diff --git a/.gitignore b/.gitignore index 27427ec..3b9d2d7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,36 +1,5 @@ -quick_install.sh -Miniconda3-latest-Linux-x86_64.sh -miniconda3 -openbis-downloads -bfabric-downloads -sftp-health2030 -sftp-viollier -token.pickle -credentials.json -snake-envs -samples* -working/samples -working/samples.tsv -working/lsf.* -working/references/*.fasta.* -working/references/*.log -working/references/*.benchmark -working/variants/ -working/qa.csv -working/qa_report.html -working/qa_vals.csv -garbage/ -status/ -._* -.snapshot -.ipynb_checkpoints - -#logs -slurm-* -.snakemake -cluster_logs - - -# vim backup files: +ssh .*.sw? -.swp +secrets +pangolin_src/fgcz-gstore.uzh.ch +uploader/__pycache__ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f03db23 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +FROM debian:buster-slim + +RUN addgroup --gid 1029 bs-pangolin && adduser --ingroup bs-pangolin --uid 542576 bs-pangolin + +WORKDIR /root + +RUN mkdir /home/bs-pangolin/.ssh +RUN chown -R bs-pangolin:bs-pangolin /home/bs-pangolin/.ssh + +RUN apt-get update && apt-get install -y vim wget lftp rsync gawk ssh git gpg expect + +USER bs-pangolin +WORKDIR /app/ +RUN mkdir -p setup +COPY pangolin_src/setup /app/setup +RUN /app/setup/setup.sh + +USER root +COPY --chown=bs-pangolin:bs-pangolin pangolin_src /app/pangolin_src +COPY --chown=bs-pangolin:bs-pangolin uploader /app/uploader +USER bs-pangolin + +WORKDIR /app/pangolin_src + + + +ENTRYPOINT ["/app/pangolin_src/entrypoint.sh"] diff --git a/README.md b/README.md index 5c1ed29..4671d45 100644 --- a/README.md +++ b/README.md @@ -28,3 +28,126 @@ for the code used in wastewater analysis, see: Short Oral presentation done at the [ECCVID conference in September 2020](https://www.escmid.org/research_projects/escmid_conferences/past_escmid_conferences/eccvid/): - https://youtu.be/BJ-un88CT9A + +## automation + +the automation is a collection of scripts connected together by a loop that is configured to monitor, log and kickstart all necessary processes. The automation is completely dockerized. + +The automation runs on VM `wisedb.nexus.ethz.ch`. Access is granted by the VM administrators and uses the ETH LDAP credentials. + +The docker container dedicated to run the automation is named `revseq-revseq-1` and runs indefinitely in an internal loop. + +### Setup + +#### Pre-requisites +- [Docker engine](https://docs.docker.com/engine/install/) + +#### Configuration +Automation configuration relies on the file `pangolin_src/config/server.conf`. The file contains all necessary variables for the automation, fully commented for clarity. + +Please refer directly to the file to define directories and automation behaviors as necessary. + +#### Resources and secrets +The automation relies on a set of resources and secret to successfully connect to external services. The necessary files are as follows: +- a `resource` directory with the files + - `config`, the ssh config file with an entry for the FGCZ sftp server + - `id_ed25519_spsp_uploads.pub`, the public key to connect to SPSP for the uploads + - `id_ed25519_wisedb_backups.pub`, the public key to connect to bs-bewi08 for the backups + - `id_ed25519_wisedb.pub`, the main public key used by the container to connect to external services + - `id_euler_ed25519.pub`, the public key to connect to Euler's rsync daemon + - `known_hosts`, the ssh file containing the accepted public keys of the FGCZ SFTP server, euler and bs-bewi08. The remaining hosts are automatically added during the container deployment +- a `secrets` directory with the files + - `bs-pangolin@d@bs-bewi08`, a file wih the credentials to login to bs-bewi08 + - `bs-pangolin@euler.ethz.ch`, a file wih the credentials to login to Euler + - `ABC9FC14AAC952E7767FD14A48B70E724BAFE0A3.asc`, the GPG key provided by SPSP for the uploads. Please refer to [SendCrypt manual](https://gitlab.sib.swiss/clinbio/sendcrypt/sendcrypt-cli/-/tree/main?ref_type=heads) for further details + - `bs-pangolin_spsp-uploads_gpg-key.gpg`, the GPG created locally for the SPSP uploads. Please refer to [SendCrypt manual](https://gitlab.sib.swiss/clinbio/sendcrypt/sendcrypt-cli/-/tree/main?ref_type=heads) for further details + - `default.sendcrypt-profile`, the profile to use for the SPSP uploads + - `fgcz-gstore.uzh.ch`, a file with the credentials to login to the FGCZ SFTP server + - `gpg_key_secrets`, a file containing the password of the local GPG key for the SPSP uploads + - `id_ed25519_spsp_uploads`, the private key to connect to SPSP for the uploads + - `id_ed25519_wisedb` the main private key to connect to external services + - `id_ed25519_wisedb_backups`, the private key to connect to bs-bewi08 for the backups + - `id_euler_ed25519`, the private key to connect to Euler + - `rsync.pass.euler`, the password to access the rsync daemon on Euler + +#### Docker configuration + +docker-compose needs to be adapted to the host machine filesystem for a successful deploy. Sections `services-pangolin-build-context`, `service-pangolin-volumes` and `secrets` rely on host absolute paths and are currently configured to run on the dedicated VM + +### Deployment + +It is suggested to deploy the automation using `docker-compose up --detach` + +### Steps + +The automation loop is started by `quasimodo.sh` which is in charge of running `carillon.sh` every loop. `carillon.sh` code can be divide in multiple phases. + +1. Data Sync: + - Provided the sync is activated in the configuration, the script runs a forced command on Euler using the script `batman.sh`, starting an rsync job that syncs any new raw data from FGCZ SFTP server to Euler directory `bfabric-downloads` + - The exit status of the sync procedure is downloaded from Euler and checked for success before running the next procedure + - The script runs a forced command on Euler to execute `sort_samples_bfabric.py` through the command `belfry.sh sortsamples --recent`. The command checks the consistency and completeness of the synced plates and creates the directory structure necessary for Vpipe +2. Vpipe run check: + - The automation checks if any Vpipe run is ongoing on Euler and logs the status + - If there is no ongoing Vpipe run, the results are backupped on bs-bewi08 + - If there is no ongoing Vpipe run, the latest batch successfully analysed by Vpipe is added to the queue of samples to upload to SPSP +3. Start Vpipe run: + - If the previous steps detected and successfully handled a new full plate to analyse and there is no ongoing Vpipe run on Euler, the automation submits a new Vpipe job on Euler to analyse the new data and logs the new submission +4. VILOCA run check: + - The automation checks if any VILOCA run is ongoing on Euler and logs the status + - If it detects the previous VILOCA run as ended because of TIMEOUT, the VILOCA snakemake directory is unlocked and the run is retried + - If there is no ongoing VILOCA run, the results are backupped on bs-bewi08 +5. Start VILOCA run: + - If the Vpipe logs shows a new batch successfully analysed by Vpipe and there is no ongoing VILOCA run on Euler, the automation submits a new VILOCA job on Euler using a forced command to analyse the new data and logs the new submission +6. Upload samples to SPSP: + - The automation keeps track of the number of files submitted to SPSP and the estimated total size uploaded for any day. As first step, the automation checks if the quotas set in the configuration are reached, before attempting any upload + - If no quota is reached and the uploads are activated, the automation triggers the upload scripts. + - The scripts take in input a full list of samples to upload and a list of samples that have been already uploaded + - A temporary list of samples to upload is generated and used to download from Euler the related cram files + - using the file `uploader/submission_metadata.py`, an SPSP metadata line is created for each sample to submit. Metadata lines are sanity checked before submission and the procedure will fail if provided unexpected samples + - The directory containing the cram files and the metadata is submitted to SPSP using the provided tool SendCrypt + - The temporary folders created by SendCrypt are deleted + - The metadata and logs from the submission are archived and backupped to bs-bewi08 +7. Amplicon coverage: + - If the Vpipe logs shows a new batch successfully analysed by Vpipe, the automation triggers the scripts necessary to calculate the amplicons coverage using a forced commands + - The results (a csv table of the coverage per amplicon and a heatmap visualization of the table) are backupped on bs-bewi08 + +# Processing +## Sample collection +Samples from the WWTPs under surveillance are collected by Eawag during a week, usually from Tuesday afternoon to the next Tuesday morning. Frequency of collection and number of WWTPs under surveillance depend on the current terms of agreement. Eawag prepares the samples and ships them to the sequencing center (FGCZ) on Tuesday. + +## Sequencing +FGCZ receives the samples, completes the library prepapration and the sequencing. The sequencing platform and library prep kit used change depending on the available technologies and the scientific evidence on the detection efficiency of SARS-CoV-2. + +## Raw data retrieval +Raw sequencing results are delivered by FGCZ to their LFTP server. The automation regularly checks the server for new data to process. If new data is available, the entire delivery is mirrored on Euler, checked for consistency with the metadata, and V-pipe is ran on the samples. The automation is able to differentiate between Aviti and NextSeq sequencing and run V-pipe with the specific settings. + +## Lollipop (temporary) +The current automated V-pipe analysis does not include the Lollipop deconvolution steps. Until those steps are integrated in the automated V-pipe run, Lollipop needs to be manually ran. + +To do so, a user is required to wait for the automated Slurm email confirming that a run on a new batch has successfully completed, then run the script `cowabunga.sh` to hardlink the necessary data to the dedicated Lollipop directory. More specifically, the two commands to run are: +- `./cowabunga.sh autoaddwastewater year` to list all batches received in the current year and update the sample list with the samples from the new batches +- `./cowabung.sh bring_results` to hardlink the necessary V-pipe output to the Lollipop directory `work-vp-test` in preparation of the Lollipop run + +After running `cowabunga.sh`, the user can navigate to the working directory `work-vp-test` and run Lollipop using the command `sbatch vpipe-test.sbatch`. + +## Postprocessing +Lollipop results need to be postprocessed to generate and visualize the curves, as well as share the results with BAG. + +Similarly to the automated V-pipe step, Lollipop reports the status of a run through automated slurm emails. If a Lollipop run is succesful, the post-processing can start. + +All postprocessing happens on a dedicated machine called `jupyterhub05`. The machine runs a jupyter notebook server pre-configured with the necessary kernels. Kernels definitions are available on the server. The necessary notebooks are `ww_cov_uploader_V-pipe` and `ww_cov_SwitzerlandMap`. + +- `ww_cov_uploader_V-pipe` loads the deconvolution and tallymut results, processing them for plotting and upload + - A user needs first to run the notebook up until the cell `Multiple choice time`. The curves preview should then be shared for review before proceeding + - If the review finds abnormalities, the cell `db.rollback()` should be run to cancel any update to the cov-spectrum database + - If the review finds no abnormalities, the cell `db.rollback()` should be *skipped* and the remainder of the notebook can be run, committing the changes to the cov-spectrum database and uploading the results on Polybox for BAG and for the public +- `ww_cov_SwitzerlandMap` loads the deconvolution and tallymut results, processing them for plotting cake plots with the average relative abundances for the week on each WWTP on a map of Switzerland + - The plot needs to be included in the email reports. The plot caption can be copy-pasted from previous weeks unless the method or the WWTPs change + +After running both notebooks, the email report can be written. The mandatory content is as follows: +- Last collection date available for each WWTP. This information is provided as output of the early cells of the notebook `ww_cov_uploader_V-pipe` +- Comment on the current status, mentioning the dominant variant (if any) and the observable relative abundance trends +- Signature +- Switzerland map plot with its caption + +The email report must be reviewed by an additional person before submission. Submission should be done by sending the email to the dedicated mailing list. \ No newline at end of file diff --git a/V-pipe b/V-pipe deleted file mode 160000 index e8be5f1..0000000 --- a/V-pipe +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e8be5f1fad7a970e3299e142f329f037f3890fe4 diff --git a/V-pipe-test b/V-pipe-test deleted file mode 160000 index e8be5f1..0000000 --- a/V-pipe-test +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e8be5f1fad7a970e3299e142f329f037f3890fe4 diff --git a/batman.sh b/batman.sh deleted file mode 100755 index 9a48dc2..0000000 --- a/batman.sh +++ /dev/null @@ -1,241 +0,0 @@ -#!/bin/bash - -clusterdir=/cluster/project/pangolin -working=working -sampleset=sampleset - -# -# Input validator -# -validateBatchName() { - if [[ "$1" =~ ^(20[0-9][0-9][0-1][0-9][0-3][0-9]_[[:alnum:]-]{4,})$ ]]; then - return; - else - echo "bad batchname ${1}" - exit 1; - fi -} - -validateTags() { - local IFS="$1" - for b in $2; do - validateBatchName "${b}" - done -} - - -RXJOB='Job <([[:digit:]]+)> is submitted' -# Generic job. -# Job <129052039> is submitted to queue . - -now=$(date '+%Y%m%d') -lastmonth=$(date '+%Y%m' --date='-1 month') -thismonth=$(date '+%Y%m') -twoweeksago=$(date '+%Y%m%d' --date='-2 weeks') - -if [[ "$1" == "--limited" ]]; then - shift - case "$1" in - rsync|df) - # sub-commands allowed in limited mode. - ;; - *) - echo "sub-command ${1} not allowed in limited mode" >&2 - exit 2 - ;; - esac -fi - -case "$1" in - rsync) - # rsync daemon : see ${SSH_ORIGINAL_COMMAND} - rsync --server --daemon . - ;; - logrotate) - # rotate logs - ~/log/rotate - ;; - addsamples) - lst="${clusterdir}/${working}/samples.tsv" - case "$2" in - --recent) - lst="${clusterdir}/${working}/samples.recent.tsv" - echo "syncing recent: ${lastmonth}, ${thismonth}" - ;; - *) - echo "Unkown parameter ${2}" > /dev/stderr - exit 2 - ;; - esac - mkdir -p --mode=2770 "${clusterdir}/${working}/samples/" - #cp -vrf --link ${clusterdir}/${sampleset}/*/ ${clusterdir}/${working}/samples/ ## failure: "no rule to create {SAMPLE}/extract/R1.fastq" - cat ${clusterdir}/${sampleset}/samples.{${lastmonth},${thismonth}}*.tsv | sort -u > "${clusterdir}/${working}/samples.recent.tsv" - sort -u ${clusterdir}/${sampleset}/samples.*.tsv > "${clusterdir}/${working}/samples.tsv" - cut -f1 "${lst}" | xargs -P 8 -i cp -vrf --link "${clusterdir}/${sampleset}/{}/" "${clusterdir}/${working}/samples/" - ;; - vpipe) - declare -A job - list=('seq' 'seqqa' 'snv' 'snvqa' 'hugemem' 'hugememqa') - shorah=1 - hold= - tag= - while [[ -n $2 ]]; do - case "$2" in - --no-shorah) - shorah=0 - ;; - --hold) - hold='-H' - ;; - --tag) - shift - if [[ ! "${2}" =~ ^[[:alnum:]]+$ ]]; then - # if it's not just letters and number, test if it is a list of valid batches - validateTags ';' "$2" - fi - tag="${2%;}" - ;; - --recent) -# # TODO switch between full cohort and only recent -# #recent="..." - ;; - *) - echo "Unkown parameter ${2}" > /dev/stderr - exit 2 - ;; - esac - shift - done - # start first job - cd ${clusterdir}/${working}/ - # use -H to put on hold for analysis - if [[ "$(sed "s/@TAG@/<${tag}>/g" vpipe-no-shorah.bsub | bsub -J "COVID-vpipe-<${tag}>-cons" ${hold})" =~ ${RXJOB} ]]; then - job['seq']=${BASH_REMATCH[1]} - # schedule a gatherqa no mater what happens - [[ "$(sed "s/@TAG@/<${tag}>/g" qa-launcher | bsub -J "COVID-vpipe-<${tag}>-qa" ${hold} -w "ended(${job['seq']})")" =~ ${RXJOB} ]] && job['seqqa']=${BASH_REMATCH[1]} - # if no fail schedule a full job with snv - if (( shorah )) && [[ "$(bsub ${hold} -w "done(${job['seq']})" -ti < vpipe.bsub)" =~ ${RXJOB} ]]; then - job['snv']=${BASH_REMATCH[1]} - # schedule a gatherqa no matter what happens to snv - [[ "$(bsub ${hold} -w "done(${job['seq']})&&ended(${job['snv']})" < qa-launcher)" =~ ${RXJOB} ]] && job['snvqa']=${BASH_REMATCH[1]} - # schedule a hugemem job if snvjob failed - if [[ "$(bsub ${hold} -w "done(${job['seq']})&&exit(${job['snv']})" -ti < vpipe-hugemem.bsub)" =~ ${RXJOB} ]]; then - job['hugemem']=${BASH_REMATCH[1]} - # schedule a qa afterward - [[ "$(bsub -w "done(${job['seq']})&&exit(${job['snv']})&&ended(${job['hugemem']})" -ti < qa-launcher)" =~ ${RXJOB} ]] && job['hugememqa']=${BASH_REMATCH[1]} - fi - fi - fi >&2 - # write job chain list - for v in "${list[@]}"; do - printf "%s\t%s\n" "${v}" "${job[$v]}" - done - ;; - job) - if [[ $2 =~ ^([[:digit:]]+)$ ]]; then - bjobs $2 | gawk -v I=$2 '$1==I{print $3}' - else - bjobs - fi - ;; - purgelogs) - find ${clusterdir}/${working}/cluster_logs/ -type f -mtime +28 -name '*.log' -print0 | xargs -0 rm -- - ;; - scratch) - temp_scratch="${SCRATCH}/pangolin/temp" - olderthan=60 - purge=0 - loop=0 - filter= - while [[ -n $2 ]]; do - case "$2" in - --minutes-ago) - if [[ ! "${3}" =~ ^[[:digit:]]+$ ]]; then - echo "parameter of ${2} must be a number of minutes (digits only), got <${3}> instead" > /dev/stderr - exit 2 - fi - shift - olderthan=$2 - ;; - --loop) - loop=1 - ;& - --purge) - purge=1 - ;; - *) - echo "Unkown parameter ${2}" > /dev/stderr - exit 2 - ;; - esac - shift - done - - if (( purge )); then - echo "purging in ${temp_scratch}..." - else - echo "listing in ${temp_scratch}..." - fi - - count_all=0 - count_old=0 - while read s; do - (( ++count_all )) - if [[ -r "${clusterdir}/${working}/samples/${s}/upload_prepared.touch" && $(find "${clusterdir}/${working}/samples/${s}/upload_prepared.touch" '!' -newermt "${olderthan} minutes ago") ]]; then - (( ++count_old )) - if (( purge )); then - rm -rvf "${temp_scratch}/samples/${s}" - else - echo "${s}"; - fi - fi; - done < <((cd "${temp_scratch}/" && find samples/ -type f ${filter} ) | grep -oP '(?<=samples/)[^/]+/[^/]+(?=/)' | sort -u) | tee /dev/stderr | wc -l 2>&1 - echo "Samples: ${count_old} old / ${count_all} total" - - if (( loop )); then - echo -e '\n\e[38;5;45;1mYou just keep on trying\e[0m\n\e[38;5;208;1mTill you run out of cake\e[0m' - if sleep "$(( 5 + olderthan))m"; then - # loop if no breaks - exec "${0}" scratch --minutes-ago "${olderthan}" --loop - fi - echo "I'm not even angry" - fi - ;; - completion) - if [[ $2 =~ ^([[:digit:]]+)$ ]]; then - bpeek $2 | gawk '$0~/^\[.*\]$/{date=$0};$0~/^[[:digit:]]+ of [[:digit:]]+ steps \([[:digit:]]+%\) done$/{print $0 "\t" date}' - fi - ;; - df) - #df ${clusterdir} ${SCRATCH} - lquota -2 ${clusterdir} - lquota -2 ${SCRATCH} - ;; - garbage) - validateBatchName "$2" - cd ${clusterdir} - - for f in ${sampleset}/*/${2}; do - garbage=$(dirname "${f//${sampleset}/garbage}") - mkdir --mode=0770 -p "${garbage}" - mv -v "${f%/}" "${garbage}/" - done - for f in ${working}/samples/*/${2}/; do - garbage="${f//${working}\/samples/garbage}" - mkdir --mode=0770 -p "${garbage%/}/"{raw_data,extracted_data}/ - mv -vf "${f%/}/raw_data/"* "${garbage%/}/raw_data/" - rm "${f%/}/raw_data/"*.fa* - rmdir "${f%/}/raw_data/" - mv -vf "${f%/}/extracted_data/"* "${garbage%/}/extracted_data/" - rm "${f%/}/extracted_data/"*{.log,.benchmark,_fastqc.html} - rmdir "${f%/}/extracted_data/" - mv -vf "${f%/}/"* "${garbage%/}/" - rmdir "${f%/}" - done - mv "${sampleset}/batch.${2}.yaml" "${sampleset}/samples.${2}.tsv" "${sampleset}/missing.${2}.txt" "${sampleset}/projects.${2}.tsv" garbage/ - ;; - *) - echo "Unkown sub-command ${1}" > /dev/stderr - exit 2 - ;; -esac diff --git a/belfry b/belfry deleted file mode 100755 index 5b5d49d..0000000 --- a/belfry +++ /dev/null @@ -1,581 +0,0 @@ -#!/bin/bash - -cluster_user="${USER%%@*}" -scriptdir="$(dirname $(which $0))" - -declare -A lab -. ${scriptdir}/server.conf - -: ${basedir:=$(pwd)} -: ${download:?} -: ${sampleset:=sampleset} -: ${working:=working} -: ${releasedir:?} -: ${storgrp:?} -: ${parallel:=16} -: ${parallelpull:=${parallel}} -: ${contimeout:=300} -: ${retries:=10} -: ${iotimeout:=300} -: ${protocolyaml:=/references/primers.yaml} - -if [[ $(realpath $scriptdir) != $(realpath $basedir) ]]; then - echo "$scriptdir vs $basedir" -fi - -if [[ ! $mode =~ ^[0-7]{,4}$ ]]; then - echo "Invalid characters <${mode//[0-7]/}> in <${mode}>" - echo 'mode should be an octal chmod value, see `mkdir --help` for informations' - mode= -fi - - -cd ${basedir} - -umask 0002 - -statusdir="${basedir}/status" -mkdir ${mode:+--mode=${mode}} -p ${statusdir} - -baseconda=$HOME/ -timeoutforeground= -#--foreground - -now=$(date '+%Y%m%d') -lastmonth=$(date '+%Y%m' --date='-1 month') -thismonth=$(date '+%Y%m') -twoweeksago=$(date '+%Y%m%d' --date='-2 weeks') - -# -# Input validator -# -validateBatchDate() { - if [[ "$1" =~ ^(20[0-9][0-9][0-1][0-9][0-3][0-9])$ ]]; then - return; - else - echo "bad batchdate ${1}" - exit 1; - fi -} - -validateBatchName() { - if [[ "$1" =~ ^(20[0-9][0-9][0-1][0-9][0-3][0-9]_[[:alnum:]-]{4,})$ ]]; then - return; - else - echo "bad batchname ${1}" - exit 1; - fi -} - - -# -# rsync parallel helpers -# -callpushrsync() { - . server.conf - - local arglist=( ) - if (( ${#@} )); then - arglist=( "${@/#/${basedir}/${sampleset}/}" ) - else - #arglist=( "${basedir}/${sampleset}/" ) - echo "rsync job didn't receive list" - exit 1; - fi - exec timeout ${timeoutforeground} --signal=INT --kill-after=5 $((rsynctimeout+contimeout+5)) \ - rsync --timeout=${iotimeout} \ - --password-file ~/rsync.pass.euler \ - -e "ssh -i ${HOME}/.ssh/id_ed25519_belfry -l ${cluster_user} -oConnectTimeout=${contimeout}" \ - -izrltH --fuzzy --fuzzy --inplace \ - -p --chmod=Dg+s,ug+rw,o-rwx,Fa-x \ - -g --chown=:'bsse-covid19-pangolin-euler' \ - "${arglist[@]}" \ - belfry@euler.ethz.ch::${sampleset}/ -} -export -f callpushrsync - - -callpullrsync() { - . server.conf - - local arglist=( ) - if (( ${#@} )); then - arglist=( "${@/#/belfry@euler.ethz.ch::${working}/samples/}" ) - else - #arglist=( "belfry@euler.ethz.ch::${working}/samples/" ) - echo "rsync job didn't receive list" - exit 1; - fi - exec timeout ${timeoutforeground} --signal=INT --kill-after=5 $((rsynctimeout+contimeout+5)) \ - rsync --timeout=${iotimeout} \ - --password-file ~/rsync.pass.euler \ - -e "ssh -i ${HOME}/.ssh/id_ed25519_belfry -l ${cluster_user} -oConnectTimeout=${contimeout}" \ - -izrltH --fuzzy --fuzzy --inplace \ - --link-dest=${basedir}/${sampleset}/ \ - "${arglist[@]}" \ - --exclude='uploads/*' \ - --exclude='raw_uploads/*.tmp.*' \ - --exclude='raw_data/*_R[12].fastq.gz' \ - --exclude='extracted_data/R[12]_fastqc.html' \ - --exclude='variants/SNVs/REGION_*/reads.fas' \ - --exclude='variants/SNVs/REGION_*/w-*.reads.fas' \ - --exclude='variants/SNVs/REGION_*/raw_reads/w-*.reads.fas.gz' \ - --exclude='*.out.log' \ - --exclude='*.err.log' \ - --exclude='*.benchmark' \ - ${basedir}/${working}/samples/ -} -export -f callpullrsync - - -callpullrsync_noshorah() { - . server.conf - - local arglist=( ) - if (( ${#@} )); then - arglist=( "${@/#/belfry@euler.ethz.ch::${working}/samples/}" ) - else - #arglist=( "belfry@euler.ethz.ch::${working}/samples/" ) - echo "rsync job didn't receive list" - exit 1; - fi - exec timeout ${timeoutforeground} --signal=INT --kill-after=5 $((rsynctimeout+contimeout+5)) \ - rsync --timeout=${iotimeout} \ - --password-file ~/rsync.pass.euler \ - -e "ssh -i ${HOME}/.ssh/id_ed25519_belfry -l ${cluster_user} -oConnectTimeout=${contimeout}" \ - -izrltH --fuzzy --fuzzy --inplace \ - --link-dest=${basedir}/${sampleset}/ \ - "${arglist[@]}" \ - --exclude='uploads/*' \ - --exclude='raw_uploads/*.tmp.*' \ - --exclude='raw_data/*_R[12].fastq.gz' \ - --exclude='extracted_data/R[12]_fastqc.html' \ - --exclude='variants/' \ - --exclude='visualization/' \ - --exclude='*.out.log' \ - --exclude='*.err.log' \ - --exclude='*.benchmark' \ - ${basedir}/${working}/samples/ -} -export -f callpullrsync_noshorah - - -# -# sync helper -# -checksyncoutput() { - local new="${statusdir}/sync${1}_new" - local last="${statusdir}/sync${1}_last" - - if [[ "${2}" =~ Total:\ +([[:digit:]]+)\ +directories,\ +([[:digit:]]+)\ +files,.*?New:\ +([[:digit:]]+)\ +files, ]]; then - echo "Newfiles downloaded" - echo -e "${BASH_REMATCH[2]}\n${BASH_REMATCH[1]}" > ${last} - flock -x -o ${last} -c "sleep 1" - echo "${BASH_REMATCH[3]}" > ${new} - else - echo "Same old shit" - touch ${last} - fi 2>&1 -} - - -# -# main handler -# -case "$1" in - syncopenbis) - echo "Sync GFB - OpenBIS..." - # Total: 10654 directories, 11687 files, 0 symlink - # New: 605 files, 0 symlinks - # 7674153291 bytes transferred - . $baseconda/miniconda3/bin/activate "" - if [[ "${2}" = "--recent" ]]; then - param=( "${lastmonth}*" "${thismonth}*" ) - echo "syncing recent: ${param[*]}" - else - param=( ) - fi - syncoutput="$(/usr/bin/time ${scriptdir}/sync_sftp -c gfb.conf "${param[@]}"|tee /dev/stderr)" - checksyncoutput "openbis" "$syncoutput" - conda deactivate - ;; - syncfgcz) - echo "Sync FGCZ - bfabric" - . $baseconda/miniconda3/bin/activate "" - . <(grep '^projlist=' fgcz.conf) - if [[ "${2}" = "--recent" ]]; then - limitlast='3 weeks ago' - ./exclude_list_bfabric -c fgcz.conf -r "${twoweeksago}" -o ${statusdir}/fgcz.exclude.lst - param=( '-e' "${statusdir}/fgcz.exclude.lst" "${projlist[@]}" ) - echo -ne "syncing recent: ${limitlast}\texcluding: " - wc -l ${statusdir}/fgcz.exclude.lst - else - param=( "${projlist[@]}" ) - fi - syncoutput="$(/usr/bin/time ${scriptdir}/sync_sftp -c fgcz.conf ${limitlast:+ -N "${limitlast}"} "${param[@]}"|tee /dev/stderr)" - checksyncoutput "fgcz" "$syncoutput" - conda deactivate - ;; - synch2030) - echo "Sync Health2030" - . $baseconda/miniconda3/bin/activate "" - # short test - only list dirs - if [[ ( ( ! -e ${statusdir}/synch2030_ended ) && ( ! -e ${statusdir}/synch2030_started ) ) || ( ${statusdir}/synch2030_ended -nt ${statusdir}/synch2030_started ) ]]; then - ${scriptdir}/list_sftp -c h2030.conf -l ${statusdir}/synch2030.${now} && \ - if [[ -s ${statusdir}/synch2030.${now} ]]; then - ln -sf synch2030.${now} ${statusdir}/synch2030_started - fi - # did we get new list? - if [[ ( -e ${statusdir}/synch2030_started ) && ( ( ! -e ${statusdir}/synch2030_ended ) || ( ${statusdir}/synch2030_started -nt ${statusdir}/synch2030_ended ) ) ]]; then - cat ${statusdir}/synch2030_started - else - echo "no new dirs" - fi - fi - # long sync - full mirror - if [[ ( -e ${statusdir}/synch2030_started ) && ( ( ! -e ${statusdir}/synch2030_ended ) || ( ${statusdir}/synch2030_started -nt ${statusdir}/synch2030_ended ) ) ]]; then - syncoutput="$(/usr/bin/time ${scriptdir}/sync_sftp -c h2030.conf |tee /dev/stderr)" - checksyncoutput "h2030" "$syncoutput" - ${scriptdir}/check_sums -c h2030.conf 'md5.txt' $( < ${statusdir}/synch2030_started) - # 2: internal error, grep error - # 1: no failed checksum found - # 0: failed checksum found - if [[ "$?" == "1" ]]; then - touch ${statusdir}/synch2030_ended - fi - fi - conda deactivate - ;; - syncviollier) - echo "Sync Viollier" - . $baseconda/miniconda3/bin/activate "" - param=( 'sample_metadata' ) - if [[ "${2}" = "--recent" ]]; then - param+=( "raw_sequences/${lastmonth}*" "raw_sequences/${thismonth}*" ) - echo "syncing recent: ${param[*]}" - echo "NOT SUPPORTED" - exit 1 - else - param+=( 'raw_sequences' ) - echo "syncing all: ${param[*]}" - fi - # HACK delete previous partial downloads - . <(grep '^download=' viollier.conf) - echo "Removing partial files from:" "${param[@]/#/${download}/}" - find "${param[@]/#/${download}/}" -type f -name '*.filepart' -print0 | xargs -r -0 rm -v - syncoutput="$(/usr/bin/time ${scriptdir}/sync_sftp -c viollier.conf "${param[@]}"|tee /dev/stderr)" - checksyncoutput "viollier" "$syncoutput" - conda deactivate - ;; - uploadviollier) - echo "Uploading Viollier" - . $baseconda/miniconda3/bin/activate "" - param=( 'consensus_sequences' 'raw_othercenters' ) - - ${scriptdir}/upload_sftp -c viollier.conf "${param[@]}" - conda deactivate - ;; - purgeviollier) - if (( ! ${lab[viollier]} )); then - echo "Viollier disabled; no purging" - exit 0 - fi - echo "Purge Viollier" - . $baseconda/miniconda3/bin/activate "" - param=( 'raw_sequences' ) - recent=$(date --date='2 week ago' '+%Y%m%d') - - ${scriptdir}/purge_sftp -r "${recent}" -c viollier.conf "${param[@]}" - conda deactivate - ;; - uploadrequests) - echo "Handling raw-read upload requests for Viollier" - . $baseconda/miniconda3/bin/activate "vineyard" - ${scriptdir}/handle_request_raw_viollier -c viollier.conf - conda deactivate - ;; - sortsamples) - . $baseconda/miniconda3/bin/activate pybis - cd $basedir - summary="" - recent="" - shrtrecent="" - force="${sort_force}" - while [[ -n $2 ]]; do - case "$2" in - --summary) - summary='--summary' - ;; - --force) - force='--force' - ;; - --recent) - recent="--recent=${lastmonth}" - shrtrecent="-r ${lastmonth}" - ;; - *) - echo "Unkown parameter ${2}" > /dev/stderr - exit 2 - ;; - esac - shift - done - fail=0 - if (( ${lab[gfb]} )); then - ${scriptdir}/sort_samples_pybis -c gfb.conf --protocols=${basedir}/${working}/${protocolyaml} --assume-same-protocol ${force} ${summary} ${recent} && bash ${basedir}/${sampleset}/movedatafiles.sh || fail=1 - else - echo "Skipping gfb" - fi - if (( ${lab[fgcz]} )); then -# # TODO config file - . <(grep '^google_sheet_patches=' fgcz.conf) - - (( google_sheet_patches )) && ${scriptdir}/google_sheet_patches - ${scriptdir}/sort_samples_bfabric_tsv -c fgcz.conf --no-fastqc --protocols=${basedir}/${working}/${protocolyaml} --libkit-override=${basedir}/${sampleset}/patch.fgcz-libkit.tsv ${force} ${recent} && bash ${basedir}/${sampleset}/movedatafiles.sh || fail=1 - else - echo "Skipping fgcz" - fi - if (( ${lab[h2030]} )) && [[ -e ${statusdir}/synch2030_ended && -e ${statusdir}/synch2030_started && ${statusdir}/synch2030_ended -nt ${statusdir}/synch2030_started ]]; then - # NOTE always recent/based on last sync), no support for --force, move done immediately/no separate movedatafiles.sh - # TODO support for protocols - ${scriptdir}/sort_h2030 -c h2030.conf $(< ${statusdir}/synch2030_started ) || fail=1 - else - echo "Skipping h2030" - fi - if (( ${lab[viollier]} )); then - # NOTE always --force, short options only - # HACK hardcoded paths due to multiple directories - ${scriptdir}/sort_viollier -c viollier.conf -4 ${basedir}/${working}/${protocolyaml} ${shrtrecent} sftp-viollier/raw_sequences/*/ && bash ${basedir}/${sampleset}/movedatafiles.sh || fail=1 - else - echo "Skipping viollier" - fi - (( fail == 0 )) && touch ${statusdir}/sortsamples_success || touch ${statusdir}/sortsamples_fail - conda deactivate - ;; - fixopenbisrights) - validateBatchDate "$2" - # NOTE ACLs should autopropagate - - #dir=( ${basedir}/${download}/${2}* ) - #if (( ${#dir[@]} > 1 )); then - # echo "${#dir[@]} directories" - # echo 'mode directories' - # find "${dir[@]}" -type d -print0 | xargs -0 chmod u+rwx,g+rwXs,o-rwx - # echo 'mode files' - # find "${dir[@]}" -type f -print0 | xargs -0 chmod 0660 - # #echo 'group' - # #chgrp -R "${storgrp}" "${dir[@]}" - #fi - ;; - pushsampleset) - if [[ "${2}" = "--recent" ]]; then - sheets=( ${basedir}/${sampleset}/samples.${lastmonth}*.tsv ${basedir}/${sampleset}/samples.${thismonth}*.tsv ) - # BUG: will generate non-globed pattern if months are missing - echo "pushing recent: ${param[*]##/}" - elif [[ "${2}" = "--batch" ]]; then - validateBatchName "${3}" - sheets=( "${basedir}/${sampleset}/samples.${3}.tsv" ) - else - sheets=( ${basedir}/${sampleset}/samples.*.tsv ) - fi - err=0 - timeout ${timeoutforeground} --signal=INT --kill-after=5 $((rsynctimeout+contimeout+5)) \ - rsync --timeout=${iotimeout} \ - --password-file ~/rsync.pass.euler \ - -e "ssh -i ${HOME}/.ssh/id_ed25519_belfry -l ${cluster_user} -oConnectTimeout=${contimeout}" \ - -izrltH --fuzzy --fuzzy --inplace \ - -p --chmod=Dg+s,ug+rw,o-rwx,Fa-x \ - -g --chown=:'bsse-covid19-pangolin-euler' \ - ${basedir}/${sampleset}/samples.*.tsv \ - ${basedir}/${sampleset}/batch.*.yaml \ - ${basedir}/${sampleset}/projects.*.tsv \ - ${basedir}/${sampleset}/missing.*.txt \ - ${basedir}/${sampleset}/patch.*.tsv \ - belfry@euler.ethz.ch::${sampleset}/ || (( ++err )) - cut -s --fields=1 "${sheets[@]}"|sort -u| \ - gawk -v P=$(( parallel * 4 )) '{i=(NR-1);b=i%P;o[b]=(o[b] " \"" $1 "\"")};END{for(i=0;i /dev/stderr - exit 2 - ;; -esac diff --git a/carillon b/carillon deleted file mode 100755 index acfaf91..0000000 --- a/carillon +++ /dev/null @@ -1,393 +0,0 @@ -#!/bin/bash - -scriptdir="$(dirname $(which $0))" -cluster_user="${USER%%@*}" - -. ${scriptdir}/server.conf - -: ${run_shorah:=0} -: ${staging:=1} - -if [[ $(realpath $scriptdir) != $(realpath $basedir) ]]; then - echo "$scriptdir vs $basedir" -fi - -if [[ ! $mode =~ ^[0-7]{,4}$ ]]; then - echo "Invalid characters <${mode//[0-7]/}> in <${mode}>" - echo 'mode should be an octal chmod value, see `mkdir --help` for informations' - mode= -fi - -cd ${basedir} - -umask 0002 - -statusdir="${basedir}/status" -mkdir ${mode:+--mode=${mode}} -p ${statusdir} -lockfile=${statusdir}/carillon_lock - -remote_batman="ssh -ni ${HOME}/.ssh/id_ed25519_batman -l ${cluster_user} euler.ethz.ch --" -remote_belfry="ssh -ni ${HOME}/.ssh/id_ed25519_belfry bs-bewi09.ethz.ch --" - -touch ${statusdir}/oh_hai_im_lopping - -if false; then # disable block -# -# Avoid re-entry -# - -if lockfile -2 -r 1 "${lockfile}"; then - # save the PID in the lockfile - chmod u+w "${lockfile}" - echo "$$" > "${lockfile}" - chmod 0444 "${lockfile}" -else - PID=$(<"${lockfile}") - - echo -e "Previous run still running (PID=${pid})\n" - - if [ -d "/proc/${pid}" ]; then - ps --forest -o 'pid,ppid,sess,user,stime,time,%cpu,ni,args' --pid "${pid}" --ppid "${pid}" -g "$(ps -o 'sid=' --pid "${pid}" --ppid "${pid}"|uniq)" - else - echo "*CANNOT FIND PID ${pid}*" - echo "perhaps process has crashed" - echo "to force restart, manually delete lock file :" - echo -e "\trm ${lockfile}" - exit 2; - fi - exit 1; -fi -fi # disable block - - - -# -# Phase 1: periodic data sync -# - -echo '=========' -echo 'Data sync' -echo '=========' - -[[ -n $skipsync ]] && echo "Hack: ${skipsync} will be skiped." -#[[ $skipsync != h2030 ]] && ${scriptdir}/belfry synch2030 # --recent : there are no old archive at Health2030 anyway -#[[ $skipsync != violler ]] && ${scriptdir}/belfry syncviollier # --recent : there are no old archive, we're even supposed to perform the deletion -[[ $skipsync != fgcz ]] && ${scriptdir}/belfry syncfgcz --recent -#[[ $skipsync != gfb ]] && ${scriptdir}/belfry syncopenbis --recent -${scriptdir}/belfry sortsamples --recent $([[ ${statusdir}/syncopenbis_last -nt ${statusdir}/syncopenbis_new ]] && echo '--summary') - -# uploading requests require prior successful download -if [[ ( -e ${statusdir}/sortsamples_fail ) && ( ${statusdir}/sortsamples_fail -nt ${statusdir}/sortsamples_success ) ]]; then - echo 'warning: cannot serve upload requests sampleset data not successfully fetched yet' > /dev/stderr -else - ${scriptdir}/belfry uploadrequests -fi - -echo '---------' -if [[ ! ${statusdir}/syncopenbis_last -nt ${statusdir}/syncopenbis_new ]]; then - echo "Data was copied: $(<${statusdir}/syncopenbis_new)" - limit=$(date --date='1 week ago' '+%Y%m%d') - echo "Fixing perms back until $limit:" - for d in $(echo ${basedir}/${download}/20* | grep -oP "(?<=/${download}/)(20[0-9][0-9][0-1][0-9][0-3][0-9])" | sort -r -u); do - if [[ "$d" < "$limit" ]]; then - echo '...done' - break - else - echo "...${d}..." - fi - ${scriptdir}/belfry fixopenbisrights "${d}" - done -fi - - - -# -# Phase 2: update status of current run and trigger backups -# - -echo "=================" -echo "Check current run" -echo "=================" - -if [[ ( -e ${statusdir}/vpipe_started ) && ( ( ! -e ${statusdir}/vpipe_ended ) || ( ${statusdir}/vpipe_started -nt ${statusdir}/vpipe_ended ) ) ]]; then - stillrunning=0 - while read j id; do - # skip missing - if [[ -z "${id}" ]]; then - echo "$j : (not started)" - continue - fi - - # skip already finished - if [[ ( -e ${statusdir}/vpipe_${j}_ended ) && ( ${statusdir}/vpipe_${j}_ended -nt ${statusdir}/vpipe_started ) ]]; then - old="$(<${statusdir}/vpipe_${j}_ended)" - if [[ "${id}" == "${old}" ]]; then - echo "$j : $id already finished" - else - echo "$j : mismatch $id vs $old" - fi - continue - fi - - # cluster status - stat=$(${remote_batman} job "${id}" || echo "(no answer)") - if [[ ( -n "${stat}" ) && ( ! "${stat}" =~ (EXIT|DONE) ) ]]; then - # running - echo -n "$j : $id : $stat" - (( ++stillrunning )) - if [[ "${stat}" == 'RUN' && ( ! "$j" =~ qa$ ) ]]; then - echo -ne '\t' - ${remote_batman} completion "${id}" | tail -n 1 - else - echo '' - fi - sleep 1 - continue - fi - - # not running - echo "$j : $id finishing" - - case "$j" in - seqqa) - ${scriptdir}/belfry pullsamples_noshorah --recent - if [[ ( -e ${statusdir}/pullsamples_noshorah_fail ) && ( ${statusdir}/pullsamples_noshorah_fail -nt ${statusdir}/pullsamples_noshorah_success ) ]]; then - echo "pulling data failed" - (( ++stillrunning )) - continue - fi - ;; - esac - - echo "${id}" > ${statusdir}/vpipe_${j}_ended - done < ${statusdir}/vpipe_started - - if (( stillrunning == 0 )); then - # HACK temporarily avoid copying over ShoRAH files - #${scriptdir}/belfry pullsamples --recent - #if [[ ( ! -e ${statusdir}/pullsamples_fail ) || ( ${statusdir}/pullsamples_success -nt ${statusdir}/pullsamples_fail ) ]]; then - ${scriptdir}/belfry pullsamples_noshorah --recent - if [[ ( ! -e ${statusdir}/pullsamples_noshorah_fail ) || ( ${statusdir}/pullsamples_noshorah_success -nt ${statusdir}/pullsamples_noshorah_fail ) ]]; then - echo "$(basename $(realpath ${statusdir}/vpipe_started))" > ${statusdir}/vpipe_ended - else - echo "pulling data failed" - fi - fi -else - echo 'No current run.' -fi - - - -# -# Phase 3: Reports and sequences -# - -echo "================" -echo "Results handling" -echo "================" - -if [[ ( -e ${statusdir}/pullsamples_noshorah_success || -e ${statusdir}/pullsamples_success ) && ( ( ! -e ${statusdir}/qa_report_success ) || ( ${statusdir}/pullsamples_noshorah_success -nt ${statusdir}/qa_report_success ) || ( ${statusdir}/pullsamples_success -nt ${statusdir}/qa_report_success ) ) ]]; then - #${scriptdir}/belfry qa_report - touch ${statusdir}/qa_report_success -else - echo 'no newer results' -fi - -if [[ ( -e ${statusdir}/qa_report_success ) && ( ( ! -e ${statusdir}/pushseq_success ) || ( ${statusdir}/qa_report_success -nt ${statusdir}/pushseq_success ) ) ]]; then - : - ${scriptdir}/belfry pushseq - #${scriptdir}/belfry gitaddseq -else - echo 'no upload needed' -fi - - - -# -# Phase 4: restart runs if new data -# - -echo "=============" -echo "Start new run" -echo "=============" - -# TODO support a yaml with regex -rxsample='(^[[:digit:]]{6,}_)|(^[[:digit:]]{8,})|(^Y[[:digit:]]{8,})|([[:digit:]]{2}_20[[:digit:]]{2}_[01]?[[:digit:]]_[0-3]?[[:digit:]])|(^KLZHC[oO][vV])|(^B[aA][[:digit:]]{4,})|(^USB_20[[:digit:]]{2}_[01]?[[:digit:]]_[[:digit:]]{2}_.{8})' -#rxsample='^[[:digit:]]{6,}_' - -scanmissingsamples() { - while read sample batch other; do - # look for only guaranteed samples - [[ $sample =~ $rxsample ]] || continue - # check the presence of fasta on each sample - if [[ -e ${basedir}/${working}/samples/${sample}/${batch}/upload_prepared.touch ]]; then - # this will check for: - # - references/ref_majority.fasta - # - references/consensus.bcftools.fasta & .chain - # - references/frameshift_deletions_check.tsv - # etc. - # see V-pipe's rule 'prepare_upload' in publish.smk - echo -n '.' - else - echo -e "\r+${batch/_/:}\t!${sample}\e[K" - true - return 0 - fi; - done < $1 - false -} - -# like "$*" but with a different field separator than default. -join_by() { local IFS="$1"; shift; echo "$*"; } - -if [[ ( ( ! -e ${statusdir}/vpipe_ended ) && ( ! -e ${statusdir}/vpipe_started ) ) || ( ${statusdir}/vpipe_ended -nt ${statusdir}/vpipe_started ) ]]; then - clearline=0 - mustrun=0 - runreason=( ) - declare -A flowcell - ref=$(date --reference="${statusdir}/vpipe_started" '+%Y%m%d') - now=$(date '+%Y%m%d') - limit=$(date --date='2 weeks ago' '+%Y%m%d') - echo "Check batch against ${ref}:" - for t in ${basedir}/${sampleset}/samples.*.tsv; do - if [[ ! $t =~ samples.([[:digit:]]{8})_([[:alnum:]]{5,}(-[[:digit:]]+)?).tsv$ ]]; then - echo "oops: Can't parse <${t}> ?!" > /dev/stderr - fi - - # check Duplicates - b="${BASH_REMATCH[1]}" - f="${BASH_REMATCH[2]}" - if [[ -n "${flowcell[$f]}" ]]; then - echo "error: Duplicate flowcell $f : ${flowcell[$f]} vs $b" > /dev/stderr - exit 2 - else - flowcell[$f]=$b - fi - - # check dates - if (( clearline )) && [[ "$limit" < "$b" ]]; then - echo -ne "\n" - clearline=0 - fi - if [[ "$ref" < "$b" || "$ref" == "$b" ]]; then - echo "!$b:$f" - (( ++mustrun )) - runreason+=( "${b}_${f}" ) - elif [[ "$limit" < "$b" ]] && scanmissingsamples $t; then - (( ++mustrun )) - runreason+=( "${b}_${f}" ) - else - if [[ "$limit" < "$b" ]]; then - echo -e "\r($b:$f)\e[K" - else - echo -ne "($b:$f)\t" - clearline=1 - fi - fi - # sanity check - if [[ "$now" < "$b" ]]; then - echo "oops: in the future $b vs $now" - fi - done - - # are we allowed to submit jobs ? - if (( donotsubmit )); then - echo -e '\e[35;1mWill NOT submit jobs\e[0m...' > /dev/stderr - if (( mustrun )); then - echo 'submit blocked' > ${statusdir}/submit_fail - echo -e '...\e[33;1mbut there are new jobs that should be started !!!\e[0m' > /dev/stderr - else - echo '...and there is nothing to run anyway' > /dev/stderr - fi - # start jobs ? - elif (( mustrun )); then - echo 'Will start new job' - - # Sanity check - if [[ ( -e ${statusdir}/sortsamples_fail ) && ( ${statusdir}/sortsamples_fail -nt ${statusdir}/sortsamples_success ) ]]; then - if (( staging )); then - echo -e '\e[33;1mwarning: sampleset data not successfully fetched yet, using staging\e[0m' > /dev/stderr - else - echo 'data fetch error' > ${statusdir}/submit_fail - echo -e '\e[31;1merror: sampleset data not successfully fetched yet\e[0m' > /dev/stderr - exit 1 - fi - fi - if [[ ( ! -e ${statusdir}/syncopenbis_new ) || ( ( -e ${statusdir}/vpipe_started ) && ( ${statusdir}/vpipe_started -nt ${statusdir}/syncopenbis_new ) ) ]]; then - echo 'oops: something fishy: no downloaded data newer than last run ?' > /dev/stderr - fi - # point of comparison for dates: - if [[ -e ${statusdir}/vpipe_ended ]]; then - lastrun=${statusdir}/vpipe_ended - else - # find the most recent 'new' sync status - lastsync=( $(ls -t ${statusdir}/sync*_new) ) - if [[ -e "${lastsync[0]}" ]]; then - lastrun="${lastsync[0]}" - else - # last fall back: sort success - lastrun=${statusdir}/sortsamples_success - fi - fi - - # push sampleset data - if [[ ( ! -e ${statusdir}/pushsampleset_success ) || ( ${lastrun} -nt ${statusdir}/pushsampleset_success ) ]]; then - ${scriptdir}/belfry pushsampleset --recent - else - echo 'oops: Sampleset already pushed' > /dev/stderr - fi - - # must run - if [[ ( -e ${statusdir}/pushsampleset_fail ) && ( ${statusdir}/pushsampleset_fail -nt ${statusdir}/pushsampleset_success ) ]]; then - echo 'error: Pushing sampleset did not succeed' > /dev/stderr - else - echo 'starting jobs' - if (( run_shorah )); then - shorah="" - else - shorah="--no-shorah" - fi - ${remote_batman} addsamples --recent && \ - ${remote_batman} vpipe ${shorah} --recent --tag "$(join_by ';' "${runreason[@]}")" > ${statusdir}/vpipe.${now} && \ - if [[ -s ${statusdir}/vpipe.${now} ]]; then - ln -sf vpipe.${now} ${statusdir}/vpipe_started - cat ${statusdir}/vpipe_started - printf "%s\t$(date '+%H%M%S')\n" "${runreason[@]}" | tee -a ${statusdir}/vpipe_new.${now} - if [[ -n "${mailto[*]}" ]]; then - ( - echo '(Possibly new) samples not having consensus sequences yet found in batches:' - printf ' - %s\n' "${runreason[@]}" - echo -e '\nStarting V-pipe on Euler:' - cat ${statusdir}/vpipe_started - ) | mail -s '[Automation-carillon] Starting V-pipe on Euler' "${mailto[@]}" - # -r "${mailfrom}" - fi - fi - fi - else - echo 'No new jobs to start' - - # check for left-over .staging files - staging_tsv=( sampleset/samples.202*.tsv.staging ) - if [[ "${staging_tsv[*]}" =~ \* ]]; then - # no staging files => safe to purge - ${scriptdir}/belfry purgeviollier - else - echo 'possible unsubmitted failed import?' > ${statusdir}/submit_fail - echo -e '\e[31;1mLeft-over staging files\e[0m' > /dev/stderr - printf ' - %s\n' "${staging_tsv[@]##*/}" > /dev/stderr - fi - - fi -else - echo 'There is already run going on' -fi - -# -# Closing words -# - -${scriptdir}/belfry df -${remote_batman} df -date -R diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..6bf5644 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,44 @@ +services: + pangolin: + build: + context: /data/projects/wastewater_automation/pangolin + volumes: + - /data/projects/wastewater_automation/workdir:/app/workdir:rw + - /data/projects/wastewater_automation/resources:/app/resources:ro + - /data/projects/dataset:/app/dataset:rw + - /data/projects/wastewater_automation/pangolin:/app/codebase:ro + secrets: + - bs-pangolin_login_euler + - bs-pangolin_login_backups + - fgcz_login + - id_ed25519_wisedb + - id_ed25519_wisedb_backups + - rsync.pass.euler + - spsp_gpg_key + - bs_pangolin_gpg_key + - bs_pangolin_gpg_key_password + - sendcrypt_profile + - spsp_uploads_ssh_private_key +secrets: + bs-pangolin_login_euler: + file: /data/projects/wastewater_automation/secrets/bs-pangolin@euler.ethz.ch + bs-pangolin_login_backups: + file: /data/projects/wastewater_automation/secrets/bs-pangolin@d@bs-bewi08 + fgcz_login: + file: /data/projects/wastewater_automation/secrets/fgcz-gstore.uzh.ch + id_ed25519_wisedb: + file: /data/projects/wastewater_automation/secrets/id_ed25519_wisedb + id_ed25519_wisedb_backups: + file: /data/projects/wastewater_automation/secrets/id_ed25519_wisedb_backups + rsync.pass.euler: + file: /data/projects/wastewater_automation/secrets/rsync.pass.euler + spsp_uploads_ssh_private_key: + file: /data/projects/wastewater_automation/secrets/id_ed25519_spsp_uploads + spsp_gpg_key: + file: /data/projects/wastewater_automation/secrets/ABC9FC14AAC952E7767FD14A48B70E724BAFE0A3.asc + bs_pangolin_gpg_key: + file: /data/projects/wastewater_automation/secrets/bs-pangolin_spsp-uploads_gpg-key.gpg + bs_pangolin_gpg_key_password: + file: /data/projects/wastewater_automation/secrets/gpg_key_secrets + sendcrypt_profile: + file: /data/projects/wastewater_automation/secrets/default.sendcrypt-profile diff --git a/pangolin_src/.dockerignore b/pangolin_src/.dockerignore new file mode 100644 index 0000000..c037f3e --- /dev/null +++ b/pangolin_src/.dockerignore @@ -0,0 +1 @@ +miniconda3 diff --git a/pangolin_src/.gitignore b/pangolin_src/.gitignore new file mode 100644 index 0000000..71d150b --- /dev/null +++ b/pangolin_src/.gitignore @@ -0,0 +1,36 @@ +quick_install.sh +Miniconda3-latest-Linux-x86_64.sh +miniconda3 +openbis-downloads +bfabric-downloads +sftp-health2030 +sftp-viollier +token.pickle +credentials.json +snake-envs +samples* +working/samples +working/samples.tsv +working/.snakemake/ +working/lsf.* +working/references/*.fasta.* +working/references/*.log +working/references/*.benchmark +working/variants/ +working/qa.csv +working/qa_report.html +working/qa_vals.csv +garbage/ +status/ +._* +.snapshot +.ipynb_checkpoints + +#logs +slurm-* + +# vim backup files: +.*.sw? + +# local dev +miniconda3 diff --git a/pangolin_src/.gitmodules b/pangolin_src/.gitmodules new file mode 100644 index 0000000..0a12af1 --- /dev/null +++ b/pangolin_src/.gitmodules @@ -0,0 +1,7 @@ +[submodule "V-pipe"] + path = V-pipe + url = https://github.com/cbg-ethz/V-pipe + +[submodule "V-pipe-test"] + path = V-pipe-test + url = https://github.com/cbg-ethz/V-pipe diff --git a/pangolin_src/README.md b/pangolin_src/README.md new file mode 100644 index 0000000..5c1ed29 --- /dev/null +++ b/pangolin_src/README.md @@ -0,0 +1,30 @@ +# pangolin + +This is a [SARS-CoV-2 sequencing project done at the BSSE](https://ethz.ch/en/news-and-events/eth-news/news/2020/04/analyses-for-getting-to-grips-with-the-pandemic.html) +relying on [V-pipe](https://cbg-ethz.github.io/V-pipe/sars-cov-2/) ([doi:10.1093/bioinformatics/btab015](https://doi.org/10.1093/bioinformatics/btab015)) +to analyse Illumina data done on [positive swabs test samples](https://bsse.ethz.ch/cevo/research/sars-cov-2/swiss-sequencing-consortium---viollier.html) +for the [genomic surveillance of the COVID-19 pandemic in Switzerland](https://sciencetaskforce.ch/en/nextstrain-phylogenetic-analysis/) +and on [wastewater samples](https://bsse.ethz.ch/news-and-events/d-bsse-news/2021/01/sars-cov-2-variants-detected-in-wastewater-samples.html) +provided by [EPFL](https://actu.epfl.ch/news/covid-19-using-wastewater-to-track-the-pandemic/) and [Eawag](https://www.eawag.ch/en/department/sww/projects/sars-cov2-in-wastewater/). + +## pipeline + +for the V-pipe pipeline, see: + + - tutorial: https://cbg-ethz.github.io/V-pipe/tutorial/sars-cov2/ ([video tutorial](https://youtu.be/pIby1UooK94)) + - SARS-CoV-2 subpage: https://cbg-ethz.github.io/V-pipe/sars-cov-2/ + - main website: https://cbg-ethz.github.io/V-pipe/ + - github repo: https://github.com/cbg-ethz/V-pipe + +## wastewater + +for the code used in wastewater analysis, see: + + - https://github.com/cbg-ethz/cojac + + +## video presentation + +Short Oral presentation done at the [ECCVID conference in September 2020](https://www.escmid.org/research_projects/escmid_conferences/past_escmid_conferences/eccvid/): + + - https://youtu.be/BJ-un88CT9A diff --git a/pangolin_src/batman.sh b/pangolin_src/batman.sh new file mode 100755 index 0000000..810d86d --- /dev/null +++ b/pangolin_src/batman.sh @@ -0,0 +1,555 @@ +#!/bin/bash + +scriptdir=/cluster/project/pangolin/test_automation/pangolin/pangolin_src +. ${scriptdir}/config/server.conf + +status=${clusterdir_old}/status +vilocadir=${remote_viloca_basedir}/${viloca_processing} + +eval "$(/cluster/project/pangolin/test_automation/miniconda3/bin/conda shell.bash hook)" + +# +# Input validator +# +validateBatchName() { + if [[ "$1" =~ ^(20[0-9][0-9][0-1][0-9][0-3][0-9]_[[:alnum:]-]{4,})$ ]]; then + return; + else + echo "bad batchname ${1}" + exit 1; + fi +} + +validateTags() { + local IFS="$1" + for b in $2; do + validateBatchName "${b}" + done +} + +# +# sync helper +# +checksyncoutput() { + local new="${status}/sync${1}_new" + local last="${status}/sync${1}_last" + + if [[ "${2}" =~ Total:\ +([[:digit:]]+)\ +directories,\ +([[:digit:]]+)\ +files,.*?New:\ +([[:digit:]]+)\ +files, ]]; then + echo "Newfiles downloaded" + echo -e "${BASH_REMATCH[2]}\n${BASH_REMATCH[1]}" > ${last} + flock -x -o ${last} -c "sleep 1" + echo "${BASH_REMATCH[3]}" > ${new} + else + echo "No files to sync found" + touch ${last} + fi 2>&1 +} + + +RXJOB='Job <([[:digit:]]+)> is submitted' +# Generic job. +# Job <129052039> is submitted to queue . + +now=$(date '+%Y%m%d') +lastmonth=$(date '+%Y%m' --date='-1 month') +thismonth=$(date '+%Y%m') +twoweeksago=$(date '+%Y%m%d' --date='-2 weeks') +year=$(date '+%Y') +lastyear=$(date '+%Y%m' --date='-1 year') + +if [[ "$1" == "--limited" ]]; then + shift + case "$1" in + rsync|df) + # sub-commands allowed in limited mode. + ;; + *) + echo "sub-command ${1} not allowed in limited mode" >&2 + exit 2 + ;; + esac +fi + +case "$1" in + rsync) + # rsync daemon : see ${SSH_ORIGINAL_COMMAND} + rsync --server --daemon . + ;; + logrotate) + # rotate logs + ~/log/rotate + ;; + addsamples) + lst="${clusterdir_old}/${working}/samples.tsv" + aviti=0 + case "$2" in + --recent) + lst="${clusterdir_old}/${working}/samples.recent.tsv" + echo "syncing recent: ${lastmonth}, ${thismonth}" + cat ${clusterdir_old}/${sampleset}/samples.{${lastmonth},${thismonth}}*.tsv | sort -u > "${clusterdir_old}/${working}/samples.recent.tsv" + ;; + --year) + lst="${clusterdir_old}/${working}/samples.recent.tsv" + echo "syncing year: ${year}" + cat ${clusterdir_old}/${sampleset}/samples.${year}*.tsv | sort -u > "${clusterdir_old}/${working}/samples.recent.tsv" + ;; + *) + echo "Unkown parameter ${2}" > /dev/stderr + exit 2 + ;; + + esac + mkdir -p --mode=2770 "${clusterdir_old}/${working}/samples/" + #cp -vrf --link ${clusterdir}/${sampleset}/*/ ${clusterdir}/${working}/samples/ ## failure: "no rule to create {SAMPLE}/extract/R1.fastq" + sort -u ${clusterdir_old}/${sampleset}/samples.*.tsv > "${clusterdir_old}/${working}/samples.tsv" + cut -f1 "${lst}" | xargs -P 8 -i cp -vrf --link "${clusterdir_old}/${sampleset}/{}/" "${clusterdir_old}/${working}/samples/" + lst="${clusterdir_old}/${working}/samples.tsv" + # Add abstractions and generalized to allow for new sequencing methods + mv ${clusterdir_old}/${working}/samples_aviti.tsv ${clusterdir_old}/${working}/samples_aviti.tsv.old + touch ${clusterdir_old}/${working}/samples_aviti.tsv + #mv ${clusterdir_old}/${working}/samples.tsv ${clusterdir_old}/${working}/samples.tsv_old + while IFS=$'\t' read -r col1 col2 col3 col4; do + if [ "${#col2}" -eq 19 ]; then + batch_date=${col2%%_*} + if [ "$batch_date" -gt $aviti_date ]; then + echo -e "${col1}\t${col2}\t${col3}\t${col4}" >> ${clusterdir_old}/${working}/samples_aviti.tsv + else + echo "skipping Aviti batch ${col2} as too old" + fi + else + echo -e "${col1}\t${col2}\t${col3}\t${col4}" >> ${clusterdir_old}/${working}/samples_pre-aviti.tsv + fi + done < ${clusterdir_old}/${working}/samples.tsv + ;; + vpipe) + declare -A job + list=('seq' 'seqqa' 'snv' 'snvqa' 'hugemem' 'hugememqa') + shorah=1 + hold= + tag= + aviti=0 + while [[ -n $2 ]]; do + case "$2" in + --no-shorah) + shorah=0 + ;; + --hold) + # use --hold to put on hold for analysis + hold='--hold' + ;; + --tag) + shift + if [[ ! "${2}" =~ ^[[:alnum:]]+$ ]]; then + # if it's not just letters and number, test if it is a list of valid batches + validateTags ';' "$2" + fi + tag="${2%;}" + ;; + --recent) +# # TODO switch between full cohort and only recent +# #recent="..." + ;; + --aviti) + aviti=1 + shorah=0 + ;; + esac + shift + done + # start first job + cd ${clusterdir_old}/${working}/ + if (( aviti )); then + echo "Processing Aviti" + job['seq']="$(sed "s/@TAG@/<${tag}>/g" vpipe_aviti.sbatch | sbatch --parsable ${hold} --job-name="COVID-AVITI-vpipe-<${tag}>-cons")" + else + job['seq']="$(sed "s/@TAG@/<${tag}>/g" vpipe-no-shorah.sbatch | sbatch --parsable ${hold} --job-name="COVID-vpipe-<${tag}>-cons")" + fi + if [[ -n "${job['seq']}" ]]; then + # schedule a gatherqa no mater what happens + job['seqqa']="$(sbatch --parsable ${hold} --job-name="COVID-qa-<${tag}>" --dependency="afterany:${job['seq']}" qa-launcher)" + # if no fail schedule a full job with snv + if (( shorah )); then + job['snv']="$(sbatch --parsable ${hold} --dependency="afterok:${job['seq']}" --kill-on-invalid-dep=yes vpipe.sbatch)" + if [[ -n "${job['snv']}" ]]; then + # schedule a gatherqa no matter what happens to snv + job['snvqa']="$(sbatch --parsable ${hold} --dependency="afterok:${job['seq']},afterany:${job['snv']}" --kill-on-invalid-dep=yes qa-launcher)" + # schedule a hugemem job if snvjob failed + job['hugemem']="$(sbatch --parsable ${hold} --dependency="afterok:${job['seq']},afternotok:${job['snv']}" --kill-on-invalid-dep=yes vpipe-hugemem.sbatch)" + # schedule a qa afterward + [[ -n "${job['hugemem']}" ]] && \ + job['hugememqa']="$(sbatch --parsable ${hold} --dependency="afterok:${job['seq']},afternotok:${job['snv']},afterany:${job['hugemem']}" --kill-on-invalid-dep=yes qa-launcher)" + fi + fi + fi >&2 + # write job chain list + for v in "${list[@]}"; do + printf "%s\t%s\n" "${v}" "${job[$v]}" + done + ;; + + job) + if [[ $2 =~ ^([[:digit:]]+)$ ]]; then + read output other < <(sacct -j "$2" --format State --noheader) + echo "${output}" + else + squeue + fi + + ;; + purgelogs) + find ${clusterdir_old}/${working}/cluster_logs/ -type f -mtime +28 -name '*.log' -print0 | xargs -0 rm -- + ;; + scratch) + temp_scratch="${SCRATCH}/pangolin/temp" + olderthan=60 + purge=0 + loop=0 + filter= + while [[ -n $2 ]]; do + case "$2" in + --minutes-ago) + if [[ ! "${3}" =~ ^[[:digit:]]+$ ]]; then + echo "parameter of ${2} must be a number of minutes (digits only), got <${3}> instead" > /dev/stderr + exit 2 + fi + shift + olderthan=$2 + ;; + --loop) + loop=1 + ;& + --purge) + purge=1 + ;; + *) + echo "Unkown parameter ${2}" > /dev/stderr + exit 2 + ;; + esac + shift + done + + if (( purge )); then + echo "purging in ${temp_scratch}..." + else + echo "listing in ${temp_scratch}..." + fi + + count_all=0 + count_old=0 + while read s; do + (( ++count_all )) + if [[ -r "${clusterdir_old}/${working}/samples/${s}/upload_prepared.touch" && $(find "${clusterdir_old}/${working}/samples/${s}/upload_prepared.touch" '!' -newermt "${olderthan} minutes ago") ]]; then + (( ++count_old )) + if (( purge )); then + rm -rvf "${temp_scratch}/samples/${s}" + else + echo "${s}"; + fi + fi; + done < <((cd "${temp_scratch}/" && find samples/ -type f ${filter} ) | grep -oP '(?<=samples/)[^/]+/[^/]+(?=/)' | sort -u) | tee /dev/stderr | wc -l 2>&1 + echo "Samples: ${count_old} old / ${count_all} total" + + if (( loop )); then + echo -e '\n\e[38;5;45;1mYou just keep on trying\e[0m\n\e[38;5;208;1mTill you run out of cake\e[0m' + if sleep "$(( 5 + olderthan))m"; then + # loop if no breaks + exec "${0}" scratch --minutes-ago "${olderthan}" --loop + fi + echo "I'm not even angry" + fi + ;; + completion) + if [[ $2 =~ ^([[:digit:]]+)$ ]]; then + gawk '$0~/^\[.*\]$/{date=$0};$0~/^[[:digit:]]+ of [[:digit:]]+ steps \([[:digit:].]+%\) done$/{print $0 "\t" date}' "${clusterdir_old}/${working}/slurm-${2}.out" + fi + ;; + df) + #df ${clusterdir_old} ${SCRATCH} + lquota -2 ${clusterdir_old} + lquota -2 ${SCRATCH} + ;; + garbage) + validateBatchName "$2" + cd ${clusterdir_old} + + for f in ${sampleset}/*/${2}; do + garbage=$(dirname "${f//${sampleset}/garbage}") + mkdir --mode=0770 -p "${garbage}" + mv -v "${f%/}" "${garbage}/" + done + for f in ${working}/samples/*/${2}/; do + garbage="${f//${working}\/samples/garbage}" + mkdir --mode=0770 -p "${garbage%/}/"{raw_data,extracted_data}/ + mv -vf "${f%/}/raw_data/"* "${garbage%/}/raw_data/" + rm "${f%/}/raw_data/"*.fa* + rmdir "${f%/}/raw_data/" + mv -vf "${f%/}/extracted_data/"* "${garbage%/}/extracted_data/" + rm "${f%/}/extracted_data/"*{.log,.benchmark,_fastqc.html} + rmdir "${f%/}/extracted_data/" + mv -vf "${f%/}/"* "${garbage%/}/" + rmdir "${f%/}" + done + mv "${sampleset}/batch.${2}.yaml" "${sampleset}/samples.${2}.tsv" "${sampleset}/missing.${2}.txt" "${sampleset}/projects.${2}.tsv" garbage/ + ;; + viloca) + cd ${vilocadir}/ + conda activate 'viloca' + . run_workflow.sh + # write job chain list + #for v in "${list[@]}"; do + # printf "%s\t%s\n" "${v}" "${job[$v]}" + #done + conda deactivate + ;; + check_viloca) + cd ${vilocadir}/ + grep -rq snake.err -e "JOB.*CANCELLED.*DUE TO TIME LIMIT" + ;; + unlock_viloca) + cd ${vilocadir}/ + conda activate 'viloca' + snakemake --unlock + conda deactivate + ;; + archive_viloca_run) + validateBatchName $2 + cd ${remote_viloca_basedir} + if [ ! -d results_archive ]; then + mkdir results_archive + fi + mkdir "results_archive/${2}" + mv "${vilocadir}/results/*" "results_archive/${2}" + ;; + create_sample_list_viloca) + validateBatchName $2 + echo ",sample,batch" > ${remote_viloca_basedir}/${viloca_staging} + cat ${clusterdir_old}/${sampleset}/samples.${2}.tsv | awk '{print $1,$2}' | tr " " "," | sed 's/^/,/' >> ${remote_viloca_basedir}/${viloca_staging} + ;; + finalize_staging_viloca) + mv ${remote_viloca_basedir}/${viloca_staging} ${remote_viloca_basedir}/${viloca_samples} + ;; + scanmissingsamples_viloca) + validateBatchName "$2" + for i in $(cat ${clusterdir_old}/${sampleset}/samples.${2}.tsv | awk '{print $1}') + do + if [ ! -d ${remote_viloca_basedir}/results_archive/${2}/${i} ] + then + echo ${i} + fi + done + ;; + sync_fgcz) + while [[ -n $2 ]]; do + case "$2" in + --https) + type='https' + ;; + --ftp) + type='ftp' + ;; + *) + echo "Unkown parameter ${2}" > /dev/stderr + exit 2 + ;; + esac + shift + done + bfabricdir=${clusterdir_old}/bfabric-downloads + cd ${bfabricdir} + sync_fgcz_statusdir=${status}/sync + mkdir -p $sync_fgcz_statusdir + fgcz_config=${clusterdir}/config/fgcz.conf + + echo "Sync FGCZ - bfabric" + + echo "Syncing from node $(hostname)" + conda activate sync + . <(grep '^projlist=' ${fgcz_config}) + if [[ "${3}" = "--recent" ]]; then + limitlast='3 weeks ago' + ${clusterdir}/exclude_list_bfabric.py -c ${fgcz_config} -r "${twoweeksago}" -o ${sync_fgcz_statusdir}/fgcz.exclude.lst + param=( '-e' "${sync_fgcz_statusdir}/fgcz.exclude.lst" "${projlist[@]}" ) + echo -ne "syncing recent: ${limitlast}\texcluding: " + wc -l ${sync_fgcz_statusdir}/fgcz.exclude.lst + else + param=( "${projlist[@]}" ) + fi + fail=0 + if [[ "${type}" = "https" ]]; then + syncoutput="$(${clusterdir}/sync_sftp.sh -c ${fgcz_config} ${limitlast:+ -N "${limitlast}"} -H "${param[@]}"|tee /dev/stderr)" || fail=1 + else + syncoutput="$(${clusterdir}/sync_sftp.sh -c ${fgcz_config} ${limitlast:+ -N "${limitlast}"} "${param[@]}"|tee /dev/stderr)" || fail=1 + fi + checksyncoutput "fgcz" "$syncoutput" + (( fail == 0 )) && touch ${sync_fgcz_statusdir}/sync_fgcz_success || touch ${sync_fgcz_statusdir}/sync_fgcz_fail + conda deactivate + ;; + sortsamples) + conda activate pybis + cd ${clusterdir} + sortsamples_statusdir=${status}/sortsamples + mkdir -p $sortsamples_statusdir + summary="" + recent="" + shrtrecent="" + force="${sort_force}" + while [[ -n $2 ]]; do + case "$2" in + --summary) + summary='--summary' + ;; + --force) + force='--force' + ;; + --recent) + recent="--recent=${lastmonth}" + shrtrecent="-r ${lastmonth}" + ;; + --year) + recent="--recent=${year}" + shrtrecent="-r ${year}" + ;; + *) + echo "Unkown parameter ${2}" > /dev/stderr + exit 2 + ;; + esac + shift + done + fail=0 + if (( ${lab[gfb]} == 1 )); then + ${clusterdir}/sort_samples_pybis.py -c ${clusterdir}/config/gfb.conf --protocols=${clusterdir_old}/${working}/${protocolyaml} --assume-same-protocol ${force} ${summary} ${recent} && bash ${clusterdir}/movedatafiles.sh || fail=1 + else + echo "Skipping gfb" + fi + if (( ${lab[fgcz]} == 1 )); then + . <(grep '^google_sheet_patches=' ${clusterdir}/config/fgcz.conf) + + (( google_sheet_patches )) && ${clusterdir}/google_sheet_patches.py + #${clusterdir}/sort_samples_bfabric_tsv.py -c ${clusterdir}/config/fgcz.conf --no-fastqc --protocols=${clusterdir_old}/${working}/${protocolyaml} --libkit-override=${clusterdir_old}/${sampleset}/patch.fgcz-libkit.tsv ${force} ${recent} && bash ${clusterdir}/movedatafiles.sh || fail=1 + ${clusterdir}/sort_samples_bfabric_tsv_aviti.py -c ${clusterdir}/config/fgcz.conf --no-fastqc --protocols=${clusterdir_old}/${working}/${protocolyaml} --libkit-override=${clusterdir_old}/${sampleset}/patch.fgcz-libkit.tsv ${force} ${recent} && bash ${clusterdir}/movedatafiles.sh || fail=1 + + else + echo "Skipping fgcz" + fi + if (( ${lab[h2030]} == 1 )) && [[ -e ${clusterdir}/synch2030_ended && -e ${clusterdir}/synch2030_started && ${clusterdir}/synch2030_ended -nt ${clusterdir}/synch2030_started ]]; then + # NOTE always recent/based on last sync), no support for --force, move done immediately/no separate movedatafiles.sh + # TODO support for protocols + ${clusterdir}/sort_h2030 -c ${clusterdir}/config/h2030.conf $(< ${clusterdir}/synch2030_started ) || fail=1 + else + echo "Skipping h2030" + fi + if (( ${lab[viollier]} == 1 )); then + # NOTE always --force, short options only + # HACK hardcoded paths due to multiple directories + ${clusterdir}/sort_viollier -c ${clusterdir}/config/viollier.conf -4 ${clusterdir}/${working}/${protocolyaml} ${shrtrecent} sftp-viollier/raw_sequences/*/ && bash ${clusterdir}/$movedatafiles.sh || fail=1 + else + echo "Skipping viollier" + fi + (( fail == 0 )) && touch ${sortsamples_statusdir}/sortsamples_success || touch ${sortsamples_statusdir}/sortsamples_fail + conda deactivate + ;; + scanmissingsamples) + sample_list=$2 + while read sample batch other; do + # look for only guaranteed samples + [[ $sample =~ $rxsample ]] || continue + # check the presence of fasta on each sample + echo -n ${sample}/${batch} + #ls ${clusterdir_old}/${working}/samples/${sample}/${batch} + if [[ -e ${clusterdir_old}/${working}/samples/${sample}/${batch}/upload_prepared.touch ]]; then + # this will check for: + # - references/ref_majority.fasta + # - references/consensus.bcftools.fasta & .chain + # - references/frameshift_deletions_check.tsv + # etc. + # see V-pipe's rule 'prepare_upload' in publish.smk + echo ' - .' + else + echo -e "\r+${batch/_/:}\t!${sample}\e[K" + true + exit 0 + fi + done < $sample_list + exit 1 + ;; + listsampleset) + case "$2" in + --recent) + ls -tr ${clusterdir_old}/${sampleset}/samples.20*.tsv | tail -n 12 + ;; + --all) + ls ${clusterdir_old}/${sampleset}/samples.20*.tsv + ;; + *) + echo "Unkown parameter ${2}" > /dev/stderr + exit 2 + ;; + esac + ;; + list_batch_samples) + validateBatchName "$2" + cat ${clusterdir_old}/${sampleset}/samples.${2}.tsv + ;; + amplicon_coverage) + case "$2" in + --batch) + validateBatchName "$3" + echo "Running amplicon coverage on batch $3" + amplicon_coverage_sample_list=${clusterdir_old}/${sampleset}/samples.${3}.tsv + amplicon_coverage_outdir=${remote_amplicon_coverage_workdir}/${3} + ;; + --timeframe) + echo "Running amplicon coverage for any sample within the timeframe $3" + startdate=${3%%-*} + enddate=${3##*-} + amplicon_coverage_sample_list=${remote_amplicon_coverage_tempdir}/samples.${3}.tsv + if [ -f ${amplicon_coverage_sample_list} ]; then + rm "${amplicon_coverage_sample_list}" + fi + alldates=$(cat ${clusterdir_old}/${working}/samples.tsv | awk '{print $2}' | awk -F'_' '{print $1}' | awk -v var=$enddate 'NR==1 { print } NR != 1 && $1 <= var { print }' | awk -v var=$startdate 'NR==1 { print } NR != 1 && $1 >= var { print }' | sort | uniq) + for i in $alldates; do + grep ${i} ${clusterdir_old}/${working}/samples.wastewateronly.tsv >> ${amplicon_coverage_sample_list} + done + amplicon_coverage_outdir=${remote_amplicon_coverage_workdir}/manual_${3} + ;; + --libkit) + echo "Running amplicon coverage for any sample with library kit $3" + grep ${3} ${clusterdir_old}/${working}/samples.wastewateronly.tsv > ${remote_amplicon_coverage_tempdir}/samples.${3}.tsv + amplicon_coverage_sample_list=${remote_amplicon_coverage_tempdir}/samples.${3}.tsv + amplicon_coverage_outdir=${remote_amplicon_coverage_workdir}/manual_${3} + ;; + esac + conda activate amplicon_coverage + cd ${remote_amplicon_coverage_workdir} + if [ ! -d ${amplicon_coverage_outdir} ]; then + mkdir ${amplicon_coverage_outdir} + python ${remote_amplicon_coverage_code}/amplicon_covs.py \ + -pv \ + -s ${amplicon_coverage_sample_list} \ + -r ${remote_primers_bed} \ + -o ${amplicon_coverage_outdir} \ + -f ${clusterdir_old}/${working}/samples || rmdir ${amplicon_coverage_outdir} + else + echo "ERROR: the amplicon coverage output directory ${amplicon_coverage_outdir} already exists. SKIPPING" + exit 5 + fi + ;; + get_vpipe_commit) + cd ${vpipe_code} + branch=$(git status | head -n 1 | sed -e 's/# On branch //') + commit=$(git log -n 1 ${branch} | head -n 1) + echo "Branch: ${branch}\n${commit}" + ;; + get_viloca_commit) + cd ${remote_viloca_basedir}/${viloca_processing} + branch=$(git status | head -n 1 | sed -e 's/# On branch //') + commit=$(git log -n 1 ${branch} | head -n 1) + echo "Branch: ${branch}\n${commit}" + ;; + *) + echo "Unkown sub-command ${1}" > /dev/stderr + exit 2 + ;; +esac diff --git a/pangolin_src/belfry.sh b/pangolin_src/belfry.sh new file mode 100755 index 0000000..3daa6dc --- /dev/null +++ b/pangolin_src/belfry.sh @@ -0,0 +1,494 @@ +#!/usr/bin/env bash + +scriptdir=/app/pangolin_src + +if [[ $(uname) == Darwin ]]; then + date=gdate +else + date=date +fi +now=$($date '+%Y%m%d') +lastmonth=$($date '+%Y%m' --date='-1 month') +thismonth=$($date '+%Y%m') +twoweeksago=$($date '+%Y%m%d' --date='-2 weeks') +oneweekago=$($date '+%Y%m%d' --date='-1 weeks') + + +declare -A lab +. ${scriptdir}/config/server.conf + +: ${basedir:=$(pwd)} +: ${download:?} +: ${sampleset:=sampleset} +: ${working:=working} +: ${storgrp:?} +: ${parallel:=16} +: ${parallelpull:=${parallel}} +: ${contimeout:=300} +: ${retries:=10} +: ${iotimeout:=300} +: ${protocolyaml:=/references/primers.yaml} +: ${viloca_basedir:?} +: ${viloca_samples:?} +: ${viloca_results:?} + +if [[ $(realpath $scriptdir) != $(realpath $basedir) ]]; then + echo "$scriptdir vs $basedir" +fi + +if [[ ! $mode =~ ^[0-7]{,4}$ ]]; then + echo "Invalid characters <${mode//[0-7]/}> in <${mode}>" + echo 'mode should be an octal chmod value, see `mkdir --help` for informations' + mode= +fi + +set -e + +# cd ${basedir} + +umask 0002 + +mkdir ${mode:+--mode=${mode}} -p ${statusdir} +mkdir ${mode:+--mode=${mode}} -p ${viloca_statusdir} +mkdir ${mode:+--mode=${mode}} -p ${uploader_statusdir} + +timeoutforeground= +#--foreground +# + +# +# Input validator +# +validateBatchDate() { + if [[ "$1" =~ ^(20[0-9][0-9][0-1][0-9][0-3][0-9])$ ]]; then + return; + else + echo "bad batchdate ${1}" + exit 1; + fi +} + +validateBatchName() { + if [[ "$1" =~ ^(20[0-9][0-9][0-1][0-9][0-3][0-9]_[[:alnum:]-]{4,})$ ]]; then + return; + else + echo "bad batchname ${1}" + exit 1; + fi +} + + +# +# rsync parallel helpers +# +callpushrsync() { + scriptdir=/app/pangolin_src + . ${scriptdir}/config/server.conf + + local arglist=( ) + if (( ${#@} )); then + arglist=( "${@/#/${basedir}/${sampleset}/}" ) + else + #arglist=( "${basedir}/${sampleset}/" ) + echo "rsync job didn't receive list" + exit 1; + fi + exec timeout ${timeoutforeground} --signal=INT --kill-after=5 $((rsynctimeout+contimeout+5)) \ + rsync --timeout=${iotimeout} \ + --password-file ${rsync_pass} \ + -e "ssh -i ${HOME}/.ssh/id_ed25519_wisedb -l ${cluster_user} -oConnectTimeout=${contimeout}" \ + -izrltH --fuzzy --fuzzy --inplace \ + -p --chmod=Dg+s,ug+rw,o-rwx,Fa-x \ + -g --chown=:'bsse-covid19-pangolin-euler' \ + "${arglist[@]}" \ + belfry@euler.ethz.ch::${sampleset}/ +} +export -f callpushrsync + + +callpullrsync_fordb() { + scriptdir=/app/pangolin_src + . ${scriptdir}/config/server.conf + + local arglist=( ) + if (( ${#@} )); then + arglist=( "${@/#/belfry@euler.ethz.ch::${working}/samples/}" ) + else + #arglist=( "belfry@euler.ethz.ch::${working}/samples/" ) + echo "rsync job didn't receive list" + exit 1; + fi + exec timeout ${timeoutforeground} --signal=INT --kill-after=5 $((rsynctimeout+contimeout+5)) \ + rsync --timeout=${iotimeout} \ + --password-file ${rsync_pass} \ + -e "ssh -i ${HOME}/.ssh/id_ed25519_wisedb -l ${cluster_user} -oConnectTimeout=${contimeout}" \ + -izrltHLK --fuzzy --fuzzy --inplace \ + --link-dest=${local_dataset}/${working}/samples \ + --exclude='alignments/' \ + --exclude='extracted_data/' \ + --exclude='preprocessed_data/' \ + --exclude='raw_data/' \ + --exclude='raw_uploads/' \ + --exclude='references/' \ + --exclude='variants/' \ + --exclude='visualization/' \ + --exclude='*.out.log' \ + --exclude='*.err.log' \ + --exclude='*.benchmark' \ + --exclude='*fastq.gz' \ + "${arglist[@]}" \ + ${local_dataset}/${working}/samples/ +} +export -f callpullrsync_fordb + +callpullrsync_viloca() { + scriptdir=/app/pangolin_src + . ${scriptdir}/config/server.conf + + local arglist=( ) + if (( ${#@} )); then + arglist=( "${@/#/belfry@euler.ethz.ch::${work_viloca}/${viloca_results}}" ) + else + #arglist=( "belfry@euler.ethz.ch::${working}/samples/" ) + echo "rsync job didn't receive list" + exit 1; + fi + exec timeout ${timeoutforeground} --signal=INT --kill-after=5 $((rsynctimeout+contimeout+5)) \ + rsync --timeout=${iotimeout} \ + --password-file ${rsync_pass} \ + -e "ssh -i ${HOME}/.ssh/id_ed25519_wisedb -l ${cluster_user} -oConnectTimeout=${contimeout}" \ + -izrltH --fuzzy --fuzzy --inplace \ + --link-dest=${$backupdir}/${viloca_backup_subdir}/ \ + "${arglist[@]}" \ + ${backupdir}/${viloca_backup_subdir} +} +export -f callpullrsync_viloca + +callpullrsync_rsync() { + scriptdir=/app/pangolin_src + . ${scriptdir}/config/server.conf + + local arglist=( ) + if (( ${#@} )); then + arglist=( "${@/#/belfry@euler.ethz.ch::${bfabric_downloads}}" ) + else + #arglist=( "belfry@euler.ethz.ch::${working}/samples/" ) + echo "rsync job didn't receive list" + exit 1; + fi + exec timeout ${timeoutforeground} --signal=INT --kill-after=5 $((rsynctimeout+contimeout+5)) \ + rsync --timeout=${iotimeout} \ + --password-file ${rsync_pass} \ + -e "ssh -i ${HOME}/.ssh/id_ed25519_wisedb -l ${cluster_user} -oConnectTimeout=${contimeout}" \ + -izrltH --fuzzy --fuzzy --inplace \ + --link-dest=${backupdir}/${sync_backup_subdir} \ + "${arglist[@]}" \ + ${backupdir}/${sync_backup_subdir} +} +export -f callpullrsync_rsync + +# +# sync helper +# +checksyncoutput() { + local new="${statusdir}/sync${1}_new" + local last="${statusdir}/sync${1}_last" + + if [[ "${2}" =~ Total:\ +([[:digit:]]+)\ +directories,\ +([[:digit:]]+)\ +files,.*?New:\ +([[:digit:]]+)\ +files, ]]; then + echo "Newfiles downloaded" + echo -e "${BASH_REMATCH[2]}\n${BASH_REMATCH[1]}" > ${last} + flock -x -o ${last} -c "sleep 1" + echo "${BASH_REMATCH[3]}" > ${new} + else + echo "No files to sync found" + touch ${last} + fi 2>&1 +} + + +set -e + +echo belfry.sh $1 + +# +# main handler +# +case "$1" in + qa_report) + . $baseconda/mambaforge/bin/activate qa_report + cd ${basedir}/${working}/ + if python qa_report.py; then + echo qa report succeeded + touch ${statusdir}/qa_report_success; + else + echo qa report failed + touch ${statusdir}/qa_report_fail + fi + mamba deactivate + ;; + pushseq) + . $baseconda/mambaforge/bin/activate "wastewater" + ./upload_viollier && touch ${statusdir}/pushseq_success || touch ${statusdir}/pushseq_fail + mamba deactivate + ;; + gitaddseq) + . $baseconda/mambaforge/bin/activate "wastewater" + cd ${releasedir} + git add qa* + find samples -name '*.fasta' -type f -print0 | xargs -0 git add + mamba deactivate + ;; + df) + df -h ${basedir} + echo + df -h --inode ${basedir} + ;; + garbage) + validateBatchName "$2" + + for f in ${sampleset}/*/${2}; do + garbage=$(dirname "${f//${sampleset}/garbage}") + mkdir ${mode:+--mode=${mode}} -p "${garbage}" + mv -v "${f%/}" "${garbage}/" + done + for f in ${working}/samples/*/${2}/; do + garbage="${f//${working}\/samples/garbage}" + mkdir ${mode:+--mode=${mode}} -p "${garbage%/}/"{raw_data,extracted_data}/ + mv -vf "${f%/}/raw_data/"* "${garbage%/}/raw_data/" + rm "${f%/}/raw_data/"*.fa* + rmdir "${f%/}/raw_data/" + #mv -vf "${f%/}/extracted_data/"* "${garbage%/}/extracted_data/" + #rm "${f%/}/extracted_data/"*_fastqc.html + #rmdir "${f%/}/extracted_data/" + mv -vf "${f%/}/"* "${garbage%/}/" + rmdir "${f%/}" + done + mv "${sampleset}/batch.${2}.yaml" "${sampleset}/samples.${2}.tsv" "${sampleset}/missing.${2}.txt" "${sampleset}/projects.${2}.tsv" garbage/ + ;; + pull_sync_status) + echo "Pulling the updated status of the raw data sync" + err=0 + rsync \ + --password-file ${rsync_pass} \ + -e "ssh -i ${HOME}/.ssh/id_ed25519_wisedb -l ${cluster_user} " \ + -izrltH --fuzzy --fuzzy --inplace \ + -p --chmod=Dg+s,ug+rw,o-rwx \ + -g --chown=:"${storgrp}" \ + belfry@euler.ethz.ch::${remote_status}/sync/ \ + ${statusdir}/remote_sync || (( ++err )) + if (( err )); then + echo "Error: ${err} rsync job(s) failed" + touch ${statusdir}/pull_sync_status_fail + else + touch ${statusdir}/pull_sync_status_success + fi + ;; + pull_sortsamples_status) + echo "Pulling the updated status of the sortsamples procedure" + err=0 + rsync \ + --password-file ${rsync_pass} \ + -e "ssh -i ${HOME}/.ssh/id_ed25519_wisedb -l ${cluster_user} " \ + -izrltH --fuzzy --fuzzy --inplace \ + -p --chmod=Dg+s,ug+rw,o-rwx \ + -g --chown=:"${storgrp}" \ + belfry@euler.ethz.ch::${remote_status}/sortsamples/* \ + ${statusdir}/remote_sortsamples || (( ++err )) + if (( err )); then + echo "Error: ${err} rsync job(s) failed" + touch ${statusdir}/pull_sortsamples_status_fail + else + touch ${statusdir}/pull_sortsamples_status_success + fi + ;; + pushsamplelist_viloca) + echo "Pushing the VILOCA sample list to the remote" + err=0 + rsync \ + --password-file ${rsync_pass} \ + -e "ssh -i ${HOME}/.ssh/id_ed25519_wisedb -l ${cluster_user} " \ + -izrltH --fuzzy --fuzzy --inplace \ + -p --chmod=Dg+s,ug+rw,o-rwx \ + -g --chown=:"${storgrp}" \ + ${viloca_basedir}/${viloca_samples} \ + belfry@euler.ethz.ch::${viloca_processing}/ || (( ++err )) + if (( err )); then + echo "Error: ${err} rsync job(s) failed" + touch ${viloca_statusdir}/pushsamplelist_viloca_fail + else + touch ${viloca_statusdir}/pushsamplelist_viloca_success + fi + ;; + queue_upload) + echo "Adding new samples to the upload list" + validateBatchName "$2" + cd ${uploader_workdir} + sheets=( "belfry@euler.ethz.ch::${sampleset}/samples.${2}.tsv" ) + rsync \ + --password-file ${HOME}/.ssh/rsync.pass.euler \ + -e "ssh -i ${HOME}/.ssh/id_ed25519_wisedb -l ${cluster_user} " \ + -izrltHLK --fuzzy --fuzzy --inplace \ + "${sheets[@]}" \ + ${basedir}/tmp/belfrysheets/ + if [ -f ${uploader_sampleset}/samples.${2}.tsv ]; then + # Add the new batch on top of the list. This ensures that the most recent batches are uploaded first, in case of retrospective uploads + cat ${uploader_sampleset}/samples.${2}.tsv | awk '{print $1,$2}' | sed -e 's/ /\t/' | cat - ${uploader_workdir}/${uploaderlist} > ${uploader_workdir}/${uploaderlist}_temp.txt && \ + mv ${uploader_workdir}/${uploaderlist}_temp.txt ${uploader_workdir}/${uploaderlist} && \ + # Remove possible duplicates after updating the upload list + cat -n ${uploader_workdir}/${uploaderlist} | sort -uk2 | sort -n | cut -f2- > ${uploader_workdir}/.working_${uploaderlist} && \ + mv ${uploader_workdir}/.working_${uploaderlist} ${uploader_workdir}/${uploaderlist} + else + echo "WARNING: the sampleset for the selected batch has not been synced yet.\nPossible causes: the sample is not complete; errors in the sync process; the name is incorrect." + fi + ;; + upload) + . ${scriptdir}/config/server.conf + echo "uploading ${uploader_sample_number} samples from the list of samples to upload" + source ${baseconda}/etc/profile.d/conda.sh + conda activate sendcrypt + cd ${uploader_workdir} + . ${uploader_code}/prepare.sh -N ${uploader_sample_number} -c ${scriptdir}/config/server.conf -b ${uploader_blacklist} + if [ -f ${uploader_tempdir}/cram_to_download.txt ]; then + rm ${uploader_tempdir}/cram_to_download.txt + fi + while IFS= read -r line; do + line2=$(echo ${line} | sed 's/ /\//g') + echo "${line2}/uploads/dehuman.cram" >> ${uploader_tempdir}/cram_to_download.txt + done < "${uploader_tempdir}/to_upload.txt" + echo "Downloading from Euler the necessary cram files" + rsync \ + --password-file ${rsync_pass} \ + -e "ssh -i ${HOME}/.ssh/id_ed25519_wisedb -l ${cluster_user} -oConnectTimeout=${contimeout}" \ + -irltHLK --fuzzy --fuzzy --inplace \ + --files-from=${uploader_tempdir}/cram_to_download.txt \ + --link-dest=${local_dataset}/${working} \ + --exclude='alignments/' \ + --exclude='extracted_data/' \ + --exclude='preprocessed_data/' \ + --exclude='raw_data/' \ + --exclude='raw_uploads/' \ + --exclude='references/' \ + --exclude='variants/' \ + --exclude='visualization/' \ + --exclude='*.out.log' \ + --exclude='*.err.log' \ + --exclude='*.benchmark' \ + --exclude='*fastq.gz' \ + belfry@euler.ethz.ch::${working}/samples \ + ${local_dataset}/${working}/samples/ + rsync \ + --password-file ${rsync_pass} \ + -e "ssh -i ${HOME}/.ssh/id_ed25519_wisedb -l ${cluster_user} -oConnectTimeout=${contimeout}" \ + -irltHLK --fuzzy --fuzzy --inplace \ + belfry@euler.ethz.ch::lollipop/variants/timeline.tsv \ + ${local_dataset}/${working} + rsync \ + --password-file ${rsync_pass} \ + -e "ssh -i ${HOME}/.ssh/id_ed25519_wisedb -l ${cluster_user} -oConnectTimeout=${contimeout}" \ + -irltHLK --fuzzy --fuzzy --inplace \ + belfry@euler.ethz.ch::${working}/qa.csv \ + ${local_dataset}/${working} + archive_now="${uploader_archive}/$(date +"%Y-%m-%d"-%H-%M-%S)" + mkdir -p $archive_now + ${uploader_code}/upload.sh ${archive_now} + metadata_len=$(wc -l ${uploader_target}/meta_data.tsv | awk '{print $1}') + if [ "${metadata_len}" == "1" ]; then + echo "Nothing to upload" | tee ${archive_now}/sencrypt.log + exit 0 + fi + if [ "${metadata_len}" == "0" ]; then + echo "ERROR: empty metadata file. Exiting." + exit 1 + fi + ( echo "Running sendCrypt" && \ + ${sendcrypt_exec} update && \ + ${sendcrypt_exec} version | tee ${archive_now}/sencrypt_version_used.txt && \ + ${sendcrypt_exec} send ${uploader_target} | tee ${archive_now}/sencrypt.log) || \ + (echo "ERROR: the upload failed" | tee ${archive_now}/sendcrypt_failed && \ + exit 1) + cat ${archive_now}/uploaded_run.txt >> ${uploader_uploaded} && \ + cp ${uploader_target}/meta_data.tsv ${archive_now} + for i in $(find ${local_dataset}/${working} -iname *cram); do + rm $i + done + ;; + clean_sendcrypt_temp) + echo "Clearning the sendcrypt temporary directories in ${uploader_tempdir}" + dir=$(ls -d ${uploader_tempdir}/*) + for i in $(find ${uploader_tempdir} -iname sendcrypt.*); do + if [ -d ${i} ]; then + echo "deleting ${i}" + rm -r ${i} + fi + rm -r ${uploader_tempdir}/target/* + done + ;; + pullsamples_for_db) + shopt -s globstar + # fetch remote sheets + mkdir -p ${basedir}/tmp/belfrysheets/ + if [[ "${2}" = "--recent" ]]; then + sheets=( "belfry@euler.ethz.ch::${sampleset}/samples.${lastmonth}*.tsv" "belfry@euler.ethz.ch::${sampleset}/samples.${thismonth}*.tsv" ) + elif [[ "${2}" = "--batch" ]]; then + validateBatchName "${3}" + sheets=( "belfry@euler.ethz.ch::${sampleset}/samples.${3}.tsv" ) + else + sheets=( "belfry@euler.ethz.ch::${sampleset}/samples.2*.tsv" ) + fi + rsync \ + --password-file ${HOME}/.ssh/rsync.pass.euler \ + -e "ssh -i ${HOME}/.ssh/id_ed25519_wisedb -l ${cluster_user} " \ + -izrltHLK --fuzzy --fuzzy --inplace \ + "${sheets[@]}" \ + ${basedir}/tmp/belfrysheets/ + if [[ "${2}" = "--recent" ]]; then + sheets=( ${basedir}/tmp/belfrysheets/samples.${lastmonth}*.tsv ${basedir}/tmp/belfrysheets/samples.${thismonth}*.tsv ) + # BUG: will generate non-globed pattern if months are missing + echo "pulling recent: ${param[*]##/}" + elif [[ "${2}" = "--batch" ]]; then + validateBatchName "${3}" + sheets=( "${basedir}/tmp/belfrysheets/samples.${3}.tsv" ) + elif [[ "${2}" = "--catchup" ]]; then + sheets=( "${basedir}/tmp/belfrysheets/samples.catchup.tsv" ) + else + sheets=( ${basedir}/tmp/belfrysheets/samples.2*.tsv ) + fi + echo "pulling: ${sheets[*]##/}" + err=0 + echo "samples:" + cut -s --fields=1 "${sheets[@]}"|sort -u| \ + gawk -v P=$(( parallelpull * 4 )) '{i=(NR-1);b=i%P;o[b]=(o[b] " \"" $1 "\"")};END{for(i=0;i /dev/stderr + exit 2 + ;; +esac diff --git a/pangolin_src/carillon.sh b/pangolin_src/carillon.sh new file mode 100755 index 0000000..19e4ffc --- /dev/null +++ b/pangolin_src/carillon.sh @@ -0,0 +1,589 @@ +#!/bin/bash + +scriptdir=/app/pangolin_src + +set -e + +. ${scriptdir}/config/server.conf + +: ${run_shorah:=0} +: ${staging:=1} + +if [[ $(realpath $scriptdir) != $(realpath $basedir) ]]; then + echo "The scripts are in $scriptdir" + echo "The base working directory for the automation is $basedir" +fi + +if [[ ! $mode =~ ^[0-7]{,4}$ ]]; then + echo "Invalid characters <${mode//[0-7]/}> in <${mode}>" + echo 'mode should be an octal chmod value, see `mkdir --help` for informations' + mode= +fi + +umask 0002 + +now=$(date '+%Y%m%d') + +mkdir ${mode:+--mode=${mode}} -p ${statusdir} +mkdir ${mode:+--mode=${mode}} -p ${viloca_statusdir} +mkdir ${mode:+--mode=${mode}} -p ${uploader_statusdir} +mkdir ${mode:+--mode=${mode}} -p ${amplicon_coverage_statusdir} + +touch ${statusdir}/oh_hai_im_looping + +source /home/bs-pangolin/.ssh/${cluster_user}@${cluster} +source /home/bs-pangolin/.ssh/${bck_user}@${bckhost} +remote_batman="ssh -o StrictHostKeyChecking=no -ni ${privkey} ${cluster_user}@${cluster} --" +remote_backup="ssh -o StrictHostKeyChecking=no -ni ${bck_privkey} ${bck_user}@${bckhost} --" + +echo "The current automation run is based on: " +${scriptdir}/belfry.sh get_pangolin_commit + +# +# Phase 1: periodic data sync +# + +echo '=========' +echo 'Data sync' +echo '=========' + +set -e + +if [[ -n $skipsync ]]; then + echo "${skipsync} will be skipped." +fi + +if [[ "${skipsync}" != "fgcz" ]]; then + ${remote_batman} sync_fgcz + ${scriptdir}/belfry.sh pull_sync_status + if [[ ( -e ${statusdir}/pull_sync_status_fail ) && ( ${statusdir}/pull_sync_status_fail -nt ${statusdir}/pull_sync_status_success ) ]]; then + echo "\e[31;1Pulling sync status files failed\e[0m" + echo "The automation will not be aware of any new deliveries" + else + if [ $backup_fgcz_raw -eq "1" ]; then + ${remote_backup} pull_fgcz_data --recent + if [[ ( -e ${statusdir}/pull_sync_status_fail ) && ( ${statusdir}/pull_sync_status_fail -nt ${statusdir}/pull_sync_status_success ) ]]; then + echo "\e[31;1Backup of fgcz raw data failed\e[0m" + echo "The system will retry next loop" + fi + else + echo "\e[33;1mBackup of FGCZ raw data DISABLED\e[0m" + fi + fi +fi +${remote_batman} sortsamples --recent $([[ ${statusdir}/syncopenbis_last -nt ${statusdir}/syncopenbis_new ]] && echo '--summary') +${scriptdir}/belfry.sh pull_sortsamples_status +if [[ ( -e ${statusdir}/pull_sortsamples_status_fail ) && ( ${statusdir}/pull_sortsamples_status_fail -nt ${statusdir}/pull_sortsamples_status_success ) ]]; then + echo "\e[31;1Pulling sortsamples status files failed\e[0m" + echo "The automation will not be aware of any new deliveries" +fi + + +# +# Phase 2: update status of current run and trigger backups +# + +echo "=================" +echo "Check current run" +echo "=================" + +if [[ ( -e ${statusdir}/vpipe_started ) && ( ( ! -e ${statusdir}/vpipe_ended ) || ( ${statusdir}/vpipe_started -nt ${statusdir}/vpipe_ended ) ) ]]; then + stillrunning=0 + echo "CHECK" + while read j id; do + echo $j $id + # skip missing + if [[ -z "${id}" ]]; then + echo "$j : (not started)" + continue + fi + + # skip already finished + if [[ ( -e ${statusdir}/vpipe_${j}_ended ) && ( ${statusdir}/vpipe_${j}_ended -nt ${statusdir}/vpipe_started ) ]]; then + old="$(<${statusdir}/vpipe_${j}_ended)" + if [[ "${id}" == "${old}" ]]; then + echo "$j : $id already finished" + else + echo "$j : mismatch $id vs $old" + fi + continue + fi + + # cluster status + stat=$(${remote_batman} job "${id}" || echo "(no answer)") + # HACK leaky abstraction ; keep in sync with profiles/smk-simple-slurm/status-sacct.sh + if [[ ( -n "${stat}" ) && ( "${stat}" =~ ^(RUNNING|PENDING|COMPLETING|CONFIGURING|SUSPENDED|\(no answer \)).* ) ]]; then + # running + echo -n "$j : $id : $stat" + (( ++stillrunning )) + if [[ "${stat}" =~ ^RUN && ( ! "$j" =~ qa$ ) ]]; then + echo -ne '\t' + ${remote_batman} completion "${id}" | tail -n 1 + else + echo '' + fi + sleep 1 + continue + fi + + # not running + echo "$j : $id finishing" + + case "$j" in + seqqa) + if [ $backup_vpipe -eq "1" ]; then + ${remote_backup} pullsamples_noshorah --recent + if [[ ( -e ${statusdir}/pullsamples_noshorah_fail ) && ( ${statusdir}/pullsamples_noshorah_fail -nt ${statusdir}/pullsamples_noshorah_success ) ]]; then + echo "\e[31;1mpulling data for database failed\e[0m" + (( ++stillrunning )) + continue + fi + else + echo "\e[33;1mBackup of V-PIPE data DISABLED\e[0m" + fi + #${scriptdir}/belfry.sh pullsamples_for_db --recent + #if [[ ( -e ${statusdir}/pullsamples_for_db_fail ) && ( ${statusdir}/pullsamples_for_db_fail -nt ${statusdir}/pullsamples_for_db_success ) ]]; then + # echo "\e[31;1mpulling data for database failed\e[0m" + # (( ++stillrunning )) + # continue + #fi + ;; + esac + + echo "${id}" > ${statusdir}/vpipe_${j}_ended + done < ${statusdir}/vpipe_started + echo done + + if (( stillrunning == 0 )); then + if [ $backup_vpipe -eq "1" ]; then + ${remote_backup} pullsamples_noshorah --recent + if [[ ( ! -e ${statusdir}/pullsamples_noshorah_success ) || ( ${statusdir}/pullsamples_noshorah_success -nt ${statusdir}/pullsamples_noshorah_fail ) ]]; then + echo "Pulling data success!" + else + echo "\e[31;1mpulling data failed\e[0m" + fi + else + echo "\e[33;1mBackup of VPIPE data DISABLED\e[0m" + fi + #${scriptdir}/belfry.sh pullsamples_for_db --recent + #if [[ ( ! -e ${statusdir}/pullsamples_for_db_success ) || ( ${statusdir}/pullsamples_for_db_success -nt ${statusdir}/pullsamples_for_db_fail ) ]]; then + # echo "Pulling data for database and uploads success!" + #else + # echo "\e[31;1mpulling data for database and uploads failed\e[0m" + #fi + + echo "$(basename $(realpath ${statusdir}/vpipe_started))" > ${statusdir}/vpipe_ended + vpipe_lastfile=$(cat ${statusdir}/vpipe_ended) + vpipe_enddate=${vpipe_lastfile##*.} + lastbatch_vpipe=$(cat ${statusdir}/vpipe_new.${vpipe_enddate} | head -n 1 | awk '{print $1}') + if [ $run_uploader -eq "1" ]; then + # queue the samples for upload. This will be handled in a dedicated section + echo Queueing last vpipe batch ${lastbatch_vpipe} for upload + ${scriptdir}/belfry.sh queue_upload ${lastbatch_vpipe} + fi + fi +else + echo 'No current run.' +fi + +# +# Phase 3: restart runs if new data +# + +echo "=============" +echo "Start new run" +echo "=============" + +# TODO support a yaml with regex +rxsample='([[:digit:]]{2}_20[[:digit:]]{2}_[01]?[[:digit:]]_[0-3]?[[:digit:]])' + +# like "$*" but with a different field separator than default. +join_by() { local IFS="$1"; shift; echo "$*"; } + +if [[ ( ( ! -e ${statusdir}/vpipe_ended ) && ( ! -e ${statusdir}/vpipe_started ) ) || ( ${statusdir}/vpipe_ended -nt ${statusdir}/vpipe_started ) ]]; then + echo check missing samples + clearline=0 + runreason=( ) + declare -A flowcell + # if test -e ${statusdir}/vpipe_started; then + ref=$(date --reference="${statusdir}/vpipe_started" '+%Y%m%d') + limit=$(date --date='2 weeks ago' '+%Y%m%d') + echo "Check batch against ${ref}:" + #for t in ${cluster_mount}/${sampleset}/samples.20*.tsv; do + for t in $(${remote_batman} listsampleset --all) + do + if [[ ! $t =~ samples.([[:digit:]]{8})_([[:alnum:]]{5,}(-[[:digit:]]+)?).tsv$ ]]; then + echo "oops: Can't parse <${t}> ?!" > /dev/stderr + fi + + # check Duplicates + b="${BASH_REMATCH[1]}" + f="${BASH_REMATCH[2]}" + + if [[ -n "${flowcell[$f]}" ]]; then + echo "error: Duplicate flowcell $f : ${flowcell[$f]} vs $b" > /dev/stderr + exit 2 + else + flowcell[$f]=$b + fi + + # check dates + if (( clearline )) && [[ "$limit" < "$b" ]]; then + echo -ne "\n" + clearline=0 + fi + if [[ "$ref" < "$b" || "$ref" == "$b" ]]; then + echo "!$b:$f" + (( ++mustrun )) + runreason+=( "${b}_${f}" ) + elif [[ "$limit" < "$b" ]]; then + if ${remote_batman} scanmissingsamples $t; then + (( ++mustrun )) + runreason+=( "${b}_${f}" ) + fi + else + if [[ "$limit" < "$b" ]]; then + echo -e "\r($b:$f)\e[K" + else + echo -ne "($b:$f)\t" + clearline=1 + fi + fi + # sanity check + if [[ "$now" < "$b" ]]; then + echo "oops: in the future $b vs $now" + fi + done + + # are we allowed to submit jobs ? + if (( donotsubmit )); then + echo -e '\e[35;1mWill NOT submit jobs\e[0m...' > /dev/stderr + if (( mustrun )); then + echo 'submit blocked' > ${statusdir}/submit_fail + echo -e '...\e[33;1mbut there are new jobs that should be started !!!\e[0m' > /dev/stderr + else + echo '...and there is nothing to run anyway' > /dev/stderr + fi + # start jobs ? + elif (( mustrun )); then + echo 'Will start new job' + + # Sanity check + if [[ ( -e ${statusdir}/remote_sortsamples/sortsamples_fail ) && ( ${statusdir}/remote_sortsamples/sortsamples_fail -nt ${statusdir}/remote_sortsamples/sortsamples_success ) ]]; then + if (( staging )); then + echo -e '\e[33;1mwarning: sampleset data not successfully fetched yet, using staging\e[0m' > /dev/stderr + else + echo 'data fetch error' > ${statusdir}/submit_fail + echo -e '\e[31;1merror: sampleset data not successfully fetched yet\e[0m' > /dev/stderr + exit 1 + fi + fi + if [[ ( ! -e ${statusdir}/syncopenbis_new ) || ( ( -e ${statusdir}/vpipe_started ) && ( ${statusdir}/vpipe_started -nt ${statusdir}/syncopenbis_new ) ) ]]; then + echo 'oops: something fishy: no downloaded data newer than last run ?' > /dev/stderr + fi + # point of comparison for dates: + if [[ -e ${statusdir}/vpipe_ended ]]; then + lastrun=${statusdir}/vpipe_ended + else + # find the most recent 'new' sync status + lastsync=( $(ls -t ${statusdir}/sync*_new) ) + if [[ -e "${lastsync[0]}" ]]; then + lastrun="${lastsync[0]}" + else + # last fall back: sort success + lastrun=${statusdir}/remote_sortsamples/sortsamples_success + fi + fi + + # must run + echo 'starting jobs' + if (( run_shorah )); then + shorah="" + else + shorah="--no-shorah" + fi + if (( skipaviti )); then + aviti="" + elif [ ${now} -ge ${aviti_date} ]; then + aviti="--aviti" + else + aviti="" + fi + ${remote_batman} addsamples --recent && \ + ${remote_batman} vpipe ${shorah} ${aviti} --tag "$(join_by ';' "${runreason[@]}")" > ${statusdir}/vpipe.${now} && \ + if [[ -s ${statusdir}/vpipe.${now} ]]; then + ln -sf ${statusdir}/vpipe.${now} ${statusdir}/vpipe_started + cat ${statusdir}/vpipe_started + printf "%s\t$(date '+%H%M%S')\n" "${runreason[@]}" | tee -a ${statusdir}/vpipe_new.${now} + ${remote_batman} get_vpipe_commit | tee -a ${statusdir}/vpipe_new.${now} + if [[ -n "${mailto[*]}" ]]; then + ( + echo '(Possibly new) samples not having consensus sequences yet found in batches:' + printf ' - %s\n' "${runreason[@]}" + echo -e '\nStarting V-pipe on Euler:' + cat ${statusdir}/vpipe_started + ) | mail -s '[Automation-carillon] Starting V-pipe on Euler' "${mailto[@]}" + # -r "${mailfrom}" + fi + fi + else + echo 'No new jobs to start' + fi +else + echo 'There is already a vpipe run going on' +fi + +# +# Phase 4: run viloca on new samples if no viloca instance is running +# +if [ "$run_viloca" -eq "1" ]; then + + echo "========================" + echo "Check current VILOCA run" + echo "========================" + + + if [[ ( -e ${viloca_statusdir}/viloca_started ) && ( ( ! -e ${viloca_statusdir}/viloca_ended ) || ( ${viloca_statusdir}/viloca_started -nt ${viloca_statusdir}/viloca_ended ) ) ]]; then + stillrunning=0 + timelimit_reached=0 + # skip missing + id=$(cat ${viloca_statusdir}/viloca_started) + id=${id#"Submitted batch job "} + if [[ -z "${id}" ]]; then + echo "VILOCA - $id : (not started)" + fi + # skip already finished + if [[ ( -e ${viloca_statusdir}/viloca_${j}_ended ) && ( ${viloca_statusdir}/viloca_${j}_ended -nt ${viloca_statusdir}/viloca_started ) ]]; then + old="$(<${viloca_statusdir}/viloca_${j}_ended)" + if [[ "${id}" == "${old}" ]]; then + echo "VILOCA : $id already finished" + stillrunning=0 + else + echo "VILOCA : mismatch $id vs $old" + fi + fi + # cluster status + stat=$(${remote_batman} job "${id}" || echo "(no answer)") + if [[ ( -n "${stat}" ) && ( "${stat}" =~ ^(RUNNING|PENDING|COMPLETING|CONFIGURING|SUSPENDED|\(no answer \)).* ) ]]; then # running + echo -n "VILOCA : $id : $stat" + echo "VILOCA : $id still running" + (( ++stillrunning )) + fi + if [[ ( -n "${stat}" ) && ( "${stat}" =~ ^( TIMEOUT ) ) ]]; then + echo -n "VILOCA : $id : $stat\n" + echo "VILOCA : $id reached time limit" + (( ++timelimit_reached )) + fi + if (( stillrunning == 0 )); then + if (( timelimit_reached > 0)); then + echo "Previous VILOCA run cancelled due to time limit. Restarting it" + ${remote_batman} unlock_viloca && \ + ${remote_batman} viloca > ${viloca_statusdir}/viloca.${now} && \ + if [[ -s ${viloca_statusdir}/viloca.${now} ]]; then + cat ${viloca_statusdir}/viloca.${now} > ${viloca_statusdir}/viloca_started + cat ${viloca_statusdir}/viloca_started + printf "%s\t$(date '+%H%M%S')\n" "${runreason[@]}" | tee -a ${viloca_statusdir}/viloca_new.${now} + ${remote_batman} get_viloca_commit | tee -a ${viloca_statusdir}/viloca_new.${now} + fi + else + lastbatch_viloca=$(cat $(ls -Art ${viloca_statusdir}/viloca_new* | tail -n 1) | head -n 1) + echo "Archiving the VILOCA run on batch ${lastbatch_viloca} to make space in the results directory for a new run" + ${remote_batman} archive_viloca_run ${lastbatch_viloca} || echo -e '...\e[33;1mFAILED TO ARCHIVE THE VILOCA RUN on batch ${lastbatch_viloca}\e[0m' + echo "${id}" > ${viloca_statusdir}/viloca_${j}_ended + echo "$(basename $(realpath ${viloca_statusdir}/viloca_started))" > ${viloca_statusdir}/viloca_ended + if [ $backup_viloca -eq "1" ]; then + ${remote_backup} pullresults_viloca --batch ${lastbatch_viloca} > ${viloca_statusdir}/backup_viloca_status_${now} + if [[ ( ! -e ${viloca_statusdir}/backup_viloca_status_${now} ) || $(cat ${viloca_statusdir}/backup_viloca_status_${now}) -eq "SUCCESS" ]]; then + echo "Backup of VILOCA results on bs-bewi08 success!" + else + echo "\e[31;1mBackup of VILOCA results on bs-bewi08 failed\e[0m" + fi + else + echo "\e[33;1mBackup of VILOCA results on bs-bewi08 DISABLED\e[0m" + fi + fi + else + echo VILOCA still running + fi + else + echo 'No current VILOCA run.' + fi + + # + # Phase 5: restart VILOCA runs if new data + # + + echo "====================" + echo "Start new VILOCA run" + echo "====================" + mustrun_viloca=0 + if [[ ( ( ! -e ${viloca_statusdir}/viloca_ended ) && ( ! -e ${viloca_statusdir}/viloca_started ) ) || ( ${viloca_statusdir}/viloca_ended -nt ${viloca_statusdir}/viloca_started ) ]]; then + lastbatch_viloca=$(cat $(ls -Art ${viloca_statusdir}/viloca_new* | tail -n 1) | head -n 1) + echo "Last batch analysed by VILOCA is ${lastbatch_viloca}" + vpipe_enddate=$(cat ${statusdir}/vpipe_ended) + vpipe_enddate=${vpipe_enddate#*.} + lastbatch_vpipe=$(cat ${statusdir}/vpipe_new.${vpipe_enddate} | head -n 1 | awk '{print $1}' | tail -n 1) + echo "The most recent completed V-Pipe run is on batch ${lastbatch_vpipe}" + if [[ $lastbatch_viloca != $lastbatch_vpipe ]]; then + echo "There is a new most recent batch that VILOCA can run on" + t=$(${remote_batman} listsampleset --all | grep samples.${lastbatch_vpipe}.tsv) + if [[ ! $t =~ samples.([[:digit:]]{8})_([[:alnum:]]{5,}(-[[:digit:]]+)?).tsv$ ]]; then + echo "oops: Can't parse <${t}> ?!" > /dev/stderr + fi + ${remote_batman} create_sample_list_viloca ${lastbatch_vpipe} + (( ++mustrun_viloca )) + else + echo "No new batch to run VILOCA on" + echo "Checking if the previous batch was successful" + not_processed=($(${remote_batman} scanmissingsamples_viloca $lastbatch_viloca)) + if [ "${not_processed}" -gt "0" ]; then + echo "Not all samples have been successfully completed. Repeating the run" + (( ++mustrun_viloca )) + else + echo "Previous batch appears successful" + echo "Nothing to do for VILOCA" + fi + fi + + # are we allowed to submit jobs ? + if (( donotsubmit_viloca == 1 )); then + echo -e '\e[35;1mWill NOT submit VILOCA jobs\e[0m...' > /dev/stderr + if (( mustrun_viloca > 0 )); then + echo 'VILOCA submit blocked' > ${viloca_statusdir}/viloca_submit_fail + echo -e '...\e[33;1mbut there are new VILOCA jobs that should be started !!!\e[0m' > /dev/stderr + else + echo '...and there is nothing VILOCA-related to run anyway' > /dev/stderr + fi + # start jobs ? + elif (( mustrun_viloca > 0 )); then + echo 'New VILOCA job waiting. Checking if Viloca is already running...' + if [[ ( -e ${viloca_statusdir}/viloca_started ) && ( ( ! -e ${viloca_statusdir}/viloca_ended ) || ( ${viloca_statusdir}/viloca_started -nt ${viloca_statusdir}/viloca_ended ) ) ]]; then + echo "BUT there is already a VILOCA instance running! Retrying during the next loop" + else + echo 'starting VILOCA jobs' + # we keep the staging file until the actual run so that, if anything goes wrong and VILOCA + # does not start for a while, the staging will be constantly updated with the latest batch + # and VILOCA will run only on the latest once it restarts + ${remote_batman} finalize_staging_viloca + # must run + ${remote_batman} viloca > ${viloca_statusdir}/viloca.${now} && \ + if [[ -s ${viloca_statusdir}/viloca.${now} ]]; then + cat ${viloca_statusdir}/viloca.${now} | tee ${viloca_statusdir}/viloca_started + echo ${lastbatch_vpipe} > ${viloca_statusdir}/viloca_new.${now} + printf "%s\t$(date '+%H%M%S')\n" "${runreason[@]}" | tee -a ${viloca_statusdir}/viloca_new.${now} + ${remote_batman} get_viloca_commit | tee -a ${viloca_statusdir}/viloca_new.${now} + if [[ -n "${mailto[*]}" ]]; then + ( + echo '(Possibly new) samples not having VILOCA results yet found:' + printf ' - %s\n' "${lastbatch_vpipe}" + echo -e '\nStarting VILOCA on Euler:' + cat ${viloca_statusdir}/viloca_started + ) | mail -s '[Automation-carillon] Starting VILOCA on Euler' "${mailto[@]}" + # -r "${mailfrom}" + fi + else + echo "ERROR: could not create ${viloca_statusdir}/viloca.${now}" + fi + fi + else + echo 'No new VILOCA run to submit' + fi + else + echo 'There is already A VILOCA run going on' + fi +else + echo 'Skipping VILOCA as per configuration' +fi + + + +# +# Phase 6: run uploader on new chunk +# +if [ ${donotsubmit_uploader} -eq "0" ]; then + + + echo "====================" + echo "Start new UPLOADER run" + echo "====================" + echo "Checking the upload quotas:" + echo "----" + if [[ ! -f ${uploader_number_status}.${now} ]]; then + echo "No status file with the amount of samples uploaded found. Assuming first run of the day" + echo 0 > ${uploader_number_status}.${now} + fi + uploaded_number=$(cat ${uploader_number_status}.${now}) + echo "Daily sample number: ${uploaded_number}/${upload_number_quota}" + echo "Daily size: $((${uploaded_number} * ${upload_avg_size}))/${upload_size_quota} MB" + echo "----" + next_number=$((${uploaded_number} + ${uploader_sample_number})) + echo "asked to upload ${uploader_sample_number} new samples, consisting of about $((${uploader_sample_number} * ${upload_avg_size})) MB" + if [ "${next_number}" -gt "${upload_number_quota}" ] || [ "$((${next_number} * ${upload_avg_size}))" -gt "${upload_size_quota}" ]; then + echo "We reached the daily submission quota imposed by SPSP for UPLOADS. Resuming tomorrow" + touch ${uploader_statusdir}/uploader_quota_hit.${now} + else + echo 'starting UPLOADER job' + ${scriptdir}/belfry.sh upload && \ + echo $(( ${uploaded_number} + ${uploader_sample_number} )) > ${uploader_number_status}.${now} + if [ ${clean_sendcrypt_temp} -eq "1" ]; then + echo "Cleaning the temporary folders generated by SendCrypt" + ${scriptdir}/belfry.sh clean_sendcrypt_temp + fi + fi + + if [ $backup_uploader -eq "1" ]; then + ${scriptdir}/belfry.sh backup_uploader > ${uploader_statusdir}/backup_uploader_status_${now} + if [[ ( ! -e ${uploader_statusdir}/backup_uploader_status_${now} ) || $(cat ${uploader_statusdir}/backup_uploader_status_${now}) -eq "SUCCESS" ]]; then + echo "Backup of UPLADER results on bs-bewi08 success!" + else + echo "\e[31;1mBackup of UPLOADER results on bs-bewi08 failed\e[0m" + fi + else + echo "\e[33;1mBackup of UPLOADER results on bs-bewi08 DISABLED\e[0m" + fi +else + echo "Skipping UPLOADER as per configuration" +fi + + +# +# Phase 7: run amplicon coverage on new batch +# +if [ $run_amplicon_coverage -eq "1" ]; then + echo "====================" + echo "Start new AMPLICON COVERAGE run" + echo "====================" + lastbatch_amplicon_coverage=$(cat $(ls -Art ${amplicon_coverage_statusdir}/amplicon_coverage_new* | tail -n 1) | head -n 1) + echo "Last batch analysed by AMPLICON COVERAGE is ${lastbatch_amplicon_coverage}" + vpipe_enddate=$(cat ${statusdir}/vpipe_ended) + vpipe_enddate=${vpipe_enddate#*.} + lastbatch_vpipe=$(cat ${statusdir}/vpipe_new.${vpipe_enddate} | head -n 1 | awk '{print $1}' | tail -n 1) + echo "The most recent completed V-Pipe run is on batch ${lastbatch_vpipe}" + if [[ $lastbatch_amplicon_coverage != $lastbatch_vpipe ]]; then + echo "There is a new most recent batch that AMPLICON COVERAGE can run on" + ${remote_batman} amplicon_coverage --batch ${lastbatch_vpipe} && echo ${lastbatch_vpipe} > ${amplicon_coverage_statusdir}/amplicon_coverage_new.${now} + else + echo "No new batch to run AMPLICON COVERAGE on" + fi + if [ ${backup_amplicon_cov} -eq "1" ]; then + ${remote_backup} pullresults_amplicon_cov > ${amplicon_coverage_statusdir}/backup_ampliconcov_status_${now} + if [[ ( ! -e ${amplicon_coverage_statusdir}/backup_ampliconcov_status_${now} ) || $(cat ${amplicon_coverage_statusdir}/backup_ampliconcov_status_${now} | tail -n 1) -eq "SUCCESS" ]]; then + echo "Backup of AMPLICON COVERAGE results on bs-bewi08 success!" + else + echo "\e[31;1mBackup of AMPLICON COVERAGE results on bs-bewi08 failed\e[0m" + fi + else + echo "\e[33;1mBackup of AMPLICON COVERAGE results on bs-bewi08 DISABLED\e[0m" + fi +else + echo "Skipping AMPLICON COVERAGE as per configuration" +fi + + +# +# Closing words +# + +${scriptdir}/belfry.sh df +${remote_batman} df +date -R diff --git a/check_sums b/pangolin_src/check_sums.sh similarity index 97% rename from check_sums rename to pangolin_src/check_sums.sh index 0f2c360..069ab9a 100755 --- a/check_sums +++ b/pangolin_src/check_sums.sh @@ -1,6 +1,6 @@ #!/bin/bash -configfile=server.conf +configfile=config/server.conf usage() { echo "Usage: $0 [-c ] " 1>&2; exit $1; } diff --git a/pangolin_src/conda_envs/amplicon_coverage.yaml b/pangolin_src/conda_envs/amplicon_coverage.yaml new file mode 100644 index 0000000..c5e59ea --- /dev/null +++ b/pangolin_src/conda_envs/amplicon_coverage.yaml @@ -0,0 +1,89 @@ +name: amplicon_coverage +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - brotli=1.1.0=hd590300_1 + - brotli-bin=1.1.0=hd590300_1 + - bzip2=1.0.8=hd590300_5 + - ca-certificates=2023.11.17=hbcca054_0 + - certifi=2023.11.17=pyhd8ed1ab_0 + - contourpy=1.2.0=py39ha90811c_0 + - cycler=0.12.1=pyhd8ed1ab_0 + - expat=2.5.0=hcb278e6_1 + - fonttools=4.46.0=py39hf860d4a_0 + - freetype=2.12.1=h267a509_2 + - gdbm=1.18=h0a1914f_2 + - importlib-resources=6.1.1=pyhd8ed1ab_0 + - importlib_resources=6.1.1=pyhd8ed1ab_0 + - kiwisolver=1.4.5=py39ha90811c_1 + - lcms2=2.16=hb7c19ff_0 + - lerc=4.0.0=h27087fc_0 + - libblas=3.9.0=20_linux64_openblas + - libbrotlicommon=1.1.0=hd590300_1 + - libbrotlidec=1.1.0=hd590300_1 + - libbrotlienc=1.1.0=hd590300_1 + - libcblas=3.9.0=20_linux64_openblas + - libdeflate=1.19=hd590300_0 + - libexpat=2.5.0=hcb278e6_1 + - libffi=3.4.2=h7f98852_5 + - libgcc-ng=13.2.0=h807b86a_3 + - libgfortran-ng=13.2.0=h69a702a_3 + - libgfortran5=13.2.0=ha4646dd_3 + - libgomp=13.2.0=h807b86a_3 + - libjpeg-turbo=3.0.0=hd590300_1 + - liblapack=3.9.0=20_linux64_openblas + - libopenblas=0.3.25=pthreads_h413a1c8_0 + - libpng=1.6.39=h753d276_0 + - libsqlite=3.44.2=h2797004_0 + - libstdcxx-ng=13.2.0=h7e041cc_3 + - libtiff=4.6.0=ha9c0a0a_2 + - libwebp-base=1.3.2=hd590300_0 + - libxcb=1.15=h0b41bf4_0 + - libzlib=1.2.13=hd590300_5 + - matplotlib=3.8.2=py39h4162558_0 + - matplotlib-base=3.8.2=py39h4e7d633_0 + - munkres=1.1.4=pyh9f0ad1d_0 + - ncurses=6.4=h59595ed_2 + - numpy=1.26.2=py39h6dedee3_0 + - openjpeg=2.5.0=h488ebb8_3 + - openssl=3.2.0=hd590300_1 + - packaging=23.2=pyhd8ed1ab_0 + - pandas=2.1.3=py39h3417b97_0 + - patsy=0.5.4=pyhd8ed1ab_0 + - pillow=10.1.0=py39hcf8a34e_0 + - pip=23.3.1=pyhd8ed1ab_0 + - pthread-stubs=0.4=h36c2ea0_1001 + - pyparsing=3.1.1=pyhd8ed1ab_0 + - pypy3.9=7.3.13=h9557127_1 + - python=3.9.18=0_73_pypy + - python-dateutil=2.8.2=pyhd8ed1ab_0 + - python-tzdata=2023.3=pyhd8ed1ab_0 + - python_abi=3.9=4_pypy39_pp73 + - pytz=2023.3.post1=pyhd8ed1ab_0 + - readline=8.2=h8228510_1 + - scipy=1.11.4=py39h6dedee3_0 + - seaborn=0.13.0=hd8ed1ab_0 + - seaborn-base=0.13.0=pyhd8ed1ab_0 + - setuptools=68.2.2=pyhd8ed1ab_0 + - six=1.16.0=pyh6c4a22f_0 + - sqlite=3.44.2=h2c6b66d_0 + - statsmodels=0.14.0=py39hf7fbf1e_2 + - tk=8.6.13=noxft_h4845f30_101 + - tornado=6.3.3=py39hf860d4a_1 + - tzdata=2023c=h71feb2d_0 + - unicodedata2=15.1.0=py39hf860d4a_0 + - wheel=0.42.0=pyhd8ed1ab_0 + - xorg-kbproto=1.0.7=h7f98852_1002 + - xorg-libx11=1.8.7=h8ee46fc_0 + - xorg-libxau=1.0.11=hd590300_0 + - xorg-libxdmcp=1.1.3=h7f98852_0 + - xorg-xextproto=7.3.0=h0b41bf4_1003 + - xorg-xproto=7.0.31=h7f98852_1007 + - xz=5.2.6=h166bdaf_0 + - zipp=3.17.0=pyhd8ed1ab_0 + - zlib=1.2.13=hd590300_5 + - zstd=1.5.5=hfc55251_0 diff --git a/pangolin_src/conda_envs/pybis.yaml b/pangolin_src/conda_envs/pybis.yaml new file mode 100644 index 0000000..a783438 --- /dev/null +++ b/pangolin_src/conda_envs/pybis.yaml @@ -0,0 +1,92 @@ +name: pybis +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - aiohttp=3.8.5=py311h5eee18b_0 + - aiosignal=1.2.0=pyhd3eb1b0_0 + - async-timeout=4.0.2=py311h06a4308_0 + - attrs=23.1.0=py311h06a4308_0 + - blas=1.0=mkl + - blinker=1.6.2=py311h06a4308_0 + - bottleneck=1.3.5=py311hbed6279_0 + - brotli-python=1.0.9=py311h6a678d5_7 + - bzip2=1.0.8=h7b6447c_0 + - ca-certificates=2023.08.22=h06a4308_0 + - cachetools=4.2.2=pyhd3eb1b0_0 + - certifi=2023.7.22=py311h06a4308_0 + - cffi=1.15.1=py311h5eee18b_3 + - charset-normalizer=2.0.4=pyhd3eb1b0_0 + - click=8.1.7=py311h06a4308_0 + - cryptography=41.0.3=py311hdda0065_0 + - frozenlist=1.4.0=py311h5eee18b_0 + - google-api-core=2.10.1=py311h06a4308_0 + - google-api-python-client=2.106.0=pyhd8ed1ab_0 + - google-auth=2.22.0=py311h06a4308_0 + - google-auth-httplib2=0.1.1=pyhd8ed1ab_0 + - google-auth-oauthlib=0.5.2=py311h06a4308_0 + - googleapis-common-protos=1.56.4=py311h06a4308_0 + - httplib2=0.22.0=pyhd8ed1ab_0 + - idna=3.4=py311h06a4308_0 + - iniconfig=1.1.1=pyhd3eb1b0_0 + - intel-openmp=2023.1.0=hdb19cb5_46305 + - ld_impl_linux-64=2.38=h1181459_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=11.2.0=h1234567_1 + - libgomp=11.2.0=h1234567_1 + - libprotobuf=3.20.3=he621ea3_0 + - libstdcxx-ng=11.2.0=h1234567_1 + - libuuid=1.41.5=h5eee18b_0 + - mkl=2023.1.0=h213fc3f_46343 + - mkl-service=2.4.0=py311h5eee18b_1 + - mkl_fft=1.3.8=py311h5eee18b_0 + - mkl_random=1.2.4=py311hdb19cb5_0 + - multidict=6.0.2=py311h5eee18b_0 + - ncurses=6.4=h6a678d5_0 + - numexpr=2.8.7=py311h65dcdc2_0 + - numpy=1.26.0=py311h08b1b3b_0 + - numpy-base=1.26.0=py311hf175353_0 + - oauthlib=3.2.2=py311h06a4308_0 + - openssl=3.0.12=h7f8727e_0 + - packaging=23.1=py311h06a4308_0 + - pandas=2.1.1=py311ha02d727_0 + - pip=23.3=py311h06a4308_0 + - pluggy=1.0.0=py311h06a4308_1 + - protobuf=3.20.3=py311h6a678d5_0 + - pyasn1=0.4.8=pyhd3eb1b0_0 + - pyasn1-modules=0.2.8=py_0 + - pycparser=2.21=pyhd3eb1b0_0 + - pyjwt=2.4.0=py311h06a4308_0 + - pyopenssl=23.2.0=py311h06a4308_0 + - pyparsing=3.0.9=py311h06a4308_0 + - pysocks=1.7.1=py311h06a4308_0 + - pytest=7.4.0=py311h06a4308_0 + - python=3.11.5=h955ad1f_0 + - python-dateutil=2.8.2=pyhd3eb1b0_0 + - python-tzdata=2023.3=pyhd3eb1b0_0 + - pytz=2023.3.post1=py311h06a4308_0 + - pyyaml=6.0.1=py311h5eee18b_0 + - readline=8.2=h5eee18b_0 + - requests=2.31.0=py311h06a4308_0 + - requests-oauthlib=1.3.0=py_0 + - rsa=4.7.2=pyhd3eb1b0_1 + - setuptools=68.0.0=py311h06a4308_0 + - six=1.16.0=pyhd3eb1b0_1 + - sqlite=3.41.2=h5eee18b_0 + - tabulate=0.8.10=py311h06a4308_0 + - tbb=2021.8.0=hdb19cb5_0 + - texttable=1.6.4=pyhd3eb1b0_0 + - tk=8.6.12=h1ccaba5_0 + - tzdata=2023c=h04d1e81_0 + - uritemplate=4.1.1=pyhd8ed1ab_0 + - urllib3=1.26.18=py311h06a4308_0 + - wheel=0.41.2=py311h06a4308_0 + - xz=5.4.2=h5eee18b_0 + - yaml=0.2.5=h7b6447c_0 + - yarl=1.8.1=py311h5eee18b_0 + - zlib=1.2.13=h5eee18b_0 + - pip: + - pybis==1.36.3 diff --git a/pangolin_src/conda_envs/sendcrypt.yaml b/pangolin_src/conda_envs/sendcrypt.yaml new file mode 100644 index 0000000..b3c838f --- /dev/null +++ b/pangolin_src/conda_envs/sendcrypt.yaml @@ -0,0 +1,66 @@ +name: sendcrypt +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - blas=1.0=mkl + - bottleneck=1.3.5=py312ha883a20_0 + - bzip2=1.0.8=h7b6447c_0 + - c-ares=1.19.1=h5eee18b_0 + - ca-certificates=2023.08.22=h06a4308_0 + - curl=8.4.0=hdbd6064_0 + - expat=2.5.0=h6a678d5_0 + - gdbm=1.18=hd4cb3f1_4 + - gettext=0.21.0=h39681ba_1 + - git=2.40.1=pl5340h36fbf9e_1 + - gnupg=2.3.2=h7853c96_0 + - icu=73.1=h6a678d5_0 + - intel-openmp=2023.1.0=hdb19cb5_46306 + - krb5=1.20.1=h143b758_1 + - ld_impl_linux-64=2.38=h1181459_1 + - libassuan=2.5.5=h9c3ff4c_0 + - libcurl=8.4.0=h251f7ec_0 + - libedit=3.1.20221030=h5eee18b_0 + - libev=4.33=h7f8727e_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=11.2.0=h1234567_1 + - libgcrypt=1.9.3=h27cfd23_0 + - libgomp=11.2.0=h1234567_1 + - libgpg-error=1.42=h2531618_0 + - libiconv=1.16=h7f8727e_2 + - libksba=1.3.5=hf484d3e_1000 + - libnghttp2=1.57.0=h2d74bed_0 + - libssh2=1.10.0=hdbd6064_2 + - libstdcxx-ng=11.2.0=h1234567_1 + - libuuid=1.41.5=h5eee18b_0 + - libxml2=2.10.4=hf1b16e4_1 + - mkl=2023.1.0=h213fc3f_46344 + - mkl-service=2.4.0=py312h5eee18b_1 + - ncurses=6.4=h6a678d5_0 + - npth=1.6=hf484d3e_1000 + - ntbtls=0.1.2=hdbcaa40_1000 + - numexpr=2.8.7=py312hf827012_0 + - numpy=1.26.0=py312hc5e2394_0 + - numpy-base=1.26.0=py312h0da6c21_0 + - openssl=3.0.12=h7f8727e_0 + - pandas=2.1.1=py312h526ad5a_0 + - pcre2=10.42=hebb0a14_0 + - perl=5.34.0=h5eee18b_2 + - pip=23.3=py312h06a4308_0 + - python=3.12.0=h996f2a0_0 + - python-dateutil=2.8.2=pyhd3eb1b0_0 + - python-tzdata=2023.3=pyhd3eb1b0_0 + - pytz=2023.3.post1=py312h06a4308_0 + - readline=8.2=h5eee18b_0 + - setuptools=68.0.0=py312h06a4308_0 + - six=1.16.0=pyhd3eb1b0_1 + - sqlite=3.41.2=h5eee18b_0 + - tbb=2021.8.0=hdb19cb5_0 + - tk=8.6.12=h1ccaba5_0 + - tzdata=2023c=h04d1e81_0 + - wheel=0.41.2=py312h06a4308_0 + - xz=5.4.2=h5eee18b_0 + - zlib=1.2.13=h5eee18b_0 diff --git a/pangolin_src/conda_envs/sync.yaml b/pangolin_src/conda_envs/sync.yaml new file mode 100644 index 0000000..c8e2c91 --- /dev/null +++ b/pangolin_src/conda_envs/sync.yaml @@ -0,0 +1,38 @@ +name: sync +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - bzip2=1.0.8=hd590300_5 + - ca-certificates=2023.7.22=hbcca054_0 + - expat=2.5.0=hcb278e6_1 + - gettext=0.21.1=h27087fc_0 + - ld_impl_linux-64=2.40=h41732ed_0 + - lftp=4.9.2=hedb13fd_1 + - libexpat=2.5.0=hcb278e6_1 + - libffi=3.4.2=h7f98852_5 + - libgcc-ng=13.2.0=h807b86a_2 + - libgomp=13.2.0=h807b86a_2 + - libiconv=1.17=h166bdaf_0 + - libnsl=2.0.1=hd590300_0 + - libsqlite=3.44.0=h2797004_0 + - libstdcxx-ng=13.2.0=h7e041cc_2 + - libuuid=2.38.1=h0b41bf4_0 + - libzlib=1.2.13=hd590300_5 + - ncurses=6.4=h59595ed_2 + - openssl=3.1.4=hd590300_0 + - pip=23.3.1=pyhd8ed1ab_0 + - python=3.12.0=hab00c5b_0_cpython + - python_abi=3.12=4_cp312 + - pyyaml=6.0.1=py312h98912ed_1 + - readline=8.2=h8228510_1 + - setuptools=68.2.2=pyhd8ed1ab_0 + - tk=8.6.13=noxft_h4845f30_101 + - tzdata=2023c=h71feb2d_0 + - wheel=0.41.3=pyhd8ed1ab_0 + - xz=5.2.6=h166bdaf_0 + - yaml=0.2.5=h7f98852_2 + - zlib=1.2.13=hd590300_5 diff --git a/pangolin_src/conda_envs/viloca.yaml b/pangolin_src/conda_envs/viloca.yaml new file mode 100644 index 0000000..077c185 --- /dev/null +++ b/pangolin_src/conda_envs/viloca.yaml @@ -0,0 +1,217 @@ +name: viloca +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - aioeasywebdav=2.4.0=pyha770c72_0 + - aiohttp=3.8.6=py310h2372a71_1 + - aiosignal=1.3.1=pyhd8ed1ab_0 + - amply=0.1.6=pyhd8ed1ab_0 + - appdirs=1.4.4=pyh9f0ad1d_0 + - archspec=0.2.2=pyhd8ed1ab_0 + - async-timeout=4.0.3=pyhd8ed1ab_0 + - attmap=0.13.2=pyhd8ed1ab_0 + - attrs=23.1.0=pyh71513ae_1 + - backports=1.0=pyhd8ed1ab_3 + - backports.functools_lru_cache=1.6.5=pyhd8ed1ab_0 + - bcrypt=4.0.1=py310hcb5633a_1 + - boltons=23.0.0=pyhd8ed1ab_0 + - boto3=1.28.79=pyhd8ed1ab_0 + - botocore=1.31.79=pyhd8ed1ab_0 + - brotli-python=1.1.0=py310hc6cd4ac_1 + - bzip2=1.0.8=hd590300_5 + - c-ares=1.21.0=hd590300_0 + - ca-certificates=2023.7.22=hbcca054_0 + - cachetools=5.3.2=pyhd8ed1ab_0 + - certifi=2023.7.22=pyhd8ed1ab_0 + - cffi=1.16.0=py310h2fee648_0 + - charset-normalizer=3.3.2=pyhd8ed1ab_0 + - coin-or-cbc=2.10.10=h9002f0b_0 + - coin-or-cgl=0.60.7=h516709c_0 + - coin-or-clp=1.17.8=h1ee7a9c_0 + - coin-or-osi=0.108.8=ha2443b9_0 + - coin-or-utils=2.11.9=hee58242_0 + - coincbc=2.10.10=0_metapackage + - colorama=0.4.6=pyhd8ed1ab_0 + - conda=23.10.0=py310hff52083_0 + - conda-libmamba-solver=23.11.0=pyhd8ed1ab_0 + - conda-package-handling=2.2.0=pyh38be061_0 + - conda-package-streaming=0.9.0=pyhd8ed1ab_0 + - configargparse=1.7=pyhd8ed1ab_0 + - connection_pool=0.0.3=pyhd3deb0d_0 + - cryptography=41.0.5=py310h75e40e8_0 + - datrie=0.8.2=py310h2372a71_7 + - defusedxml=0.7.1=pyhd8ed1ab_0 + - docutils=0.20.1=py310hff52083_2 + - dpath=2.1.6=pyha770c72_0 + - dropbox=11.36.2=pyhd8ed1ab_0 + - exceptiongroup=1.1.3=pyhd8ed1ab_0 + - filechunkio=1.8=py_2 + - filelock=3.13.1=pyhd8ed1ab_0 + - fmt=10.1.1=h00ab1b0_0 + - frozenlist=1.4.0=py310h2372a71_1 + - ftputil=5.0.4=pyhd8ed1ab_0 + - gitdb=4.0.11=pyhd8ed1ab_0 + - gitpython=3.1.40=pyhd8ed1ab_0 + - google-api-core=2.12.0=pyhd8ed1ab_0 + - google-api-python-client=2.106.0=pyhd8ed1ab_0 + - google-auth=2.23.4=pyhca7485f_0 + - google-auth-httplib2=0.1.1=pyhd8ed1ab_0 + - google-cloud-core=2.3.3=pyhd8ed1ab_0 + - google-cloud-storage=2.13.0=pyhca7485f_0 + - google-crc32c=1.1.2=py310hc5c09a0_5 + - google-resumable-media=2.6.0=pyhd8ed1ab_0 + - googleapis-common-protos=1.61.0=pyhd8ed1ab_0 + - grpcio=1.59.2=py310h1b8f574_0 + - htslib=1.18=h81da01d_0 + - httplib2=0.22.0=pyhd8ed1ab_0 + - humanfriendly=10.0=pyhd8ed1ab_6 + - icu=73.2=h59595ed_0 + - idna=3.4=pyhd8ed1ab_0 + - importlib_resources=6.1.1=pyhd8ed1ab_0 + - iniconfig=2.0.0=pyhd8ed1ab_0 + - jinja2=3.1.2=pyhd8ed1ab_1 + - jmespath=1.0.1=pyhd8ed1ab_0 + - jsonpatch=1.33=pyhd8ed1ab_0 + - jsonpointer=2.4=py310hff52083_3 + - jsonschema=4.19.2=pyhd8ed1ab_0 + - jsonschema-specifications=2023.7.1=pyhd8ed1ab_0 + - jupyter_core=5.5.0=py310hff52083_0 + - keyutils=1.6.1=h166bdaf_0 + - krb5=1.21.2=h659d440_0 + - ld_impl_linux-64=2.40=h41732ed_0 + - libabseil=20230802.1=cxx17_h59595ed_0 + - libarchive=3.7.2=h039dbb9_0 + - libblas=3.9.0=19_linux64_openblas + - libcblas=3.9.0=19_linux64_openblas + - libcrc32c=1.1.2=h9c3ff4c_0 + - libcurl=8.4.0=hca28451_0 + - libdeflate=1.19=hd590300_0 + - libedit=3.1.20191231=he28a2e2_2 + - libev=4.33=h516909a_1 + - libffi=3.4.2=h7f98852_5 + - libgcc-ng=13.2.0=h807b86a_2 + - libgfortran-ng=13.2.0=h69a702a_2 + - libgfortran5=13.2.0=ha4646dd_2 + - libgomp=13.2.0=h807b86a_2 + - libgrpc=1.59.2=hd6c4280_0 + - libiconv=1.17=h166bdaf_0 + - liblapack=3.9.0=19_linux64_openblas + - liblapacke=3.9.0=19_linux64_openblas + - libmamba=1.5.3=had39da4_1 + - libmambapy=1.5.3=py310h39ff949_1 + - libnghttp2=1.55.1=h47da74e_0 + - libnsl=2.0.1=hd590300_0 + - libopenblas=0.3.24=pthreads_h413a1c8_0 + - libprotobuf=4.24.4=hf27288f_0 + - libre2-11=2023.06.02=h7a70373_0 + - libshorah=1.99.4=py310he0067ce_1 + - libsodium=1.0.18=h36c2ea0_1 + - libsolv=0.7.26=hfc55251_0 + - libsqlite=3.44.0=h2797004_0 + - libssh2=1.11.0=h0841786_0 + - libstdcxx-ng=13.2.0=h7e041cc_2 + - libuuid=2.38.1=h0b41bf4_0 + - libxml2=2.11.5=h232c23b_1 + - libzlib=1.2.13=hd590300_5 + - logmuse=0.2.6=pyh8c360ce_0 + - lz4-c=1.9.4=hcb278e6_0 + - lzo=2.10=h516909a_1000 + - mamba=1.5.3=py310h51d5547_1 + - markdown-it-py=3.0.0=pyhd8ed1ab_0 + - markupsafe=2.1.3=py310h2372a71_1 + - mdurl=0.1.0=pyhd8ed1ab_0 + - multidict=6.0.4=py310h2372a71_1 + - nbformat=5.9.2=pyhd8ed1ab_0 + - ncurses=6.4=h59595ed_2 + - numpy=1.26.0=py310hb13e2d6_0 + - oauth2client=4.1.3=py_0 + - openssl=3.1.4=hd590300_0 + - packaging=23.2=pyhd8ed1ab_0 + - pandas=2.1.2=py310hcc13569_0 + - paramiko=3.3.1=pyhd8ed1ab_0 + - peppy=0.35.7=pyhd8ed1ab_0 + - pip=23.3.1=pyhd8ed1ab_0 + - pkgutil-resolve-name=1.3.10=pyhd8ed1ab_1 + - plac=1.4.1=pyhd8ed1ab_1 + - platformdirs=3.11.0=pyhd8ed1ab_0 + - pluggy=1.3.0=pyhd8ed1ab_0 + - ply=3.11=py_1 + - prettytable=3.9.0=pyhd8ed1ab_0 + - protobuf=4.24.4=py310h620c231_0 + - psutil=5.9.5=py310h2372a71_1 + - pulp=2.7.0=py310hff52083_1 + - pyasn1=0.5.0=pyhd8ed1ab_0 + - pyasn1-modules=0.3.0=pyhd8ed1ab_0 + - pybind11-abi=4=hd8ed1ab_3 + - pycosat=0.6.6=py310h2372a71_0 + - pycparser=2.21=pyhd8ed1ab_0 + - pygments=2.16.1=pyhd8ed1ab_0 + - pynacl=1.5.0=py310h2372a71_3 + - pyopenssl=23.3.0=pyhd8ed1ab_0 + - pyparsing=3.1.1=pyhd8ed1ab_0 + - pysftp=0.2.9=py_1 + - pysocks=1.7.1=pyha2e5f31_6 + - pytest=7.4.3=pyhd8ed1ab_0 + - python=3.10.13=hd12c33a_0_cpython + - python-dateutil=2.8.2=pyhd8ed1ab_0 + - python-fastjsonschema=2.18.1=pyhd8ed1ab_0 + - python-irodsclient=1.1.9=pyhd8ed1ab_0 + - python-tzdata=2023.3=pyhd8ed1ab_0 + - python_abi=3.10=4_cp310 + - pytz=2023.3.post1=pyhd8ed1ab_0 + - pyu2f=0.1.5=pyhd8ed1ab_0 + - pyyaml=6.0.1=py310h2372a71_1 + - re2=2023.06.02=h2873b5e_0 + - readline=8.2=h8228510_1 + - referencing=0.30.2=pyhd8ed1ab_0 + - reproc=14.2.4.post0=hd590300_1 + - reproc-cpp=14.2.4.post0=h59595ed_1 + - requests=2.31.0=pyhd8ed1ab_0 + - reretry=0.11.8=pyhd8ed1ab_0 + - rich=13.6.0=pyhd8ed1ab_0 + - rpds-py=0.12.0=py310hcb5633a_0 + - rsa=4.9=pyhd8ed1ab_0 + - ruamel.yaml=0.17.40=py310h2372a71_0 + - ruamel.yaml.clib=0.2.7=py310h2372a71_2 + - s3transfer=0.7.0=pyhd8ed1ab_0 + - setuptools=68.2.2=pyhd8ed1ab_0 + - setuptools-scm=8.0.4=pyhd8ed1ab_0 + - six=1.16.0=pyh6c4a22f_0 + - slacker=0.14.0=py_0 + - smart_open=6.4.0=pyhd8ed1ab_0 + - smmap=5.0.0=pyhd8ed1ab_0 + - snakemake=7.30.2=hdfd78af_1 + - snakemake-minimal=7.30.2=pyhdfd78af_1 + - stone=3.3.1=pyhd8ed1ab_0 + - stopit=1.1.2=py_0 + - tabulate=0.9.0=pyhd8ed1ab_1 + - throttler=1.2.2=pyhd8ed1ab_0 + - tk=8.6.13=noxft_h4845f30_101 + - tomli=2.0.1=pyhd8ed1ab_0 + - toposort=1.10=pyhd8ed1ab_0 + - tqdm=4.66.1=pyhd8ed1ab_0 + - traitlets=5.13.0=pyhd8ed1ab_0 + - truststore=0.8.0=pyhd8ed1ab_0 + - typing-extensions=4.8.0=hd8ed1ab_0 + - typing_extensions=4.8.0=pyha770c72_0 + - tzdata=2023c=h71feb2d_0 + - ubiquerg=0.6.3=pyhd8ed1ab_0 + - uritemplate=4.1.1=pyhd8ed1ab_0 + - urllib3=1.26.18=pyhd8ed1ab_0 + - veracitools=0.1.3=py_0 + - wcwidth=0.2.9=pyhd8ed1ab_0 + - wheel=0.41.3=pyhd8ed1ab_0 + - wrapt=1.15.0=py310h2372a71_1 + - xz=5.2.6=h166bdaf_0 + - yaml=0.2.5=h7f98852_2 + - yaml-cpp=0.8.0=h59595ed_0 + - yarl=1.9.2=py310h2372a71_1 + - yte=1.5.1=pyha770c72_2 + - zipp=3.17.0=pyhd8ed1ab_0 + - zlib=1.2.13=hd590300_5 + - zstandard=0.22.0=py310h1275a96_0 + - zstd=1.5.5=hfc55251_0 diff --git a/fgcz.conf b/pangolin_src/config/fgcz.conf similarity index 79% rename from fgcz.conf rename to pangolin_src/config/fgcz.conf index 706038b..f51abdc 100644 --- a/fgcz.conf +++ b/pangolin_src/config/fgcz.conf @@ -2,22 +2,22 @@ # lab's name for the batch YAML description lab=fgcz # name of the _SFTP_ server, as used when fetching data files (i.e.: same name as in netrc file) -fileserver=fgcz-gstore.uzh.ch +fileserver=fgcz-gstore.uzh.ch # port used by SFTP, in case of non standard (you can also use ~/.ssh/config for that) -srvport=666 +srvport_sftp=666 +srvport_https=443 +protocol=sftp # experiment name in sftp store #expname=/projects/p23212 expname=/projects -#projlist=( 'p23224' 'p23212' 'p24991' 'p25650' 'p26177' 'p30045' ) -#Limit projects only to wastewater: -projlist=( 'p23224' 'p26177' 'p30045' 'p24991' ) -# base dircetory (default: cwd) -basedir=/links/shared/covid19-pangolin/backup -# sub-directory to hold the unsorted downloaded datasets -download=bfabric-downloads +expname_https=/p00/projects +expname_sftp=/projects +projlist=( 'p23224' 'p35759' ) +# directory to hold the unsorted downloaded datasets +download=/cluster/project/pangolin/bfabric-downloads ## sub-directory to hold the sorted samples set #sampleset=staging-fcgz -sampleset=sampleset +sampleset=/cluster/project/pangolin/sampleset # explicit list of orders spread among multiple batches that should be merged fuselist=MS547_COV19_o23391,MS547_COV19_o23391_copy,NOV584_COV19_o23860,NOV593_COV19_o23860,NOV671_COV19_o24453,NOV671_COV19_o24453_copy,MiSeq_210820_MS639_o25795_DataDelivery_part,MiSeq_210820_MS640_o25795_DataDelivery_part,MiSeq_210820_MS639_o25796_part,MiSeq_210820_MS640_o25796_part,NovaSeq_20220708_NOV1343_o28874_DataDelivery,NovaSeq_20220715_NOV1355_o28874_DataDelivery # delay after which orders aren't considered for merging anymore @@ -36,9 +36,9 @@ iotimeout=300 # number of attempts to reconnect (Default: 10) #retries=10 # mark bad runs -badlist=MiSeq_210122_MS562_o23881_DataDelivery,Fastqc_53742_2021-01-27--15-56-09,MiSeq_210129_MS564_o23775_DataDelivery,NOV657_COV19_o24384,MiSeq_210317_MS586_ww_o24329_o24329_DataDelivery,NovaSeq_20210806_NOV873_o25679_DataDelivery,NovaSeq_20210806_NOV873_o25697_DataDelivery,Fastqc_64627_2021-12-19--17-54-32,Fastqc_64710_2021-12-22--11-02-11,NovaSeq_20211217_NOV1084_o27040_DataDelivery,NovaSeq_20211217_NOV1084_o26977,NovaSeq_20211217_NOV1084_o27023,NovaSeq_20211217_NOV1084_o27026,Fastqc_64771_2021-12-25--11-15-42,Fastqc_64774_2021-12-25--11-17-46,Fastqc_64703_2021-12-22--10-58-32,Fastqc_64706_2021-12-22--11-00-41,Fastqc_64636_2021-12-19--17-58-55,Fastqc_64713_2021-12-22--11-03-19,Fastqc_64777_2021-12-25--11-19-09,Fastqc_64630_2021-12-19--17-56-31,Fastqc_64780_2021-12-25--11-20-17,Fastqc_64633_2021-12-19--17-57-51,NovaSeq_20221007_NOV1461_o29645_DataDelivery +badlist=MiSeq_210122_MS562_o23881_DataDelivery,Fastqc_53742_2021-01-27--15-56-09,MiSeq_210129_MS564_o23775_DataDelivery,NOV657_COV19_o24384,MiSeq_210317_MS586_ww_o24329_o24329_DataDelivery,NovaSeq_20210806_NOV873_o25679_DataDelivery,NovaSeq_20210806_NOV873_o25697_DataDelivery,Fastqc_64627_2021-12-19--17-54-32,Fastqc_64710_2021-12-22--11-02-11,NovaSeq_20211217_NOV1084_o27040_DataDelivery,NovaSeq_20211217_NOV1084_o26977,NovaSeq_20211217_NOV1084_o27023,NovaSeq_20211217_NOV1084_o27026,Fastqc_64771_2021-12-25--11-15-42,Fastqc_64774_2021-12-25--11-17-46,Fastqc_64703_2021-12-22--10-58-32,Fastqc_64706_2021-12-22--11-00-41,Fastqc_64636_2021-12-19--17-58-55,Fastqc_64713_2021-12-22--11-03-19,Fastqc_64777_2021-12-25--11-19-09,Fastqc_64630_2021-12-19--17-56-31,Fastqc_64780_2021-12-25--11-20-17,Fastqc_64633_2021-12-19--17-57-51,NovaSeq_20221007_NOV1461_o29645_DataDelivery,o34397_NextSeq_240301_NS2k-272,o34397_Fastqc_2024-03-03--23-31-04,o34397_FastqScreen_2024-03-03--23-31-24,o34220_NextSeq_240216_NS2k-265,o34220_Fastqc_2024-02-19--15-50-33,o34220_FastqScreen_2024-02-19--15-50-53,o34676_Fastqc_2024-04-08--09-00-47,o34676_FastqScreen_2024-04-08--09-01-56,o34676_NovaSeq_240405_NOV1970,o35541_Aviti_2023-10-22_AV013, o36056_Aviti_240816_AV021 forcelist=NOV641,NOV674_COV19_repeats,NOV741_COVID # if libkit isn't mentionned, set protocol to this: -fallbackproto=v41 +fallbackproto=v3 # do we download patchmap lists from google sheet? google_sheet_patches=0 diff --git a/gfb.conf b/pangolin_src/config/gfb.conf similarity index 94% rename from gfb.conf rename to pangolin_src/config/gfb.conf index 713228b..dea1732 100644 --- a/gfb.conf +++ b/pangolin_src/config/gfb.conf @@ -14,9 +14,9 @@ expname=/BSSE_STADLER_COVID/STADLER_COVID/COVID_V # the type for which need to search the experiment thourgh samtype=ILLUMINA_FLOW_LANE # base dircetory (default: cwd) -basedir=/links/shared/covid19-pangolin/backup +basedir=/data/backup # sub-directory to hold the unsorted downloaded datasets -download=openbis-downloads +download=/mnt/cluster/openbis-downloads # sub-directory to hold the sorted samples set sampleset=sampleset # linking instead of copying ? diff --git a/h2030.conf b/pangolin_src/config/h2030.conf similarity index 95% rename from h2030.conf rename to pangolin_src/config/h2030.conf index 9aed37a..4e2dd98 100644 --- a/h2030.conf +++ b/pangolin_src/config/h2030.conf @@ -11,7 +11,7 @@ expname=/downloads # base dircetory basedir=/links/shared/covid19-pangolin/backup # sub-directory to hold the unsorted downloaded datasets -download=sftp-health2030 +download=/mnt/cluster/sftp-health2030 ## sub-directory to hold the sorted samples set sampleset=sampleset ## linking instead of copying ? diff --git a/harvester_config_nopw.yml b/pangolin_src/config/harvester_config_nopw.yml similarity index 100% rename from harvester_config_nopw.yml rename to pangolin_src/config/harvester_config_nopw.yml diff --git a/rotate.conf b/pangolin_src/config/rotate.conf similarity index 100% rename from rotate.conf rename to pangolin_src/config/rotate.conf diff --git a/pangolin_src/config/rsyncd.conf b/pangolin_src/config/rsyncd.conf new file mode 100644 index 0000000..0054bbd --- /dev/null +++ b/pangolin_src/config/rsyncd.conf @@ -0,0 +1,29 @@ +read only = no +use chroot = no +transfer logging = true +log format = %h %o %f %l %b +log file = log/rsyncd.log + +charset = utf8 + +uid = bs-pangolin +auth users = dryak +secrets file = rsyncd.secrets + +[sampleset] + path = /cluster/project/pangolin/sampleset/ + comment = Samples set + auth users = belfry:wo batman:ro dryak:rw ceciliav:ro + incoming chmod = Dg+s,ug+rw,o-rwx + +[working] + path = /cluster/project/pangolin/working/ + comment = Working directory + auth users = belfry:ro batman:ro dryak:rw ceciliav:ro + outgoing chmod = Dg+s,ug+rw,o-rwx + +[catchup] + path = /cluster/project/pangolin/work-catchup-dehuman/ + comment = Workdir for catchup + auth users = belfry:ro batman:ro dryak:rw + outgoing chmod = Dg+s,ug+rw,o-rwx diff --git a/pangolin_src/config/server.conf b/pangolin_src/config/server.conf new file mode 100755 index 0000000..ebb1754 --- /dev/null +++ b/pangolin_src/config/server.conf @@ -0,0 +1,178 @@ +[ _ ] +# labs: SFTP synchronisation +# (except in case of abnormally slow operation, sync can be left active) +skipsync= +# labs: sample sorting and import +# (set to 0 for labs whose sequences we aren't currently importing, e.g. due to problems) +declare -A lab +lab=( ['gfb']=0 ['fgcz']=1 ['h2030']=0 ['viollier']=0 ) +# NOTE: h2030 isn't sequencing as the number of cases is lower. +# pass '--force' to force overwriting any existing file when moving +sort_force=--force +# base wastewater dir +wwdir=/app +# base directory +basedir=${wwdir}/workdir +# sub-directory of openbis download for rights fixing +download=openbis-downloads +# sub-directory containing the scripts +scriptdir=${wwdir}/pangolin_src +# directory containing the entire git repository, for logging purposes +codebase=${wwdir}/codebase +# sub-directory to hold the sorted samples set +# working sub-directory +# linking instead of copying ? +# --reflink for CoW filesystems (ZFS, BTRFS) +# --hardlink for most unix-like filesystems +link=--link +# Installation directory of miniconda3 +baseconda=${wwdir}/miniconda3 +# File containing the Rsync password +rsync_pass=/home/bs-pangolin/.ssh/rsync.pass.euler +# YAML file containing the primers +protocolyaml=references/primers.yaml +# Regular expression to recognize sample names +rxsample='(^[[:digit:]]{6,}_)|(^[[:digit:]]{8,})|(^Y[[:digit:]]{8,})|([[:digit:]]{2}_20[[:digit:]]{2}_[01]?[[:digit:]]_[0-3]?[[:digit:]])|(^KLZHC[oO][vV])|(^B[aA][[:digit:]]{4,})|(^USB_20[[:digit:]]{2}_[01]?[[:digit:]]_[[:digit:]]{2}_.{8})' +# Date after which all samples are processed as Aviti +aviti_date=20260601 +# Flag to temporary skip Aviti in case the sequencing center reverts back to Illumina +skipaviti=0 + +# group on the storage (inside download and sampleset) +# storgrp=bsse-covid19-pangolin@d.ethz.ch +storgrp=bs-pangolin-group +# parallel copy jobs +parallel=8 +# parallel backup copy jobs +parallelpull=4 +# whereto push the sequences at the end +#releasedir=/links/shared/covid19-pangolin/pangolin/consensus_data/batch/ +#releasedir=$(realpath $(pwd)/batch) +# timeout before rsync considers the transfer failed in seconds +rsynctimeout=2000 +# SSH connection timeout +contimeout=300 +# IO timeout +iotimeout=300 +# suspend jobs submission in case of problems +donotsubmit=0 +# skip or run ShoRAH step of V-pipe +run_shorah=0 +# mail +mailfrom='Automation-carillon' +mailto= +# timeout for the whole automation loop +runtimeout=7200 +# timeout to wait for 'sync' command: 0: no timeout, "": default, otherwise seconds +synctimeout=0 +# directory containing the local copy of the dataset +local_dataset=/app/dataset + +# Cluster config +# username used on the cluster (while we're switching over) +cluster_user=bs-pangolin +cluster=euler.ethz.ch +clusterdir="/cluster/project/pangolin/test_automation/pangolin/pangolin_src" +clusterdir_old="/cluster/project/pangolin" +working="working" +sampleset="sampleset" +# rsyncd module to sync from the remote status folder +remote_status=remote_status +# rsyncd module to sync from the remote bfabric sync folder +bfabric_downloads=bfabric-downloads +# rsyncd module to sync to the remote VILOCA work folder +work_viloca=work-viloca +#rsyncd module to sync to the remote uploader work folder +#work_uploader=work-uploader +# Directory containing the git repository of V-pipe +vpipe_code=/cluster/project/pangolin/V-pipe + +# BSSE folder backups config +# Do we want to backup the Vpipe results? +backup_vpipe=0 +# Do we want to backup the VILOCA results? +backup_viloca=0 +# Do we want to backup the uploader results? +backup_uploader=0 +# Do we want to backup the FGCZ mirror? +backup_fgcz_raw=0 +# Do we want to backup the amplicon coverage results? +backup_amplicon_cov=0 +# the user of the remote machine used for backup +bck_user=bs-pangolin +# the hostname of the remote machine used for backup +bckhost=d@bs-bewi08 +# Remote directory for the uploader results +remote_uploader_backup_dir=/links/shared/covid19-pangolin/backup/uploader_results + +# VILOCA-specific configuration +run_viloca=0 +donotsubmit_viloca=0 +# Local VILOCA work directory +viloca_basedir=${basedir}/viloca +# Remote VILOCA base directory +remote_viloca_basedir=${clusterdir_old}/work-viloca +# Remote processing directory for VILOCA +viloca_processing=SARS-CoV-2-wastewater-sample-processing-VILOCA +# Remote results subdirectory for VILOCA +viloca_results=results +# Name of the sample list file for VILOCA +viloca_samples=samples.csv +# Staging file for VILOCA sample list +viloca_staging=${viloca_samples}.staging + +# Upload-specific configuration +# Do we run the uploader section at all, including adding new samples to the list? +run_uploader=0 +# Location of the sendCrypt executable for the uploads +sendcrypt_exec=${HOME}/.sendcrypt/sendcrypt +# Do we submit the data to upload? A "1" means that the uploader will only update the list +donotsubmit_uploader=0 +# Clean sendcrypt temporary directories +clean_sendcrypt_temp=1 +# Filename of the list of samples to upload +uploaderlist=batches_to_upload.tsv +# Remote work directory for the uploader +uploader_workdir=${basedir}/uploader +# Directory containing the samples to be uploaded +uploader_dataset=/app/dataset +# Directory containing the sampleset files of the samples to upload +uploader_sampleset=${basedir}/tmp/belfrysheets/ +# Number of samples to upload every try +uploader_sample_number=100 +# Daily quota of files to upload on SPSP +upload_number_quota=1000000000000 +# Daily quota of size to upload on SPSP in MB +upload_size_quota=500000 +# Average expected size in MB of an sample to upload +upload_avg_size=500 +# Remote folder containing the uploader code +uploader_code=${wwdir}/uploader +# Remote temporary directory for files that should be dropped or overwritten between runs +uploader_tempdir=${uploader_workdir}/temp +# Remote sub directory of the staging directory where the files to upload should be copied +uploader_target=${uploader_tempdir}/target +# Remote archival directory for the upload files +uploader_archive=/app/dataset/archive +# Remote location of the full list of uploaded samples +uploader_uploaded=${uploader_workdir}/all_uploaded.tsv +# Do we check for updates for sendcrypt every time the uploader runs? +update_sendcrypt=1 +# Blacklist including files to exclude from upload. Reasons are included in a separate column +uploader_blacklist=${uploader_workdir}/blacklist.tsv + +# Amplicon coverage-specific configuration +run_amplicon_coverage=1 +remote_amplicon_coverage_workdir=${clusterdir_old}/work-amplicon-coverage +remote_amplicon_coverage_code=${remote_amplicon_coverage_workdir}/amplicon_cov +remote_primers_bed=${remote_amplicon_coverage_code}/ArticV531primers.bed +remote_amplicon_coverage_tempdir=${remote_amplicon_coverage_workdir}/temp + +# Status tracking configuration +statusdir=${basedir}/status +viloca_statusdir=${statusdir}/viloca +uploader_statusdir=${statusdir}/uploader +lockfile=${statusdir}/carillon_lock +# statusfile containing the number of uploaded files for the day +uploader_number_status=${uploader_statusdir}/uploader_number +amplicon_coverage_statusdir=${statusdir}/amplicon_coverage diff --git a/viollier.conf b/pangolin_src/config/viollier.conf similarity index 96% rename from viollier.conf rename to pangolin_src/config/viollier.conf index e9d53c4..6141d7f 100644 --- a/viollier.conf +++ b/pangolin_src/config/viollier.conf @@ -12,7 +12,7 @@ expname=/ # base dircetory basedir=/links/shared/covid19-pangolin/backup # sub-directory to hold the unsorted downloaded datasets -download=sftp-viollier +download=/mnt/cluster/sftp-viollier ## sub-directory to hold the sorted samples set sampleset=sampleset ## linking instead of copying ? diff --git a/cowabunga.sh b/pangolin_src/cowabunga.sh similarity index 62% rename from cowabunga.sh rename to pangolin_src/cowabunga.sh index b0fe030..8094080 100755 --- a/cowabunga.sh +++ b/pangolin_src/cowabunga.sh @@ -23,7 +23,7 @@ validateBatchName() { validateProto() { case "$1" in - v3|v4|v41) + v3|v4|v41|v532) return ;; *) @@ -37,16 +37,34 @@ validateProto() { lastmonth=$(date '+%Y%m' --date='-1 month') thismonth=$(date '+%Y%m') +thisyear=$(date '+%Y') case "$1" in autoaddww|autoaddwastewater) + if [[ -z "$2" ]] || [[ "$2" != "last_month" && "$2" != "this_month" && "$2" != "year" && "$2" != "all" ]]; then + echo "Usage: $0 $1 " 1>&2; + echo "Accepted timeframes: last_month, this_month, year, all" + exit 0 + fi + + timeframe=$2 + if [[ "$timeframe" == "last_month" ]]; then + custom_date=$lastmonth + elif [[ "$timeframe" == "this_month" ]]; then + custom_date=$thismonth + elif [[ "$timeframe" == "year" ]]; then + custom_date=$thisyear + elif [[ "$timeframe" == "all" ]]; then + custom_date="2" + fi + projects=( 'p23224' 'p24991' 'p26177' 'p30045' ) projpat="$( ( IFS='|';echo "${projects[*]}" ) )" - gawk -v d="${lastmonth}" '$2 ${working}/samples.wastewateronly.tsv.old + gawk -v d="${custom_date}" '$2 ${working}/samples.wastewateronly.tsv.old { # assemble samples.wastewateronly.tsv { - printf '%s\n' ${sampleset}/projects.${thismonth}* ${sampleset}/projects.${lastmonth}* | grep -vF '*' | sort -r | while read p; do + printf '%s\n' ${sampleset}/projects.${custom_date}*.tsv | grep -vF '*' | sort -r | while read p; do echo -n "${p} - ${p//projects/samples} ... " >&2 gawk -v projpat="${projpat}" '(FILENAME~/\/projects\./)&&($2~projpat){ww[$1]++;nww++};(FILENAME~/\/samples\./)&&(ww[$1]);END{print nww >> "/dev/stderr"}' "${p}" "${p//projects/samples}" done | tee >(cut -f4 >&3) # pass the protos to the second part bellow @@ -101,7 +119,7 @@ case "$1" in if grep -qF "${BATCH}" ${working}/samples.wastewateronly${PROTO:+.${PROTO}}.tsv; then echo "using:" ls -l ${working}/samples.wastewateronly${PROTO:+.${PROTO}}{,.lastweek,.thisweek}.tsv - else + else echo "backing up lastweek:" cp -v ${working}/samples.wastewateronly${PROTO:+.${PROTO}}{,.lastweek}.tsv fi @@ -136,7 +154,7 @@ case "$1" in if (( overwrite )); then echo "Entirely rebuilding results/ directory" else - echo "Appending missing samples to results/ directory" + echo "Appending missing samples to results/ directory" fi mkdir -p ${worktest}/results/ @@ -150,12 +168,23 @@ case "$1" in fi mkdir -p ${worktest}/results/${s}/${b}/ - cp -alv ${working}/samples/${s}/${b}/{references,alignments} ${worktest}/results/${s}/${b}/ + if grep -q ${s} lollipop_blacklist.txt || grep -q ${b} lollipop_blacklist.txt; then + echo "Skipping ${worktest}/results/${s}/${b}/ in lollipop_blacklist.txt" + else + cp -alv ${working}/samples/${s}/${b}/{references,alignments} ${worktest}/results/${s}/${b}/ + fi done < ${worktest}/${TSV} + for i in $(cat lollipop_blacklist.txt); do + while read -r line + do + [[ ! $line =~ $i ]] && echo "$line" + done <${worktest}/${TSV} > ${worktest}/${TSV}_temp + mv ${worktest}/${TSV}_temp ${worktest}/${TSV} + done ;; fetch_cooc) - proto=${2:-v41} + proto=${2:-v532} validateProto "${proto}" echo "fetching ${proto}:" @@ -163,6 +192,22 @@ case "$1" in echo "done" ;; + export) + base_ww=/cluster/project/pangolin/work-vp-test + base_main=/cluster/project/pangolin/working + export_base=/cluster/project/pangolin/export + + timeline="${base_ww}/variants/timeline.tsv" # TODO marge with main in the future + mainresults="${base_main}/samples/" # TODO switch to results in the future + tail -n +2 "${timeline}" | while read s b o; do + #mkdir -p --mode=0775 ${export_base}/{,${s}/{,${b}/{,uploads/,alignments/}}} + install --preserve-timestamps --mode=0775 -D --directory "${export_base}/${s}/${b}/"{uploads,alignments}/ + install --preserve-timestamps --mode=0664 -D --target-directory="${export_base}/${s}/${b}/alignments/" "${mainresults}/${s}/${b}/alignments/"{dehuman.count,REF_aln_stats.yaml} + cp --link -vf "${mainresults}/${s}/${b}/uploads/dehuman.cram"* "${export_base}/${s}/${b}/uploads/" + chmod o+r "${export_base}/${s}/${b}/uploads/dehuman.cram"* + done + ;; + *) echo "Unkown subcommand ${1}" > /dev/stderr exit 2 diff --git a/dbsse-user-setup.md b/pangolin_src/dbsse-user-setup.md similarity index 100% rename from dbsse-user-setup.md rename to pangolin_src/dbsse-user-setup.md diff --git a/pangolin_src/dockerignore b/pangolin_src/dockerignore new file mode 100644 index 0000000..c037f3e --- /dev/null +++ b/pangolin_src/dockerignore @@ -0,0 +1 @@ +miniconda3 diff --git a/pangolin_src/entrypoint.sh b/pangolin_src/entrypoint.sh new file mode 100755 index 0000000..a051998 --- /dev/null +++ b/pangolin_src/entrypoint.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +####### +echo "Setting up the credentials" +cp /run/secrets/bs-pangolin_login_euler /home/bs-pangolin/.ssh/bs-pangolin@euler.ethz.ch +cp /run/secrets/bs-pangolin_login_backups /home/bs-pangolin/.ssh/bs-pangolin@d@bs-bewi08 +cp /run/secrets/fgcz_login ~/.ssh/fgcz-gstore.uzh.ch +cp /run/secrets/id_ed25519_wisedb /run/secrets/id_ed25519_wisedb_backups /app/resources/config /app/resources/id_ed25519_wisedb.pub /app/resources/id_ed25519_wisedb_backups.pub /run/secrets/rsync.pass.euler /app/resources/id_ed25519_spsp_uploads.pub ~/.ssh +ssh-keyscan spsp.sib.swiss >> ~/.ssh/known_hosts +ssh-keyscan bs-bewi08.ethz.ch >> ~/.ssh/known_hosts +ssh-keyscan fgcz-gstore.uzh.ch >> ~/.ssh/known_hosts +ssh-keyscan euler.ethz.ch >> ~/.ssh/known_hosts +cp /run/secrets/sendcrypt_profile /home/bs-pangolin/.sendcrypt/profiles/default.env +cp /run/secrets/spsp_uploads_ssh_private_key /home/bs-pangolin/.ssh/id_ed25519_spsp_uploads + +echo "Setting up the GPG keys for the SPSP uploads" +cp /run/secrets/spsp_gpg_key /home/bs-pangolin/.ssh/spsp_gpg_key +cp /run/secrets/bs_pangolin_gpg_key /home/bs-pangolin/.ssh/bs_pangolin_gpg_key +cp /run/secrets/bs_pangolin_gpg_key_password /home/bs-pangolin/.ssh/bs_pangolin_gpg_key_password +gpg --import /home/bs-pangolin/.ssh/spsp_gpg_key +echo "ABC9FC14AAC952E7767FD14A48B70E724BAFE0A3:6:" | gpg --import-ownertrust + +cat /home/bs-pangolin/.ssh/bs_pangolin_gpg_key_password | gpg --batch --yes --passphrase-fd 0 --import /home/bs-pangolin/.ssh/bs_pangolin_gpg_key +echo "B2E046C543F6FDD84C4A5A307ABF7E3B5AAAE4A6:6:" | gpg --import-ownertrust +/app/pangolin_src/quasimodo.sh +#sleep 10d diff --git a/euler-user-setup.md b/pangolin_src/euler-user-setup.md similarity index 100% rename from euler-user-setup.md rename to pangolin_src/euler-user-setup.md diff --git a/exclude_list_bfabric b/pangolin_src/exclude_list_bfabric.py similarity index 94% rename from exclude_list_bfabric rename to pangolin_src/exclude_list_bfabric.py index d0c5c9c..749c8ec 100755 --- a/exclude_list_bfabric +++ b/pangolin_src/exclude_list_bfabric.py @@ -13,7 +13,7 @@ # parse command line argparser = argparse.ArgumentParser(description="Build an excluse list for folders to sync on bfabric") argparser.add_argument('-c', '--config', metavar='CONF', required=False, - default='server.conf', + default='config/server.conf', type=str, dest='config', help="configuration file to load") argparser.add_argument('-r', '--recent', metavar='ONLYAFTER', required=True, type=str, dest='recent', help="Only process batches whose date is posterior to the argument") @@ -51,11 +51,11 @@ '''folders to use even with missing order''' -rxbatchdate=re.compile('batch\.(?P\d+)_[^.]+\.yaml$') -rxdatelike=re.compile('_(?P2\d{3})-(?P[01]?\d)-(?P[0-3]?\d)--') +rxbatchdate=re.compile(r'batch\.(?P\d+)_[^.]+\.yaml$') +rxdatelike=re.compile(r'_(?P2\d{3})-(?P[01]?\d)-(?P[0-3]?\d)--') # glob all projects -if re.search('/p\d+/?$', expname): +if re.search(r'/p\d+/?$', expname): # project name included in the SFTP path, we don't need to scan projects='' extrapath=0 diff --git a/google_sheet_patches b/pangolin_src/google_sheet_patches.py similarity index 100% rename from google_sheet_patches rename to pangolin_src/google_sheet_patches.py diff --git a/handle_request_raw_viollier b/pangolin_src/handle_request_raw_viollier.sh similarity index 99% rename from handle_request_raw_viollier rename to pangolin_src/handle_request_raw_viollier.sh index 8cad253..6485b82 100755 --- a/handle_request_raw_viollier +++ b/pangolin_src/handle_request_raw_viollier.sh @@ -1,6 +1,6 @@ #!/bin/bash -configfile=viollier.conf +configfile=configs/viollier.conf usage() { echo "Usage: $0 [ -f ] [-c ]" 1>&2; exit $1; } diff --git a/list_sftp b/pangolin_src/list_sftp.sh similarity index 97% rename from list_sftp rename to pangolin_src/list_sftp.sh index e712ba9..edae874 100755 --- a/list_sftp +++ b/pangolin_src/list_sftp.sh @@ -1,6 +1,6 @@ #!/bin/bash -configfile=server.conf +configfile=config/server.conf usage() { echo "Usage: $0 [ -l ] [-c ] [filter [...]]" 1>&2; exit $1; } diff --git a/pangolin_src/log.txt b/pangolin_src/log.txt new file mode 100644 index 0000000..ad0e07d --- /dev/null +++ b/pangolin_src/log.txt @@ -0,0 +1,122 @@ +06428153 06428153 +455292e6 455292e6 +USB_2022-10-28_02c700e2 02c700e2 +USB_2022-10-28_06ec3eaf 06ec3eaf +USB_2022-10-28_06f5d6c3 06f5d6c3 +USB_2022-10-28_06fb759b 06fb759b +USB_2022-10-28_074399e9 074399e9 +USB_2022-10-28_086530e1 086530e1 +USB_2022-10-28_09777393 09777393 +USB_2022-10-28_0e820f32 0e820f32 +USB_2022-10-28_0ff198eb 0ff198eb +USB_2022-10-28_1129a4a4 1129a4a4 +USB_2022-10-28_175f2f6f 175f2f6f +USB_2022-10-28_2328aee9 2328aee9 +USB_2022-10-28_29808bba 29808bba +USB_2022-10-28_2a6e17ca 2a6e17ca +USB_2022-10-28_2a8c2a6d 2a8c2a6d +USB_2022-10-28_2b775387 2b775387 +USB_2022-10-28_2e241d96 2e241d96 +USB_2022-10-28_2f0438ac 2f0438ac +USB_2022-10-28_329f506d 329f506d +USB_2022-10-28_352f211b 352f211b +USB_2022-10-28_3635ff6e 3635ff6e +USB_2022-10-28_3861def0 3861def0 +USB_2022-10-28_39141479 39141479 +USB_2022-10-28_3eb2adc5 3eb2adc5 +USB_2022-10-28_3f37e8e5 3f37e8e5 +USB_2022-10-28_3f8e3933 3f8e3933 +USB_2022-10-28_3fc8a644 3fc8a644 +USB_2022-10-28_421dbd17 421dbd17 +USB_2022-10-28_4317dce0 4317dce0 +USB_2022-10-28_43950f4a 43950f4a +USB_2022-10-28_44b4df00 44b4df00 +USB_2022-10-28_4623409e 4623409e +USB_2022-10-28_4723d3da 4723d3da +USB_2022-10-28_475a58cd 475a58cd +USB_2022-10-28_48ea3c01 48ea3c01 +USB_2022-10-28_4d20aedb 4d20aedb +USB_2022-10-28_5267ae70 5267ae70 +USB_2022-10-28_53f854f1 53f854f1 +USB_2022-10-28_55d81e4b 55d81e4b +USB_2022-10-28_56034ef2 56034ef2 +USB_2022-10-28_56c628c0 56c628c0 +USB_2022-10-28_5a9eae4b 5a9eae4b +USB_2022-10-28_5ad0a9d0 5ad0a9d0 +USB_2022-10-28_5b70eb64 5b70eb64 +USB_2022-10-28_5bc7fcc4 5bc7fcc4 +USB_2022-10-28_602656c7 602656c7 +USB_2022-10-28_625e0f5c 625e0f5c +USB_2022-10-28_626bb617 626bb617 +USB_2022-10-28_638e1a8e 638e1a8e +USB_2022-10-28_65cee940 65cee940 +USB_2022-10-28_673b6ab6 673b6ab6 +USB_2022-10-28_67fc62bc 67fc62bc +USB_2022-10-28_691eb7fe 691eb7fe +USB_2022-10-28_6c3e823e 6c3e823e +USB_2022-10-28_6ce95475 6ce95475 +USB_2022-10-28_7006553a 7006553a +USB_2022-10-28_70dcbbab 70dcbbab +USB_2022-10-28_7498aa55 7498aa55 +USB_2022-10-28_78c53c2f 78c53c2f +USB_2022-10-28_7a6bdfdc 7a6bdfdc +USB_2022-10-28_7c9422f3 7c9422f3 +USB_2022-10-28_7f5d5a92 7f5d5a92 +USB_2022-10-28_829d99ed 829d99ed +USB_2022-10-28_85339425 85339425 +USB_2022-10-28_879f7468 879f7468 +USB_2022-10-28_87d6ae24 87d6ae24 +USB_2022-10-28_8b921a8d 8b921a8d +USB_2022-10-28_8c47cf7a 8c47cf7a +USB_2022-10-28_8f9b9836 8f9b9836 +USB_2022-10-28_8fbb5096 8fbb5096 +USB_2022-10-28_91c064fa 91c064fa +USB_2022-10-28_93508ce8 93508ce8 +USB_2022-10-28_95e689e7 95e689e7 +USB_2022-10-28_97f4392d 97f4392d +USB_2022-10-28_98ea126c 98ea126c +USB_2022-10-28_9accd506 9accd506 +USB_2022-10-28_a039b49f a039b49f +USB_2022-10-28_a14af161 a14af161 +USB_2022-10-28_a28294dd a28294dd +USB_2022-10-28_a3e00176 a3e00176 +USB_2022-10-28_a91e6a38 a91e6a38 +USB_2022-10-28_a982ae48 a982ae48 +USB_2022-10-28_ab871ca4 ab871ca4 +USB_2022-10-28_ae4f0562 ae4f0562 +USB_2022-10-28_b1c9ca82 b1c9ca82 +USB_2022-10-28_b3e83a35 b3e83a35 +USB_2022-10-28_b8f682a2 b8f682a2 +USB_2022-10-28_bbb31a19 bbb31a19 +USB_2022-10-28_bbdc13f8 bbdc13f8 +USB_2022-10-28_c12ccbf8 c12ccbf8 +USB_2022-10-28_c2501231 c2501231 +USB_2022-10-28_c5819b28 c5819b28 +USB_2022-10-28_c598255e c598255e +USB_2022-10-28_c60153cb c60153cb +USB_2022-10-28_c8ad9b27 c8ad9b27 +USB_2022-10-28_c8eb9e24 c8eb9e24 +USB_2022-10-28_c978288c c978288c +USB_2022-10-28_c9b6dc45 c9b6dc45 +USB_2022-10-28_c9f6eb57 c9f6eb57 +USB_2022-10-28_cabacd25 cabacd25 +USB_2022-10-28_d23ae8c2 d23ae8c2 +USB_2022-10-28_d65d8c91 d65d8c91 +USB_2022-10-28_d7e173ac d7e173ac +USB_2022-10-28_d94490ad d94490ad +USB_2022-10-28_d98a176f d98a176f +USB_2022-10-28_e0b92d21 e0b92d21 +USB_2022-10-28_e664b36b e664b36b +USB_2022-10-28_e67d23f1 e67d23f1 +USB_2022-10-28_eb012c01 eb012c01 +USB_2022-10-28_ec757eb6 ec757eb6 +USB_2022-10-28_eccba265 eccba265 +USB_2022-10-28_ed417a2b ed417a2b +USB_2022-10-28_f0625d67 f0625d67 +USB_2022-10-28_f1394a2a f1394a2a +USB_2022-10-28_f1773d26 f1773d26 +USB_2022-10-28_f32dd63c f32dd63c +USB_2022-10-28_f7f38bd6 f7f38bd6 +USB_2022-10-28_f91d1cf4 f91d1cf4 +USB_2022-10-28_fbfe00ca fbfe00ca +USB_2022-10-28_fc900762 fc900762 diff --git a/pangolin_src/log/batman.sh b/pangolin_src/log/batman.sh new file mode 100755 index 0000000..dcd6c22 --- /dev/null +++ b/pangolin_src/log/batman.sh @@ -0,0 +1,241 @@ +#!/bin/bash + +clusterdir=/cluster/project/pangolin +working=working +sampleset=sampleset + +# +# Input validator +# +validateBatchName() { + if [[ "$1" =~ ^(20[0-9][0-9][0-1][0-9][0-3][0-9]_[[:alnum:]-]{4,})$ ]]; then + return; + else + echo "bad batchname ${1}" + exit 1; + fi +} + +validateTags() { + local IFS="$1" + for b in $2; do + validateBatchName "${b}" + done +} + + +RXJOB='Job <([[:digit:]]+)> is submitted' +# Generic job. +# Job <129052039> is submitted to queue . + +now=$(date '+%Y%m%d') +lastmonth=$(date '+%Y%m' --date='-1 month') +thismonth=$(date '+%Y%m') +twoweeksago=$(date '+%Y%m%d' --date='-2 weeks') + +if [[ "$1" == "--limited" ]]; then + shift + case "$1" in + rsync|df) + # sub-commands allowed in limited mode. + ;; + *) + echo "sub-command ${1} not allowed in limited mode" >&2 + exit 2 + ;; + esac +fi + +case "$1" in + rsync) + # rsync daemon : see ${SSH_ORIGINAL_COMMAND} + rsync --server --daemon . + ;; + logrotate) + # rotate logs + ~/log/rotate + ;; + addsamples) + lst="${clusterdir}/${working}/samples.tsv" + case "$2" in + --recent) + lst="${clusterdir}/${working}/samples.recent.tsv" + echo "syncing recent: ${lastmonth}, ${thismonth}" + ;; + *) + echo "Unkown parameter ${2}" > /dev/stderr + exit 2 + ;; + esac + mkdir -p --mode=2770 "${clusterdir}/${working}/samples/" + #cp -vrf --link ${clusterdir}/${sampleset}/*/ ${clusterdir}/${working}/samples/ ## failure: "no rule to create {SAMPLE}/extract/R1.fastq" + cat ${clusterdir}/${sampleset}/samples.{${lastmonth},${thismonth}}*.tsv | sort -u > "${clusterdir}/${working}/samples.recent.tsv" + sort -u ${clusterdir}/${sampleset}/samples.*.tsv > "${clusterdir}/${working}/samples.tsv" + cut -f1 "${lst}" | xargs -P 8 -i cp -vrf --link "${clusterdir}/${sampleset}/{}/" "${clusterdir}/${working}/samples/" + ;; + vpipe) + declare -A job + list=('seq' 'seqqa' 'snv' 'snvqa' 'hugemem' 'hugememqa') + shorah=1 + hold= + tag= + while [[ -n $2 ]]; do + case "$2" in + --no-shorah) + shorah=0 + ;; + --hold) + hold='-H' + ;; + --tag) + shift + if [[ ! "${2}" =~ ^[[:alnum:]]+$ ]]; then + # if it's not just letters and number, test if it is a list of valid batches + validateTags ';' "$2" + fi + tag="${2%;}" + ;; + --recent) +# # TODO switch between full cohort and only recent +# #recent="..." + ;; + *) + echo "Unkown parameter ${2}" > /dev/stderr + exit 2 + ;; + esac + shift + done + # start first job + cd ${clusterdir}/${working}/ + # use -H to put on hold for analysis + if [[ "$(sed "s/@TAG@/<${tag}>/g" vpipe-no-shorah.bsub | bsub -J "COVID-vpipe-<${tag}>-cons" ${hold})" =~ ${RXJOB} ]]; then + job['seq']=${BASH_REMATCH[1]} + # schedule a gatherqa no mater what happens + [[ "$(sed "s/@TAG@/<${tag}>/g" qa-launcher | bsub -J "COVID-vpipe-<${tag}>-qa" ${hold} -w "ended(${job['seq']})")" =~ ${RXJOB} ]] && job['seqqa']=${BASH_REMATCH[1]} + # if no fail schedule a full job with snv + if (( shorah )) && [[ "$(bsub ${hold} -w "done(${job['seq']})" -ti < vpipe.bsub)" =~ ${RXJOB} ]]; then + job['snv']=${BASH_REMATCH[1]} + # schedule a gatherqa no matter what happens to snv + [[ "$(bsub ${hold} -w "done(${job['seq']})&&ended(${job['snv']})" < qa-launcher)" =~ ${RXJOB} ]] && job['snvqa']=${BASH_REMATCH[1]} + # schedule a hugemem job if snvjob failed + if [[ "$(bsub ${hold} -w "done(${job['seq']})&&exit(${job['snv']})" -ti < vpipe-hugemem.bsub)" =~ ${RXJOB} ]]; then + job['hugemem']=${BASH_REMATCH[1]} + # schedule a qa afterward + [[ "$(bsub -w "done(${job['seq']})&&exit(${job['snv']})&&ended(${job['hugemem']})" -ti < qa-launcher)" =~ ${RXJOB} ]] && job['hugememqa']=${BASH_REMATCH[1]} + fi + fi + fi >&2 + # write job chain list + for v in "${list[@]}"; do + printf "%s\t%s\n" "${v}" "${job[$v]}" + done + ;; + job) + if [[ $2 =~ ^([[:digit:]]+)$ ]]; then + bjobs $2 | gawk -v I=$2 '$1==I{print $3}' + else + bjobs + fi + ;; + purgelogs) + find ${clusterdir}/${working}/cluster_logs/ -type f -mtime +28 -name '*.log' -print0 | xargs -0 rm -- + ;; + scratch) + temp_scratch="${SCRATCH}/pangolin/temp" + olderthan=60 + purge=0 + loop=0 + filter= + while [[ -n $2 ]]; do + case "$2" in + --minutes-ago) + if [[ ! "${3}" =~ ^[[:digit:]]+$ ]]; then + echo "parameter of ${2} must be a number of minutes (digits only), got <${3}> instead" > /dev/stderr + exit 2 + fi + shift + olderthan=$2 + ;; + --loop) + loop=1 + ;& + --purge) + purge=1 + ;; + *) + echo "Unkown parameter ${2}" > /dev/stderr + exit 2 + ;; + esac + shift + done + + if (( purge )); then + echo "purging in ${temp_scratch}..." + else + echo "listing in ${temp_scratch}..." + fi + + count_all=0 + count_old=0 + while read s; do + (( ++count_all )) + if [[ -r "${clusterdir}/${working}/samples/${s}/upload_prepared.touch" && $(find "${clusterdir}/${working}/samples/${s}/upload_prepared.touch" '!' -newermt "${olderthan} minutes ago") ]]; then + (( ++count_old )) + if (( purge )); then + rm -rvf "${temp_scratch}/samples/${s}" + else + echo "${s}"; + fi + fi; + done < <((cd "${temp_scratch}/" && find samples/ -type f ${filter} ) | grep -oP '(?<=samples/)[^/]+/[^/]+(?=/)' | sort -u) | tee /dev/stderr | wc -l 2>&1 + echo "Samples: ${count_old} old / ${count_all} total" + + if (( loop )); then + echo -e '\n\e[38;5;45;1mYou just keep on trying\e[0m\n\e[38;5;208;1mTill you run out of cake\e[0m' + if sleep "$(( 5 + olderthan))m"; then + # loop if no breaks + exec "${0}" scratch --minutes-ago "${olderthan}" --loop + fi + echo "I'm not even angry" + fi + ;; + completion) + if [[ $2 =~ ^([[:digit:]]+)$ ]]; then + bpeek $2 | gawk '$0~/^\[.*\]$/{date=$0};$0~/^[[:digit:]]+ of [[:digit:]]+ steps \([[:digit:]]+%\) done$/{print $0 "\t" date}' + fi + ;; + df) + #df ${clusterdir} ${SCRATCH} + lquota -2 ${clusterdir} + lquota -2 ${SCRATCH} + ;; + garbage) + validateBatchName "$2" + cd ${clusterdir} + + for f in ${sampleset}/*/${2}; do + garbage=$(dirname "${f//${sampleset}/garbage}") + mkdir --mode=0770 -p "${garbage}" + mv -v "${f%/}" "${garbage}/" + done + for f in ${working}/samples/*/${2}/; do + garbage="${f//${working}\/samples/garbage}" + mkdir --mode=0770 -p "${garbage%/}/"{raw_data,extracted_data}/ + mv -vf "${f%/}/raw_data/"* "${garbage%/}/raw_data/" + rm "${f%/}/raw_data/"*.fa* + rmdir "${f%/}/raw_data/" + mv -vf "${f%/}/extracted_data/"* "${garbage%/}/extracted_data/" + rm "${f%/}/extracted_data/"*{.log,.benchmark,_fastqc.html} + rmdir "${f%/}/extracted_data/" + mv -vf "${f%/}/"* "${garbage%/}/" + rmdir "${f%/}" + done + mv "${sampleset}/batch.${2}.yaml" "${sampleset}/samples.${2}.tsv" "${sampleset}/missing.${2}.txt" "${sampleset}/projects.${2}.tsv" garbage/ + ;; + *) + echo "Unkown sub-command ${1}" > /dev/stderr + exit 2 + ;; +esac diff --git a/pangolin_src/log/belfry.sh b/pangolin_src/log/belfry.sh new file mode 100755 index 0000000..0d0ee29 --- /dev/null +++ b/pangolin_src/log/belfry.sh @@ -0,0 +1,677 @@ +#!/usr/bin/env bash + +scriptdir="$(dirname $(realpath $(which $0)))" + +if [[ $(uname) == Darwin ]]; then + date=gdate +else + date=date +fi + + +declare -A lab +. ${scriptdir}/config/server.conf + +: ${basedir:=$(pwd)} +: ${download:?} +: ${sampleset:=sampleset} +: ${working:=working} +: ${releasedir:?} +: ${storgrp:?} +: ${parallel:=16} +: ${parallelpull:=${parallel}} +: ${contimeout:=300} +: ${retries:=10} +: ${iotimeout:=300} +: ${protocolyaml:=/references/primers.yaml} + +if [[ $(realpath $scriptdir) != $(realpath $basedir) ]]; then + echo "$scriptdir vs $basedir" +fi + +if [[ ! $mode =~ ^[0-7]{,4}$ ]]; then + echo "Invalid characters <${mode//[0-7]/}> in <${mode}>" + echo 'mode should be an octal chmod value, see `mkdir --help` for informations' + mode= +fi + +set -e + +# cd ${basedir} + +umask 0002 + +statusdir="${basedir}/status" +mkdir ${mode:+--mode=${mode}} -p ${statusdir} + +baseconda=${scriptdir} + +timeoutforeground= +#--foreground +# + +now=$($date '+%Y%m%d') +lastmonth=$($date '+%Y%m' --date='-1 month') +thismonth=$($date '+%Y%m') +twoweeksago=$($date '+%Y%m%d' --date='-2 weeks') +oneweekago=$($date '+%Y%m%d' --date='-1 weeks') + +# +# Input validator +# +validateBatchDate() { + if [[ "$1" =~ ^(20[0-9][0-9][0-1][0-9][0-3][0-9])$ ]]; then + return; + else + echo "bad batchdate ${1}" + exit 1; + fi +} + +validateBatchName() { + if [[ "$1" =~ ^(20[0-9][0-9][0-1][0-9][0-3][0-9]_[[:alnum:]-]{4,})$ ]]; then + return; + else + echo "bad batchname ${1}" + exit 1; + fi +} + + +# +# rsync parallel helpers +# +callpushrsync() { + . config/server.conf + + local arglist=( ) + if (( ${#@} )); then + arglist=( "${@/#/${basedir}/${sampleset}/}" ) + else + #arglist=( "${basedir}/${sampleset}/" ) + echo "rsync job didn't receive list" + exit 1; + fi + exec timeout ${timeoutforeground} --signal=INT --kill-after=5 $((rsynctimeout+contimeout+5)) \ + rsync --timeout=${iotimeout} \ + --password-file ~/rsync.pass.euler \ + -e "ssh -i ${HOME}/.ssh/id_ed25519_belfry -l ${cluster_user} -oConnectTimeout=${contimeout}" \ + -izrltH --fuzzy --fuzzy --inplace \ + -p --chmod=Dg+s,ug+rw,o-rwx,Fa-x \ + -g --chown=:'bsse-covid19-pangolin-euler' \ + "${arglist[@]}" \ + belfry@euler.ethz.ch::${sampleset}/ +} +export -f callpushrsync + + +callpullrsync() { + . config/server.conf + + local arglist=( ) + if (( ${#@} )); then + arglist=( "${@/#/belfry@euler.ethz.ch::${working}/samples/}" ) + else + #arglist=( "belfry@euler.ethz.ch::${working}/samples/" ) + echo "rsync job didn't receive list" + exit 1; + fi + exec timeout ${timeoutforeground} --signal=INT --kill-after=5 $((rsynctimeout+contimeout+5)) \ + rsync --timeout=${iotimeout} \ + --password-file ~/rsync.pass.euler \ + -e "ssh -i ${HOME}/.ssh/id_ed25519_belfry -l ${cluster_user} -oConnectTimeout=${contimeout}" \ + -izrltH --fuzzy --fuzzy --inplace \ + --link-dest=${basedir}/${sampleset}/ \ + "${arglist[@]}" \ + --exclude='uploads/*' \ + --exclude='raw_uploads/*.tmp.*' \ + --exclude='raw_data/*_R[12].fastq.gz' \ + --exclude='extracted_data/R[12]_fastqc.html' \ + --exclude='variants/SNVs/REGION_*/reads.fas' \ + --exclude='variants/SNVs/REGION_*/w-*.reads.fas' \ + --exclude='variants/SNVs/REGION_*/raw_reads/w-*.reads.fas.gz' \ + --exclude='*.out.log' \ + --exclude='*.err.log' \ + --exclude='*.benchmark' \ + ${basedir}/${working}/samples/ +} +export -f callpullrsync + + +callpullrsync_noshorah() { + . config/server.conf + + local arglist=( ) + if (( ${#@} )); then + arglist=( "${@/#/belfry@euler.ethz.ch::${working}/samples/}" ) + else + #arglist=( "belfry@euler.ethz.ch::${working}/samples/" ) + echo "rsync job didn't receive list" + exit 1; + fi + exec timeout ${timeoutforeground} --signal=INT --kill-after=5 $((rsynctimeout+contimeout+5)) \ + rsync --timeout=${iotimeout} \ + --password-file ~/rsync.pass.euler \ + -e "ssh -i ${HOME}/.ssh/id_ed25519_belfry -l ${cluster_user} -oConnectTimeout=${contimeout}" \ + -izrltH --fuzzy --fuzzy --inplace \ + --link-dest=${basedir}/${sampleset}/ \ + "${arglist[@]}" \ + --exclude='uploads/*' \ + --exclude='raw_uploads/*.tmp.*' \ + --exclude='raw_data/*_R[12].fastq.gz' \ + --exclude='extracted_data/R[12]_fastqc.html' \ + --exclude='variants/' \ + --exclude='visualization/' \ + --exclude='*.out.log' \ + --exclude='*.err.log' \ + --exclude='*.benchmark' \ + ${basedir}/${working}/samples/ +} +export -f callpullrsync_noshorah + + +# +# sync helper +# +checksyncoutput() { + local new="${statusdir}/sync${1}_new" + local last="${statusdir}/sync${1}_last" + + if [[ "${2}" =~ Total:\ +([[:digit:]]+)\ +directories,\ +([[:digit:]]+)\ +files,.*?New:\ +([[:digit:]]+)\ +files, ]]; then + echo "Newfiles downloaded" + echo -e "${BASH_REMATCH[2]}\n${BASH_REMATCH[1]}" > ${last} + flock -x -o ${last} -c "sleep 1" + echo "${BASH_REMATCH[3]}" > ${new} + else + echo "Same old shit" + touch ${last} + fi 2>&1 +} + + +set -e + +# +# main handler +# +case "$1" in + syncopenbis) + echo "Sync GFB - OpenBIS..." + if [[ "${2}" = "--recent" ]]; then + param=( "${lastmonth}*" "${thismonth}*" ) + echo "syncing recent: ${param[*]}" + elif [[ "${2}" = "--thismonth" ]]; then + param=( "${thismonth}*" ) + echo "syncing this month: ${param[*]}" + elif [[ "${2}" = "--lastweek" ]]; then + param=( "${oneweekago}*" ) + echo "syncing lastweek: ${param[*]}" + elif [[ "${2}" = "--twoweeksago" ]]; then + param=( "${twoweeksago}*" ) + echo "syncing lastweek: ${param[*]}" + else + param=( ) + fi + syncoutput="$(/usr/bin/time ${scriptdir}/sync_sftp.sh -c ${scriptdir}/config/gfb.conf "${param[@]}"|tee /dev/stderr)" + checksyncoutput "openbis" "$syncoutput" + ${baseconda}/miniconda3/bin/deactivate + ;; + syncfgcz) + echo "Sync FGCZ - bfabric" + . $baseconda/miniconda3/bin/activate "" + . <(grep '^projlist=' config/fgcz.conf) + if [[ "${2}" = "--recent" ]]; then + limitlast='3 weeks ago' + ./exclude_list_bfabric -c config/fgcz.conf -r "${twoweeksago}" -o ${statusdir}/fgcz.exclude.lst + param=( '-e' "${statusdir}/fgcz.exclude.lst" "${projlist[@]}" ) + echo -ne "syncing recent: ${limitlast}\texcluding: " + wc -l ${statusdir}/fgcz.exclude.lst + else + param=( "${projlist[@]}" ) + fi + syncoutput="$(/usr/bin/time ${scriptdir}/sync_sftp.sh -c config/fgcz.conf ${limitlast:+ -N "${limitlast}"} "${param[@]}"|tee /dev/stderr)" + checksyncoutput "fgcz" "$syncoutput" + ${baseconda}/miniconda3/bin/deactivate + ;; + synch2030) + echo "Sync Health2030" + . $baseconda/miniconda3/bin/activate "" + # short test - only list dirs + if [[ ( ( ! -e ${statusdir}/synch2030_ended ) && ( ! -e ${statusdir}/synch2030_started ) ) || ( ${statusdir}/synch2030_ended -nt ${statusdir}/synch2030_started ) ]]; then + ${scriptdir}/list_sftp -c config/h2030.conf -l ${statusdir}/synch2030.${now} && \ + if [[ -s ${statusdir}/synch2030.${now} ]]; then + ln -sf synch2030.${now} ${statusdir}/synch2030_started + fi + # did we get new list? + if [[ ( -e ${statusdir}/synch2030_started ) && ( ( ! -e ${statusdir}/synch2030_ended ) || ( ${statusdir}/synch2030_started -nt ${statusdir}/synch2030_ended ) ) ]]; then + cat ${statusdir}/synch2030_started + else + echo "no new dirs" + fi + fi + # long sync - full mirror + if [[ ( -e ${statusdir}/synch2030_started ) && ( ( ! -e ${statusdir}/synch2030_ended ) || ( ${statusdir}/synch2030_started -nt ${statusdir}/synch2030_ended ) ) ]]; then + syncoutput="$(/usr/bin/time ${scriptdir}/sync_sftp.sh -c config/h2030.conf |tee /dev/stderr)" + checksyncoutput "h2030" "$syncoutput" + ${scriptdir}/check_sums -c config/h2030.conf 'md5.txt' $( < ${statusdir}/synch2030_started) + # 2: internal error, grep error + # 1: no failed checksum found + # 0: failed checksum found + if [[ "$?" == "1" ]]; then + touch ${statusdir}/synch2030_ended + fi + fi + ${baseconda}/miniconda3/bin/deactivate + ;; + syncviollier) + echo "Sync Viollier" + . $baseconda/miniconda3/bin/activate "" + param=( 'sample_metadata' ) + if [[ "${2}" = "--recent" ]]; then + param+=( "raw_sequences/${lastmonth}*" "raw_sequences/${thismonth}*" ) + echo "syncing recent: ${param[*]}" + echo "NOT SUPPORTED" + exit 1 + else + param+=( 'raw_sequences' ) + echo "syncing all: ${param[*]}" + fi + # HACK delete previous partial downloads + . <(grep '^download=' config/viollier.conf) + echo "Removing partial files from:" "${param[@]/#/${download}/}" + find "${param[@]/#/${download}/}" -type f -name '*.filepart' -print0 | xargs -r -0 rm -v + syncoutput="$(/usr/bin/time ${scriptdir}/sync_sftp.sh -c config/viollier.conf "${param[@]}"|tee /dev/stderr)" + checksyncoutput "viollier" "$syncoutput" + ${baseconda}/miniconda3/bin/deactivate + ;; + uploadviollier) + echo "Uploading Viollier" + . $baseconda/miniconda3/bin/activate "" + param=( 'consensus_sequences' 'raw_othercenters' ) + + ${scriptdir}/upload_sftp -c config/viollier.conf "${param[@]}" + ${baseconda}/miniconda3/bin/deactivate + ;; + purgeviollier) + if (( ! ${lab[viollier]} )); then + echo "Viollier disabled; no purging" + exit 0 + fi + echo "Purge Viollier" + . $baseconda/miniconda3/bin/activate "" + param=( 'raw_sequences' ) + recent=$($date --date='2 week ago' '+%Y%m%d') + + ${scriptdir}/purge_sftp -r "${recent}" -c config/viollier.conf "${param[@]}" + ${baseconda}/miniconda3/bin/deactivate + ;; + uploadrequests) + echo "Handling raw-read upload requests for Viollier" + . $baseconda/miniconda3/bin/activate "vineyard" + ${scriptdir}/handle_request_raw_viollier -c config/viollier.conf + ${baseconda}/miniconda3/bin/deactivate + ;; + sortsamples) + . $baseconda/miniconda3/bin/activate pybis + # cd $basedir + summary="" + recent="" + shrtrecent="" + force="${sort_force}" + while [[ -n $2 ]]; do + case "$2" in + --summary) + summary='--summary' + ;; + --force) + force='--force' + ;; + --recent) + recent="--recent=${lastmonth}" + shrtrecent="-r ${lastmonth}" + ;; + *) + echo "Unkown parameter ${2}" > /dev/stderr + exit 2 + ;; + esac + shift + done + fail=0 + if (( ${lab[gfb]} )); then + ${scriptdir}/sort_samples_pybis.py -c ${scriptdir}/config/gfb.conf --protocols=${working}/${protocolyaml} --assume-same-protocol ${force} ${summary} ${recent} && bash ${basedir}/${sampleset}/movedatafiles.sh || fail=1 + else + echo "Skipping gfb" + fi + if (( ${lab[fgcz]} )); then +# # TODO config file + . <(grep '^google_sheet_patches=' fgcz.conf) + + (( google_sheet_patches )) && ${scriptdir}/google_sheet_patches + ${scriptdir}/sort_samples_bfabric_tsv -c fgcz.conf --no-fastqc --protocols=${working}/${protocolyaml} --libkit-override=${basedir}/${sampleset}/patch.fgcz-libkit.tsv ${force} ${recent} && bash ${basedir}/${sampleset}/movedatafiles.sh || fail=1 + else + echo "Skipping fgcz" + fi + if (( ${lab[h2030]} )) && [[ -e ${statusdir}/synch2030_ended && -e ${statusdir}/synch2030_started && ${statusdir}/synch2030_ended -nt ${statusdir}/synch2030_started ]]; then + # NOTE always recent/based on last sync), no support for --force, move done immediately/no separate movedatafiles.sh + # TODO support for protocols + ${scriptdir}/sort_h2030 -c h2030.conf $(< ${statusdir}/synch2030_started ) || fail=1 + else + echo "Skipping h2030" + fi + if (( ${lab[viollier]} )); then + # NOTE always --force, short options only + # HACK hardcoded paths due to multiple directories + ${scriptdir}/sort_viollier -c viollier.conf -4 ${working}/${protocolyaml} ${shrtrecent} sftp-viollier/raw_sequences/*/ && bash ${basedir}/${sampleset}/movedatafiles.sh || fail=1 + else + echo "Skipping viollier" + fi + (( fail == 0 )) && touch ${statusdir}/sortsamples_success || touch ${statusdir}/sortsamples_fail + ${baseconda}/miniconda3/bin/deactivate + ;; + fixopenbisrights) + validateBatchDate "$2" + # NOTE ACLs should autopropagate + + #dir=( ${basedir}/${download}/${2}* ) + #if (( ${#dir[@]} > 1 )); then + # echo "${#dir[@]} directories" + # echo 'mode directories' + # find "${dir[@]}" -type d -print0 | xargs -0 chmod u+rwx,g+rwXs,o-rwx + # echo 'mode files' + # find "${dir[@]}" -type f -print0 | xargs -0 chmod 0660 + # #echo 'group' + # #chgrp -R "${storgrp}" "${dir[@]}" + #fi + ;; + pushsampleset) + if [[ "${2}" = "--recent" ]]; then + sheets=( ${basedir}/${sampleset}/samples.${lastmonth}*.tsv ${basedir}/${sampleset}/samples.${thismonth}*.tsv ) + # BUG: will generate non-globed pattern if months are missing + echo "pushing recent: ${param[*]##/}" + elif [[ "${2}" = "--batch" ]]; then + validateBatchName "${3}" + sheets=( "${basedir}/${sampleset}/samples.${3}.tsv" ) + else + sheets=( ${basedir}/${sampleset}/samples.*.tsv ) + fi + err=0 + timeout ${timeoutforeground} --signal=INT --kill-after=5 $((rsynctimeout+contimeout+5)) \ + rsync --timeout=${iotimeout} \ + --password-file ~/rsync.pass.euler \ + -e "ssh -i ${HOME}/.ssh/id_ed25519_belfry -l ${cluster_user} -oConnectTimeout=${contimeout}" \ + -izrltH --fuzzy --fuzzy --inplace \ + -p --chmod=Dg+s,ug+rw,o-rwx,Fa-x \ + -g --chown=:'bsse-covid19-pangolin-euler' \ + ${basedir}/${sampleset}/samples.*.tsv \ + ${basedir}/${sampleset}/batch.*.yaml \ + ${basedir}/${sampleset}/projects.*.tsv \ + ${basedir}/${sampleset}/missing.*.txt \ + ${basedir}/${sampleset}/patch.*.tsv \ + belfry@euler.ethz.ch::${sampleset}/ || (( ++err )) + cut -s --fields=1 "${sheets[@]}"|sort -u| \ + gawk -v P=$(( parallel * 4 )) '{i=(NR-1);b=i%P;o[b]=(o[b] " \"" $1 "\"")};END{for(i=0;i /dev/stderr + exit 2 + ;; +esac diff --git a/pangolin_src/log/carillon.sh b/pangolin_src/log/carillon.sh new file mode 100755 index 0000000..0454653 --- /dev/null +++ b/pangolin_src/log/carillon.sh @@ -0,0 +1,375 @@ +#!/bin/bash + +scriptdir="$(dirname $(realpath $(which $0)))" + +. ${scriptdir}/config/server.conf + +: ${run_shorah:=0} +: ${staging:=1} + +if [[ $(realpath $scriptdir) != $(realpath $basedir) ]]; then + echo "$scriptdir vs $basedir" +fi + +if [[ ! $mode =~ ^[0-7]{,4}$ ]]; then + echo "Invalid characters <${mode//[0-7]/}> in <${mode}>" + echo 'mode should be an octal chmod value, see `mkdir --help` for informations' + mode= +fi + +mkdir -p ${basedir} +# cd ${basedir} + +umask 0002 + +statusdir="${basedir}/status" +mkdir ${mode:+--mode=${mode}} -p ${statusdir} +lockfile=${statusdir}/carillon_lock + + +touch ${statusdir}/oh_hai_im_lopping + + +# +# Phase 1: periodic data sync +# + +echo '=========' +echo 'Data sync' +echo '=========' + +set -e + +[[ -n $skipsync ]] && echo "Hack: ${skipsync} will be skiped." + + +# UWE: UNDO: +# [[ $skipsync != gfb ]] && ${scriptdir}/belfry.sh syncopenbis --twoweeksago # --recent +# ${scriptdir}/belfry.sh sortsamples --recent $([[ ${statusdir}/syncopenbis_last -nt ${statusdir}/syncopenbis_new ]] && echo '--summary') + +# uploading requests require prior successful download +if [[ ( -e ${statusdir}/sortsamples_fail ) && ( ${statusdir}/sortsamples_fail -nt ${statusdir}/sortsamples_success ) ]]; then + echo 'warning: cannot serve upload requests sampleset data not successfully fetched yet' > /dev/stderr +#else + #${scriptdir}/belfry.sh uploadrequests +fi + +echo '---------' +if [[ ! ${statusdir}/syncopenbis_last -nt ${statusdir}/syncopenbis_new ]]; then + echo "Data was copied: $(<${statusdir}/syncopenbis_new)" + limit=$(date --date='1 week ago' '+%Y%m%d') + echo "Fixing perms back until $limit:" + for d in $(echo ${basedir}/${download}/20* | grep -oP "(?<=/${download}/)(20[0-9][0-9][0-1][0-9][0-3][0-9])" | sort -r -u); do + if [[ "$d" < "$limit" ]]; then + echo '...done' + break + else + echo "...${d}..." + fi + ${scriptdir}/belfry.sh fixopenbisrights "${d}" + done +fi + +source ./secrets/${cluster_user}@${cluster} + +remote_batman="ssh -ni ${privkey} ${cluster_user}@${cluster} --" + + +# +# Phase 2: update status of current run and trigger backups +# + +echo "=================" +echo "Check current run" +echo "=================" + +if [[ ( -e ${statusdir}/vpipe_started ) && ( ( ! -e ${statusdir}/vpipe_ended ) || ( ${statusdir}/vpipe_started -nt ${statusdir}/vpipe_ended ) ) ]]; then + stillrunning=0 + echo "CHECK" + while read j id; do + echo $j $id + # skip missing + if [[ -z "${id}" ]]; then + echo "$j : (not started)" + continue + fi + + # skip already finished + if [[ ( -e ${statusdir}/vpipe_${j}_ended ) && ( ${statusdir}/vpipe_${j}_ended -nt ${statusdir}/vpipe_started ) ]]; then + old="$(<${statusdir}/vpipe_${j}_ended)" + if [[ "${id}" == "${old}" ]]; then + echo "$j : $id already finished" + else + echo "$j : mismatch $id vs $old" + fi + continue + fi + + # cluster status + stat=$(${remote_batman} job "${id}" || echo "(no answer)") + if [[ ( -n "${stat}" ) && ( ! "${stat}" =~ (EXIT|DONE) ) ]]; then + # running + echo -n "$j : $id : $stat" + (( ++stillrunning )) + if [[ "${stat}" == 'RUN' && ( ! "$j" =~ qa$ ) ]]; then + echo -ne '\t' + ${remote_batman} completion "${id}" | tail -n 1 + else + echo '' + fi + sleep 1 + continue + fi + + # not running + echo "$j : $id finishing" + + case "$j" in + seqqa) + ${scriptdir}/belfry.sh pullsamples_noshorah --recent + if [[ ( -e ${statusdir}/pullsamples_noshorah_fail ) && ( ${statusdir}/pullsamples_noshorah_fail -nt ${statusdir}/pullsamples_noshorah_success ) ]]; then + echo "pulling data failed" + (( ++stillrunning )) + continue + fi + ;; + esac + + echo "${id}" > ${statusdir}/vpipe_${j}_ended + done < ${statusdir}/vpipe_started + echo done + + echo $stillrunning + + if (( stillrunning == 0 )); then + # HACK temporarily avoid copying over ShoRAH files + #${scriptdir}/belfry.sh pullsamples --recent + #if [[ ( ! -e ${statusdir}/pullsamples_fail ) || ( ${statusdir}/pullsamples_success -nt ${statusdir}/pullsamples_fail ) ]]; then + ${scriptdir}/belfry.sh pullsamples_noshorah --recent + if [[ ( ! -e ${statusdir}/pullsamples_noshorah_fail ) || ( ${statusdir}/pullsamples_noshorah_success -nt ${statusdir}/pullsamples_noshorah_fail ) ]]; then + echo "$(basename $(realpath ${statusdir}/vpipe_started))" > ${statusdir}/vpipe_ended + else + echo "pulling data failed" + fi + fi +else + echo 'No current run.' +fi + + + +# +# Phase 3: Reports and sequences +# + +echo "================" +echo "Results handling" +echo "================" + +if [[ ( -e ${statusdir}/pullsamples_noshorah_success || -e ${statusdir}/pullsamples_success ) && ( ( ! -e ${statusdir}/qa_report_success ) || ( ${statusdir}/pullsamples_noshorah_success -nt ${statusdir}/qa_report_success ) || ( ${statusdir}/pullsamples_success -nt ${statusdir}/qa_report_success ) ) ]]; then + #${scriptdir}/belfry.sh qa_report + touch ${statusdir}/qa_report_success +else + echo 'no newer results' +fi + +if [[ ( -e ${statusdir}/qa_report_success ) && ( ( ! -e ${statusdir}/pushseq_success ) || ( ${statusdir}/qa_report_success -nt ${statusdir}/pushseq_success ) ) ]]; then + : + ${scriptdir}/belfry.sh pushseq + #${scriptdir}/belfry.sh gitaddseq +else + echo 'no upload needed' +fi + + + +# +# Phase 4: restart runs if new data +# + +echo "=============" +echo "Start new run" +echo "=============" + +# TODO support a yaml with regex +rxsample='(^[[:digit:]]{6,}_)|(^[[:digit:]]{8,})|(^Y[[:digit:]]{8,})|([[:digit:]]{2}_20[[:digit:]]{2}_[01]?[[:digit:]]_[0-3]?[[:digit:]])|(^KLZHC[oO][vV])|(^B[aA][[:digit:]]{4,})|(^USB_20[[:digit:]]{2}_[01]?[[:digit:]]_[[:digit:]]{2}_.{8})' +#rxsample='^[[:digit:]]{6,}_' + +scanmissingsamples() { + while read sample batch other; do + # look for only guaranteed samples + [[ $sample =~ $rxsample ]] || continue + # check the presence of fasta on each sample + if [[ -e ${basedir}/${working}/samples/${sample}/${batch}/upload_prepared.touch ]]; then + # this will check for: + # - references/ref_majority.fasta + # - references/consensus.bcftools.fasta & .chain + # - references/frameshift_deletions_check.tsv + # etc. + # see V-pipe's rule 'prepare_upload' in publish.smk + echo -n '.' + else + echo -e "\r+${batch/_/:}\t!${sample}\e[K" + true + return 0 + fi; + done < $1 + false +} + +# like "$*" but with a different field separator than default. +join_by() { local IFS="$1"; shift; echo "$*"; } + +if [[ ( ( ! -e ${statusdir}/vpipe_ended ) && ( ! -e ${statusdir}/vpipe_started ) ) || ( ${statusdir}/vpipe_ended -nt ${statusdir}/vpipe_started ) ]]; then + echo 1 + clearline=0 + mustrun=0 + runreason=( ) + declare -A flowcell + if test -e ${statusdir}/vpipe_started; then + ref=$(date --reference="${statusdir}/vpipe_started" '+%Y%m%d') + now=$(date '+%Y%m%d') + limit=$(date --date='2 weeks ago' '+%Y%m%d') + echo "Check batch against ${ref}:" + for t in ${basedir}/${sampleset}/samples.*.tsv; do + if [[ ! $t =~ samples.([[:digit:]]{8})_([[:alnum:]]{5,}(-[[:digit:]]+)?).tsv$ ]]; then + echo "oops: Can't parse <${t}> ?!" > /dev/stderr + fi + + # check Duplicates + b="${BASH_REMATCH[1]}" + f="${BASH_REMATCH[2]}" + if [[ -n "${flowcell[$f]}" ]]; then + echo "error: Duplicate flowcell $f : ${flowcell[$f]} vs $b" > /dev/stderr + exit 2 + else + flowcell[$f]=$b + fi + + # check dates + if (( clearline )) && [[ "$limit" < "$b" ]]; then + echo -ne "\n" + clearline=0 + fi + if [[ "$ref" < "$b" || "$ref" == "$b" ]]; then + echo "!$b:$f" + (( ++mustrun )) + runreason+=( "${b}_${f}" ) + elif [[ "$limit" < "$b" ]] && scanmissingsamples $t; then + (( ++mustrun )) + runreason+=( "${b}_${f}" ) + else + if [[ "$limit" < "$b" ]]; then + echo -e "\r($b:$f)\e[K" + else + echo -ne "($b:$f)\t" + clearline=1 + fi + fi + # sanity check + if [[ "$now" < "$b" ]]; then + echo "oops: in the future $b vs $now" + fi + done + fi + + # are we allowed to submit jobs ? + if (( donotsubmit )); then + echo -e '\e[35;1mWill NOT submit jobs\e[0m...' > /dev/stderr + if (( mustrun )); then + echo 'submit blocked' > ${statusdir}/submit_fail + echo -e '...\e[33;1mbut there are new jobs that should be started !!!\e[0m' > /dev/stderr + else + echo '...and there is nothing to run anyway' > /dev/stderr + fi + # start jobs ? + elif (( mustrun )); then + echo 'Will start new job' + + # Sanity check + if [[ ( -e ${statusdir}/sortsamples_fail ) && ( ${statusdir}/sortsamples_fail -nt ${statusdir}/sortsamples_success ) ]]; then + if (( staging )); then + echo -e '\e[33;1mwarning: sampleset data not successfully fetched yet, using staging\e[0m' > /dev/stderr + else + echo 'data fetch error' > ${statusdir}/submit_fail + echo -e '\e[31;1merror: sampleset data not successfully fetched yet\e[0m' > /dev/stderr + exit 1 + fi + fi + if [[ ( ! -e ${statusdir}/syncopenbis_new ) || ( ( -e ${statusdir}/vpipe_started ) && ( ${statusdir}/vpipe_started -nt ${statusdir}/syncopenbis_new ) ) ]]; then + echo 'oops: something fishy: no downloaded data newer than last run ?' > /dev/stderr + fi + # point of comparison for dates: + if [[ -e ${statusdir}/vpipe_ended ]]; then + lastrun=${statusdir}/vpipe_ended + else + # find the most recent 'new' sync status + lastsync=( $(ls -t ${statusdir}/sync*_new) ) + if [[ -e "${lastsync[0]}" ]]; then + lastrun="${lastsync[0]}" + else + # last fall back: sort success + lastrun=${statusdir}/sortsamples_success + fi + fi + + # push sampleset data + if [[ ( ! -e ${statusdir}/pushsampleset_success ) || ( ${lastrun} -nt ${statusdir}/pushsampleset_success ) ]]; then + ${scriptdir}/belfry.sh pushsampleset --recent + else + echo 'oops: Sampleset already pushed' > /dev/stderr + fi + + # must run + if [[ ( -e ${statusdir}/pushsampleset_fail ) && ( ${statusdir}/pushsampleset_fail -nt ${statusdir}/pushsampleset_success ) ]]; then + echo 'error: Pushing sampleset did not succeed' > /dev/stderr + else + echo 'starting jobs' + if (( run_shorah )); then + shorah="" + else + shorah="--no-shorah" + fi + ${remote_batman} addsamples --recent && \ + ${remote_batman} vpipe ${shorah} --recent --tag "$(join_by ';' "${runreason[@]}")" > ${statusdir}/vpipe.${now} && \ + if [[ -s ${statusdir}/vpipe.${now} ]]; then + ln -sf vpipe.${now} ${statusdir}/vpipe_started + cat ${statusdir}/vpipe_started + printf "%s\t$(date '+%H%M%S')\n" "${runreason[@]}" | tee -a ${statusdir}/vpipe_new.${now} + if [[ -n "${mailto[*]}" ]]; then + ( + echo '(Possibly new) samples not having consensus sequences yet found in batches:' + printf ' - %s\n' "${runreason[@]}" + echo -e '\nStarting V-pipe on Euler:' + cat ${statusdir}/vpipe_started + ) | mail -s '[Automation-carillon] Starting V-pipe on Euler' "${mailto[@]}" + # -r "${mailfrom}" + fi + fi + fi + else + echo 'No new jobs to start' + + # check for left-over .staging files + staging_tsv=( sampleset/samples.202*.tsv.staging ) + if [[ "${staging_tsv[*]}" =~ \* ]]; then + # no staging files => safe to purge + ${scriptdir}/belfry.sh purgeviollier + else + echo 'possible unsubmitted failed import?' > ${statusdir}/submit_fail + echo -e '\e[31;1mLeft-over staging files\e[0m' > /dev/stderr + printf ' - %s\n' "${staging_tsv[@]##*/}" > /dev/stderr + fi + + fi +else + echo 'There is already run going on' +fi + +# +# Closing words +# + +${scriptdir}/belfry.sh df +${remote_batman} df +date -R diff --git a/pangolin_src/log/check_sums.sh b/pangolin_src/log/check_sums.sh new file mode 100755 index 0000000..069ab9a --- /dev/null +++ b/pangolin_src/log/check_sums.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +configfile=config/server.conf + +usage() { echo "Usage: $0 [-c ] " 1>&2; exit $1; } + +while getopts "l:c:h" o; do + case "${o}" in + c) configfile=${OPTARG} + if [[ ! -r ${configfile} ]]; then + echo "Cannot read ${configfile}" 1>&2 + usage 2 + fi + ;; + h) usage 0 ;; + *) usage 2 ;; + esac +done +shift $((OPTIND-1)) + + +. ${configfile} + +: ${basedir:=$(pwd)} +: ${download:?} + +if (( ${#@} < 2)); then + usage 2 +else + sumfile=$1 + shift + case ${sumfile,,} in + *xxh*) + sum=xxh64sum + ;; + *md5*) + sum=md5sum + ;; + *sha256*) + sum=sha256sum + ;; + *sha*) + sum=sha1sum + ;; + *) + echo "Unrecognized sum type: ${sumfile}" 1>&2 + exit 2 + ;; + esac +fi + +success=1 +checkfiles=( ) +for d in "${@}"; do + if [[ ! -d "${basedir}/${download}/${d}" ]]; then + echo "No ${d}" 1>&2 + success=0 + continue + elif [[ ! -r "${basedir}/${download}/${d}/${sumfile}" ]]; then + echo "No ${sumfile} in ${d}" 1>&2 + success=0 + continue + else + checkfiles+=( "${basedir}/${download}/${d}/${sumfile}.check" ) + echo -ne "\r\e[Kchecksum ${d}/${sumfile}\r" + # TODO skip past checks if successful + # TODO make parallel + ( cd ${basedir}/${download}/${d}/; ${sum} -c "${sumfile}" > "${sumfile}.check" ) + fi +done + +echo -ne "\r\e[K" +#echo "success: ${success}" +#echo "${#checkfiles[@]}: ${checkfiles[*]}" + +(( (success > 0) && (${#checkfiles[@]} > 0) )) || exit 2 + +echo -n "FAILED: " +exec grep -cvP ': OK$' "${checkfiles[@]}" diff --git a/pangolin_src/log/cowabunga.sh b/pangolin_src/log/cowabunga.sh new file mode 100755 index 0000000..595ac92 --- /dev/null +++ b/pangolin_src/log/cowabunga.sh @@ -0,0 +1,170 @@ +#!/bin/bash + +umask 0007 + +scriptdir="$(dirname $(which $0))" +baseconda="$scriptdir/" + +working=working +worktest=work-vp-test +sampleset=sampleset + +# +# Input validator +# +validateBatchName() { + if [[ "$1" =~ ^(20[0-9][0-9][0-1][0-9][0-3][0-9]_[[:alnum:]-]{4,})$ ]]; then + return; + else + echo "bad batchname ${1}" + exit 1; + fi +} + +validateProto() { + case "$1" in + v3|v4|v41) + return + ;; + *) + echo "bad proto ${1}" + exit 1; + ;; + esac +} + + + +lastmonth=$(date '+%Y%m' --date='-1 month') +thismonth=$(date '+%Y%m') + +case "$1" in + autoaddww|autoaddwastewater) + projects=( 'p23224' 'p24991' 'p26177' ) + projpat="$( ( IFS='|';echo "${projects[*]}" ) )" + gawk -v d="${lastmonth}" '$2 ${working}/samples.wastewateronly.tsv.old + { + # assemble samples.wastewateronly.tsv + { + printf '%s\n' ${sampleset}/projects.${thismonth}* ${sampleset}/projects.${lastmonth}* | grep -vF '*' | sort -r | while read p; do + echo -n "${p} - ${p//projects/samples} ... " >&2 + gawk -v projpat="${projpat}" '(FILENAME~/\/projects\./)&&($2~projpat){ww[$1]++;nww++};(FILENAME~/\/samples\./)&&(ww[$1]);END{print nww >> "/dev/stderr"}' "${p}" "${p//projects/samples}" + done | tee >(cut -f4 >&3) # pass the protos to the second part bellow + cat ${working}/samples.wastewateronly.tsv.old + } > ${working}/samples.wastewateronly.tsv + } 3>&1 | sort | uniq -c | while read cnt PROTO o; do + echo "proto: ${PROTO} (${cnt})" + gawk -v proto="${PROTO}" '$4==proto' ${working}/samples.wastewateronly.tsv > ${working}/samples.wastewateronly.${PROTO}.tsv + done; + ;; + + shufflebatches) + if [[ -z "$2" || "$2" == "--help" ]]; then + echo "Usage: $0 $1 " 1>&2; + exit 0 + fi + + BATCH=$2 + validateBatchName "${BATCH}" + + # check existing + if [[ ! -e "sampleset/samples.${BATCH}.tsv" ]]; then + echo "Cannot find batch ${BATCH}" >&2 + exit 1 + fi + + echo "batch ${BATCH}" + + # majority votre proto + read cnt PROTO o < <(cut -f4 sampleset/samples.${BATCH}.tsv | sort | uniq -c | tee /dev/stderr) + echo "proto: ${PROTO}" + + # + # Proto validate + specific files + # + case "${PROTO}" in + v3) bedfile="cojac/nCoV-2019.insert.${PROTO^^}.bed" ;; + v4) bedfile="cojac/SARS-CoV-2.insert.${PROTO^^}.txt" ;; + v41) bedfile="cojac/SARS-CoV-2.insert.${PROTO^^}.bed" ;; + *) + echo "bad proto ${PROTO}" + exit 1; + ;; + esac + allprototsv=( ${working}/samples.wastewateronly.{v41,v4,v3}.tsv ) + if [[ ! -r "${bedfile}" ]]; then + echo "missing bedfile ${bedfile}" + exit 1; + fi + + # 'lastweek' backup + if grep -qF "${BATCH}" ${working}/samples.wastewateronly${PROTO:+.${PROTO}}.tsv; then + echo "using:" + ls -l ${working}/samples.wastewateronly${PROTO:+.${PROTO}}{,.lastweek,.thisweek}.tsv + else + echo "backing up lastweek:" + cp -v ${working}/samples.wastewateronly${PROTO:+.${PROTO}}{,.lastweek}.tsv + fi + + (grep -vP '^(Y\d{2,}|NC|\d{6,})' "sampleset/samples.${BATCH}.tsv" | tee ${working}/samples.wastewateronly${PROTO:+.${PROTO}}.thisweek.tsv ; cat ${working}/samples.wastewateronly${PROTO:+.${PROTO}}.lastweek.tsv) > ${working}/samples.wastewateronly${PROTO:+.${PROTO}}.tsv + cat "${allprototsv[@]}" > ${working}/samples.wastewateronly.tsv + ;; + + bring_results) + TSV=samples.wastewateronly.tsv + overwrite=0 + if [[ -n "$2" ]]; then + case "$2" in + --overwrite) + overwrite=1 + ;; + *) + echo "Unkown parameter ${2}" > /dev/stderr + exit 2 + ;; + esac + fi + + # bring current list in + if cp ${scriptdir}/${working}/${TSV} ${scriptdir}/${worktest}/; then + ln -sf "${TSV}" ${worktest}/samples.tsv + else + echo "Cannot find ${TSV}" + exit 1 + fi + + if (( overwrite )); then + echo "Entirely rebuilding results/ directory" + else + echo "Appending missing samples to results/ directory" + fi + + mkdir -p ${worktest}/results/ + + while read s b o; do + rmdir --ignore-fail-on-non-empty ${worktest}/results/${s}/${b}/ + if (( overwrite )); then + rm -rvf ${worktest}/results/${s}/${b}/ + elif [[ -e ${worktest}/results/${s}/${b}/ ]]; then + continue + fi + + mkdir -p ${worktest}/results/${s}/${b}/ + cp -alv ${working}/samples/${s}/${b}/{references,alignments} ${worktest}/results/${s}/${b}/ + done < ${worktest}/${TSV} + ;; + + fetch_cooc) + proto=${2:-v41} + validateProto "${proto}" + echo "fetching ${proto}:" + + gawk -v proto="${proto}" '$4==proto' ${worktest}/samples.tsv | while read s b o; do cat ${worktest}/results/${s}/${b}/signatures/cooc.yaml; echo -n '.' >&2; done > ${worktest}/cooc.${proto}.yaml + echo "done" + ;; + + *) + echo "Unkown subcommand ${1}" > /dev/stderr + exit 2 + ;; +esac diff --git a/pangolin_src/log/handle_request_raw_viollier.sh b/pangolin_src/log/handle_request_raw_viollier.sh new file mode 100755 index 0000000..6485b82 --- /dev/null +++ b/pangolin_src/log/handle_request_raw_viollier.sh @@ -0,0 +1,131 @@ +#!/bin/bash + +configfile=configs/viollier.conf + +usage() { echo "Usage: $0 [ -f ] [-c ]" 1>&2; exit $1; } + +# Helper +fail() { + printf '\e[31;1mArgh: %s\e[0m\n' "$1" 1>&2 + [[ -n "$2" ]] && echo "$2" 1>&2 + exit 1 +} + +oops() { + printf '\e[33;1mOops:%s\e[0m\n' "$1" 1>&2 +} + +title() { + printf '\e[34;1m======================\n%s\n======================\e[0m\n\n' "$1" +} + +message() { + printf '\e[37;1m%s\t%s\e[0m\n' "$1" "$2" +} + +status() { + printf '\e[36;1m%s\e[0m\n' "$1" +} + +force=0 +while getopts "c:fh" o; do + case "${o}" in + c) configfile=${OPTARG} + if [[ ! -r ${configfile} ]]; then + echo "Cannot read ${configfile}" 1>&2 + usage 1 + fi + ;; + f) force=1 ;; + h) usage 0 ;; + *) usage 1 ;; + esac +done +shift $((OPTIND-1)) + + +. ${configfile} + +: ${fileserver:?} +# ${srvport} +: ${expname:?} +: ${basedir:=$(pwd)} +: ${download:?} +: ${sampleset:?} + + +target_base='raw_othercenters' +req_file='fastq_requests.tsv' +msg_file='request_results.txt' + +[[ "${1}" == '-h' || "${1}" == '--help' ]] && usage 0 + + +if [[ $( < ~/.netrc ) =~ machine[[:space:]]+${fileserver}[[:space:]]?login[[:space:]]+([^[:space:]]+) ]]; then + username="${BASH_REMATCH[1]}" +elif [[ $( < ~/.ssh/config ) =~ Host[[:space:]]+${fileserver}[[:space:]]+ ]]; then + username= +else + fail "cannot find login for machine ${fileserver} in ~/.netrc nor ~/.ssh/config" +fi + + +cd ${basedir} + +# download new request file and compare with old one +if lftp -c "set xfer:clobber on; get1 sftp://${username:+${username}@}${fileserver}${srvport:+:${srvport}}/${expname}/${target_base}/${req_file} -o ${download}/${target_base}/"; then + if (( force )); then + status "Forcing update" + # check content + elif [[ -e "${download}/${target_base}/${req_file}.old" ]] && diff -q "${download}/${target_base}/${req_file}"{,.old}; then + # same: nothing to do + status "No new uploads in ${req_file}" + grep -P '^(\([[:alpha:]]+|[[:space:]]+|summary)' ${download}/${target_base}/${msg_file} + exit 0 + fi +else + # error + fail "Cannot download request file sftp://${username:+${username}@}${fileserver}${srvport:+:${srvport}}/${expname}/${target_base}/${req_file}" "Please check that ${configfile} is correctly setup" +fi + +# create result file +tee ${download}/${target_base}/${msg_file} <&1; then + printf '\e[36;1mValidation successful\e[0m\n' 1>&2 + echo -e "successfully validated input\n" +else + # error + printf '\e[31;1mValidation failed!\e[0m' 1>&2 + echo -e "error while trying to validate the request, see above\n" +fi | tee -a ${download}/${target_base}/${msg_file} + +tee ${download}/${target_base}/${msg_file} <&1; then + # upload success + printf '\e[36;1mSuccess\e[0m\n' 1>&2 + echo -e "successfully uploaded to ${subdir}\n" + mv -v "${download}/${target_base}/${req_file}"{,.old} + mv -v "${download}/${target_base}/${validated_file}"{,.old} +else + # error + printf '\e[31;1mArgh:\e[0m' 1>&2 + echo -e "error while trying to upload, see above\n" +fi | tee -a ${download}/${target_base}/${msg_file} + +(echo $'\nsummary of troubles'; grep -oP '(?<=[\(])(zero yield|[[:alpha:]]+)' ${download}/${target_base}/${msg_file}|sort|uniq -c || echo $'\tno troubles found' ) | tee -a ${download}/${target_base}/${msg_file} + +exec lftp -c "set xfer:clobber on; put ${download}/${target_base}/${msg_file} -o sftp://${username:+${username}@}${fileserver}${srvport:+:${srvport}}/${expname}/${target_base}/" diff --git a/pangolin_src/log/list_sftp.sh b/pangolin_src/log/list_sftp.sh new file mode 100755 index 0000000..edae874 --- /dev/null +++ b/pangolin_src/log/list_sftp.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +configfile=config/server.conf + +usage() { echo "Usage: $0 [ -l ] [-c ] [filter [...]]" 1>&2; exit $1; } + +while getopts "l:c:h" o; do + case "${o}" in + c) configfile=${OPTARG} + if [[ ! -r ${configfile} ]]; then + echo "Cannot read ${configfile}" 1>&2 + usage 1 + fi + ;; + l) listfile=${OPTARG} ;; + h) usage 0 ;; + *) usage 1 ;; + esac +done +shift $((OPTIND-1)) + + +. ${configfile} + +: ${fileserver:?} +# ${srvport} +: ${expname:?} +: ${basedir:=$(pwd)} +: ${download:?} + +if [[ $( < ~/.netrc ) =~ machine[[:space:]]+${fileserver}[[:space:]]?login[[:space:]]+([^[:space:]]+) ]]; then + username="${BASH_REMATCH[1]}" +elif [[ $( < ~/.ssh/config ) =~ Host[[:space:]]+${fileserver}[[:space:]]+ ]]; then + username= +else + echo "cannot find login for machine ${fileserver} in ~/.netrc" >&2 + exit 1 +fi + +if (( ${#@} )); then + dir=( "${@/#/ /${expname}/}" ) + source="${dir[*]}" +else + source="/${expname}/" +fi + +missing=( ) +for d in $(lftp -c "connect sftp://${username:+${username}@}${fileserver}${srvport:+:${srvport}}; cls -q1B ${source}"); do + if [[ ! -d "${basedir}/${download}/${d}" ]]; then + missing+=( "${d}" ) + : #echo "${d} missing" + else + : #echo "${d} present" + fi +done + +if (( ${#missing[@]} > 0 )) && [[ -n ${listfile} ]]; then + printf "%s\n" "${missing[@]}" > ${listfile} +fi diff --git a/patch_rename b/pangolin_src/log/patch_rename.sh similarity index 99% rename from patch_rename rename to pangolin_src/log/patch_rename.sh index 9bd2940..14108da 100755 --- a/patch_rename +++ b/pangolin_src/log/patch_rename.sh @@ -2,7 +2,7 @@ scriptdir="$(dirname $(which $0))" -. ${scriptdir}/server.conf +. ${scriptdir}/config/server.conf : ${basedir:=$(pwd)} : ${sampleset:=sampleset} diff --git a/purge_sftp b/pangolin_src/log/purge_sftp.sh similarity index 98% rename from purge_sftp rename to pangolin_src/log/purge_sftp.sh index 6daa904..9db4cec 100755 --- a/purge_sftp +++ b/pangolin_src/log/purge_sftp.sh @@ -1,6 +1,6 @@ #!/bin/bash -configfile=server.conf +configfile=config/server.conf usage() { echo "Usage: $0 [ -d ] [ -i ] [ -r ] [-c ] [filter [...]]" 1>&2; exit $1; } diff --git a/purge_sftp_fasta b/pangolin_src/log/purge_sftp_fasta.sh similarity index 98% rename from purge_sftp_fasta rename to pangolin_src/log/purge_sftp_fasta.sh index ee556ae..90b9f10 100755 --- a/purge_sftp_fasta +++ b/pangolin_src/log/purge_sftp_fasta.sh @@ -1,6 +1,6 @@ #!/bin/bash -configfile=server.conf +configfile=config/server.conf usage() { echo "Usage: $0 [ -d ] [ -r ] [-c ] [filter [...]]" 1>&2; exit $1; } diff --git a/quasimodo b/pangolin_src/log/quasimodo.sh similarity index 96% rename from quasimodo rename to pangolin_src/log/quasimodo.sh index de00c5c..b176ddd 100755 --- a/quasimodo +++ b/pangolin_src/log/quasimodo.sh @@ -31,7 +31,7 @@ drop_cache() { printf '[%u]\t[%s]\tcache droped\n' "$(date --utc '+%s')" "$(date --rfc-3339=seconds)" >>drop_cache.log # sync the buffer to have more to drop - synctimeout=$(timeout -k 5 -s INT ${shorttimeout} grep -oP '(?<=^synctimeout=).*$' server.conf) + synctimeout=$(timeout -k 5 -s INT ${shorttimeout} grep -oP '(?<=^synctimeout=).*$' config/server.conf) if [[ -z '$synctimeout' ]]; then synctimeout=$runtimeout fi @@ -52,7 +52,7 @@ drop_cache() { } ring_carillon() { - newtimeout=$(timeout -k 5 -s INT ${shorttimeout} grep -oP '(?<=^runtimeout=).*$' server.conf) + newtimeout=$(timeout -k 5 -s INT ${shorttimeout} grep -oP '(?<=^runtimeout=).*$' config/server.conf) if [[ -n "${runtimeout}" ]]; then runtimeout=${newtimeout} fi @@ -85,7 +85,7 @@ ring_carillon() { scriptdir="$(dirname $(which $0))" ${scriptdir}/belfry df cluster_user="${USER%%@*}" - cluster_user=$(timeout -k 5 -s INT ${shorttimeout} grep -oP '(?<=^cluster_user=).*$' server.conf) + cluster_user=$(timeout -k 5 -s INT ${shorttimeout} grep -oP '(?<=^cluster_user=).*$' config/server.conf) remote_batman="ssh -ni ${HOME}/.ssh/id_ed25519_batman -l ${cluster_user} euler.ethz.ch --" timeout -k 5 -s INT $shorttimeout ${remote_batman} df date -R diff --git a/sort_h2030 b/pangolin_src/log/sort_h2030.sh similarity index 98% rename from sort_h2030 rename to pangolin_src/log/sort_h2030.sh index 6a77884..2d3dba3 100755 --- a/sort_h2030 +++ b/pangolin_src/log/sort_h2030.sh @@ -1,6 +1,6 @@ #!/bin/bash -configfile=server.conf +configfile=config/server.conf usage() { echo "Usage: $0 [-c ] " 1>&2; exit $1; } diff --git a/sort_samples_email b/pangolin_src/log/sort_samples_email.sh similarity index 100% rename from sort_samples_email rename to pangolin_src/log/sort_samples_email.sh diff --git a/sort_viollier b/pangolin_src/log/sort_viollier.sh similarity index 99% rename from sort_viollier rename to pangolin_src/log/sort_viollier.sh index 88c98a0..cd5c5d8 100755 --- a/sort_viollier +++ b/pangolin_src/log/sort_viollier.sh @@ -1,6 +1,6 @@ #!/bin/bash -configfile=server.conf +configfile=config/server.conf usage() { echo "Usage: $0 [-c ] [-r ] [ -4 ] [ ]" 1>&2; exit $1; } diff --git a/pangolin_src/log/sync_sftp.sh b/pangolin_src/log/sync_sftp.sh new file mode 100755 index 0000000..3b235ba --- /dev/null +++ b/pangolin_src/log/sync_sftp.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +configfile=config/server.conf + +usage() { echo "Usage: $0 [-c ] [ -N ] [ -e ] [filter [...]]" 1>&2; exit $1; } + + +while getopts "c:N:e:h" o; do + case "${o}" in + c) configfile=${OPTARG} + if [[ ! -r ${configfile} ]]; then + echo "Cannot read ${configfile}" 1>&2 + usage 1 + fi + ;; + e) exrxfile="${OPTARG}" ;; + N) newerthan="${OPTARG}" ;; + h) usage 0 ;; + *) usage 1 ;; + esac +done +shift $((OPTIND-1)) + + +. ${configfile} + +: ${fileserver:?} +# ${srvport} +# ${prefix} +: ${expname:?} +: ${basedir:=$(pwd)} +: ${download:?} +: ${parallel:=16} +: ${contimeout:=300} +: ${retries:=10} +: ${iotimeout:=300} + +. secrets/${fileserver} + +# add host rsa key if not done yet: +grep --silent \\[${fileserver}\\]:${srvport} ~/.ssh/known_hosts || { ssh-keyscan -t rsa -p ${srvport} ${fileserver} >> ~/.ssh/known_hosts; } + +if (( ${#@} )); then + dir=( "${@/#/ --directory=${prefix:+${prefix}/}${expname}/}" ) + source="${dir[*]}" +else + source="--directory=${prefix:+${prefix}/}${expname}/*" +fi + +umask 0002 + + +connect="connect sftp://${user}:${password}@${fileserver}:${srvport}" +settings="set cmd:move-background false; set net:timeout $(( contimeout / retries)); set net:max-retries ${retries}; set net:reconnect-interval-base 8; set xfer:timeout ${iotimeout}" +mirror="mirror -v --continue --no-perms --parallel=${parallel} --loop --target-directory=${basedir}/${download} ${source} ${newerthan:+ --newer-than="'${newerthan}'"}${exrxfile:+ --exclude-rx-from="'${exrxfile}'"}" + +exec lftp -c "$settings; $connect; $mirror" diff --git a/upload_sftp b/pangolin_src/log/upload_sftp.sh similarity index 97% rename from upload_sftp rename to pangolin_src/log/upload_sftp.sh index bec3c07..b63fb35 100755 --- a/upload_sftp +++ b/pangolin_src/log/upload_sftp.sh @@ -1,6 +1,6 @@ #!/bin/bash -configfile=server.conf +configfile=config/server.conf usage() { echo "Usage: $0 [-c ] directories [...] " 1>&2; exit $1; } diff --git a/upload_viollier b/pangolin_src/log/upload_viollier.sh similarity index 98% rename from upload_viollier rename to pangolin_src/log/upload_viollier.sh index 6cfbe11..c8f3595 100755 --- a/upload_viollier +++ b/pangolin_src/log/upload_viollier.sh @@ -1,6 +1,6 @@ #!/bin/bash -configfile=viollier.conf +configfile=config/viollier.conf usage() { echo "Usage: $0 [-c ]" 1>&2; exit $1; } diff --git a/upload_viollier_raw b/pangolin_src/log/upload_viollier_raw.sh similarity index 98% rename from upload_viollier_raw rename to pangolin_src/log/upload_viollier_raw.sh index 773ea3c..0db34db 100755 --- a/upload_viollier_raw +++ b/pangolin_src/log/upload_viollier_raw.sh @@ -1,6 +1,6 @@ #!/bin/bash -configfile=viollier.conf +configfile=config/viollier.conf usage() { echo "Usage: $0 [-c ] [ -s ] [ -q ] TSVFILE " 1>&2; exit $1; } diff --git a/pangolin_src/movedatafiles.sh b/pangolin_src/movedatafiles.sh new file mode 100644 index 0000000..bc36549 --- /dev/null +++ b/pangolin_src/movedatafiles.sh @@ -0,0 +1,1811 @@ +scriptdir=/cluster/project/pangolin/test_automation/pangolin/pangolin_src +. ${scriptdir}/config/server.conf + +link='--link' +mode='' # e.g.: --mode=0770 + +# Helper +fail() { + printf '\e[31;1mArgh: %s\e[0m\n' "$1" 1>&2 + [[ -n "$2" ]] && echo "$2" 1>&2 + exit 1 +} + +warn() { + printf '\e[33;1mArgh: %s\e[0m\n' "$1" 1>&2 + [[ -n "$2" ]] && echo "$2" 1>&2 +} + +ALLOK=1 +X() { + ALLOK=0 +} + +# sanity checks +if [[ ! -d "/cluster/project/pangolin/sampleset" ]]; then + fail 'No sampleset directory:' "/cluster/project/pangolin/sampleset" +fi +if [[ ! -d "/cluster/project/pangolin/bfabric-downloads" ]]; then + fail 'No download directory:' "/cluster/project/pangolin/bfabric-downloads" +fi + +if [[ ! -d "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899" ]]; then + fail 'Not a directory:' "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899" +fi +echo -ne '\r[················]\r' + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A1_05_2023_09_27/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/A1_05_2023_09_27_S56_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A1_05_2023_09_27/20231013_HLLJMDRX3/raw_data/A1_05_2023_09_27_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/A1_05_2023_09_27_S56_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A1_05_2023_09_27/20231013_HLLJMDRX3/raw_data/A1_05_2023_09_27_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A2_10_2023_09_30/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/A2_10_2023_09_30_S3_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A2_10_2023_09_30/20231013_HLLJMDRX3/raw_data/A2_10_2023_09_30_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/A2_10_2023_09_30_S3_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A2_10_2023_09_30/20231013_HLLJMDRX3/raw_data/A2_10_2023_09_30_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A3_15_2023_09_29/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/A3_15_2023_09_29_S6_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A3_15_2023_09_29/20231013_HLLJMDRX3/raw_data/A3_15_2023_09_29_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/A3_15_2023_09_29_S6_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A3_15_2023_09_29/20231013_HLLJMDRX3/raw_data/A3_15_2023_09_29_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A4_16_2023_10_03/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/A4_16_2023_10_03_S9_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A4_16_2023_10_03/20231013_HLLJMDRX3/raw_data/A4_16_2023_10_03_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/A4_16_2023_10_03_S9_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A4_16_2023_10_03/20231013_HLLJMDRX3/raw_data/A4_16_2023_10_03_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A5_18_2023_09_30/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/A5_18_2023_09_30_S31_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A5_18_2023_09_30/20231013_HLLJMDRX3/raw_data/A5_18_2023_09_30_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/A5_18_2023_09_30_S31_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A5_18_2023_09_30/20231013_HLLJMDRX3/raw_data/A5_18_2023_09_30_R2.fastq.gz"||X +echo -ne "\r[▏···············]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A6_25_2023_09_27/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/A6_25_2023_09_27_S39_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A6_25_2023_09_27/20231013_HLLJMDRX3/raw_data/A6_25_2023_09_27_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/A6_25_2023_09_27_S39_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A6_25_2023_09_27/20231013_HLLJMDRX3/raw_data/A6_25_2023_09_27_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A7_32_2023_10_02/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/A7_32_2023_10_02_S52_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A7_32_2023_10_02/20231013_HLLJMDRX3/raw_data/A7_32_2023_10_02_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/A7_32_2023_10_02_S52_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A7_32_2023_10_02/20231013_HLLJMDRX3/raw_data/A7_32_2023_10_02_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A8_34_2023_09_28/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/A8_34_2023_09_28_S38_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A8_34_2023_09_28/20231013_HLLJMDRX3/raw_data/A8_34_2023_09_28_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/A8_34_2023_09_28_S38_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A8_34_2023_09_28/20231013_HLLJMDRX3/raw_data/A8_34_2023_09_28_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A9_35_2023_10_01/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/A9_35_2023_10_01_S7_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A9_35_2023_10_01/20231013_HLLJMDRX3/raw_data/A9_35_2023_10_01_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/A9_35_2023_10_01_S7_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A9_35_2023_10_01/20231013_HLLJMDRX3/raw_data/A9_35_2023_10_01_R2.fastq.gz"||X +echo -ne "\r[▎···············]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B1_05_2023_09_29/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/B1_05_2023_09_29_S17_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B1_05_2023_09_29/20231013_HLLJMDRX3/raw_data/B1_05_2023_09_29_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/B1_05_2023_09_29_S17_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B1_05_2023_09_29/20231013_HLLJMDRX3/raw_data/B1_05_2023_09_29_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B2_10_2023_10_01/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/B2_10_2023_10_01_S53_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B2_10_2023_10_01/20231013_HLLJMDRX3/raw_data/B2_10_2023_10_01_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/B2_10_2023_10_01_S53_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B2_10_2023_10_01/20231013_HLLJMDRX3/raw_data/B2_10_2023_10_01_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B3_15_2023_09_30/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/B3_15_2023_09_30_S25_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B3_15_2023_09_30/20231013_HLLJMDRX3/raw_data/B3_15_2023_09_30_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/B3_15_2023_09_30_S25_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B3_15_2023_09_30/20231013_HLLJMDRX3/raw_data/B3_15_2023_09_30_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B4_17_2023_09_26/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/B4_17_2023_09_26_S70_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B4_17_2023_09_26/20231013_HLLJMDRX3/raw_data/B4_17_2023_09_26_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/B4_17_2023_09_26_S70_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B4_17_2023_09_26/20231013_HLLJMDRX3/raw_data/B4_17_2023_09_26_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B5_18_2023_10_01/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/B5_18_2023_10_01_S4_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B5_18_2023_10_01/20231013_HLLJMDRX3/raw_data/B5_18_2023_10_01_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/B5_18_2023_10_01_S4_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B5_18_2023_10_01/20231013_HLLJMDRX3/raw_data/B5_18_2023_10_01_R2.fastq.gz"||X +echo -ne "\r[▍···············]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B6_25_2023_09_29/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/B6_25_2023_09_29_S37_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B6_25_2023_09_29/20231013_HLLJMDRX3/raw_data/B6_25_2023_09_29_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/B6_25_2023_09_29_S37_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B6_25_2023_09_29/20231013_HLLJMDRX3/raw_data/B6_25_2023_09_29_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B7_32_2023_10_03/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/B7_32_2023_10_03_S10_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B7_32_2023_10_03/20231013_HLLJMDRX3/raw_data/B7_32_2023_10_03_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/B7_32_2023_10_03_S10_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B7_32_2023_10_03/20231013_HLLJMDRX3/raw_data/B7_32_2023_10_03_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B8_34_2023_09_29/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/B8_34_2023_09_29_S23_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B8_34_2023_09_29/20231013_HLLJMDRX3/raw_data/B8_34_2023_09_29_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/B8_34_2023_09_29_S23_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B8_34_2023_09_29/20231013_HLLJMDRX3/raw_data/B8_34_2023_09_29_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B9_36_2023_09_28/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/B9_36_2023_09_28_S32_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B9_36_2023_09_28/20231013_HLLJMDRX3/raw_data/B9_36_2023_09_28_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/B9_36_2023_09_28_S32_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B9_36_2023_09_28/20231013_HLLJMDRX3/raw_data/B9_36_2023_09_28_R2.fastq.gz"||X +echo -ne "\r[▌···············]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C1_05_2023_09_30/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/C1_05_2023_09_30_S57_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C1_05_2023_09_30/20231013_HLLJMDRX3/raw_data/C1_05_2023_09_30_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/C1_05_2023_09_30_S57_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C1_05_2023_09_30/20231013_HLLJMDRX3/raw_data/C1_05_2023_09_30_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C2_12_2023_09_26/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/C2_12_2023_09_26_S55_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C2_12_2023_09_26/20231013_HLLJMDRX3/raw_data/C2_12_2023_09_26_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/C2_12_2023_09_26_S55_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C2_12_2023_09_26/20231013_HLLJMDRX3/raw_data/C2_12_2023_09_26_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C3_15_2023_10_01/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/C3_15_2023_10_01_S34_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C3_15_2023_10_01/20231013_HLLJMDRX3/raw_data/C3_15_2023_10_01_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/C3_15_2023_10_01_S34_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C3_15_2023_10_01/20231013_HLLJMDRX3/raw_data/C3_15_2023_10_01_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C4_17_2023_09_28/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/C4_17_2023_09_28_S28_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C4_17_2023_09_28/20231013_HLLJMDRX3/raw_data/C4_17_2023_09_28_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/C4_17_2023_09_28_S28_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C4_17_2023_09_28/20231013_HLLJMDRX3/raw_data/C4_17_2023_09_28_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C5_18_2023_10_02/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/C5_18_2023_10_02_S61_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C5_18_2023_10_02/20231013_HLLJMDRX3/raw_data/C5_18_2023_10_02_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/C5_18_2023_10_02_S61_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C5_18_2023_10_02/20231013_HLLJMDRX3/raw_data/C5_18_2023_10_02_R2.fastq.gz"||X +echo -ne "\r[▋···············]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C6_25_2023_09_30/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/C6_25_2023_09_30_S66_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C6_25_2023_09_30/20231013_HLLJMDRX3/raw_data/C6_25_2023_09_30_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/C6_25_2023_09_30_S66_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C6_25_2023_09_30/20231013_HLLJMDRX3/raw_data/C6_25_2023_09_30_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C7_33_2023_09_28/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/C7_33_2023_09_28_S58_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C7_33_2023_09_28/20231013_HLLJMDRX3/raw_data/C7_33_2023_09_28_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/C7_33_2023_09_28_S58_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C7_33_2023_09_28/20231013_HLLJMDRX3/raw_data/C7_33_2023_09_28_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C8_34_2023_09_30/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/C8_34_2023_09_30_S15_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C8_34_2023_09_30/20231013_HLLJMDRX3/raw_data/C8_34_2023_09_30_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/C8_34_2023_09_30_S15_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C8_34_2023_09_30/20231013_HLLJMDRX3/raw_data/C8_34_2023_09_30_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C9_36_2023_09_30/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/C9_36_2023_09_30_S60_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C9_36_2023_09_30/20231013_HLLJMDRX3/raw_data/C9_36_2023_09_30_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/C9_36_2023_09_30_S60_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C9_36_2023_09_30/20231013_HLLJMDRX3/raw_data/C9_36_2023_09_30_R2.fastq.gz"||X +echo -ne "\r[▊···············]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D1_05_2023_10_01/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/D1_05_2023_10_01_S29_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D1_05_2023_10_01/20231013_HLLJMDRX3/raw_data/D1_05_2023_10_01_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/D1_05_2023_10_01_S29_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D1_05_2023_10_01/20231013_HLLJMDRX3/raw_data/D1_05_2023_10_01_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D2_12_2023_09_28/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/D2_12_2023_09_28_S1_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D2_12_2023_09_28/20231013_HLLJMDRX3/raw_data/D2_12_2023_09_28_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/D2_12_2023_09_28_S1_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D2_12_2023_09_28/20231013_HLLJMDRX3/raw_data/D2_12_2023_09_28_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D3_15_2023_10_02/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/D3_15_2023_10_02_S51_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D3_15_2023_10_02/20231013_HLLJMDRX3/raw_data/D3_15_2023_10_02_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/D3_15_2023_10_02_S51_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D3_15_2023_10_02/20231013_HLLJMDRX3/raw_data/D3_15_2023_10_02_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D4_17_2023_09_29/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/D4_17_2023_09_29_S24_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D4_17_2023_09_29/20231013_HLLJMDRX3/raw_data/D4_17_2023_09_29_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/D4_17_2023_09_29_S24_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D4_17_2023_09_29/20231013_HLLJMDRX3/raw_data/D4_17_2023_09_29_R2.fastq.gz"||X +echo -ne "\r[▉···············]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D5_19_2023_09_27/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/D5_19_2023_09_27_S64_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D5_19_2023_09_27/20231013_HLLJMDRX3/raw_data/D5_19_2023_09_27_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/D5_19_2023_09_27_S64_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D5_19_2023_09_27/20231013_HLLJMDRX3/raw_data/D5_19_2023_09_27_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D6_25_2023_10_01/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/D6_25_2023_10_01_S16_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D6_25_2023_10_01/20231013_HLLJMDRX3/raw_data/D6_25_2023_10_01_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/D6_25_2023_10_01_S16_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D6_25_2023_10_01/20231013_HLLJMDRX3/raw_data/D6_25_2023_10_01_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D7_33_2023_09_30/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/D7_33_2023_09_30_S65_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D7_33_2023_09_30/20231013_HLLJMDRX3/raw_data/D7_33_2023_09_30_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/D7_33_2023_09_30_S65_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D7_33_2023_09_30/20231013_HLLJMDRX3/raw_data/D7_33_2023_09_30_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D8_34_2023_10_01/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/D8_34_2023_10_01_S22_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D8_34_2023_10_01/20231013_HLLJMDRX3/raw_data/D8_34_2023_10_01_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/D8_34_2023_10_01_S22_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D8_34_2023_10_01/20231013_HLLJMDRX3/raw_data/D8_34_2023_10_01_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D9_36_2023_10_01/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/D9_36_2023_10_01_S5_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D9_36_2023_10_01/20231013_HLLJMDRX3/raw_data/D9_36_2023_10_01_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/D9_36_2023_10_01_S5_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D9_36_2023_10_01/20231013_HLLJMDRX3/raw_data/D9_36_2023_10_01_R2.fastq.gz"||X +echo -ne "\r[█···············]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E1_05_2023_10_02/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/E1_05_2023_10_02_S13_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E1_05_2023_10_02/20231013_HLLJMDRX3/raw_data/E1_05_2023_10_02_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/E1_05_2023_10_02_S13_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E1_05_2023_10_02/20231013_HLLJMDRX3/raw_data/E1_05_2023_10_02_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E2_12_2023_09_29/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/E2_12_2023_09_29_S50_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E2_12_2023_09_29/20231013_HLLJMDRX3/raw_data/E2_12_2023_09_29_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/E2_12_2023_09_29_S50_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E2_12_2023_09_29/20231013_HLLJMDRX3/raw_data/E2_12_2023_09_29_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E3_16_2023_09_28/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/E3_16_2023_09_28_S68_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E3_16_2023_09_28/20231013_HLLJMDRX3/raw_data/E3_16_2023_09_28_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/E3_16_2023_09_28_S68_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E3_16_2023_09_28/20231013_HLLJMDRX3/raw_data/E3_16_2023_09_28_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E4_17_2023_09_30/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/E4_17_2023_09_30_S2_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E4_17_2023_09_30/20231013_HLLJMDRX3/raw_data/E4_17_2023_09_30_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/E4_17_2023_09_30_S2_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E4_17_2023_09_30/20231013_HLLJMDRX3/raw_data/E4_17_2023_09_30_R2.fastq.gz"||X +echo -ne "\r[█▏··············]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E5_19_2023_09_29/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/E5_19_2023_09_29_S43_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E5_19_2023_09_29/20231013_HLLJMDRX3/raw_data/E5_19_2023_09_29_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/E5_19_2023_09_29_S43_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E5_19_2023_09_29/20231013_HLLJMDRX3/raw_data/E5_19_2023_09_29_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E6_25_2023_10_02/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/E6_25_2023_10_02_S11_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E6_25_2023_10_02/20231013_HLLJMDRX3/raw_data/E6_25_2023_10_02_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/E6_25_2023_10_02_S11_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E6_25_2023_10_02/20231013_HLLJMDRX3/raw_data/E6_25_2023_10_02_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E7_33_2023_10_01/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/E7_33_2023_10_01_S44_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E7_33_2023_10_01/20231013_HLLJMDRX3/raw_data/E7_33_2023_10_01_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/E7_33_2023_10_01_S44_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E7_33_2023_10_01/20231013_HLLJMDRX3/raw_data/E7_33_2023_10_01_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E8_35_2023_09_26/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/E8_35_2023_09_26_S54_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E8_35_2023_09_26/20231013_HLLJMDRX3/raw_data/E8_35_2023_09_26_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/E8_35_2023_09_26_S54_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E8_35_2023_09_26/20231013_HLLJMDRX3/raw_data/E8_35_2023_09_26_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E9_36_2023_10_02/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/E9_36_2023_10_02_S42_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E9_36_2023_10_02/20231013_HLLJMDRX3/raw_data/E9_36_2023_10_02_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/E9_36_2023_10_02_S42_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E9_36_2023_10_02/20231013_HLLJMDRX3/raw_data/E9_36_2023_10_02_R2.fastq.gz"||X +echo -ne "\r[█▎··············]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F1_10_2023_09_26/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/F1_10_2023_09_26_S36_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F1_10_2023_09_26/20231013_HLLJMDRX3/raw_data/F1_10_2023_09_26_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/F1_10_2023_09_26_S36_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F1_10_2023_09_26/20231013_HLLJMDRX3/raw_data/F1_10_2023_09_26_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F2_12_2023_09_30/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/F2_12_2023_09_30_S46_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F2_12_2023_09_30/20231013_HLLJMDRX3/raw_data/F2_12_2023_09_30_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/F2_12_2023_09_30_S46_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F2_12_2023_09_30/20231013_HLLJMDRX3/raw_data/F2_12_2023_09_30_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F3_16_2023_09_30/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/F3_16_2023_09_30_S45_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F3_16_2023_09_30/20231013_HLLJMDRX3/raw_data/F3_16_2023_09_30_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/F3_16_2023_09_30_S45_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F3_16_2023_09_30/20231013_HLLJMDRX3/raw_data/F3_16_2023_09_30_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F4_17_2023_10_01/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/F4_17_2023_10_01_S41_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F4_17_2023_10_01/20231013_HLLJMDRX3/raw_data/F4_17_2023_10_01_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/F4_17_2023_10_01_S41_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F4_17_2023_10_01/20231013_HLLJMDRX3/raw_data/F4_17_2023_10_01_R2.fastq.gz"||X +echo -ne "\r[█▍··············]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F5_19_2023_09_30/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/F5_19_2023_09_30_S19_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F5_19_2023_09_30/20231013_HLLJMDRX3/raw_data/F5_19_2023_09_30_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/F5_19_2023_09_30_S19_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F5_19_2023_09_30/20231013_HLLJMDRX3/raw_data/F5_19_2023_09_30_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F6_32_2023_09_28/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/F6_32_2023_09_28_S33_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F6_32_2023_09_28/20231013_HLLJMDRX3/raw_data/F6_32_2023_09_28_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/F6_32_2023_09_28_S33_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F6_32_2023_09_28/20231013_HLLJMDRX3/raw_data/F6_32_2023_09_28_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F7_33_2023_10_02/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/F7_33_2023_10_02_S27_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F7_33_2023_10_02/20231013_HLLJMDRX3/raw_data/F7_33_2023_10_02_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/F7_33_2023_10_02_S27_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F7_33_2023_10_02/20231013_HLLJMDRX3/raw_data/F7_33_2023_10_02_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F8_35_2023_09_28/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/F8_35_2023_09_28_S20_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F8_35_2023_09_28/20231013_HLLJMDRX3/raw_data/F8_35_2023_09_28_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/F8_35_2023_09_28_S20_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F8_35_2023_09_28/20231013_HLLJMDRX3/raw_data/F8_35_2023_09_28_R2.fastq.gz"||X +echo -ne "\r[█▌··············]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F9_36_2023_10_03/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/F9_36_2023_10_03_S8_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F9_36_2023_10_03/20231013_HLLJMDRX3/raw_data/F9_36_2023_10_03_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/F9_36_2023_10_03_S8_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F9_36_2023_10_03/20231013_HLLJMDRX3/raw_data/F9_36_2023_10_03_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G1_10_2023_09_28/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/G1_10_2023_09_28_S59_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G1_10_2023_09_28/20231013_HLLJMDRX3/raw_data/G1_10_2023_09_28_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/G1_10_2023_09_28_S59_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G1_10_2023_09_28/20231013_HLLJMDRX3/raw_data/G1_10_2023_09_28_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G2_12_2023_10_01/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/G2_12_2023_10_01_S40_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G2_12_2023_10_01/20231013_HLLJMDRX3/raw_data/G2_12_2023_10_01_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/G2_12_2023_10_01_S40_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G2_12_2023_10_01/20231013_HLLJMDRX3/raw_data/G2_12_2023_10_01_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G3_16_2023_10_01/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/G3_16_2023_10_01_S12_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G3_16_2023_10_01/20231013_HLLJMDRX3/raw_data/G3_16_2023_10_01_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/G3_16_2023_10_01_S12_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G3_16_2023_10_01/20231013_HLLJMDRX3/raw_data/G3_16_2023_10_01_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G4_18_2023_09_27/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/G4_18_2023_09_27_S69_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G4_18_2023_09_27/20231013_HLLJMDRX3/raw_data/G4_18_2023_09_27_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/G4_18_2023_09_27_S69_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G4_18_2023_09_27/20231013_HLLJMDRX3/raw_data/G4_18_2023_09_27_R2.fastq.gz"||X +echo -ne "\r[█▋··············]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G5_19_2023_10_01/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/G5_19_2023_10_01_S62_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G5_19_2023_10_01/20231013_HLLJMDRX3/raw_data/G5_19_2023_10_01_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/G5_19_2023_10_01_S62_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G5_19_2023_10_01/20231013_HLLJMDRX3/raw_data/G5_19_2023_10_01_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G6_32_2023_09_30/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/G6_32_2023_09_30_S67_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G6_32_2023_09_30/20231013_HLLJMDRX3/raw_data/G6_32_2023_09_30_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/G6_32_2023_09_30_S67_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G6_32_2023_09_30/20231013_HLLJMDRX3/raw_data/G6_32_2023_09_30_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G7_33_2023_10_03/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/G7_33_2023_10_03_S35_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G7_33_2023_10_03/20231013_HLLJMDRX3/raw_data/G7_33_2023_10_03_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/G7_33_2023_10_03_S35_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G7_33_2023_10_03/20231013_HLLJMDRX3/raw_data/G7_33_2023_10_03_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G8_35_2023_09_29/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/G8_35_2023_09_29_S26_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G8_35_2023_09_29/20231013_HLLJMDRX3/raw_data/G8_35_2023_09_29_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/G8_35_2023_09_29_S26_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G8_35_2023_09_29/20231013_HLLJMDRX3/raw_data/G8_35_2023_09_29_R2.fastq.gz"||X +echo -ne "\r[█▊··············]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H1_10_2023_09_29/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/H1_10_2023_09_29_S30_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H1_10_2023_09_29/20231013_HLLJMDRX3/raw_data/H1_10_2023_09_29_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/H1_10_2023_09_29_S30_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H1_10_2023_09_29/20231013_HLLJMDRX3/raw_data/H1_10_2023_09_29_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H2_15_2023_09_27/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/H2_15_2023_09_27_S14_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H2_15_2023_09_27/20231013_HLLJMDRX3/raw_data/H2_15_2023_09_27_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/H2_15_2023_09_27_S14_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H2_15_2023_09_27/20231013_HLLJMDRX3/raw_data/H2_15_2023_09_27_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H3_16_2023_10_02/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/H3_16_2023_10_02_S49_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H3_16_2023_10_02/20231013_HLLJMDRX3/raw_data/H3_16_2023_10_02_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/H3_16_2023_10_02_S49_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H3_16_2023_10_02/20231013_HLLJMDRX3/raw_data/H3_16_2023_10_02_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H4_18_2023_09_29/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/H4_18_2023_09_29_S63_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H4_18_2023_09_29/20231013_HLLJMDRX3/raw_data/H4_18_2023_09_29_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/H4_18_2023_09_29_S63_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H4_18_2023_09_29/20231013_HLLJMDRX3/raw_data/H4_18_2023_09_29_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H5_19_2023_10_02/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/H5_19_2023_10_02_S48_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H5_19_2023_10_02/20231013_HLLJMDRX3/raw_data/H5_19_2023_10_02_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/H5_19_2023_10_02_S48_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H5_19_2023_10_02/20231013_HLLJMDRX3/raw_data/H5_19_2023_10_02_R2.fastq.gz"||X +echo -ne "\r[█▉··············]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H6_32_2023_10_01/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/H6_32_2023_10_01_S47_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H6_32_2023_10_01/20231013_HLLJMDRX3/raw_data/H6_32_2023_10_01_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/H6_32_2023_10_01_S47_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H6_32_2023_10_01/20231013_HLLJMDRX3/raw_data/H6_32_2023_10_01_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H7_34_2023_09_26/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/H7_34_2023_09_26_S21_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H7_34_2023_09_26/20231013_HLLJMDRX3/raw_data/H7_34_2023_09_26_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/H7_34_2023_09_26_S21_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H7_34_2023_09_26/20231013_HLLJMDRX3/raw_data/H7_34_2023_09_26_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H8_35_2023_09_30/"{,"20231013_HLLJMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/H8_35_2023_09_30_S18_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H8_35_2023_09_30/20231013_HLLJMDRX3/raw_data/H8_35_2023_09_30_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33104_NovaSeq_231013_NOV1899/H8_35_2023_09_30_S18_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H8_35_2023_09_30/20231013_HLLJMDRX3/raw_data/H8_35_2023_09_30_R2.fastq.gz"||X +if [[ ! -d "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904" ]]; then + fail "Not a directory:" "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904" +fi + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A1_05_2023_10_03/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/A1_05_2023_10_03_S13_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A1_05_2023_10_03/20231020_HLNKVDRX3/raw_data/A1_05_2023_10_03_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/A1_05_2023_10_03_S13_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A1_05_2023_10_03/20231020_HLNKVDRX3/raw_data/A1_05_2023_10_03_R2.fastq.gz"||X +echo -ne "\r[██··············]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A2_10_2023_10_07/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/A2_10_2023_10_07_S16_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A2_10_2023_10_07/20231020_HLNKVDRX3/raw_data/A2_10_2023_10_07_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/A2_10_2023_10_07_S16_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A2_10_2023_10_07/20231020_HLNKVDRX3/raw_data/A2_10_2023_10_07_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A3_15_2023_10_03/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/A3_15_2023_10_03_S51_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A3_15_2023_10_03/20231020_HLNKVDRX3/raw_data/A3_15_2023_10_03_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/A3_15_2023_10_03_S51_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A3_15_2023_10_03/20231020_HLNKVDRX3/raw_data/A3_15_2023_10_03_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A4_16_2023_10_10/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/A4_16_2023_10_10_S58_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A4_16_2023_10_10/20231020_HLNKVDRX3/raw_data/A4_16_2023_10_10_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/A4_16_2023_10_10_S58_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A4_16_2023_10_10/20231020_HLNKVDRX3/raw_data/A4_16_2023_10_10_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A5_18_2023_10_07/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/A5_18_2023_10_07_S54_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A5_18_2023_10_07/20231020_HLNKVDRX3/raw_data/A5_18_2023_10_07_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/A5_18_2023_10_07_S54_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A5_18_2023_10_07/20231020_HLNKVDRX3/raw_data/A5_18_2023_10_07_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A6_25_2023_10_03/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/A6_25_2023_10_03_S2_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A6_25_2023_10_03/20231020_HLNKVDRX3/raw_data/A6_25_2023_10_03_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/A6_25_2023_10_03_S2_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A6_25_2023_10_03/20231020_HLNKVDRX3/raw_data/A6_25_2023_10_03_R2.fastq.gz"||X +echo -ne "\r[██▏·············]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A7_32_2023_10_10/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/A7_32_2023_10_10_S1_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A7_32_2023_10_10/20231020_HLNKVDRX3/raw_data/A7_32_2023_10_10_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/A7_32_2023_10_10_S1_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A7_32_2023_10_10/20231020_HLNKVDRX3/raw_data/A7_32_2023_10_10_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A8_34_2023_10_06/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/A8_34_2023_10_06_S20_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A8_34_2023_10_06/20231020_HLNKVDRX3/raw_data/A8_34_2023_10_06_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/A8_34_2023_10_06_S20_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A8_34_2023_10_06/20231020_HLNKVDRX3/raw_data/A8_34_2023_10_06_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A9_36_2023_10_04/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/A9_36_2023_10_04_S26_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A9_36_2023_10_04/20231020_HLNKVDRX3/raw_data/A9_36_2023_10_04_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/A9_36_2023_10_04_S26_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A9_36_2023_10_04/20231020_HLNKVDRX3/raw_data/A9_36_2023_10_04_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B1_05_2023_10_05/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/B1_05_2023_10_05_S21_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B1_05_2023_10_05/20231020_HLNKVDRX3/raw_data/B1_05_2023_10_05_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/B1_05_2023_10_05_S21_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B1_05_2023_10_05/20231020_HLNKVDRX3/raw_data/B1_05_2023_10_05_R2.fastq.gz"||X +echo -ne "\r[██▎·············]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B2_10_2023_10_08/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/B2_10_2023_10_08_S37_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B2_10_2023_10_08/20231020_HLNKVDRX3/raw_data/B2_10_2023_10_08_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/B2_10_2023_10_08_S37_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B2_10_2023_10_08/20231020_HLNKVDRX3/raw_data/B2_10_2023_10_08_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B3_15_2023_10_07/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/B3_15_2023_10_07_S22_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B3_15_2023_10_07/20231020_HLNKVDRX3/raw_data/B3_15_2023_10_07_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/B3_15_2023_10_07_S22_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B3_15_2023_10_07/20231020_HLNKVDRX3/raw_data/B3_15_2023_10_07_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B4_17_2023_10_02/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/B4_17_2023_10_02_S29_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B4_17_2023_10_02/20231020_HLNKVDRX3/raw_data/B4_17_2023_10_02_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/B4_17_2023_10_02_S29_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B4_17_2023_10_02/20231020_HLNKVDRX3/raw_data/B4_17_2023_10_02_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B5_18_2023_10_08/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/B5_18_2023_10_08_S36_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B5_18_2023_10_08/20231020_HLNKVDRX3/raw_data/B5_18_2023_10_08_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/B5_18_2023_10_08_S36_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B5_18_2023_10_08/20231020_HLNKVDRX3/raw_data/B5_18_2023_10_08_R2.fastq.gz"||X +echo -ne "\r[██▍·············]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B6_25_2023_10_05/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/B6_25_2023_10_05_S3_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B6_25_2023_10_05/20231020_HLNKVDRX3/raw_data/B6_25_2023_10_05_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/B6_25_2023_10_05_S3_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B6_25_2023_10_05/20231020_HLNKVDRX3/raw_data/B6_25_2023_10_05_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B7_33_2023_10_04/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/B7_33_2023_10_04_S43_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B7_33_2023_10_04/20231020_HLNKVDRX3/raw_data/B7_33_2023_10_04_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/B7_33_2023_10_04_S43_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B7_33_2023_10_04/20231020_HLNKVDRX3/raw_data/B7_33_2023_10_04_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B8_34_2023_10_07/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/B8_34_2023_10_07_S38_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B8_34_2023_10_07/20231020_HLNKVDRX3/raw_data/B8_34_2023_10_07_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/B8_34_2023_10_07_S38_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B8_34_2023_10_07/20231020_HLNKVDRX3/raw_data/B8_34_2023_10_07_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B9_36_2023_10_06/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/B9_36_2023_10_06_S30_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B9_36_2023_10_06/20231020_HLNKVDRX3/raw_data/B9_36_2023_10_06_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/B9_36_2023_10_06_S30_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B9_36_2023_10_06/20231020_HLNKVDRX3/raw_data/B9_36_2023_10_06_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C1_05_2023_10_07/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/C1_05_2023_10_07_S34_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C1_05_2023_10_07/20231020_HLNKVDRX3/raw_data/C1_05_2023_10_07_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/C1_05_2023_10_07_S34_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C1_05_2023_10_07/20231020_HLNKVDRX3/raw_data/C1_05_2023_10_07_R2.fastq.gz"||X +echo -ne "\r[██▌·············]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C2_12_2023_10_02/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/C2_12_2023_10_02_S60_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C2_12_2023_10_02/20231020_HLNKVDRX3/raw_data/C2_12_2023_10_02_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/C2_12_2023_10_02_S60_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C2_12_2023_10_02/20231020_HLNKVDRX3/raw_data/C2_12_2023_10_02_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C3_15_2023_10_08/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/C3_15_2023_10_08_S15_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C3_15_2023_10_08/20231020_HLNKVDRX3/raw_data/C3_15_2023_10_08_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/C3_15_2023_10_08_S15_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C3_15_2023_10_08/20231020_HLNKVDRX3/raw_data/C3_15_2023_10_08_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C4_17_2023_10_04/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/C4_17_2023_10_04_S35_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C4_17_2023_10_04/20231020_HLNKVDRX3/raw_data/C4_17_2023_10_04_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/C4_17_2023_10_04_S35_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C4_17_2023_10_04/20231020_HLNKVDRX3/raw_data/C4_17_2023_10_04_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C5_18_2023_10_09/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/C5_18_2023_10_09_S19_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C5_18_2023_10_09/20231020_HLNKVDRX3/raw_data/C5_18_2023_10_09_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/C5_18_2023_10_09_S19_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C5_18_2023_10_09/20231020_HLNKVDRX3/raw_data/C5_18_2023_10_09_R2.fastq.gz"||X +echo -ne "\r[██▋·············]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C6_25_2023_10_07/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/C6_25_2023_10_07_S59_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C6_25_2023_10_07/20231020_HLNKVDRX3/raw_data/C6_25_2023_10_07_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/C6_25_2023_10_07_S59_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C6_25_2023_10_07/20231020_HLNKVDRX3/raw_data/C6_25_2023_10_07_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C7_33_2023_10_06/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/C7_33_2023_10_06_S11_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C7_33_2023_10_06/20231020_HLNKVDRX3/raw_data/C7_33_2023_10_06_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/C7_33_2023_10_06_S11_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C7_33_2023_10_06/20231020_HLNKVDRX3/raw_data/C7_33_2023_10_06_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C8_34_2023_10_08/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/C8_34_2023_10_08_S17_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C8_34_2023_10_08/20231020_HLNKVDRX3/raw_data/C8_34_2023_10_08_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/C8_34_2023_10_08_S17_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C8_34_2023_10_08/20231020_HLNKVDRX3/raw_data/C8_34_2023_10_08_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C9_36_2023_10_08/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/C9_36_2023_10_08_S10_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C9_36_2023_10_08/20231020_HLNKVDRX3/raw_data/C9_36_2023_10_08_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/C9_36_2023_10_08_S10_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C9_36_2023_10_08/20231020_HLNKVDRX3/raw_data/C9_36_2023_10_08_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D1_05_2023_10_08/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/D1_05_2023_10_08_S31_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D1_05_2023_10_08/20231020_HLNKVDRX3/raw_data/D1_05_2023_10_08_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/D1_05_2023_10_08_S31_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D1_05_2023_10_08/20231020_HLNKVDRX3/raw_data/D1_05_2023_10_08_R2.fastq.gz"||X +echo -ne "\r[██▊·············]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D2_12_2023_10_04/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/D2_12_2023_10_04_S67_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D2_12_2023_10_04/20231020_HLNKVDRX3/raw_data/D2_12_2023_10_04_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/D2_12_2023_10_04_S67_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D2_12_2023_10_04/20231020_HLNKVDRX3/raw_data/D2_12_2023_10_04_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D3_15_2023_10_09/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/D3_15_2023_10_09_S56_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D3_15_2023_10_09/20231020_HLNKVDRX3/raw_data/D3_15_2023_10_09_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/D3_15_2023_10_09_S56_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D3_15_2023_10_09/20231020_HLNKVDRX3/raw_data/D3_15_2023_10_09_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D4_17_2023_10_06/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/D4_17_2023_10_06_S39_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D4_17_2023_10_06/20231020_HLNKVDRX3/raw_data/D4_17_2023_10_06_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/D4_17_2023_10_06_S39_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D4_17_2023_10_06/20231020_HLNKVDRX3/raw_data/D4_17_2023_10_06_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D5_19_2023_10_03/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/D5_19_2023_10_03_S63_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D5_19_2023_10_03/20231020_HLNKVDRX3/raw_data/D5_19_2023_10_03_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/D5_19_2023_10_03_S63_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D5_19_2023_10_03/20231020_HLNKVDRX3/raw_data/D5_19_2023_10_03_R2.fastq.gz"||X +echo -ne "\r[██▉·············]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D6_25_2023_10_08/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/D6_25_2023_10_08_S55_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D6_25_2023_10_08/20231020_HLNKVDRX3/raw_data/D6_25_2023_10_08_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/D6_25_2023_10_08_S55_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D6_25_2023_10_08/20231020_HLNKVDRX3/raw_data/D6_25_2023_10_08_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D7_33_2023_10_08/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/D7_33_2023_10_08_S69_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D7_33_2023_10_08/20231020_HLNKVDRX3/raw_data/D7_33_2023_10_08_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/D7_33_2023_10_08_S69_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D7_33_2023_10_08/20231020_HLNKVDRX3/raw_data/D7_33_2023_10_08_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D8_35_2023_10_02/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/D8_35_2023_10_02_S57_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D8_35_2023_10_02/20231020_HLNKVDRX3/raw_data/D8_35_2023_10_02_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/D8_35_2023_10_02_S57_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D8_35_2023_10_02/20231020_HLNKVDRX3/raw_data/D8_35_2023_10_02_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D9_36_2023_10_09/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/D9_36_2023_10_09_S47_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D9_36_2023_10_09/20231020_HLNKVDRX3/raw_data/D9_36_2023_10_09_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/D9_36_2023_10_09_S47_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D9_36_2023_10_09/20231020_HLNKVDRX3/raw_data/D9_36_2023_10_09_R2.fastq.gz"||X +echo -ne "\r[███·············]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E1_05_2023_10_09/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/E1_05_2023_10_09_S9_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E1_05_2023_10_09/20231020_HLNKVDRX3/raw_data/E1_05_2023_10_09_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/E1_05_2023_10_09_S9_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E1_05_2023_10_09/20231020_HLNKVDRX3/raw_data/E1_05_2023_10_09_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E2_12_2023_10_06/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/E2_12_2023_10_06_S24_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E2_12_2023_10_06/20231020_HLNKVDRX3/raw_data/E2_12_2023_10_06_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/E2_12_2023_10_06_S24_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E2_12_2023_10_06/20231020_HLNKVDRX3/raw_data/E2_12_2023_10_06_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E3_16_2023_10_04/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/E3_16_2023_10_04_S25_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E3_16_2023_10_04/20231020_HLNKVDRX3/raw_data/E3_16_2023_10_04_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/E3_16_2023_10_04_S25_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E3_16_2023_10_04/20231020_HLNKVDRX3/raw_data/E3_16_2023_10_04_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E4_17_2023_10_07/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/E4_17_2023_10_07_S14_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E4_17_2023_10_07/20231020_HLNKVDRX3/raw_data/E4_17_2023_10_07_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/E4_17_2023_10_07_S14_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E4_17_2023_10_07/20231020_HLNKVDRX3/raw_data/E4_17_2023_10_07_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E5_19_2023_10_05/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/E5_19_2023_10_05_S62_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E5_19_2023_10_05/20231020_HLNKVDRX3/raw_data/E5_19_2023_10_05_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/E5_19_2023_10_05_S62_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E5_19_2023_10_05/20231020_HLNKVDRX3/raw_data/E5_19_2023_10_05_R2.fastq.gz"||X +echo -ne "\r[███▏············]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E6_25_2023_10_09/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/E6_25_2023_10_09_S68_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E6_25_2023_10_09/20231020_HLNKVDRX3/raw_data/E6_25_2023_10_09_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/E6_25_2023_10_09_S68_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E6_25_2023_10_09/20231020_HLNKVDRX3/raw_data/E6_25_2023_10_09_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E7_33_2023_10_09/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/E7_33_2023_10_09_S64_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E7_33_2023_10_09/20231020_HLNKVDRX3/raw_data/E7_33_2023_10_09_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/E7_33_2023_10_09_S64_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E7_33_2023_10_09/20231020_HLNKVDRX3/raw_data/E7_33_2023_10_09_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E8_35_2023_10_04/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/E8_35_2023_10_04_S42_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E8_35_2023_10_04/20231020_HLNKVDRX3/raw_data/E8_35_2023_10_04_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/E8_35_2023_10_04_S42_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E8_35_2023_10_04/20231020_HLNKVDRX3/raw_data/E8_35_2023_10_04_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E9_36_2023_10_10/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/E9_36_2023_10_10_S52_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E9_36_2023_10_10/20231020_HLNKVDRX3/raw_data/E9_36_2023_10_10_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/E9_36_2023_10_10_S52_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E9_36_2023_10_10/20231020_HLNKVDRX3/raw_data/E9_36_2023_10_10_R2.fastq.gz"||X +echo -ne "\r[███▎············]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F1_10_2023_10_02/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/F1_10_2023_10_02_S48_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F1_10_2023_10_02/20231020_HLNKVDRX3/raw_data/F1_10_2023_10_02_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/F1_10_2023_10_02_S48_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F1_10_2023_10_02/20231020_HLNKVDRX3/raw_data/F1_10_2023_10_02_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F2_12_2023_10_07/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/F2_12_2023_10_07_S33_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F2_12_2023_10_07/20231020_HLNKVDRX3/raw_data/F2_12_2023_10_07_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/F2_12_2023_10_07_S33_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F2_12_2023_10_07/20231020_HLNKVDRX3/raw_data/F2_12_2023_10_07_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F3_16_2023_10_06/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/F3_16_2023_10_06_S4_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F3_16_2023_10_06/20231020_HLNKVDRX3/raw_data/F3_16_2023_10_06_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/F3_16_2023_10_06_S4_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F3_16_2023_10_06/20231020_HLNKVDRX3/raw_data/F3_16_2023_10_06_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F4_17_2023_10_08/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/F4_17_2023_10_08_S5_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F4_17_2023_10_08/20231020_HLNKVDRX3/raw_data/F4_17_2023_10_08_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/F4_17_2023_10_08_S5_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F4_17_2023_10_08/20231020_HLNKVDRX3/raw_data/F4_17_2023_10_08_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F5_19_2023_10_07/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/F5_19_2023_10_07_S46_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F5_19_2023_10_07/20231020_HLNKVDRX3/raw_data/F5_19_2023_10_07_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/F5_19_2023_10_07_S46_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F5_19_2023_10_07/20231020_HLNKVDRX3/raw_data/F5_19_2023_10_07_R2.fastq.gz"||X +echo -ne "\r[███▍············]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F6_32_2023_10_06/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/F6_32_2023_10_06_S7_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F6_32_2023_10_06/20231020_HLNKVDRX3/raw_data/F6_32_2023_10_06_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/F6_32_2023_10_06_S7_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F6_32_2023_10_06/20231020_HLNKVDRX3/raw_data/F6_32_2023_10_06_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F7_33_2023_10_10/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/F7_33_2023_10_10_S28_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F7_33_2023_10_10/20231020_HLNKVDRX3/raw_data/F7_33_2023_10_10_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/F7_33_2023_10_10_S28_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F7_33_2023_10_10/20231020_HLNKVDRX3/raw_data/F7_33_2023_10_10_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F8_35_2023_10_06/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/F8_35_2023_10_06_S50_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F8_35_2023_10_06/20231020_HLNKVDRX3/raw_data/F8_35_2023_10_06_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/F8_35_2023_10_06_S50_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F8_35_2023_10_06/20231020_HLNKVDRX3/raw_data/F8_35_2023_10_06_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G1_10_2023_10_04/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/G1_10_2023_10_04_S45_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G1_10_2023_10_04/20231020_HLNKVDRX3/raw_data/G1_10_2023_10_04_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/G1_10_2023_10_04_S45_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G1_10_2023_10_04/20231020_HLNKVDRX3/raw_data/G1_10_2023_10_04_R2.fastq.gz"||X +echo -ne "\r[███▌············]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G2_12_2023_10_08/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/G2_12_2023_10_08_S44_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G2_12_2023_10_08/20231020_HLNKVDRX3/raw_data/G2_12_2023_10_08_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/G2_12_2023_10_08_S44_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G2_12_2023_10_08/20231020_HLNKVDRX3/raw_data/G2_12_2023_10_08_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G3_16_2023_10_08/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/G3_16_2023_10_08_S40_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G3_16_2023_10_08/20231020_HLNKVDRX3/raw_data/G3_16_2023_10_08_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/G3_16_2023_10_08_S40_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G3_16_2023_10_08/20231020_HLNKVDRX3/raw_data/G3_16_2023_10_08_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G4_18_2023_10_03/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/G4_18_2023_10_03_S65_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G4_18_2023_10_03/20231020_HLNKVDRX3/raw_data/G4_18_2023_10_03_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/G4_18_2023_10_03_S65_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G4_18_2023_10_03/20231020_HLNKVDRX3/raw_data/G4_18_2023_10_03_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G5_19_2023_10_08/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/G5_19_2023_10_08_S8_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G5_19_2023_10_08/20231020_HLNKVDRX3/raw_data/G5_19_2023_10_08_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/G5_19_2023_10_08_S8_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G5_19_2023_10_08/20231020_HLNKVDRX3/raw_data/G5_19_2023_10_08_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G6_32_2023_10_08/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/G6_32_2023_10_08_S53_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G6_32_2023_10_08/20231020_HLNKVDRX3/raw_data/G6_32_2023_10_08_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/G6_32_2023_10_08_S53_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G6_32_2023_10_08/20231020_HLNKVDRX3/raw_data/G6_32_2023_10_08_R2.fastq.gz"||X +echo -ne "\r[███▋············]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G7_34_2023_10_02/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/G7_34_2023_10_02_S6_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G7_34_2023_10_02/20231020_HLNKVDRX3/raw_data/G7_34_2023_10_02_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/G7_34_2023_10_02_S6_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G7_34_2023_10_02/20231020_HLNKVDRX3/raw_data/G7_34_2023_10_02_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G8_35_2023_10_07/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/G8_35_2023_10_07_S41_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G8_35_2023_10_07/20231020_HLNKVDRX3/raw_data/G8_35_2023_10_07_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/G8_35_2023_10_07_S41_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G8_35_2023_10_07/20231020_HLNKVDRX3/raw_data/G8_35_2023_10_07_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H1_10_2023_10_06/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/H1_10_2023_10_06_S18_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H1_10_2023_10_06/20231020_HLNKVDRX3/raw_data/H1_10_2023_10_06_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/H1_10_2023_10_06_S18_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H1_10_2023_10_06/20231020_HLNKVDRX3/raw_data/H1_10_2023_10_06_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H2_15_2023_10_05/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/H2_15_2023_10_05_S32_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H2_15_2023_10_05/20231020_HLNKVDRX3/raw_data/H2_15_2023_10_05_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/H2_15_2023_10_05_S32_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H2_15_2023_10_05/20231020_HLNKVDRX3/raw_data/H2_15_2023_10_05_R2.fastq.gz"||X +echo -ne "\r[███▊············]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H3_16_2023_10_09/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/H3_16_2023_10_09_S49_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H3_16_2023_10_09/20231020_HLNKVDRX3/raw_data/H3_16_2023_10_09_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/H3_16_2023_10_09_S49_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H3_16_2023_10_09/20231020_HLNKVDRX3/raw_data/H3_16_2023_10_09_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H4_18_2023_10_05/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/H4_18_2023_10_05_S61_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H4_18_2023_10_05/20231020_HLNKVDRX3/raw_data/H4_18_2023_10_05_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/H4_18_2023_10_05_S61_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H4_18_2023_10_05/20231020_HLNKVDRX3/raw_data/H4_18_2023_10_05_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H5_19_2023_10_09/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/H5_19_2023_10_09_S66_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H5_19_2023_10_09/20231020_HLNKVDRX3/raw_data/H5_19_2023_10_09_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/H5_19_2023_10_09_S66_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H5_19_2023_10_09/20231020_HLNKVDRX3/raw_data/H5_19_2023_10_09_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H6_32_2023_10_09/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/H6_32_2023_10_09_S23_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H6_32_2023_10_09/20231020_HLNKVDRX3/raw_data/H6_32_2023_10_09_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/H6_32_2023_10_09_S23_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H6_32_2023_10_09/20231020_HLNKVDRX3/raw_data/H6_32_2023_10_09_R2.fastq.gz"||X +echo -ne "\r[███▉············]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H7_34_2023_10_04/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/H7_34_2023_10_04_S12_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H7_34_2023_10_04/20231020_HLNKVDRX3/raw_data/H7_34_2023_10_04_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/H7_34_2023_10_04_S12_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H7_34_2023_10_04/20231020_HLNKVDRX3/raw_data/H7_34_2023_10_04_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H8_35_2023_10_08/"{,"20231020_HLNKVDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/H8_35_2023_10_08_S27_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H8_35_2023_10_08/20231020_HLNKVDRX3/raw_data/H8_35_2023_10_08_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33160_NovaSeq_231020_NOV1904/H8_35_2023_10_08_S27_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H8_35_2023_10_08/20231020_HLNKVDRX3/raw_data/H8_35_2023_10_08_R2.fastq.gz"||X +if [[ ! -d "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897" ]]; then + fail "Not a directory:" "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897" +fi +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A1_05_2023_09_19/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/A1_05_2023_09_19_S5_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A1_05_2023_09_19/20231006_HLLLTDRX3/raw_data/A1_05_2023_09_19_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/A1_05_2023_09_19_S5_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A1_05_2023_09_19/20231006_HLLLTDRX3/raw_data/A1_05_2023_09_19_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A2_10_2023_09_23/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/A2_10_2023_09_23_S53_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A2_10_2023_09_23/20231006_HLLLTDRX3/raw_data/A2_10_2023_09_23_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/A2_10_2023_09_23_S53_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A2_10_2023_09_23/20231006_HLLLTDRX3/raw_data/A2_10_2023_09_23_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A3_15_2023_09_19/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/A3_15_2023_09_19_S29_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A3_15_2023_09_19/20231006_HLLLTDRX3/raw_data/A3_15_2023_09_19_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/A3_15_2023_09_19_S29_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A3_15_2023_09_19/20231006_HLLLTDRX3/raw_data/A3_15_2023_09_19_R2.fastq.gz"||X +echo -ne "\r[████············]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A4_16_2023_09_26/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/A4_16_2023_09_26_S38_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A4_16_2023_09_26/20231006_HLLLTDRX3/raw_data/A4_16_2023_09_26_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/A4_16_2023_09_26_S38_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A4_16_2023_09_26/20231006_HLLLTDRX3/raw_data/A4_16_2023_09_26_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A5_18_2023_09_23/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/A5_18_2023_09_23_S54_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A5_18_2023_09_23/20231006_HLLLTDRX3/raw_data/A5_18_2023_09_23_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/A5_18_2023_09_23_S54_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A5_18_2023_09_23/20231006_HLLLTDRX3/raw_data/A5_18_2023_09_23_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A6_25_2023_09_19/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/A6_25_2023_09_19_S7_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A6_25_2023_09_19/20231006_HLLLTDRX3/raw_data/A6_25_2023_09_19_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/A6_25_2023_09_19_S7_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A6_25_2023_09_19/20231006_HLLLTDRX3/raw_data/A6_25_2023_09_19_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A7_33_2023_09_25/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/A7_33_2023_09_25_S17_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A7_33_2023_09_25/20231006_HLLLTDRX3/raw_data/A7_33_2023_09_25_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/A7_33_2023_09_25_S17_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A7_33_2023_09_25/20231006_HLLLTDRX3/raw_data/A7_33_2023_09_25_R2.fastq.gz"||X +echo -ne "\r[████▏···········]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A8_35_2023_09_20/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/A8_35_2023_09_20_S20_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A8_35_2023_09_20/20231006_HLLLTDRX3/raw_data/A8_35_2023_09_20_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/A8_35_2023_09_20_S20_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A8_35_2023_09_20/20231006_HLLLTDRX3/raw_data/A8_35_2023_09_20_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A9_36_2023_09_26/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/A9_36_2023_09_26_S11_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A9_36_2023_09_26/20231006_HLLLTDRX3/raw_data/A9_36_2023_09_26_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/A9_36_2023_09_26_S11_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A9_36_2023_09_26/20231006_HLLLTDRX3/raw_data/A9_36_2023_09_26_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B1_05_2023_09_21/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/B1_05_2023_09_21_S21_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B1_05_2023_09_21/20231006_HLLLTDRX3/raw_data/B1_05_2023_09_21_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/B1_05_2023_09_21_S21_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B1_05_2023_09_21/20231006_HLLLTDRX3/raw_data/B1_05_2023_09_21_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B2_10_2023_09_24/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/B2_10_2023_09_24_S43_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B2_10_2023_09_24/20231006_HLLLTDRX3/raw_data/B2_10_2023_09_24_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/B2_10_2023_09_24_S43_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B2_10_2023_09_24/20231006_HLLLTDRX3/raw_data/B2_10_2023_09_24_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B3_15_2023_09_23/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/B3_15_2023_09_23_S32_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B3_15_2023_09_23/20231006_HLLLTDRX3/raw_data/B3_15_2023_09_23_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/B3_15_2023_09_23_S32_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B3_15_2023_09_23/20231006_HLLLTDRX3/raw_data/B3_15_2023_09_23_R2.fastq.gz"||X +echo -ne "\r[████▎···········]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B4_17_2023_09_18/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/B4_17_2023_09_18_S59_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B4_17_2023_09_18/20231006_HLLLTDRX3/raw_data/B4_17_2023_09_18_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/B4_17_2023_09_18_S59_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B4_17_2023_09_18/20231006_HLLLTDRX3/raw_data/B4_17_2023_09_18_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B5_18_2023_09_24/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/B5_18_2023_09_24_S19_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B5_18_2023_09_24/20231006_HLLLTDRX3/raw_data/B5_18_2023_09_24_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/B5_18_2023_09_24_S19_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B5_18_2023_09_24/20231006_HLLLTDRX3/raw_data/B5_18_2023_09_24_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B6_25_2023_09_21/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/B6_25_2023_09_21_S39_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B6_25_2023_09_21/20231006_HLLLTDRX3/raw_data/B6_25_2023_09_21_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/B6_25_2023_09_21_S39_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B6_25_2023_09_21/20231006_HLLLTDRX3/raw_data/B6_25_2023_09_21_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B7_33_2023_09_26/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/B7_33_2023_09_26_S62_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B7_33_2023_09_26/20231006_HLLLTDRX3/raw_data/B7_33_2023_09_26_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/B7_33_2023_09_26_S62_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B7_33_2023_09_26/20231006_HLLLTDRX3/raw_data/B7_33_2023_09_26_R2.fastq.gz"||X +echo -ne "\r[████▍···········]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B8_35_2023_09_22/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/B8_35_2023_09_22_S35_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B8_35_2023_09_22/20231006_HLLLTDRX3/raw_data/B8_35_2023_09_22_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/B8_35_2023_09_22_S35_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B8_35_2023_09_22/20231006_HLLLTDRX3/raw_data/B8_35_2023_09_22_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C1_05_2023_09_23/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/C1_05_2023_09_23_S47_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C1_05_2023_09_23/20231006_HLLLTDRX3/raw_data/C1_05_2023_09_23_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/C1_05_2023_09_23_S47_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C1_05_2023_09_23/20231006_HLLLTDRX3/raw_data/C1_05_2023_09_23_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C2_12_2023_09_18/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/C2_12_2023_09_18_S13_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C2_12_2023_09_18/20231006_HLLLTDRX3/raw_data/C2_12_2023_09_18_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/C2_12_2023_09_18_S13_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C2_12_2023_09_18/20231006_HLLLTDRX3/raw_data/C2_12_2023_09_18_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C3_15_2023_09_24/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/C3_15_2023_09_24_S14_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C3_15_2023_09_24/20231006_HLLLTDRX3/raw_data/C3_15_2023_09_24_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/C3_15_2023_09_24_S14_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C3_15_2023_09_24/20231006_HLLLTDRX3/raw_data/C3_15_2023_09_24_R2.fastq.gz"||X +echo -ne "\r[████▌···········]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C4_17_2023_09_20/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/C4_17_2023_09_20_S15_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C4_17_2023_09_20/20231006_HLLLTDRX3/raw_data/C4_17_2023_09_20_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/C4_17_2023_09_20_S15_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C4_17_2023_09_20/20231006_HLLLTDRX3/raw_data/C4_17_2023_09_20_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C5_18_2023_09_25/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/C5_18_2023_09_25_S3_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C5_18_2023_09_25/20231006_HLLLTDRX3/raw_data/C5_18_2023_09_25_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/C5_18_2023_09_25_S3_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C5_18_2023_09_25/20231006_HLLLTDRX3/raw_data/C5_18_2023_09_25_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C6_25_2023_09_23/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/C6_25_2023_09_23_S63_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C6_25_2023_09_23/20231006_HLLLTDRX3/raw_data/C6_25_2023_09_23_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/C6_25_2023_09_23_S63_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C6_25_2023_09_23/20231006_HLLLTDRX3/raw_data/C6_25_2023_09_23_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C7_34_2023_09_18/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/C7_34_2023_09_18_S27_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C7_34_2023_09_18/20231006_HLLLTDRX3/raw_data/C7_34_2023_09_18_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/C7_34_2023_09_18_S27_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C7_34_2023_09_18/20231006_HLLLTDRX3/raw_data/C7_34_2023_09_18_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C8_35_2023_09_23/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/C8_35_2023_09_23_S41_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C8_35_2023_09_23/20231006_HLLLTDRX3/raw_data/C8_35_2023_09_23_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/C8_35_2023_09_23_S41_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C8_35_2023_09_23/20231006_HLLLTDRX3/raw_data/C8_35_2023_09_23_R2.fastq.gz"||X +echo -ne "\r[████▋···········]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D1_05_2023_09_24/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/D1_05_2023_09_24_S61_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D1_05_2023_09_24/20231006_HLLLTDRX3/raw_data/D1_05_2023_09_24_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/D1_05_2023_09_24_S61_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D1_05_2023_09_24/20231006_HLLLTDRX3/raw_data/D1_05_2023_09_24_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D2_12_2023_09_20/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/D2_12_2023_09_20_S28_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D2_12_2023_09_20/20231006_HLLLTDRX3/raw_data/D2_12_2023_09_20_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/D2_12_2023_09_20_S28_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D2_12_2023_09_20/20231006_HLLLTDRX3/raw_data/D2_12_2023_09_20_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D3_15_2023_09_25/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/D3_15_2023_09_25_S49_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D3_15_2023_09_25/20231006_HLLLTDRX3/raw_data/D3_15_2023_09_25_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/D3_15_2023_09_25_S49_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D3_15_2023_09_25/20231006_HLLLTDRX3/raw_data/D3_15_2023_09_25_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D4_17_2023_09_22/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/D4_17_2023_09_22_S58_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D4_17_2023_09_22/20231006_HLLLTDRX3/raw_data/D4_17_2023_09_22_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/D4_17_2023_09_22_S58_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D4_17_2023_09_22/20231006_HLLLTDRX3/raw_data/D4_17_2023_09_22_R2.fastq.gz"||X +echo -ne "\r[████▊···········]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D5_19_2023_09_19/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/D5_19_2023_09_19_S60_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D5_19_2023_09_19/20231006_HLLLTDRX3/raw_data/D5_19_2023_09_19_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/D5_19_2023_09_19_S60_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D5_19_2023_09_19/20231006_HLLLTDRX3/raw_data/D5_19_2023_09_19_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D6_25_2023_09_24/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/D6_25_2023_09_24_S44_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D6_25_2023_09_24/20231006_HLLLTDRX3/raw_data/D6_25_2023_09_24_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/D6_25_2023_09_24_S44_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D6_25_2023_09_24/20231006_HLLLTDRX3/raw_data/D6_25_2023_09_24_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D7_34_2023_09_20/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/D7_34_2023_09_20_S33_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D7_34_2023_09_20/20231006_HLLLTDRX3/raw_data/D7_34_2023_09_20_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/D7_34_2023_09_20_S33_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D7_34_2023_09_20/20231006_HLLLTDRX3/raw_data/D7_34_2023_09_20_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D8_35_2023_09_24/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/D8_35_2023_09_24_S46_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D8_35_2023_09_24/20231006_HLLLTDRX3/raw_data/D8_35_2023_09_24_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/D8_35_2023_09_24_S46_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D8_35_2023_09_24/20231006_HLLLTDRX3/raw_data/D8_35_2023_09_24_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E1_05_2023_09_25/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/E1_05_2023_09_25_S23_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E1_05_2023_09_25/20231006_HLLLTDRX3/raw_data/E1_05_2023_09_25_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/E1_05_2023_09_25_S23_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E1_05_2023_09_25/20231006_HLLLTDRX3/raw_data/E1_05_2023_09_25_R2.fastq.gz"||X +echo -ne "\r[████▉···········]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E2_12_2023_09_22/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/E2_12_2023_09_22_S10_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E2_12_2023_09_22/20231006_HLLLTDRX3/raw_data/E2_12_2023_09_22_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/E2_12_2023_09_22_S10_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E2_12_2023_09_22/20231006_HLLLTDRX3/raw_data/E2_12_2023_09_22_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E3_16_2023_09_20/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/E3_16_2023_09_20_S40_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E3_16_2023_09_20/20231006_HLLLTDRX3/raw_data/E3_16_2023_09_20_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/E3_16_2023_09_20_S40_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E3_16_2023_09_20/20231006_HLLLTDRX3/raw_data/E3_16_2023_09_20_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E4_17_2023_09_23/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/E4_17_2023_09_23_S12_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E4_17_2023_09_23/20231006_HLLLTDRX3/raw_data/E4_17_2023_09_23_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/E4_17_2023_09_23_S12_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E4_17_2023_09_23/20231006_HLLLTDRX3/raw_data/E4_17_2023_09_23_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E5_19_2023_09_21/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/E5_19_2023_09_21_S64_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E5_19_2023_09_21/20231006_HLLLTDRX3/raw_data/E5_19_2023_09_21_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/E5_19_2023_09_21_S64_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E5_19_2023_09_21/20231006_HLLLTDRX3/raw_data/E5_19_2023_09_21_R2.fastq.gz"||X +echo -ne "\r[█████···········]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E6_25_2023_09_25/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/E6_25_2023_09_25_S37_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E6_25_2023_09_25/20231006_HLLLTDRX3/raw_data/E6_25_2023_09_25_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/E6_25_2023_09_25_S37_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E6_25_2023_09_25/20231006_HLLLTDRX3/raw_data/E6_25_2023_09_25_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E7_34_2023_09_22/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/E7_34_2023_09_22_S50_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E7_34_2023_09_22/20231006_HLLLTDRX3/raw_data/E7_34_2023_09_22_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/E7_34_2023_09_22_S50_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E7_34_2023_09_22/20231006_HLLLTDRX3/raw_data/E7_34_2023_09_22_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E8_36_2023_09_20/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/E8_36_2023_09_20_S45_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E8_36_2023_09_20/20231006_HLLLTDRX3/raw_data/E8_36_2023_09_20_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/E8_36_2023_09_20_S45_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E8_36_2023_09_20/20231006_HLLLTDRX3/raw_data/E8_36_2023_09_20_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F1_10_2023_09_18/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/F1_10_2023_09_18_S42_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F1_10_2023_09_18/20231006_HLLLTDRX3/raw_data/F1_10_2023_09_18_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/F1_10_2023_09_18_S42_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F1_10_2023_09_18/20231006_HLLLTDRX3/raw_data/F1_10_2023_09_18_R2.fastq.gz"||X +echo -ne "\r[█████▏··········]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F2_12_2023_09_23/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/F2_12_2023_09_23_S25_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F2_12_2023_09_23/20231006_HLLLTDRX3/raw_data/F2_12_2023_09_23_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/F2_12_2023_09_23_S25_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F2_12_2023_09_23/20231006_HLLLTDRX3/raw_data/F2_12_2023_09_23_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F3_16_2023_09_22/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/F3_16_2023_09_22_S4_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F3_16_2023_09_22/20231006_HLLLTDRX3/raw_data/F3_16_2023_09_22_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/F3_16_2023_09_22_S4_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F3_16_2023_09_22/20231006_HLLLTDRX3/raw_data/F3_16_2023_09_22_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F4_17_2023_09_24/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/F4_17_2023_09_24_S52_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F4_17_2023_09_24/20231006_HLLLTDRX3/raw_data/F4_17_2023_09_24_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/F4_17_2023_09_24_S52_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F4_17_2023_09_24/20231006_HLLLTDRX3/raw_data/F4_17_2023_09_24_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F5_19_2023_09_23/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/F5_19_2023_09_23_S2_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F5_19_2023_09_23/20231006_HLLLTDRX3/raw_data/F5_19_2023_09_23_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/F5_19_2023_09_23_S2_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F5_19_2023_09_23/20231006_HLLLTDRX3/raw_data/F5_19_2023_09_23_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F6_33_2023_09_20/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/F6_33_2023_09_20_S9_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F6_33_2023_09_20/20231006_HLLLTDRX3/raw_data/F6_33_2023_09_20_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/F6_33_2023_09_20_S9_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F6_33_2023_09_20/20231006_HLLLTDRX3/raw_data/F6_33_2023_09_20_R2.fastq.gz"||X +echo -ne "\r[█████▎··········]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F7_34_2023_09_23/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/F7_34_2023_09_23_S36_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F7_34_2023_09_23/20231006_HLLLTDRX3/raw_data/F7_34_2023_09_23_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/F7_34_2023_09_23_S36_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F7_34_2023_09_23/20231006_HLLLTDRX3/raw_data/F7_34_2023_09_23_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F8_36_2023_09_22/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/F8_36_2023_09_22_S34_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F8_36_2023_09_22/20231006_HLLLTDRX3/raw_data/F8_36_2023_09_22_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/F8_36_2023_09_22_S34_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F8_36_2023_09_22/20231006_HLLLTDRX3/raw_data/F8_36_2023_09_22_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G1_10_2023_09_20/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/G1_10_2023_09_20_S56_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G1_10_2023_09_20/20231006_HLLLTDRX3/raw_data/G1_10_2023_09_20_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/G1_10_2023_09_20_S56_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G1_10_2023_09_20/20231006_HLLLTDRX3/raw_data/G1_10_2023_09_20_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G2_12_2023_09_24/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/G2_12_2023_09_24_S22_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G2_12_2023_09_24/20231006_HLLLTDRX3/raw_data/G2_12_2023_09_24_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/G2_12_2023_09_24_S22_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G2_12_2023_09_24/20231006_HLLLTDRX3/raw_data/G2_12_2023_09_24_R2.fastq.gz"||X +echo -ne "\r[█████▍··········]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G3_16_2023_09_24/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/G3_16_2023_09_24_S65_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G3_16_2023_09_24/20231006_HLLLTDRX3/raw_data/G3_16_2023_09_24_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/G3_16_2023_09_24_S65_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G3_16_2023_09_24/20231006_HLLLTDRX3/raw_data/G3_16_2023_09_24_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G4_18_2023_09_19/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/G4_18_2023_09_19_S48_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G4_18_2023_09_19/20231006_HLLLTDRX3/raw_data/G4_18_2023_09_19_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/G4_18_2023_09_19_S48_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G4_18_2023_09_19/20231006_HLLLTDRX3/raw_data/G4_18_2023_09_19_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G5_19_2023_09_24/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/G5_19_2023_09_24_S24_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G5_19_2023_09_24/20231006_HLLLTDRX3/raw_data/G5_19_2023_09_24_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/G5_19_2023_09_24_S24_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G5_19_2023_09_24/20231006_HLLLTDRX3/raw_data/G5_19_2023_09_24_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G6_33_2023_09_22/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/G6_33_2023_09_22_S16_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G6_33_2023_09_22/20231006_HLLLTDRX3/raw_data/G6_33_2023_09_22_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/G6_33_2023_09_22_S16_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G6_33_2023_09_22/20231006_HLLLTDRX3/raw_data/G6_33_2023_09_22_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G7_34_2023_09_24/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/G7_34_2023_09_24_S6_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G7_34_2023_09_24/20231006_HLLLTDRX3/raw_data/G7_34_2023_09_24_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/G7_34_2023_09_24_S6_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G7_34_2023_09_24/20231006_HLLLTDRX3/raw_data/G7_34_2023_09_24_R2.fastq.gz"||X +echo -ne "\r[█████▌··········]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G8_36_2023_09_24/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/G8_36_2023_09_24_S31_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G8_36_2023_09_24/20231006_HLLLTDRX3/raw_data/G8_36_2023_09_24_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/G8_36_2023_09_24_S31_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G8_36_2023_09_24/20231006_HLLLTDRX3/raw_data/G8_36_2023_09_24_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H1_10_2023_09_22/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/H1_10_2023_09_22_S1_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H1_10_2023_09_22/20231006_HLLLTDRX3/raw_data/H1_10_2023_09_22_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/H1_10_2023_09_22_S1_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H1_10_2023_09_22/20231006_HLLLTDRX3/raw_data/H1_10_2023_09_22_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H2_15_2023_09_21/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/H2_15_2023_09_21_S57_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H2_15_2023_09_21/20231006_HLLLTDRX3/raw_data/H2_15_2023_09_21_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/H2_15_2023_09_21_S57_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H2_15_2023_09_21/20231006_HLLLTDRX3/raw_data/H2_15_2023_09_21_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H3_16_2023_09_25/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/H3_16_2023_09_25_S51_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H3_16_2023_09_25/20231006_HLLLTDRX3/raw_data/H3_16_2023_09_25_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/H3_16_2023_09_25_S51_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H3_16_2023_09_25/20231006_HLLLTDRX3/raw_data/H3_16_2023_09_25_R2.fastq.gz"||X +echo -ne "\r[█████▋··········]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H4_18_2023_09_21/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/H4_18_2023_09_21_S18_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H4_18_2023_09_21/20231006_HLLLTDRX3/raw_data/H4_18_2023_09_21_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/H4_18_2023_09_21_S18_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H4_18_2023_09_21/20231006_HLLLTDRX3/raw_data/H4_18_2023_09_21_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H5_19_2023_09_25/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/H5_19_2023_09_25_S26_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H5_19_2023_09_25/20231006_HLLLTDRX3/raw_data/H5_19_2023_09_25_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/H5_19_2023_09_25_S26_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H5_19_2023_09_25/20231006_HLLLTDRX3/raw_data/H5_19_2023_09_25_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H6_33_2023_09_24/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/H6_33_2023_09_24_S55_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H6_33_2023_09_24/20231006_HLLLTDRX3/raw_data/H6_33_2023_09_24_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/H6_33_2023_09_24_S55_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H6_33_2023_09_24/20231006_HLLLTDRX3/raw_data/H6_33_2023_09_24_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H7_35_2023_09_18/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/H7_35_2023_09_18_S30_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H7_35_2023_09_18/20231006_HLLLTDRX3/raw_data/H7_35_2023_09_18_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/H7_35_2023_09_18_S30_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H7_35_2023_09_18/20231006_HLLLTDRX3/raw_data/H7_35_2023_09_18_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H8_36_2023_09_25/"{,"20231006_HLLLTDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/H8_36_2023_09_25_S8_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H8_36_2023_09_25/20231006_HLLLTDRX3/raw_data/H8_36_2023_09_25_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33039_NovaSeq_231006_NOV1897/H8_36_2023_09_25_S8_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H8_36_2023_09_25/20231006_HLLLTDRX3/raw_data/H8_36_2023_09_25_R2.fastq.gz"||X +if [[ ! -d "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917" ]]; then + fail "Not a directory:" "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917" +fi +echo -ne "\r[█████▊··········]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A1_05_2023_10_11/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/A1_05_2023_10_11_S63_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A1_05_2023_10_11/20231103_HLTHYDRX3/raw_data/A1_05_2023_10_11_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/A1_05_2023_10_11_S63_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A1_05_2023_10_11/20231103_HLTHYDRX3/raw_data/A1_05_2023_10_11_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A2_10_2023_10_14/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/A2_10_2023_10_14_S66_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A2_10_2023_10_14/20231103_HLTHYDRX3/raw_data/A2_10_2023_10_14_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/A2_10_2023_10_14_S66_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A2_10_2023_10_14/20231103_HLTHYDRX3/raw_data/A2_10_2023_10_14_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A3_15_2023_10_13/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/A3_15_2023_10_13_S40_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A3_15_2023_10_13/20231103_HLTHYDRX3/raw_data/A3_15_2023_10_13_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/A3_15_2023_10_13_S40_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A3_15_2023_10_13/20231103_HLTHYDRX3/raw_data/A3_15_2023_10_13_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A4_16_2023_10_17/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/A4_16_2023_10_17_S41_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A4_16_2023_10_17/20231103_HLTHYDRX3/raw_data/A4_16_2023_10_17_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/A4_16_2023_10_17_S41_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A4_16_2023_10_17/20231103_HLTHYDRX3/raw_data/A4_16_2023_10_17_R2.fastq.gz"||X +echo -ne "\r[█████▉··········]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A5_18_2023_10_14/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/A5_18_2023_10_14_S28_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A5_18_2023_10_14/20231103_HLTHYDRX3/raw_data/A5_18_2023_10_14_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/A5_18_2023_10_14_S28_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A5_18_2023_10_14/20231103_HLTHYDRX3/raw_data/A5_18_2023_10_14_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A6_25_2023_10_11/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/A6_25_2023_10_11_S52_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A6_25_2023_10_11/20231103_HLTHYDRX3/raw_data/A6_25_2023_10_11_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/A6_25_2023_10_11_S52_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A6_25_2023_10_11/20231103_HLTHYDRX3/raw_data/A6_25_2023_10_11_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A7_32_2023_10_16/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/A7_32_2023_10_16_S11_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A7_32_2023_10_16/20231103_HLTHYDRX3/raw_data/A7_32_2023_10_16_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/A7_32_2023_10_16_S11_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A7_32_2023_10_16/20231103_HLTHYDRX3/raw_data/A7_32_2023_10_16_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A8_34_2023_10_12/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/A8_34_2023_10_12_S42_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A8_34_2023_10_12/20231103_HLTHYDRX3/raw_data/A8_34_2023_10_12_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/A8_34_2023_10_12_S42_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A8_34_2023_10_12/20231103_HLTHYDRX3/raw_data/A8_34_2023_10_12_R2.fastq.gz"||X +echo -ne "\r[██████··········]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A9_35_2023_10_15/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/A9_35_2023_10_15_S43_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A9_35_2023_10_15/20231103_HLTHYDRX3/raw_data/A9_35_2023_10_15_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/A9_35_2023_10_15_S43_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A9_35_2023_10_15/20231103_HLTHYDRX3/raw_data/A9_35_2023_10_15_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B1_05_2023_10_13/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/B1_05_2023_10_13_S39_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B1_05_2023_10_13/20231103_HLTHYDRX3/raw_data/B1_05_2023_10_13_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/B1_05_2023_10_13_S39_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B1_05_2023_10_13/20231103_HLTHYDRX3/raw_data/B1_05_2023_10_13_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B2_10_2023_10_15/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/B2_10_2023_10_15_S24_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B2_10_2023_10_15/20231103_HLTHYDRX3/raw_data/B2_10_2023_10_15_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/B2_10_2023_10_15_S24_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B2_10_2023_10_15/20231103_HLTHYDRX3/raw_data/B2_10_2023_10_15_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B3_15_2023_10_14/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/B3_15_2023_10_14_S29_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B3_15_2023_10_14/20231103_HLTHYDRX3/raw_data/B3_15_2023_10_14_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/B3_15_2023_10_14_S29_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B3_15_2023_10_14/20231103_HLTHYDRX3/raw_data/B3_15_2023_10_14_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B4_17_2023_10_10/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/B4_17_2023_10_10_S51_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B4_17_2023_10_10/20231103_HLTHYDRX3/raw_data/B4_17_2023_10_10_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/B4_17_2023_10_10_S51_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B4_17_2023_10_10/20231103_HLTHYDRX3/raw_data/B4_17_2023_10_10_R2.fastq.gz"||X +echo -ne "\r[██████▏·········]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B5_18_2023_10_15/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/B5_18_2023_10_15_S1_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B5_18_2023_10_15/20231103_HLTHYDRX3/raw_data/B5_18_2023_10_15_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/B5_18_2023_10_15_S1_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B5_18_2023_10_15/20231103_HLTHYDRX3/raw_data/B5_18_2023_10_15_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B6_25_2023_10_13/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/B6_25_2023_10_13_S59_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B6_25_2023_10_13/20231103_HLTHYDRX3/raw_data/B6_25_2023_10_13_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/B6_25_2023_10_13_S59_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B6_25_2023_10_13/20231103_HLTHYDRX3/raw_data/B6_25_2023_10_13_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B7_32_2023_10_17/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/B7_32_2023_10_17_S23_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B7_32_2023_10_17/20231103_HLTHYDRX3/raw_data/B7_32_2023_10_17_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/B7_32_2023_10_17_S23_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B7_32_2023_10_17/20231103_HLTHYDRX3/raw_data/B7_32_2023_10_17_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B8_34_2023_10_13/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/B8_34_2023_10_13_S67_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B8_34_2023_10_13/20231103_HLTHYDRX3/raw_data/B8_34_2023_10_13_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/B8_34_2023_10_13_S67_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B8_34_2023_10_13/20231103_HLTHYDRX3/raw_data/B8_34_2023_10_13_R2.fastq.gz"||X +echo -ne "\r[██████▎·········]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B9_36_2023_10_12/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/B9_36_2023_10_12_S45_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B9_36_2023_10_12/20231103_HLTHYDRX3/raw_data/B9_36_2023_10_12_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/B9_36_2023_10_12_S45_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B9_36_2023_10_12/20231103_HLTHYDRX3/raw_data/B9_36_2023_10_12_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C1_05_2023_10_14/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/C1_05_2023_10_14_S57_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C1_05_2023_10_14/20231103_HLTHYDRX3/raw_data/C1_05_2023_10_14_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/C1_05_2023_10_14_S57_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C1_05_2023_10_14/20231103_HLTHYDRX3/raw_data/C1_05_2023_10_14_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C2_12_2023_10_10/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/C2_12_2023_10_10_S4_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C2_12_2023_10_10/20231103_HLTHYDRX3/raw_data/C2_12_2023_10_10_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/C2_12_2023_10_10_S4_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C2_12_2023_10_10/20231103_HLTHYDRX3/raw_data/C2_12_2023_10_10_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C3_15_2023_10_15/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/C3_15_2023_10_15_S25_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C3_15_2023_10_15/20231103_HLTHYDRX3/raw_data/C3_15_2023_10_15_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/C3_15_2023_10_15_S25_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C3_15_2023_10_15/20231103_HLTHYDRX3/raw_data/C3_15_2023_10_15_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C4_17_2023_10_12/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/C4_17_2023_10_12_S53_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C4_17_2023_10_12/20231103_HLTHYDRX3/raw_data/C4_17_2023_10_12_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/C4_17_2023_10_12_S53_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C4_17_2023_10_12/20231103_HLTHYDRX3/raw_data/C4_17_2023_10_12_R2.fastq.gz"||X +echo -ne "\r[██████▍·········]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C5_18_2023_10_16/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/C5_18_2023_10_16_S60_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C5_18_2023_10_16/20231103_HLTHYDRX3/raw_data/C5_18_2023_10_16_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/C5_18_2023_10_16_S60_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C5_18_2023_10_16/20231103_HLTHYDRX3/raw_data/C5_18_2023_10_16_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C6_25_2023_10_14/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/C6_25_2023_10_14_S6_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C6_25_2023_10_14/20231103_HLTHYDRX3/raw_data/C6_25_2023_10_14_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/C6_25_2023_10_14_S6_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C6_25_2023_10_14/20231103_HLTHYDRX3/raw_data/C6_25_2023_10_14_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C7_33_2023_10_12/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/C7_33_2023_10_12_S31_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C7_33_2023_10_12/20231103_HLTHYDRX3/raw_data/C7_33_2023_10_12_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/C7_33_2023_10_12_S31_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C7_33_2023_10_12/20231103_HLTHYDRX3/raw_data/C7_33_2023_10_12_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C8_34_2023_10_14/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/C8_34_2023_10_14_S36_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C8_34_2023_10_14/20231103_HLTHYDRX3/raw_data/C8_34_2023_10_14_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/C8_34_2023_10_14_S36_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C8_34_2023_10_14/20231103_HLTHYDRX3/raw_data/C8_34_2023_10_14_R2.fastq.gz"||X +echo -ne "\r[██████▌·········]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C9_36_2023_10_14/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/C9_36_2023_10_14_S9_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C9_36_2023_10_14/20231103_HLTHYDRX3/raw_data/C9_36_2023_10_14_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/C9_36_2023_10_14_S9_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C9_36_2023_10_14/20231103_HLTHYDRX3/raw_data/C9_36_2023_10_14_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D1_05_2023_10_15/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/D1_05_2023_10_15_S46_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D1_05_2023_10_15/20231103_HLTHYDRX3/raw_data/D1_05_2023_10_15_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/D1_05_2023_10_15_S46_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D1_05_2023_10_15/20231103_HLTHYDRX3/raw_data/D1_05_2023_10_15_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D2_12_2023_10_12/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/D2_12_2023_10_12_S10_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D2_12_2023_10_12/20231103_HLTHYDRX3/raw_data/D2_12_2023_10_12_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/D2_12_2023_10_12_S10_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D2_12_2023_10_12/20231103_HLTHYDRX3/raw_data/D2_12_2023_10_12_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D3_15_2023_10_16/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/D3_15_2023_10_16_S22_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D3_15_2023_10_16/20231103_HLTHYDRX3/raw_data/D3_15_2023_10_16_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/D3_15_2023_10_16_S22_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D3_15_2023_10_16/20231103_HLTHYDRX3/raw_data/D3_15_2023_10_16_R2.fastq.gz"||X +echo -ne "\r[██████▋·········]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D4_17_2023_10_13/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/D4_17_2023_10_13_S47_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D4_17_2023_10_13/20231103_HLTHYDRX3/raw_data/D4_17_2023_10_13_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/D4_17_2023_10_13_S47_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D4_17_2023_10_13/20231103_HLTHYDRX3/raw_data/D4_17_2023_10_13_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D5_19_2023_10_11/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/D5_19_2023_10_11_S54_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D5_19_2023_10_11/20231103_HLTHYDRX3/raw_data/D5_19_2023_10_11_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/D5_19_2023_10_11_S54_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D5_19_2023_10_11/20231103_HLTHYDRX3/raw_data/D5_19_2023_10_11_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D6_25_2023_10_15/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/D6_25_2023_10_15_S69_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D6_25_2023_10_15/20231103_HLTHYDRX3/raw_data/D6_25_2023_10_15_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/D6_25_2023_10_15_S69_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D6_25_2023_10_15/20231103_HLTHYDRX3/raw_data/D6_25_2023_10_15_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D7_33_2023_10_14/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/D7_33_2023_10_14_S58_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D7_33_2023_10_14/20231103_HLTHYDRX3/raw_data/D7_33_2023_10_14_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/D7_33_2023_10_14_S58_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D7_33_2023_10_14/20231103_HLTHYDRX3/raw_data/D7_33_2023_10_14_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D8_34_2023_10_15/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/D8_34_2023_10_15_S68_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D8_34_2023_10_15/20231103_HLTHYDRX3/raw_data/D8_34_2023_10_15_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/D8_34_2023_10_15_S68_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D8_34_2023_10_15/20231103_HLTHYDRX3/raw_data/D8_34_2023_10_15_R2.fastq.gz"||X +echo -ne "\r[██████▊·········]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D9_36_2023_10_16/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/D9_36_2023_10_16_S19_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D9_36_2023_10_16/20231103_HLTHYDRX3/raw_data/D9_36_2023_10_16_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/D9_36_2023_10_16_S19_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D9_36_2023_10_16/20231103_HLTHYDRX3/raw_data/D9_36_2023_10_16_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E1_05_2023_10_16/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/E1_05_2023_10_16_S37_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E1_05_2023_10_16/20231103_HLTHYDRX3/raw_data/E1_05_2023_10_16_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/E1_05_2023_10_16_S37_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E1_05_2023_10_16/20231103_HLTHYDRX3/raw_data/E1_05_2023_10_16_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E2_12_2023_10_13/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/E2_12_2023_10_13_S18_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E2_12_2023_10_13/20231103_HLTHYDRX3/raw_data/E2_12_2023_10_13_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/E2_12_2023_10_13_S18_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E2_12_2023_10_13/20231103_HLTHYDRX3/raw_data/E2_12_2023_10_13_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E3_16_2023_10_12/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/E3_16_2023_10_12_S16_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E3_16_2023_10_12/20231103_HLTHYDRX3/raw_data/E3_16_2023_10_12_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/E3_16_2023_10_12_S16_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E3_16_2023_10_12/20231103_HLTHYDRX3/raw_data/E3_16_2023_10_12_R2.fastq.gz"||X +echo -ne "\r[██████▉·········]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E4_17_2023_10_14/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/E4_17_2023_10_14_S64_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E4_17_2023_10_14/20231103_HLTHYDRX3/raw_data/E4_17_2023_10_14_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/E4_17_2023_10_14_S64_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E4_17_2023_10_14/20231103_HLTHYDRX3/raw_data/E4_17_2023_10_14_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E5_19_2023_10_13/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/E5_19_2023_10_13_S34_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E5_19_2023_10_13/20231103_HLTHYDRX3/raw_data/E5_19_2023_10_13_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/E5_19_2023_10_13_S34_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E5_19_2023_10_13/20231103_HLTHYDRX3/raw_data/E5_19_2023_10_13_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E6_25_2023_10_16/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/E6_25_2023_10_16_S61_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E6_25_2023_10_16/20231103_HLTHYDRX3/raw_data/E6_25_2023_10_16_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/E6_25_2023_10_16_S61_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E6_25_2023_10_16/20231103_HLTHYDRX3/raw_data/E6_25_2023_10_16_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E7_33_2023_10_15/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/E7_33_2023_10_15_S17_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E7_33_2023_10_15/20231103_HLTHYDRX3/raw_data/E7_33_2023_10_15_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/E7_33_2023_10_15_S17_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E7_33_2023_10_15/20231103_HLTHYDRX3/raw_data/E7_33_2023_10_15_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E8_35_2023_10_10/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/E8_35_2023_10_10_S3_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E8_35_2023_10_10/20231103_HLTHYDRX3/raw_data/E8_35_2023_10_10_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/E8_35_2023_10_10_S3_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E8_35_2023_10_10/20231103_HLTHYDRX3/raw_data/E8_35_2023_10_10_R2.fastq.gz"||X +echo -ne "\r[███████·········]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E9_36_2023_10_15/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/E9_36_2023_10_15_S38_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E9_36_2023_10_15/20231103_HLTHYDRX3/raw_data/E9_36_2023_10_15_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/E9_36_2023_10_15_S38_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E9_36_2023_10_15/20231103_HLTHYDRX3/raw_data/E9_36_2023_10_15_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F1_10_2023_10_10/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/F1_10_2023_10_10_S14_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F1_10_2023_10_10/20231103_HLTHYDRX3/raw_data/F1_10_2023_10_10_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/F1_10_2023_10_10_S14_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F1_10_2023_10_10/20231103_HLTHYDRX3/raw_data/F1_10_2023_10_10_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F2_12_2023_10_14/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/F2_12_2023_10_14_S2_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F2_12_2023_10_14/20231103_HLTHYDRX3/raw_data/F2_12_2023_10_14_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/F2_12_2023_10_14_S2_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F2_12_2023_10_14/20231103_HLTHYDRX3/raw_data/F2_12_2023_10_14_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F3_16_2023_10_14/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/F3_16_2023_10_14_S48_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F3_16_2023_10_14/20231103_HLTHYDRX3/raw_data/F3_16_2023_10_14_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/F3_16_2023_10_14_S48_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F3_16_2023_10_14/20231103_HLTHYDRX3/raw_data/F3_16_2023_10_14_R2.fastq.gz"||X +echo -ne "\r[███████▏········]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F4_17_2023_10_15/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/F4_17_2023_10_15_S27_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F4_17_2023_10_15/20231103_HLTHYDRX3/raw_data/F4_17_2023_10_15_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/F4_17_2023_10_15_S27_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F4_17_2023_10_15/20231103_HLTHYDRX3/raw_data/F4_17_2023_10_15_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F5_19_2023_10_14/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/F5_19_2023_10_14_S5_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F5_19_2023_10_14/20231103_HLTHYDRX3/raw_data/F5_19_2023_10_14_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/F5_19_2023_10_14_S5_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F5_19_2023_10_14/20231103_HLTHYDRX3/raw_data/F5_19_2023_10_14_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F6_32_2023_10_12/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/F6_32_2023_10_12_S50_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F6_32_2023_10_12/20231103_HLTHYDRX3/raw_data/F6_32_2023_10_12_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/F6_32_2023_10_12_S50_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F6_32_2023_10_12/20231103_HLTHYDRX3/raw_data/F6_32_2023_10_12_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F7_33_2023_10_16/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/F7_33_2023_10_16_S44_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F7_33_2023_10_16/20231103_HLTHYDRX3/raw_data/F7_33_2023_10_16_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/F7_33_2023_10_16_S44_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F7_33_2023_10_16/20231103_HLTHYDRX3/raw_data/F7_33_2023_10_16_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F8_35_2023_10_12/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/F8_35_2023_10_12_S8_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F8_35_2023_10_12/20231103_HLTHYDRX3/raw_data/F8_35_2023_10_12_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/F8_35_2023_10_12_S8_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F8_35_2023_10_12/20231103_HLTHYDRX3/raw_data/F8_35_2023_10_12_R2.fastq.gz"||X +echo -ne "\r[███████▎········]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F9_36_2023_10_17/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/F9_36_2023_10_17_S13_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F9_36_2023_10_17/20231103_HLTHYDRX3/raw_data/F9_36_2023_10_17_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/F9_36_2023_10_17_S13_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F9_36_2023_10_17/20231103_HLTHYDRX3/raw_data/F9_36_2023_10_17_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G1_10_2023_10_12/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/G1_10_2023_10_12_S33_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G1_10_2023_10_12/20231103_HLTHYDRX3/raw_data/G1_10_2023_10_12_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/G1_10_2023_10_12_S33_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G1_10_2023_10_12/20231103_HLTHYDRX3/raw_data/G1_10_2023_10_12_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G2_12_2023_10_15/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/G2_12_2023_10_15_S32_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G2_12_2023_10_15/20231103_HLTHYDRX3/raw_data/G2_12_2023_10_15_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/G2_12_2023_10_15_S32_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G2_12_2023_10_15/20231103_HLTHYDRX3/raw_data/G2_12_2023_10_15_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G3_16_2023_10_15/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/G3_16_2023_10_15_S20_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G3_16_2023_10_15/20231103_HLTHYDRX3/raw_data/G3_16_2023_10_15_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/G3_16_2023_10_15_S20_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G3_16_2023_10_15/20231103_HLTHYDRX3/raw_data/G3_16_2023_10_15_R2.fastq.gz"||X +echo -ne "\r[███████▍········]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G4_18_2023_10_11/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/G4_18_2023_10_11_S26_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G4_18_2023_10_11/20231103_HLTHYDRX3/raw_data/G4_18_2023_10_11_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/G4_18_2023_10_11_S26_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G4_18_2023_10_11/20231103_HLTHYDRX3/raw_data/G4_18_2023_10_11_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G5_19_2023_10_15/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/G5_19_2023_10_15_S65_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G5_19_2023_10_15/20231103_HLTHYDRX3/raw_data/G5_19_2023_10_15_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/G5_19_2023_10_15_S65_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G5_19_2023_10_15/20231103_HLTHYDRX3/raw_data/G5_19_2023_10_15_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G6_32_2023_10_14/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/G6_32_2023_10_14_S35_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G6_32_2023_10_14/20231103_HLTHYDRX3/raw_data/G6_32_2023_10_14_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/G6_32_2023_10_14_S35_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G6_32_2023_10_14/20231103_HLTHYDRX3/raw_data/G6_32_2023_10_14_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G7_33_2023_10_17/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/G7_33_2023_10_17_S12_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G7_33_2023_10_17/20231103_HLTHYDRX3/raw_data/G7_33_2023_10_17_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/G7_33_2023_10_17_S12_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G7_33_2023_10_17/20231103_HLTHYDRX3/raw_data/G7_33_2023_10_17_R2.fastq.gz"||X +echo -ne "\r[███████▌········]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G8_35_2023_10_13/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/G8_35_2023_10_13_S56_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G8_35_2023_10_13/20231103_HLTHYDRX3/raw_data/G8_35_2023_10_13_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/G8_35_2023_10_13_S56_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G8_35_2023_10_13/20231103_HLTHYDRX3/raw_data/G8_35_2023_10_13_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H1_10_2023_10_13/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/H1_10_2023_10_13_S49_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H1_10_2023_10_13/20231103_HLTHYDRX3/raw_data/H1_10_2023_10_13_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/H1_10_2023_10_13_S49_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H1_10_2023_10_13/20231103_HLTHYDRX3/raw_data/H1_10_2023_10_13_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H2_15_2023_10_11/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/H2_15_2023_10_11_S15_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H2_15_2023_10_11/20231103_HLTHYDRX3/raw_data/H2_15_2023_10_11_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/H2_15_2023_10_11_S15_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H2_15_2023_10_11/20231103_HLTHYDRX3/raw_data/H2_15_2023_10_11_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H3_16_2023_10_16/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/H3_16_2023_10_16_S62_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H3_16_2023_10_16/20231103_HLTHYDRX3/raw_data/H3_16_2023_10_16_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/H3_16_2023_10_16_S62_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H3_16_2023_10_16/20231103_HLTHYDRX3/raw_data/H3_16_2023_10_16_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H4_18_2023_10_13/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/H4_18_2023_10_13_S30_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H4_18_2023_10_13/20231103_HLTHYDRX3/raw_data/H4_18_2023_10_13_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/H4_18_2023_10_13_S30_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H4_18_2023_10_13/20231103_HLTHYDRX3/raw_data/H4_18_2023_10_13_R2.fastq.gz"||X +echo -ne "\r[███████▋········]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H5_19_2023_10_16/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/H5_19_2023_10_16_S70_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H5_19_2023_10_16/20231103_HLTHYDRX3/raw_data/H5_19_2023_10_16_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/H5_19_2023_10_16_S70_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H5_19_2023_10_16/20231103_HLTHYDRX3/raw_data/H5_19_2023_10_16_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H6_32_2023_10_15/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/H6_32_2023_10_15_S21_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H6_32_2023_10_15/20231103_HLTHYDRX3/raw_data/H6_32_2023_10_15_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/H6_32_2023_10_15_S21_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H6_32_2023_10_15/20231103_HLTHYDRX3/raw_data/H6_32_2023_10_15_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H7_34_2023_10_10/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/H7_34_2023_10_10_S7_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H7_34_2023_10_10/20231103_HLTHYDRX3/raw_data/H7_34_2023_10_10_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/H7_34_2023_10_10_S7_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H7_34_2023_10_10/20231103_HLTHYDRX3/raw_data/H7_34_2023_10_10_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H8_35_2023_10_14/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/H8_35_2023_10_14_S55_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H8_35_2023_10_14/20231103_HLTHYDRX3/raw_data/H8_35_2023_10_14_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33236_NovaSeq_231103_NOV1917/H8_35_2023_10_14_S55_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H8_35_2023_10_14/20231103_HLTHYDRX3/raw_data/H8_35_2023_10_14_R2.fastq.gz"||X +if [[ ! -d "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919" ]]; then + fail "Not a directory:" "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919" +fi +echo -ne "\r[███████▊········]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A1_10_2023_10_24/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/A1_10_2023_10_24_S37_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A1_10_2023_10_24/20231110_HKHFMDRX3/raw_data/A1_10_2023_10_24_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/A1_10_2023_10_24_S37_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A1_10_2023_10_24/20231110_HKHFMDRX3/raw_data/A1_10_2023_10_24_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A2_12_2023_10_28/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/A2_12_2023_10_28_S69_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A2_12_2023_10_28/20231110_HKHFMDRX3/raw_data/A2_12_2023_10_28_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/A2_12_2023_10_28_S69_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A2_12_2023_10_28/20231110_HKHFMDRX3/raw_data/A2_12_2023_10_28_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A3_35_2023_10_26/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/A3_35_2023_10_26_S26_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A3_35_2023_10_26/20231110_HKHFMDRX3/raw_data/A3_35_2023_10_26_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/A3_35_2023_10_26_S26_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A3_35_2023_10_26/20231110_HKHFMDRX3/raw_data/A3_35_2023_10_26_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A4_05_2023_10_30/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/A4_05_2023_10_30_S17_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A4_05_2023_10_30/20231110_HKHFMDRX3/raw_data/A4_05_2023_10_30_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/A4_05_2023_10_30_S17_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A4_05_2023_10_30/20231110_HKHFMDRX3/raw_data/A4_05_2023_10_30_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A5_18_2023_10_28/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/A5_18_2023_10_28_S22_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A5_18_2023_10_28/20231110_HKHFMDRX3/raw_data/A5_18_2023_10_28_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/A5_18_2023_10_28_S22_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A5_18_2023_10_28/20231110_HKHFMDRX3/raw_data/A5_18_2023_10_28_R2.fastq.gz"||X +echo -ne "\r[███████▉········]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A6_34_2023_10_24/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/A6_34_2023_10_24_S58_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A6_34_2023_10_24/20231110_HKHFMDRX3/raw_data/A6_34_2023_10_24_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/A6_34_2023_10_24_S58_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A6_34_2023_10_24/20231110_HKHFMDRX3/raw_data/A6_34_2023_10_24_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A7_25_2023_10_29/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/A7_25_2023_10_29_S23_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A7_25_2023_10_29/20231110_HKHFMDRX3/raw_data/A7_25_2023_10_29_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/A7_25_2023_10_29_S23_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A7_25_2023_10_29/20231110_HKHFMDRX3/raw_data/A7_25_2023_10_29_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A8_32_2023_10_28/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/A8_32_2023_10_28_S19_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A8_32_2023_10_28/20231110_HKHFMDRX3/raw_data/A8_32_2023_10_28_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/A8_32_2023_10_28_S19_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A8_32_2023_10_28/20231110_HKHFMDRX3/raw_data/A8_32_2023_10_28_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A9_33_2023_10_31/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/A9_33_2023_10_31_S25_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A9_33_2023_10_31/20231110_HKHFMDRX3/raw_data/A9_33_2023_10_31_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/A9_33_2023_10_31_S25_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A9_33_2023_10_31/20231110_HKHFMDRX3/raw_data/A9_33_2023_10_31_R2.fastq.gz"||X +echo -ne "\r[████████········]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B1_10_2023_10_26/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/B1_10_2023_10_26_S46_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B1_10_2023_10_26/20231110_HKHFMDRX3/raw_data/B1_10_2023_10_26_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/B1_10_2023_10_26_S46_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B1_10_2023_10_26/20231110_HKHFMDRX3/raw_data/B1_10_2023_10_26_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B2_12_2023_10_29/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/B2_12_2023_10_29_S61_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B2_12_2023_10_29/20231110_HKHFMDRX3/raw_data/B2_12_2023_10_29_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/B2_12_2023_10_29_S61_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B2_12_2023_10_29/20231110_HKHFMDRX3/raw_data/B2_12_2023_10_29_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B3_35_2023_10_27/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/B3_35_2023_10_27_S63_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B3_35_2023_10_27/20231110_HKHFMDRX3/raw_data/B3_35_2023_10_27_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/B3_35_2023_10_27_S63_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B3_35_2023_10_27/20231110_HKHFMDRX3/raw_data/B3_35_2023_10_27_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B4_15_2023_10_25/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/B4_15_2023_10_25_S32_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B4_15_2023_10_25/20231110_HKHFMDRX3/raw_data/B4_15_2023_10_25_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/B4_15_2023_10_25_S32_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B4_15_2023_10_25/20231110_HKHFMDRX3/raw_data/B4_15_2023_10_25_R2.fastq.gz"||X +echo -ne "\r[████████▏·······]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B5_18_2023_10_29/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/B5_18_2023_10_29_S45_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B5_18_2023_10_29/20231110_HKHFMDRX3/raw_data/B5_18_2023_10_29_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/B5_18_2023_10_29_S45_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B5_18_2023_10_29/20231110_HKHFMDRX3/raw_data/B5_18_2023_10_29_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B6_34_2023_10_26/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/B6_34_2023_10_26_S9_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B6_34_2023_10_26/20231110_HKHFMDRX3/raw_data/B6_34_2023_10_26_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/B6_34_2023_10_26_S9_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B6_34_2023_10_26/20231110_HKHFMDRX3/raw_data/B6_34_2023_10_26_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B7_25_2023_10_30/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/B7_25_2023_10_30_S42_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B7_25_2023_10_30/20231110_HKHFMDRX3/raw_data/B7_25_2023_10_30_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/B7_25_2023_10_30_S42_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B7_25_2023_10_30/20231110_HKHFMDRX3/raw_data/B7_25_2023_10_30_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B8_32_2023_10_29/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/B8_32_2023_10_29_S33_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B8_32_2023_10_29/20231110_HKHFMDRX3/raw_data/B8_32_2023_10_29_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/B8_32_2023_10_29_S33_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B8_32_2023_10_29/20231110_HKHFMDRX3/raw_data/B8_32_2023_10_29_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B9_36_2023_10_26/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/B9_36_2023_10_26_S67_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B9_36_2023_10_26/20231110_HKHFMDRX3/raw_data/B9_36_2023_10_26_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/B9_36_2023_10_26_S67_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B9_36_2023_10_26/20231110_HKHFMDRX3/raw_data/B9_36_2023_10_26_R2.fastq.gz"||X +echo -ne "\r[████████▎·······]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C1_10_2023_10_27/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/C1_10_2023_10_27_S36_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C1_10_2023_10_27/20231110_HKHFMDRX3/raw_data/C1_10_2023_10_27_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/C1_10_2023_10_27_S36_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C1_10_2023_10_27/20231110_HKHFMDRX3/raw_data/C1_10_2023_10_27_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C2_17_2023_10_24/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/C2_17_2023_10_24_S47_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C2_17_2023_10_24/20231110_HKHFMDRX3/raw_data/C2_17_2023_10_24_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/C2_17_2023_10_24_S47_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C2_17_2023_10_24/20231110_HKHFMDRX3/raw_data/C2_17_2023_10_24_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C3_35_2023_10_28/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/C3_35_2023_10_28_S40_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C3_35_2023_10_28/20231110_HKHFMDRX3/raw_data/C3_35_2023_10_28_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/C3_35_2023_10_28_S40_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C3_35_2023_10_28/20231110_HKHFMDRX3/raw_data/C3_35_2023_10_28_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C4_15_2023_10_27/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/C4_15_2023_10_27_S66_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C4_15_2023_10_27/20231110_HKHFMDRX3/raw_data/C4_15_2023_10_27_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/C4_15_2023_10_27_S66_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C4_15_2023_10_27/20231110_HKHFMDRX3/raw_data/C4_15_2023_10_27_R2.fastq.gz"||X +echo -ne "\r[████████▍·······]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C5_18_2023_10_30/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/C5_18_2023_10_30_S56_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C5_18_2023_10_30/20231110_HKHFMDRX3/raw_data/C5_18_2023_10_30_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/C5_18_2023_10_30_S56_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C5_18_2023_10_30/20231110_HKHFMDRX3/raw_data/C5_18_2023_10_30_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C6_34_2023_10_27/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/C6_34_2023_10_27_S52_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C6_34_2023_10_27/20231110_HKHFMDRX3/raw_data/C6_34_2023_10_27_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/C6_34_2023_10_27_S52_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C6_34_2023_10_27/20231110_HKHFMDRX3/raw_data/C6_34_2023_10_27_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C7_16_2023_10_26/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/C7_16_2023_10_26_S28_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C7_16_2023_10_26/20231110_HKHFMDRX3/raw_data/C7_16_2023_10_26_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/C7_16_2023_10_26_S28_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C7_16_2023_10_26/20231110_HKHFMDRX3/raw_data/C7_16_2023_10_26_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C8_32_2023_10_30/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/C8_32_2023_10_30_S21_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C8_32_2023_10_30/20231110_HKHFMDRX3/raw_data/C8_32_2023_10_30_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/C8_32_2023_10_30_S21_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C8_32_2023_10_30/20231110_HKHFMDRX3/raw_data/C8_32_2023_10_30_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C9_36_2023_10_28/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/C9_36_2023_10_28_S2_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C9_36_2023_10_28/20231110_HKHFMDRX3/raw_data/C9_36_2023_10_28_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/C9_36_2023_10_28_S2_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C9_36_2023_10_28/20231110_HKHFMDRX3/raw_data/C9_36_2023_10_28_R2.fastq.gz"||X +echo -ne "\r[████████▌·······]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D1_10_2023_10_28/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/D1_10_2023_10_28_S50_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D1_10_2023_10_28/20231110_HKHFMDRX3/raw_data/D1_10_2023_10_28_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/D1_10_2023_10_28_S50_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D1_10_2023_10_28/20231110_HKHFMDRX3/raw_data/D1_10_2023_10_28_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D2_17_2023_10_26/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/D2_17_2023_10_26_S11_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D2_17_2023_10_26/20231110_HKHFMDRX3/raw_data/D2_17_2023_10_26_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/D2_17_2023_10_26_S11_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D2_17_2023_10_26/20231110_HKHFMDRX3/raw_data/D2_17_2023_10_26_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D3_35_2023_10_29/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/D3_35_2023_10_29_S16_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D3_35_2023_10_29/20231110_HKHFMDRX3/raw_data/D3_35_2023_10_29_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/D3_35_2023_10_29_S16_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D3_35_2023_10_29/20231110_HKHFMDRX3/raw_data/D3_35_2023_10_29_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D4_15_2023_10_28/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/D4_15_2023_10_28_S59_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D4_15_2023_10_28/20231110_HKHFMDRX3/raw_data/D4_15_2023_10_28_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/D4_15_2023_10_28_S59_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D4_15_2023_10_28/20231110_HKHFMDRX3/raw_data/D4_15_2023_10_28_R2.fastq.gz"||X +echo -ne "\r[████████▋·······]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D5_19_2023_10_25/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/D5_19_2023_10_25_S65_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D5_19_2023_10_25/20231110_HKHFMDRX3/raw_data/D5_19_2023_10_25_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/D5_19_2023_10_25_S65_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D5_19_2023_10_25/20231110_HKHFMDRX3/raw_data/D5_19_2023_10_25_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D6_34_2023_10_28/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/D6_34_2023_10_28_S5_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D6_34_2023_10_28/20231110_HKHFMDRX3/raw_data/D6_34_2023_10_28_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/D6_34_2023_10_28_S5_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D6_34_2023_10_28/20231110_HKHFMDRX3/raw_data/D6_34_2023_10_28_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D7_16_2023_10_28/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/D7_16_2023_10_28_S27_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D7_16_2023_10_28/20231110_HKHFMDRX3/raw_data/D7_16_2023_10_28_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/D7_16_2023_10_28_S27_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D7_16_2023_10_28/20231110_HKHFMDRX3/raw_data/D7_16_2023_10_28_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D8_32_2023_10_31/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/D8_32_2023_10_31_S18_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D8_32_2023_10_31/20231110_HKHFMDRX3/raw_data/D8_32_2023_10_31_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/D8_32_2023_10_31_S18_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D8_32_2023_10_31/20231110_HKHFMDRX3/raw_data/D8_32_2023_10_31_R2.fastq.gz"||X +echo -ne "\r[████████▊·······]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D9_36_2023_10_29/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/D9_36_2023_10_29_S49_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D9_36_2023_10_29/20231110_HKHFMDRX3/raw_data/D9_36_2023_10_29_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/D9_36_2023_10_29_S49_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D9_36_2023_10_29/20231110_HKHFMDRX3/raw_data/D9_36_2023_10_29_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E1_10_2023_10_29/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/E1_10_2023_10_29_S57_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E1_10_2023_10_29/20231110_HKHFMDRX3/raw_data/E1_10_2023_10_29_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/E1_10_2023_10_29_S57_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E1_10_2023_10_29/20231110_HKHFMDRX3/raw_data/E1_10_2023_10_29_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E2_17_2023_10_27/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/E2_17_2023_10_27_S7_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E2_17_2023_10_27/20231110_HKHFMDRX3/raw_data/E2_17_2023_10_27_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/E2_17_2023_10_27_S7_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E2_17_2023_10_27/20231110_HKHFMDRX3/raw_data/E2_17_2023_10_27_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E3_05_2023_10_25/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/E3_05_2023_10_25_S54_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E3_05_2023_10_25/20231110_HKHFMDRX3/raw_data/E3_05_2023_10_25_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/E3_05_2023_10_25_S54_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E3_05_2023_10_25/20231110_HKHFMDRX3/raw_data/E3_05_2023_10_25_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E4_15_2023_10_29/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/E4_15_2023_10_29_S44_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E4_15_2023_10_29/20231110_HKHFMDRX3/raw_data/E4_15_2023_10_29_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/E4_15_2023_10_29_S44_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E4_15_2023_10_29/20231110_HKHFMDRX3/raw_data/E4_15_2023_10_29_R2.fastq.gz"||X +echo -ne "\r[████████▉·······]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E5_19_2023_10_27/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/E5_19_2023_10_27_S60_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E5_19_2023_10_27/20231110_HKHFMDRX3/raw_data/E5_19_2023_10_27_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/E5_19_2023_10_27_S60_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E5_19_2023_10_27/20231110_HKHFMDRX3/raw_data/E5_19_2023_10_27_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E6_34_2023_10_29/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/E6_34_2023_10_29_S15_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E6_34_2023_10_29/20231110_HKHFMDRX3/raw_data/E6_34_2023_10_29_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/E6_34_2023_10_29_S15_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E6_34_2023_10_29/20231110_HKHFMDRX3/raw_data/E6_34_2023_10_29_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E7_16_2023_10_29/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/E7_16_2023_10_29_S68_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E7_16_2023_10_29/20231110_HKHFMDRX3/raw_data/E7_16_2023_10_29_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/E7_16_2023_10_29_S68_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E7_16_2023_10_29/20231110_HKHFMDRX3/raw_data/E7_16_2023_10_29_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E8_33_2023_10_26/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/E8_33_2023_10_26_S64_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E8_33_2023_10_26/20231110_HKHFMDRX3/raw_data/E8_33_2023_10_26_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/E8_33_2023_10_26_S64_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E8_33_2023_10_26/20231110_HKHFMDRX3/raw_data/E8_33_2023_10_26_R2.fastq.gz"||X +echo -ne "\r[█████████·······]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E9_36_2023_10_30/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/E9_36_2023_10_30_S51_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E9_36_2023_10_30/20231110_HKHFMDRX3/raw_data/E9_36_2023_10_30_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/E9_36_2023_10_30_S51_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E9_36_2023_10_30/20231110_HKHFMDRX3/raw_data/E9_36_2023_10_30_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F1_12_2023_10_24/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/F1_12_2023_10_24_S10_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F1_12_2023_10_24/20231110_HKHFMDRX3/raw_data/F1_12_2023_10_24_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/F1_12_2023_10_24_S10_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F1_12_2023_10_24/20231110_HKHFMDRX3/raw_data/F1_12_2023_10_24_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F2_17_2023_10_28/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/F2_17_2023_10_28_S41_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F2_17_2023_10_28/20231110_HKHFMDRX3/raw_data/F2_17_2023_10_28_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/F2_17_2023_10_28_S41_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F2_17_2023_10_28/20231110_HKHFMDRX3/raw_data/F2_17_2023_10_28_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F3_05_2023_10_27/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/F3_05_2023_10_27_S38_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F3_05_2023_10_27/20231110_HKHFMDRX3/raw_data/F3_05_2023_10_27_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/F3_05_2023_10_27_S38_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F3_05_2023_10_27/20231110_HKHFMDRX3/raw_data/F3_05_2023_10_27_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F4_15_2023_10_30/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/F4_15_2023_10_30_S20_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F4_15_2023_10_30/20231110_HKHFMDRX3/raw_data/F4_15_2023_10_30_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/F4_15_2023_10_30_S20_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F4_15_2023_10_30/20231110_HKHFMDRX3/raw_data/F4_15_2023_10_30_R2.fastq.gz"||X +echo -ne "\r[█████████▏······]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F5_19_2023_10_28/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/F5_19_2023_10_28_S13_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F5_19_2023_10_28/20231110_HKHFMDRX3/raw_data/F5_19_2023_10_28_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/F5_19_2023_10_28_S13_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F5_19_2023_10_28/20231110_HKHFMDRX3/raw_data/F5_19_2023_10_28_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F6_25_2023_10_27/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/F6_25_2023_10_27_S30_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F6_25_2023_10_27/20231110_HKHFMDRX3/raw_data/F6_25_2023_10_27_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/F6_25_2023_10_27_S30_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F6_25_2023_10_27/20231110_HKHFMDRX3/raw_data/F6_25_2023_10_27_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F7_16_2023_10_30/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/F7_16_2023_10_30_S43_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F7_16_2023_10_30/20231110_HKHFMDRX3/raw_data/F7_16_2023_10_30_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/F7_16_2023_10_30_S43_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F7_16_2023_10_30/20231110_HKHFMDRX3/raw_data/F7_16_2023_10_30_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F8_33_2023_10_28/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/F8_33_2023_10_28_S34_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F8_33_2023_10_28/20231110_HKHFMDRX3/raw_data/F8_33_2023_10_28_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/F8_33_2023_10_28_S34_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F8_33_2023_10_28/20231110_HKHFMDRX3/raw_data/F8_33_2023_10_28_R2.fastq.gz"||X +echo -ne "\r[█████████▎······]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F9_36_2023_10_31/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/F9_36_2023_10_31_S8_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F9_36_2023_10_31/20231110_HKHFMDRX3/raw_data/F9_36_2023_10_31_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/F9_36_2023_10_31_S8_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F9_36_2023_10_31/20231110_HKHFMDRX3/raw_data/F9_36_2023_10_31_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G1_12_2023_10_26/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/G1_12_2023_10_26_S70_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G1_12_2023_10_26/20231110_HKHFMDRX3/raw_data/G1_12_2023_10_26_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/G1_12_2023_10_26_S70_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G1_12_2023_10_26/20231110_HKHFMDRX3/raw_data/G1_12_2023_10_26_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G2_17_2023_10_29/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/G2_17_2023_10_29_S24_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G2_17_2023_10_29/20231110_HKHFMDRX3/raw_data/G2_17_2023_10_29_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/G2_17_2023_10_29_S24_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G2_17_2023_10_29/20231110_HKHFMDRX3/raw_data/G2_17_2023_10_29_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G3_05_2023_10_28/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/G3_05_2023_10_28_S39_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G3_05_2023_10_28/20231110_HKHFMDRX3/raw_data/G3_05_2023_10_28_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/G3_05_2023_10_28_S39_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G3_05_2023_10_28/20231110_HKHFMDRX3/raw_data/G3_05_2023_10_28_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G4_18_2023_10_25/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/G4_18_2023_10_25_S6_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G4_18_2023_10_25/20231110_HKHFMDRX3/raw_data/G4_18_2023_10_25_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/G4_18_2023_10_25_S6_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G4_18_2023_10_25/20231110_HKHFMDRX3/raw_data/G4_18_2023_10_25_R2.fastq.gz"||X +echo -ne "\r[█████████▍······]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G5_19_2023_10_29/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/G5_19_2023_10_29_S48_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G5_19_2023_10_29/20231110_HKHFMDRX3/raw_data/G5_19_2023_10_29_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/G5_19_2023_10_29_S48_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G5_19_2023_10_29/20231110_HKHFMDRX3/raw_data/G5_19_2023_10_29_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G6_25_2023_10_25/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/G6_25_2023_10_25_S55_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G6_25_2023_10_25/20231110_HKHFMDRX3/raw_data/G6_25_2023_10_25_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/G6_25_2023_10_25_S55_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G6_25_2023_10_25/20231110_HKHFMDRX3/raw_data/G6_25_2023_10_25_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G7_16_2023_10_31/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/G7_16_2023_10_31_S14_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G7_16_2023_10_31/20231110_HKHFMDRX3/raw_data/G7_16_2023_10_31_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/G7_16_2023_10_31_S14_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G7_16_2023_10_31/20231110_HKHFMDRX3/raw_data/G7_16_2023_10_31_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G8_33_2023_10_29/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/G8_33_2023_10_29_S35_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G8_33_2023_10_29/20231110_HKHFMDRX3/raw_data/G8_33_2023_10_29_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/G8_33_2023_10_29_S35_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G8_33_2023_10_29/20231110_HKHFMDRX3/raw_data/G8_33_2023_10_29_R2.fastq.gz"||X +echo -ne "\r[█████████▌······]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H1_12_2023_10_27/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/H1_12_2023_10_27_S29_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H1_12_2023_10_27/20231110_HKHFMDRX3/raw_data/H1_12_2023_10_27_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/H1_12_2023_10_27_S29_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H1_12_2023_10_27/20231110_HKHFMDRX3/raw_data/H1_12_2023_10_27_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H2_35_2023_10_24/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/H2_35_2023_10_24_S4_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H2_35_2023_10_24/20231110_HKHFMDRX3/raw_data/H2_35_2023_10_24_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/H2_35_2023_10_24_S4_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H2_35_2023_10_24/20231110_HKHFMDRX3/raw_data/H2_35_2023_10_24_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H3_05_2023_10_29/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/H3_05_2023_10_29_S62_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H3_05_2023_10_29/20231110_HKHFMDRX3/raw_data/H3_05_2023_10_29_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/H3_05_2023_10_29_S62_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H3_05_2023_10_29/20231110_HKHFMDRX3/raw_data/H3_05_2023_10_29_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H4_18_2023_10_27/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/H4_18_2023_10_27_S31_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H4_18_2023_10_27/20231110_HKHFMDRX3/raw_data/H4_18_2023_10_27_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/H4_18_2023_10_27_S31_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H4_18_2023_10_27/20231110_HKHFMDRX3/raw_data/H4_18_2023_10_27_R2.fastq.gz"||X +echo -ne "\r[█████████▋······]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H5_19_2023_10_30/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/H5_19_2023_10_30_S12_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H5_19_2023_10_30/20231110_HKHFMDRX3/raw_data/H5_19_2023_10_30_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/H5_19_2023_10_30_S12_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H5_19_2023_10_30/20231110_HKHFMDRX3/raw_data/H5_19_2023_10_30_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H6_25_2023_10_28/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/H6_25_2023_10_28_S53_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H6_25_2023_10_28/20231110_HKHFMDRX3/raw_data/H6_25_2023_10_28_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/H6_25_2023_10_28_S53_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H6_25_2023_10_28/20231110_HKHFMDRX3/raw_data/H6_25_2023_10_28_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H7_32_2023_10_26/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/H7_32_2023_10_26_S1_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H7_32_2023_10_26/20231110_HKHFMDRX3/raw_data/H7_32_2023_10_26_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/H7_32_2023_10_26_S1_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H7_32_2023_10_26/20231110_HKHFMDRX3/raw_data/H7_32_2023_10_26_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H8_33_2023_10_30/"{,"20231110_HKHFMDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/H8_33_2023_10_30_S3_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H8_33_2023_10_30/20231110_HKHFMDRX3/raw_data/H8_33_2023_10_30_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33365_NovaSeq_231110_NOV1919/H8_33_2023_10_30_S3_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H8_33_2023_10_30/20231110_HKHFMDRX3/raw_data/H8_33_2023_10_30_R2.fastq.gz"||X +if [[ ! -d "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917" ]]; then + fail "Not a directory:" "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917" +fi + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A1_05_2023_10_17/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/A1_05_2023_10_17_S135_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A1_05_2023_10_17/20231103_HLTHYDRX3/raw_data/A1_05_2023_10_17_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/A1_05_2023_10_17_S135_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A1_05_2023_10_17/20231103_HLTHYDRX3/raw_data/A1_05_2023_10_17_R2.fastq.gz"||X +echo -ne "\r[█████████▊······]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A2_10_2023_10_21/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/A2_10_2023_10_21_S131_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A2_10_2023_10_21/20231103_HLTHYDRX3/raw_data/A2_10_2023_10_21_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/A2_10_2023_10_21_S131_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A2_10_2023_10_21/20231103_HLTHYDRX3/raw_data/A2_10_2023_10_21_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A3_15_2023_10_19/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/A3_15_2023_10_19_S123_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A3_15_2023_10_19/20231103_HLTHYDRX3/raw_data/A3_15_2023_10_19_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/A3_15_2023_10_19_S123_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A3_15_2023_10_19/20231103_HLTHYDRX3/raw_data/A3_15_2023_10_19_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A4_16_2023_10_24/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/A4_16_2023_10_24_S132_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A4_16_2023_10_24/20231103_HLTHYDRX3/raw_data/A4_16_2023_10_24_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/A4_16_2023_10_24_S132_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A4_16_2023_10_24/20231103_HLTHYDRX3/raw_data/A4_16_2023_10_24_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A5_18_2023_10_21/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/A5_18_2023_10_21_S90_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A5_18_2023_10_21/20231103_HLTHYDRX3/raw_data/A5_18_2023_10_21_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/A5_18_2023_10_21_S90_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A5_18_2023_10_21/20231103_HLTHYDRX3/raw_data/A5_18_2023_10_21_R2.fastq.gz"||X +echo -ne "\r[█████████▉······]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A6_25_2023_10_17/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/A6_25_2023_10_17_S85_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A6_25_2023_10_17/20231103_HLTHYDRX3/raw_data/A6_25_2023_10_17_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/A6_25_2023_10_17_S85_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A6_25_2023_10_17/20231103_HLTHYDRX3/raw_data/A6_25_2023_10_17_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A7_32_2023_10_23/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/A7_32_2023_10_23_S118_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A7_32_2023_10_23/20231103_HLTHYDRX3/raw_data/A7_32_2023_10_23_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/A7_32_2023_10_23_S118_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A7_32_2023_10_23/20231103_HLTHYDRX3/raw_data/A7_32_2023_10_23_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A8_34_2023_10_18/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/A8_34_2023_10_18_S121_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A8_34_2023_10_18/20231103_HLTHYDRX3/raw_data/A8_34_2023_10_18_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/A8_34_2023_10_18_S121_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A8_34_2023_10_18/20231103_HLTHYDRX3/raw_data/A8_34_2023_10_18_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"A9_35_2023_10_22/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/A9_35_2023_10_22_S94_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/A9_35_2023_10_22/20231103_HLTHYDRX3/raw_data/A9_35_2023_10_22_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/A9_35_2023_10_22_S94_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/A9_35_2023_10_22/20231103_HLTHYDRX3/raw_data/A9_35_2023_10_22_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B1_05_2023_10_19/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/B1_05_2023_10_19_S97_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B1_05_2023_10_19/20231103_HLTHYDRX3/raw_data/B1_05_2023_10_19_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/B1_05_2023_10_19_S97_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B1_05_2023_10_19/20231103_HLTHYDRX3/raw_data/B1_05_2023_10_19_R2.fastq.gz"||X +echo -ne "\r[██████████······]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B2_10_2023_10_22/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/B2_10_2023_10_22_S81_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B2_10_2023_10_22/20231103_HLTHYDRX3/raw_data/B2_10_2023_10_22_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/B2_10_2023_10_22_S81_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B2_10_2023_10_22/20231103_HLTHYDRX3/raw_data/B2_10_2023_10_22_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B3_15_2023_10_21/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/B3_15_2023_10_21_S104_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B3_15_2023_10_21/20231103_HLTHYDRX3/raw_data/B3_15_2023_10_21_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/B3_15_2023_10_21_S104_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B3_15_2023_10_21/20231103_HLTHYDRX3/raw_data/B3_15_2023_10_21_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B4_17_2023_10_16/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/B4_17_2023_10_16_S93_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B4_17_2023_10_16/20231103_HLTHYDRX3/raw_data/B4_17_2023_10_16_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/B4_17_2023_10_16_S93_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B4_17_2023_10_16/20231103_HLTHYDRX3/raw_data/B4_17_2023_10_16_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B5_18_2023_10_22/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/B5_18_2023_10_22_S101_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B5_18_2023_10_22/20231103_HLTHYDRX3/raw_data/B5_18_2023_10_22_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/B5_18_2023_10_22_S101_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B5_18_2023_10_22/20231103_HLTHYDRX3/raw_data/B5_18_2023_10_22_R2.fastq.gz"||X +echo -ne "\r[██████████▏·····]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B6_25_2023_10_19/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/B6_25_2023_10_19_S114_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B6_25_2023_10_19/20231103_HLTHYDRX3/raw_data/B6_25_2023_10_19_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/B6_25_2023_10_19_S114_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B6_25_2023_10_19/20231103_HLTHYDRX3/raw_data/B6_25_2023_10_19_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B7_32_2023_10_24/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/B7_32_2023_10_24_S117_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B7_32_2023_10_24/20231103_HLTHYDRX3/raw_data/B7_32_2023_10_24_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/B7_32_2023_10_24_S117_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B7_32_2023_10_24/20231103_HLTHYDRX3/raw_data/B7_32_2023_10_24_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B8_34_2023_10_20/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/B8_34_2023_10_20_S111_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B8_34_2023_10_20/20231103_HLTHYDRX3/raw_data/B8_34_2023_10_20_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/B8_34_2023_10_20_S111_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B8_34_2023_10_20/20231103_HLTHYDRX3/raw_data/B8_34_2023_10_20_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"B9_36_2023_10_18/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/B9_36_2023_10_18_S112_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/B9_36_2023_10_18/20231103_HLTHYDRX3/raw_data/B9_36_2023_10_18_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/B9_36_2023_10_18_S112_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/B9_36_2023_10_18/20231103_HLTHYDRX3/raw_data/B9_36_2023_10_18_R2.fastq.gz"||X +echo -ne "\r[██████████▎·····]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C1_05_2023_10_21/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/C1_05_2023_10_21_S96_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C1_05_2023_10_21/20231103_HLTHYDRX3/raw_data/C1_05_2023_10_21_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/C1_05_2023_10_21_S96_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C1_05_2023_10_21/20231103_HLTHYDRX3/raw_data/C1_05_2023_10_21_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C2_12_2023_10_16/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/C2_12_2023_10_16_S91_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C2_12_2023_10_16/20231103_HLTHYDRX3/raw_data/C2_12_2023_10_16_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/C2_12_2023_10_16_S91_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C2_12_2023_10_16/20231103_HLTHYDRX3/raw_data/C2_12_2023_10_16_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C3_15_2023_10_22/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/C3_15_2023_10_22_S79_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C3_15_2023_10_22/20231103_HLTHYDRX3/raw_data/C3_15_2023_10_22_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/C3_15_2023_10_22_S79_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C3_15_2023_10_22/20231103_HLTHYDRX3/raw_data/C3_15_2023_10_22_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C4_17_2023_10_18/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/C4_17_2023_10_18_S89_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C4_17_2023_10_18/20231103_HLTHYDRX3/raw_data/C4_17_2023_10_18_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/C4_17_2023_10_18_S89_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C4_17_2023_10_18/20231103_HLTHYDRX3/raw_data/C4_17_2023_10_18_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C5_18_2023_10_23/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/C5_18_2023_10_23_S82_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C5_18_2023_10_23/20231103_HLTHYDRX3/raw_data/C5_18_2023_10_23_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/C5_18_2023_10_23_S82_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C5_18_2023_10_23/20231103_HLTHYDRX3/raw_data/C5_18_2023_10_23_R2.fastq.gz"||X +echo -ne "\r[██████████▍·····]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C6_25_2023_10_21/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/C6_25_2023_10_21_S125_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C6_25_2023_10_21/20231103_HLTHYDRX3/raw_data/C6_25_2023_10_21_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/C6_25_2023_10_21_S125_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C6_25_2023_10_21/20231103_HLTHYDRX3/raw_data/C6_25_2023_10_21_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C7_33_2023_10_18/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/C7_33_2023_10_18_S106_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C7_33_2023_10_18/20231103_HLTHYDRX3/raw_data/C7_33_2023_10_18_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/C7_33_2023_10_18_S106_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C7_33_2023_10_18/20231103_HLTHYDRX3/raw_data/C7_33_2023_10_18_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C8_34_2023_10_21/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/C8_34_2023_10_21_S119_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C8_34_2023_10_21/20231103_HLTHYDRX3/raw_data/C8_34_2023_10_21_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/C8_34_2023_10_21_S119_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C8_34_2023_10_21/20231103_HLTHYDRX3/raw_data/C8_34_2023_10_21_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"C9_36_2023_10_20/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/C9_36_2023_10_20_S126_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/C9_36_2023_10_20/20231103_HLTHYDRX3/raw_data/C9_36_2023_10_20_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/C9_36_2023_10_20_S126_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/C9_36_2023_10_20/20231103_HLTHYDRX3/raw_data/C9_36_2023_10_20_R2.fastq.gz"||X +echo -ne "\r[██████████▌·····]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D1_05_2023_10_22/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/D1_05_2023_10_22_S136_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D1_05_2023_10_22/20231103_HLTHYDRX3/raw_data/D1_05_2023_10_22_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/D1_05_2023_10_22_S136_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D1_05_2023_10_22/20231103_HLTHYDRX3/raw_data/D1_05_2023_10_22_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D2_12_2023_10_18/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/D2_12_2023_10_18_S102_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D2_12_2023_10_18/20231103_HLTHYDRX3/raw_data/D2_12_2023_10_18_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/D2_12_2023_10_18_S102_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D2_12_2023_10_18/20231103_HLTHYDRX3/raw_data/D2_12_2023_10_18_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D3_15_2023_10_23/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/D3_15_2023_10_23_S122_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D3_15_2023_10_23/20231103_HLTHYDRX3/raw_data/D3_15_2023_10_23_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/D3_15_2023_10_23_S122_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D3_15_2023_10_23/20231103_HLTHYDRX3/raw_data/D3_15_2023_10_23_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D4_17_2023_10_20/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/D4_17_2023_10_20_S138_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D4_17_2023_10_20/20231103_HLTHYDRX3/raw_data/D4_17_2023_10_20_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/D4_17_2023_10_20_S138_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D4_17_2023_10_20/20231103_HLTHYDRX3/raw_data/D4_17_2023_10_20_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D5_19_2023_10_17/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/D5_19_2023_10_17_S129_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D5_19_2023_10_17/20231103_HLTHYDRX3/raw_data/D5_19_2023_10_17_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/D5_19_2023_10_17_S129_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D5_19_2023_10_17/20231103_HLTHYDRX3/raw_data/D5_19_2023_10_17_R2.fastq.gz"||X +echo -ne "\r[██████████▋·····]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D6_25_2023_10_22/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/D6_25_2023_10_22_S107_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D6_25_2023_10_22/20231103_HLTHYDRX3/raw_data/D6_25_2023_10_22_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/D6_25_2023_10_22_S107_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D6_25_2023_10_22/20231103_HLTHYDRX3/raw_data/D6_25_2023_10_22_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D7_33_2023_10_20/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/D7_33_2023_10_20_S109_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D7_33_2023_10_20/20231103_HLTHYDRX3/raw_data/D7_33_2023_10_20_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/D7_33_2023_10_20_S109_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D7_33_2023_10_20/20231103_HLTHYDRX3/raw_data/D7_33_2023_10_20_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D8_34_2023_10_22/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/D8_34_2023_10_22_S76_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D8_34_2023_10_22/20231103_HLTHYDRX3/raw_data/D8_34_2023_10_22_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/D8_34_2023_10_22_S76_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D8_34_2023_10_22/20231103_HLTHYDRX3/raw_data/D8_34_2023_10_22_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"D9_36_2023_10_22/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/D9_36_2023_10_22_S92_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/D9_36_2023_10_22/20231103_HLTHYDRX3/raw_data/D9_36_2023_10_22_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/D9_36_2023_10_22_S92_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/D9_36_2023_10_22/20231103_HLTHYDRX3/raw_data/D9_36_2023_10_22_R2.fastq.gz"||X +echo -ne "\r[██████████▊·····]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E1_05_2023_10_23/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/E1_05_2023_10_23_S77_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E1_05_2023_10_23/20231103_HLTHYDRX3/raw_data/E1_05_2023_10_23_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/E1_05_2023_10_23_S77_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E1_05_2023_10_23/20231103_HLTHYDRX3/raw_data/E1_05_2023_10_23_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E2_12_2023_10_20/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/E2_12_2023_10_20_S130_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E2_12_2023_10_20/20231103_HLTHYDRX3/raw_data/E2_12_2023_10_20_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/E2_12_2023_10_20_S130_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E2_12_2023_10_20/20231103_HLTHYDRX3/raw_data/E2_12_2023_10_20_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E3_16_2023_10_18/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/E3_16_2023_10_18_S98_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E3_16_2023_10_18/20231103_HLTHYDRX3/raw_data/E3_16_2023_10_18_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/E3_16_2023_10_18_S98_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E3_16_2023_10_18/20231103_HLTHYDRX3/raw_data/E3_16_2023_10_18_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E4_17_2023_10_21/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/E4_17_2023_10_21_S84_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E4_17_2023_10_21/20231103_HLTHYDRX3/raw_data/E4_17_2023_10_21_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/E4_17_2023_10_21_S84_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E4_17_2023_10_21/20231103_HLTHYDRX3/raw_data/E4_17_2023_10_21_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E5_19_2023_10_19/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/E5_19_2023_10_19_S103_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E5_19_2023_10_19/20231103_HLTHYDRX3/raw_data/E5_19_2023_10_19_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/E5_19_2023_10_19_S103_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E5_19_2023_10_19/20231103_HLTHYDRX3/raw_data/E5_19_2023_10_19_R2.fastq.gz"||X +echo -ne "\r[██████████▉·····]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E6_25_2023_10_23/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/E6_25_2023_10_23_S127_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E6_25_2023_10_23/20231103_HLTHYDRX3/raw_data/E6_25_2023_10_23_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/E6_25_2023_10_23_S127_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E6_25_2023_10_23/20231103_HLTHYDRX3/raw_data/E6_25_2023_10_23_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E8_35_2023_10_16/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/E8_35_2023_10_16_S88_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E8_35_2023_10_16/20231103_HLTHYDRX3/raw_data/E8_35_2023_10_16_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/E8_35_2023_10_16_S88_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E8_35_2023_10_16/20231103_HLTHYDRX3/raw_data/E8_35_2023_10_16_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"E9_36_2023_10_23/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/E9_36_2023_10_23_S128_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/E9_36_2023_10_23/20231103_HLTHYDRX3/raw_data/E9_36_2023_10_23_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/E9_36_2023_10_23_S128_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/E9_36_2023_10_23/20231103_HLTHYDRX3/raw_data/E9_36_2023_10_23_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F1_10_2023_10_16/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/F1_10_2023_10_16_S75_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F1_10_2023_10_16/20231103_HLTHYDRX3/raw_data/F1_10_2023_10_16_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/F1_10_2023_10_16_S75_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F1_10_2023_10_16/20231103_HLTHYDRX3/raw_data/F1_10_2023_10_16_R2.fastq.gz"||X +echo -ne "\r[███████████·····]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F2_12_2023_10_21/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/F2_12_2023_10_21_S87_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F2_12_2023_10_21/20231103_HLTHYDRX3/raw_data/F2_12_2023_10_21_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/F2_12_2023_10_21_S87_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F2_12_2023_10_21/20231103_HLTHYDRX3/raw_data/F2_12_2023_10_21_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F3_16_2023_10_20/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/F3_16_2023_10_20_S134_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F3_16_2023_10_20/20231103_HLTHYDRX3/raw_data/F3_16_2023_10_20_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/F3_16_2023_10_20_S134_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F3_16_2023_10_20/20231103_HLTHYDRX3/raw_data/F3_16_2023_10_20_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F4_17_2023_10_22/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/F4_17_2023_10_22_S133_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F4_17_2023_10_22/20231103_HLTHYDRX3/raw_data/F4_17_2023_10_22_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/F4_17_2023_10_22_S133_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F4_17_2023_10_22/20231103_HLTHYDRX3/raw_data/F4_17_2023_10_22_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F5_19_2023_10_21/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/F5_19_2023_10_21_S137_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F5_19_2023_10_21/20231103_HLTHYDRX3/raw_data/F5_19_2023_10_21_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/F5_19_2023_10_21_S137_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F5_19_2023_10_21/20231103_HLTHYDRX3/raw_data/F5_19_2023_10_21_R2.fastq.gz"||X +echo -ne "\r[███████████▏····]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F6_32_2023_10_18/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/F6_32_2023_10_18_S124_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F6_32_2023_10_18/20231103_HLTHYDRX3/raw_data/F6_32_2023_10_18_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/F6_32_2023_10_18_S124_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F6_32_2023_10_18/20231103_HLTHYDRX3/raw_data/F6_32_2023_10_18_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F7_33_2023_10_23/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/F7_33_2023_10_23_S78_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F7_33_2023_10_23/20231103_HLTHYDRX3/raw_data/F7_33_2023_10_23_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/F7_33_2023_10_23_S78_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F7_33_2023_10_23/20231103_HLTHYDRX3/raw_data/F7_33_2023_10_23_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F8_35_2023_10_18/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/F8_35_2023_10_18_S74_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F8_35_2023_10_18/20231103_HLTHYDRX3/raw_data/F8_35_2023_10_18_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/F8_35_2023_10_18_S74_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F8_35_2023_10_18/20231103_HLTHYDRX3/raw_data/F8_35_2023_10_18_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"F9_36_2023_10_24/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/F9_36_2023_10_24_S80_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/F9_36_2023_10_24/20231103_HLTHYDRX3/raw_data/F9_36_2023_10_24_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/F9_36_2023_10_24_S80_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/F9_36_2023_10_24/20231103_HLTHYDRX3/raw_data/F9_36_2023_10_24_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G1_10_2023_10_18/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/G1_10_2023_10_18_S86_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G1_10_2023_10_18/20231103_HLTHYDRX3/raw_data/G1_10_2023_10_18_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/G1_10_2023_10_18_S86_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G1_10_2023_10_18/20231103_HLTHYDRX3/raw_data/G1_10_2023_10_18_R2.fastq.gz"||X +echo -ne "\r[███████████▎····]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G2_12_2023_10_22/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/G2_12_2023_10_22_S120_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G2_12_2023_10_22/20231103_HLTHYDRX3/raw_data/G2_12_2023_10_22_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/G2_12_2023_10_22_S120_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G2_12_2023_10_22/20231103_HLTHYDRX3/raw_data/G2_12_2023_10_22_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G3_16_2023_10_22/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/G3_16_2023_10_22_S115_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G3_16_2023_10_22/20231103_HLTHYDRX3/raw_data/G3_16_2023_10_22_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/G3_16_2023_10_22_S115_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G3_16_2023_10_22/20231103_HLTHYDRX3/raw_data/G3_16_2023_10_22_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G4_18_2023_10_17/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/G4_18_2023_10_17_S73_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G4_18_2023_10_17/20231103_HLTHYDRX3/raw_data/G4_18_2023_10_17_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/G4_18_2023_10_17_S73_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G4_18_2023_10_17/20231103_HLTHYDRX3/raw_data/G4_18_2023_10_17_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G5_19_2023_10_22/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/G5_19_2023_10_22_S71_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G5_19_2023_10_22/20231103_HLTHYDRX3/raw_data/G5_19_2023_10_22_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/G5_19_2023_10_22_S71_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G5_19_2023_10_22/20231103_HLTHYDRX3/raw_data/G5_19_2023_10_22_R2.fastq.gz"||X +echo -ne "\r[███████████▍····]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G6_32_2023_10_20/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/G6_32_2023_10_20_S140_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G6_32_2023_10_20/20231103_HLTHYDRX3/raw_data/G6_32_2023_10_20_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/G6_32_2023_10_20_S140_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G6_32_2023_10_20/20231103_HLTHYDRX3/raw_data/G6_32_2023_10_20_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G7_33_2023_10_24/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/G7_33_2023_10_24_S100_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G7_33_2023_10_24/20231103_HLTHYDRX3/raw_data/G7_33_2023_10_24_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/G7_33_2023_10_24_S100_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G7_33_2023_10_24/20231103_HLTHYDRX3/raw_data/G7_33_2023_10_24_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"G8_35_2023_10_20/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/G8_35_2023_10_20_S83_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/G8_35_2023_10_20/20231103_HLTHYDRX3/raw_data/G8_35_2023_10_20_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/G8_35_2023_10_20_S83_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/G8_35_2023_10_20/20231103_HLTHYDRX3/raw_data/G8_35_2023_10_20_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H1_10_2023_10_20/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/H1_10_2023_10_20_S113_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H1_10_2023_10_20/20231103_HLTHYDRX3/raw_data/H1_10_2023_10_20_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/H1_10_2023_10_20_S113_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H1_10_2023_10_20/20231103_HLTHYDRX3/raw_data/H1_10_2023_10_20_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H2_15_2023_10_17/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/H2_15_2023_10_17_S110_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H2_15_2023_10_17/20231103_HLTHYDRX3/raw_data/H2_15_2023_10_17_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/H2_15_2023_10_17_S110_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H2_15_2023_10_17/20231103_HLTHYDRX3/raw_data/H2_15_2023_10_17_R2.fastq.gz"||X +echo -ne "\r[███████████▌····]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H3_16_2023_10_23/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/H3_16_2023_10_23_S105_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H3_16_2023_10_23/20231103_HLTHYDRX3/raw_data/H3_16_2023_10_23_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/H3_16_2023_10_23_S105_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H3_16_2023_10_23/20231103_HLTHYDRX3/raw_data/H3_16_2023_10_23_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H4_18_2023_10_19/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/H4_18_2023_10_19_S99_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H4_18_2023_10_19/20231103_HLTHYDRX3/raw_data/H4_18_2023_10_19_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/H4_18_2023_10_19_S99_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H4_18_2023_10_19/20231103_HLTHYDRX3/raw_data/H4_18_2023_10_19_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H5_19_2023_10_23/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/H5_19_2023_10_23_S139_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H5_19_2023_10_23/20231103_HLTHYDRX3/raw_data/H5_19_2023_10_23_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/H5_19_2023_10_23_S139_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H5_19_2023_10_23/20231103_HLTHYDRX3/raw_data/H5_19_2023_10_23_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H6_32_2023_10_22/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/H6_32_2023_10_22_S108_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H6_32_2023_10_22/20231103_HLTHYDRX3/raw_data/H6_32_2023_10_22_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/H6_32_2023_10_22_S108_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H6_32_2023_10_22/20231103_HLTHYDRX3/raw_data/H6_32_2023_10_22_R2.fastq.gz"||X +echo -ne "\r[███████████▋····]\r" + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H7_34_2023_10_16/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/H7_34_2023_10_16_S116_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H7_34_2023_10_16/20231103_HLTHYDRX3/raw_data/H7_34_2023_10_16_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/H7_34_2023_10_16_S116_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H7_34_2023_10_16/20231103_HLTHYDRX3/raw_data/H7_34_2023_10_16_R2.fastq.gz"||X + +mkdir ${mode} -p "/cluster/project/pangolin/sampleset/"{,"H8_35_2023_10_21/"{,"20231103_HLTHYDRX3/"{,raw_data,extracted_data}}} +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/H8_35_2023_10_21_S72_R1_001.fastq.gz" "/cluster/project/pangolin/sampleset/H8_35_2023_10_21/20231103_HLTHYDRX3/raw_data/H8_35_2023_10_21_R1.fastq.gz"||X +cp -vf ${link} "/cluster/project/pangolin/bfabric-downloads/p23224/o33290_NovaSeq_231103_NOV1917/H8_35_2023_10_21_S72_R2_001.fastq.gz" "/cluster/project/pangolin/sampleset/H8_35_2023_10_21/20231103_HLTHYDRX3/raw_data/H8_35_2023_10_21_R2.fastq.gz"||X + +echo -e '\r\e[K[████████████████] done.' +if (( !ALLOK )); then + echo Some errors + exit 1 +fi; + + +mv -v /cluster/project/pangolin/sampleset/samples.20231013_HLLJMDRX3.tsv.staging /cluster/project/pangolin/sampleset/samples.20231013_HLLJMDRX3.tsv +mv -v /cluster/project/pangolin/sampleset/samples.20231020_HLNKVDRX3.tsv.staging /cluster/project/pangolin/sampleset/samples.20231020_HLNKVDRX3.tsv +mv -v /cluster/project/pangolin/sampleset/samples.20231006_HLLLTDRX3.tsv.staging /cluster/project/pangolin/sampleset/samples.20231006_HLLLTDRX3.tsv +mv -v /cluster/project/pangolin/sampleset/samples.20231103_HLTHYDRX3.tsv.staging /cluster/project/pangolin/sampleset/samples.20231103_HLTHYDRX3.tsv +mv -v /cluster/project/pangolin/sampleset/samples.20231110_HKHFMDRX3.tsv.staging /cluster/project/pangolin/sampleset/samples.20231110_HKHFMDRX3.tsv + +echo All Ok +exit 0 + diff --git a/pangolin_src/nohup.out b/pangolin_src/nohup.out new file mode 100644 index 0000000..d5f1b43 --- /dev/null +++ b/pangolin_src/nohup.out @@ -0,0 +1,168 @@ +/home/bs-pangolin/pangolin vs /data/backup +========= +Data sync +========= +/home/bs-pangolin/pangolin vs /data/backup +Handling raw-read upload requests for Viollier +deactivated +--------- +================= +Check current run +================= +No current run. +================ +Results handling +================ +no newer results +push data to vilolier -- deactivated +============= +Start new run +============= +Check batch against 20221222: +/data/backup/sampleset/samples.20230106_HN752DRX2.tsv +x 20221222 20230106 +!20230106:HN752DRX2 +Will start new job +/home/bs-pangolin/pangolin vs /data/backup +pushing recent: +/data/backup/sampleset/samples.20230106_HN752DRX2.tsv /data/backup/sampleset/samples.202302*.tsv +cut: '/data/backup/sampleset/samples.202302*.tsv': No such file or directory +.d..t...... USB_2023_01_02_da4d51fa--CP386--G4--type--opt/20230106_HN752DRX2/raw_data/ +&2 + [[ -n "$2" ]] && echo "$2" 1>&2 + exit 1 +} + +oops() { + printf '\e[33;1mOops:%s\e[0m\n' "$1" 1>&2 +} + +title() { + printf '\e[34;1m======================\n%s\n======================\e[0m\n\n' "$1" +} + +message() { + printf '\e[37;1m%s\t%s\e[0m\n' "$1" "$2" +} + +status() { + printf '\e[36;1m%s\e[0m\n' "$1" +} + + +# fetch batch description +if [[ -z "${1}" || "${1}" == "--help" ]]; then + cat < ] +Applies samplename patch after the fact. +USAGE + exit +fi + +# forced patchname ? +patchfile= +if [[ "${1}" == "--patch" ]]; then + shift + patchfile="${1}" + shift +fi + +# get batch +batchname="${1}" +byml="${sampleset}/batch.${batchname}.yaml" +[[ -r "${byml}" ]] || fail "Cannot open ${byml}" + +# get patchname (if not provided) +if [[ -z "${patchfile}" ]]; then + echo "autodetecting patchmap for ${batchname}" + # parse batch description + batch=$(<${byml}) + [[ "${batch}" =~ lab:[[:space:]]+([[:alnum:]]+) ]] || fail "Cannot find lab" $'file was:\n'"${batch}" + lab="${BASH_REMATCH[1]}" + case "$lab" in + fgcz) + echo "patch remapping for fgcz" + if [[ "${2}" == "--force" ]]; then + patchfile="${3}" + echo "forcing patchfile ${patchfile}" + else + # TODO fallback to projects.{batch}.tsv + [[ "${batch}" =~ order:[[:space:]]+o?([[:digit:]]+) ]] || fail "Cannot find order" $'file was:\n'"${batch}" + order="${BASH_REMATCH[1]}" + echo "order ${order}" + [[ "${batch}" =~ project:[[:space:]]+p?([[:digit:]]+) ]] || fail "Cannot find project" $'file was:\n'"${batch}" + project="${BASH_REMATCH[1]}" + echo "project ${project}" + patchfile="${sampleset}/patch.${project}.${order}.tsv" + fi + ;; + *) + fail "Unknown lab $lab" + ;; + esac +fi + +# look for patchfile +[[ -r "${patchfile}" ]] || fail "Cannot open ${patchfile}" +echo "patchfile: ${patchfile}" + +# patch samples in sampleset and working +declare -A map +while read old new trash; do + map[$old]="${new}" + + if [[ -e "${working}/samples/${new}" ]]; then + oops "$old -> $new already moved" + continue + fi + + longold="${old}-${batchname}" + longnew="${new}-${batchname}" + + mkdir -p "${sampleset}/${new}" + mv -v "${sampleset}/${old}/${batchname}" "${sampleset}/${new}/" + touch --reference="${sampleset}/${old}" "${sampleset}/${new}" + rmdir "${sampleset}/${old}" + + mkdir -p "${working}/samples/${new}" + mv -v "${working}/samples/${old}/${batchname}" "${working}/samples/${new}/" + + mv "${working}/samples/${new}/${batchname}/alignments/basecnt.tsv.gz"{,.old} + zcat "${working}/samples/${new}/${batchname}/alignments/basecnt.tsv.gz.old" | sed "1s/${longold}/${longnew}/g" | gzip >"${working}/samples/${new}/${batchname}/alignments/basecnt.tsv.gz" + touch --reference="${working}/samples/${new}/${batchname}/alignments/basecnt.tsv.gz.old" "${working}/samples/${new}/${batchname}/alignments/basecnt.tsv.gz" + rm "${working}/samples/${new}/${batchname}/alignments/basecnt.tsv.gz.old" + + mv "${working}/samples/${new}/${batchname}/alignments/coverage.tsv.gz"{,.old} + zcat "${working}/samples/${new}/${batchname}/alignments/coverage.tsv.gz.old" | sed "1s/${longold}/${longnew}/g" | gzip >"${working}/samples/${new}/${batchname}/alignments/coverage.tsv.gz" + touch --reference="${working}/samples/${new}/${batchname}/alignments/coverage.tsv.gz.old" "${working}/samples/${new}/${batchname}/alignments/coverage.tsv.gz" + rm "${working}/samples/${new}/${batchname}/alignments/coverage.tsv.gz.old" + + sed -si "1s/${longold}/${longnew}/g" "${working}/samples/${new}/${batchname}/alignments/REF_aln_stats.yaml" "${working}/samples/${new}/${batchname}/references/"*.fasta + sed -i "s@${old}([-/])${batchname}@${new}\{1}${batchname}@g;s/^${longold:0:6}/${longnew:0:6}/g" "${working}/samples/${new}/${batchname}/references/"*.matcher + sed -i "s/${longold}/${longnew}/g" "${working}/samples/${new}/${batchname}/references/frameshift_deletions_check.tsv" + + touch --reference="${working}/samples/${old}" "${working}/samples/${new}" + rmdir "${working}/samples/${old}" +done < "${patchfile}" + +# patch sample tsv list +mv "${sampleset}/samples.${batchname}.tsv"{,.old} +while read old batch len proto; do + new="${old}" + if [[ -z ${map[$old]} ]]; then + oops "tsv $new not in patch list" + else + new="${map[$old]}" + fi + printf "%s\t%s\t%u\t%s\n" "${new}" "${batch}" "${len}" "${proto}" +done < "${sampleset}/samples.${batchname}.tsv.old" > "${sampleset}/samples.${batchname}.tsv" +#rm "${sampleset}/samples.${batchname}.tsv.old" + +# patch projects tsv list +mv "${sampleset}/projects.${batchname}.tsv"{,.old} +while read old project order folder plate; do + new="${old}" + if [[ -z ${map[$old]} ]]; then + oops "tsv $new not in patch list" + else + new="${map[$old]}" + fi + printf "%s\t%s\t%s\t%s\t%s\n" "${new}" "${project}" "${order}" "${folder}" "${plate}" +done < "${sampleset}/projects.${batchname}.tsv.old" > "${sampleset}/projects.${batchname}.tsv" +#rm "${sampleset}/projects.${batchname}.tsv.old" diff --git a/profiles/custom-lsf/bsub_wrap.sh b/pangolin_src/profiles/custom-lsf/bsub_wrap.sh similarity index 100% rename from profiles/custom-lsf/bsub_wrap.sh rename to pangolin_src/profiles/custom-lsf/bsub_wrap.sh diff --git a/profiles/custom-lsf/config.yaml b/pangolin_src/profiles/custom-lsf/config.yaml similarity index 100% rename from profiles/custom-lsf/config.yaml rename to pangolin_src/profiles/custom-lsf/config.yaml diff --git a/pangolin_src/profiles/custom-lsf/status.sh b/pangolin_src/profiles/custom-lsf/status.sh new file mode 100755 index 0000000..93c015e --- /dev/null +++ b/pangolin_src/profiles/custom-lsf/status.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +#RXJOB='Job <([[:digit:]]+)> is submitted' +RXJOB='<([[:digit:]]+)>' + +if [[ "$*" =~ $RXJOB ]]; then + J="${BASH_REMATCH[1]}" +else + echo "Cannot find JobID in '$*'" >&2 + exit +fi + +state="$(bjobs $J | gawk -v I=$J '$1==I{print $3}')" + +case "${state}" in + EXIT) + echo "failed" + ;; + DONE) + echo "success" + ;; + RUN) + echo "running" + ;; + PEND) + echo "running" + ;; + *) + echo "Weird status ${state}" >&2 + echo "running" + ;; +esac + +exit 0 diff --git a/profiles/smk-simple-slurm/config.yaml b/pangolin_src/profiles/smk-simple-slurm/config.yaml similarity index 93% rename from profiles/smk-simple-slurm/config.yaml rename to pangolin_src/profiles/smk-simple-slurm/config.yaml index 165472d..673eeaf 100644 --- a/profiles/smk-simple-slurm/config.yaml +++ b/pangolin_src/profiles/smk-simple-slurm/config.yaml @@ -1,6 +1,6 @@ cluster: mkdir -p cluster_logs/{rule} && - sbatch + ${{SNAKEMAKE_PROFILE:-../profiles}}/smk-simple-slurm/sbatch_wrap.sh --parsable --job-name=COVID-vpipe-{rule}-{wildcards} --cpus-per-task={threads} diff --git a/profiles/smk-simple-slurm/config.yaml.orig b/pangolin_src/profiles/smk-simple-slurm/config.yaml.orig similarity index 100% rename from profiles/smk-simple-slurm/config.yaml.orig rename to pangolin_src/profiles/smk-simple-slurm/config.yaml.orig diff --git a/profiles/smk-simple-slurm/sbatch_wrap.sh b/pangolin_src/profiles/smk-simple-slurm/sbatch_wrap.sh similarity index 100% rename from profiles/smk-simple-slurm/sbatch_wrap.sh rename to pangolin_src/profiles/smk-simple-slurm/sbatch_wrap.sh diff --git a/pangolin_src/profiles/smk-simple-slurm/status-sacct.sh b/pangolin_src/profiles/smk-simple-slurm/status-sacct.sh new file mode 100755 index 0000000..46564e0 --- /dev/null +++ b/pangolin_src/profiles/smk-simple-slurm/status-sacct.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +# Check status of Slurm job + +jobid="$1" + +if [[ "$jobid" == Submitted ]] +then + echo smk-simple-slurm: Invalid job ID: "$jobid" >&2 + echo smk-simple-slurm: Did you remember to add the flag --parsable to your sbatch call? >&2 + exit 1 +fi + +read output other < <(sacct -j "$jobid" --format State --noheader) + +if [[ $output =~ ^(COMPLETED).* ]] +then + echo success +elif [[ $output =~ ^(RUNNING|PENDING|COMPLETING|CONFIGURING|SUSPENDED).* ]] +then + echo running +else + echo failed +fi diff --git a/pangolin_src/purge_sftp.sh b/pangolin_src/purge_sftp.sh new file mode 100755 index 0000000..9db4cec --- /dev/null +++ b/pangolin_src/purge_sftp.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +configfile=config/server.conf + +usage() { echo "Usage: $0 [ -d ] [ -i ] [ -r ] [-c ] [filter [...]]" 1>&2; exit $1; } + +dryrun=0 +ignoredownload=0 +while getopts "r:c:dih" o; do + case "${o}" in + c) configfile=${OPTARG} + if [[ ! -r ${configfile} ]]; then + echo "Cannot read ${configfile}" 1>&2 + usage 1 + fi + ;; + d) dryrun=1 ;; + i) ignoredownload=1 ;; + r) recent=${OPTARG} + if [[ ! ${recent} =~ ^20[[:digit:]]{2}-?([01][[:digit:]]-?([0-3][[:digit:]])?)?$ ]]; then + echo "Wrong format ${recent}" 1>&2 + usage 2 + fi + ;; + h) usage 0 ;; + *) usage 1 ;; + esac +done +shift $((OPTIND-1)) + + +. ${configfile} + +: ${fileserver:?} +# ${srvport} +: ${expname:?} +: ${basedir:=$(pwd)} +: ${download:?} + + +if [[ $( < ~/.netrc ) =~ machine[[:space:]]+${fileserver}[[:space:]]?login[[:space:]]+([^[:space:]]+) ]]; then + username="${BASH_REMATCH[1]}" +elif [[ $( < ~/.ssh/config ) =~ Host[[:space:]]+${fileserver}[[:space:]]+ ]]; then + username= +else + echo "cannot find login for machine ${fileserver} in ~/.netrc" >&2 + exit 1 +fi + +du_option= +if (( ${#@} )); then + dir=( "${@/#/ /${expname}/}" ) + source="${dir[*]}" + if (( ${#dir[@]} > 1)); then + du_option='c' + fi +else + source="/${expname}/" +fi + +if [[ -z "${recent}" ]]; then + echo -e "\e[31;1mWarning, no date limit set to purge ${fileserver}\e[0m" +else + echo -e "\e[37;1mPurging older than ${recent} from ${fileserver}\e[0m" +fi + + +purgelist=( ) +for d in $(lftp -c "connect sftp://${username:+${username}@}${fileserver}${srvport:+:${srvport}}; cls -q1 ${source};"); do + b=$(basename "${d}") + if [[ ! -d "${basedir}/${download}/${d}" ]] && (( ! ignoredownload )); then + echo "${d} not present!" + continue + elif [[ ! "${b}" =~ ^(20[[:digit:]]{2})-?([01][[:digit:]])-?([0-3][[:digit:]]) ]]; then + echo "${b} cannot parse" + continue + elif [[ -n "${recent}" && "${b}" > "${recent}" ]]; then # lexicographic order, so it works with subsets + echo "${b} skip" + else + purgelist+=( "${d}" ) + echo "${b} to be purged" + fi +done + +if (( ${#purgelist[@]} > 0 )) && [[ -n "${purgelist[*]}" ]]; then + if [[ "${purgelist[*]}" =~ [\;\"\'] ]]; then + echo "<${purgelist[*]}> invalid character ${BASH_REMATCH[0]}" + exit 1 + fi + if (( dryrun )); then + echo "Dry-run, not actually deleting" + exit 0 + fi + exec lftp -c "connect sftp://${username:+${username}@}${fileserver}${srvport:+:${srvport}}; du -s${du_option}h ${source}; rm -r ${purgelist[*]}; sleep 5s; du -s${du_option}h ${source};" +else + echo "Nothing to do" + exec lftp -c "connect sftp://${username:+${username}@}${fileserver}${srvport:+:${srvport}}; du -s${du_option}h ${source};" +fi diff --git a/pangolin_src/purge_sftp_fasta.sh b/pangolin_src/purge_sftp_fasta.sh new file mode 100755 index 0000000..90b9f10 --- /dev/null +++ b/pangolin_src/purge_sftp_fasta.sh @@ -0,0 +1,95 @@ +#!/bin/bash + +configfile=config/server.conf + +usage() { echo "Usage: $0 [ -d ] [ -r ] [-c ] [filter [...]]" 1>&2; exit $1; } + +dryrun=0 +while getopts "r:c:dh" o; do + case "${o}" in + c) configfile=${OPTARG} + if [[ ! -r ${configfile} ]]; then + echo "Cannot read ${configfile}" 1>&2 + usage 1 + fi + ;; + d) dryrun=1 ;; + r) recent=${OPTARG} + if [[ ! ${recent} =~ ^20[[:digit:]]{2}([01][[:digit:]]([0-3][[:digit:]])?)?$ ]]; then + echo "Wrong format ${recent}" 1>&2 + usage 2 + fi + ;; + h) usage 0 ;; + *) usage 1 ;; + esac +done +shift $((OPTIND-1)) + + +. ${configfile} + +: ${fileserver:?} +# ${srvport} +: ${expname:?} +: ${basedir:=$(pwd)} +: ${download:?} + + +if [[ $( < ~/.netrc ) =~ machine[[:space:]]+${fileserver}[[:space:]]?login[[:space:]]+([^[:space:]]+) ]]; then + username="${BASH_REMATCH[1]}" +elif [[ $( < ~/.ssh/config ) =~ Host[[:space:]]+${fileserver}[[:space:]]+ ]]; then + username= +else + echo "cannot find login for machine ${fileserver} in ~/.netrc" >&2 + exit 1 +fi + +du_option= +if (( ${#@} )); then + dir=( "${@/#/ /${expname}/}" ) + source="${dir[*]/%//*.fas*}" + if (( ${#dir[@]} > 1)); then + du_option='c' + fi +else + source="/${expname}/*.fas*" +fi + +if [[ -z "${recent}" ]]; then + echo -e "\e[31;1mError no date limit set to purge ${fileserver}\e[0m" + exit 2 +else + echo -e "\e[37;1mPurging older than ${recent} from ${fileserver}\e[0m" +fi + + +purgelist=( ) +for f in $(lftp -c "connect sftp://${username:+${username}@}${fileserver}${srvport:+:${srvport}}; cls -q1 ${source};"); do + b=$(basename "${f}") + if [[ ! "${f}" =~ -(20[[:digit:]]{2}[01][[:digit:]][0-3][[:digit:]]_[[:alnum:]]{4,})\.fas(ta)?$ ]]; then + echo "${b} cannot parse" + continue + elif [[ -n "${recent}" && "${BASH_REMATCH[1]}" > "${recent}" ]]; then # lexicographic order, so it works with subsets + : # echo "${b} skip" + else + purgelist+=( "${f}" ) + echo "${b} to be purged" + [[ -f "${basedir}/${download}/${f}" ]] && rm "${basedir}/${download}/${f}" + fi +done + +if (( ${#purgelist[@]} > 0 )) && [[ -n "${purgelist[*]}" ]]; then + if [[ "${purgelist[*]}" =~ [\;\"\'] ]]; then + echo "<${purgelist[*]}> invalid character ${BASH_REMATCH[0]}" + exit 1 + fi + if (( dryrun )); then + echo "Dry-run, not actually deleting" + exit 0 + fi + exec lftp -c "connect sftp://${username:+${username}@}${fileserver}${srvport:+:${srvport}}; du -s${du_option}h ${dir[*]}; rm -r ${purgelist[*]}; sleep 5s; du -s${du_option}h ${dir[*]};" +else + echo "Nothing to do" + exec lftp -c "connect sftp://${username:+${username}@}${fileserver}${srvport:+:${srvport}}; du -s${du_option}h ${dir[*]};" +fi diff --git a/pangolin_src/quasimodo.sh b/pangolin_src/quasimodo.sh new file mode 100755 index 0000000..1f7a002 --- /dev/null +++ b/pangolin_src/quasimodo.sh @@ -0,0 +1,82 @@ +#!/bin/bash + + +usage() { echo "Usage: $0 -s + -s : singleshot - stops at the first loop if it fails + -h : this help" 1>&2; exit $1; } + +singleshot=0 +while getopts "sh" o; do + case "${o}" in + s) singleshot=1 ;; + h) usage 0 ;; + *) usage 1 ;; + esac +done + +scriptdir=/app/pangolin_src +. ${scriptdir}/config/server.conf +runtimeout=3600 +shorttimeout=300 + +ring_carillon() { + now=$(date '+%Y%m%d') + newtimeout=$(timeout -k 5 -s INT ${shorttimeout} grep -oP '(?<=^runtimeout=).*$' config/server.conf) + if [[ -n "${runtimeout}" ]]; then + runtimeout=${newtimeout} + fi + if timeout -k 5 -s INT ${shorttimeout} touch b0rk && [[ -f b0rk ]]; then + rm b0rk + else + echo "Aargh: problem writing on storage !!!" + # TODO use carillon phases + ${scriptdir}/belfry.sh df + cluster_user="${USER%%@*}" + cluster_user=$(timeout -k 5 -s INT ${shorttimeout} grep -oP '(?<=^cluster_user=).*$' config/server.conf) + remote_batman="ssh -o StrictHostKeyChecking=no -ni ${HOME}/.ssh/id_ed25519_batman -l ${cluster_user} euler.ethz.ch --" + timeout -k 5 -s INT $shorttimeout ${remote_batman} df + date -R + return 1 + fi + # run the carrillon script + echo "Starting loop for: $runtimeout sec" + timeout -k 5 -s INT $runtimeout ${scriptdir}/carillon.sh | tee -a ${statusdir}/carillon/carillon_${now}.log + local retval=$? + timeout -k 5 -s INT $shorttimeout touch ${statusdir}/loop_done + + # report NFS status + #dmesg -LTk| grep -P 'nfs:.*server \S* (OK|not responding)' --colour=always|tail -n 1 + + return $retval +} + + +stopfile="${statusdir}/stop" + +# remove previous abort file +if [[ -e "${stopfile}" ]]; then + echo "(removing previous stop file)" + rm "${stopfile}" +fi + +echo 'First run...' +ring_carillon || (( singleshot == 0 )) || exit 1 + +while sleep 1200; do + # re-enter directory (in case a NFS crash has rendered the previous CWD handle stale) + workpath=$(dirname $(which $0)) + cd ~ + cd "${workpath}" + + echo 'loop...' + #######/usr/bin/kinit -l 1h -k -t $HOME/$USER.keytab ${USER%@*}@D.ETHZ.CH; + ring_carillon + + # check for abort + if [[ -e "${stopfile}" ]]; then + rm "${stopfile}" + exit 0 + fi + + date -R; +done diff --git a/rotate b/pangolin_src/rotate similarity index 100% rename from rotate rename to pangolin_src/rotate diff --git a/pangolin_src/rsyncd.conf b/pangolin_src/rsyncd.conf new file mode 100644 index 0000000..6fc0aed --- /dev/null +++ b/pangolin_src/rsyncd.conf @@ -0,0 +1,53 @@ +read only = no +use chroot = no +transfer logging = true +log format = %h %o %f %l %b +log file = log/rsyncd.log + +charset = utf8 + +uid = bs-pangolin +auth users = dryak +secrets file = rsyncd.secrets + +[sampleset] + path = /cluster/project/pangolin/sampleset/ + comment = Samples set + auth users = belfry:wo batman:ro dryak:rw ceciliav:ro auguste:ro + incoming chmod = Dg+s,ug+rw,o-rwx + +[working] + path = /cluster/project/pangolin/working/ + comment = Working directory + auth users = belfry:ro batman:ro dryak:rw ceciliav:ro auguste:ro + outgoing chmod = Dg+s,ug+rw,o-rwx + +[catchup] + path = /cluster/project/pangolin/work-catchup-dehuman/ + comment = Workdir for catchup + auth users = belfry:ro batman:ro dryak:rw + outgoing chmod = Dg+s,ug+rw,o-rwx + +[work_viloca] + path = /cluster/project/pangolin/work-viloca/ + comment = Workdir for Viloca + auth users = belfry:wo batman:ro dryak:rw + incoming chmod = Dg+s,ug+rw,o-rwx + +[remote_status] + path = /cluster/project/pangolin/status/ + comment = Workdir for sync status files + auth users = belfry:ro batman:ro dryak:rw + outgoing chmod = Dg+s,ug+rw,o-rwx + +[bfabric_downloads] + path = /cluster/project/pangolin/bfabric-downloads + comment = Workdir for sync bfabric raw data + auth users = belfry:ro batman:ro dryak:rw + outgoing chmod = Dg+s,ug+rw,o-rwx + +[work_uploader] + path = /cluster/project/pangolin/work-uploader/ + comment = Workdir for the uploader + auth users = belfry:wo batman:ro dryak:rw + incoming chmod = Dg+s,ug+rw,o-rwx \ No newline at end of file diff --git a/pangolin_src/setup.rst b/pangolin_src/setup.rst new file mode 100644 index 0000000..e69de29 diff --git a/pangolin_src/setup/.gitignore b/pangolin_src/setup/.gitignore new file mode 100644 index 0000000..28ad299 --- /dev/null +++ b/pangolin_src/setup/.gitignore @@ -0,0 +1 @@ +Miniconda3*.sh diff --git a/conda-qa_report.yaml b/pangolin_src/setup/conda-qa_report.yaml similarity index 100% rename from conda-qa_report.yaml rename to pangolin_src/setup/conda-qa_report.yaml diff --git a/conda_pybis_env.yaml b/pangolin_src/setup/conda_pybis_env.yaml similarity index 95% rename from conda_pybis_env.yaml rename to pangolin_src/setup/conda_pybis_env.yaml index eb49e65..362edb4 100644 --- a/conda_pybis_env.yaml +++ b/pangolin_src/setup/conda_pybis_env.yaml @@ -15,6 +15,7 @@ dependencies: - requests - urllib3 - click + - python-dateutil #- zope.interface #- certifi #- chardet diff --git a/pangolin_src/setup/conda_sendcrypt.yaml b/pangolin_src/setup/conda_sendcrypt.yaml new file mode 100644 index 0000000..5def7ef --- /dev/null +++ b/pangolin_src/setup/conda_sendcrypt.yaml @@ -0,0 +1,12 @@ +name: sendcrypt +channels: + - defaults + - conda-forge + - bioconda +dependencies: + - python + - pandas + - numpy + - git + - gnupg + - zip \ No newline at end of file diff --git a/conda_vineyard.yaml b/pangolin_src/setup/conda_sync.yaml similarity index 64% rename from conda_vineyard.yaml rename to pangolin_src/setup/conda_sync.yaml index 0a0c4fe..675210b 100644 --- a/conda_vineyard.yaml +++ b/pangolin_src/setup/conda_sync.yaml @@ -1,11 +1,8 @@ -name: vineyard +name: sync channels: - defaults - conda-forge - - bioconda dependencies: - python>=3.10 - - psycopg2 - - pyyaml - lftp - - curl + - pyyaml diff --git a/pangolin_src/setup/conda_viloca.yaml b/pangolin_src/setup/conda_viloca.yaml new file mode 100644 index 0000000..69aeb4f --- /dev/null +++ b/pangolin_src/setup/conda_viloca.yaml @@ -0,0 +1,8 @@ +name: viloca +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - libshorah + - snakemake=7.30.2 diff --git a/pangolin_src/setup/setup.sh b/pangolin_src/setup/setup.sh new file mode 100755 index 0000000..2c9cb28 --- /dev/null +++ b/pangolin_src/setup/setup.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +set -eu + +if ! test -d /app/miniconda3; then + URL="https://repo.anaconda.com/miniconda/Miniconda3-py310_22.11.1-1-Linux-x86_64.sh" + SCRIPT=$(basename $URL) + test -f ${SCRIPT} || wget $URL + bash ${SCRIPT} -b -p /app/miniconda3 +fi + + +# activate conda from .bashrc +# >>> conda initialize >>> +export PATH="/app/miniconda3/bin:$PATH" +echo $PATH +# <<< conda initialize <<< +conda init + +conda install --yes -c conda-forge mamba + +for ENV in /app/setup/conda*.yaml; do + echo create env for $ENV + mamba env create -f $ENV + echo +done + + + +echo Installing and setting up SendCryptCli for submission to SPSP +bash -c "$(wget -qO- https://gitlab.sib.swiss/clinbio/sendcrypt/sendcrypt-cli/-/raw/main/tools/install.sh)" +echo -e 'export PATH="$HOME/.sendcrypt:$PATH"' >> ~/.bashrc +source ~/.bashrc + + diff --git a/pangolin_src/setup/setup_cluster.txt b/pangolin_src/setup/setup_cluster.txt new file mode 100644 index 0000000..9ebbe13 --- /dev/null +++ b/pangolin_src/setup/setup_cluster.txt @@ -0,0 +1,17 @@ + +/cluster/project/pangolin/s3c_setup/ + - working + - sampleset + - batman.sh + - V-pipe (cloned from https://github.com/cbg-ethz/V-pipe.git, rubicon branch) + - miniconda + - env conda_sync + - env conda_sendcrypt + - rsyncd.conf + - fgcz.conf + - secret fgcz-gstore.uzh.ch + - secret gpg keys + - gpg_key_secrets + - id_ed25519_wisedb.pub + - Add the public key to the list of authorized keys + - diff --git a/pangolin_src/sort_h2030.sh b/pangolin_src/sort_h2030.sh new file mode 100755 index 0000000..2d3dba3 --- /dev/null +++ b/pangolin_src/sort_h2030.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +configfile=config/server.conf + +usage() { echo "Usage: $0 [-c ] " 1>&2; exit $1; } + +while getopts "c:h" o; do + case "${o}" in + c) configfile=${OPTARG} + if [[ ! -r ${configfile} ]]; then + echo "Cannot read ${configfile}" 1>&2 + usage 2 + fi + ;; + h) usage 0 ;; + *) usage 2 ;; + esac +done +shift $((OPTIND-1)) + +duplicates=1 + +. ${configfile} + +: ${basedir:=$(pwd)} +: ${download:?} +: ${sampleset:?} + + + +if (( ${#@} < 1)); then + usage 2 +fi + +rx_runfold='^([[:digit:]]{2}[01][[:digit:]][0-3][[:digit:]])_([[:alnum:]]+)_([[:digit:]]+)_(0+-|[AB])([[:alnum:]]+)' # e.g.: 200430_M01761_0414_000000000-J3JCT or 201023_A00730_0259_BHTVCCDRXX +# ^-[1]: date ^-[2]: instr ^-[3]: run num ^-(prefix)^-[5]: flowcell + +success=1 +for d in "${@}"; do + if [[ ! -d "${basedir}/${download}/${d}" ]]; then + echo "No ${d}" 1>&2 + success=0 + continue + fi + csum=( "${basedir}/${download}/${d}/"*.check ) + if [[ "${csum[*]}" =~ [\*\[] ]]; then + echo "No successful check sum in ${d}" 1>&2 + success=0 + continue + fi + + # 552 2021-01-25 23:30:27 ./sort_samples_dumb -f sftp-health2030/210122_A00485_0093_BH2FM5DRXY/ -s '_H2FM5DRXY' -b 20210121_fused -o sample-test/ + # 613 2021-01-29 16:42:24 ./sort_samples_dumb -f sftp-health2030/210128_A00485_0096_AH27VTDRXY/ -b 20210128_H27VTDRXY -o sample-test/ + + if [[ ${d} =~ ${rx_runfold} ]]; then + flowcell="${BASH_REMATCH[5]}" + batch="20${BASH_REMATCH[1]}_${flowcell}" + quiet='' + dupes='' + if [[ -r "${sampleset}/samples.${batch}.tsv" ]]; then + quiet='-q' + echo "${batch} : ${d}" + fi + if (( duplicates )); then + dupes='-D' + fi + ./sort_samples_dumb ${quiet} ${dupes} -f ${basedir}/${download}/${d} -b "${batch}" -g -o "${sampleset}" ${mode:+-m ${mode}}|| success=0 + echo "type: h2030 +lab: h2030 +libkit: Illumina_COVIDSeq +fused: false +runfolder: ${d%%/}" > ${sampleset}/batch.${batch}.yaml + + echo -ne "\e[35;1m" + (gawk '$3!=""{print;l=$3};$3==""{print $0 l;print "Sample "$1"-"$2" is empty">"/dev/stderr"}' ${sampleset}/samples.${batch}.tsv > ${sampleset}/samples.${batch}.tsv.patched) 2>&1 + echo -ne "\e[0m" + mv ${sampleset}/samples.${batch}.tsv{.patched,} + else + echo "Can't parse ${d}" + fi +done + +if (( success )); then + echo "All ok" + exit 0 +else + echo "Some problems" + exit 1 +fi diff --git a/sort_samples_bfabric_tsv b/pangolin_src/sort_samples_bfabric_tsv.py similarity index 93% rename from sort_samples_bfabric_tsv rename to pangolin_src/sort_samples_bfabric_tsv.py index 2915dbd..38a7e29 100755 --- a/sort_samples_bfabric_tsv +++ b/pangolin_src/sort_samples_bfabric_tsv.py @@ -26,7 +26,7 @@ def bar(v, m=128): # parse command line argparser = argparse.ArgumentParser(description="Fetch metadata from bfabric relying on the built-in metadata.tsv") argparser.add_argument('-c', '--config', metavar='CONF', required=False, - default='server.conf', + default='config/server.conf', type=str, dest='config', help="configuration file to load") argparser.add_argument('-f', '--force', required=False, action='store_true', dest='force', help="Force overwriting any existing file when moving") @@ -82,9 +82,10 @@ def load_proto(protoyaml): with open(args.config) as f: config.read_string(f""" [DEFAULT] lab={os.path.splitext(os.path.basename(args.config))[0]} -basedir={os.getcwd()} -sampleset=sampleset -download=bfabric-downloads +basedir=/cluster/project/pangolin +basedir_test=/cluster/project/pangolin/test_automation/pangolin/pangolin_src +sampleset=/cluster/project/pangolin/sampleset +download=/cluster/project/pangolin/bfabric-downloads link=--link mode= badlist= @@ -100,6 +101,8 @@ def load_proto(protoyaml): '''name of the lab, to put in the batch YAML''' basedir=config['_']['basedir'].strip("\"'") '''base dircetory''' +basedir_test=config['_']['basedir_test'].strip("\"'") +'''test base directory''' expname=config['_']['expname'].strip("\"'") '''projects name in SFTP''' download=config['_']['download'].strip("\"'") @@ -225,7 +228,7 @@ def load_proto(protoyaml): if not os.path.isfile(j): # (newest style) multiple stat files, one for each barcode format in the run # We need to identify which one is related to our files based on dataset.tsv - statsfiles = glob.glob(os.path.join(srch,'DmxStats','Stats_*.standard.json')) + statsfiles = glob.glob(os.path.join(srch,'DmxStats','Stats_*.json')) if len(statsfiles) == 0: continue else: @@ -240,7 +243,13 @@ def load_proto(protoyaml): # build the stats filename based on the barcode lengths j = os.path.join(srch, 'DmxStats', f'Stats_i1-{barcode1}_i2-{barcode2}.standard.json') if not os.path.isfile(j): - continue + # After 13-05-2024 FGCZ updated the filename structure of the Stats json file + j = os.path.join(srch, 'DmxStats', f'Stats_L1_i1-{barcode1}_i2-{barcode2}.json') + if not os.path.isfile(j): + #After 24-06-2024 FGCZ updated the filename structure of the Stats json file for Aviti sequencing + j = os.path.join(srch, 'DmxStats', f'Stats_L1_L2_i1-{barcode1}_i2-{barcode2}.json') + if not os.path.isfile(j): + continue if name in badlist: print(f"\x1b[35;1mskipping {name} in bad list\x1b[0m") @@ -312,26 +321,40 @@ def load_proto(protoyaml): namematch={} with open(j, 'rt') as f: - stats = json.loads(f.read()); + try: + stats = json.loads(f.read()); + except json.JSONDecodeError as e: + print(f"\x1b[31;1mError: Cannot parse JSON file {j}, Invalid JSON syntax:\x1b[0m", e) + continue # parse flowcell try: m=rxcell.search(stats['Flowcell']).groupdict() flowcell=m['cell'] except: - print(f"{name} cannot parse: {stats['Flowcell']}") - continue + try: + m=rxcell.search(stats['FlowCellID']).groupdict() + flowcell=m['cell'] + except: + print(f"{name} cannot parse: {stats.get('Flowcell')}") + continue # parse run folder - runfolder=stats['RunId'] + try: + runfolder=stats['RunId'] + except: + runfolder=stats['RunID'] try: m=rxrun.search(runfolder).groupdict() rundate=f"20{m['date']}" # NOTE runfolders are yymmdd, not yyyymmdd if flowcell != m['cell']: - print(f"{name} Warning: cell missmatch: {flowcell} vs {m['cell']}") + print(f"{name} Warning: cell missmatch: {flowcell} vs {m.get('cell')}") except: - print(f"{name} cannot parse: {runfolder}") - continue + try: + rundate=name.replace("-","").split("_")[2] + except: + print(f"{name} cannot parse: {runfolder}") + continue order2runfolders[order]=list(set([runfolder] + order2runfolders.get(order, []))) # skip older @@ -347,19 +370,38 @@ def load_proto(protoyaml): # parse information about reads lane={} - for l in stats['ReadInfosForLanes']: # lane - lanenum=l['LaneNumber'] + try: + laneinfo=stats['ReadInfosForLanes'] + except: + laneinfo=stats["Lanes"] + for l in laneinfo: # lane + try: + lanenum=l['LaneNumber'] + except: + lanenum=l["Lane"] ends=rlen=0 - for r in l['ReadInfos']: # read phases (indexes, reads) - if r['IsIndexedRead']: continue + try: + readinfo=l['ReadInfos'] + except: + readinfo=l["Reads"] + for r in readinfo: # read phases (indexes, reads) + try: + if r['IsIndexedRead']: continue + except: + if args.verbose: + print("Cannot find field 'IsIndexedRead' in the Stats file. If this is Aviti, this is not an error") # sanity check - if rlen and rlen != r['NumCycles']: + try: + ncycles=r['NumCycles'] + except: + ncycles=len(r['Cycles']) + if rlen and rlen != ncycles: print(f"{name} Warning: read lenght changes from {rlen} to {r['NumCycles']} we currently only support symetric read lenghts") # gather info ends+=1 - if rlen < r['NumCycles']: rlen=r['NumCycles'] + if rlen < ncycles: rlen=ncycles # sanity check if ends < 1 or ends > 2: @@ -371,6 +413,7 @@ def load_proto(protoyaml): samples={} badyield=0 badsamples=set() + for l in stats['ConversionResults']: # lane lanenum=l['LaneNumber'] ends=lane[lanenum]['ends'] @@ -499,7 +542,7 @@ def load_proto(protoyaml): # shell script file with all moving instructions inside -sh=open(os.path.join(basedir,sampleset,'movedatafiles.sh'), 'wt') +sh=open(os.path.join(basedir_test,'movedatafiles.sh'), 'wt') # generic header: only for stand-alone files. print(r''' diff --git a/pangolin_src/sort_samples_bfabric_tsv_aviti.py b/pangolin_src/sort_samples_bfabric_tsv_aviti.py new file mode 100755 index 0000000..163141e --- /dev/null +++ b/pangolin_src/sort_samples_bfabric_tsv_aviti.py @@ -0,0 +1,827 @@ +#!/usr/bin/env python3 + +import sys +import os +import glob +import io +import re +import configparser +import argparse +import csv +import json +import yaml +import hashlib +import math +import datetime +import zipfile + +# progress bar unicode +def bar(v, m=128): + f=v&7 + return ('\u2588' * (v >> 3))+(chr(0x2590 - f) if f else '')+('\u00b7' * ((m-v) >> 3)) + +# column in dataset.tsv under which we need to look for plates +plate_column = 'Tube [Characteristic]' # 'Platename [Characteristic]' +lib_column = 'LibraryPrepKit' # e.g.: "SARS-CoV-2 ARTIC V4.1 NEB Ultra II" + +# parse command line +argparser = argparse.ArgumentParser(description="Fetch metadata from bfabric relying on the built-in metadata.tsv") +argparser.add_argument('-c', '--config', metavar='CONF', required=False, + default='config/server.conf', + type=str, dest='config', help="configuration file to load") +argparser.add_argument('-f', '--force', required=False, + action='store_true', dest='force', help="Force overwriting any existing file when moving") +argparser.add_argument('-Q', '--no-fastqc', required=False, + action='store_true', dest='nofqc', help="Skip importing fastqc dirs") +argparser.add_argument('-s', '--summary', required=False, + action='store_true', dest='summary', help="Only display a summary of datasets, not an exhaustive list of all samples") +argparser.add_argument('-v', '--verbose', required=False, + action='store_true', dest='verbose', help="Explicitely list every single parse folder") +argparser.add_argument('-r', '--recent', metavar='ONLYAFTER', required=False, + dest='recent', help="Only process batches whose date is posterior to the argument") +argparser.add_argument("-4", "--protocols", metavar="PROTOCOLSYAML", required=False, default=None, + type=str, dest="protoyaml", help="Generate 4-column samples.tsv, using 'name' and 'alias' from the supplied protocols YAML file") +argparser.add_argument("-l", "--libkit-override", metavar="PROTOCOLSTSV", required=False, default=None, + type=str, dest="libkittsv", help="Patch map to override LibraryPrepKit for certain projects/orders") + +args = argparser.parse_args() + + +def load_proto(protoyaml): + """load a protocols YAML file and build a mapping of full name strings to the short keys""" + with open(protoyaml) as f: + py = yaml.load(f, Loader=yaml.BaseLoader) + pmap = {} + for k, p in py.items(): + if "name" in p: + pmap[p.get("name")] = k + for a in p.get("alias", []): + assert ( + a not in pmap + ), f"duplicate alias <{a}> in protocols YAML file <{protoyaml}>, last see in <{pmap[a]}>" + pmap[a] = k + return pmap + + +proto = load_proto(args.protoyaml) if args.protoyaml else None + + +libkitoverride = { } +if args.libkittsv: + with open(args.libkittsv,'rt',encoding='utf-8', newline='') as pf: + libkitoverride = { (prj,ordr):newval for (prj,ordr,newval,*o) in csv.reader(pf, delimiter="\t") } + + +# Load defaults from config file +config = configparser.ConfigParser(strict=False) # non-strict: support repeated section headers +config.SECTCRE = re.compile(r'\[ *(?P
[^]]+?) *\]') # support spaces in section headers +with open(args.config) as f: config.read_string(f""" +[DEFAULT] +lab={os.path.splitext(os.path.basename(args.config))[0]} +basedir=/cluster/project/pangolin +basedir_test=/cluster/project/pangolin/test_automation/pangolin/pangolin_src +sampleset=/cluster/project/pangolin/sampleset +download=/cluster/project/pangolin/bfabric-downloads +link=--link +mode= +badlist= +forcelist= +nofuselist= +fuselist= +fusedays=30 +fallbackproto= +[_] +""" + f.read()) # add defaults + a pseudo-section "_" right before the ini file, to support bash-style section_header-less config files + +lab=config['_']['lab'].strip("\"'") +'''name of the lab, to put in the batch YAML''' +basedir=config['_']['basedir'].strip("\"'") +'''base dircetory''' +basedir_test=config['_']['basedir_test'].strip("\"'") +'''test base directory''' +expname=config['_']['expname'].strip("\"'") +'''projects name in SFTP''' +download=config['_']['download'].strip("\"'") +'''sub-directory to hold the unsorted downloaded datasets''' +sampleset=config['_']['sampleset'].strip("\"'") +'''sub-directory to hold the sorted samples set''' +link=config['_']['link'].strip("\"'") +''' +linking instead of copying ? + --reflink for CoW filesystems (ZFS, BTRFS) + --hardlink for most unix-like filesystems +''' +badlist=(config['_']['badlist'].strip("\"'").split(',')) if len(config['_']['badlist']) else list() +'''folders to skip''' +forcelist=(config['_']['forcelist'].strip("\"'").split(',')) if len(config['_']['forcelist']) else list() +'''folders to use even with missing order''' +fuselist=(config['_']['fuselist'].strip("\"'").split(',')) if len(config['_']['fuselist']) else list() +'''folders to always merge''' +# nofuselist=(config['_']['nofuselist'].strip("\"'").split(',')) if len(config['_']['nofuselist']) else list() +# '''folders to never merge''' +# fusedays=int(config['_']['fusedays'].strip("\"'")) +# '''delay after which orders aren't considered for merging anymore''' + +fallbackproto=config['_']['fallbackproto'].strip("\"'") + +# parse the chmod parameter +try: + cf_mode=config['_']['mode'].strip("\"'") + mkdirmode=int(cf_mode, base=8) if cf_mode else None +except: + print(f"cannot parse <{config['_']['mode']}> as an octal chmod value. see `mkdir --help` for informations") + sys.exit(2) + +# glob all projects +if re.search('/p\d+/?$', expname): + # project name included in the SFTP path, we don't need to scan + projects='' + extrapath=0 +else: + # whole storage in SFTP path, we need to scan for projects + projects='p[0-9][0-9][0-9]*' + extrapath=1 + +# RegEx to parse some specific string +rxorder=re.compile(r'(?:^|/|_)(?Po\d+)') # match the order patter either after a "_", at the start of the string (wether it's the actual start or the initial part after a path) +rxrun=re.compile('^(?P20)?(?P\d{6})_(?P\w+)_(?P\d+)_(?P(?:0+-)|[AB])?(?P\w+(?(prefix)|-\d+))$') # e.g.: 200430_M01761_0414_000000000-J3JCT or 201023_A00730_0259_BHTVCCDRXX or 20210709_FS10001154_41_BPA73113-1417 +rxcell=re.compile('(?:0+-)?(?P\w+(?:-\d+)?)$') # e.g.: '000000000-CTT3D' or 'HTVCCDRXX' or 'BPA73113-1417' +rxsuffix=re.compile('(?:_S\d+)?(?:_L\d+)?$') # e.g.: ..._Plate_2_011120EG27_A4_S5_L001 +rxfqext=re.compile('\.fastq\.gz$') + + + +################################ +# # +# Phase 1: Gather the data # +# # +################################ + +# look for dataset files + +# FastQC +if args.verbose: + print("\x1b[37;1mLooking for FastQC folders\x1b[31;0m") +fastqc={} # maps orders to FastQC directories (or in the absence of order number: checksum of the input_dataset) +fastqc_samples={} # maps which samples are present in which FastQC directory (some might be missing) +for d in glob.glob(os.path.join(basedir,download,projects,'Fastqc_*')): + name = d.split(os.sep)[-1] + if name in badlist: + print(f"skipping {name} in bad list") + continue + t = os.path.join(d,'input_dataset.tsv'); + # FastQC_Result also listed dataset.tsv of Fastqc_ directories + if not (os.path.isdir(os.path.join(d,'FastQC_Result')) and + os.path.isfile(t)): + if args.verbose: + print(f"{name} no FastQC_Result") + continue + f = os.path.join(d.split(os.sep)[-1],'FastQC_Result') # Holds the _fastqc.html files + with open(t,'rt',encoding='utf-8', newline='') as tf: # this file has the same content as the original experiment + o=None # keep tracking of the order -> FastQC mapping + fastqc_samples[f]={} # list of files (some files didn't get their respective FastQF + for r in csv.DictReader(tf, dialect='excel-tab'): + fastqc_samples[f][r['Name']] = True + if (not o) and ('Order Id [B-Fabric]' in r): + o = r['Order Id [B-Fabric]'] + if o: + # match by Order Id, but not all have it + fastqc[o]=f + if args.verbose: + print(f"{name} - order {o}") + continue + # match by (input_) dataset.tsv content + md5_hash = hashlib.md5(usedforsecurity=False) + with open(t,'rb') as tf: + md5_hash.update(tf.read()) + fastqc[md5_hash.digest()]=f + if args.verbose: + print(f"{name} - checksum {md5_hash.digest()}") + +# Samples +if args.verbose: + print("\x1b[37;1mLooking for sample folders\x1b[31;0m") +totsam=0 +batches={} +order2runs={} # table that keeps track of orders and how many runs each has. +order2runfolders={} +plate2runs={} # table that leeps track of which run each plate has ended up in. +for srch in glob.glob(os.path.join(basedir,download,projects,'*')): + if "Aviti" in srch: + print(srch, f"\x1b[32;1mDetected Aviti sample\x1b[0m") + pathparts = srch.split(os.sep) + path = os.sep.join(pathparts) + name = pathparts[-1] + prj = pathparts[-2] if extrapath else '' + # (new style) look for bcl2fastq's json file + j = os.path.join(srch, 'Stats','Stats.json') + if not os.path.isfile(j): + # (old style) look within the Reports generated by bcl2fastq + j = os.path.join(srch, 'stats','Reports','Stats.json') + if not os.path.isfile(j): + # (newest style) multiple stat files, one for each barcode format in the run + # We need to identify which one is related to our files based on dataset.tsv + statsfiles = glob.glob(os.path.join(srch,'DmxStats','Stats_*.json')) + if len(statsfiles) == 0: + continue + else: + # We assume that the entire run uses the same barcode length for all samples + # TODO: generalize + with open(os.path.join(path,'dataset.tsv'),'rt',encoding='utf-8', newline='') as tf: + r = next(csv.DictReader(tf, dialect='excel-tab'), None) + if 'barcode1' in r: + barcode1 = len(f"{r['barcode1']}") + if 'barcode2' in r: + barcode2 = len(f"{r['barcode2']}") + if "Aviti" in srch: + #After 24-06-2024 FGCZ updated the filename structure of the Stats json file for Aviti sequencing + j = os.path.join(srch, 'DmxStats', f'Stats_L1_L2_i1-{barcode1}_i2-{barcode2}.json') + if not os.path.isfile(j): + print(j, "Cannot find the Stats file") + continue + else: + # build the stats filename based on the barcode lengths + j = os.path.join(srch, 'DmxStats', f'Stats_i1-{barcode1}_i2-{barcode2}.standard.json') + if not os.path.isfile(j): + # After 13-05-2024 FGCZ updated the filename structure of the Stats json file + j = os.path.join(srch, 'DmxStats', f'Stats_L1_i1-{barcode1}_i2-{barcode2}.json') + if not os.path.isfile(j): + #After 24-06-2024 FGCZ updated the filename structure of the Stats json file for Aviti sequencing + j = os.path.join(srch, 'DmxStats', f'Stats_L1_L2_i1-{barcode1}_i2-{barcode2}.json') + if not os.path.isfile(j): + print(srch, "Cannot find the Stats file") + continue + if name in badlist: + print(f"\x1b[35;1mskipping {name} in bad list\x1b[0m") + continue + order=name # default for corner cases when we don't have an actual order + # Load the order2runs.tsv file to find out if this order is spead over many runs... + o2rt = os.path.join(srch, 'order2runs.txt') + plates=None + if os.path.isfile(o2rt): + with open(o2rt,'rt',encoding='utf-8') as tf: + o2r_runs={} + head = re.split('[ \t]+', tf.readline().rstrip()) + for lraw in tf: + l=lraw.rstrip() + if len(l) == 0: + continue + r = dict(zip(head,re.split('[ \t]+',l))) + #for r in csv.DictReader(tf, dialect='excel-tab'): + order=o2r_order=f"o{r['Order']}" + o2r_plates=r.get('Plate','').split(';') # multiple plates split + for ru in r['Run'].split(';'): # multiple runs + o2r_runs[ru]=o2r_plates # keep track which run has which plates + if name.find(ru) != -1: # e.g.: NOV1039 is the run of name/delivery folder NovaSeq_20211119_NOV1039_o26712_DataDelivery + plates=list(set(o2r_plates + ([] if plates is None else plates))) + for p in o2r_plates: + plate2runs[p] = list(set(r['Run'].split(';') + plate2runs.get(p, []))) + order2runs[order]=o2r_runs + # try parsing order from path string + try: + m=rxorder.search(name).groupdict() + order=m['order'] + except: + if name in forcelist: + # no order but force processing + print(f"\x1b[35;1m({name} in force list) \x1b[0m", end='') + # search the dataset TSV in case we got a real order number + with open(os.path.join(path,'dataset.tsv'),'rt',encoding='utf-8', newline='') as tf: + r = next(csv.DictReader(tf, dialect='excel-tab'), None) + if 'Order Id [B-Fabric]' in r: + order = f"o{r['Order Id [B-Fabric]']}" + print(order, end='') + # otherwise we leave the above default (name) + print() + else: + print(f"\x1b[31;1mcan't parse {name}\x1b[0m") + continue + ######################################## + # # + # Parse the Demultiplex stats JSON # + # # + ######################################## + # first, retrieve the bfabric ID - sample name matches from dataset.tsv + # This must happen only if the columns exist, as older or malformed dataset.tsv files can lack the required columns + t=os.path.join(path,'dataset.tsv') + namematch=None + try: + with open(t,'rt',encoding='utf-8', newline='') as tf: + tfc=csv.DictReader(tf, dialect='excel-tab') + namematch = {cp['Sample Id [B-Fabric]']: cp['Name'] for cp in tfc} + except KeyError: + if args.verbose: + print("problems with the column matching in dataset.tsv. Excluding the bfabric ID conversion") + namematch={} + with open(j, 'rt') as f: + try: + stats = json.loads(f.read()); + except json.JSONDecodeError as e: + print(f"\x1b[31;1mError: Cannot parse JSON file {j}, Invalid JSON syntax:\x1b[0m", e) + continue + # parse flowcell + if "Aviti" in srch: + try: + m=rxcell.search(stats['FlowCellID']).groupdict() + flowcell=m['cell'] + except: + print(f"{name} cannot parse: {stats.get('Flowcell')}") + continue + else: + try: + m=rxcell.search(stats['Flowcell']).groupdict() + flowcell=m['cell'] + except: + print(f"{name} cannot parse: {stats.get('Flowcell')}") + continue + # parse run folder + if "Aviti" in srch: + runfolder=stats['RunID'] + try: + with zipfile.ZipFile(os.path.join(basedir,download,prj,name,"DmxStats","SequencerReport.zip")) as zf: + with io.TextIOWrapper(zf.open("reports_L1_L2_i1-"+str(barcode1)+"_i2-"+str(barcode2)+"/RunParameters.json")) as f: + try: + stats_zip = json.loads(f.read()); + except json.JSONDecodeError as e: + print(f"\x1b[31;1mError: Cannot parse JSON file {j}, Invalid JSON syntax:\x1b[0m", e) + continue + rundate=stats_zip["Date"].replace("-","").split("T")[0] + except: + print(f"{name} cannot parse: {runfolder}") + continue + else: + runfolder=stats['RunId'] + try: + m=rxrun.search(runfolder).groupdict() + rundate=f"20{m['date']}" # NOTE runfolders are yymmdd, not yyyymmdd + if flowcell != m['cell']: + print(f"{name} Warning: cell missmatch: {flowcell} vs {m.get('cell')}") + except: + continue + order2runfolders[order]=list(set([runfolder] + order2runfolders.get(order, []))) + # skip older + if args.recent: + if rundate < args.recent: + print(f"Skipping {rundate} {order} {flowcell}") + continue + # NOTE for this skip to work, all recplicate of an order must be in the fuselist, otherwise merge can be accidentally missed if the replicate span across a month border + # not skipped, advertise libkit forcing + if (prj, order) in libkitoverride: + print(name, f"\x1b[32;1mforcing {lib_column} of {order} {name} as {libkitoverride[(prj, order)]}\x1b[0m") + # parse information about reads + lane={} + if "Aviti" in srch: + laneinfo=stats["Lanes"] + else: + laneinfo=stats['ReadInfosForLanes'] + for l in laneinfo: # lane + if "Aviti" in srch: + lanenum=l['Lane'] + readinfo=l["Reads"] + else: + lanenum=l["LaneNumber"] + readinfo=l['ReadInfos'] + ends=rlen=0 + for r in readinfo: # read phases (indexes, reads) + if "Aviti" in srch: + ncycles=len(r['Cycles']) + try: + if "I" in r['Read']: continue + except: + print(f"\x1b[31;1mError: Invalid indication if the read is indexed or not\x1b[0m") + else: + if r['IsIndexedRead']: continue + ncycles=r['NumCycles'] + if rlen and rlen != ncycles: + print(f"{name} Warning: read lenght changes from {rlen} to {r['NumCycles']} we currently only support symetric read lenghts") + # gather info + ends+=1 + if rlen < ncycles: rlen=ncycles + # sanity check + if ends < 1 or ends > 2: + print(f"{name} Error: we currently only support single or paired ends, but found {ends} reads") + lane[lanenum]={'ends': ends, 'rlen': rlen-1} + # parse info about samples + samples={} + badyield=0 + badsamples=set() + #with zipfile.ZipFile(os.path.join(basedir,download,prj,name,"DmxStats","SequencerReport.zip")) as zf: + # with io.TextIOWrapper(zf.open("reports_L1_L2_i1-"+str(barcode1)+"_i2-"+str(barcode2)+"/RunParameters.json")) as f: + # try: + # stats_zip = json.loads(f.read()); + # except json.JSONDecodeError as e: + # print(f"\x1b[31;1mError: Cannot parse JSON file {j}, Invalid JSON syntax:\x1b[0m", e) + # continue + if "Aviti" in srch: + try: + if len(stats['Lanes']) > 1: + print("ERROR: multiple lanes processing is not implemented yet for Aviti") + continue + for l in stats['Lanes']: + lanenum=l['Lane'] + ends=lane[lanenum]['ends'] + rlen=lane[lanenum]['rlen'] + for s in stats['SampleStats']: # sample in lane + samname=s['SampleName'] + # In case the json file is listing using the bfabric IDs instead of the sample names, convert + if samname in namematch: + samname = namematch[samname] + # filter out fastq files with zero reads + if s['NumPolonies'] == 0: + badyield+=1; + badsamples.add(samname) + continue + samples[samname]={'ends': ends, 'rlen': rlen} + totsam+=1 + except: + continue + else: + for l in stats['ConversionResults']: # lane + lanenum=l['LaneNumber'] + ends=lane[lanenum]['ends'] + rlen=lane[lanenum]['rlen'] + for s in l['DemuxResults']: # sample in lane + samname=s['SampleName'] + # In case the json file is listing using the bfabric IDs instead of the sample names, convert + if samname in namematch: + samname = namematch[samname] + # filter out fastq files with zero reads + if s['NumberReads'] == 0: + badyield+=1; + badsamples.add(samname) + continue + samples[samname]={'ends': ends, 'rlen': rlen} + totsam+=1 + # Check readcounts + if badyield: + print(name, f"\x1b[33;1m{badyield} samples with bad yield !\x1b[0m", sep='\t') + # Need multiple samples + if len(samples) < 2: + print(name, f"\x1b[31;1mOnly {len(samples)}!\x1b[0m", sep='\t') + continue + # + # dataset.tsv + # + t=os.path.join(path,'dataset.tsv') + to=None + tplates=None + with open(t,'rt',encoding='utf-8', newline='') as tf: + tfc=csv.DictReader(tf, dialect='excel-tab') + r = next(tfc) + # check proto + if lib_column not in r: + print(name, f"\u001b[33;1mwarning, missing {lib_column} in <{t}>\u001b[0m") + # check plates + if plate_column in r: + tplates=list(set([r[plate_column]] + [cp[plate_column] for cp in tfc])) + # and now check for order column + if 'Order Id [B-Fabric]' in r: + to = r['Order Id [B-Fabric]'] + # + # Build per batch / samples data + # + b={'name':name, 'prj': prj, 'path':path, 'dataset':t, 'flowcell': flowcell, 'runfolder': runfolder,'rundate':rundate,'samples':samples,'badyield':badyield,'badsamples':badsamples} + # add plates, which ever method managed to find them + if tplates is not None and len(tplates)>0: + b.update({ 'plates':tplates }) + elif plates is not None and len(plates)>0: + b.update({ 'plates':plates }) + # if both attempts at counting plates were successfull, check if they agree + if tplates is not None and plates is not None: + diff=list(set(tplates).symmetric_difference(set(plates))) + if len(diff)>0: + if args.verbose: + print(name, f"\x1b[33mnon-concording plates list: dataset <{';'.join(list(set(tplates).difference(set(plates))))}< and order2run >{';'.join(list(set(plates).difference(set(tplates))))}>\x1b[0m") + else: + print(name, f"\x1b[33mnon-concording plates list: between dataset and order2run\x1b[0m") + # handle orders replicate and fuse them + if order in batches: + key=f"{order}:{name}" + days=abs(datetime.datetime.strptime(rundate, '%Y%m%d').date()-datetime.datetime.strptime(batches[order]['rundate'], '%Y%m%d').date()) // datetime.timedelta(days=1) + # single scenario / ultra-simple fuse logic: is it in list? (No more autoguessing) + if name in fuselist or batches[order]['name'] in fuselist: + print(name, f"\x1b[36;1m{rundate}_{flowcell} is REPLICATE of order {order}'s {batches[order]['name']} - {batches[order]['rundate']}_{batches[order]['flowcell']} (reason: in fuse list)\x1b[0m (note: {days} days appart)") + batches[order]['dupe']=True + b['appendto']=order + else: + print(name, f"\x1b[36mnot fusing {rundate}_{flowcell} with {order}'s {batches[order]['rundate']}_{batches[order]['flowcell']}: not in fuse list {days}\x1b[0m (note: {days} days appart)") + else: + key=order + # Now, link with FastQC, based on dataset.tsv and... + # ...based on order column if present ? + if to is not None: + # sanity check: is the folder the same? + if to in fastqc: + b['fastqc']=fastqc[to] + if to[0] != 'o': + to = f"o{to}" + if to != order: + print(name, f"\x1b[31;1mFolder order {order} vs dataset.tsv order {to}\x1b[0m") + batches[key]=b + if args.verbose: + print(f"{name}: {order} - {runfolder}") + continue + # ...match by (input_) dataset.tsv content + md5_hash = hashlib.md5(usedforsecurity=False) + with open(t,'rb') as tf: + md5_hash.update(tf.read()) + md5=md5_hash.digest() + if md5 in fastqc: + b['fastqc']=fastqc[md5] + batches[key]=b + if args.verbose: + print(f"{name}: {order} - {runfolder}") + + +################################ +# # +# Phase 2: Output the thing # +# # +################################ + +# create sampleset directory if missing +if not os.path.isdir(os.path.join(basedir,sampleset)): + try: + kwmkdir={ 'mode': mkdirmode } if mkdirmode else { } + os.mkdir(os.path.join(basedir,sampleset),**kwmkdir) + except FileExistsError: + pass + + +# shell script file with all moving instructions inside +sh=open(os.path.join(basedir_test,'movedatafiles.sh'), 'wt') + +# generic header: only for stand-alone files. +print(r''' +link='%(link)s' +mode='%(mode)s' # e.g.: --mode=0770 + +# Helper +fail() { + printf '\e[31;1mArgh: %%s\e[0m\n' "$1" 1>&2 + [[ -n "$2" ]] && echo "$2" 1>&2 + exit 1 +} + +warn() { + printf '\e[33;1mArgh: %%s\e[0m\n' "$1" 1>&2 + [[ -n "$2" ]] && echo "$2" 1>&2 +} + +ALLOK=1 +X() { + ALLOK=0 +} + +# sanity checks +[[ -d '%(sampleset)s' ]] || fail 'No sampleset directory:' '%(sampleset)s' +[[ -d '%(download)s' ]] || fail 'No download directory:' '%(download)s' +''' % {'link':link,'mode':(f"--mode={mkdirmode:04o}" if mkdirmode else ''), 'sampleset':sampleset,'download':download}, file=sh) + +# helper function to handle items that can be either single string or list (or empty) +listify = lambda x: x if type(x) is list else [] if x is None else [x] +delistify = lambda x: x if type(x) is str else None if len(x) == 0 else next(iter(x)) if len(x) == 1 else sorted(x, reverse=True) + +lastbar=-1 +cursam=0 +otsv={} # opened file handles for samples.tsv (for appending) +osprj={} # opened file handles for projects.tsv (mapping samples name to projects and orders) (for appending) +oyaml={} # old dictionnaries with previous version of batch.yaml (to be updated and used to overwrite the file) +omis={} # old list with previous bad samples (to be update and used to overwrite file) +osamseen={} # old list of previously sample seen with reads (non bad) - used to update omis +for b in batches: + name=batches[b]['name'] + prj=batches[b]['prj'] + rundate=batches[b]['rundate'] + flowcell=batches[b]['flowcell'] + + # skip older + if args.recent: + if rundate < args.recent: + totsam -= len(batches[b]['samples']) + #print(f"Skipping {rundate} {order} {flowcell}") + continue + + # either classic '20201120_JDNB4' or merged '20201124_o23391' + dupe=False + if 'dupe' in batches[b]: + dupe=True + order=b + batch=f"{rundate}_{order}" + elif 'appendto' in batches[b]: + dupe=True + order = batches[b]['appendto'] + rundate = batches[order]['rundate'] + batch=f"{rundate}_{order}" + elif name in fuselist: + # Check that this match expectations of fuselist + print(b, f"\x1b[31;1m should have a replicate according to fuselit but only {name} - {rundate}_{flowcell} found, no replicate found - dropping folder!\x1b[0m") + continue + else: + order=b + batch=f"{rundate}_{flowcell}" + + print(r"[[ -d '%(download)s/%(prj)s/%(id)s' ]] || fail 'Not a directory:' '%(download)s/%(prj)s/%(id)s'" % {'download':download,'prj':prj,'id':name}, file=sh) + qcdir=None + if (not args.nofqc) and (not dupe) and ('fastqc' in batches[b]): + qcdir=batches[b]['fastqc'] + print(r"[[ -d '%(download)s/%(prj)s/%(qc)s' ]] || fail 'No download directory:' '%(download)s/%(prj)s/%(qc)s'" % {'download':download,'prj':prj,'qc': qcdir}, file=sh) + + # patch file exist ? + patchmap = { } + patchtsv=os.path.join(basedir,sampleset,f"patch.{prj.removeprefix('p')}.{order.removeprefix('o')}.tsv") + if os.path.isfile(patchtsv): + print(f"\x1b[32;1mpatching {order} {name} with {patchtsv}\x1b[0m") + with open(patchtsv,'rt',encoding='utf-8', newline='') as pf: + patchmap = { old:new for (old,new,*r) in csv.reader(pf, delimiter="\t") } + + # patch file exist ? + forcelibkit = libkitoverride.get( (prj, order), None ) + + # batch TSV file and info YAML + runfolder=None + if batch not in otsv: + otsv[batch]=tsv=open(os.path.join(basedir,sampleset,f'samples.{batch}.tsv.staging'), 'wt') + osprj[batch]=sprj=open(os.path.join(basedir,sampleset,f'projects.{batch}.tsv'), 'wt') + plts = { 'plates': delistify(batches[b]['plates']) } if batches[b].get('plates') is not None else { } + oyaml[batch]=yamldict={'type':'bfabric','lab':lab,'order':order,'project':prj,'name':name,'fused':dupe, + 'runfolder': batches[b]['runfolder'], + 'fastqcfolder': batches[b]['fastqc'] if 'fastqc' in batches[b] else None, + 'folder': batches[b]['name'], + **plts, + } + badsamples=batches[b]['badsamples'] + # keep bad lists organised by orders within each batch (as some could be mixed weirdly) + if batch not in omis: + omis[batch]={order: badsamples} + osamseen[batch]={order: set(batches[b]['samples'].keys()).difference(badsamples)} + else: + omis[batch].update({order: badsamples}) + osamseen[batch].update({order: set(batches[b]['samples'].keys()).difference(badsamples)}) + else: + tsv=otsv[batch] + sprj=osprj[batch] + # update the yaml instead of simply combining + yamldict=oyaml[batch] + combine_plts = delistify(set(listify(yamldict.get('plates', [])) + batches[b].get('plates', []))) + plts = { 'plates': combine_plts } if combine_plts is not None else { } + yamldict.update({ + 'project': delistify(set([prj] + listify(yamldict['project']))), + 'order': delistify(set([order] + listify(yamldict['order']))), + 'name': name, + #bfabfolder=[batches[b]['name'],batches[order]['name']] + 'folder': delistify(set([batches[b]['name']] + listify(yamldict['folder']))), + #runfolder=[batches[b]['runfolder'],batches[order]['runfolder']] + 'runfolder': delistify(set([batches[b]['runfolder']] + listify(yamldict['runfolder']))), + #fastqcfolder=[] + #if 'fastqc' in batches[b]: + # fastqcfolder+=[batches[b]['fastqc']] + #if 'fastqc' in batches[order]: + # fastqcfolder+=[batches[order]['fastqc']] + #if len(fastqcfolder) == 0: + # fastqcfolder=None + 'fastqcfolder': delistify(set(listify(yamldict['fastqcfolder']) + ([batches[b]['fastqc']] if 'fastqc' in batches[b] else []))), + **plts, + }) + oyaml[batch]=yamldict + + # intersections etc. + # NOTE two fused sequencing set do not necessarily contain the same samples, so the exact venn diagram gets a bit crazierer + # list of new samples *with* reads (non bad) that where added now. + newnonbadsamples=set(batches[b]['samples'].keys()).difference(batches[b]['badsamples']) + if order in omis[batch]: + omis[batch].update({order: ( + # from the old bad sample : remove those whose samples now have reads + omis[batch][order].difference(newnonbadsamples) + ).union( + # from the new bad samples : remove those whose previous sample had reads + batches[b]['badsamples'].difference(osamseen[batch][order]) + )}) + osamseen[batch][order].update(newnonbadsamples) + else: + omis[batch].update({order: batches[b]['badsamples'].difference(batches[order]['samples'].keys()) }) + osamseen[batch].update({order: newnonbadsamples }) + + # combine missing between different orders + badsamples=[ ] + for x in omis[batch].values(): + badsamples += x + + # YAML with batch infos + with open(os.path.join(basedir,sampleset,f'batch.{batch}.yaml'), 'wt') as yml: + print(yaml.dump(yamldict, sort_keys=False), file=yml) + # list of missing files + misfname=os.path.join(basedir,sampleset,f'missing.{batch}.txt') + if len(badsamples): + with open(misfname, 'wt') as missing: + missing.writelines("%s\n" % patchmap.get(bad, bad) for bad in sorted(set(badsamples))) + else: + if os.path.isfile(misfname): + os.remove(misfname) + + with open(batches[b]['dataset'],'rt',encoding='utf-8') as tf: + for r in csv.DictReader(tf, dialect='excel-tab'): + # progress + prgbar=math.floor(cursam*128/totsam) + if lastbar != prgbar: + print(f"echo -ne '\\r[{bar(prgbar)}]\\r'", file=sh) + lastbar=prgbar + cursam+=1 + + # match read TSV to known samples + fulname=samname=r['Name'] + if samname in batches[b]['samples']: + batsamname=samname + else: + olen=len(samname) + # try removing typical trailing stuff: Sample num, Lane num + samname=rxsuffix.sub('', samname) + if samname in batches[b]['samples']: + batsamname=samname + else: + # try if one of the batch's sample has a name which is a subset + slen=len(samname) + mlen=0 + matchname=None + for batsamname in batches[b]['samples']: + tlen=len(batsamname) + if tlen < slen: + if (samname[:tlen] == batsamname) and (mlen < tlen): + mlen=tlen + matchname=batsamname + else: + if (samname == batsamname[:slen]) and (mlen < slen): + mlen=slen + matchname=batsamname + if matchname != None: + print(f"{batches[b]['name']} {samname} fuzzy matched to {matchname}") + batsamname=matchname + else: + print(f"{batches[b]['name']} Can't match {samname}") + continue + if dupe: + fulname=f"{fulname}_{flowcell}" + + # use patch map to adjust name + if samname in patchmap: + samname = patchmap[samname] + + # files + ends=batches[b]['samples'][batsamname]['ends'] + rlen=batches[b]['samples'][batsamname]['rlen'] + r1=r['Read1 [File]'].split(os.sep)[-1] + if ends==2: + r2=r['Read2 [File]'].split(os.sep)[-1] + + # plate + plate = [ r[plate_column] ] if plate_column in r else [] + + # tsv line + if ('appendto' not in batches[b]) or (batsamname not in batches[order]['samples']): + tf = [samname, batch, rlen] + library = forcelibkit if forcelibkit else r.get(lib_column, None) + # HACK parse ASCII + if library == "None": + library = None + if proto: + if library: + assert ( + library in proto + ), f"Cannot find library kit <{library}> in protocols YAML file <{args.protoyaml}> while processing <{batches[b]['dataset']}>" + tf += [proto[library]] + elif fallbackproto: + tf += [fallbackproto] + print(*tf, sep="\t", file=tsv) + + # map name to project / order / folder + print(samname, prj, order, batches[b]['name'], *plate, sep='\t', file=sprj) + + # move script + print(r''' +mkdir ${mode} -p "%(sampleset)s/"{,"%(sname)s/"{,"%(batch)s/"{,raw_data,extracted_data}}} +cp -v%(force)s ${link} '%(download)s/%(prj)s/%(id)s/%(read)s' '%(sampleset)s/%(sname)s/%(batch)s/raw_data/%(destname)s'||X''' % {'force': ('f' if args.force else ''),'download':download,'prj':prj,'id':name,'sname':samname,'batch':batch,'sampleset':sampleset,'read':r1,'destname':f"{fulname}_R1.fastq.gz"}, file=sh) + if ends==2: + print(r"cp -v%(force)s ${link} '%(download)s/%(prj)s/%(id)s/%(read)s' '%(sampleset)s/%(sname)s/%(batch)s/raw_data/%(destname)s'||X" % {'force': ('f' if args.force else ''),'download':download,'prj':prj,'id':name,'sname':samname,'batch':batch,'sampleset':sampleset,'read':r2,'link':link,'destname':f"{fulname}_R2.fastq.gz"}, file=sh) + if qcdir and fulname in fastqc_samples[qcdir]: + fqc=rxfqext.sub('_fastqc.html',r1) + print(r"cp -v%(force)s ${link} '%(download)s/%(prj)s/%(fastqc)s/%(fqc)s' '%(sampleset)s/%(sname)s/%(batch)s/extracted_data/R1_fastqc.html'||X" %{'force': ('f' if args.force else ''),'download':download,'prj':prj,'fastqc':qcdir,'fqc':fqc,'sampleset':sampleset,'sname':samname,'batch':batch}, file=sh) + if ends==2: + fqc=rxfqext.sub('_fastqc.html',r2) + print(r"cp -v%(force)s ${link} '%(download)s/%(prj)s/%(fastqc)s/%(fqc)s' '%(sampleset)s/%(sname)s/%(batch)s/extracted_data/R2_fastqc.html'||X" %{'force': ('f' if args.force else ''),'download':download,'prj':prj,'fastqc':qcdir,'fqc':fqc,'sampleset':sampleset,'sname':samname,'batch':batch}, file=sh) + +print(f""" +echo -e '\\r\\e[K[{bar(128)}] done.' +if (( !ALLOK )); then + echo Some errors + exit 1 +fi; + +""", file=sh) + +for b in otsv.keys(): + print(f"mv -v {sampleset}/samples.{b}.tsv.staging {sampleset}/samples.{b}.tsv", file=sh) + +print(""" +echo All Ok +exit 0 +""", file=sh) diff --git a/pangolin_src/sort_samples_email.sh b/pangolin_src/sort_samples_email.sh new file mode 100755 index 0000000..4abc48f --- /dev/null +++ b/pangolin_src/sort_samples_email.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +date=20200409 +link=--link +target=../sampleset + +#Data Set Type: FASTQ_GZ Index: GATATCCA-GGATTAAC, External Sample Name: 100848_112_H5, Contact Person: Timothy Vaughan;Sarah Nadeau;Jeremie Scire;Sophie Seidel +#https://openbis-dsu.ethz.ch/openbis?viewMode=SIMPLE#entity=DATA_SET&permId=20200425131608350-60692023 + +truncate --size 0 ${target}/samples.tsv.new +cat e-mail.txt | while read line; do + if [[ $line =~ ^Data\ Set\ Type:\ FASTQ_GZ ]]; then + if [[ $line =~ Sample\ Name:\ ((([[:digit:]]{6}|neg)_[[:digit:]]+_[[:alpha:]][[:digit:]]{,2})|(pos|H2O|EMPTY)[^,]+) ]]; then + temp=${BASH_REMATCH[1]} + tmp2=${temp//[\.\/]/_} + name=${tmp2//__/_} + id= + continue + else + echo -e '\e[31;1mInfo line parsing error:\e[0m' + echo $line + exit 1 + fi + elif [[ $line =~ ^https ]]; then + if [[ $line =~ Id=([[:digit:]]+-[[:digit:]]+)$ ]]; then + id=${BASH_REMATCH[1]} + else + echo -e '\e[31;1mURL line parsing error:\e[0m' + echo $line + exit 1 + fi + elif [[ -z "$line" ]]; then + continue + else + id= + name= + fi + + if [[ -z "$name" || -z "$id" ]]; then + echo -e '\e[31;1mParsing fail:\e[0m' + echo $line + exit 1 + fi + +# $ 20200425131508718-60692018/original/BSSE_QGF_137972_000000000_J3JCY_1_MM_1/ : +# BSSE_QGF_137972_000000000_J3JCY_1_100845_112_G6_ACAGTTGA_GCTCCGAT_S89_L001_MM_1_metadata.tsv +# BSSE_QGF_137972_000000000_J3JCY_1_100845_112_G6_ACAGTTGA_GCTCCGAT_S89_L001_R2_001_MM_1.fastq.gz +# BSSE_QGF_137972_000000000_J3JCY_1_100845_112_G6_ACAGTTGA_GCTCCGAT_S89_L001_R1_001_MM_1.fastq.gz + + if [[ ! -d "$id" ]]; then + echo -e '\e[31;1mNot a directory:\e[0m' + echo $id + exit 1 + fi + + fastq=( ${id}/original/*/*_${name}_*_R[1-2]_*.fastq.gz ) + + if [[ "${fastq[*]}" =~ \* ]]; then + echo -e '\e[31;1mCannot list fastq files:\e[0m' + echo $id : $name + exit 1 + elif (( ${#fastq[@]} != 2 )); then + echo -e '\e[31;1mNumber of fastq files not 2:\e[0m' + echo "${#fastq[@]} : ${fastq[@]}" + exit 1 + fi + + dst=${target}/$name/$date/raw_data + mkdir -p "$dst" + for file in "${fastq[@]}"; do + destname="${file##*/}" + if [[ $file =~ _L[[:digit:]]+_R[[:digit:]](_[[:digit:]]+(_[^\.]+)?.fastq.gz)$ ]]; then + destname="${destname//${BASH_REMATCH[1]}/.fastq.gz}" + fi + cp -v ${link} "${file}" "${dst}/${destname}" + done + + echo -e "${name}\t${date}\t250" >> ${target}/samples.tsv.new +done diff --git a/sort_samples_pybis b/pangolin_src/sort_samples_pybis.py similarity index 96% rename from sort_samples_pybis rename to pangolin_src/sort_samples_pybis.py index 186c1ae..87ad51b 100755 --- a/sort_samples_pybis +++ b/pangolin_src/sort_samples_pybis.py @@ -45,7 +45,7 @@ def report_progress(prgrs, prgmax, summary, sh=None): "--config", metavar="CONF", required=False, - default="server.conf", + default="config/server.conf", type=str, dest="config", help="configuration file to load", @@ -149,7 +149,7 @@ def load_config(args): samtype=ILLUMINA_FLOW_LANE basedir={os.getcwd()} sampleset=sampleset - download=openbis-downloads + download=/mnt/cluster/openbis-downloads link=--link mode= enforce_fetching= @@ -212,11 +212,10 @@ def get_entry(name): rxcell = re.compile(r"(?<=/)(?P(?:\w+-)?(?P\w+)(?:\:(?P\d+))?)$") rxclean = re.compile( r"[\W_]+" -) # or [\-\.\:\/] : characters that are converted to (single) '_' for file-system friendliness +) # or [\-\.\:\/] : characters that are converted to '_' for file-system friendliness rxclean_new = re.compile( - r"(?:[^\-A-Za-z0-9]|(?<=[^-])-(?=[^-]))+" + r"[^\-A-Za-z0-9_]+|(?<=[^-])-(?=[^-])" ) # more recently, double dashes are not sanitized but included -# TODO we currently presume double-underscores ARE converted to single '_'. To be clarified if this code ever gets de-mothballed in the future. (To whomever is handling the '27 bird flu: good luck) rxrecent = re.compile(r"^(?P\d{4})(?:(?P\d{2})(?P\d{2})?)?") samkwargs = {} @@ -237,16 +236,25 @@ def get_entry(name): # we use ~/.netrc to obtain credentials # (we need that config file anyway to download the data files from openbis' fileserver) -username = os.environ.get("USERNAME") or netrc.netrc().authenticators(fileserver)[0] -password = os.environ.get("PASSWORD") or netrc.netrc().authenticators(fileserver)[2] -# username, password = netrc.netrc().authenticators(fileserver)[0::2] -if 0: - # console interactive - import getpass +secrets = {} +try: + with open(f"secrets/{fileserver}") as fh: + for line in fh: + line = line.strip() + if not line: + continue + key, _, value = line.partition("=") + value = value.strip('"') + secrets[key] = value +except IOError: + pass + - username = os.getlogin() - password = getpass.getpass() +username = secrets["user"] or os.environ.get("USERNAME") or netrc.netrc().authenticators(fileserver)[0] +password = secrets["password"] or os.environ.get("PASSWORD") or netrc.netrc().authenticators(fileserver)[2] + +print("connect to", apiurl) o = Openbis(apiurl, verify_certificates=True, use_cache=True) o.login( @@ -254,6 +262,7 @@ def get_entry(name): ) # save the session token in ~/.pybis/example.com.token atexit.register(o.logout) + # shell script file with all moving instructions inside sh = open(os.path.join(basedir, sampleset, "movedatafiles.sh"), "wt") print( diff --git a/pangolin_src/sort_viollier.sh b/pangolin_src/sort_viollier.sh new file mode 100755 index 0000000..cd5c5d8 --- /dev/null +++ b/pangolin_src/sort_viollier.sh @@ -0,0 +1,161 @@ +#!/bin/bash + +configfile=config/server.conf + +usage() { echo "Usage: $0 [-c ] [-r ] [ -4 ] [ ]" 1>&2; exit $1; } + +protocolsyaml= + +while getopts "c:r:4:h" o; do + case "${o}" in + c) configfile=${OPTARG} + if [[ ! -r ${configfile} ]]; then + echo "Cannot read ${configfile}" 1>&2 + usage 2 + fi + ;; + r) recent=${OPTARG} + if [[ ! ${recent} =~ ^20[[:digit:]]{2}([01][[:digit:]]([0-3][[:digit:]])?)?$ ]]; then + echo "Wrong format ${recent}" 1>&2 + usage 2 + fi + ;; + 4) protocolsyaml=${OPTARG} + if [[ ! -r ${protocolsyaml} ]]; then + echo "Cannot read ${protocolsyaml}" 1>&2 + usage 2 + fi + ;; + h) usage 0 ;; + *) usage 2 ;; + esac +done +shift $((OPTIND-1)) + +duplicates=1 + +. ${configfile} + +: ${lab:?} +: ${basedir:=$(pwd)} +: ${download:?} +: ${sampleset:?} + + + +if (( ${#@} < 1)); then + scan=( ${basedir}/${download}/*/ ) +else + scan=( "${@}" ) +fi + +rx_folder='^(20[[:digit:]]{2}[01][[:digit:]][0-3][[:digit:]])_([[:alnum:]]+)$' # e.g.: 20210528_061936 +# ^-[1]: date ^-[2]: time + +# appended batches: make shared header (cf. sort_samples_jobinfo) +# and also setup combined error status (GRANDTOTAL_ALLOK) +cat > ${basedir}/${sampleset}/movedatafiles.sh <&2 + [[ -n "\$2" ]] && echo "\$2" 1>&2 + exit 1 +} + +warn() { + printf '\e[33;1mArgh: %s\e[0m\n' "\$1" 1>&2 + [[ -n "\$2" ]] && echo "\$2" 1>&2 +} + +GRANDTOTAL_ALLOK=1 +ALLOK=1 +X() { + ALLOK=0 +} + +# sanity checks +[[ -d '${sampleset}' ]] || fail 'No sampleset directory:' '${sampleset}' +SH_HEAD + +success=1 +b=0 +for d in "${scan[@]}"; do + # progress bar + (( v = b * 128 / ${#scan[@]} )) + bar1="$(for ((i=0;i< (v>>3);++i)) do echo -n $'\u2588'; done)" + if (( (v&7) > 0 )); then + bar2="$(echo -ne "\u$(printf '%04x' $((0x2590 - (v&7) )) )")" + else + bar2='' + fi + bar3="$(for ((i=0;i< ( (128-v)>>3);++i)) do echo -n $'\u22C5' ; done)" # normally: \u00b7 + cat >> ${basedir}/${sampleset}/movedatafiles.sh <&2 + success=0 + continue + fi + # skip older + if [[ -n "${recent}" && "$(basename ${d})" < "${recent}" ]]; then + continue + fi + # folder ? + if [[ $(basename ${d}) =~ ${rx_folder} ]]; then + # check for left-over (windows') partial download files + if [[ -n "$(find "${d}" -type f -name '*.filepart' | tee /dev/stderr)" ]]; then + echo -e "\e[33;1mIncompletely downloaded files in $(basename ${d})\e[0m skipping..." + touch status/syncviollier_failed + continue + fi + # Remove irrelevant samples from sample sheet + if [[ "$(< ${d}/SampleSheetUsed.csv)" =~ [[:space:]]Error ]]; then + echo -e "Filtering out 'Error' samples" + mv ${d}/SampleSheetUsed.csv{,.err} + grep -vP '^Error' ${d}/SampleSheetUsed.csv.err > ${d}/SampleSheetUsed.csv + fi + # Search for patchmap if available + patch= + if [[ -e "${d%%/}.patchmap.tsv" ]]; then + echo "Patchmap found for $(basename ${d})" + patchmap="--patchmap=${d%%/}.patchmap.tsv" + fi + ./sort_samples_jobinfo --summary --sourcedir=${d} ${patchmap} --outdir=${basedir}/${sampleset} ${mode:+--mode=${mode}} --force --forcelanes --batch ${lab} --append --staging ${protocolsyaml:+--protocols=${protocolsyaml}} + # between appended batches: combine error status into overall status & reset local error status + cat >> ${basedir}/${sampleset}/movedatafiles.sh <> ${basedir}/${sampleset}/movedatafiles.sh <] [ -N ] [ -e ] [filter [...]]" 1>&2; exit $1; } + + +while getopts "c:e:N:H:h" o; do + case "${o}" in + c) configfile=${OPTARG} + if [[ ! -r ${configfile} ]]; then + echo "Cannot read ${configfile}" 1>&2 + usage 1 + fi + ;; + e) exrxfile="${OPTARG}" ;; + N) newerthan="${OPTARG}" ;; + H) https=1 ;; + h) usage 0 ;; + *) usage 1 ;; + esac +done +shift $((OPTIND-1)) + + +. ${configfile} + +: ${fileserver:?} +# ${srvport} +# ${prefix} +: ${expname:?} +: ${basedir:=$(pwd)} +: ${download:?} +: ${parallel:=16} +: ${contimeout:=300} +: ${retries:=10} +: ${iotimeout:=300} + +. /cluster/home/bs-pangolin/.ssh/${fileserver} + +# add host rsa key if not done yet: +grep --silent \\[${fileserver}\\]:${srvport} ~/.ssh/known_hosts || { ssh-keyscan -t rsa -p ${srvport} ${fileserver} >> ~/.ssh/known_hosts; } + +if (( ${#@} )); then + #dir=( "${@/#/ --directory=${prefix:+${prefix}/}${expname}/}" ) + dir=( "${@/#/ --directory=${prefix:+${prefix}/}}" ) + source="${dir[*]}" +else + source="--directory=${prefix:+${prefix}/}${expname}/*" +fi + +umask 0002 + +if [[ "$https" -eq "1" ]]; then + protocol="https" + srvport=${srvport_https} +else + protocol="sftp" + srvport=${srvport_sftp} + connect_prepass="connect ${protocol}://${user}" + connect_postpass="@${fileserver}${srvport:+:${srvport}}" + connect="${connect_prepass}${password:+:${password}}${connect_postpass}" + connect_echo="${connect_prepass}${password:+:}${connect_postpass}" + echo "${connect_echo}" + settings="set cmd:move-background false; set net:timeout $(( contimeout / retries)); set net:max-retries ${retries}; set net:reconnect-interval-base 8; set xfer:timeout ${iotimeout}" + mirror="mirror --ignore-time -v --continue --no-perms --parallel=${parallel} --loop ${source} -O ${download} ${newerthan:+ --newer-than="'${newerthan}'"}${exrxfile:+ --exclude-rx-from="'${exrxfile}'"}" + echo lftp -c "$settings; ${connect_echo}; cd $expname; $mirror" + exec lftp -c "$settings; $connect; cd $expname; $mirror" +fi diff --git a/sync_sftp b/pangolin_src/upload_sftp.sh similarity index 60% rename from sync_sftp rename to pangolin_src/upload_sftp.sh index 9aa3555..b63fb35 100755 --- a/sync_sftp +++ b/pangolin_src/upload_sftp.sh @@ -1,21 +1,19 @@ #!/bin/bash -configfile=server.conf +configfile=config/server.conf -usage() { echo "Usage: $0 [-c ] [ -N ] [ -e ] [filter [...]]" 1>&2; exit $1; } +usage() { echo "Usage: $0 [-c ] directories [...] " 1>&2; exit $1; } -while getopts "c:N:e:h" o; do +while getopts "c:h" o; do case "${o}" in c) configfile=${OPTARG} if [[ ! -r ${configfile} ]]; then echo "Cannot read ${configfile}" 1>&2 - usage 1 + usage 2 fi ;; - e) exrxfile="${OPTARG}" ;; - N) newerthan="${OPTARG}" ;; h) usage 0 ;; - *) usage 1 ;; + *) usage 2 ;; esac done shift $((OPTIND-1)) @@ -24,7 +22,6 @@ shift $((OPTIND-1)) : ${fileserver:?} # ${srvport} -# ${prefix} : ${expname:?} : ${basedir:=$(pwd)} : ${download:?} @@ -43,13 +40,13 @@ else fi if (( ${#@} )); then - dir=( "${@/#/ --directory=${prefix:+${prefix}/}${expname}/}" ) + dir=( "${@/#/ --directory=${download}/}" ) source="${dir[*]}" else - source="--directory=${prefix:+${prefix}/}${expname}/*" + source="--directory=${download}/*" + echo "directories to upload are currently mandatory !" >&2 + exit 2 fi -umask 0002 - -#exec echo "${exrxfile:+ --exclude-rx-from="${exrxfile}"}" -exec lftp -c "set cmd:move-background false; set net:timeout $(( contimeout / retries)); set net:max-retries ${retries}; set net:reconnect-interval-base 8; set xfer:timeout ${iotimeout}; connect sftp://${username:+${username}@}${fileserver}${srvport:+:${srvport}}; mirror --continue --no-perms --parallel=${parallel} --loop --target-directory=${basedir}/${download} ${source} ${newerthan:+ --newer-than="'${newerthan}'"}${exrxfile:+ --exclude-rx-from="'${exrxfile}'"}" +cd ${basedir} +exec lftp -c "set cmd:move-background false; set net:timeout $(( contimeout / retries)); set net:max-retries ${retries}; set net:reconnect-interval-base 8; set xfer:timeout ${iotimeout}; connect sftp://${username:+${username}@}${fileserver}${srvport:+:${srvport}}; mirror --reverse --continue --no-perms --no-umask --parallel=${parallel} --loop --target-directory=${expname} ${source}" diff --git a/pangolin_src/upload_viollier.sh b/pangolin_src/upload_viollier.sh new file mode 100755 index 0000000..c8f3595 --- /dev/null +++ b/pangolin_src/upload_viollier.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +configfile=config/viollier.conf + +usage() { echo "Usage: $0 [-c ]" 1>&2; exit $1; } + +while getopts "c:h" o; do + case "${o}" in + c) configfile=${OPTARG} + if [[ ! -r ${configfile} ]]; then + echo "Cannot read ${configfile}" 1>&2 + usage 1 + fi + ;; + h) usage 0 ;; + *) usage 1 ;; + esac +done +shift $((OPTIND-1)) + + +. ${configfile} + +: ${lab:?} +: ${fileserver:?} +# ${srvport} +: ${expname:?} +: ${basedir:=$(pwd)} +: ${download:?} +: ${sampleset:?} +: ${working:=working} + +target=consensus_sequences + +[[ "${1}" == '-h' || "${1}" == '--help' ]] && usage 0 + +if [[ ! -d ${basedir}/${download}/ ]]; then + echo "missing directory ${download}" >&2 + exit 1 +elif [[ ! -d ${basedir}/${working}/ ]]; then + echo "missing working ${working}" >&2 + exit 1 +fi + +lastmonth=$(date '+%Y%m' --date='-1 month') +month3ago=$(date '+%Y%m%d' --date='-3 month') + +mkdir -p ${basedir}/${download}/${target} +cd ${basedir} +error=0 +tsvs=( ) +for byml in $(grep -l "lab: ${lab}" ${sampleset}/batch.*.yaml); do + if [[ ! $byml =~ batch\.(20[[:digit:]]{2}[01][[:digit:]][0-3][[:digit:]]_[[:alnum:]]{4,})\.yaml$ ]]; then + echo "cannot parse $byml name" >&2 + error=1 + continue + fi + batch=${BASH_REMATCH[1]} + + if [[ ${lastmonth} > ${batch} ]]; then + echo "skipping ${batch}" + continue + fi + echo "batch: ${batch}" + + tsv="${sampleset}/samples.${batch}.tsv" + if [[ ! -r ${tsv} ]]; then + echo "Cannot read ${tsv}" + error=1 + continue + fi + + tsvs+=( ${tsv} ) +done + +if (( ${#tsvs[@]} == 0 )); then + echo "Nothing to do" + exit $error +fi + +cat "${tsvs[@]}" | while read sample batch len; do + cp --link -vrf ${working}/samples/${sample}/${batch}/references/ref_majority_dels.fasta ${download}/${target}/${sample}-${batch}.fasta || error=1 +done + +(( error )) && exit 0 + +./upload_sftp -c ${configfile} ${target} && exec ./purge_sftp_fasta -r ${month3ago} -c ${configfile} ${target} diff --git a/pangolin_src/upload_viollier_raw.sh b/pangolin_src/upload_viollier_raw.sh new file mode 100755 index 0000000..0db34db --- /dev/null +++ b/pangolin_src/upload_viollier_raw.sh @@ -0,0 +1,113 @@ +#!/bin/bash + +configfile=config/viollier.conf + +usage() { echo "Usage: $0 [-c ] [ -s ] [ -q ] TSVFILE " 1>&2; exit $1; } + +subdir=$(date '+%Y-%m-%d') +verbose='v' + +while getopts "c:s:qh" o; do + case "${o}" in + c) configfile=${OPTARG} + if [[ ! -r ${configfile} ]]; then + echo "Cannot read ${configfile}" 1>&2 + usage 1 + fi + ;; + q) verbose='' ;; + s) subdir=${OPTARG} ;; + h) usage 0 ;; + *) usage 1 ;; + esac +done +shift $((OPTIND-1)) + + +. ${configfile} + +: ${fileserver:?} +# ${srvport} +: ${expname:?} +: ${basedir:=$(pwd)} +: ${download:?} +: ${sampleset:?} + + +target_base=raw_othercenters +target=${target_base}/${subdir} + +(( ${#@} == 0 )) && usage 2 +[[ "${1}" == '-h' || "${1}" == '--help' ]] && usage 0 + +if [[ ! -d ${basedir}/${download}/ ]]; then + echo "missing directory ${download}" >&2 + exit 1 +elif [[ ! -d ${basedir}/${sampleset}/ ]]; then + echo "missing sampleset ${sampleset}" >&2 + exit 1 +fi + +if [[ ! -r "${1}" ]]; then + echo "Cannot read file ${1}" >&2 + exit 2 +fi + +mkdir -p ${basedir}/${download}/${target} +cd ${basedir} +error=0 +while read aufxx ethid rest; do + # remove DOS/Windows-style CR+LF line ending + auf=${aufxx%%$'\r'} + # skip empty lines + if [[ -z ${auf} ]]; then + continue + elif [[ ! ${auf} =~ ^[[:digit:]]{8,} ]]; then + echo "cannot parse ${auf}" >&2 + error=1 + continue + fi + aufnum=${BASH_REMATCH[0]} + + if [[ -z ${ethid} ]]; then + rx="^([[:digit:]]*[[:alpha:]]+[[:alnum:]-]+_)?${auf}[^\t]*" + ethid=$(grep -hoP "${rx}" ${sampleset}/samples.*.tsv) + if [[ -z ${ethid} ]]; then + ethid=$(grep -hoP "${rx}" ${sampleset}/missing.*.txt) + if [[ -z ${ethid} ]]; then + echo "(missing ${aufnum})" + else + echo "(zero yield for ${aufnum} in ${ethid//$'\n'/ })" + fi + continue + elif [[ ${ethid} == *$'\n'* ]]; then + echo -e "(${aufnum}: multiple matches: ${ethid//$'\n'/ }\nUploading all matches)" + fi + elif [[ ! ${ethid} =~ ^[[:digit:]]{6,} ]]; then + echo "cannot parse ${ethid}" >&2 + error=1 + continue + fi + + [[ -d ${download}/${target}/${aufnum}/ ]] && rm -rf ${download}/${target}/${aufnum}/ + mkdir -p ${download}/${target}/${aufnum} + # copy each duplicate sample in a separate subdirectory + for singleethid in ${ethid} + do + echo "${singleethid} -> ${aufnum}" + for b in ${sampleset}/${singleethid}/*; do + bname="$(basename "${b}")" + # NOTE make sure to distinguish samples by their internal ETHID, in case that they come from the same batch, e.g.: + # + # if '34889065' maps to both '34889065_A9' and '34889065_C9' from the same batch '20211203_HTHMGDRXY': + # 'sampleset/34889065_A9/20211203_HTHMGDRXY' => '...{target}/34889065/20211203_HTHMGDRXY-34889065_A9' + # 'sampleset/34889065_C9/20211203_HTHMGDRXY' => '...{target}/34889065/20211203_HTHMGDRXY-34889065_C9' + # + cp --link -${verbose}rf ${b} ${download}/${target}/${aufnum}/${bname}-${singleethid} || error=1 + done + done +done < <( cat ${1}; echo; ) + +(( error )) && exit 0 + +exec ./upload_sftp -c ${configfile} ${target_base} diff --git a/validate_request_raw_viollier.py b/pangolin_src/validate_request_raw_viollier.py similarity index 100% rename from validate_request_raw_viollier.py rename to pangolin_src/validate_request_raw_viollier.py diff --git a/pangolin_src/work-catchup-dehuman/ketchup.sh b/pangolin_src/work-catchup-dehuman/ketchup.sh new file mode 100755 index 0000000..a0a8a28 --- /dev/null +++ b/pangolin_src/work-catchup-dehuman/ketchup.sh @@ -0,0 +1,250 @@ +#!/usr/bin/env bash + +umask 0007 + +scriptdir="$(dirname $(which $0))" +baseconda="$scriptdir/../" + +# +# Input validator +# + +validateDate() { + if [[ "$1" =~ ^(20[0-9][0-9][0-1][0-9]([0-3][0-9]?)?)$ ]]; then + return; + else + echo "bad date ${1}" + exit 1; + fi +} + +validateBatchName() { + if [[ "$1" =~ ^(20[0-9][0-9][0-1][0-9][0-3][0-9]_[[:alnum:]-]{4,})$ ]]; then + return; + else + echo "bad batchname ${1}" + exit 1; + fi +} + +validateTags() { + for b in "${@}"; do + validateBatchName "${b}" + done +} + + +lastmonth=$(date '+%Y%m' --date='-1 month') + +case "$1" in + samples) + list_tsv=( ) + case "$2" in + --recent) + list_tsv=( "${scriptdir}/../working/samples.recent.tsv" ) + shift + ;; + --date) + shift + validateDate "${2}" + list_tsv=( $(echo "../sampleset/samples.${2}"*".tsv" ) ) + shift + ;; + --tsv) + shift + if [[ ! -r "${2}" ]]; then + echo "Cannot read ${2}" + exit 1 + fi + # TODO validate + list_tsv=( "${2}" ) + shift + ;; + esac + shift + + # get batches + if [[ -n "${*}" ]]; then + validateTags "${@}" + for b in "${@}"; do + s="${scriptdir}/../sampleset/samples.${b}.tsv" + if [[ ! -s "${s}" ]]; then + echo "missing sample file ${s}" > /dev/stderr + exit 2 + fi + list_tsv+=( "${s}" ) + done + fi + + # check everything is normal + if (( ${#list_tsv[@]} == 0 )); then + echo "usage: ${0} samples [ --recent | --date | --tsv ] ..." > /dev/stderr + exit 2 + fi + + echo "importing samples from:" + printf ' - %s\n' "${list_tsv[@]}" + sort -u "${list_tsv[@]}" > "${scriptdir}/samples.catchup.tsv" + wc -l "${scriptdir}/samples.catchup.tsv" + + rm -rf "${scriptdir}/samples/" + mkdir -p "${scriptdir}/samples/" + cut -f1 "${scriptdir}/samples.catchup.tsv" | sort -u | while read s; do + o="${scriptdir}/../working/samples/${s}" + if [[ -d "${o}" ]]; then + ln -fvs "$(realpath --relative-to="${scriptdir}/samples/" "${o}")" "${scriptdir}/samples/"; + fi; + done + find "${scriptdir}/.snakemake/metadata/" "${scriptdir}/.snakemake/incomplete/" "${scriptdir}/cluster_logs/" -type f -print0 | xargs -r0 rm + echo -e '\n\e[38;5;45;1mI ve experiments to run\e[0m\n\e[38;5;208;1mThere is research to be done\e[0m' + ;; + + scratch) + olderthan=60 + purge=0 + loop=0 + filter= + while [[ -n $2 ]]; do + case "$2" in + --minutes-ago) + if [[ ! "${3}" =~ ^[[:digit:]]+$ ]]; then + echo "parameter of ${2} must be a number of minutes (digits only), got <${3}> instead" > /dev/stderr + exit 2 + fi + shift + olderthan=$2 + ;; + --loop) + loop=1 + filter='-name dehuman.sam' + ### dh_aln.sam + ### reject_R2.fastq.gz + ### reject_R1.fastq.gz + ### host_aln.sam + ### dehuman.filter + ;& + --purge) + purge=1 + ;; + *) + echo "Unkown parameter ${2}" > /dev/stderr + exit 2 + ;; + esac + shift + done + + hourago="${SCRATCH}/hourago.touch" + temp_scratch="/cluster/scratch/bs-pangolin/pangolin/temp" + touch --date="${olderthan} minutes ago" "${hourago}" + + if (( purge )); then + echo "purging in ${temp_scratch}..." + else + echo "listing in ${temp_scratch}..." + fi + + (cd "${temp_scratch}/" && find samples/ -type f ${filter} ) | grep -oP '(?<=samples/)[^/]+/[^/]+(?=/)' | sort -u | while read s; do + if [[ -s "${scriptdir}/samples/${s}/raw_uploads/dehuman.cram.md5" && "${scriptdir}/samples/${s}/raw_uploads/dehuman.cram.md5" -ot "${hourago}" ]]; then + #if [[ -r "${scriptdir}/../working/samples/${s}/upload_prepared.touch" && "${scriptdir}/../working/samples/${s}/upload_prepared.touch" -ot "${hourago}" ]]; then + if (( purge )); then + rm -rvf "${temp_scratch}/samples/${s}" + else + echo "${s}"; + fi + fi; + done | tee /dev/stderr | wc -l 2>&1 + + if (( loop )); then + lquota -2 ${SCRATCH} + echo -e '\n\e[38;5;45;1mYou just keep on trying\e[0m\n\e[38;5;208;1mTill you run out of cake\e[0m' + if sleep "$(( 5 + olderthan))m"; then + # loop if no breaks + exec "${0}" scratch --minutes-ago "${olderthan}" --loop + fi + echo "I'm not even angry" + fi + ;; + + check_reads) + . $baseconda/miniconda3/bin/activate "qa" + while read s b o; do + declare -a r; + r=( $(gawk -v s="${s}" -v b="${b}" 'BEGIN{FS=","};FNR==1{for(f=1;f cr )); then + echo "${cram}" | tee -a cram_error.txt; + else + echo "."; + fi; + done < samples.catchup.tsv + ;; + + delete_raw) + cmd='printf %s\n' + msg='\r\e[1mConsider %s/%b for deletion\e[0m\n' + if [[ -n "$2" ]]; then + case "$2" in + --do-it) + cmd='rm -v' + msg='\r\e[36;1mDelete raw from %s/%b\e[0m\n' + ;; + *) + echo "Unkown parameter ${2}" > /dev/stderr + exit 2 + ;; + esac + fi + + sort -u "${scriptdir}/samples.catchup.tsv" | while read s b o; do + if [[ "${b}" > "${lastmonth}" ]]; then + echo -e "\e[31;1mSkipping ${s}/${b} : too recent (${lastmonth})\e[0m" + continue + fi + + if [[ + -s "${scriptdir}/samples/${s}/${b}/raw_uploads/dehuman.cram.md5" + && -s "${scriptdir}/samples/${s}/${b}/raw_uploads/dehuman.cram" + && -e "${scriptdir}/samples/${s}/${b}/upload_prepared.touch" + && -s "${scriptdir}/samples/${s}/${b}/alignments/REF_aln_trim.bam" + && -s "${scriptdir}/samples/${s}/${b}/references/consensus.bcftools.fasta" + ]]; then + printf "${msg}" "${s}" "${b}" + else + echo -n '.' + continue + fi + + ${cmd} "${scriptdir}/samples/${s}/${b}/"{raw_data,preprocessed_data}/*.fastq.gz "${scriptdir}/../sampleset/${s}/${b}/raw_data/"*.fastq.gz "${scriptdir}/samples/${s}/${b}/alignments/"REF_aln.bam* + (cd "${scriptdir}/samples/" && rmdir --ignore-fail-on-non-empty --parents "${s}/${b}/raw_data" ) + (cd "${scriptdir}/../sampleset/" && rmdir --ignore-fail-on-non-empty --parents "${s}/${b}/raw_data" ) + done + echo -e '\r\e[K\e[38;5;45;1mThis was a triumph! I m making a note here; "Huge success"\e[0m' + ;; + + still_around) + sort -u "${scriptdir}/samples.catchup.tsv" | while read s b o; do + if [[ -d "${scriptdir}/../sampleset/${s}/${b}/raw_data/" || -f "${scriptdir}/samples/${s}/${b}/alignments/REF_aln.bam" ]]; then + echo -e "\r\e[K${s}/${b} still around" + else + echo -n '.' + fi + done + echo -e '\r\e[K\e[38;5;208;1mThe cake is a lie!\e[0m' + ;; + + df) + exec "${scriptdir}/../batman.sh" "df" + echo "Cannot find batman.sh" + exit 2 + ;; + + *) + echo "Unkown subcommand ${1}" > /dev/stderr + exit 2 + ;; +esac diff --git a/work-catchup-dehuman/vpipe-catchup.bsub b/pangolin_src/work-catchup-dehuman/vpipe-catchup.bsub similarity index 100% rename from work-catchup-dehuman/vpipe-catchup.bsub rename to pangolin_src/work-catchup-dehuman/vpipe-catchup.bsub diff --git a/work-catchup-dehuman/vpipe-catchup.sbatch b/pangolin_src/work-catchup-dehuman/vpipe-catchup.sbatch similarity index 80% rename from work-catchup-dehuman/vpipe-catchup.sbatch rename to pangolin_src/work-catchup-dehuman/vpipe-catchup.sbatch index 5fd512e..726810f 100755 --- a/work-catchup-dehuman/vpipe-catchup.sbatch +++ b/pangolin_src/work-catchup-dehuman/vpipe-catchup.sbatch @@ -1,6 +1,6 @@ #!/bin/bash #SBATCH --job-name="CATCHUP-vpipe" -#SBATCH --mail-user="carrara@nexus.ethz.ch" # pelin.icer@bsse.ethz.ch david.dreifuss@bsse.ethz.ch" # kim.jablonski@bsse.ethz.ch louis.duplessis@bsse.ethz.ch" # sarah.nadeau@bsse.ethz.ch chaoran.chen@bsse.ethz.ch +#SBATCH --mail-user="ivan.topolsky@bsse.ethz.ch" # carrara@nexus.ethz.ch" # pelin.icer@bsse.ethz.ch david.dreifuss@bsse.ethz.ch" # kim.jablonski@bsse.ethz.ch louis.duplessis@bsse.ethz.ch" # sarah.nadeau@bsse.ethz.ch chaoran.chen@bsse.ethz.ch #SBATCH --mail-type=END #SBATCH --mem-per-cpu=16384 ####### --partition=light diff --git a/work-catchup-dehuman/vpipe.config b/pangolin_src/work-catchup-dehuman/vpipe.config similarity index 100% rename from work-catchup-dehuman/vpipe.config rename to pangolin_src/work-catchup-dehuman/vpipe.config diff --git a/profiles/custom-lsf/status.sh b/profiles/custom-lsf/status.sh deleted file mode 100755 index 84fcf97..0000000 --- a/profiles/custom-lsf/status.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash - -#RXJOB='Job <([[:digit:]]+)> is submitted' -RXJOB='<([[:digit:]]+)>' - -if [[ "$*" =~ $RXJOB ]]; then - J="${BASH_REMATCH[1]}" -else - echo "Cannot find JobID in '$*'" >&2 - exit -fi - -state="$(bjobs $J | gawk -v I=$J '$1==I{print $3}')" - -case "${state}" in - EXIT) - echo "failed" - ;; - DONE) - echo "success" - ;; - RUN) - echo "running" - ;; - PEND) - echo "running" - ;; - *) - echo "Weird status ${state}" >&2 - echo "running" - ;; -esac - -exit 0 diff --git a/profiles/smk-simple-slurm/status-sacct.sh b/profiles/smk-simple-slurm/status-sacct.sh deleted file mode 100755 index 7adfe40..0000000 --- a/profiles/smk-simple-slurm/status-sacct.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash - -# Check status of Slurm job - -jobid="$1" - -if [[ "$jobid" == Submitted ]] -then - echo smk-simple-slurm: Invalid job ID: "$jobid" >&2 - echo smk-simple-slurm: Did you remember to add the flag --parsable to your sbatch call? >&2 - exit 1 -fi - -read output other < <(sacct -j "$jobid" --format State --noheader) - -if [[ $output =~ ^(COMPLETED).* ]] -then - echo success -elif [[ $output =~ ^(RUNNING|PENDING|COMPLETING|CONFIGURING|SUSPENDED).* || -z "$output" ]] -# HACK: if sacct has crashed (e.g. failure to connect to slurmdb) and doesn't return any output, consider the jobs as still running -then - echo running -else - echo failed - echo "failure reason: ${output}" >&2 -fi - -# CD COMPLETED Job has terminated all processes on all nodes with an exit code of zero. - -# BF BOOT_FAIL Job terminated due to launch failure, typically due to a hardware failure (e.g. unable to boot the node or block and the job can not be requeued). -# CA CANCELLED Job was explicitly cancelled by the user or system administrator. The job may or may not have been initiated. -# DL DEADLINE Job terminated on deadline. -# F FAILED Job terminated with non-zero exit code or other failure condition. -# NF NODE_FAIL Job terminated due to failure of one or more allocated nodes. -# OOM OUT_OF_MEMORY Job experienced out of memory error. -# TO TIMEOUT Job terminated upon reaching its time limit. -# PR PREEMPTED Job terminated due to preemption. -# RQ REQUEUED Job was requeued. - # RS RESIZING Job is about to change size. -# RV REVOKED Sibling was removed from cluster due to other cluster starting the job. - -# PD PENDING Job is awaiting resource allocation. -# R RUNNING Job currently has an allocation. -# S SUSPENDED Job has an allocation, but execution has been suspended and CPUs have been released for other jobs. diff --git a/python/.gitignore b/python/.gitignore deleted file mode 100644 index bbe4d0d..0000000 --- a/python/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -venv* -*.pyc -__pycache__ -.*.sw? -.defaults -sars_cov_2.db diff --git a/python/init.sh b/python/init.sh deleted file mode 100755 index eeb0d00..0000000 --- a/python/init.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -# -# init.sh -# Copyright (C) 2021 Uwe Schmitt -# -# Distributed under terms of the MIT license. -# - -set -e - -# https://stackoverflow.com/questions/59895/ -SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -cd ${SCRIPT_DIR} - -test -d venv || python -m venv venv - -source venv/bin/activate - -pip install requirements.txt diff --git a/python/requirements.txt b/python/requirements.txt deleted file mode 100644 index e0a76f3..0000000 --- a/python/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -sqlalchemy -psycopg2 diff --git a/python/upload_to_db.py b/python/upload_to_db.py deleted file mode 100644 index a762ff3..0000000 --- a/python/upload_to_db.py +++ /dev/null @@ -1,232 +0,0 @@ -#!/usr/bin/env python - -""" -This scripte reads a summary.json file created by v-pipe and imports the -computed consensus sequences into the s3c database. - -usage: - - for testing: - - $ python upload_to_db /path/to/summary_zip_file - - this will create sqlite3 db 'sars_cov_2.db' in the current working directory - - for productive usage: - - $ DBUSER=abcde DBPASSWORD=xyde python upload_to_db /path/to/summary_zip_file - -the environment variable DBHOST, DBPORT and DATABASE can also be set to overrun -standard settings. -""" - -import json -import os -import sys -import tempfile -import time -import warnings -import zipfile -from datetime import datetime - -import sqlalchemy -from sqlalchemy import ( - Column, - DateTime, - Integer, - LargeBinary, - MetaData, - String, - Table, - bindparam, - create_engine, - exc, - insert, -) -from sqlalchemy.pool import SingletonThreadPool - - -def connect_to_db(connection_string, *, verbose=False, attempts=5, delay=1): - - if connection_string.startswith("sqlite"): - from sqlite3 import dbapi2 as sqlite - - engine = create_engine( - connection_string, - module=sqlite, - echo=verbose, - poolclass=SingletonThreadPool, - ) - - else: - # pool_pre_ping: check if connection is valid befor submitting a - # command, if not: remove old connections from pool and setup fresh one. - engine = create_engine(connection_string, echo=verbose, pool_pre_ping=True) - - stored_exception = None - for attempt in range(attempts): - try: - engine.connect() - except exc.OperationalError as e: - stored_exception = e - warnings.warn( - "could not connect, sleep for {} sencds and try again".format(delay) - ) - time.sleep(delay) - continue - else: - break - - else: - raise TimeoutError( - "could not connect to {}. Error is {}".format( - connection_string, stored_exception - ) - ) from None - - return engine - - -def create_tables_if_not_exists(engine): - meta = MetaData() - - consensus_table = Table( - "new_sequence", - meta, - Column("id", Integer, primary_key=True), - Column("batch", String), - Column("sample", String), - Column("header", String), - Column("created", DateTime(timezone=True)), - Column("checksum_seguid", String), - Column("checksum_crc64", String), - Column("sequence", String), - ) - - raw_data_table = Table( - "raw_data", - meta, - Column("id", Integer, primary_key=True), - Column("batch", String), - Column("sample", String), - Column("data", LargeBinary), - ) - - meta.create_all(engine) - return consensus_table, raw_data_table - - -def import_data(engine, consensus_table, raw_data_table, summary_file, batch_size=1000): - - target = tempfile.mkdtemp() - with zipfile.ZipFile(summary_file, "r") as zip_ref: - zip_ref.extractall(target) - - summary_file = os.path.join(target, "summary.json") - - with open(summary_file) as fh: - try: - summary_data = json.load(fh) - except json.JSONDecodeError as e: - raise ValueError(f"{summary_file} is no valid json file: {e}") from None - - prepared_stmt = insert(consensus_table).values( - batch=bindparam("batch"), - sample=bindparam("sample"), - header=bindparam("header"), - created=bindparam("created"), - checksum_seguid=bindparam("checksum_seguid"), - checksum_crc64=bindparam("checksum_crc64"), - sequence=bindparam("sequence"), - ) - - rows = [] - count = 0 - - for entry in summary_data: - sample = entry["sample"] - batch = entry["batch"] - created = datetime.fromisoformat(entry["created"]) - for sequence_data in entry["sequences"]: - header = sequence_data["header"] - seguid = sequence_data["seguid"] - crc64 = sequence_data["crc64"] - sequence = sequence_data["sequence"] - rows.append( - dict( - batch=batch, - sample=sample, - header=header, - created=created, - checksum_seguid=seguid, - checksum_crc64=crc64, - sequence=sequence, - ) - ) - if len(rows) >= batch_size: - engine.execute(prepared_stmt, rows) - count += len(rows) - rows = [] - - raw_data_path = entry["raw_data"] - if raw_data_path: - with open(os.path.join(target, raw_data_path), "rb") as fh: - data = fh.read() - print(batch, sample) - engine.execute( - insert(raw_data_table).values(batch=batch, sample=sample, data=data) - ) - - engine.execute(prepared_stmt, rows) - count += len(rows) - return count - - -if __name__ == "__main__": - - if len(sys.argv) < 2: - raise ValueError("please provide path with summary file") - if len(sys.argv) > 2: - warnings.warn( - "you provided more argments than the summary file. will ignore them." - ) - - summary_file = sys.argv[1] - if not os.path.exists(summary_file): - raise IOError(f"provided file {summary_file} does not exist.") - - try: - with open(summary_file) as fh: - pass - except IOError: - raise IOError(f"counld not open {summary_file} for reading.") from None - - if os.path.exists(".defaults"): - with open(".defaults") as fh: - default = dict(line.strip().split("=") for line in fh if line.strip()) - else: - default = {} - - DBUSER = os.getenv("DBUSER") - DBPASSWORD = os.getenv("DBPASSWORD") - DBHOST = os.getenv("DBHOST", default.get("DBHOST")) - DBPORT = os.getenv("DBPORT", default.get("DBPORT")) - DATABASE = os.getenv("DATABASE", default.get("DATABASE")) - - if DBUSER is not None and DBPASSWORD is not None: - connection_string = ( - f"postgresql://{DBUSER}:{DBPASSWORD}@{DBHOST}:{DBPORT}/{DATABASE}" - ) - else: - warnings.warn( - "you did not set DBUSER and DBPASSWORD variables. will write" - " to local sqlite3 db sars_cov_2.db instead." - ) - path = "sars_cov_2.db" - connection_string = "sqlite+pysqlite:///{}".format(path) - - engine = connect_to_db(connection_string) - consensus_table, raw_data_table = create_tables_if_not_exists(engine) - count = import_data(engine, consensus_table, raw_data_table, summary_file) - - print(f"imported {count} sequence(s).") diff --git a/rsyncd.conf b/rsyncd.conf deleted file mode 100644 index 5d86561..0000000 --- a/rsyncd.conf +++ /dev/null @@ -1,29 +0,0 @@ -read only = no -use chroot = no -transfer logging = true -log format = %h %o %f %l %b -log file = log/rsyncd.log - -charset = utf8 - -uid = bs-pangolin -auth users = dryak -secrets file = rsyncd.secrets - -[sampleset] - path = /cluster/project/pangolin/sampleset/ - comment = Samples set - auth users = belfry:wo batman:ro dryak:rw ceciliav:ro auguste:ro - incoming chmod = Dg+s,ug+rw,o-rwx - -[working] - path = /cluster/project/pangolin/working/ - comment = Working directory - auth users = belfry:ro batman:ro dryak:rw ceciliav:ro auguste:ro - outgoing chmod = Dg+s,ug+rw,o-rwx - -[catchup] - path = /cluster/project/pangolin/work-catchup-dehuman/ - comment = Workdir for catchup - auth users = belfry:ro batman:ro dryak:rw - outgoing chmod = Dg+s,ug+rw,o-rwx diff --git a/run_container.sh b/run_container.sh new file mode 100755 index 0000000..91e01e2 --- /dev/null +++ b/run_container.sh @@ -0,0 +1 @@ +docker run -it -v /data/wastewater_automation/workdir:/app/workdir /bin/bash diff --git a/server.conf b/server.conf deleted file mode 100755 index 25efbaa..0000000 --- a/server.conf +++ /dev/null @@ -1,50 +0,0 @@ -[ _ ] -# labs: SFTP synchronisation -# (except in case of abnormally slow operation, sync can be left active) -skipsync=gfb -# labs: sample sorting and import -# (swith 0 for labs whose sequences we aren't currently importing, e.g. due to problems) -lab=( ['gfb']=0 ['fgcz']=1 ['h2030']=0 ['viollier']=0 ) -# NOTE: h2030 isn't sequencing as the number of cases is lower. -# pass '--force' to force overwriting any existing file when moving -sort_force=--force -# base dircetory -basedir=/links/shared/covid19-pangolin/backup -# sub-directory of openbis download for rights fixing -download=openbis-downloads -# sub-directory to hold the sorted samples set -sampleset=sampleset -# working sub-directory -working=working -# linking instead of copying ? -# --reflink for CoW filesystems (ZFS, BTRFS) -# --hardlink for most unix-like filesystems -link=--link -# group on the storage (inside download and sampleset) -storgrp=bsse-covid19-pangolin@d.ethz.ch -# parallel copy jobs -parallel=8 -# parallel backup copy jobs -parallelpull=32 -# whereto push the sequences at the end -releasedir=/links/shared/covid19-pangolin/pangolin/consensus_data/batch/ -# timeout before rsync considers the transfer failed in seconds -rsynctimeout=2000 -# SSH connection timeout -contimeout=300 -# IO timeout -iotimeout=300 -# suspend jobs submission in case of problems -donotsubmit=0 -# skip or run ShoRAH step of V-pipe -run_shorah=0 -# mail -mailfrom='Automation-carillon' -#mailto=( ivan.topolsky@bsse.ethz.ch , kim.jablonski@bsse.ethz.ch , sarah.nadeau@bsse.ethz.ch , chaoran.chen@bsse.ethz.ch , louis.duplessis@bsse.ethz.ch ) -# timeout for the whole automation loop -runtimeout=7200 -# timeout to wait for 'sync' command: 0: no timeout, "": default, otherwise seconds -synctimeout=0 - -# username used on the cluster (while we're switching over) -cluster_user=bs-pangolin diff --git a/sort_samples_demultiplexstats b/sort_samples_demultiplexstats deleted file mode 120000 index b91cc8f..0000000 --- a/sort_samples_demultiplexstats +++ /dev/null @@ -1 +0,0 @@ -V-pipe/utils/sort_samples_demultiplexstats \ No newline at end of file diff --git a/sort_samples_dumb b/sort_samples_dumb deleted file mode 120000 index 9450e95..0000000 --- a/sort_samples_dumb +++ /dev/null @@ -1 +0,0 @@ -V-pipe/utils/sort_samples_dumb \ No newline at end of file diff --git a/sort_samples_jobinfo b/sort_samples_jobinfo deleted file mode 120000 index 3909b92..0000000 --- a/sort_samples_jobinfo +++ /dev/null @@ -1 +0,0 @@ -V-pipe/utils/sort_samples_jobinfo \ No newline at end of file diff --git a/uploader/create_metadata_line.py b/uploader/create_metadata_line.py new file mode 100644 index 0000000..065cdac --- /dev/null +++ b/uploader/create_metadata_line.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 + +# Script to generate a single metadata line for WasteWater upload to SPSP + +import os +import argparse +import sys +import re +sys.path.append("/app/uploader") +import submission_metadata as meta + +# parse command line +def parse_args(): + """ Set up the parsing of command-line arguments """ + + parser = argparse.ArgumentParser(description='Validate Viollier raw data upload requests against the database') + parser.add_argument('-s', '--samplename', required=True, help = "Samplename of the sample to upload") + parser.add_argument('-b', '--batchname', required=True, help = "Batchname of the batch to upload") + parser.add_argument('-u', '--update', required=False, default="No", help = "If the field _is_assembly_update_ should be Yes or No") + parser.add_argument('-o', '--outfile', required=True, help="metadata output file to write the line to") + return parser.parse_args() + +# samplename="KLZHCov220123" +# batchname="20220204_HVFYNDRXY" +# update="No" + +def load_locations(locationfile): + with open(locationfile, 'r') as file: + #locations = file.readlines() + myline = [line.rstrip() for line in file] + myline = [ " ".join(element.split()) for element in myline ] + locations = [re.split(r'\t|\s', line) for line in myline] + return locations + +def load_timeline(timelinefile, samplename): + with open(timelinefile, "r") as file: + myline = [line.rstrip() for line in file] + timeline = [re.split(r'\t', line) for line in myline] + for line in timeline: + if (samplename in line): + timeinfo = line + return timeinfo + +def read_qa(samplename, qafile): + with open(qafile, 'r') as file: + myline = [line.rstrip() for line in file] + sampleline = [re.split(r',', line) for line in myline] + for line in sampleline: + if (samplename in line): + samplecov = line + return samplecov + +def verify_mandatory_fields(line, meta, samplename): + line = line.split("\t") + if (line[1]!="2697049"): + sys.exit("Error: The metadata line for " + samplename + " has an unexpected species code") + date = line[3].split("-") + if (len(date)!=3): + sys.exit("Error: The metadata line for " + samplename + " has a date with an unexpected format") + if (date[0] not in meta.projyears): + sys.exit("Error: The metadata line for " + samplename + " has a date with an unepxted year") + cantonfull = line[4].split("/") + if (cantonfull[2]==""): + sys.exit("Error: The metadata line for " + samplename + " has a location general field with an empty canton") + if (line[7]!="Environment"): + sys.exit("Error: The metadata line for " + samplename + " has an unexpected isolation source description") + # line[8] is not necessary as it's built from the cantonfull we already checked + if (line[13]!="Surveillance"): + sys.exit("Error: The metadata line for " + samplename + " has an unexpected sequencing purpose") + if (line[14]!="Metagenome"): + sys.exit("Error: The metadata line for " + samplename + " has an unexpected sequencing investigation type") + if (line[15]==""): + sys.exit("Error: The metadata line for " + samplename + " has an empty cram file field") + cram = line[15].split(".") + if (cram[1]!="cram"): + sys.exit("Error: The metadata line for " + samplename + " has a cram file with an unexpected extension") + # line[16] is already verified inline with a try/except + if (line[18]!=meta.seqplatform): + sys.exit("Error: The metadata line for " + samplename + " has a unexpected sequencing platform") + if (line[19]!=meta.assembly): + sys.exit("Error: The metadata line for " + samplename + " has a unexpected assembly method") + if (line[21]!=meta.reportinglab): + sys.exit("Error: The metadata line for " + samplename + " has a unexpected reporting lab name") + if (line[22] not in meta.collectinglab.values()): + sys.exit("Error: The metadata line for " + samplename + " has a unexpected collecting lab name") + +def verify_strain_name(strain, meta): + pieces = strain.split("/") + if (len(pieces)!=4): + sys.exit("Error: strain name " + strain + " does not have 4 fields separated by /") + if (pieces[0]!="hCoV-19"): + sys.exit("Error: strain name " + strain + " does not have the string hCoV-19 as first field") + if (pieces[1]!="Switzerland"): + sys.exit("Error: strain name " + strain + " does not have the string Switzerland as second field") + if (pieces[3] not in meta.projyears): + sys.exit("Error: strain name " + strain + " does not have an accepted year as second field") + pieces2 = pieces[2].split("-") + if (len(pieces2)!=3): + sys.exit("Error: strain name " + strain + " does not have 3 elements separated by - in the third field") + if (pieces2[0] not in meta.cantons): + sys.exit("Error: strain name " + strain + " does not have an accepted canton code") + if (pieces2[1] not in meta.submitting): + sys.exit("Error: strain name " + strain + " does not have an accepted submitting lab") + if (pieces2[2] == ""): + sys.exit("Error: strain name " + strain + " has an empty sample name") + +def get_authors_by_date(meta, collectingcode, center, ethz, date): + try: + date = int(date) + except: + sys.exit("Error: found a non-numeric date in the timeline file") + allkeys = meta.authors.keys() + for k in allkeys: + if (collectingcode in k): + try: + startdate = int(k.split("_")[1]) + except: + sys.exit("Error: found a non-numeric date in the authors settings for " + collectingcode) + try: + enddate = int(k.split("_")[2])+1 + except: + sys.exit("Error: found a non-numeric date in the authors settings for " + collectingcode) + if (date in range(startdate, enddate)): + collectingauthorcode = k + if (center in k): + try: + startdate = int(k.split("_")[1]) + except: + sys.exit("Error: found a non-numeric date in the authors settings for " + center) + try: + enddate = int(k.split("_")[2])+1 + except: + sys.exit("Error: found a non-numeric date in the authors settings for " + center) + if (date in range(startdate, enddate)): + centerauthorcode = k + if (ethz in k): + try: + startdate = int(k.split("_")[1]) + except: + sys.exit("Error: found a non-numeric date in the authors settings for " + ethz) + try: + enddate = int(k.split("_")[2])+1 + except: + sys.exit("Error: found a non-numeric date in the authors settings for " + ethz) + if (date in range(startdate, enddate)): + ethzauthorcode = k + try: + collectingauthorcode + except: + sys.exit("Error: no date range available for authors of " + collectingcode + " for date " + str(date)) + try: + centerauthorcode + except: + sys.exit("Error: no date range available for authors of " + center + " for date " + str(date)) + try: + ethzauthorcode + except: + sys.exit("Error: no date range available for authors of " + ethz + " for date " + str(date)) + return [collectingauthorcode, centerauthorcode, ethzauthorcode] + +def main(): + args = parse_args() + if (args.samplename == "" or args.batchname == "" or args.outfile == ""): + sys.exit("Error, empty sample name") + + if (args.update != "Yes" and args.update != "No"): + sys.exit("Error: wrong value for option --update") + + try: + locations = load_locations(meta.locations) + except: + sys.exit("Error: cannot load the locations file") + try: + locations[locations.index(['KLZHCov', 'Kanton', 'Zürich'])] = ['KLZHCov', 'Zürich', "(ZH)"] + locations[locations.index(['KLZHCov_Promega', 'Kanton', 'Zürich/Promega'])] = ['KLZHCov_Promega', 'Zürich', "(ZH)"] + #locations[locations.index(['Ba', 'Basel', '(catchment', 'area', 'ARA', 'Basel)'])] = ['Ba', 'Basel', '(BS)'] + except: + sys.exit("We have exceptions in place for KLZHCov, KLZHCov_Promega. It looks like one of them is not anymore in the location list") + try: + mydata = load_timeline(meta.timelinefile, args.samplename) + except: + sys.exit("Error: cannot load the timeline file") + if (len(mydata) == 6) and (mydata[4] in meta.exceptions.keys()): + mydata.append(meta.exceptions[mydata[4]]) + elif (len(mydata) == 4) and (mydata[0].split("_")[0] in meta.exceptions.keys()): + name = mydata[0].split("_")[0] + date_list = list(args.batchname.split("_")[0]) + date = "" + for i in range(0, len(date_list)): + if i in [4, 6]: + date = date + "-" + date_list[i] + else: + date = date + date_list[i] + mydata.extend([name, date, meta.exceptions[name]]) + else: + if (mydata[6] == "Basel (catchment area ARA Basel)"): + mydata[6] = "Basel (BS)" + if (mydata[6] == "Kanton Zürich"): + mydata[6] = "Zürich (ZH)" + if (mydata[6] == "Kanton Zürich/Promega"): + mydata[6] = "Zürich (ZH)" + cram = args.samplename+".cram" + strain = 'hCoV-19/Switzerland/'+mydata[6].split(" ")[1].replace("(","").replace(")","")+"-ETHZ-"+mydata[0].replace("_","").replace("-","")+"/"+mydata[5].split("-")[0] + verify_strain_name(strain, meta) + sourcename = mydata[6] + try: + sampleinfo = meta.kit[mydata[3]] + except: + sys.exit("Error: cannot recognise the primer kit code:" + mydata[3]) + + try: + samplecov = str(round(float(read_qa(args.samplename, meta.qafile)[34]))) + except: + sys.exit("Error: cannot load the qa file") + + try: + collectingcode = meta.collecting_lab[mydata[4]] + except: + sys.exit("Error: the provided plant " + mydata[4] + " has no associated collecting lab code in the configuration") + + try: + collectinglab = meta.collectinglab[collectingcode] + except: + sys.exit("Error: the provided lab code " + collectingcode + " has no associated collecting lab name in the configuration") + + authorscode = get_authors_by_date(meta, collectingcode, meta.centerused, "ethz", mydata[5].replace("-","")) + + try: + authors = meta.authors[authorscode[0]] + ", " + meta.authors[authorscode[1]] + ", " + meta.authors[authorscode[2]] + except: + sys.exit("Error: the authors list cannot be completed, either for a missing collecting lab code (" + collectingcode + "), a missing sequencing center (" + meta["centerused"] + ") or a missing entry for ETHZ") + + try: + catchment_size = str(round(float(meta.size[mydata[4]]))) + except ValueError: + catchment_size = "" + + fullline = args.update+"\t2697049\t"+strain+"\t"+mydata[5]+"\tEurope/Switzerland/"+mydata[6].split(" ")[1].replace("(","").replace(")","")+"\t"+mydata[6]+"\t\tEnvironment\tWastewater treatment plant\t"+sourcename+"\t"+catchment_size+"\t"+meta.population[mydata[4]]+"\t"+meta.region[mydata[4]]+"\tSurveillance\tMetagenome\t"+cram+"\t"+sampleinfo+"\t"+meta.seqcenter[meta.centerused]+"\t"+meta.seqplatform+"\t"+meta.assembly+"\t"+samplecov+"\t"+meta.reportinglab+"\t"+collectinglab+"\t"+authors+"\t"+meta.embargo+"\t\t\n" + verify_mandatory_fields(fullline, meta, args.samplename) + + try: + with open(args.outfile, "a") as file_object: + file_object.write(fullline) + except: + sys.ext("Error: failed to write the metadata line for sample ", args.samplename, " in output file ", args.outfile) + +if __name__ == '__main__': + main() + diff --git a/uploader/next_upload.sh b/uploader/next_upload.sh new file mode 100755 index 0000000..6916eec --- /dev/null +++ b/uploader/next_upload.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +usage() { echo "Usage: $0 [-c ] [ -N ] [ -a ]" 1>&2; exit $1; } + + +while getopts "c:N:a:h" o; do + case "${o}" in + c) configfile=${OPTARG} + if [[ ! -r ${configfile} ]]; then + echo "Cannot read ${configfile}" 1>&2 + usage 1 + fi + ;; + a) arhivedir="${OPTARG}" ;; + N) sample_number="${OPTARG}" ;; + h) usage 0 ;; + *) usage 1 ;; + esac +done +shift $((OPTIND-1)) + +# set -euo pipefail +set -uo pipefail + +. ${configfile} + +#${uploader_code}/prepare.sh -N $sample_number -c ${configfile} + +archive_now="${uploader_archive}/$(date +"%Y-%m-%d"-%H-%M-%S)" +mkdir -p $archive_now + +${uploader_code}/upload.sh ${archive_now} && \ +cat ${archive_now}/uploaded_run.txt >> ${uploader_uploaded} && \ +rm -r ${local_dataset}/${working}/samples/* + + diff --git a/uploader/prepare.sh b/uploader/prepare.sh new file mode 100755 index 0000000..d4c8cb2 --- /dev/null +++ b/uploader/prepare.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +usage() { echo "Usage: $0 [-c ] [ -N ] [-b ]" 1>&2; exit $1; } + + +while getopts "c:N:b:h" o; do + case "${o}" in + c) configfile=${OPTARG} + if [[ ! -r ${configfile} ]]; then + echo "Cannot read ${configfile}" 1>&2 + usage 1 + fi + ;; + N) sample_number="${OPTARG}" ;; + b) blacklist="${OPTARG}" ;; + h) usage 0 ;; + *) usage 1 ;; + esac +done +shift $((OPTIND-1)) + +set -eu + +. ${configfile} + +if [ -f ${uploader_tempdir}/to_upload.txt ] +then + rm "${uploader_tempdir}/to_upload.txt" +fi +echo "Preparing the list of files to upload for this batch" +{ python3 - ${uploader_workdir}/${uploaderlist} ${uploader_uploaded} ${sample_number} ${uploader_tempdir}/to_upload.txt ${blacklist} < "${uploadlist}.tmp" && \ + mv "${uploadlist}.tmp" "${uploadlist}" && \ + echo ${batch_to_upload} >> ${uploadedbatches}) || \ + (echo "ERROR in handling the logging of the batches" && \ + exit 5) + diff --git a/uploader/sendcrypt_install.txt b/uploader/sendcrypt_install.txt new file mode 100644 index 0000000..7cf40c0 --- /dev/null +++ b/uploader/sendcrypt_install.txt @@ -0,0 +1,20 @@ +From the gitlab repository for sendCrypt, commit c203a915 (https://gitlab.sib.swiss/clinbio/sendcrypt/sendcrypt-cli) + +sh -c "$(wget -qO- https://gitlab.sib.swiss/clinbio/sendcrypt/sendcrypt-cli/-/raw/main/tools/install.sh)" + +echo -e 'export PATH="$HOME/.sendcrypt:$PATH"' >> ~/.bashrc +cp secrets default.sendcrypt-profile ~/.sendcrypt/profiles + +gpg --import secrets +gpg --edit-key trust +#set trust level to 5 + +gpg --full-generate-key +#use "RSA and RSA" and "4096" +#the expiry date has been set to two years (22 Sep 2025) +#the name associated to the kay has been set to XXX +#the address associated to the key has been set to bs-pangolin@ethz.ch + +#send key to openPGP server +gpg --send-keys + diff --git a/uploader/submission_metadata.py b/uploader/submission_metadata.py new file mode 100644 index 0000000..bb57739 --- /dev/null +++ b/uploader/submission_metadata.py @@ -0,0 +1,232 @@ +cantons = ["BL","ZH","SG","AI","ZG","GE","TG","OW","UR","GL","TI","SO","GR","FR","SZ","BS","JU","AG","AR","VS","NW","NE","VD","LU","BE","SH"] +mandatory_fields = ["species","strain_name","isolation_date","location_general","isolation_source_description","isolation_source_detailed","sequencing_purpose","sequencing_investigation_type","orig_fastq_name_forward","library_preparation_kit","sequencing_platform","assembly_method","reporting_lab_name","collecting_lab_name","reporting_authors"] +kit = { + "v3": "SARS-CoV-2 ARTIC V3", + "V3": "SARS-CoV-2 ARTIC V3", + "v4": "SARS-CoV-2 ARTIC V4", + "V4": "SARS-CoV-2 ARTIC V4", + "v41": "SARS-CoV-2 ARTIC V4.1", + "V41": "SARS-CoV-2 ARTIC V4.1", + "v4.1": "SARS-CoV-2 ARTIC V4.1", + "V4.1": "SARS-CoV-2 ARTIC V4.1", + "v532": "SARS-CoV-2 ARTIC V5.3.2", + "V532": "SARS-CoV-2 ARTIC V5.3.2", + "v5.3.2": "SARS-CoV-2 ARTIC V5.3.2", + "V5.3.2": "SARS-CoV-2 ARTIC V5.3.2", +} +collecting_lab = { + "1": "eawag", + "2": "eawag", + "3": "eawag", + "4": "eawag", + "5": "eawag", + "6": "eawag", + "7": "eawag", + "8": "eawag", + "9": "eawag", + "10": "eawag", + "12": "eawag", + "11.1": "eawag", + "11.2": "eawag", + "13": "eawag", + "14": "eawag", + "15": "eawag", + "16": "eawag", + "17": "eawag", + "18": "eawag", + "19": "eawag", + "20": "eawag", + "21": "eawag", + "22": "eawag", + "23": "eawag", + #"24": "eawag", + "25": "eawag", + "26": "eawag", + "32":"eawag", + "33":"eawag", + "34":"eawag", + "35":"eawag", + "36":"eawag", + "37":"eawag", + "99": "eawag", + "ba": "basel", + "klzhcov": "kzurich", + "klzhCov_Promega": "kzurich", + "558600": "microsynth", + "624801": "microsynth", + "680000": "microsynth", +} +population = { + "1": "27000", + "2": "30000", + "3": "12000", + "4": "15000", + "5": "124000", + "6": "25000", + "7": "54000", + "8": "14000", + "9": "51000", + "10": "471000", + "12": "248000", + "11.1": "43000", + "11.2": "43000", + "13": "55000", + "14": "225000", + "15": "274000", + "16": "454000", + "17": "55000", + "18": "", + "19": "64000", + "20": "", + "21": "", + "22": "45000", + "23": "", + #"24": "", + "25": "62000", + "26": "", + "32":"225000", + "33":"", + "34":"646000", + "35":"254000", + "36":"32000", + "37":"33000", + "ba": "273075", + "klzhcov": "473361", + "klzhCov_Promega": "473361", + "558600": "247824", + "624801": "32480", + "680000": "16320", +} +size = { + "1": "54.00219643288182", + "2": "", + "3": "", + "4": "57.96259543880482", + "5": "198.70257279974336", + "6": "118.04924802536674", + "7": "", + "8": "391.2999739700438", + "9": "384.43053891241505", + "10": "102.34846398120602", + "12": "88.90982585503468", + "11.1": "28.89310807274574", + "11.2": "28.89310807274574", + "13": "96.68757118163049", + "14": "105.11106081093753", + "15": "65.10327564165928", + "16": "126.19405027126015", + "17": "211.00560768931132", + "18": "176.55426928711", + "19": "104.18141542388327", + "20": "", + "21": "39.77221171816035", + "22": "13.240694396501643", + "23": "97.20041181629541", + #"24": "", + "25": "449.0738556277142", + "26": "", + "32":"164.77508882825475", + "33":"144.5003117634239", + "34":"50.868649327111015", + "35":"238.28580403442436", + "36":"112.93752962464872", + "37":"87.95865237426803", + "99": "", + "ba": "", + "klzhcov": "", + "klzhCov_Promega": "", + "558600": "", + "624801": "", + "680000": "", +} +region = { + "1": "", + "2": "", + "3": "", + "4": "", + "5": "", + "6": "", + "7": "", + "8": "", + "9": "", + "10": "", + "12": "", + "11.1": "", + "11.2": "", + "13": "", + "14": "", + "15": "", + "16": "", + "17": "", + "18": "", + "19": "", + "20": "", + "21": "", + "22": "", + "23": "", + #"24": "", + "25": "", + "26": "", + "32":"", + "33":"", + "34":"", + "35":"", + "36":"", + "37":"", + "99": "", + "ba": "Basel-Stadt/Switzerland, Basel-Landschaft/Switzerland, Germany, France", + "klzhcov": "Kanton Zurich", + "klzhCov_Promega": "Kanton Zurich", + "558600": "", + "624801": "", + "680000": "", +} +collectinglab = { + "eawag": "Eawag, Swiss Federal Institute of Aquatic Science and Technology", + "basel": "State Laboratory of Basel-Stadt, Basel, Switzerland; University of Basel, Basel, Switzerland; Division of Infectious Diseases and Hospital Epidemiology, University Hospital Basel, Basel, Switzerland", + "microsynth": "Microsynth AG, Schützenstrasse 15, 9436 Balgach, Switzerland", + "kzurich": "Cantonal laboratory Zurich", +} +## for eawag we have changes in authors depending on date. What we have is eawag_ where indicates that any entry oldere than that should have that specific list +authors = { + "eawag_20000101_20181131": "Christoph Ort, Tamar Kohn, Timothy R. Julian, Lea Caduff", + "eawag_20181209_20200131": "Christoph Ort, Tamar Kohn, Timothy R. Julian, Lea Caduff, Elyse Stachler", + "eawag_20200201_20200229": "Christoph Ort, Tamar Kohn, Timothy R. Julian, Lea Caduff, Pravin Ganesanandamoorth, Xavier Fernandez-Cassi, Elyse Stachler", + "eawag_20200301_20200831": "Christoph Ort, Tamar Kohn, Timothy R. Julian, Lea Caduff, Pravin Ganesanandamoorth, Xavier Fernandez-Cassi, Elyse Stachler, Carola Bänziger", + "eawag_20200901_20200930": "Christoph Ort, Tamar Kohn, Timothy R. Julian, Lea Caduff, Pravin Ganesanandamoorth, Xavier Fernandez-Cassi, Elyse Stachler, Carola Bänziger, Anina Kull", + "eawag_20201001_20201031": "Christoph Ort, Tamar Kohn, Timothy R. Julian, Lea Caduff, Pravin Ganesanandamoorth, Xavier Fernandez-Cassi, Elyse Stachler, Carola Bänziger, Anina Kull, Federica Cariti", + "eawag_20201101_20210131": "Christoph Ort, Tamar Kohn, Timothy R. Julian, Lea Caduff, Pravin Ganesanandamoorth, Xavier Fernandez-Cassi, Elyse Stachler, Anina Kull, Federica Cariti", + "eawag_20210201_20210531": "Christoph Ort, Tamar Kohn, Timothy R. Julian, Lea Caduff, Pravin Ganesanandamoorth, Xavier Fernandez-Cassi, Elyse Stachler, Anina Kull, Federica Cariti, Alexander J. Devaux", + "eawag_20210601_20210831": "Christoph Ort, Tamar Kohn, Timothy R. Julian, Lea Caduff, Pravin Ganesanandamoorth, Xavier Fernandez-Cassi, Elyse Stachler, Anina Kull, Federica Cariti, Alexander J. Devaux, Blanche Wies", + "eawag_20210901_20210930": "Christoph Ort, Tamar Kohn, Timothy R. Julian, Lea Caduff, Pravin Ganesanandamoorth, Xavier Fernandez-Cassi, Elyse Stachler, Federica Cariti, Alexander J. Devaux, Charlie Gan", + "eawag_20211001_20211031": "Christoph Ort, Tamar Kohn, Timothy R. Julian, Lea Caduff, Pravin Ganesanandamoorth, Xavier Fernandez-Cassi, Federica Cariti, Alexander J. Devaux, Charlie Gan", + "eawag_20211101_20211231": "Christoph Ort, Tamar Kohn, Timothy R. Julian, Lea Caduff, Pravin Ganesanandamoorth, Federica Cariti, Alexander J. Devaux, Franziska Böni, Johannes Rusch, Charlie Gan", + "eawag_20220101_20220229": "Christoph Ort, Tamar Kohn, Timothy R. Julian, Lea Caduff, Federica Cariti, Alexander J. Devaux, Franziska Böni, Johannes Rusch, Charlie Gan", + "eawag_20220301_20220430": "Christoph Ort, Tamar Kohn, Timothy R. Julian, Lea Caduff, Federica Cariti, Alexander J. Devaux, Franziska Böni, Johannes Rusch, Laura Brülisauer, Camila Morales Undurraga, Charlie Gan", + "eawag_20220501_20220930": "Christoph Ort, Tamar Kohn, Timothy R. Julian, Lea Caduff, Alexander J. Devaux, Franziska Böni, Johannes Rusch, Laura Brülisauer, Camila Morales Undurraga, Charlie Gan", + "eawag_20221001_20230131": "Christoph Ort, Tamar Kohn, Timothy R. Julian, Lea Caduff, Alexander J. Devaux, Franziska Böni, Johannes Rusch, Laura Brülisauer, Aurélie Holschneider, Charlie Gan", + "eawag_20230201_20230228": "Christoph Ort, Tamar Kohn, Timothy R. Julian, Lea Caduff, Alexander J. Devaux, Franziska Böni, Laura Brülisauer, Seju Kang, Aurélie Holschneider, Charlie Gan", + "eawag_20230301_20230331": "Christoph Ort, Tamar Kohn, Timothy R. Julian, Lea Caduff, Alexander J. Devaux, Seju Kang, Ayazhan Dauletova, Camille Hablützel, Rachel McLeod, Aurélie Holschneider, Charlie Gan", + "eawag_20230401_25000101": "Christoph Ort, Tamar Kohn, Timothy R. Julian, Lea Caduff, Seju Kang, Ayazhan Dauletova, Camille Hablützel, Rachel McLeod, Daniela Yordanova, Jolinda de Korne, Charlie Gan", + "basel_20000101_25000101": "Claudia Bagutti, Evelyn Ilg Hampe, Sarah Tschudin Sutter", + "microsynth_20000101_25000101": "Christoph Gruenig, Maria-Luise Deflorian", + "kzurich_20000101_25000101": "Nadine Gerber, Natalie Meyer, René Köppel", + "ethz_20000101_25000101": "Katharina Jahn, Pelin Burcak Icer, David Dreifuss, Ivan Topolsky, Lara Fuhrmann, Kim Philipp Jablonski, Anika John, Matteo Carrara, Franziska Singer, Chaoran Chen, Sarah Nadeau, Niko Beerenwinkel, Tanja Stadler", + "fgcz_20000101_25000101": "Catharine Aquino, Lennart Opitz, Tim Sykes", +} +seqplatform = "Combination Illumina MiSeq, Illumina NovaSeq 5000/6000, NextSeq2000" +reportinglab = "Department of Biosystems Science and Engineering, ETH Zurich; Mattenstrasse 26, 4058 Basel" +qafile = "/app/dataset/working/qa.csv" +seqcenter = { + "fgcz": "Functional Genomics Center Zurich", +} +centerused = "fgcz" +locations = "/app/workdir/uploader/resources/ww_locations.tsv" +basedir = "/app/uploader" +samplesfolder = "/app/dataset/working/samples" +timelinefile = "/app/dataset/working/timeline.tsv" +assembly = "V-pipe" +embargo = "" +projyears = ["2020", "2021", "2022", "2023", "2024"] +submitting = "ETHZ" +exceptions = {"558600":"Lausanne (VD)", "624801":"Sierre/Noes (VS)", "680000":"Porrentruy (JU)"} diff --git a/uploader/upload.sh b/uploader/upload.sh new file mode 100755 index 0000000..a3e516d --- /dev/null +++ b/uploader/upload.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +scriptdir=/app/pangolin_src + +if [ -z $1 ] || [ ! -d $1 ]; then + echo "ERROR: the script uploader.sh requires the archive folder as parameter. Parameter not set of directory no found." + exit 1 +fi + +. ${scriptdir}/config/server.conf +cd ${uploader_code} +source ${baseconda}/etc/profile.d/conda.sh +conda activate sendcrypt + +set -eu + +# don't keep non-matching globs: +shopt -s nullglob + +function bash_traceback() { + local lasterr="$?" + set +o xtrace + local code="-1" + local bash_command=${BASH_COMMAND} + echo "Error in ${BASH_SOURCE[1]}:${BASH_LINENO[0]} ('$bash_command' exited with status $lasterr)" >&2 + if [ ${#FUNCNAME[@]} -gt 2 ]; then + # Print out the stack trace described by $function_stack + echo "Traceback of ${BASH_SOURCE[1]} (most recent call last):" >&2 + for ((i=0; i < ${#FUNCNAME[@]} - 1; i++)); do + local funcname="${FUNCNAME[$i]}" + [ "$i" -eq "0" ] && funcname=$bash_command + echo -e " ${BASH_SOURCE[$i+1]}:${BASH_LINENO[$i]}\\t$funcname" >&2 + done + fi + echo "Exiting with status ${code}" >&2 + exit "${code}" +} + +# provide an error handler whenever a command exits nonzero +# propagate ERR trap handler functions, expansions and subshells +set -o errtrace + +export TMPDIR=${uploader_tempdir} + +echo "Creating the necessary files and directories" +if [ -f ${TMPDIR}/uploaded_run.txt ]; then + rm ${TMPDIR}/uploaded_run.txt +fi + +export target=${TMPDIR}/target +if [ -d $target ]; then + rm -rf $target +fi +mkdir -p $target +tsv=${target}/meta_data.tsv +echo $tsv +archive_now=$1 + +echo "Initializing the submission metadata" +echo -e "is_assembly_update\tspecies\tstrain_name\tisolation_date\tlocation_general\tlocation_city\tlocation_geocoordinates\tisolation_source_description\tisolation_source_detailed\tisolation_source_name\tisolation_source_size_catchment_area\tisolation_source_population_size_catchment_area\tisolation_source_regions_catchment_area\tsequencing_purpose\tsequencing_investigation_type\torig_fastq_name_forward\tlibrary_preparation_kit\tsequencing_lab_name\tsequencing_platform\tassembly_method\traw_dataset_coverage\treporting_lab_name\tcollecting_lab_name\treporting_authors\traw_dataset_embargo\tgenbank_identifier\tENA_accession"> $tsv + +echo "Retrieving CRAM files and adding their metadata line" + +cat ${TMPDIR}/to_upload.txt | +while read samplename batch; do + samplename=$(echo $samplename | tr -d '"') + if [[ $samplename =~ [A-H][0-9]_24_.* ]]; then + echo "skipped ski resort $samplename" | tee -a ${archive_now}/not_found.txt + continue + fi + if grep -q ${samplename} ${uploader_blacklist}; then + echo "skipping ${samplename} as per blacklist" + continue + fi + batch=$(echo $batch | tr -d '"') + echo "$samplename $batch" + X=${uploader_dataset}/working/samples/${samplename}/${batch}/uploads/dehuman.cram + if [ -f $X ]; then + cp $(realpath $X) $target/${samplename}.cram + python3 ${uploader_code}/create_metadata_line.py -s ${samplename} -b ${batch} -o $tsv + echo $samplename >> ${archive_now}/uploaded_run.txt + else + echo "not found $samplename" | tee -a ${archive_now}/not_found.txt + fi +done + +echo $tsv + +ls -l $target +echo +echo "Everything is ready to run sendCrypt" + diff --git a/work-catchup-dehuman/ketchup.sh b/work-catchup-dehuman/ketchup.sh deleted file mode 100755 index b6d0681..0000000 --- a/work-catchup-dehuman/ketchup.sh +++ /dev/null @@ -1,250 +0,0 @@ -#!/usr/bin/env bash - -umask 0007 - -scriptdir="$(dirname $(which $0))" -baseconda="$scriptdir/../" - -# -# Input validator -# - -validateDate() { - if [[ "$1" =~ ^(20[0-9][0-9][0-1][0-9]([0-3][0-9]?)?)$ ]]; then - return; - else - echo "bad date ${1}" - exit 1; - fi -} - -validateBatchName() { - if [[ "$1" =~ ^(20[0-9][0-9][0-1][0-9][0-3][0-9]_[[:alnum:]-]{4,})$ ]]; then - return; - else - echo "bad batchname ${1}" - exit 1; - fi -} - -validateTags() { - for b in "${@}"; do - validateBatchName "${b}" - done -} - - -lastmonth=$(date '+%Y%m' --date='-1 month') - -case "$1" in - samples) - list_tsv=( ) - case "$2" in - --recent) - list_tsv=( "${scriptdir}/../working/samples.recent.tsv" ) - shift - ;; - --date) - shift - validateDate "${2}" - list_tsv=( $(echo "../sampleset/samples.${2}"*".tsv" ) ) - shift - ;; - --tsv) - shift - if [[ ! -r "${2}" ]]; then - echo "Cannot read ${2}" - exit 1 - fi - # TODO validate - list_tsv=( "${2}" ) - shift - ;; - esac - shift - - # get batches - if [[ -n "${*}" ]]; then - validateTags "${@}" - for b in "${@}"; do - s="${scriptdir}/../sampleset/samples.${b}.tsv" - if [[ ! -s "${s}" ]]; then - echo "missing sample file ${s}" > /dev/stderr - exit 2 - fi - list_tsv+=( "${s}" ) - done - fi - - # check everything is normal - if (( ${#list_tsv[@]} == 0 )); then - echo "usage: ${0} samples [ --recent | --date | --tsv ] ..." > /dev/stderr - exit 2 - fi - - echo "importing samples from:" - printf ' - %s\n' "${list_tsv[@]}" - sort -u "${list_tsv[@]}" > "${scriptdir}/samples.catchup.tsv" - wc -l "${scriptdir}/samples.catchup.tsv" - - rm -rf "${scriptdir}/samples/" - mkdir -p "${scriptdir}/samples/" - cut -f1 "${scriptdir}/samples.catchup.tsv" | sort -u | while read s; do - o="${scriptdir}/../working/samples/${s}" - if [[ -d "${o}" ]]; then - ln -fvs "$(realpath --relative-to="${scriptdir}/samples/" "${o}")" "${scriptdir}/samples/"; - fi; - done - find "${scriptdir}/.snakemake/metadata/" "${scriptdir}/.snakemake/incomplete/" "${scriptdir}/cluster_logs/" -type f -print0 | xargs -r0 rm - echo -e '\n\e[38;5;45;1mI ve experiments to run\e[0m\n\e[38;5;208;1mThere is research to be done\e[0m' - ;; - - scratch) - olderthan=60 - purge=0 - loop=0 - filter= - while [[ -n $2 ]]; do - case "$2" in - --minutes-ago) - if [[ ! "${3}" =~ ^[[:digit:]]+$ ]]; then - echo "parameter of ${2} must be a number of minutes (digits only), got <${3}> instead" > /dev/stderr - exit 2 - fi - shift - olderthan=$2 - ;; - --loop) - loop=1 - filter='-name dehuman.sam' - ### dh_aln.sam - ### reject_R2.fastq.gz - ### reject_R1.fastq.gz - ### host_aln.sam - ### dehuman.filter - ;& - --purge) - purge=1 - ;; - *) - echo "Unkown parameter ${2}" > /dev/stderr - exit 2 - ;; - esac - shift - done - - hourago="${SCRATCH}/hourago.touch" - temp_scratch="/cluster/scratch/bs-pangolin/pangolin/temp" - touch --date="${olderthan} minutes ago" "${hourago}" - - if (( purge )); then - echo "purging in ${temp_scratch}..." - else - echo "listing in ${temp_scratch}..." - fi - - (cd "${temp_scratch}/" && find samples/ -type f ${filter} ) | grep -oP '(?<=samples/)[^/]+/[^/]+(?=/)' | sort -u | while read s; do - if [[ -s "${scriptdir}/samples/${s}/raw_uploads/dehuman.cram.md5" && "${scriptdir}/samples/${s}/raw_uploads/dehuman.cram.md5" -ot "${hourago}" ]]; then - #if [[ -r "${scriptdir}/../working/samples/${s}/upload_prepared.touch" && "${scriptdir}/../working/samples/${s}/upload_prepared.touch" -ot "${hourago}" ]]; then - if (( purge )); then - rm -rvf "${temp_scratch}/samples/${s}" - else - echo "${s}"; - fi - fi; - done | tee /dev/stderr | wc -l 2>&1 - - if (( loop )); then - lquota -2 ${SCRATCH} - echo -e '\n\e[38;5;45;1mYou just keep on trying\e[0m\n\e[38;5;208;1mTill you run out of cake\e[0m' - if sleep "$(( 5 + olderthan))m"; then - # loop if no breaks - exec "${0}" scratch --minutes-ago "${olderthan}" --loop - fi - echo "I'm not even angry" - fi - ;; - - check_reads) - . $baseconda/miniconda3/bin/activate "qa" - while read s b o; do - declare -a r; - r=( $(gawk -v s="${s}" -v b="${b}" 'BEGIN{FS=","};FNR==1{for(f=1;f cr )); then - echo "${cram}" | tee -a cram_error.txt; - else - echo "."; - fi; - done < samples.catchup.tsv - ;; - - delete_raw) - cmd='printf %s\n' - msg='\r\e[1mConsider %s/%b for deletion\e[0m\n' - if [[ -n "$2" ]]; then - case "$2" in - --do-it) - cmd='rm -v' - msg='\r\e[36;1mDelete raw from %s/%b\e[0m\n' - ;; - *) - echo "Unkown parameter ${2}" > /dev/stderr - exit 2 - ;; - esac - fi - - sort -u "${scriptdir}/samples.catchup.tsv" | while read s b o; do - if [[ "${b}" > "${lastmonth}" ]]; then - echo -e "\e[31;1mSkipping ${s}/${b} : too recent (${lastmonth})\e[0m" - continue - fi - - if [[ - -s "${scriptdir}/samples/${s}/${b}/raw_uploads/dehuman.cram.md5" - && -s "${scriptdir}/samples/${s}/${b}/raw_uploads/dehuman.cram" - && -e "${scriptdir}/samples/${s}/${b}/upload_prepared.touch" - && -s "${scriptdir}/samples/${s}/${b}/alignments/REF_aln_trim.bam" - && -s "${scriptdir}/samples/${s}/${b}/references/consensus.bcftools.fasta" - ]]; then - printf "${msg}" "${s}" "${b}" - else - echo -n '.' - continue - fi - - ${cmd} "${scriptdir}/samples/${s}/${b}/"{raw_data,preprocessed_data}/*.fastq.gz "${scriptdir}/../sampleset/${s}/${b}/raw_data/"*.fastq.gz "${scriptdir}/samples/${s}/${b}/alignments/"REF_aln.bam* - (cd "${scriptdir}/samples/" && rmdir --ignore-fail-on-non-empty --parents "${s}/${b}/raw_data" ) - (cd "${scriptdir}/../sampleset/" && rmdir --ignore-fail-on-non-empty --parents "${s}/${b}/raw_data" ) - done - echo -e '\r\e[K\e[38;5;45;1mThis was a triumph! I m making a note here; "Huge success"\e[0m' - ;; - - still_around) - sort -u "${scriptdir}/samples.catchup.tsv" | while read s b o; do - if [[ -d "${scriptdir}/../sampleset/${s}/${b}/raw_data/" || -f "${scriptdir}/samples/${s}/${b}/alignments/REF_aln.bam" ]]; then - echo -e "\r\e[K${s}/${b} still around" - else - echo -n '.' - fi - done - echo -e '\r\e[K\e[38;5;208;1mThe cake is a lie!\e[0m' - ;; - - df) - exec "${scriptdir}/../batman.sh" "df" - echo "Cannot find batman.sh" - exit 2 - ;; - - *) - echo "Unkown subcommand ${1}" > /dev/stderr - exit 2 - ;; -esac diff --git a/working/bsub_wrap.sh b/working/bsub_wrap.sh deleted file mode 100755 index 575c21e..0000000 --- a/working/bsub_wrap.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -exec bsub "${@:1:$((${#@}-1))}" < "${@:$((${#@}))}" - -# this causes snakemake to pipe the intended comming into bsub, instead of executing a separate shell script -# circumvents the shell script not being visible yet on target node diff --git a/working/gather1qa b/working/gather1qa index 748f296..1ed755a 100644 --- a/working/gather1qa +++ b/working/gather1qa @@ -1,53 +1,48 @@ #!/bin/bash -#BSUB -L /bin/bash -#BSUB -J COVID-qa -#BSUB -u "ivan.topolsky@bsse.ethz.ch" # singer@nexus.ethz.ch carrara@nexus.ethz.ch louis.duplessis@bsse.ethz.ch sarah.nadeau@bsse.ethz.ch" # chaoran.chen@bsse.ethz.ch -##### -N -#BSUB -n 1 -#BSUB -M 4096 -#BSUB -R rusage[mem=4096] -#BSUB -W 4:00 +#SBATCH --job-name="COVID-qa" +#SBATCH --mail-user="ivan.topolsky@bsse.ethz.ch" # carrara@nexus.ethz.ch # singer@nexus.ethz.ch louis.duplessis@bsse.ethz.ch sarah.nadeau@bsse.ethz.ch" # chaoran.chen@bsse.ethz.ch +#SBATCH --mail-type=FAIL +#SBATCH --cpus-per-task=1 +#SBATCH --mem-per-cpu=4096 +#SBATCH --time="4:00:00" batches=() +# if script wasn't editted on-the-fly, but parameters where passed +#if (( ${#batches[@]} == 0 )) && (( $# )); then +if (( $# )); then + batches+=( "$@" ) +fi + umask 0007 -if (( $LSB_JOBINDEX )); then - echo "JobIdx $LSB_JOBINDEX" +idx="$SLURM_ARRAY_TASK_ID" + +if [[ "$idx" =~ ^[0-9]+$ ]] && (( idx >= 0 )); then + echo "JobIdx $idx" - batch=${batches[$LSB_JOBINDEX]} + batch=${batches[$idx]} # check if samplename is ok if [[ -z "${batch}" ]]; then - echo "wrong job index ${LSB_JOBINDEX}" + echo "wrong job index ${idx}" echo "Max is ${#batches[@]}" exit 1; fi input_tsv="${TMPDIR:+$TMPDIR/}samples.qa-${batch}.tsv" output_csv="qa/qa.${batch}.csv" echo "batch: ${batch}" + echo "output: ${output_csv}" gawk -v b="${batch}" '$2==b' < samples.tsv > ${input_tsv} echo "tsv: ${input_tsv}" wc -l ${input_tsv} else echo "to run with all, use:" - echo " - bsub -J \"COVID-qa[1-${#batches[@]}]\" < gather1qa" + echo " - sbatch --array=$(( ${batch[0]:--1} * -1))-$(( ${#batches[@]} - 1)) gather1qa $*" exit 1 fi -#. ../miniconda3/bin/activate 'qa' -# channels: -# - conda-forge -# - bioconda -# - defaults -# dependencies: -# - emboss -# - samtools - -searchsamtools=( ../snake-envs/*/bin/samtools ) -samtools=${searchsamtools[0]} - return_fail=0 # Helper @@ -240,7 +235,6 @@ while read subject sublvl len; do [[ $prinseq =~ Good\ sequences\ \(pairs\):\ ([[:digit:],]+) ]] && qadata['goodpairs']="${BASH_REMATCH[1]//,/}" || oops "missing good pairs" # 2.2 count bamfile - #qadata['alnreads']=$(${samtools} view -c ${sample}/alignments/REF_aln.bam) alnstat=$(<${sample}//alignments/REF_aln_stats.yaml) for k in ${alnkeys[@]}; do S="${alnsearch[${k}]}" diff --git a/working/qa-launcher b/working/qa-launcher index 4a52c3a..92bc042 100755 --- a/working/qa-launcher +++ b/working/qa-launcher @@ -1,4 +1,11 @@ #!/bin/bash +#SBATCH --job-name="COVID-qa" +#SBATCH --mail-user="ivan.topolsky@bsse.ethz.ch carrara@nexus.ethz.ch" +#SBATCH --mail-type=END +#SBATCH --cpus-per-task=1 +#SBATCH --mem-per-cpu=1024 +#SBATCH --time="4:00:00" +#SBATCH --oversubscribe umask 0007 @@ -16,9 +23,6 @@ validateBatchName() { fi } -RXJOB='Job <([[:digit:]]+)> is submitted' -# Generic job. -# Job <129052039> is submitted to queue . usage() { echo "Usage: $0 [ -r | -a | ... ]" 1>&2; exit $1; } @@ -46,6 +50,7 @@ for b in "${batches[@]}"; do validateBatchName "${b}" done +echo "Batches: ${batches[*]}" echo "Total batches: ${#batches[*]}" @@ -53,17 +58,12 @@ echo "Total batches: ${#batches[*]}" # submit to LSF # -if [[ "$(sed -r 's@^(batches=).*$@\1( "" '"$(printf '"%s" ' "${batches[@]}" )"')@' gather1qa | bsub -J "COVID-qa[1-${#batches[@]}]" | tee /dev/stderr)" =~ ${RXJOB} ]]; then - job="${BASH_REMATCH[1]}" -else - echo "Failed job submission" -fi -# schedule a assembly no mater what happens -echo "Waiting for ${job} to finish..." -bwait -w "ended(${job})" +# run sub-jobs and waits for completion +sbatch --wait --parsable --array="0-$(( ${#batches[@]} - 1 ))" gather1qa "${batches[@]}" echo "Assembling..." +# TODO this can be simplified by using xsv to merge the csv files ( for b in "${batches[@]}"; do csv="qa/qa.${b}.csv" diff --git a/working/qa_report.ipynb b/working/qa_report.ipynb deleted file mode 100644 index 946484c..0000000 --- a/working/qa_report.ipynb +++ /dev/null @@ -1,1167 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "coveragefile = 'variants/coverage.tsv'\n", - "input_qafile = 'qa.csv'\n", - "output_qafile = 'qa_vals.csv'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from IPython.core.display import display, HTML\n", - "display(HTML(\n", - " ''\n", - "))\n", - "\n", - "display(HTML('''\n", - "'''))\n", - "\n", - "import warnings\n", - "warnings.filterwarnings('ignore')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "from matplotlib.colors import ListedColormap\n", - "import seaborn as sns\n", - "sns.set_style('ticks')\n", - "\n", - "import plotly.graph_objs as go\n", - "from plotly.offline import init_notebook_mode, iplot\n", - "import plotly.figure_factory as ff\n", - "import plotly.io as pio\n", - "\n", - "\n", - "init_notebook_mode()\n", - "\n", - "from ipywidgets import interact\n", - "from IPython.display import display, Markdown, Latex" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import re\n", - "def extract_sample_conditions(samples):\n", - " '''Function to parse a list of sample names and return their condition.\n", - " Parameters:\n", - " samples: list of sample names\n", - " Returns:\n", - " (condition, condition_extended): tuple of np.arrays. \n", - " condition is a np.array where 0 = negative control,\n", - " 1 = positive control, 2 = experimental condition, -1 = unassigned. \n", - " condition_extended is a np.array where \n", - " 0 = H2O negative control, 1 = empty well negative control, 2 = negative PCR tests,\n", - " 3 = negative controls of Functional Genomics ZH, \n", - " 4 = Twist positive control, 5 = Positive Controls (TO CONFIRM),\n", - " 6 = ETHZ_ID sample, 7 = wastewater sample, 8 = functional genomics center sample, \n", - " 9= Basel Sequencing of UZH Virology lab, 10 = Labormedizinisches Zentrum Dr Risch, 11 = USZ Tier sample,\n", - " -1 = unassigned\n", - " '''\n", - " # make a list of regexps\n", - " re_list = [\n", - " # regexp for H2O negative controls eg: H2O_CP002_A7\n", - " re.compile(\"^H2O\"),\n", - " # regexp for Empty wells negative controls eg: EMPTY_CP002_A11\n", - " re.compile(\"^((EMPTY)|(empty))\"),\n", - " # regexp for samples of negative tests eg: neg_109_B2\n", - " re.compile(\"^neg_\"),\n", - " # regexp for negative controls of Functional Genomics ZH eg: NTC_NA_NTC_NA\n", - " re.compile(\"^NTC_NA_NTC_NA\"),\n", - " # regexp for samples of Twist positive controls eg: pos_MN908947_3_1_100000_CP0002\n", - " re.compile(\"^(pos_)|(Twist_control)\"),\n", - " # regexp for samples of Positive Controls (TO CONFIRM) eg: CoV_ctrl_1_1_10000\n", - " re.compile(\"CoV_ctrl_\"),\n", - " # regexp for ETH ID samples eg: 160000_434_D02\n", - " re.compile(\"^[0-9]{6}(_Plate){0,1}_(p){0,1}[0-9]+_\"),\n", - " # regexp for wastewater samples eg: 09_2020_03_24_B\n", - " re.compile(\"^[0-9]{2}_202[0-9]_\"),\n", - " # regexp for functional genomics eg: 30430668_Plate_8_041120tb3_D7\n", - " re.compile(\"^[0-9]{8}_Plate_[0-9]+\"),\n", - " # regexp for Basel sequencing for some UZH lab (UZH Virology ?) eg: A2_722\n", - " re.compile(\"^[A-Z][0-9]_[0-9]+\"),\n", - " # regexp for lone sample by the lab \"Labormedizinisches Zentrum Dr Risch\": 674597001\n", - " re.compile(\"^674597001\"),\n", - " # regexp for USZ Tier eg: USZ_5_Tier\n", - " re.compile(\"^USZ_[0-9]_Tier\")\n", - " ]\n", - "\n", - " # make 0-1 array with n_samples lines and p_conditions columns\n", - " re_matches = [[int(regex.match(sam) is not None) for sam in samples] for regex in re_list]\n", - " re_matches_array = np.array(re_matches)\n", - "\n", - " # check that samples are assigned to 1 condition max\n", - " if np.max(np.sum(re_matches_array, axis=0)) > 1:\n", - " print(\"Warning: some samples were assigned to multiple conditions.\")\n", - "\n", - " # assign to each its extended condition\n", - " condition_extended = np.argmax(re_matches_array, axis=0)\n", - " condition_extended = condition_extended - (np.sum(re_matches_array, axis=0) == 0) # set unassigned to -1\n", - "\n", - " # assign to each its condition\n", - " condition = np.in1d(condition_extended, [0,1,2,3])*0\n", - " condition = condition + np.in1d(condition_extended, [4,5])*1\n", - " condition = condition + np.in1d(condition_extended, [6,7,8,9,10,11])*2\n", - " condition = condition + np.in1d(condition_extended, [-1])*(-1)\n", - " \n", - " return (condition, condition_extended)\n", - " " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "data = pd.read_csv(input_qafile)\n", - "data = data.set_index('sample')\n", - "data = data.reset_index().rename(columns={'index':'sample'})\n", - "\n", - "coverage = pd.read_csv(coveragefile, sep='\\t',index_col=['ref','pos']).droplevel('ref',axis=0)\n", - "coverage = coverage.T\n", - "\n", - "data['input_reads'] = data['input_r1'] + data['input_r2']\n", - "data['goodreads'] = data['goodpairs'] * 2\n", - "data['conditions'], data['conditions_extended'] = extract_sample_conditions(data['sample'])", - "\n", - "data['group'] = 'Real'\n", - "data.loc[data.conditions_extended == 0] = 'H2O'\n", - "data.loc[data.conditions_extended == 1] = 'Empty'\n", - "data.loc[data.conditions_extended == 2] = 'Negative'\n", - "data.loc[data.conditions == 2] = 'Positive'\n", - "\n", - "real_samples = data.loc[data.conditions == 2]\n", - "h2o_samples = data.loc[data.conditions_extended == 0]\n", - "emp_samples = data.loc[data.conditions_extended == 1]\n", - "neg_samples = data.loc[data.conditions_extended == 2]\n", - "pos_samples = data.loc[data.conditions == 2]\n", - "\n", - "subsets_dict = dict(All=data, Real=real_samples, Empty=emp_samples, H2O=h2o_samples, Negative=neg_samples, \n", - " Positive=pos_samples)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "sizes = dict()\n", - "for key in subsets_dict:\n", - " sizes[key] = subsets_dict[key].shape[0]\n", - "sizes = pd.DataFrame(sizes, index=range(1))\n", - "\n", - "groupsize = sizes.T.reset_index().rename(columns={'index': 'group', 0: 'size'})" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "coverage_quartiles = coverage.T.describe().loc[['25%', '50%', '75%']]\n", - "\n", - "coverage_quartiles = coverage_quartiles.T.reset_index().rename(columns={'index': 'sample'})\n", - "\n", - "coverage_quartiles['sample'] = coverage_quartiles['sample'].apply(lambda s: s.split('-')[0])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "coverage = coverage.reset_index(drop=True)\n", - "real_mean = coverage.loc[data['group']=='Real'].quantile(q=0.5, axis=0)\n", - "real_1q = coverage.loc[data['group']=='Real'].quantile(q=0.25, axis=0)\n", - "real_3q = coverage.loc[data['group']=='Real'].quantile(q=0.75, axis=0)\n", - "empty_mean = coverage.loc[data['group']=='Empty'].quantile(q=0.5, axis=0)\n", - "empty_1q = coverage.loc[data['group']=='Empty'].quantile(q=0.25, axis=0)\n", - "empty_3q = coverage.loc[data['group']=='Empty'].quantile(q=0.75, axis=0)\n", - "h2o_mean = coverage.loc[data['group']=='H2O'].quantile(q=0.5, axis=0)\n", - "h2o_1q = coverage.loc[data['group']=='H2O'].quantile(q=0.25, axis=0)\n", - "h2o_3q = coverage.loc[data['group']=='H2O'].quantile(q=0.75, axis=0)\n", - "neg_mean = coverage.loc[data['group']=='Negative'].quantile(q=0.5, axis=0)\n", - "pos_mean = coverage.loc[data['group']=='Positive'].quantile(q=0.5, axis=0)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "coverage_pos_percentiles = pd.DataFrame(columns={'real'})" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "quantity = 'basequal'\n", - "read_no = 1\n", - "quantity = f'r{read_no}_{quantity}'\n", - "\n", - "percs = dict()\n", - "for key in subsets_dict:\n", - " percs[key] = subsets_dict[key][f'fastqc_{quantity}'].value_counts(normalize=True)\n", - "\n", - "fqcpercs = pd.DataFrame(percs).fillna(0) * 100\n", - "#fqcpercs = fqcpercs.loc[['PASS','WARNING','FAIL','']]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "upper_bound_reals = 25\n", - "lower_bound_controls = 90 \n", - "\n", - "percs = dict()\n", - "nums = dict()\n", - "for key in subsets_dict:\n", - " percs[key] = (1. - subsets_dict[key]['goodpairs']/(subsets_dict[key][['input_r1', 'input_r2']].sum(axis=1)/2)) * 100\n", - " nums[key] = (subsets_dict[key][['input_r1', 'input_r2']].sum()/2) - subsets_dict[key]['goodpairs']\n", - " \n", - "percs = pd.DataFrame(percs)\n", - "nums = pd.DataFrame(nums)\n", - "\n", - "rejreads = percs\n", - "rejreads['sample'] = data['sample']\n", - "rejreads['Absolute'] = ((data[['input_r1', 'input_r2']].sum(axis=1)/2) - data['goodpairs']).astype(int)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "lower_bound_reals = 90\n", - "upper_bound_controls = 10\n", - "\n", - "percs = dict()\n", - "for key in subsets_dict:\n", - " percs[key] = subsets_dict[key]['alnreads'] / (subsets_dict[key]['goodpairs']*2.) * 100\n", - "\n", - "percs = pd.DataFrame(percs)\n", - "\n", - "alnreads = percs\n", - "alnreads['sample'] = data['sample']\n", - "alnreads['Absolute'] = data['alnreads']" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "bwa_insert_dist = data[['sample', 'bwa_insert_min', 'bwa_insert_mean', 'bwa_insert_max']]\n", - "bwa_insert_dist = bwa_insert_dist.rename(columns={'bwa_insert_min':'min', 'bwa_insert_mean':'mean', 'bwa_insert_max':'max'})" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "vals_N = dict()\n", - "vals_lc = dict()\n", - "for key in subsets_dict:\n", - " vals_N[key] = subsets_dict[key]['consensus_N']\n", - " vals_lc[key] = subsets_dict[key]['consensus_lower']\n", - "\n", - "vals_N = pd.DataFrame(vals_N)\n", - "vals_lc = pd.DataFrame(vals_lc)\n", - "\n", - "lcbases = vals_lc\n", - "lcbases['sample'] = data['sample']\n", - "\n", - "nbases = vals_N\n", - "nbases['sample'] = data['sample']" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "whole_genome_length = 29903\n", - "vals = dict()\n", - "for key in subsets_dict:\n", - " vals[key] = whole_genome_length - subsets_dict[key]['consensus_N'] - subsets_dict[key]['match_id']\n", - " \n", - "vals = pd.DataFrame(vals)\n", - "\n", - "diffbases = vals\n", - "diffbases['sample'] = data['sample']" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "vals_snv = dict()\n", - "vals_f = dict()\n", - "vals_maj = dict()\n", - "for key in subsets_dict:\n", - " vals_snv[key] = subsets_dict[key]['shorah_snv']\n", - " vals_f[key] = subsets_dict[key]['shorah_filtered']\n", - " vals_maj[key] = subsets_dict[key]['shorah_majority']\n", - "\n", - "vals_snv = pd.DataFrame(vals_snv)\n", - "vals_f = pd.DataFrame(vals_f)\n", - "vals_maj = pd.DataFrame(vals_maj)\n", - "\n", - "filtered_snvs = vals_f\n", - "filtered_snvs['sample'] = data['sample']\n", - "maj_snvs = vals_maj\n", - "maj_snvs['sample'] = data['sample']" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "flags = pd.DataFrame({'sample':data['sample']})\n", - "flags_values = pd.DataFrame({'sample':data['sample']})\n", - "\n", - "flags['coverage'] = 'PASS'\n", - "assert (coverage_quartiles['sample'] == flags['sample']).all()\n", - "coverage_3q_min_warning = 2000\n", - "coverage_3q_min_fail = 1000\n", - "coverage_failed = coverage_quartiles.loc[coverage_quartiles['75%'] < coverage_3q_min_warning]['sample'].index\n", - "flags.loc[coverage_failed, 'coverage'] = 'WARNING'\n", - "coverage_failed = coverage_quartiles.loc[coverage_quartiles['75%'] < coverage_3q_min_fail]['sample'].index\n", - "flags.loc[coverage_failed, 'coverage'] = 'FAIL'\n", - "flags_values['coverage'] = coverage_quartiles['75%']\n", - "\n", - "flags['r1_basequal'] = data['fastqc_r1_basequal']\n", - "flags['r2_basequal'] = data['fastqc_r2_basequal']\n", - "flags_values['r1_basequal'] = flags['r1_basequal']\n", - "flags_values['r2_basequal'] = flags['r2_basequal']\n", - "\n", - "flags['rejreads'] = 'PASS'\n", - "rejupper_bound_warning = 20\n", - "rejupper_bound_fail = 40 # or minimum of negative controls?\n", - "rejreads_failed = rejreads.loc[rejreads['All'] > rejupper_bound_warning].index\n", - "flags.loc[rejreads_failed, 'rejreads'] = 'WARNING'\n", - "rejreads_failed = rejreads.loc[rejreads['All'] > rejupper_bound_fail].index\n", - "flags.loc[rejreads_failed, 'rejreads'] = 'FAIL'\n", - "flags_values['rejreads'] = rejreads['All']\n", - "\n", - "flags['alnreads'] = 'PASS'\n", - "alnlower_bound = 90\n", - "alnreads_failed = alnreads.loc[alnreads['All'] < alnlower_bound].index\n", - "flags.loc[alnreads_failed, 'alnreads'] = 'FAIL'\n", - "flags_values['alnreads'] = alnreads['All']\n", - "\n", - "flags['insertsize'] = 'PASS'\n", - "insupper_bound = 500\n", - "inslower_bound = 300\n", - "insertsize_failed = bwa_insert_dist.loc[(bwa_insert_dist['min'] < inslower_bound) | (bwa_insert_dist['max'] > insupper_bound)].index\n", - "flags.loc[insertsize_failed, 'insertsize'] = 'FAIL'\n", - "flags_values['insertsize'] = bwa_insert_dist['mean']\n", - "\n", - "flags['consensus_N'] = 'PASS'\n", - "undupper_bound = 10000\n", - "nbases_failed = nbases.loc[nbases['All'] > undupper_bound].index\n", - "flags.loc[nbases_failed, 'consensus_N'] = 'FAIL'\n", - "flags_values['consensus_N'] = nbases['All']\n", - "\n", - "flags['consensus_lcbases'] = 'PASS'\n", - "ambupper_bound = 20000\n", - "lcbases_failed = lcbases.loc[lcbases['All'] > ambupper_bound].index\n", - "flags.loc[lcbases_failed, 'consensus_lcbases'] = 'FAIL'\n", - "flags_values['consensus_lcbases'] = lcbases['All']\n", - "\n", - "flags['consensus_diffbases'] = 'PASS'\n", - "diffupper_bound_warning = 50\n", - "diffupper_bound_fail = 100\n", - "diffbases_warning = diffbases.loc[diffbases['All'] > diffupper_bound_warning].index\n", - "flags.loc[diffbases_warning, 'consensus_diffbases'] = 'WARNING'\n", - "diffbases_failed = diffbases.loc[diffbases['All'] > diffupper_bound_fail].index\n", - "flags.loc[diffbases_failed, 'consensus_diffbases'] = 'FAIL'\n", - "flags_values['consensus_diffbases'] = diffbases['All']\n", - "\n", - "flags = flags.rename(columns={'snvs_filtered': 'snvs'})\n", - "flags['snvs'] = 'PASS'\n", - "snvsupper_bound = 1000\n", - "filtered_failed = filtered_snvs.loc[filtered_snvs['All'] > snvsupper_bound].index\n", - "flags.loc[filtered_failed, 'snvs'] = 'FAIL'\n", - "flags_values['snvs'] = filtered_snvs['All']\n", - "\n", - "flags['snvs_majority'] = 'PASS'\n", - "majupper_bounds = np.max([np.ones(diffbases['All'].shape[0]), diffbases['All'].values], axis=0) * 10\n", - "majlower_bounds = (np.max([np.ones(diffbases['All'].shape[0]), diffbases['All'].values], axis=0) * 0.1).astype(int)\n", - "maj_failed = maj_snvs.loc[(maj_snvs['All'].values > majupper_bounds) | (maj_snvs['All'].values < majlower_bounds)].index\n", - "flags.loc[maj_failed, 'snvs_majority'] = 'FAIL'\n", - "flags_values['snvs_majority'] = maj_snvs['All']\n", - "\n", - "flags = flags.rename(columns={'sample': 'Sample'})\n", - "flags_values = flags_values.rename(columns={'sample': 'Sample'})" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "splt=data['batch'].str.split(\"_\", n = 1, expand = True) \n", - "data['date']=splt[0]\n", - "data['flowcell']=splt[1]\n", - "date = str(pd.to_datetime(data.loc[0, 'date'], format='%Y%m%d').date())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Summary" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# List of real samples that failed for consensus usage\n", - "# Sample is failed if any of the flags is a fail\n", - "sample_status = pd.DataFrame(dict(samples=flags['Sample'], status='PASS'))\n", - "for index, row in flags.iterrows():\n", - " if np.any(np.array(row.values[:-2]) == 'FAIL'):\n", - " sample_status.loc[index,'status'] = 'FAIL'\n", - " elif np.any(np.array(row.values[:-2]) == 'WARNING'):\n", - " sample_status.loc[index,'status'] = 'WARNING'\n", - " \n", - "failed_reals = sample_status.loc[data['group']=='Real'].loc[sample_status['status']=='FAIL']['samples']\n", - "\n", - "print(f'\\tFAILED {len(failed_reals)} real samples:\\n')\n", - "for sample in failed_reals:\n", - " print('\\t' + sample)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "group_colors = dict(Real='#80b1d3', Empty='#ffffb3', H2O='#bebada', Negative='#fb8072', Positive='#8dd3c7')\n", - "\n", - "layout = go.Layout(\n", - " title = 'Sample group sizes',\n", - " xaxis = go.layout.XAxis(\n", - " title = 'Group',\n", - " showticklabels=True),\n", - " yaxis = go.layout.YAxis(\n", - " title = 'Number of samples'\n", - " ),\n", - " margin = go.layout.Margin(t = 50),\n", - " template=\"plotly_white\"\n", - " \n", - ")\n", - "\n", - "colors = ['#3182bd'] + list(group_colors.values())\n", - "fig = go.Figure()\n", - "\n", - "# Add traces\n", - "fig.add_trace(go.Bar(x=groupsize['group'], y=groupsize['size'],\n", - " marker=dict(color=colors)))\n", - "\n", - "fig.update_layout(layout)\n", - "fig.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "colorscale=[[0, \"red\"], [0.111111111111, \"yellow\"], [0.222222222222, \"green\"], \n", - " [0.333333333333, \"rgb(141,211,199)\"], [0.444444444444, \"rgb(255,255,179)\"], \n", - " [0.555555555556, \"rgb(190,186,218)\"], [0.666666666667, \"rgb(190,186,218)\"], \n", - " [0.777777777778, \"rgb(128,177,211)\"], [0.888888888889, \"rgb(128,177,211)\"], \n", - " [1, \"rgb(49,54,149)\"]]\n", - "passcolordicts = {'FAIL': 0., 'WARNING': 1./9., 'PASS': 2./9.}\n", - "groupcolordicts = {'Real': 7./9., 'Empty': 6./9., 'H2O': 5./9., 'Negative': 4./9., 'Positive': 3./9.}\n", - "colors = np.array([[passcolordicts[v] for v in flags[s].values] for s in list(flags.columns[1:])])\n", - "colors = np.concatenate([np.array([[groupcolordicts[group] for group in data['group']]]), colors])\n", - "colors = np.concatenate([np.array([[8./9. for s in list(flags.columns)]]), colors.T])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "flags2 = flags.rename(columns={'coverage': 'Coverage', 'r1_basequal': 'Base quality R1', 'r2_basequal': 'Base quality R2',\n", - " 'rejreads': 'Rejected reads', 'alnreads': 'Aligned reads', 'insertsize': 'Insert size', \n", - " 'consensus_N': 'Consensus
undetermined',\n", - " 'consensus_lcbases': 'Consensus
ambiguousity', 'consensus_diffbases': 'Consensus
sequence',\n", - " 'snvs': 'Number of
SNVs', 'snvs_majority': 'Majority
SNVs'})\n", - "from copy import deepcopy\n", - "flags3 = deepcopy(flags2)\n", - "flags3.loc[np.where(np.array(['pos' in f for f in flags3['Sample']]))[0],'Sample'] = ['_'.join(a.split('_', 3)[:2]) for a in flags3.loc[np.where(np.array(['pos' in f for f in flags3['Sample']]))[0],'Sample'].tolist()]\n", - "flags3.loc[np.where(np.array(['EMPTY' in f for f in flags3['Sample']]))[0],'Sample'] = ['_'.join(a.split('_', 3)[:2]) for a in flags3.loc[np.where(np.array(['EMPTY' in f for f in flags3['Sample']]))[0],'Sample'].tolist()]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "descriptions = {'Sample':\"\",\n", - " 'Coverage': f'Third quartile of the coverage across the whole genome.
WARNING if < {coverage_3q_min_warning}
FAIL if < {coverage_3q_min_fail}',\n", - " 'Base quality R1': 'FastQC per base quality flag for read 1',\n", - " 'Base quality R2': 'FastQC per base quality flag for read 2',\n", - " 'Rejected reads': f'Percentage of reads rejected by Prinseq.
WARNING if > {rejupper_bound_warning}%
FAIL if > {rejupper_bound_fail}%',\n", - " 'Aligned reads': f'Percentage of kept reads that were aligned.
FAIL if < {alnlower_bound}%',\n", - " 'Insert size': f'Minimum and maximum estimated insert sizes.
FAIL if minimum < {inslower_bound} or maximum > {insupper_bound}',\n", - " 'Consensus
undetermined': f'Number of bases considered undetermined, i.e., with support in less than 5 reads.
FAIL if > {undupper_bound}',\n", - " 'Consensus
ambiguosity': f'Number of bases considered ambiguous, i.e., with support in less than 50 reads.
FAIL if > {ambupper_bound}',\n", - " 'Consensus
sequence': f'Number of bases different from the reference sequence. \"N\" calls not included.
WARNING if > {diffupper_bound_warning}
FAIL if > {diffupper_bound_fail}',\n", - " 'Number of
SNVs': f'Number of single-nucleotide variants
FAIL if > {snvsupper_bound}',\n", - " 'Majority
SNVs': f'Number of consensus single-nucleotide variants
FAIL if 10x larger than the consensus sequence'}\n", - "\n", - "hover=[]\n", - "hover.append(['' + desc + '' for desc in list(descriptions.values())])\n", - "for i in range(flags_values.shape[0]):\n", - " row = [flags_values.iloc[i]['Sample']]\n", - " for j in range(1, flags_values.shape[1]):\n", - " val = str(flags_values.iloc[i, j])\n", - " if flags_values.columns[j] == 'rejreads' or flags_values.columns[j] == 'alnreads':\n", - " val = f\"{flags_values.iloc[i, j]:.2f}\"\n", - " row.append('' + list(descriptions.values())[j] + '' + '

' +\n", - " '' + str(flags_values.iloc[i, 0]) + ': ' + val + '

' + \n", - " 'Coverage 3Q: ' + str(flags_values.iloc[i]['coverage']) + '
' +\n", - " f\"% Rejected reads: {flags_values.iloc[i]['rejreads']:.2f}
\" +\n", - " f\"% Aligned reads: {flags_values.iloc[i]['alnreads']:.2f}\")\n", - " \n", - " hover.append(row)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "flags_values.to_csv(output_qafile, index=False)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": false - }, - "outputs": [], - "source": [ - "table_simple = ff.create_table(flags3, text=hover, hoverinfo='text', )\n", - "# hovertemplate=\"%{text}

%{y}

\" +\n", - "# \"\")\n", - "table_simple.update_layout(autosize=False,\n", - " width=1400,)\n", - "d = table_simple.to_dict()\n", - "d['data'][0]['z'] = colors\n", - "d['data'][0]['colorscale'] = colorscale\n", - "\n", - "pio.show(d, config={'displayModeBar': False})" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Coverage" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "delta_qs = coverage_quartiles.T.iloc[1:].T.diff(axis=1)\n", - "delta_qs['sample'] = coverage_quartiles['sample']\n", - "delta_qs['25%'] = coverage_quartiles['25%']" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": false - }, - "outputs": [], - "source": [ - "layout = go.Layout(\n", - " title = 'Coverage quartiles per sample',\n", - " xaxis = go.layout.XAxis(\n", - " title = 'Samples',\n", - " showticklabels=False),\n", - " yaxis = go.layout.YAxis(\n", - " title = 'Coverage'\n", - " ),\n", - " template=\"plotly_white\",\n", - " barmode='relative',\n", - " )\n", - "\n", - "colors = ['#6baed6', '#3182bd', '#08519c']\n", - "fig = go.Figure()\n", - "\n", - "# Add traces\n", - "fig.add_trace(go.Bar(x=coverage_quartiles['sample'], y=delta_qs['25%'],\n", - " #mode='lines',\n", - " text=coverage_quartiles['25%'],\n", - " hovertemplate=\"(%{x}, %{text})\",\n", - " name='1Q', marker=dict(color=colors[0])))\n", - "fig.add_trace(go.Bar(x=coverage_quartiles['sample'], y=delta_qs['50%'],\n", - " #mode='lines',\n", - " text=coverage_quartiles['50%'],\n", - " hovertemplate=\"(%{x}, %{text})\",\n", - " name='Median', marker=dict(color=colors[1])))\n", - "fig.add_trace(go.Bar(x=coverage_quartiles['sample'], y=delta_qs['75%'],\n", - " #mode='lines',\n", - " text=coverage_quartiles['75%'],\n", - " hovertemplate=\"(%{x}, %{text})\",\n", - " name='3Q', marker=dict(color=colors[2])))\n", - "fig.add_trace(go.Scatter(x=coverage_quartiles['sample'], y=np.ones((coverage_quartiles.shape[0])) * coverage_3q_min_warning,\n", - " mode='lines',\n", - " name='Soft lower bound\\nfor 3Q', marker=dict(color='#CCCC00'), line = dict(dash='dot')))\n", - "fig.add_trace(go.Scatter(x=coverage_quartiles['sample'], y=np.ones((coverage_quartiles.shape[0])) * coverage_3q_min_fail,\n", - " mode='lines',\n", - " name='Hard lower bound\\nfor 3Q', marker=dict(color='red'), line = dict(dash='dot')))\n", - "\n", - "fig.update_layout(layout)\n", - "\n", - "fig.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "group_percentiles = dict(\n", - " Real=pd.DataFrame({'25%':coverage.loc[data['group']=='Real'].quantile(q=0.25, axis=0),\n", - " '50%':coverage.loc[data['group']=='Real'].quantile(q=0.5, axis=0),\n", - " '75%':coverage.loc[data['group']=='Real'].quantile(q=0.75, axis=0)}),\n", - " Empty=pd.DataFrame({'25%':coverage.loc[data['group']=='Empty'].quantile(q=0.25, axis=0),\n", - " '50%':coverage.loc[data['group']=='Empty'].quantile(q=0.5, axis=0),\n", - " '75%':coverage.loc[data['group']=='Empty'].quantile(q=0.75, axis=0)}),\n", - " H2O=pd.DataFrame({'25%':coverage.loc[data['group']=='H2O'].quantile(q=0.25, axis=0),\n", - " '50%':coverage.loc[data['group']=='H2O'].quantile(q=0.5, axis=0),\n", - " '75%':coverage.loc[data['group']=='H2O'].quantile(q=0.75, axis=0)}),\n", - " Negative=pd.DataFrame({'25%':coverage.loc[data['group']=='Negative'].quantile(q=0.25, axis=0),\n", - " '50%':coverage.loc[data['group']=='Negative'].quantile(q=0.5, axis=0),\n", - " '75%':coverage.loc[data['group']=='Negative'].quantile(q=0.75, axis=0)}),\n", - " Positive=pd.DataFrame({'25%':coverage.loc[data['group']=='Positive'].quantile(q=0.25, axis=0),\n", - " '50%':coverage.loc[data['group']=='Positive'].quantile(q=0.5, axis=0),\n", - " '75%':coverage.loc[data['group']=='Positive'].quantile(q=0.75, axis=0)}))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": false - }, - "outputs": [], - "source": [ - "colors = ['#6baed6', '#3182bd', '#08519c']\n", - "\n", - "groups = data['group'].unique()\n", - "vals = []\n", - "for i, g in enumerate(groups):\n", - " a = [False, False, False] * len(groups)\n", - " a[i*3:i*3+3] = [True, True, True]\n", - " vals.append(a)\n", - "\n", - "# Add traces\n", - "fig = go.Figure()\n", - "for i, group in enumerate(groups):\n", - " for j, p in enumerate(['25%', '50%', '75%']):\n", - " fig.add_trace(go.Scatter(x=group_percentiles[group].index, y=group_percentiles[group][p],\n", - " mode='lines',\n", - " name=['1st quart.', 'Median', '3rd quart.'][j], marker=dict(color=colors[j]),\n", - " visible=False if i > 0 else True))\n", - "\n", - "updatemenus = list([\n", - " dict(active=0,\n", - " showactive = True,\n", - " buttons=list([dict(label=group,\n", - " method=\"update\",\n", - " args =[{\"visible\": vals[i]}])\n", - " for i, group in enumerate(groups)\n", - " ]),\n", - " direction=\"down\",\n", - " pad={\"r\": 10, \"t\": 10},\n", - " x=0.05,\n", - " xanchor=\"left\",\n", - " y=1.12,\n", - " yanchor=\"top\")])\n", - "\n", - "layout = go.Layout(\n", - " title = 'Coverage quartiles per position per group',\n", - " xaxis = go.layout.XAxis(\n", - " title = 'Position',\n", - " showticklabels=True),\n", - " yaxis = go.layout.YAxis(\n", - " title = 'Coverage'\n", - " ),\n", - " template=\"plotly_white\",\n", - " updatemenus=updatemenus\n", - ")\n", - "\n", - "fig.update_layout(layout)\n", - "# Add annotation\n", - "fig.update_layout(\n", - " annotations=[\n", - " dict(text=\"Group:\", showarrow=False,\n", - " x=0., y=1.085, yref=\"paper\", align=\"left\")\n", - " ]\n", - ")\n", - "\n", - "fig.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Prinseq" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "layout = go.Layout(\n", - " title = 'Reads rejected by Prinseq',\n", - " xaxis = go.layout.XAxis(\n", - " title = 'Samples',\n", - " showticklabels=False),\n", - " yaxis = go.layout.YAxis(\n", - " title = 'Reads rejected (%)'\n", - " ),\n", - " template=\"plotly_white\"\n", - " \n", - ")\n", - "\n", - "# fig = go.Figure(data=[{'y': coverage_quartiles['coverage'], 'x': coverage_quartiles['sample']}], layout=layout)\n", - "# iplot(fig)\n", - "colors = ['#3182bd']\n", - "fig = go.Figure()\n", - "\n", - "# Add traces\n", - "fig.add_trace(go.Bar(x=rejreads['sample'], y=rejreads['All'], name=\"Observed\",\n", - " marker=dict(color=colors[0]),\n", - " text = rejreads[\"Absolute\"].values,\n", - " hoverinfo = 'y+text',\n", - " hovertemplate=\n", - " \"%{x}

\" +\n", - " \"Percentage: %{y:.2f}%
\" +\n", - " \"Absolute: %{text}
\"\n", - " \"\",\n", - " ))\n", - "fig.add_trace(go.Scatter(x=rejreads['sample'], y=np.ones((rejreads.shape[0])) * rejupper_bound_warning,\n", - " mode='lines',\n", - " name='Soft upper bound', marker=dict(color='#CCCC00'), line = dict(dash='dot')))\n", - "fig.add_trace(go.Scatter(x=rejreads['sample'], y=np.ones((rejreads.shape[0])) * rejupper_bound_fail,\n", - " mode='lines',\n", - " name='Hard upper bound', marker=dict(color='red'), line = dict(dash='dot')))\n", - "fig.update_layout(layout)\n", - "fig.show()\n", - "# fig = px.line(coverage_quartiles, x=\"sample\", y=\"coverage\",\n", - "# color='percentile', hover_name=\"sample\", template=\"plotly_white\")\n", - "\n", - "# iplot(fig)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Alignment" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": false - }, - "outputs": [], - "source": [ - "layout = go.Layout(\n", - " title = 'Reads aligned by BWA',\n", - " xaxis = go.layout.XAxis(\n", - " title = 'Samples',\n", - " showticklabels=False),\n", - " yaxis = go.layout.YAxis(\n", - " title = 'Reads aligned (%)'\n", - " ),\n", - " template=\"plotly_white\"\n", - " \n", - ")\n", - "\n", - "colors = ['#3182bd']\n", - "fig = go.Figure()\n", - "\n", - "# Add traces\n", - "fig.add_trace(go.Bar(x=alnreads['sample'], y=alnreads['All'].values, name=\"Observed\",\n", - " marker=dict(color=colors[0]),\n", - " text = alnreads[\"Absolute\"].values,\n", - " hoverinfo = 'y+text',\n", - " hovertemplate=\n", - " \"%{x}

\" +\n", - " \"Percentage: %{y:.2f}%
\" +\n", - " \"Absolute: %{text}
\"\n", - " \"\",\n", - " ))\n", - "fig.add_trace(go.Scatter(x=alnreads['sample'], y=np.ones((alnreads.shape[0])) * 90,\n", - " mode='lines',\n", - " name='Lower bound', marker=dict(color='red'), line = dict(dash='dot')))\n", - "\n", - "fig.update_layout(layout)\n", - "fig.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "delta_is = bwa_insert_dist.T.iloc[1:].T.diff(axis=1)\n", - "delta_is['sample'] = bwa_insert_dist['sample']\n", - "delta_is['min'] = bwa_insert_dist['min']" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "layout = go.Layout(\n", - " title = 'Insert sizes estimated by BWA',\n", - " xaxis = go.layout.XAxis(\n", - " title = 'Samples',\n", - " showticklabels=False),\n", - " yaxis = go.layout.YAxis(\n", - " title = 'Insert sizes',\n", - " range = [0, 600]\n", - " ),\n", - " template=\"plotly_white\",\n", - " barmode='stack'\n", - ")\n", - "\n", - "colors = ['#6baed6', '#3182bd', '#08519c']\n", - "fig = go.Figure()\n", - "\n", - "# Add traces\n", - "fig.add_trace(go.Bar(x=bwa_insert_dist['sample'], y=delta_is['min'],\n", - " #mode='lines',\n", - " text=bwa_insert_dist['min'],\n", - " hovertemplate=\"(%{x}, %{text})\",\n", - " name='Min', marker=dict(color=colors[0])))\n", - "fig.add_trace(go.Bar(x=bwa_insert_dist['sample'], y=delta_is['mean'],\n", - " #mode='lines',\n", - " text=bwa_insert_dist['mean'],\n", - " hovertemplate=\"(%{x}, %{text})\",\n", - " name='Mean', marker=dict(color=colors[1])))\n", - "fig.add_trace(go.Bar(x=bwa_insert_dist['sample'], y=delta_is['max'],\n", - " #mode='lines',\n", - " text=bwa_insert_dist['max'],\n", - " hovertemplate=\"(%{x}, %{text})\",\n", - " name='Max', marker=dict(color=colors[2])))\n", - "\n", - "fig.add_trace(go.Scatter(x=rejreads['sample'], y=np.ones((rejreads.shape[0])) * 300,\n", - " mode='lines',\n", - " name='Lower bound', marker=dict(color='red'), line = dict(dash='dot')))\n", - "fig.add_trace(go.Scatter(x=rejreads['sample'], y=np.ones((rejreads.shape[0])) * 500,\n", - " mode='lines',\n", - " name='Upper bound', marker=dict(color='red'), line = dict(dash='dot')))\n", - "\n", - "fig.update_layout(layout)\n", - "fig.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Consensus sequence" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "layout = go.Layout(\n", - " title = 'Number of undetermined (N) bases in consensus',\n", - " xaxis = go.layout.XAxis(\n", - " title = 'Samples',\n", - " showticklabels=False),\n", - " yaxis = go.layout.YAxis(\n", - " title = 'Number of N calls'\n", - " ),\n", - " template=\"plotly_white\"\n", - " \n", - ")\n", - "\n", - "colors = ['#3182bd']\n", - "fig = go.Figure()\n", - "\n", - "# Add traces\n", - "fig.add_trace(go.Bar(x=nbases['sample'], y=nbases['All'], name=\"Observed\",\n", - " marker=dict(color=colors[0])))\n", - "fig.add_trace(go.Scatter(x=nbases['sample'], y=np.ones((nbases.shape[0])) * undupper_bound,\n", - " mode='lines',\n", - " name='Upper bound', marker=dict(color='red'), line = dict(dash='dot')))\n", - "\n", - "fig.update_layout(layout)\n", - "fig.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "layout = go.Layout(\n", - " title = 'Number of ambiguous (lowercase) bases in consensus',\n", - " xaxis = go.layout.XAxis(\n", - " title = 'Samples',\n", - " showticklabels=False),\n", - " yaxis = go.layout.YAxis(\n", - " title = 'Number of lowercase calls'\n", - " ),\n", - " template=\"plotly_white\"\n", - " \n", - ")\n", - "\n", - "colors = ['#3182bd']\n", - "fig = go.Figure()\n", - "\n", - "# Add traces\n", - "fig.add_trace(go.Bar(x=lcbases['sample'], y=lcbases['All'], name=\"Observed\",\n", - " marker=dict(color=colors[0])))\n", - "fig.add_trace(go.Scatter(x=lcbases['sample'], y=np.ones((lcbases.shape[0])) * ambupper_bound,\n", - " mode='lines',\n", - " name='Upper bound', marker=dict(color='red'), line = dict(dash='dot')))\n", - "\n", - "fig.update_layout(layout)\n", - "fig.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "layout = go.Layout(\n", - " title = 'Number of different bases* between consensus and reference
*not including \"N\"',\n", - " xaxis = go.layout.XAxis(\n", - " title = 'Samples',\n", - " showticklabels=False),\n", - " yaxis = go.layout.YAxis(\n", - " title = 'Number of bases',\n", - " range = [0, diffupper_bound_fail + 10]\n", - " ),\n", - " template=\"plotly_white\"\n", - " \n", - ")\n", - "colors = ['#3182bd']\n", - "fig = go.Figure()\n", - "\n", - "# Add traces\n", - "fig.add_trace(go.Bar(x=diffbases['sample'], y=diffbases['All'], name=\"Observed\",\n", - " marker=dict(color=colors[0])))\n", - "fig.add_trace(go.Scatter(x=diffbases['sample'], y=np.ones((diffbases.shape[0])) * diffupper_bound_warning,\n", - " mode='lines',\n", - " name='Soft upper bound', marker=dict(color='#CCCC00'), line = dict(dash='dot')))\n", - "\n", - "fig.add_trace(go.Scatter(x=diffbases['sample'], y=np.ones((diffbases.shape[0])) * diffupper_bound_fail,\n", - " mode='lines',\n", - " name='Hard upper bound', marker=dict(color='red'), line = dict(dash='dot')))\n", - "fig.update_layout(layout)\n", - "fig.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## ShoRAH" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": false - }, - "outputs": [], - "source": [ - "layout = go.Layout(\n", - " title = 'SNVs called by ShoRAH (posterior probability > 80%)',\n", - " xaxis = go.layout.XAxis(\n", - " title = 'Samples',\n", - " showticklabels=False),\n", - " yaxis = go.layout.YAxis(\n", - " title = 'Number of SNVs'\n", - " ),\n", - " template=\"plotly_white\"\n", - " \n", - ")\n", - "\n", - "colors = ['#3182bd']\n", - "fig = go.Figure()\n", - "\n", - "# Add traces\n", - "fig.add_trace(go.Bar(x=filtered_snvs['sample'], y=filtered_snvs['All'], name=\"Observed\",\n", - " marker=dict(color=colors[0])))\n", - "fig.add_trace(go.Scatter(x=filtered_snvs['sample'], y=np.ones((filtered_snvs.shape[0])) * 2000,\n", - " mode='lines',\n", - " name='Upper bound', marker=dict(color='red'), line = dict(dash='dot')))\n", - "\n", - "fig.update_layout(layout)\n", - "fig.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": false - }, - "outputs": [], - "source": [ - "layout = go.Layout(\n", - " title = 'Number of SNVs from filtered ShoRAH-based majority vote',\n", - " xaxis = go.layout.XAxis(\n", - " title = 'Samples',\n", - " showticklabels=False),\n", - " yaxis = go.layout.YAxis(\n", - " title = 'Number of SNVs'\n", - " ),\n", - " template=\"plotly_white\",\n", - " yaxis_type=\"log\",\n", - " \n", - ")\n", - "\n", - "colors = ['#3182bd']\n", - "fig = go.Figure()\n", - "\n", - "# Add traces\n", - "fig.add_trace(go.Bar(x=maj_snvs['sample'], y=maj_snvs['All'], name=\"Observed\",\n", - " marker=dict(color=colors[0])))\n", - "fig.add_trace(go.Scatter(x=maj_snvs['sample'], y=majupper_bounds,\n", - " mode='markers',\n", - " name='Upper bound', marker=dict(color='red'), line = dict(dash='dot')))\n", - "\n", - "fig.update_layout(layout)\n", - "fig.show()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "conda-qa_report", - "language": "python", - "name": "conda-qa_report" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.3" - }, - "toc": { - "base_numbering": 1, - "nav_menu": {}, - "number_sections": true, - "sideBar": true, - "skip_h1_title": false, - "title_cell": "Table of Contents", - "title_sidebar": "Contents", - "toc_cell": false, - "toc_position": {}, - "toc_section_display": true, - "toc_window_display": false - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/working/references/primers.yaml b/working/references/primers.yaml index 827aab6..386468f 100644 --- a/working/references/primers.yaml +++ b/working/references/primers.yaml @@ -1,3 +1,12 @@ +v532: + name: SARS-CoV-2 ARTIC V5.3.2 + alias: + - SARS-CoV-2 ARTIC V5.3.2 NEB Ultra II + - SARS-CoV-2 ARTIC V5.3.2 NexteraXT + inserts_bedfile: references/primers/v532/SARS-CoV-2.insert.bed + primers_bedfile: references/primers/v532/SARS-CoV-2.primer.bed + primers_file: references/primers/v532/SARS-CoV-2.tsv + primers_fasta: references/primers/v532/SARS-CoV-2.primer.fasta v41: name: SARS-CoV-2 ARTIC V4.1 alias: diff --git a/working/references/primers/v532/SARS-CoV-2.insert.bed b/working/references/primers/v532/SARS-CoV-2.insert.bed new file mode 100644 index 0000000..2dbf696 --- /dev/null +++ b/working/references/primers/v532/SARS-CoV-2.insert.bed @@ -0,0 +1,96 @@ +NC_045512.2 2880 3233 SARS-CoV-2_400_INSERT_10 2 - +NC_045512.2 3213 3560 SARS-CoV-2_400_INSERT_11 1 - +NC_045512.2 3540 3883 SARS-CoV-2_400_INSERT_12 2 - +NC_045512.2 3824 4147 SARS-CoV-2_400_INSERT_13 1 - +NC_045512.2 4108 4457 SARS-CoV-2_400_INSERT_14 2 - +NC_045512.2 4425 4776 SARS-CoV-2_400_INSERT_15 1 - +NC_045512.2 4756 5089 SARS-CoV-2_400_INSERT_16 2 - +NC_045512.2 5063 5398 SARS-CoV-2_400_INSERT_17 1 - +NC_045512.2 5370 5716 SARS-CoV-2_400_INSERT_18 2 - +NC_045512.2 5696 6031 SARS-CoV-2_400_INSERT_19 1 - +NC_045512.2 78 419 SARS-CoV-2_400_INSERT_1 1 - +NC_045512.2 5923 6257 SARS-CoV-2_400_INSERT_20 2 - +NC_045512.2 6237 6562 SARS-CoV-2_400_INSERT_21 1 - +NC_045512.2 6542 6882 SARS-CoV-2_400_INSERT_22 2 - +NC_045512.2 6854 7199 SARS-CoV-2_400_INSERT_23 1 - +NC_045512.2 7179 7518 SARS-CoV-2_400_INSERT_24 2 - +NC_045512.2 7482 7819 SARS-CoV-2_400_INSERT_25 1 - +NC_045512.2 7797 8136 SARS-CoV-2_400_INSERT_26 2 - +NC_045512.2 8112 8468 SARS-CoV-2_400_INSERT_27 1 - +NC_045512.2 8436 8781 SARS-CoV-2_400_INSERT_28 2 - +NC_045512.2 8761 9107 SARS-CoV-2_400_INSERT_29 1 - +NC_045512.2 366 707 SARS-CoV-2_400_INSERT_2 2 - +NC_045512.2 9052 9397 SARS-CoV-2_400_INSERT_30 2 - +NC_045512.2 9324 9673 SARS-CoV-2_400_INSERT_31 1 - +NC_045512.2 9604 9949 SARS-CoV-2_400_INSERT_32 2 - +NC_045512.2 9929 10266 SARS-CoV-2_400_INSERT_33 1 - +NC_045512.2 10245 10587 SARS-CoV-2_400_INSERT_34 2 - +NC_045512.2 10557 10897 SARS-CoV-2_400_INSERT_35 1 - +NC_045512.2 10865 11201 SARS-CoV-2_400_INSERT_36 2 - +NC_045512.2 11181 11514 SARS-CoV-2_400_INSERT_37 1 - +NC_045512.2 11494 11832 SARS-CoV-2_400_INSERT_38 2 - +NC_045512.2 11811 12161 SARS-CoV-2_400_INSERT_39 1 - +NC_045512.2 661 1018 SARS-CoV-2_400_INSERT_3 1 - +NC_045512.2 12137 12477 SARS-CoV-2_400_INSERT_40 2 - +NC_045512.2 12444 12794 SARS-CoV-2_400_INSERT_41 1 - +NC_045512.2 12774 13121 SARS-CoV-2_400_INSERT_42 2 - +NC_045512.2 13099 13458 SARS-CoV-2_400_INSERT_43 1 - +NC_045512.2 13435 13787 SARS-CoV-2_400_INSERT_44 2 - +NC_045512.2 13767 14120 SARS-CoV-2_400_INSERT_45 1 - +NC_045512.2 14100 14427 SARS-CoV-2_400_INSERT_46 2 - +NC_045512.2 14407 14745 SARS-CoV-2_400_INSERT_47 1 - +NC_045512.2 14725 15065 SARS-CoV-2_400_INSERT_48 2 - +NC_045512.2 15045 15386 SARS-CoV-2_400_INSERT_49 1 - +NC_045512.2 995 1340 SARS-CoV-2_400_INSERT_4 2 - +NC_045512.2 15366 15716 SARS-CoV-2_400_INSERT_50 2 - +NC_045512.2 15688 16028 SARS-CoV-2_400_INSERT_51 1 - +NC_045512.2 16018 16386 SARS-CoV-2_400_INSERT_52 2 - +NC_045512.2 16311 16650 SARS-CoV-2_400_INSERT_53 1 - +NC_045512.2 16647 17004 SARS-CoV-2_400_INSERT_54 2 - +NC_045512.2 16994 17333 SARS-CoV-2_400_INSERT_55 1 - +NC_045512.2 17212 17560 SARS-CoV-2_400_INSERT_56 2 - +NC_045512.2 17507 17859 SARS-CoV-2_400_INSERT_57 1 - +NC_045512.2 17839 18181 SARS-CoV-2_400_INSERT_58 2 - +NC_045512.2 18153 18504 SARS-CoV-2_400_INSERT_59 1 - +NC_045512.2 1320 1660 SARS-CoV-2_400_INSERT_5 1 - +NC_045512.2 18484 18835 SARS-CoV-2_400_INSERT_60 2 - +NC_045512.2 18815 19170 SARS-CoV-2_400_INSERT_61 1 - +NC_045512.2 19112 19469 SARS-CoV-2_400_INSERT_62 2 - +NC_045512.2 19449 19770 SARS-CoV-2_400_INSERT_63 1 - +NC_045512.2 19750 20091 SARS-CoV-2_400_INSERT_64 2 - +NC_045512.2 20054 20408 SARS-CoV-2_400_INSERT_65 1 - +NC_045512.2 20388 20729 SARS-CoV-2_400_INSERT_66 2 - +NC_045512.2 20676 21018 SARS-CoV-2_400_INSERT_67 1 - +NC_045512.2 21018 21372 SARS-CoV-2_400_INSERT_68 2 - +NC_045512.2 21352 21696 SARS-CoV-2_400_INSERT_69 1 - +NC_045512.2 1596 1945 SARS-CoV-2_400_INSERT_6 2 - +NC_045512.2 21607 21927 SARS-CoV-2_400_INSERT_70 2 - +NC_045512.2 21894 22238 SARS-CoV-2_400_INSERT_71 1 - +NC_045512.2 22189 22517 SARS-CoV-2_400_INSERT_72 2 - +NC_045512.2 22494 22839 SARS-CoV-2_400_INSERT_73 1 - +NC_045512.2 22774 23119 SARS-CoV-2_400_INSERT_74 2 - +NC_045512.2 23109 23452 SARS-CoV-2_400_INSERT_75 1 - +NC_045512.2 23258 23609 SARS-CoV-2_400_INSERT_76 2 - +NC_045512.2 23589 23914 SARS-CoV-2_400_INSERT_77 1 - +NC_045512.2 23853 24209 SARS-CoV-2_400_INSERT_78 2 - +NC_045512.2 24189 24535 SARS-CoV-2_400_INSERT_79 1 - +NC_045512.2 1905 2259 SARS-CoV-2_400_INSERT_7 1 - +NC_045512.2 24468 24815 SARS-CoV-2_400_INSERT_80 2 - +NC_045512.2 24774 25120 SARS-CoV-2_400_INSERT_81 1 - +NC_045512.2 25082 25423 SARS-CoV-2_400_INSERT_82 2 - +NC_045512.2 25402 25744 SARS-CoV-2_400_INSERT_83 1 - +NC_045512.2 25680 26048 SARS-CoV-2_400_INSERT_84 2 - +NC_045512.2 26039 26382 SARS-CoV-2_400_INSERT_85 1 - +NC_045512.2 26362 26730 SARS-CoV-2_400_INSERT_86 2 - +NC_045512.2 26621 26989 SARS-CoV-2_400_INSERT_87 1 - +NC_045512.2 26981 27349 SARS-CoV-2_400_INSERT_88 2 - +NC_045512.2 27226 27583 SARS-CoV-2_400_INSERT_89 1 - +NC_045512.2 2252 2603 SARS-CoV-2_400_INSERT_8 2 - +NC_045512.2 27558 27927 SARS-CoV-2_400_INSERT_90 2 - +NC_045512.2 27860 28209 SARS-CoV-2_400_INSERT_91 1 - +NC_045512.2 28166 28513 SARS-CoV-2_400_INSERT_92 2 - +NC_045512.2 28493 28849 SARS-CoV-2_400_INSERT_93 1 - +NC_045512.2 28829 29203 SARS-CoV-2_400_INSERT_94 2 - +NC_045512.2 29183 29538 SARS-CoV-2_400_INSERT_95 1 - +NC_045512.2 29486 29840 SARS-CoV-2_400_INSERT_96 2 - +NC_045512.2 2563 2900 SARS-CoV-2_400_INSERT_9 1 - diff --git a/working/references/primers/v532/SARS-CoV-2.primer.bed b/working/references/primers/v532/SARS-CoV-2.primer.bed new file mode 100644 index 0000000..1193c57 --- /dev/null +++ b/working/references/primers/v532/SARS-CoV-2.primer.bed @@ -0,0 +1,192 @@ +NC_045512.2 47 78 SARS-CoV-2_400_1_LEFT_1 1 + CTCTTGTAGATCTGTTCTCTAAACGAACTTT +NC_045512.2 419 447 SARS-CoV-2_400_1_RIGHT_1 1 - AAAACGCCTTTTTCAACTTCTACTAAGC +NC_045512.2 344 366 SARS-CoV-2_400_2_LEFT_0 2 + TCGTACGTGGCTTTGGAGACTC +NC_045512.2 707 732 SARS-CoV-2_400_2_RIGHT_0 2 - TCTTCATAAGGATCAGTGCCAAGCT +NC_045512.2 638 661 SARS-CoV-2_400_3_LEFT_1 1 + AGAACGGTAATAAAGGAGCTGGT +NC_045512.2 1018 1047 SARS-CoV-2_400_3_RIGHT_0 1 - TTTGCCAATTTAATTTCAAAAGGTGTCTG +NC_045512.2 970 995 SARS-CoV-2_400_4_LEFT_0 2 + CATGAAATTGCTTGGTACACGGAAC +NC_045512.2 1340 1370 SARS-CoV-2_400_4_RIGHT_0 2 - AAATTTTAACAACAGCATTTTGGGGTAAGT +NC_045512.2 1292 1320 SARS-CoV-2_400_5_LEFT_0 1 + TTTGTGGCACTGAGAATTTGACTAAAGA +NC_045512.2 1660 1692 SARS-CoV-2_400_5_RIGHT_0 1 - ATAATGGCGATCTCTTCATTAAGTTTAAAGTC +NC_045512.2 1574 1596 SARS-CoV-2_400_6_LEFT_1 2 + GTGTTGTTGGAGAAGGTTCCGA +NC_045512.2 1945 1972 SARS-CoV-2_400_6_RIGHT_1 2 - TATTGTTATAGCGGCCTTCTGTAAAAC +NC_045512.2 1882 1905 SARS-CoV-2_400_7_LEFT_2 1 + GCTGCTCGTGTTGTACGATCAAT +NC_045512.2 2259 2284 SARS-CoV-2_400_7_RIGHT_2 1 - CTCCTTAATTTCCTTTGCACAGGTG +NC_045512.2 2229 2252 SARS-CoV-2_400_8_LEFT_0 2 + TGCTTGTGAAATTGTCGGTGGAC +NC_045512.2 2603 2629 SARS-CoV-2_400_8_RIGHT_0 2 - AAGCCCGTTAATACAAACTGGTGTAC +NC_045512.2 2533 2563 SARS-CoV-2_400_9_LEFT_0 1 + GTCTTGAAAACTGGTGATTTACAACCATTA +NC_045512.2 2900 2933 SARS-CoV-2_400_9_RIGHT_0 1 - GTAATTCAGATACTGGTTGCAAAGTTTTTATGA +NC_045512.2 2854 2880 SARS-CoV-2_400_10_LEFT_0 2 + CTCGGTACAGAAGTAAATGAGTTCGC +NC_045512.2 3233 3254 SARS-CoV-2_400_10_RIGHT_0 2 - TGTCCTCACTGCCGTCTTGTT +NC_045512.2 3184 3213 SARS-CoV-2_400_11_LEFT_0 1 + GAGCAAGAAGAAGATTGGTTAGATGATGA +NC_045512.2 3560 3584 SARS-CoV-2_400_11_RIGHT_0 1 - CGCTTAAAACACAACTACCACCCA +NC_045512.2 3510 3540 SARS-CoV-2_400_12_LEFT_0 2 + CATGCAAGTTGAATCTGATGATTACATAGC +NC_045512.2 3883 3913 SARS-CoV-2_400_12_RIGHT_0 2 - AGTTATAAATGGCTTAACTTCCTCTTTAGG +NC_045512.2 3791 3824 SARS-CoV-2_400_13_LEFT_0 1 + CTGTCTTTGATAAAAATCTCTATGACAAACTTG +NC_045512.2 4147 4180 SARS-CoV-2_400_13_RIGHT_0 1 - CTTTTTAGTAGGTATAACCACAGCAGTTAAAAC +NC_045512.2 4079 4108 SARS-CoV-2_400_14_LEFT_0 2 + GTGACATTGACATCACTTTCTTAAAGAAA +NC_045512.2 4457 4488 SARS-CoV-2_400_14_RIGHT_0 2 - CCCTTATATTTACGCTGTATAGTTGAAACTA +NC_045512.2 4403 4425 SARS-CoV-2_400_15_LEFT_0 1 + CACATGCAGAAGAAACACGCAA +NC_045512.2 4776 4803 SARS-CoV-2_400_15_RIGHT_0 1 - TCTTTATAGGAACCAGCAAGTGAGATG +NC_045512.2 4723 4756 SARS-CoV-2_400_16_LEFT_0 2 + AATGGTTATCTTACTTCTTCTTCTAAAACACCT +NC_045512.2 5089 5119 SARS-CoV-2_400_16_RIGHT_0 2 - AAATGTTTTACCTTCATGTGAATTATGAGG +NC_045512.2 5036 5063 SARS-CoV-2_400_17_LEFT_0 1 + GACAACAGTTTGGTCCAACTTATTTGG +NC_045512.2 5398 5429 SARS-CoV-2_400_17_RIGHT_0 1 - CTGTCTTATTACAGTAGGCTAAGATAAGTGC +NC_045512.2 5344 5370 SARS-CoV-2_400_18_LEFT_0 2 + GCTCTACAAGATGCTTATTACAGAGC +NC_045512.2 5716 5744 SARS-CoV-2_400_18_RIGHT_0 2 - ACTCACTAGCACAAGTAAATGTACCATG +NC_045512.2 5671 5696 SARS-CoV-2_400_19_LEFT_0 1 + CCTTTTGTTATGATGTCAGCACCAC +NC_045512.2 6031 6062 SARS-CoV-2_400_19_RIGHT_0 1 - CACATACAAACTTAAAATTATCGAAGCTTGC +NC_045512.2 5891 5923 SARS-CoV-2_400_20_LEFT_0 2 + CCATAAAACCAGTTACTTATAAATTGGATGGT +NC_045512.2 6257 6288 SARS-CoV-2_400_20_RIGHT_0 2 - CAGGTATTTGGTTTATACGTGGCTTTATTAG +NC_045512.2 6204 6237 SARS-CoV-2_400_21_LEFT_0 1 + GAAAGGAGCTAAATTGTTACATAAACCTATTGT +NC_045512.2 6562 6595 SARS-CoV-2_400_21_RIGHT_0 1 - AGGTTTCTTAATAGTAAGACTAGAATTGTCTAC +NC_045512.2 6515 6542 SARS-CoV-2_400_22_LEFT_0 2 + TAAAAATTACAGAAGAGGTTGGCCACA +NC_045512.2 6882 6915 SARS-CoV-2_400_22_RIGHT_0 2 - TAATTAAATGAAGCCTCTAGACAAAATTTACCG +NC_045512.2 6823 6854 SARS-CoV-2_400_23_LEFT_0 1 + AATTCTAGAATTAAAGCATCTATGCCGACTA +NC_045512.2 7199 7229 SARS-CoV-2_400_23_RIGHT_0 1 - CAAAAGCAGTTAAATCCCATTTAAAAGATG +NC_045512.2 7145 7179 SARS-CoV-2_400_24_LEFT_0 2 + GTTTAGATTCTTTAGACACCTATCCTTCTTTAGA +NC_045512.2 7518 7545 SARS-CoV-2_400_24_RIGHT_0 2 - ACAATAGTTGTACATTCGACTCTTGTT +NC_045512.2 7456 7482 SARS-CoV-2_400_25_LEFT_0 1 + GTGCATGTTGTAGACGGTTGTAATTC +NC_045512.2 7819 7850 SARS-CoV-2_400_25_RIGHT_0 1 - CTCTCAGGTTGTCTAAGTTAACAAAATGAGA +NC_045512.2 7768 7797 SARS-CoV-2_400_26_LEFT_0 2 + CATCTTTACTTTGATAAAGCTGGTCAAAA +NC_045512.2 8136 8169 SARS-CoV-2_400_26_RIGHT_0 2 - GAAATAAAAGTAGATAAGACATTGTCTAAGGAC +NC_045512.2 8085 8112 SARS-CoV-2_400_27_LEFT_0 1 + AAAACTCAAAACACTAGTTGCAACTGC +NC_045512.2 8468 8498 SARS-CoV-2_400_27_RIGHT_0 1 - CACATGTCAACTTAAAAGGTAAGTTATTCT +NC_045512.2 8406 8436 SARS-CoV-2_400_28_LEFT_0 2 + CGTTAAAGATTTCATGTCATTGTCTGAACA +NC_045512.2 8781 8806 SARS-CoV-2_400_28_RIGHT_0 2 - ATTAGTATAACTACCACCACGCTGG +NC_045512.2 8732 8761 SARS-CoV-2_400_29_LEFT_0 1 + CAGATACTTGTTTTGCTAACAAACATGCT +NC_045512.2 9107 9129 SARS-CoV-2_400_29_RIGHT_0 1 - ACATAACGTGTGTCAGGGCGTA +NC_045512.2 9023 9052 SARS-CoV-2_400_30_LEFT_0 2 + CAATTTTTAAAGATGCTTCTGGTAAGCCA +NC_045512.2 9397 9423 SARS-CoV-2_400_30_RIGHT_0 2 - ACAATACCACCAGCTACTATAGATGC +NC_045512.2 9299 9324 SARS-CoV-2_400_31_LEFT_1 1 + GATCTTTACCAGGAGTTTTCTGTGG +NC_045512.2 9673 9706 SARS-CoV-2_400_31_RIGHT_0 1 - ACAAATGATATAAGCAATTGTTATCCAGAAAGG +NC_045512.2 9571 9604 SARS-CoV-2_400_32_LEFT_0 2 + GGTGTTTATTCTGTTATTTACTTGTACTTGACA +NC_045512.2 9949 9971 SARS-CoV-2_400_32_RIGHT_0 2 - TTGCGAGATGACAACAAGCAGC +NC_045512.2 9896 9929 SARS-CoV-2_400_33_LEFT_0 1 + ATAATAAGTACAAGTATTTTAGTGGAGCAATGG +NC_045512.2 10266 10295 SARS-CoV-2_400_33_RIGHT_0 1 - AATGTCCAATAACCCTGAGTTGAACATTA +NC_045512.2 10215 10245 SARS-CoV-2_400_34_LEFT_0 2 + TGAAGATTTACTCATTCGTAAGTCTAATCA +NC_045512.2 10587 10615 SARS-CoV-2_400_34_RIGHT_0 2 - GTCAACAAAAGGTCCATAAAAGTTACCT +NC_045512.2 10527 10557 SARS-CoV-2_400_35_LEFT_0 1 + TTTTTGTTACATGCACCATATGGAATTACC +NC_045512.2 10897 10927 SARS-CoV-2_400_35_RIGHT_0 1 - AAATTCATCTTCTAATAAAGCACTACCCAA +NC_045512.2 10832 10865 SARS-CoV-2_400_36_LEFT_0 2 + CCGTTTTAGATATGTGTGCTTCATTAAAAGAAT +NC_045512.2 11201 11232 SARS-CoV-2_400_36_RIGHT_0 2 - GGCATATAGACCATATTAAAATAAGCTACAG +NC_045512.2 11152 11181 SARS-CoV-2_400_37_LEFT_0 1 + AAACATAAGCATGCATTTCTCTGTTTGTT +NC_045512.2 11514 11536 SARS-CoV-2_400_37_RIGHT_0 1 - ACCTCTGGCCAAAAACATGACA +NC_045512.2 11463 11494 SARS-CoV-2_400_38_LEFT_0 2 + GTGGGCTCTTATAATCTCTGTTACTTCTAAC +NC_045512.2 11832 11863 SARS-CoV-2_400_38_RIGHT_0 2 - CTTTACATCTGACATTTTAGACTGTACAGTG +NC_045512.2 11785 11811 SARS-CoV-2_400_39_LEFT_0 1 + AACATTAAATTGTTGGGTGTTGGTGG +NC_045512.2 12161 12185 SARS-CoV-2_400_39_RIGHT_0 1 - CAGAATCACCATTAGCAACAGCCT +NC_045512.2 12112 12137 SARS-CoV-2_400_40_LEFT_0 2 + TCCCTTCCATCATATGCAGCTTTTG +NC_045512.2 12477 12510 SARS-CoV-2_400_40_RIGHT_0 2 - TTTTTATATGTGTTATAGTCTGGTATGACAACC +NC_045512.2 12419 12444 SARS-CoV-2_400_41_LEFT_0 1 + CAAGAGATGGTTGTGTTCCCTTGAA +NC_045512.2 12794 12819 SARS-CoV-2_400_41_RIGHT_0 1 - AACAGTGCAAGTACAAACCTACCTC +NC_045512.2 12752 12774 SARS-CoV-2_400_42_LEFT_0 2 + GCACTGATGACAATGCGTTAGC +NC_045512.2 13121 13146 SARS-CoV-2_400_42_RIGHT_0 2 - CAATTAGTGATTGGTTGTCCCCCAC +NC_045512.2 13075 13099 SARS-CoV-2_400_43_LEFT_0 1 + GCTTTTGCTGTAGATGCTGCTAAA +NC_045512.2 13458 13480 SARS-CoV-2_400_43_RIGHT_0 1 - CACCGCAAACCCGTTTAAAAAC +NC_045512.2 13415 13435 SARS-CoV-2_400_44_LEFT_0 2 + ATCAACTCCGCGAACCCATG +NC_045512.2 13787 13815 SARS-CoV-2_400_44_RIGHT_0 2 - TGCCATTGTGTATTTAGTAAGACGTTGA +NC_045512.2 13738 13767 SARS-CoV-2_400_45_LEFT_0 1 + ACTTCTTTAAGTTTAGAATAGACGGTGAC +NC_045512.2 14120 14144 SARS-CoV-2_400_45_RIGHT_0 1 - TCTACAACAGGAACTCCACTACCT +NC_045512.2 14073 14100 SARS-CoV-2_400_46_LEFT_0 2 + CTCAATGGTAACTGGTATGATTTCGGT +NC_045512.2 14427 14457 SARS-CoV-2_400_46_RIGHT_0 2 - TGGAACACCATCAACAAATATTTTTCTCAC +NC_045512.2 14375 14407 SARS-CoV-2_400_47_LEFT_0 1 + CTTTAATGTTTTATTCTCTACAGTGTTCCCAC +NC_045512.2 14745 14775 SARS-CoV-2_400_47_RIGHT_0 1 - ATCCTGAGCAAAGAAGAAGTGTTTTAATTC +NC_045512.2 14700 14725 SARS-CoV-2_400_48_LEFT_0 2 + GACTTTGCTGTGTCTAAGGGTTTCT +NC_045512.2 15065 15095 SARS-CoV-2_400_48_RIGHT_0 2 - TTCTTTGCACTAATGGCATACTTAAGATTC +NC_045512.2 15016 15045 SARS-CoV-2_400_49_LEFT_0 1 + CACTTTTCGCATATACAAAACGTAATGTC +NC_045512.2 15386 15416 SARS-CoV-2_400_49_RIGHT_0 1 - CACTCATTAGCTAATCTATAGAAACGGTGT +NC_045512.2 15342 15366 SARS-CoV-2_400_50_LEFT_0 2 + TCACTTGTTCTTGCTCGCAAACAT +NC_045512.2 15716 15742 SARS-CoV-2_400_50_RIGHT_0 2 - TATTGAAACACACAACAGCATCGTCA +NC_045512.2 15659 15688 SARS-CoV-2_400_51_LEFT_0 1 + CTTTGTGAATGAGTTTTACGCATATTTGC +NC_045512.2 16028 16059 SARS-CoV-2_400_51_RIGHT_0 1 - AGGATGTTTAGTAAGTGGGTAAGCATCTATA +NC_045512.2 15992 16018 SARS-CoV-2_400_52_LEFT_2 2 + TGGTACACTTATGATTGAACGGTTCG +NC_045512.2 16386 16409 SARS-CoV-2_400_52_RIGHT_2 2 - GTGACATCACAACCTGGAGCATT +NC_045512.2 16285 16311 SARS-CoV-2_400_53_LEFT_0 1 + GTGCTTGCATACGTAGACCATTCTTA +NC_045512.2 16650 16679 SARS-CoV-2_400_53_RIGHT_0 1 - GACAGTTTAAATGTCTCCTCAGTAGCTTT +NC_045512.2 16624 16647 SARS-CoV-2_400_54_LEFT_1 2 + TCAAGCTTTTTGCAGCAGAAACG +NC_045512.2 17004 17033 SARS-CoV-2_400_54_RIGHT_1 2 - ACATTGCTAGAAAACTCATCTGAGATATT +NC_045512.2 16962 16994 SARS-CoV-2_400_55_LEFT_1 1 + CAAGAGCACTATGTTAGAATTACTGGCTTATA +NC_045512.2 17333 17362 SARS-CoV-2_400_55_RIGHT_1 1 - TTTCATCAAAGACAACTATATCTGCTGTC +NC_045512.2 17182 17212 SARS-CoV-2_400_56_LEFT_0 2 + CACTATGTGAGAAGGCATTAAAATATTTGC +NC_045512.2 17560 17582 SARS-CoV-2_400_56_RIGHT_0 2 - ACAATTTCAGCAGGACAACGCC +NC_045512.2 17478 17507 SARS-CoV-2_400_57_LEFT_0 1 + GGCACACTAGAACCAGAATATTTCAATTC +NC_045512.2 17859 17886 SARS-CoV-2_400_57_RIGHT_0 1 - AGTGGTTTGAGTGAATATGACATAGTC +NC_045512.2 17813 17839 SARS-CoV-2_400_58_LEFT_0 2 + GGGACTACCAACTCAAACTGTTGATT +NC_045512.2 18181 18212 SARS-CoV-2_400_58_RIGHT_0 2 - ATCATAGAGATGAGTCTTCTATAGGTCATGT +NC_045512.2 18121 18153 SARS-CoV-2_400_59_LEFT_0 1 + GTGTTGACACTAAATTCAAAACTGAAGGTTTA +NC_045512.2 18504 18527 SARS-CoV-2_400_59_RIGHT_0 1 - CGCACTACATTCCAAGGAAGTCC +NC_045512.2 18460 18484 SARS-CoV-2_400_60_LEFT_0 2 + CGCCTGGAGATCAATTTAAACACC +NC_045512.2 18835 18860 SARS-CoV-2_400_60_RIGHT_0 2 - GCATCACAACTAGCTACATGTGCAT +NC_045512.2 18789 18815 SARS-CoV-2_400_61_LEFT_0 1 + GGTAACCTACAAAGCAACCATGATCT +NC_045512.2 19170 19195 SARS-CoV-2_400_61_RIGHT_0 1 - TCCAAAATAGGCATACACCATCTGT +NC_045512.2 19087 19112 SARS-CoV-2_400_62_LEFT_2 2 + TCTATGATGCACAGCCTTGTAGTGA +NC_045512.2 19469 19495 SARS-CoV-2_400_62_RIGHT_0 2 - GTCTACAGACAGCACCACCTAAATTG +NC_045512.2 19415 19449 SARS-CoV-2_400_63_LEFT_0 1 + AGTGTCAGATATAGATTATGTACCACTAAAGTCT +NC_045512.2 19770 19796 SARS-CoV-2_400_63_RIGHT_0 1 - GCCCAAAGCTCAAATGCTACATTAAC +NC_045512.2 19721 19750 SARS-CoV-2_400_64_LEFT_0 2 + AGTTGATGGTGTTGATGTAGAATTGTTTG +NC_045512.2 20091 20121 SARS-CoV-2_400_64_RIGHT_0 2 - TAATGTGACTCCATTAAGACTAGCTTGTTT +NC_045512.2 20028 20054 SARS-CoV-2_400_65_LEFT_0 1 + GCCCGTAATGGTGTTCTTATTACAGA +NC_045512.2 20408 20441 SARS-CoV-2_400_65_RIGHT_0 1 - GTACTGTCCATAGGAATAAAATCTTCTAATTCA +NC_045512.2 20358 20388 SARS-CoV-2_400_66_LEFT_0 2 + GGTTTACATCTACTGATTGGACTAGCTAAA +NC_045512.2 20729 20758 SARS-CoV-2_400_66_RIGHT_0 2 - CACTATCACCATAATTTTGAAGGTCACAC +NC_045512.2 20650 20676 SARS-CoV-2_400_67_LEFT_1 1 + AATTACAATCTAGTCAAGCGTGGCAA +NC_045512.2 21018 21051 SARS-CoV-2_400_67_RIGHT_1 1 - CATATCACTAATAATGAGATCCCATTTATTAGC +NC_045512.2 20991 21018 SARS-CoV-2_400_68_LEFT_0 2 + ATTGGTGATTGTGCAACTGTACATACA +NC_045512.2 21372 21402 SARS-CoV-2_400_68_RIGHT_0 2 - ACTCATGTCAAATAAAGAATAGGAAGACAA +NC_045512.2 21322 21352 SARS-CoV-2_400_69_LEFT_0 1 + ATGTCATGCATGCAAATTACATATTTTGGA +NC_045512.2 21696 21722 SARS-CoV-2_400_69_RIGHT_0 1 - AGTCCTGAGTTGAATGTAAAACTGAG +NC_045512.2 21579 21607 SARS-CoV-2_400_70_LEFT_0 2 + TTTATTGCCACTAGTCTCTAGTCAGTGT +NC_045512.2 21927 21960 SARS-CoV-2_400_70_RIGHT_0 2 - AATTCACAGACTTTAATAACAACATTAGTAGCG +NC_045512.2 21866 21894 SARS-CoV-2_400_71_LEFT_0 1 + GAGGCTGGATTTTTGGTACTACTTTAGA +NC_045512.2 22238 22266 SARS-CoV-2_400_71_RIGHT_0 1 - ATGTTAATACCTATTGGCAAATCTACCA +NC_045512.2 22156 22189 SARS-CoV-2_400_72_LEFT_0 2 + GGTTATTTTAAAATATATTCTAAGCACACGCCT +NC_045512.2 22517 22547 SARS-CoV-2_400_72_RIGHT_0 2 - ATCTAACAATAGATTCTGTTGGTTGGACTC +NC_045512.2 22466 22494 SARS-CoV-2_400_73_LEFT_0 1 + CGTTGAAATCCTTCACTGTAGAAAAAGG +NC_045512.2 22839 22866 SARS-CoV-2_400_73_RIGHT_0 1 - GCTATAACGCAGCCTGTAAAATCATCT +NC_045512.2 22742 22774 SARS-CoV-2_400_74_LEFT_0 2 + ATGTCTATGCAGATTCATTTGTAATTAGAGGT +NC_045512.2 23119 23140 SARS-CoV-2_400_74_RIGHT_0 2 - TCCACAAACAGTTGCTGGTGC +NC_045512.2 23078 23109 SARS-CoV-2_400_75_LEFT_1 1 + AACCATACAGAGTAGTAGTACTTTCTTTTGA +NC_045512.2 23452 23478 SARS-CoV-2_400_75_RIGHT_1 1 - CCTGTAGAATAAACACGCCAAGTAGG +NC_045512.2 23229 23258 SARS-CoV-2_400_76_LEFT_0 2 + CAAAAAGTTTCTGCCTTTCCAACAATTTG +NC_045512.2 23609 23631 SARS-CoV-2_400_76_RIGHT_0 2 - TGACTAGCTACACTACGTGCCC +NC_045512.2 23563 23589 SARS-CoV-2_400_77_LEFT_0 1 + GCAGGTATATGCGCTAGTTATCAGAC +NC_045512.2 23914 23944 SARS-CoV-2_400_77_RIGHT_0 1 - AATTGGTGGTGTTTTGTAAATTTGTTTGAC +NC_045512.2 23823 23853 SARS-CoV-2_400_78_LEFT_0 2 + GCAATATGGCAGTTTTTGTACACAATTAAA +NC_045512.2 24209 24231 SARS-CoV-2_400_78_RIGHT_0 2 - GCACCAAAGGTCCAACCAGAAG +NC_045512.2 24160 24189 SARS-CoV-2_400_79_LEFT_0 1 + GATGAAATGATTGCTCAATACACTTCTGC +NC_045512.2 24535 24560 SARS-CoV-2_400_79_RIGHT_0 1 - TGCCTGTGATCAACCTATCAATTTG +NC_045512.2 24442 24468 SARS-CoV-2_400_80_LEFT_0 2 + ACGCTTGTTAAACAACTTAGCTCCAA +NC_045512.2 24815 24839 SARS-CoV-2_400_80_RIGHT_0 2 - CTTCACGAGGAAAGTGTGCTTTTC +NC_045512.2 24751 24774 SARS-CoV-2_400_81_LEFT_0 1 + CATGTGACTTATGTCCCTGCACA +NC_045512.2 25120 25151 SARS-CoV-2_400_81_RIGHT_0 1 - GAGATTCATTTAAATTCTTGGCAACCTCATT +NC_045512.2 25053 25082 SARS-CoV-2_400_82_LEFT_0 2 + TGATTTAGGTGACATCTCTGGCATTAATG +NC_045512.2 25423 25452 SARS-CoV-2_400_82_RIGHT_0 2 - GATTTCACCTTGCTTCAAAGTTACAGTTC +NC_045512.2 25372 25402 SARS-CoV-2_400_83_LEFT_0 1 + CATTACACATAAACGAACTTATGGATTTGT +NC_045512.2 25744 25777 SARS-CoV-2_400_83_RIGHT_0 1 - GCCAAAGCCTCATTATTATTCTTACAAAGTTTA +NC_045512.2 25653 25680 SARS-CoV-2_400_84_LEFT_2 2 + GTAACAGTTTACTCACACCTTTTGCTC +NC_045512.2 26048 26072 SARS-CoV-2_400_84_RIGHT_2 2 - TGTTCAACACCARTGTCTGTACTC +NC_045512.2 26011 26039 SARS-CoV-2_400_85_LEFT_0 1 + TCACTTCAGACTATTACCAGCTGTACTC +NC_045512.2 26382 26411 SARS-CoV-2_400_85_RIGHT_0 1 - AAAGAAGGTTTTACAAGACTCACGTTAAC +NC_045512.2 26339 26362 SARS-CoV-2_400_86_LEFT_0 2 + CATCCTTACTGCGCTTCGATTGT +NC_045512.2 26730 26756 SARS-CoV-2_400_86_RIGHT_0 2 - ACCGGTGATCCAATTTATTCTGTAAA +NC_045512.2 26593 26621 SARS-CoV-2_400_87_LEFT_1 1 + AGGTTTCCTATTCCTTACATGGATTTGT +NC_045512.2 26989 27009 SARS-CoV-2_400_87_RIGHT_1 1 - CCTTGATGTCACAGCGTCCT +NC_045512.2 26958 26981 SARS-CoV-2_400_88_LEFT_2 2 + GTGGACATCTTCGTATTGCTGGA +NC_045512.2 27349 27376 SARS-CoV-2_400_88_RIGHT_2 2 - CCATTGGTTGCTCTTCATCTAATTGAG +NC_045512.2 27200 27226 SARS-CoV-2_400_89_LEFT_2 1 + GATGTTTCATCTCGTTGACTTTCAGG +NC_045512.2 27583 27603 SARS-CoV-2_400_89_RIGHT_0 1 - GCCGTCAGGACAAGCAAAAG +NC_045512.2 27530 27558 SARS-CoV-2_400_90_LEFT_0 2 + TCATCCTCTAGCTGATAACAAATTTGCA +NC_045512.2 27927 27950 SARS-CoV-2_400_90_RIGHT_0 2 - TTCTTGGTGAAATGCAGCTACAG +NC_045512.2 27832 27860 SARS-CoV-2_400_91_LEFT_0 1 + TATCTTTTGGTTCTCACTTGAACTGCAA +NC_045512.2 28209 28237 SARS-CoV-2_400_91_RIGHT_0 1 - CGAACGTCATGATACTCTAAAAAGTCTT +NC_045512.2 28135 28166 SARS-CoV-2_400_92_LEFT_0 2 + TTCCTGTTTACCTTTTACAATTAATTGCCAG +NC_045512.2 28513 28539 SARS-CoV-2_400_92_RIGHT_0 2 - CTTCGGTAGTAGCCAATTTGGTCATC +NC_045512.2 28473 28493 SARS-CoV-2_400_93_LEFT_0 1 + TCGAGGACAAGGCGTTCCAA +NC_045512.2 28849 28873 SARS-CoV-2_400_93_RIGHT_0 1 - GCCTGGAGTTGAATTTCTTGAACT +NC_045512.2 28808 28829 SARS-CoV-2_400_94_LEFT_0 2 + GCAGTCAAGCCTCTTCTCGTT +NC_045512.2 29203 29224 SARS-CoV-2_400_94_RIGHT_0 2 - CATTCCGAAGAACGCTGAAGC +NC_045512.2 29159 29183 SARS-CoV-2_400_95_LEFT_0 1 + CTGATTACAAACATTGGCCGCAAA +NC_045512.2 29538 29559 SARS-CoV-2_400_95_RIGHT_0 1 - ATCTGCCTTGTGTGGTCTGCA +NC_045512.2 29462 29486 SARS-CoV-2_400_96_LEFT_1 2 + CTGCAGATTTGGATGATTTCTCCA +NC_045512.2 29840 29873 SARS-CoV-2_400_96_RIGHT_0 2 - TTTGTCATTCTCCTAAGAAGCTATTAAAATCAC diff --git a/working/references/primers/v532/SARS-CoV-2.primer.fasta b/working/references/primers/v532/SARS-CoV-2.primer.fasta new file mode 100644 index 0000000..15f067e --- /dev/null +++ b/working/references/primers/v532/SARS-CoV-2.primer.fasta @@ -0,0 +1,384 @@ +>SARS-CoV-2_400_1_LEFT_1::NC_045512.2:47-78 +CTCTTGTAGATCTGTTCTCTAAACGAACTTT +>SARS-CoV-2_400_1_RIGHT_1::NC_045512.2:419-447 +AAAACGCCTTTTTCAACTTCTACTAAGC +>SARS-CoV-2_400_2_LEFT_0::NC_045512.2:344-366 +TCGTACGTGGCTTTGGAGACTC +>SARS-CoV-2_400_2_RIGHT_0::NC_045512.2:707-732 +TCTTCATAAGGATCAGTGCCAAGCT +>SARS-CoV-2_400_3_LEFT_1::NC_045512.2:638-661 +AGAACGGTAATAAAGGAGCTGGT +>SARS-CoV-2_400_3_RIGHT_0::NC_045512.2:1018-1047 +TTTGCCAATTTAATTTCAAAAGGTGTCTG +>SARS-CoV-2_400_4_LEFT_0::NC_045512.2:970-995 +CATGAAATTGCTTGGTACACGGAAC +>SARS-CoV-2_400_4_RIGHT_0::NC_045512.2:1340-1370 +AAATTTTAACAACAGCATTTTGGGGTAAGT +>SARS-CoV-2_400_5_LEFT_0::NC_045512.2:1292-1320 +TTTGTGGCACTGAGAATTTGACTAAAGA +>SARS-CoV-2_400_5_RIGHT_0::NC_045512.2:1660-1692 +ATAATGGCGATCTCTTCATTAAGTTTAAAGTC +>SARS-CoV-2_400_6_LEFT_1::NC_045512.2:1574-1596 +GTGTTGTTGGAGAAGGTTCCGA +>SARS-CoV-2_400_6_RIGHT_1::NC_045512.2:1945-1972 +TATTGTTATAGCGGCCTTCTGTAAAAC +>SARS-CoV-2_400_7_LEFT_2::NC_045512.2:1882-1905 +GCTGCTCGTGTTGTACGATCAAT +>SARS-CoV-2_400_7_RIGHT_2::NC_045512.2:2259-2284 +CTCCTTAATTTCCTTTGCACAGGTG +>SARS-CoV-2_400_8_LEFT_0::NC_045512.2:2229-2252 +TGCTTGTGAAATTGTCGGTGGAC +>SARS-CoV-2_400_8_RIGHT_0::NC_045512.2:2603-2629 +AAGCCCGTTAATACAAACTGGTGTAC +>SARS-CoV-2_400_9_LEFT_0::NC_045512.2:2533-2563 +GTCTTGAAAACTGGTGATTTACAACCATTA +>SARS-CoV-2_400_9_RIGHT_0::NC_045512.2:2900-2933 +GTAATTCAGATACTGGTTGCAAAGTTTTTATGA +>SARS-CoV-2_400_10_LEFT_0::NC_045512.2:2854-2880 +CTCGGTACAGAAGTAAATGAGTTCGC +>SARS-CoV-2_400_10_RIGHT_0::NC_045512.2:3233-3254 +TGTCCTCACTGCCGTCTTGTT +>SARS-CoV-2_400_11_LEFT_0::NC_045512.2:3184-3213 +GAGCAAGAAGAAGATTGGTTAGATGATGA +>SARS-CoV-2_400_11_RIGHT_0::NC_045512.2:3560-3584 +CGCTTAAAACACAACTACCACCCA +>SARS-CoV-2_400_12_LEFT_0::NC_045512.2:3510-3540 +CATGCAAGTTGAATCTGATGATTACATAGC +>SARS-CoV-2_400_12_RIGHT_0::NC_045512.2:3883-3913 +AGTTATAAATGGCTTAACTTCCTCTTTAGG +>SARS-CoV-2_400_13_LEFT_0::NC_045512.2:3791-3824 +CTGTCTTTGATAAAAATCTCTATGACAAACTTG +>SARS-CoV-2_400_13_RIGHT_0::NC_045512.2:4147-4180 +CTTTTTAGTAGGTATAACCACAGCAGTTAAAAC +>SARS-CoV-2_400_14_LEFT_0::NC_045512.2:4079-4108 +GTGACATTGACATCACTTTCTTAAAGAAA +>SARS-CoV-2_400_14_RIGHT_0::NC_045512.2:4457-4488 +CCCTTATATTTACGCTGTATAGTTGAAACTA +>SARS-CoV-2_400_15_LEFT_0::NC_045512.2:4403-4425 +CACATGCAGAAGAAACACGCAA +>SARS-CoV-2_400_15_RIGHT_0::NC_045512.2:4776-4803 +TCTTTATAGGAACCAGCAAGTGAGATG +>SARS-CoV-2_400_16_LEFT_0::NC_045512.2:4723-4756 +AATGGTTATCTTACTTCTTCTTCTAAAACACCT +>SARS-CoV-2_400_16_RIGHT_0::NC_045512.2:5089-5119 +AAATGTTTTACCTTCATGTGAATTATGAGG +>SARS-CoV-2_400_17_LEFT_0::NC_045512.2:5036-5063 +GACAACAGTTTGGTCCAACTTATTTGG +>SARS-CoV-2_400_17_RIGHT_0::NC_045512.2:5398-5429 +CTGTCTTATTACAGTAGGCTAAGATAAGTGC +>SARS-CoV-2_400_18_LEFT_0::NC_045512.2:5344-5370 +GCTCTACAAGATGCTTATTACAGAGC +>SARS-CoV-2_400_18_RIGHT_0::NC_045512.2:5716-5744 +ACTCACTAGCACAAGTAAATGTACCATG +>SARS-CoV-2_400_19_LEFT_0::NC_045512.2:5671-5696 +CCTTTTGTTATGATGTCAGCACCAC +>SARS-CoV-2_400_19_RIGHT_0::NC_045512.2:6031-6062 +CACATACAAACTTAAAATTATCGAAGCTTGC +>SARS-CoV-2_400_20_LEFT_0::NC_045512.2:5891-5923 +CCATAAAACCAGTTACTTATAAATTGGATGGT +>SARS-CoV-2_400_20_RIGHT_0::NC_045512.2:6257-6288 +CAGGTATTTGGTTTATACGTGGCTTTATTAG +>SARS-CoV-2_400_21_LEFT_0::NC_045512.2:6204-6237 +GAAAGGAGCTAAATTGTTACATAAACCTATTGT +>SARS-CoV-2_400_21_RIGHT_0::NC_045512.2:6562-6595 +AGGTTTCTTAATAGTAAGACTAGAATTGTCTAC +>SARS-CoV-2_400_22_LEFT_0::NC_045512.2:6515-6542 +TAAAAATTACAGAAGAGGTTGGCCACA +>SARS-CoV-2_400_22_RIGHT_0::NC_045512.2:6882-6915 +TAATTAAATGAAGCCTCTAGACAAAATTTACCG +>SARS-CoV-2_400_23_LEFT_0::NC_045512.2:6823-6854 +AATTCTAGAATTAAAGCATCTATGCCGACTA +>SARS-CoV-2_400_23_RIGHT_0::NC_045512.2:7199-7229 +CAAAAGCAGTTAAATCCCATTTAAAAGATG +>SARS-CoV-2_400_24_LEFT_0::NC_045512.2:7145-7179 +GTTTAGATTCTTTAGACACCTATCCTTCTTTAGA +>SARS-CoV-2_400_24_RIGHT_0::NC_045512.2:7518-7545 +ACAATAGTTGTACATTCGACTCTTGTT +>SARS-CoV-2_400_25_LEFT_0::NC_045512.2:7456-7482 +GTGCATGTTGTAGACGGTTGTAATTC +>SARS-CoV-2_400_25_RIGHT_0::NC_045512.2:7819-7850 +CTCTCAGGTTGTCTAAGTTAACAAAATGAGA +>SARS-CoV-2_400_26_LEFT_0::NC_045512.2:7768-7797 +CATCTTTACTTTGATAAAGCTGGTCAAAA +>SARS-CoV-2_400_26_RIGHT_0::NC_045512.2:8136-8169 +GAAATAAAAGTAGATAAGACATTGTCTAAGGAC +>SARS-CoV-2_400_27_LEFT_0::NC_045512.2:8085-8112 +AAAACTCAAAACACTAGTTGCAACTGC +>SARS-CoV-2_400_27_RIGHT_0::NC_045512.2:8468-8498 +CACATGTCAACTTAAAAGGTAAGTTATTCT +>SARS-CoV-2_400_28_LEFT_0::NC_045512.2:8406-8436 +CGTTAAAGATTTCATGTCATTGTCTGAACA +>SARS-CoV-2_400_28_RIGHT_0::NC_045512.2:8781-8806 +ATTAGTATAACTACCACCACGCTGG +>SARS-CoV-2_400_29_LEFT_0::NC_045512.2:8732-8761 +CAGATACTTGTTTTGCTAACAAACATGCT +>SARS-CoV-2_400_29_RIGHT_0::NC_045512.2:9107-9129 +ACATAACGTGTGTCAGGGCGTA +>SARS-CoV-2_400_30_LEFT_0::NC_045512.2:9023-9052 +CAATTTTTAAAGATGCTTCTGGTAAGCCA +>SARS-CoV-2_400_30_RIGHT_0::NC_045512.2:9397-9423 +ACAATACCACCAGCTACTATAGATGC +>SARS-CoV-2_400_31_LEFT_1::NC_045512.2:9299-9324 +GATCTTTACCAGGAGTTTTCTGTGG +>SARS-CoV-2_400_31_RIGHT_0::NC_045512.2:9673-9706 +ACAAATGATATAAGCAATTGTTATCCAGAAAGG +>SARS-CoV-2_400_32_LEFT_0::NC_045512.2:9571-9604 +GGTGTTTATTCTGTTATTTACTTGTACTTGACA +>SARS-CoV-2_400_32_RIGHT_0::NC_045512.2:9949-9971 +TTGCGAGATGACAACAAGCAGC +>SARS-CoV-2_400_33_LEFT_0::NC_045512.2:9896-9929 +ATAATAAGTACAAGTATTTTAGTGGAGCAATGG +>SARS-CoV-2_400_33_RIGHT_0::NC_045512.2:10266-10295 +AATGTCCAATAACCCTGAGTTGAACATTA +>SARS-CoV-2_400_34_LEFT_0::NC_045512.2:10215-10245 +TGAAGATTTACTCATTCGTAAGTCTAATCA +>SARS-CoV-2_400_34_RIGHT_0::NC_045512.2:10587-10615 +GTCAACAAAAGGTCCATAAAAGTTACCT +>SARS-CoV-2_400_35_LEFT_0::NC_045512.2:10527-10557 +TTTTTGTTACATGCACCATATGGAATTACC +>SARS-CoV-2_400_35_RIGHT_0::NC_045512.2:10897-10927 +AAATTCATCTTCTAATAAAGCACTACCCAA +>SARS-CoV-2_400_36_LEFT_0::NC_045512.2:10832-10865 +CCGTTTTAGATATGTGTGCTTCATTAAAAGAAT +>SARS-CoV-2_400_36_RIGHT_0::NC_045512.2:11201-11232 +GGCATATAGACCATATTAAAATAAGCTACAG +>SARS-CoV-2_400_37_LEFT_0::NC_045512.2:11152-11181 +AAACATAAGCATGCATTTCTCTGTTTGTT +>SARS-CoV-2_400_37_RIGHT_0::NC_045512.2:11514-11536 +ACCTCTGGCCAAAAACATGACA +>SARS-CoV-2_400_38_LEFT_0::NC_045512.2:11463-11494 +GTGGGCTCTTATAATCTCTGTTACTTCTAAC +>SARS-CoV-2_400_38_RIGHT_0::NC_045512.2:11832-11863 +CTTTACATCTGACATTTTAGACTGTACAGTG +>SARS-CoV-2_400_39_LEFT_0::NC_045512.2:11785-11811 +AACATTAAATTGTTGGGTGTTGGTGG +>SARS-CoV-2_400_39_RIGHT_0::NC_045512.2:12161-12185 +CAGAATCACCATTAGCAACAGCCT +>SARS-CoV-2_400_40_LEFT_0::NC_045512.2:12112-12137 +TCCCTTCCATCATATGCAGCTTTTG +>SARS-CoV-2_400_40_RIGHT_0::NC_045512.2:12477-12510 +TTTTTATATGTGTTATAGTCTGGTATGACAACC +>SARS-CoV-2_400_41_LEFT_0::NC_045512.2:12419-12444 +CAAGAGATGGTTGTGTTCCCTTGAA +>SARS-CoV-2_400_41_RIGHT_0::NC_045512.2:12794-12819 +AACAGTGCAAGTACAAACCTACCTC +>SARS-CoV-2_400_42_LEFT_0::NC_045512.2:12752-12774 +GCACTGATGACAATGCGTTAGC +>SARS-CoV-2_400_42_RIGHT_0::NC_045512.2:13121-13146 +CAATTAGTGATTGGTTGTCCCCCAC +>SARS-CoV-2_400_43_LEFT_0::NC_045512.2:13075-13099 +GCTTTTGCTGTAGATGCTGCTAAA +>SARS-CoV-2_400_43_RIGHT_0::NC_045512.2:13458-13480 +CACCGCAAACCCGTTTAAAAAC +>SARS-CoV-2_400_44_LEFT_0::NC_045512.2:13415-13435 +ATCAACTCCGCGAACCCATG +>SARS-CoV-2_400_44_RIGHT_0::NC_045512.2:13787-13815 +TGCCATTGTGTATTTAGTAAGACGTTGA +>SARS-CoV-2_400_45_LEFT_0::NC_045512.2:13738-13767 +ACTTCTTTAAGTTTAGAATAGACGGTGAC +>SARS-CoV-2_400_45_RIGHT_0::NC_045512.2:14120-14144 +TCTACAACAGGAACTCCACTACCT +>SARS-CoV-2_400_46_LEFT_0::NC_045512.2:14073-14100 +CTCAATGGTAACTGGTATGATTTCGGT +>SARS-CoV-2_400_46_RIGHT_0::NC_045512.2:14427-14457 +TGGAACACCATCAACAAATATTTTTCTCAC +>SARS-CoV-2_400_47_LEFT_0::NC_045512.2:14375-14407 +CTTTAATGTTTTATTCTCTACAGTGTTCCCAC +>SARS-CoV-2_400_47_RIGHT_0::NC_045512.2:14745-14775 +ATCCTGAGCAAAGAAGAAGTGTTTTAATTC +>SARS-CoV-2_400_48_LEFT_0::NC_045512.2:14700-14725 +GACTTTGCTGTGTCTAAGGGTTTCT +>SARS-CoV-2_400_48_RIGHT_0::NC_045512.2:15065-15095 +TTCTTTGCACTAATGGCATACTTAAGATTC +>SARS-CoV-2_400_49_LEFT_0::NC_045512.2:15016-15045 +CACTTTTCGCATATACAAAACGTAATGTC +>SARS-CoV-2_400_49_RIGHT_0::NC_045512.2:15386-15416 +CACTCATTAGCTAATCTATAGAAACGGTGT +>SARS-CoV-2_400_50_LEFT_0::NC_045512.2:15342-15366 +TCACTTGTTCTTGCTCGCAAACAT +>SARS-CoV-2_400_50_RIGHT_0::NC_045512.2:15716-15742 +TATTGAAACACACAACAGCATCGTCA +>SARS-CoV-2_400_51_LEFT_0::NC_045512.2:15659-15688 +CTTTGTGAATGAGTTTTACGCATATTTGC +>SARS-CoV-2_400_51_RIGHT_0::NC_045512.2:16028-16059 +AGGATGTTTAGTAAGTGGGTAAGCATCTATA +>SARS-CoV-2_400_52_LEFT_2::NC_045512.2:15992-16018 +TGGTACACTTATGATTGAACGGTTCG +>SARS-CoV-2_400_52_RIGHT_2::NC_045512.2:16386-16409 +GTGACATCACAACCTGGAGCATT +>SARS-CoV-2_400_53_LEFT_0::NC_045512.2:16285-16311 +GTGCTTGCATACGTAGACCATTCTTA +>SARS-CoV-2_400_53_RIGHT_0::NC_045512.2:16650-16679 +GACAGTTTAAATGTCTCCTCAGTAGCTTT +>SARS-CoV-2_400_54_LEFT_1::NC_045512.2:16624-16647 +TCAAGCTTTTTGCAGCAGAAACG +>SARS-CoV-2_400_54_RIGHT_1::NC_045512.2:17004-17033 +ACATTGCTAGAAAACTCATCTGAGATATT +>SARS-CoV-2_400_55_LEFT_1::NC_045512.2:16962-16994 +CAAGAGCACTATGTTAGAATTACTGGCTTATA +>SARS-CoV-2_400_55_RIGHT_1::NC_045512.2:17333-17362 +TTTCATCAAAGACAACTATATCTGCTGTC +>SARS-CoV-2_400_56_LEFT_0::NC_045512.2:17182-17212 +CACTATGTGAGAAGGCATTAAAATATTTGC +>SARS-CoV-2_400_56_RIGHT_0::NC_045512.2:17560-17582 +ACAATTTCAGCAGGACAACGCC +>SARS-CoV-2_400_57_LEFT_0::NC_045512.2:17478-17507 +GGCACACTAGAACCAGAATATTTCAATTC +>SARS-CoV-2_400_57_RIGHT_0::NC_045512.2:17859-17886 +AGTGGTTTGAGTGAATATGACATAGTC +>SARS-CoV-2_400_58_LEFT_0::NC_045512.2:17813-17839 +GGGACTACCAACTCAAACTGTTGATT +>SARS-CoV-2_400_58_RIGHT_0::NC_045512.2:18181-18212 +ATCATAGAGATGAGTCTTCTATAGGTCATGT +>SARS-CoV-2_400_59_LEFT_0::NC_045512.2:18121-18153 +GTGTTGACACTAAATTCAAAACTGAAGGTTTA +>SARS-CoV-2_400_59_RIGHT_0::NC_045512.2:18504-18527 +CGCACTACATTCCAAGGAAGTCC +>SARS-CoV-2_400_60_LEFT_0::NC_045512.2:18460-18484 +CGCCTGGAGATCAATTTAAACACC +>SARS-CoV-2_400_60_RIGHT_0::NC_045512.2:18835-18860 +GCATCACAACTAGCTACATGTGCAT +>SARS-CoV-2_400_61_LEFT_0::NC_045512.2:18789-18815 +GGTAACCTACAAAGCAACCATGATCT +>SARS-CoV-2_400_61_RIGHT_0::NC_045512.2:19170-19195 +TCCAAAATAGGCATACACCATCTGT +>SARS-CoV-2_400_62_LEFT_2::NC_045512.2:19087-19112 +TCTATGATGCACAGCCTTGTAGTGA +>SARS-CoV-2_400_62_RIGHT_0::NC_045512.2:19469-19495 +GTCTACAGACAGCACCACCTAAATTG +>SARS-CoV-2_400_63_LEFT_0::NC_045512.2:19415-19449 +AGTGTCAGATATAGATTATGTACCACTAAAGTCT +>SARS-CoV-2_400_63_RIGHT_0::NC_045512.2:19770-19796 +GCCCAAAGCTCAAATGCTACATTAAC +>SARS-CoV-2_400_64_LEFT_0::NC_045512.2:19721-19750 +AGTTGATGGTGTTGATGTAGAATTGTTTG +>SARS-CoV-2_400_64_RIGHT_0::NC_045512.2:20091-20121 +TAATGTGACTCCATTAAGACTAGCTTGTTT +>SARS-CoV-2_400_65_LEFT_0::NC_045512.2:20028-20054 +GCCCGTAATGGTGTTCTTATTACAGA +>SARS-CoV-2_400_65_RIGHT_0::NC_045512.2:20408-20441 +GTACTGTCCATAGGAATAAAATCTTCTAATTCA +>SARS-CoV-2_400_66_LEFT_0::NC_045512.2:20358-20388 +GGTTTACATCTACTGATTGGACTAGCTAAA +>SARS-CoV-2_400_66_RIGHT_0::NC_045512.2:20729-20758 +CACTATCACCATAATTTTGAAGGTCACAC +>SARS-CoV-2_400_67_LEFT_1::NC_045512.2:20650-20676 +AATTACAATCTAGTCAAGCGTGGCAA +>SARS-CoV-2_400_67_RIGHT_1::NC_045512.2:21018-21051 +CATATCACTAATAATGAGATCCCATTTATTAGC +>SARS-CoV-2_400_68_LEFT_0::NC_045512.2:20991-21018 +ATTGGTGATTGTGCAACTGTACATACA +>SARS-CoV-2_400_68_RIGHT_0::NC_045512.2:21372-21402 +ACTCATGTCAAATAAAGAATAGGAAGACAA +>SARS-CoV-2_400_69_LEFT_0::NC_045512.2:21322-21352 +ATGTCATGCATGCAAATTACATATTTTGGA +>SARS-CoV-2_400_69_RIGHT_0::NC_045512.2:21696-21722 +AGTCCTGAGTTGAATGTAAAACTGAG +>SARS-CoV-2_400_70_LEFT_0::NC_045512.2:21579-21607 +TTTATTGCCACTAGTCTCTAGTCAGTGT +>SARS-CoV-2_400_70_RIGHT_0::NC_045512.2:21927-21960 +AATTCACAGACTTTAATAACAACATTAGTAGCG +>SARS-CoV-2_400_71_LEFT_0::NC_045512.2:21866-21894 +GAGGCTGGATTTTTGGTACTACTTTAGA +>SARS-CoV-2_400_71_RIGHT_0::NC_045512.2:22238-22266 +ATGTTAATACCTATTGGCAAATCTACCA +>SARS-CoV-2_400_72_LEFT_0::NC_045512.2:22156-22189 +GGTTATTTTAAAATATATTCTAAGCACACGCCT +>SARS-CoV-2_400_72_RIGHT_0::NC_045512.2:22517-22547 +ATCTAACAATAGATTCTGTTGGTTGGACTC +>SARS-CoV-2_400_73_LEFT_0::NC_045512.2:22466-22494 +CGTTGAAATCCTTCACTGTAGAAAAAGG +>SARS-CoV-2_400_73_RIGHT_0::NC_045512.2:22839-22866 +GCTATAACGCAGCCTGTAAAATCATCT +>SARS-CoV-2_400_74_LEFT_0::NC_045512.2:22742-22774 +ATGTCTATGCAGATTCATTTGTAATTAGAGGT +>SARS-CoV-2_400_74_RIGHT_0::NC_045512.2:23119-23140 +TCCACAAACAGTTGCTGGTGC +>SARS-CoV-2_400_75_LEFT_1::NC_045512.2:23078-23109 +AACCATACAGAGTAGTAGTACTTTCTTTTGA +>SARS-CoV-2_400_75_RIGHT_1::NC_045512.2:23452-23478 +CCTGTAGAATAAACACGCCAAGTAGG +>SARS-CoV-2_400_76_LEFT_0::NC_045512.2:23229-23258 +CAAAAAGTTTCTGCCTTTCCAACAATTTG +>SARS-CoV-2_400_76_RIGHT_0::NC_045512.2:23609-23631 +TGACTAGCTACACTACGTGCCC +>SARS-CoV-2_400_77_LEFT_0::NC_045512.2:23563-23589 +GCAGGTATATGCGCTAGTTATCAGAC +>SARS-CoV-2_400_77_RIGHT_0::NC_045512.2:23914-23944 +AATTGGTGGTGTTTTGTAAATTTGTTTGAC +>SARS-CoV-2_400_78_LEFT_0::NC_045512.2:23823-23853 +GCAATATGGCAGTTTTTGTACACAATTAAA +>SARS-CoV-2_400_78_RIGHT_0::NC_045512.2:24209-24231 +GCACCAAAGGTCCAACCAGAAG +>SARS-CoV-2_400_79_LEFT_0::NC_045512.2:24160-24189 +GATGAAATGATTGCTCAATACACTTCTGC +>SARS-CoV-2_400_79_RIGHT_0::NC_045512.2:24535-24560 +TGCCTGTGATCAACCTATCAATTTG +>SARS-CoV-2_400_80_LEFT_0::NC_045512.2:24442-24468 +ACGCTTGTTAAACAACTTAGCTCCAA +>SARS-CoV-2_400_80_RIGHT_0::NC_045512.2:24815-24839 +CTTCACGAGGAAAGTGTGCTTTTC +>SARS-CoV-2_400_81_LEFT_0::NC_045512.2:24751-24774 +CATGTGACTTATGTCCCTGCACA +>SARS-CoV-2_400_81_RIGHT_0::NC_045512.2:25120-25151 +GAGATTCATTTAAATTCTTGGCAACCTCATT +>SARS-CoV-2_400_82_LEFT_0::NC_045512.2:25053-25082 +TGATTTAGGTGACATCTCTGGCATTAATG +>SARS-CoV-2_400_82_RIGHT_0::NC_045512.2:25423-25452 +GATTTCACCTTGCTTCAAAGTTACAGTTC +>SARS-CoV-2_400_83_LEFT_0::NC_045512.2:25372-25402 +CATTACACATAAACGAACTTATGGATTTGT +>SARS-CoV-2_400_83_RIGHT_0::NC_045512.2:25744-25777 +GCCAAAGCCTCATTATTATTCTTACAAAGTTTA +>SARS-CoV-2_400_84_LEFT_2::NC_045512.2:25653-25680 +GTAACAGTTTACTCACACCTTTTGCTC +>SARS-CoV-2_400_84_RIGHT_2::NC_045512.2:26048-26072 +TGTTCAACACCARTGTCTGTACTC +>SARS-CoV-2_400_85_LEFT_0::NC_045512.2:26011-26039 +TCACTTCAGACTATTACCAGCTGTACTC +>SARS-CoV-2_400_85_RIGHT_0::NC_045512.2:26382-26411 +AAAGAAGGTTTTACAAGACTCACGTTAAC +>SARS-CoV-2_400_86_LEFT_0::NC_045512.2:26339-26362 +CATCCTTACTGCGCTTCGATTGT +>SARS-CoV-2_400_86_RIGHT_0::NC_045512.2:26730-26756 +ACCGGTGATCCAATTTATTCTGTAAA +>SARS-CoV-2_400_87_LEFT_1::NC_045512.2:26593-26621 +AGGTTTCCTATTCCTTACATGGATTTGT +>SARS-CoV-2_400_87_RIGHT_1::NC_045512.2:26989-27009 +CCTTGATGTCACAGCGTCCT +>SARS-CoV-2_400_88_LEFT_2::NC_045512.2:26958-26981 +GTGGACATCTTCGTATTGCTGGA +>SARS-CoV-2_400_88_RIGHT_2::NC_045512.2:27349-27376 +CCATTGGTTGCTCTTCATCTAATTGAG +>SARS-CoV-2_400_89_LEFT_2::NC_045512.2:27200-27226 +GATGTTTCATCTCGTTGACTTTCAGG +>SARS-CoV-2_400_89_RIGHT_0::NC_045512.2:27583-27603 +GCCGTCAGGACAAGCAAAAG +>SARS-CoV-2_400_90_LEFT_0::NC_045512.2:27530-27558 +TCATCCTCTAGCTGATAACAAATTTGCA +>SARS-CoV-2_400_90_RIGHT_0::NC_045512.2:27927-27950 +TTCTTGGTGAAATGCAGCTACAG +>SARS-CoV-2_400_91_LEFT_0::NC_045512.2:27832-27860 +TATCTTTTGGTTCTCACTTGAACTGCAA +>SARS-CoV-2_400_91_RIGHT_0::NC_045512.2:28209-28237 +CGAACGTCATGATACTCTAAAAAGTCTT +>SARS-CoV-2_400_92_LEFT_0::NC_045512.2:28135-28166 +TTCCTGTTTACCTTTTACAATTAATTGCCAG +>SARS-CoV-2_400_92_RIGHT_0::NC_045512.2:28513-28539 +CTTCGGTAGTAGCCAATTTGGTCATC +>SARS-CoV-2_400_93_LEFT_0::NC_045512.2:28473-28493 +TCGAGGACAAGGCGTTCCAA +>SARS-CoV-2_400_93_RIGHT_0::NC_045512.2:28849-28873 +GCCTGGAGTTGAATTTCTTGAACT +>SARS-CoV-2_400_94_LEFT_0::NC_045512.2:28808-28829 +GCAGTCAAGCCTCTTCTCGTT +>SARS-CoV-2_400_94_RIGHT_0::NC_045512.2:29203-29224 +CATTCCGAAGAACGCTGAAGC +>SARS-CoV-2_400_95_LEFT_0::NC_045512.2:29159-29183 +CTGATTACAAACATTGGCCGCAAA +>SARS-CoV-2_400_95_RIGHT_0::NC_045512.2:29538-29559 +ATCTGCCTTGTGTGGTCTGCA +>SARS-CoV-2_400_96_LEFT_1::NC_045512.2:29462-29486 +CTGCAGATTTGGATGATTTCTCCA +>SARS-CoV-2_400_96_RIGHT_0::NC_045512.2:29840-29873 +TTTGTCATTCTCCTAAGAAGCTATTAAAATCAC diff --git a/working/references/primers/v532/SARS-CoV-2.tsv b/working/references/primers/v532/SARS-CoV-2.tsv new file mode 100644 index 0000000..893f62b --- /dev/null +++ b/working/references/primers/v532/SARS-CoV-2.tsv @@ -0,0 +1,192 @@ +SARS-CoV-2_400_1_LEFT_1,CTCTTGTAGATCTGTTCTCTAAACGAACTTT +SARS-CoV-2_400_1_RIGHT_1,AAAACGCCTTTTTCAACTTCTACTAAGC +SARS-CoV-2_400_2_LEFT_0,TCGTACGTGGCTTTGGAGACTC +SARS-CoV-2_400_2_RIGHT_0,TCTTCATAAGGATCAGTGCCAAGCT +SARS-CoV-2_400_3_LEFT_1,AGAACGGTAATAAAGGAGCTGGT +SARS-CoV-2_400_3_RIGHT_0,TTTGCCAATTTAATTTCAAAAGGTGTCTG +SARS-CoV-2_400_4_LEFT_0,CATGAAATTGCTTGGTACACGGAAC +SARS-CoV-2_400_4_RIGHT_0,AAATTTTAACAACAGCATTTTGGGGTAAGT +SARS-CoV-2_400_5_LEFT_0,TTTGTGGCACTGAGAATTTGACTAAAGA +SARS-CoV-2_400_5_RIGHT_0,ATAATGGCGATCTCTTCATTAAGTTTAAAGTC +SARS-CoV-2_400_6_LEFT_1,GTGTTGTTGGAGAAGGTTCCGA +SARS-CoV-2_400_6_RIGHT_1,TATTGTTATAGCGGCCTTCTGTAAAAC +SARS-CoV-2_400_7_LEFT_2,GCTGCTCGTGTTGTACGATCAAT +SARS-CoV-2_400_7_RIGHT_2,CTCCTTAATTTCCTTTGCACAGGTG +SARS-CoV-2_400_8_LEFT_0,TGCTTGTGAAATTGTCGGTGGAC +SARS-CoV-2_400_8_RIGHT_0,AAGCCCGTTAATACAAACTGGTGTAC +SARS-CoV-2_400_9_LEFT_0,GTCTTGAAAACTGGTGATTTACAACCATTA +SARS-CoV-2_400_9_RIGHT_0,GTAATTCAGATACTGGTTGCAAAGTTTTTATGA +SARS-CoV-2_400_10_LEFT_0,CTCGGTACAGAAGTAAATGAGTTCGC +SARS-CoV-2_400_10_RIGHT_0,TGTCCTCACTGCCGTCTTGTT +SARS-CoV-2_400_11_LEFT_0,GAGCAAGAAGAAGATTGGTTAGATGATGA +SARS-CoV-2_400_11_RIGHT_0,CGCTTAAAACACAACTACCACCCA +SARS-CoV-2_400_12_LEFT_0,CATGCAAGTTGAATCTGATGATTACATAGC +SARS-CoV-2_400_12_RIGHT_0,AGTTATAAATGGCTTAACTTCCTCTTTAGG +SARS-CoV-2_400_13_LEFT_0,CTGTCTTTGATAAAAATCTCTATGACAAACTTG +SARS-CoV-2_400_13_RIGHT_0,CTTTTTAGTAGGTATAACCACAGCAGTTAAAAC +SARS-CoV-2_400_14_LEFT_0,GTGACATTGACATCACTTTCTTAAAGAAA +SARS-CoV-2_400_14_RIGHT_0,CCCTTATATTTACGCTGTATAGTTGAAACTA +SARS-CoV-2_400_15_LEFT_0,CACATGCAGAAGAAACACGCAA +SARS-CoV-2_400_15_RIGHT_0,TCTTTATAGGAACCAGCAAGTGAGATG +SARS-CoV-2_400_16_LEFT_0,AATGGTTATCTTACTTCTTCTTCTAAAACACCT +SARS-CoV-2_400_16_RIGHT_0,AAATGTTTTACCTTCATGTGAATTATGAGG +SARS-CoV-2_400_17_LEFT_0,GACAACAGTTTGGTCCAACTTATTTGG +SARS-CoV-2_400_17_RIGHT_0,CTGTCTTATTACAGTAGGCTAAGATAAGTGC +SARS-CoV-2_400_18_LEFT_0,GCTCTACAAGATGCTTATTACAGAGC +SARS-CoV-2_400_18_RIGHT_0,ACTCACTAGCACAAGTAAATGTACCATG +SARS-CoV-2_400_19_LEFT_0,CCTTTTGTTATGATGTCAGCACCAC +SARS-CoV-2_400_19_RIGHT_0,CACATACAAACTTAAAATTATCGAAGCTTGC +SARS-CoV-2_400_20_LEFT_0,CCATAAAACCAGTTACTTATAAATTGGATGGT +SARS-CoV-2_400_20_RIGHT_0,CAGGTATTTGGTTTATACGTGGCTTTATTAG +SARS-CoV-2_400_21_LEFT_0,GAAAGGAGCTAAATTGTTACATAAACCTATTGT +SARS-CoV-2_400_21_RIGHT_0,AGGTTTCTTAATAGTAAGACTAGAATTGTCTAC +SARS-CoV-2_400_22_LEFT_0,TAAAAATTACAGAAGAGGTTGGCCACA +SARS-CoV-2_400_22_RIGHT_0,TAATTAAATGAAGCCTCTAGACAAAATTTACCG +SARS-CoV-2_400_23_LEFT_0,AATTCTAGAATTAAAGCATCTATGCCGACTA +SARS-CoV-2_400_23_RIGHT_0,CAAAAGCAGTTAAATCCCATTTAAAAGATG +SARS-CoV-2_400_24_LEFT_0,GTTTAGATTCTTTAGACACCTATCCTTCTTTAGA +SARS-CoV-2_400_24_RIGHT_0,ACAATAGTTGTACATTCGACTCTTGTT +SARS-CoV-2_400_25_LEFT_0,GTGCATGTTGTAGACGGTTGTAATTC +SARS-CoV-2_400_25_RIGHT_0,CTCTCAGGTTGTCTAAGTTAACAAAATGAGA +SARS-CoV-2_400_26_LEFT_0,CATCTTTACTTTGATAAAGCTGGTCAAAA +SARS-CoV-2_400_26_RIGHT_0,GAAATAAAAGTAGATAAGACATTGTCTAAGGAC +SARS-CoV-2_400_27_LEFT_0,AAAACTCAAAACACTAGTTGCAACTGC +SARS-CoV-2_400_27_RIGHT_0,CACATGTCAACTTAAAAGGTAAGTTATTCT +SARS-CoV-2_400_28_LEFT_0,CGTTAAAGATTTCATGTCATTGTCTGAACA +SARS-CoV-2_400_28_RIGHT_0,ATTAGTATAACTACCACCACGCTGG +SARS-CoV-2_400_29_LEFT_0,CAGATACTTGTTTTGCTAACAAACATGCT +SARS-CoV-2_400_29_RIGHT_0,ACATAACGTGTGTCAGGGCGTA +SARS-CoV-2_400_30_LEFT_0,CAATTTTTAAAGATGCTTCTGGTAAGCCA +SARS-CoV-2_400_30_RIGHT_0,ACAATACCACCAGCTACTATAGATGC +SARS-CoV-2_400_31_LEFT_1,GATCTTTACCAGGAGTTTTCTGTGG +SARS-CoV-2_400_31_RIGHT_0,ACAAATGATATAAGCAATTGTTATCCAGAAAGG +SARS-CoV-2_400_32_LEFT_0,GGTGTTTATTCTGTTATTTACTTGTACTTGACA +SARS-CoV-2_400_32_RIGHT_0,TTGCGAGATGACAACAAGCAGC +SARS-CoV-2_400_33_LEFT_0,ATAATAAGTACAAGTATTTTAGTGGAGCAATGG +SARS-CoV-2_400_33_RIGHT_0,AATGTCCAATAACCCTGAGTTGAACATTA +SARS-CoV-2_400_34_LEFT_0,TGAAGATTTACTCATTCGTAAGTCTAATCA +SARS-CoV-2_400_34_RIGHT_0,GTCAACAAAAGGTCCATAAAAGTTACCT +SARS-CoV-2_400_35_LEFT_0,TTTTTGTTACATGCACCATATGGAATTACC +SARS-CoV-2_400_35_RIGHT_0,AAATTCATCTTCTAATAAAGCACTACCCAA +SARS-CoV-2_400_36_LEFT_0,CCGTTTTAGATATGTGTGCTTCATTAAAAGAAT +SARS-CoV-2_400_36_RIGHT_0,GGCATATAGACCATATTAAAATAAGCTACAG +SARS-CoV-2_400_37_LEFT_0,AAACATAAGCATGCATTTCTCTGTTTGTT +SARS-CoV-2_400_37_RIGHT_0,ACCTCTGGCCAAAAACATGACA +SARS-CoV-2_400_38_LEFT_0,GTGGGCTCTTATAATCTCTGTTACTTCTAAC +SARS-CoV-2_400_38_RIGHT_0,CTTTACATCTGACATTTTAGACTGTACAGTG +SARS-CoV-2_400_39_LEFT_0,AACATTAAATTGTTGGGTGTTGGTGG +SARS-CoV-2_400_39_RIGHT_0,CAGAATCACCATTAGCAACAGCCT +SARS-CoV-2_400_40_LEFT_0,TCCCTTCCATCATATGCAGCTTTTG +SARS-CoV-2_400_40_RIGHT_0,TTTTTATATGTGTTATAGTCTGGTATGACAACC +SARS-CoV-2_400_41_LEFT_0,CAAGAGATGGTTGTGTTCCCTTGAA +SARS-CoV-2_400_41_RIGHT_0,AACAGTGCAAGTACAAACCTACCTC +SARS-CoV-2_400_42_LEFT_0,GCACTGATGACAATGCGTTAGC +SARS-CoV-2_400_42_RIGHT_0,CAATTAGTGATTGGTTGTCCCCCAC +SARS-CoV-2_400_43_LEFT_0,GCTTTTGCTGTAGATGCTGCTAAA +SARS-CoV-2_400_43_RIGHT_0,CACCGCAAACCCGTTTAAAAAC +SARS-CoV-2_400_44_LEFT_0,ATCAACTCCGCGAACCCATG +SARS-CoV-2_400_44_RIGHT_0,TGCCATTGTGTATTTAGTAAGACGTTGA +SARS-CoV-2_400_45_LEFT_0,ACTTCTTTAAGTTTAGAATAGACGGTGAC +SARS-CoV-2_400_45_RIGHT_0,TCTACAACAGGAACTCCACTACCT +SARS-CoV-2_400_46_LEFT_0,CTCAATGGTAACTGGTATGATTTCGGT +SARS-CoV-2_400_46_RIGHT_0,TGGAACACCATCAACAAATATTTTTCTCAC +SARS-CoV-2_400_47_LEFT_0,CTTTAATGTTTTATTCTCTACAGTGTTCCCAC +SARS-CoV-2_400_47_RIGHT_0,ATCCTGAGCAAAGAAGAAGTGTTTTAATTC +SARS-CoV-2_400_48_LEFT_0,GACTTTGCTGTGTCTAAGGGTTTCT +SARS-CoV-2_400_48_RIGHT_0,TTCTTTGCACTAATGGCATACTTAAGATTC +SARS-CoV-2_400_49_LEFT_0,CACTTTTCGCATATACAAAACGTAATGTC +SARS-CoV-2_400_49_RIGHT_0,CACTCATTAGCTAATCTATAGAAACGGTGT +SARS-CoV-2_400_50_LEFT_0,TCACTTGTTCTTGCTCGCAAACAT +SARS-CoV-2_400_50_RIGHT_0,TATTGAAACACACAACAGCATCGTCA +SARS-CoV-2_400_51_LEFT_0,CTTTGTGAATGAGTTTTACGCATATTTGC +SARS-CoV-2_400_51_RIGHT_0,AGGATGTTTAGTAAGTGGGTAAGCATCTATA +SARS-CoV-2_400_52_LEFT_2,TGGTACACTTATGATTGAACGGTTCG +SARS-CoV-2_400_52_RIGHT_2,GTGACATCACAACCTGGAGCATT +SARS-CoV-2_400_53_LEFT_0,GTGCTTGCATACGTAGACCATTCTTA +SARS-CoV-2_400_53_RIGHT_0,GACAGTTTAAATGTCTCCTCAGTAGCTTT +SARS-CoV-2_400_54_LEFT_1,TCAAGCTTTTTGCAGCAGAAACG +SARS-CoV-2_400_54_RIGHT_1,ACATTGCTAGAAAACTCATCTGAGATATT +SARS-CoV-2_400_55_LEFT_1,CAAGAGCACTATGTTAGAATTACTGGCTTATA +SARS-CoV-2_400_55_RIGHT_1,TTTCATCAAAGACAACTATATCTGCTGTC +SARS-CoV-2_400_56_LEFT_0,CACTATGTGAGAAGGCATTAAAATATTTGC +SARS-CoV-2_400_56_RIGHT_0,ACAATTTCAGCAGGACAACGCC +SARS-CoV-2_400_57_LEFT_0,GGCACACTAGAACCAGAATATTTCAATTC +SARS-CoV-2_400_57_RIGHT_0,AGTGGTTTGAGTGAATATGACATAGTC +SARS-CoV-2_400_58_LEFT_0,GGGACTACCAACTCAAACTGTTGATT +SARS-CoV-2_400_58_RIGHT_0,ATCATAGAGATGAGTCTTCTATAGGTCATGT +SARS-CoV-2_400_59_LEFT_0,GTGTTGACACTAAATTCAAAACTGAAGGTTTA +SARS-CoV-2_400_59_RIGHT_0,CGCACTACATTCCAAGGAAGTCC +SARS-CoV-2_400_60_LEFT_0,CGCCTGGAGATCAATTTAAACACC +SARS-CoV-2_400_60_RIGHT_0,GCATCACAACTAGCTACATGTGCAT +SARS-CoV-2_400_61_LEFT_0,GGTAACCTACAAAGCAACCATGATCT +SARS-CoV-2_400_61_RIGHT_0,TCCAAAATAGGCATACACCATCTGT +SARS-CoV-2_400_62_LEFT_2,TCTATGATGCACAGCCTTGTAGTGA +SARS-CoV-2_400_62_RIGHT_0,GTCTACAGACAGCACCACCTAAATTG +SARS-CoV-2_400_63_LEFT_0,AGTGTCAGATATAGATTATGTACCACTAAAGTCT +SARS-CoV-2_400_63_RIGHT_0,GCCCAAAGCTCAAATGCTACATTAAC +SARS-CoV-2_400_64_LEFT_0,AGTTGATGGTGTTGATGTAGAATTGTTTG +SARS-CoV-2_400_64_RIGHT_0,TAATGTGACTCCATTAAGACTAGCTTGTTT +SARS-CoV-2_400_65_LEFT_0,GCCCGTAATGGTGTTCTTATTACAGA +SARS-CoV-2_400_65_RIGHT_0,GTACTGTCCATAGGAATAAAATCTTCTAATTCA +SARS-CoV-2_400_66_LEFT_0,GGTTTACATCTACTGATTGGACTAGCTAAA +SARS-CoV-2_400_66_RIGHT_0,CACTATCACCATAATTTTGAAGGTCACAC +SARS-CoV-2_400_67_LEFT_1,AATTACAATCTAGTCAAGCGTGGCAA +SARS-CoV-2_400_67_RIGHT_1,CATATCACTAATAATGAGATCCCATTTATTAGC +SARS-CoV-2_400_68_LEFT_0,ATTGGTGATTGTGCAACTGTACATACA +SARS-CoV-2_400_68_RIGHT_0,ACTCATGTCAAATAAAGAATAGGAAGACAA +SARS-CoV-2_400_69_LEFT_0,ATGTCATGCATGCAAATTACATATTTTGGA +SARS-CoV-2_400_69_RIGHT_0,AGTCCTGAGTTGAATGTAAAACTGAG +SARS-CoV-2_400_70_LEFT_0,TTTATTGCCACTAGTCTCTAGTCAGTGT +SARS-CoV-2_400_70_RIGHT_0,AATTCACAGACTTTAATAACAACATTAGTAGCG +SARS-CoV-2_400_71_LEFT_0,GAGGCTGGATTTTTGGTACTACTTTAGA +SARS-CoV-2_400_71_RIGHT_0,ATGTTAATACCTATTGGCAAATCTACCA +SARS-CoV-2_400_72_LEFT_0,GGTTATTTTAAAATATATTCTAAGCACACGCCT +SARS-CoV-2_400_72_RIGHT_0,ATCTAACAATAGATTCTGTTGGTTGGACTC +SARS-CoV-2_400_73_LEFT_0,CGTTGAAATCCTTCACTGTAGAAAAAGG +SARS-CoV-2_400_73_RIGHT_0,GCTATAACGCAGCCTGTAAAATCATCT +SARS-CoV-2_400_74_LEFT_0,ATGTCTATGCAGATTCATTTGTAATTAGAGGT +SARS-CoV-2_400_74_RIGHT_0,TCCACAAACAGTTGCTGGTGC +SARS-CoV-2_400_75_LEFT_1,AACCATACAGAGTAGTAGTACTTTCTTTTGA +SARS-CoV-2_400_75_RIGHT_1,CCTGTAGAATAAACACGCCAAGTAGG +SARS-CoV-2_400_76_LEFT_0,CAAAAAGTTTCTGCCTTTCCAACAATTTG +SARS-CoV-2_400_76_RIGHT_0,TGACTAGCTACACTACGTGCCC +SARS-CoV-2_400_77_LEFT_0,GCAGGTATATGCGCTAGTTATCAGAC +SARS-CoV-2_400_77_RIGHT_0,AATTGGTGGTGTTTTGTAAATTTGTTTGAC +SARS-CoV-2_400_78_LEFT_0,GCAATATGGCAGTTTTTGTACACAATTAAA +SARS-CoV-2_400_78_RIGHT_0,GCACCAAAGGTCCAACCAGAAG +SARS-CoV-2_400_79_LEFT_0,GATGAAATGATTGCTCAATACACTTCTGC +SARS-CoV-2_400_79_RIGHT_0,TGCCTGTGATCAACCTATCAATTTG +SARS-CoV-2_400_80_LEFT_0,ACGCTTGTTAAACAACTTAGCTCCAA +SARS-CoV-2_400_80_RIGHT_0,CTTCACGAGGAAAGTGTGCTTTTC +SARS-CoV-2_400_81_LEFT_0,CATGTGACTTATGTCCCTGCACA +SARS-CoV-2_400_81_RIGHT_0,GAGATTCATTTAAATTCTTGGCAACCTCATT +SARS-CoV-2_400_82_LEFT_0,TGATTTAGGTGACATCTCTGGCATTAATG +SARS-CoV-2_400_82_RIGHT_0,GATTTCACCTTGCTTCAAAGTTACAGTTC +SARS-CoV-2_400_83_LEFT_0,CATTACACATAAACGAACTTATGGATTTGT +SARS-CoV-2_400_83_RIGHT_0,GCCAAAGCCTCATTATTATTCTTACAAAGTTTA +SARS-CoV-2_400_84_LEFT_2,GTAACAGTTTACTCACACCTTTTGCTC +SARS-CoV-2_400_84_RIGHT_2,TGTTCAACACCARTGTCTGTACTC +SARS-CoV-2_400_85_LEFT_0,TCACTTCAGACTATTACCAGCTGTACTC +SARS-CoV-2_400_85_RIGHT_0,AAAGAAGGTTTTACAAGACTCACGTTAAC +SARS-CoV-2_400_86_LEFT_0,CATCCTTACTGCGCTTCGATTGT +SARS-CoV-2_400_86_RIGHT_0,ACCGGTGATCCAATTTATTCTGTAAA +SARS-CoV-2_400_87_LEFT_1,AGGTTTCCTATTCCTTACATGGATTTGT +SARS-CoV-2_400_87_RIGHT_1,CCTTGATGTCACAGCGTCCT +SARS-CoV-2_400_88_LEFT_2,GTGGACATCTTCGTATTGCTGGA +SARS-CoV-2_400_88_RIGHT_2,CCATTGGTTGCTCTTCATCTAATTGAG +SARS-CoV-2_400_89_LEFT_2,GATGTTTCATCTCGTTGACTTTCAGG +SARS-CoV-2_400_89_RIGHT_0,GCCGTCAGGACAAGCAAAAG +SARS-CoV-2_400_90_LEFT_0,TCATCCTCTAGCTGATAACAAATTTGCA +SARS-CoV-2_400_90_RIGHT_0,TTCTTGGTGAAATGCAGCTACAG +SARS-CoV-2_400_91_LEFT_0,TATCTTTTGGTTCTCACTTGAACTGCAA +SARS-CoV-2_400_91_RIGHT_0,CGAACGTCATGATACTCTAAAAAGTCTT +SARS-CoV-2_400_92_LEFT_0,TTCCTGTTTACCTTTTACAATTAATTGCCAG +SARS-CoV-2_400_92_RIGHT_0,CTTCGGTAGTAGCCAATTTGGTCATC +SARS-CoV-2_400_93_LEFT_0,TCGAGGACAAGGCGTTCCAA +SARS-CoV-2_400_93_RIGHT_0,GCCTGGAGTTGAATTTCTTGAACT +SARS-CoV-2_400_94_LEFT_0,GCAGTCAAGCCTCTTCTCGTT +SARS-CoV-2_400_94_RIGHT_0,CATTCCGAAGAACGCTGAAGC +SARS-CoV-2_400_95_LEFT_0,CTGATTACAAACATTGGCCGCAAA +SARS-CoV-2_400_95_RIGHT_0,ATCTGCCTTGTGTGGTCTGCA +SARS-CoV-2_400_96_LEFT_1,CTGCAGATTTGGATGATTTCTCCA +SARS-CoV-2_400_96_RIGHT_0,TTTGTCATTCTCCTAAGAAGCTATTAAAATCAC diff --git a/working/vpipe-hugemem.bsub b/working/vpipe-hugemem.bsub deleted file mode 100644 index c908699..0000000 --- a/working/vpipe-hugemem.bsub +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -#BSUB -L /bin/bash -#BSUB -J COVID-vpipe-hugemem -#BSUB -u "ivan.topolsky@bsse.ethz.ch" -#BSUB -N -#BSUB -M 16384 -#BSUB -R rusage[mem=16384] -#BSUB -R light -##### -q light.5d -#BSUB -W 96:0 - -umask 0007 - -# unlock if crashed before finishing -sleep 300 -./vpipe --configfile config.unlock.yaml ---dryrun --unlock - -mkdir -p cluster_logs/ -export SNAKEMAKE_PROFILE=$(realpath ../profiles/) -exec ./vpipe --profile ${SNAKEMAKE_PROFILE}/custom-lsf/ --configfile config.hugemem.yaml diff --git a/working/vpipe-hugemem.sbatch b/working/vpipe-hugemem.sbatch index a2ae1d2..7993cdf 100644 --- a/working/vpipe-hugemem.sbatch +++ b/working/vpipe-hugemem.sbatch @@ -1,9 +1,9 @@ #!/bin/bash -SBATCH --job-name="COVID-vpipe-hugemem" +#SBATCH --job-name="COVID-vpipe-hugemem" #SBATCH --mail-user="ivan.topolsky@bsse.ethz.ch" #SBATCH --mail-type=END #SBATCH --mem-per-cpu=16384 -####### --partition=light +#SBATCH --oversubscribe #SBATCH --time=96:00:00 umask 0007 diff --git a/working/vpipe-no-shorah.bsub b/working/vpipe-no-shorah.bsub deleted file mode 100644 index 5e1ffc3..0000000 --- a/working/vpipe-no-shorah.bsub +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash -#BSUB -L /bin/bash -#BSUB -J COVID-vpipe-cons -#BSUB -u "ivan.topolsky@bsse.ethz.ch carrara@nexus.ethz.ch shuqing.yu@nexus.ethz.ch schmittu@ethz.ch" -#BSUB -N -#BSUB -M 16384 -#BSUB -R rusage[mem=16384] -#BSUB -R light -##### -q light.5d -#BSUB -W 23:0 - -### batches @TAG@ - - -logfile=/cluster/scratch/bs-pangolin/log.$LSB_JOBID - -# tee all output from this script into logfile -exec > >(tee -a $logfile) 2>&1 - -function sendmail { - # email proxy at bs-stadler04 since the smtp0 server is not reachable from euler directly: - echo "see $logfile and $LSB_OUTPUTFILE" | ssh bs-pangolin@d@bs-stadler04.ethz.ch ./sendmail.sh \ - ivan.topolsky@bsse.ethz.ch carrara@nexus.ethz.ch shuqing.yu@nexus.ethz.ch \ - schmittu@ethz.ch \ - -s "VPIPE FAILED: please check $LSB_OUTPUTFILE" -} - -trap sendmail ERR - -set -e - -umask 0007 - -# unlock if crashed before finishing -./vpipe --configfile config.unlock.yaml --dryrun --unlock || exit 1 -#--cleanup-shadow - -# update the shared snake envs (with proxy when not on a login node) -#./create_envs -p -# proxy so update can work -echo "Using proxy" -. /etc/profile.d/software_stack_default.sh -module load eth_proxy -echo "${https_proxy}" - -mkdir -p cluster_logs/ -export SNAKEMAKE_PROFILE=$(realpath ../profiles/custom-lsf/) -./vpipe --profile ${SNAKEMAKE_PROFILE} --configfile config.no-shorah.yaml --omit-from snv diff --git a/working/vpipe-no-shorah.sbatch b/working/vpipe-no-shorah.sbatch index c50b091..5b7568e 100644 --- a/working/vpipe-no-shorah.sbatch +++ b/working/vpipe-no-shorah.sbatch @@ -3,10 +3,9 @@ #SBATCH --mail-user="ivan.topolsky@bsse.ethz.ch carrara@nexus.ethz.ch shuqing.yu@nexus.ethz.ch schmittu@ethz.ch" #SBATCH --mail-type=END #SBATCH --mem-per-cpu=16384 -####### --partition=light +#SBATCH --oversubscribe #SBATCH --time=23:00:00 - -### batches @TAG@ +#SBATCH --comment="batches @TAG@" logfile=/cluster/scratch/bs-pangolin/log.$LSB_JOBID @@ -42,4 +41,5 @@ echo "${https_proxy}" mkdir -p cluster_logs/ export SNAKEMAKE_PROFILE=$(realpath ../profiles/smk-simple-slurm/ ) -./vpipe --profile ${SNAKEMAKE_PROFILE} --configfile config.no-shorah.yaml --omit-from snv +./vpipe --profile ${SNAKEMAKE_PROFILE} \ + --configfile config.no-shorah.yaml --omit-from snv diff --git a/working/vpipe.bsub b/working/vpipe.bsub deleted file mode 100644 index bce0143..0000000 --- a/working/vpipe.bsub +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -#BSUB -L /bin/bash -#BSUB -J COVID-vpipe -#BSUB -u "ivan.topolsky@bsse.ethz.ch" -#BSUB -N -#BSUB -M 16384 -#BSUB -R rusage[mem=16384] -#BSUB -R light -##### -q light.5d -#BSUB -W 23:0 - -umask 0007 - -mkdir -p cluster_logs/ -export SNAKEMAKE_PROFILE=$(realpath ../profiles/) -exec ./vpipe --profile ${SNAKEMAKE_PROFILE}/custom-lsf/ --configfile config.snv.yaml diff --git a/working/vpipe.config b/working/vpipe.config index 9fa5ed3..4583d7d 100644 --- a/working/vpipe.config +++ b/working/vpipe.config @@ -33,6 +33,7 @@ upload = True [preprocessing] mem=4096 +time=300 # # Group: align @@ -57,6 +58,9 @@ threads=6 leave_msa_temp = true +[primerstrim] +mem=3072 + [consensus_sequences] #conda=/cluster/project/pangolin/test/smallgenomeutilities.yaml @@ -66,6 +70,7 @@ leave_msa_temp = true mem=8192 [basecounts] +time=240 #conda=/cluster/project/pangolin/test/smallgenomeutilities.yaml [coverage] diff --git a/working/vpipe.sbatch b/working/vpipe.sbatch index 4e3f1b1..89762ec 100644 --- a/working/vpipe.sbatch +++ b/working/vpipe.sbatch @@ -3,11 +3,14 @@ #SBATCH --mail-user="ivan.topolsky@bsse.ethz.ch" #SBATCH --mail-type=END #SBATCH --mem-per-cpu=16384 -####### --partition=light +#SBATCH --oversubscribe #SBATCH --time=23:00:00 umask 0007 mkdir -p cluster_logs/ -export SNAKEMAKE_PROFILE=$(realpath ../profiles/) -exec ./vpipe --profile ${SNAKEMAKE_PROFILE}/custom-lsf/ --configfile config.snv.yaml +# export SNAKEMAKE_PROFILE=$(realpath ../profiles/custom-lsf/) +# exec ./vpipe --profile ${SNAKEMAKE_PROFILE} \ +export SNAKEMAKE_PROFILE=$(realpath ../profiles/smk-simple-slurm/ ) +exec ./vpipe --profile ${SNAKEMAKE_PROFILE} \ + --configfile config.snv.yaml