Merge pull request #15 from c3g/devel

Devel

.dockerignore

@@ -0,0 +1 @@
+singularity/

.gitignore

@@ -1,2 +1,3 @@
 cvmfschecksum.*
 *_mount_cache.sh
+singularity/wrapper_genpipes/images

Dockerfile

@@ -8,23 +8,12 @@ WORKDIR /tmp
 ENV CCTOOLS_VERSION 7.0.16
 ENV CVMFS_VERSION latest
 ENV MODULE_VERSION 4.1.2
-RUN yum update -y && yum install -y  \
-  https://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-${CVMFS_VERSION}.noarch.rpm \
-  && yum install -y cvmfs.x86_64 wget unzip.x86_64 make.x86_64 gcc expectk dejagnu less tcl-devel.x86_64 \
+RUN yum update -y \
+  && yum install -y wget unzip.x86_64 make.x86_64 gcc expectk dejagnu less tcl-devel.x86_64 \
   && yum clean all
+RUN yum install -y  fuse
 
-# parrot
-#RUN wget https://ccl.cse.nd.edu/software/autobuild/commit/5b94c693/cctools-5b94c693-x86_64-redhat7.tar.gz && tar xvf cctools-5b94c693-x86_64-redhat7.tar.gz && mv cctools-5b94c693-x86_64-redhat7 /opt/cctools &&  rm cctools-5b94c693-x86_64-redhat7.tar.gz
-RUN wget http://ccl.cse.nd.edu/software/files/cctools-${CCTOOLS_VERSION}-x86_64-centos7.tar.gz \
-  && tar xvf cctools-${CCTOOLS_VERSION}-x86_64-centos7.tar.gz && mv cctools-${CCTOOLS_VERSION}-x86_64-centos7 \
-  /opt/cctools && rm cctools-${CCTOOLS_VERSION}-x86_64-centos7.tar.gz
-
-RUN mkdir /etc/parrot
-ADD keys /etc/parrot/keys
-RUN chmod 4755 /bin/ping
-# adding local config to containe. These will overwrite the cvmfs-config.computecanada ones
-ADD config.d /etc/parrot/config.d
-RUN mkdir /cvmfs-cache && chmod 777 /cvmfs-cache
+RUN mkdir /cvmfs-cache  /cvmfs  && chmod 777 /cvmfs-cache  /cvmfs
 
 # module
 RUN wget https://github.com/cea-hpc/modules/releases/download/v${MODULE_VERSION}/modules-${MODULE_VERSION}.tar.gz

Dockerfile.premount

@@ -0,0 +1,36 @@
+FROM centos:7.6.1810
+MAINTAINER P-O Quirion [email protected]
+
+WORKDIR /tmp
+
+# All yum cmd
+
+ENV CCTOOLS_VERSION 7.0.16
+ENV CVMFS_VERSION latest
+ENV MODULE_VERSION 4.1.2
+RUN yum update -y \
+  && yum install -y wget unzip.x86_64 make.x86_64 gcc expectk dejagnu less tcl-devel.x86_64 \
+  && yum clean all
+
+RUN mkdir /cvmfs-cache  /cvmfs  && chmod 777 /cvmfs-cache  /cvmfs
+
+# module
+RUN wget https://github.com/cea-hpc/modules/releases/download/v${MODULE_VERSION}/modules-${MODULE_VERSION}.tar.gz
+RUN tar xzf modules-${MODULE_VERSION}.tar.gz && \
+    rm modules-${MODULE_VERSION}.tar.gz \
+    && cd  modules-${MODULE_VERSION}  && ./configure && make -j 7  && make install \
+    && cd .. && rm -rf modules-${MODULE_VERSION} && rm -rf /usr/local/Modules/modulefiles/*
+RUN ["ln", "-s", "/usr/local/Modules/init/profile.sh", "/etc/profile.d/z00_module.sh"]
+#RUN echo "source /etc/profile.d/z00_module.sh" >>  /etc/bashrc
+ADD devmodule/genpipes "/usr/local/Modules/modulefiles/."
+
+#RUN echo "source /etc/profile.d/z90_genpipes.sh" >>  /etc/bashrc
+RUN ["ln", "-s", "/usr/local/etc/genpiperc", "/etc/profile.d/z90_genpipes.sh"]
+
+ADD genpiperc    /usr/local/etc/genpiperc
+ADD init_genpipes /usr/local/bin/init_genpipes
+RUN chmod 755 /usr/local/bin/init_genpipes
+
+ENTRYPOINT ["init_genpipes"]
+# docker build --tag c3genomics/genpipes:beta .
+

README.md

@@ -1,58 +1,75 @@
 # Genpipes container
 
-A container to create and run Genpipes pipelines anywhere. Only user priviledge are required.
+You can use Genpipes in a Container (GiaC) to run Genpipes on a single machine, on a a Torque/PBS cluster or on a SLURM cluster.
 
-The [Genpipes](https://bitbucket.org/mugqic/genpipes/src/master/README.md) tools are design to run on Super Computers ([documentation here](http://www.computationalgenomics.ca/tutorials/)), however, you can generate generate the pipelines scripts and run smaller experiment on a server with container technology. Containers can also be used to debug and develop [Genpipes](https://bitbucket.org/mugqic/genpipes/src/master/README.md) on you machine.
+If FUSE and [Singularity](https://singularity.lbl.gov/index.html) is installed on your machine you are all set, a simple user with no special privilege is enough.
 
-## Install a compatible container technology on your machine (Linux, Mac, or Windows)
+While you can use (GiaC) to debug and develop Genpipes on you machine on you machine, [Genpipes](https://bitbucket.org/mugqic/genpipes/src/master/README.md) is design to run analysis on Super Computers.  
 
- - [Docker](https://docs.docker.com/install/)
- - [Singularity](https://singularity.lbl.gov/index.html)
 
-Then run with docker
-```
-#!bash
-TAG=latest
-docker run --privileged -v /tmp:/tmp --network host -it -w $PWD -v $HOME:$HOME --user $UID:$GROUPS -v /etc/group:/etc/group  -v /etc/passwd:/etc/passwd  c3genomics/genpipes:$TAG
-```
-
-or singularity
-```
-#!bash
-TAG=latest
-singularity run docker://c3genomics/genpipes:$TAG
-```
+## Install a compatible container technology on your machine
+Follow installation procedure from te  [Singularity install page](https://sylabs.io/guides/3.5/admin-guide/installation.html)
 
 
 You can also [Read the genpipes documentation](https://bitbucket.org/mugqic/genpipes)
 
-Here is [the project's Docker hub page](https://hub.docker.com/r/c3genomics/genpipes/)
-
 ## What exactly is avalable in that container?
 
 The full tested and integrated C3G/MUGQIC software stack.
 [For more details: http://www.computationalgenomics.ca/cvmfs-modules/](http://www.computationalgenomics.ca/cvmfs-modules/)
 
 
-## Setup a dev enviroment
+## Setup a GiaC environment
 
 You can use this container to develop and test new version of GenPipes.
 
-Fist, clone genpipe somewhere under your $HOME folder three.
+Fist, clone genpipes somewhere under your $HOME folder three. Then get the container wrapper:
 
 ```
-git clone https://bitbucket.org/mugqic/genpipes $HOME/some/dir/genpipes
-```
-Add the followin line to your .bashrc
+git clone https://bitbucket.org/mugqic/genpipes $WORKDIR/genpipes
+
+$WORKDIR/genpipes/resources/container/get_wrapper.sh
 
-```
-export GENPIPES_DEV_DIR=$HOME/some/dir/genpipes
 ```
 
-Start the container with the normal procedure seen above. In the running container, execute the followin command:
+You can now configuere the `$WORKDIR/genpipes/resources/container/etc/wrapper.conf` file:
 
 ```
- module load dev_genpipes
+# GEN_SHARED_CVMFS should have a sufficient amount of space to load full reference files
+export GEN_SHARED_CVMFS=$HOME/cvmfs
+TMPDIR=/tmp
+export UMOUNT_RETRY=5
+BIND_LIST=
 ```
 
-Voilà, now GenPipes uses whatever commmit of branch that has been checked out in $HOME/some/dir/genpipes
+`GEN_SHARED_CVMFS` will hold a cache for GiaC [CVMFS](https://cernvm.cern.ch/portal/filesystem) system, it will hold the genomes and software being used by Genpipes. This folder will grow with Genpipes usage. You can delete in in between usage, but keep in mind that once deleted it will need to be rebuild by downloading data form the internet.
+
+`BIND_LIST` is a list of file system, separated by comma, you need Genpipes to have access to, by default, only your $HOME is mounted. For example if you are on an HPC system with a `/scratch` and `/data` space, you would have `BIND_LIST=/scratch,/data`. The string will be fed to Singularity `--bind` option, see `singularity --help` for more details.
+
+`TMPDIR` place to store GiaC temp files. These are a mainly metadata of what will be stored in the `GEN_SHARED_CVMFS` folder. The volume of data in this folder will be small.
+
+`UMOUNT_RETRY` GiaC will mount a [CVMFS](https://cernvm.cern.ch/portal/filesystem) in the `TMPDIR` folder, once GiaC is done, the folder is unmounted up to `UMOUNT_RETRY` time, usually two try does the trick, but 5 is safer.  
+
+
+You do not need any other setup on your machine.
+
+## PIPELINE USAGE
+
+# On SLURM HPC
+
+Created an ini file that fits your system and for the pipeline(s) you want to use.
+
+
+add the `--wrap` option when running the pipelines
+
+# On PBS/torque flavored HPC
+
+Created an ini file that fits your system and for the pipeline(s) you want to use.
+
+add the `--wrap` option when running the pipelines
+
+
+# On a single machine.
+
+
+Just run the pipeline with the `--wrap` and `-j batch` and `--no-json` options!

genpiperc

@@ -1,10 +1,15 @@
 #source /etc/bashrc
 #source ~/.bashrc
-if [ -z "${QUIET}" ]; then
+if [ -z "${QUIET+x}" ]; then
 echo -e "\nWait while Genpipes module are loaded. This could take a while,"
 echo -e   "  especially if the cvmfs cache is new\n"
 fi
 module use $MUGQIC_INSTALL_HOME/modulefiles
 module load mugqic/python/2.7.14
-module load mugqic/genpipes${PIPELINE_VERSION}
 
+if [ -z "${GENPIPES_DEV_DIR+x}" ]; then
+    module load mugqic/genpipes${PIPELINE_VERSION}
+else
+    module use /usr/local/Modules/modulefiles
+    module load dev_genpipes
+fi

init_genpipes

@@ -5,46 +5,24 @@
 # http://cvmfs.readthedocs.io/en/stable/cpt-configure.html#parrot-connector-to-cernvm-fs
 # It also loads the lmod software to load mugqic modules 
 
-export PARROT_ALLOW_SWITCHING_CVMFS_REPOSITORIES=yes
-export HTTP_PROXY='DIRECT'
 
-LOCAL_CONFIG_PATH=/etc/parrot
-PARROT_CVMFS_ALIEN_CACHE=/tmp/parrot.${UID}
 export MUGQIC_INSTALL_HOME=/cvmfs/soft.mugqic/CentOS6
 
 
 usage (){
-  echo -e "\nUsage: $0 [-c <PATH>] [-a <PATH>] [-p <PATH> ] [ -d <PATH> ] [-V <X.X.X> ] [ <cmd> ] " 1>&2;
+  echo -e "\nUsage: $0  [ -d <PATH> ] [-V <X.X.X> ] [ <cmd> ] " 1>&2;
   echo -e "\nOPTION"
-  echo -e "\t-a  Set the path of the cache use to store cvmfs data"
-  echo -e "\t      default: ${PARROT_CVMFS_ALIEN_CACHE}"
-  echo -e "\t-c  Set the path for local cvmfs repo config"
-  echo -e "\t      default: ${LOCAL_CONFIG_PATH}"
-  echo -e "\t-p  Set the path for the mugqic software stack"
-  echo -e "\t      default: ${MUGQIC_INSTALL_HOME}"
   echo -e "\t-d  Set a path to a genpipes repo that can supersedes"
   echo -e "\t      the soft.mugqic repo's version (developer's mode)"
   echo -e "\t-V    Genpipes version (will default to latest release)"
-  echo -e "\t-e  Execute specific command and exit"
   echo -e "\t-v  vebose mode, makes the parrot talk"
   echo
 }
 
 export QUIET="-d clear"
 
-while getopts ":a:d:c:p:vV:" opt; do
+while getopts ":d:vV:h" opt; do
   case $opt in
-    a)
-      echo "Setting parrot alien cache to $OPTARG"
-      export PARROT_CVMFS_ALIEN_CACHE=${OPTARG}
-      ;;
-    c)
-      echo "Using local cvmfs config path $OPTARG"
-      export LOCAL_CONFIG_PATH=${OPTARG}
-      ;;
-    p)
-      MUGQIC_INSTALL_HOME=${OPTARG}
-      ;;
     v)
       unset QUIET
       ;;
@@ -72,55 +50,24 @@ done
 
 
 shift $((OPTIND-1))
-# move the execline to a script
+# move the exec line to a script
 if [ $# -gt 0 ] ; then
   function finish {
     rm ${genpipe_script}
   }
   genpipe_script=$(mktemp /tmp/genpipe_script.XXXXXX)
   trap finish EXIT
   chmod 755 ${genpipe_script}
-  echo $@ > ${genpipe_script}
+  echo '#! /bin/bash' >> ${genpipe_script}
+  echo "$@" >> ${genpipe_script}
 fi
 
-CONFIG_PATH=/etc/parrot/config.d
-KEY_PATH=/etc/parrot/keys
-
-cvmfs_to_parrot() {
-  repo_name=$1 
-  # Get the CC config 
-  source ${CONFIG_PATH}/${repo_name}.conf
-  # Load the local config
-  source ${LOCAL_CONFIG_PATH}/${repo_name}.conf 2> /dev/null
-  local ref_url=${CVMFS_SERVER_URL=}
-  local ref_url=${CVMFS_SERVER_URL//@fqrn@/$repo_name}
-  local ref_url=${CVMFS_SERVER_URL//@fqrn@/$repo_name}
-
-  local ref_key=${CVMFS_KEYS_DIR:-$KEY_PATH}
-  local ref_key=$(echo "${ref_key}" | sed 's|^/cvmfs|/tmp|')
-
-  echo $ref_url $ref_key
-
-}
-
-# Reading the statum connection, 
-# could be loccally overwritten to optimise connections
-read ref_url ref_key < <(cvmfs_to_parrot  ref.mugqic)
-read soft_url soft_key < <(cvmfs_to_parrot  soft.mugqic)
-
-# The <default-repositories> option is important when cvmfs repo are 
-# already present in the cvmfs directory. Parrot will not try to remount
-# the fs in that case.
-export PARROT_CVMFS_REPO="<default-repositories> \
-	${CVMFS_CONFIG_CC} \
-	soft.mugqic:url=$soft_url,pubkey=$KEY_PATH/soft.mugqic.pub,try_local_filesystem \
-	ref.mugqic:url=$ref_url,pubkey=$KEY_PATH/ref.mugqic.pub,try_local_filesystem"
 
 
 # load cvmfs
 if [  ${genpipe_script}  ]; then
-  /opt/cctools/bin/parrot_run $QUIET --no-set-foreground bash  -lc ${genpipe_script}
+  bash -ilc "${genpipe_script}"
 else
-   /opt/cctools/bin/parrot_run $QUIET bash -il
+  bash -il
 fi
 

singularity/wrapper_genpipes/bin/container_wrapper.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+
+
+SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
+SCRIPTPATH=${SCRIPTPATH%%bin}
+SCRIPTPATH=${SCRIPTPATH%%/}
+
+source ${SCRIPTPATH}/etc/wrapper.conf
+TMPDIR=${TMPDIR:-/tmp}
+
+export GEN_LOCAL_CVMFS=$(mktemp -d $TMPDIR/cvmfs_XXXXX)
+
+GEN_REF=${GEN_LOCAL_CVMFS}/mnt/ref.mugqic
+GEN_SOFT=${GEN_LOCAL_CVMFS}/mnt/soft.mugqic
+
+mkdir -p ${GEN_SOFT}
+mkdir -p ${GEN_REF}
+
+cleaup () {
+#  echo unmounting ${GEN_LOCAL_CVMFS}/mnt
+
+  try=0
+  while  true ; do
+    fusermount -u ${GEN_REF} 2>/dev/null
+    ret=$?
+    if [[ ${ret} -eq 0 || ${try} -ge  ${UMOUNT_RETRY} ]]; then
+      break
+    fi
+    try=$(($try+1))
+    sleep 1
+  done
+
+  while  true ; do
+    fusermount -u ${GEN_SOFT} 2>/dev/null
+    ret=$?
+    if [[ ${ret} -eq 0 || ${try} -ge  ${UMOUNT_RETRY} ]]; then
+      break
+    fi
+    try=$(($try+1))
+    sleep 1
+  done
+
+  rm -r ${GEN_LOCAL_CVMFS}  2>/dev/null
+}
+
+trap cleaup EXIT
+
+export CVMFS_KEYS_DIR=${SCRIPTPATH}/etc/keys/gen
+export CVMFS_ALIEN_CACHE=${GEN_SHARED_CVMFS}/alien_cache/soft.mugqic
+mkdir -p $CVMFS_ALIEN_CACHE
+chmod 755 ${GEN_SHARED_CVMFS}/alien_cache
+${SCRIPTPATH}/lib64/ld-linux-x86-64.so.2  --library-path ${SCRIPTPATH}/lib64  \
+${SCRIPTPATH}/bin/cvmfs2 -o config=${SCRIPTPATH}/etc/genconfig.conf soft.mugqic \
+${GEN_SOFT}  >/dev/null
+
+export CVMFS_ALIEN_CACHE=${GEN_SHARED_CVMFS}/alien_cache/ref.mugqic
+mkdir -p $CVMFS_ALIEN_CACHE
+${SCRIPTPATH}/lib64/ld-linux-x86-64.so.2  --library-path ${SCRIPTPATH}/lib64  \
+${SCRIPTPATH}/bin/cvmfs2 -o config=${SCRIPTPATH}/etc/genconfig.conf ref.mugqic \
+${GEN_REF} > /dev/null
+
+
+if [ -z ${BIND_LIST+x} ]; then
+  singularity run --cleanenv -B ${GEN_LOCAL_CVMFS}/mnt:/cvmfs  \
+  ${SCRIPTPATH}/images/genpipes.sif \
+  "$@"
+else
+  singularity run --cleanenv -B ${GEN_LOCAL_CVMFS}/mnt:/cvmfs  -B "$BIND_LIST" \
+  ${SCRIPTPATH}/images/genpipes.sif \
+  "$@"
+
+fi
+
+

singularity/wrapper_genpipes/etc/genconfig.conf

@@ -0,0 +1,10 @@
+CVMFS_CACHE_BASE=${GEN_LOCAL_CVMFS}
+CVMFS_RELOAD_SOCKETS=${GEN_LOCAL_CVMFS}
+CVMFS_ALIEN_CACHE=${CVMFS_ALIEN_CACHE}
+CVMFS_QUOTA_LIMIT=-1
+CVMFS_SHARED_CACHE=no
+CVMFS_USYSLOG=${GEN_LOCAL_CVMFS}/cvmfs.log
+CVMFS_CLAIM_OWNERSHIP=yes
+CVMFS_HTTP_PROXY=DIRECT
+CVMFS_SERVER_URL="http://cvmfs-1-west-c3g.computecanada.ca:8000/cvmfs/@fqrn@;http://cvmfs-s0-genomic.vhost38.genap.ca/cvmfs/@fqrn@"
+CVMFS_KEYS_DIR=${CVMFS_KEYS_DIR}