From a692afccdc96dc8d15ddc47fd27c71cc059a158d Mon Sep 17 00:00:00 2001
From: Jean-Philippe Lenain <jlenain@users.noreply.github.com>
Date: Fri, 2 Feb 2024 17:53:28 +0100
Subject: [PATCH] Scripts for DQM (#105)

* Update user script for DQM job submission

* Work on user script for DQM job submission

* Give up on COMDIRAC initialization: commands not found on DIRAC workers

* The environment variable NECTARCAMDATA also needs to be passed into the container at runtime

* Bug in handling $NECTARCAMDATA/runs

* I was deleting $PWD on exit, which prevented any output sandbox to be retrievable !

* I forgot to specify the destination SE in dirac-dms-add-file (:facepalm:)

* Update destination SE on DIRAC to GRIF instead of LPNHE

* Use canonical quoting (SC2086)

* Changed destination SE

* Even if singularity binary available, some jobs tell that:
INFO:    Environment variable SINGULARITYENV_MPLCONFIGDIR is set, but APPTAINERENV_MPLCONFIGDIR is preferred

* Generate plots on DIRAC

* Add some doc.

---------

Co-authored-by: Jean-Philippe Lenain <jlenain@in2p3.fr>
---
 .../dqm_job_submitter/dqm_processor.sh        | 37 +++++++++----------
 .../dqm_job_submitter/submit_dqm_processor.py | 14 ++++---
 .../jlenain/submitGainFitDIRAC/GainFitter.sh  |  2 +-
 3 files changed, 27 insertions(+), 26 deletions(-)

diff --git a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
index ba0d915e..ae2cf3a6 100755
--- a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
+++ b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
@@ -5,7 +5,7 @@
 
 function usage ()
 {
-    echo "Usage: $(basename $0) -r <run number>"
+    echo "Usage: $(basename "$0") -r <run number>"
 }
 
 function help ()
@@ -36,7 +36,7 @@ while getopts ":hr:" option; do
 done
 shift $((OPTIND-1))
 
-if [ -z $runnb ]; then
+if [ -z "$runnb" ]; then
     usage
     exit 1
 fi
@@ -55,18 +55,18 @@ function exit_script() {
     # Some cleanup before leaving:
     # [ -d $CONTAINER ] && rm -rf $CONTAINER
     # [ -f $CONTAINER ] && rm -f $CONTAINER
-    [ -d $NECTARCAMDATA ] && rm -rf $NECTARCAMDATA
-    [ -d $OUTDIR ] && rm -rf $OUTDIR
-    [ -f ${OUTDIR}.tar.gz ] && rm -f ${OUTDIR}.tar.gz
-    [ -d ${OUTDIR} ] && rm -rf ${OUTDIR}
+    [ -d "$NECTARCAMDATA/runs" ] && rm -rf "$NECTARCAMDATA/runs"
+    [ -d "$OUTDIR" ] && rm -rf "$OUTDIR"
+    [ -f "${OUTDIR}.tar.gz" ] && rm -f "${OUTDIR}.tar.gz"
+    [ -d "${OUTDIR}" ] && rm -rf "${OUTDIR}"
     [ -f $WRAPPER ] && rm -f $WRAPPER
 
-    exit $return_code
+    exit "$return_code"
 }
 
-export NECTARCAMDATA=$PWD/runs
-[ ! -d $NECTARCAMDATA ] && mkdir -p $NECTARCAMDATA || exit_script $?
-mv nectarcam*.sqlite NectarCAM.Run*.fits.fz $NECTARCAMDATA/.
+export NECTARCAMDATA=$PWD
+[ ! -d "$NECTARCAMDATA/runs" ] && mkdir -p "$NECTARCAMDATA/runs" || exit_script $?
+mv nectarcam*.sqlite NectarCAM.Run*.fits.fz "$NECTARCAMDATA/runs/."
 
 # Halim's DQM code needs to use a specific output directory:
 export NECTARDIR=$PWD/$OUTDIR
@@ -79,22 +79,19 @@ export NECTARDIR=$PWD/$OUTDIR
 # Create a wrapper BASH script with cleaned environment, see https://redmine.cta-observatory.org/issues/51483
 cat > $WRAPPER <<EOF
 #!/bin/env bash
+# See https://redmine.cta-observatory.org/issues/51483: start container from a clean environment
 echo "Cleaning environment \$CLEANED_ENV" 
 [ -z "\$CLEANED_ENV" ] && exec /bin/env -i CLEANED_ENV="Done" HOME=\${HOME} SHELL=/bin/bash /bin/bash -l "\$0" "\$@" 
 
-# From https://github.com/DIRACGrid/COMDIRAC/wiki/Injob
-# initialize job for COMDIRAC commands
-export DCOMMANDS_CONFIG_DIR=$PWD
-dconfig --guess
-dinit --fromProxy
-
-# Some environment variables related to python, to be passed to container, be it for old Singularity version or recent Apptainer ones:
+# Some environment variables, related to python or nectarchain, have to be passed to container, be it for old Singularity versions or recent Apptainer ones:
 export SINGULARITYENV_MPLCONFIGDIR=/tmp
 export SINGULARITYENV_NUMBA_CACHE_DIR=/tmp
+export SINGULARITYENV_NECTARCAMDATA=$NECTARCAMDATA
 export SINGULARITYENV_NECTARDIR=$NECTARDIR
 
 export APPTAINERENV_MPLCONFIGDIR=/tmp
 export APPTAINERENV_NUMBA_CACHE_DIR=/tmp
+export APPTAINERENV_NECTARCAMDATA=$NECTARCAMDATA
 export APPTAINERENV_NECTARDIR=$NECTARDIR
 
 # Handle Singularity or Apptainer case:
@@ -111,7 +108,7 @@ echo
 echo "Running" 
 # Instantiate the nectarchain Singularity image, run our DQM example run within it:
 # cmd="\$CALLER exec --home $PWD $CONTAINER /opt/conda/envs/nectarchain/bin/python /opt/cta/nectarchain/src/nectarchain/dqm/start_dqm.py --r0 $NECTARCAMDATA $NECTARDIR -i $LISTRUNS"
-cmd="\$CALLER exec --home $PWD $CONTAINER /opt/conda/envs/nectarchain/bin/python /opt/cta/nectarchain/src/nectarchain/dqm/start_dqm.py --r0 -r $runnb $NECTARCAMDATA $NECTARDIR"
+cmd="\$CALLER exec --home $PWD $CONTAINER /opt/conda/envs/nectarchain/bin/python /opt/cta/nectarchain/src/nectarchain/dqm/start_dqm.py --r0 --plot --runnb $runnb $NECTARCAMDATA $NECTARDIR"
 echo \$cmd
 eval \$cmd
 EOF
@@ -121,7 +118,7 @@ chmod u+x $WRAPPER || exit_script $?
 
 
 # Archive the output directory and push it on DIRAC before leaving the job:
-tar zcf ${OUTDIR}.tar.gz ${OUTDIR}/ || exit_script $?
-dirac-dms-add-file ${DIRAC_OUTDIR}/${OUTDIR}.tar.gz ${OUTDIR}.tar.gz || exit_script $?
+tar zcf "${OUTDIR}.tar.gz" "${OUTDIR}/" || exit_script $?
+dirac-dms-add-file "${DIRAC_OUTDIR}/${OUTDIR}.tar.gz" "${OUTDIR}.tar.gz" CC-IN2P3-USER || exit_script $?
 
 exit_script 0
diff --git a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/submit_dqm_processor.py b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/submit_dqm_processor.py
index 03e5cb40..1ec0ed2a 100755
--- a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/submit_dqm_processor.py
+++ b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/submit_dqm_processor.py
@@ -8,15 +8,15 @@
 import sys
 from time import sleep
 
-# The magic DIRAC 2 lines
+# DIRAC client initialization
 import DIRAC
 
+DIRAC.initialize()
+
 # astropy imports
 from astropy import time
 from astropy import units as u
 
-DIRAC.initialize()
-
 # DIRAC imports
 from DIRAC.Interfaces.API.Dirac import Dirac
 from DIRAC.Interfaces.API.Job import Job
@@ -28,7 +28,7 @@
 dirac = Dirac()
 
 # Option and argument parser
-parser = argparse.ArgumentParser()
+parser = argparse.ArgumentParser(description="Submit jobs on DIRAC to run the DQM")
 parser.add_argument(
     "-d",
     "--date",
@@ -37,7 +37,11 @@
     type=str,
 )
 parser.add_argument(
-    "-r", "--run", default=None, help="only process a specific run (optional)", type=str
+    "-r",
+    "--run",
+    default=None,
+    help="only process a specific run (optional). When omitted, all the runs acquired on DATE are processed (1 job per run).",
+    type=str,
 )
 parser.add_argument(
     "--dry-run",
diff --git a/src/nectarchain/user_scripts/jlenain/submitGainFitDIRAC/GainFitter.sh b/src/nectarchain/user_scripts/jlenain/submitGainFitDIRAC/GainFitter.sh
index 5f0ac95e..c6293083 100755
--- a/src/nectarchain/user_scripts/jlenain/submitGainFitDIRAC/GainFitter.sh
+++ b/src/nectarchain/user_scripts/jlenain/submitGainFitDIRAC/GainFitter.sh
@@ -94,7 +94,7 @@ cd $ORIGPWD
 
 # Archive the output directory and push it on DIRAC before leaving the job:
 tar zcf ${OUTDIR}.tar.gz ${OUTDIR}/
-dirac-dms-add-file ${DIRAC_OUTDIR}/${OUTDIR}.tar.gz ${OUTDIR}.tar.gz LPNHE-USER
+dirac-dms-add-file ${DIRAC_OUTDIR}/${OUTDIR}.tar.gz ${OUTDIR}.tar.gz GRIF-USER
 
 # Some cleanup before leaving:
 [ -d $OUTDIR ] && rm -rf $OUTDIR