From f5b8c88fe39862eaaa990cc139efd13d850eae25 Mon Sep 17 00:00:00 2001
From: Justin Wozniak <woz@anl.gov>
Date: Wed, 12 Apr 2023 14:35:33 -0500
Subject: [PATCH 01/10] Draft train.sh for Uno

---
 Pilot1/Uno/train.sh | 57 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)
 create mode 100755 Pilot1/Uno/train.sh

diff --git a/Pilot1/Uno/train.sh b/Pilot1/Uno/train.sh
new file mode 100755
index 00000000..777971a6
--- /dev/null
+++ b/Pilot1/Uno/train.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+set -eu
+
+# UNO TRAIN SH
+
+# arg 1 CUDA_VISIBLE_DEVICES
+# arg 2 CANDLE_DATA_DIR
+# arg 3 CANDLE_CONFIG
+
+### Path to your CANDLEized model's main Python script###
+CANDLE_MODEL=/usr/local/Benchmarks/Pilot1/Uno/uno_baseline_keras2.py
+
+if (( $# < 2 )) ; then
+  echo "Uno/train.sh: Illegal number of parameters: given: ${#}"
+  echo "CUDA_VISIBLE_DEVICES and CANDLE_DATA_DIR are required"
+  exit -1
+fi
+
+CUDA_VISIBLE_DEVICES=$1 ; shift
+CANDLE_DATA_DIR=$1 ; shift
+
+if (( $# == 2 )) ; then
+  CMD=( python ${CANDLE_MODEL} )
+  echo "CMD = $CMD"
+elif (( $# >= 3 )) ; then
+  # if original $3 is a file, set candle_config and passthrough $@
+  CANDLE_CONFIG=$1
+  if [[ -f $CANDLE_CONFIG ]] ; then
+    echo "Uno/train.sh: found CANDLE_CONFIG=$CANDLE_CONFIG"
+    shift
+    CMD=( python ${CANDLE_MODEL} --config_file $CANDLE_CONFIG $@ )
+  else # simply passthrough $@
+    CMD=( python ${CANDLE_MODEL} $@ )
+  fi
+fi
+
+echo "CMD = ${CMD[@]}"
+
+# Report runtime arguments
+echo "using CUDA_VISIBLE_DEVICES ${CUDA_VISIBLE_DEVICES}"
+echo "using CANDLE_DATA_DIR ${CANDLE_DATA_DIR}"
+echo "using CANDLE_CONFIG ${CANDLE_CONFIG}"
+
+echo "train.sh: running command..."
+
+# Set up environmental variables and execute the model!
+if env CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES} \
+       CANDLE_DATA_DIR=${CANDLE_DATA_DIR} \
+       ${CMD[@]}
+then
+  echo "train.sh: OK."
+else
+  CODE=$?
+  echo "train.sh: MODEL ERROR: code=$CODE"
+fi
+
+exit $CODE

From 94f019ef75d468f5d5de46eb075172caa368da0f Mon Sep 17 00:00:00 2001
From: Justin Wozniak <woz@anl.gov>
Date: Wed, 12 Apr 2023 16:04:39 -0500
Subject: [PATCH 02/10] Set default

---
 Pilot1/Uno/train.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Pilot1/Uno/train.sh b/Pilot1/Uno/train.sh
index 777971a6..7dc158bf 100755
--- a/Pilot1/Uno/train.sh
+++ b/Pilot1/Uno/train.sh
@@ -18,6 +18,7 @@ fi
 
 CUDA_VISIBLE_DEVICES=$1 ; shift
 CANDLE_DATA_DIR=$1 ; shift
+CANDLE_CONFIG=0
 
 if (( $# == 2 )) ; then
   CMD=( python ${CANDLE_MODEL} )

From ebaeba20108a6baf69e4df50460d9d0d0013cd86 Mon Sep 17 00:00:00 2001
From: Justin Wozniak <woz@anl.gov>
Date: Wed, 12 Apr 2023 16:19:25 -0500
Subject: [PATCH 03/10] Set CODE

---
 Pilot1/Uno/train.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Pilot1/Uno/train.sh b/Pilot1/Uno/train.sh
index 7dc158bf..47277e1a 100755
--- a/Pilot1/Uno/train.sh
+++ b/Pilot1/Uno/train.sh
@@ -49,6 +49,7 @@ if env CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES} \
        CANDLE_DATA_DIR=${CANDLE_DATA_DIR} \
        ${CMD[@]}
 then
+  CODE=0
   echo "train.sh: OK."
 else
   CODE=$?

From a605d80b44ae5c2cdd57417171dc424c7a5adca5 Mon Sep 17 00:00:00 2001
From: Justin Wozniak <woz@anl.gov>
Date: Wed, 12 Apr 2023 16:27:52 -0500
Subject: [PATCH 04/10] Debug

---
 Pilot1/Uno/train.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Pilot1/Uno/train.sh b/Pilot1/Uno/train.sh
index 47277e1a..664408a6 100755
--- a/Pilot1/Uno/train.sh
+++ b/Pilot1/Uno/train.sh
@@ -22,7 +22,7 @@ CANDLE_CONFIG=0
 
 if (( $# == 2 )) ; then
   CMD=( python ${CANDLE_MODEL} )
-  echo "CMD = $CMD"
+  echo "CMD2 = ${CMD[@]}"
 elif (( $# >= 3 )) ; then
   # if original $3 is a file, set candle_config and passthrough $@
   CANDLE_CONFIG=$1
@@ -44,6 +44,7 @@ echo "using CANDLE_CONFIG ${CANDLE_CONFIG}"
 
 echo "train.sh: running command..."
 
+set -x
 # Set up environmental variables and execute the model!
 if env CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES} \
        CANDLE_DATA_DIR=${CANDLE_DATA_DIR} \

From 8ec394ac1a3ada6af1bbddc3417f05228e872597 Mon Sep 17 00:00:00 2001
From: Justin Wozniak <woz@anl.gov>
Date: Wed, 12 Apr 2023 16:30:28 -0500
Subject: [PATCH 05/10] Debug

---
 Pilot1/Uno/train.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Pilot1/Uno/train.sh b/Pilot1/Uno/train.sh
index 664408a6..3e6c20be 100755
--- a/Pilot1/Uno/train.sh
+++ b/Pilot1/Uno/train.sh
@@ -20,6 +20,7 @@ CUDA_VISIBLE_DEVICES=$1 ; shift
 CANDLE_DATA_DIR=$1 ; shift
 CANDLE_CONFIG=0
 
+set -x
 if (( $# == 2 )) ; then
   CMD=( python ${CANDLE_MODEL} )
   echo "CMD2 = ${CMD[@]}"

From 3db69361f6797b1615e1455b354fd8eb9688168f Mon Sep 17 00:00:00 2001
From: Justin Wozniak <woz@anl.gov>
Date: Wed, 12 Apr 2023 16:36:50 -0500
Subject: [PATCH 06/10] Debug

---
 Pilot1/Uno/train.sh | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/Pilot1/Uno/train.sh b/Pilot1/Uno/train.sh
index 3e6c20be..e7c1cccc 100755
--- a/Pilot1/Uno/train.sh
+++ b/Pilot1/Uno/train.sh
@@ -10,8 +10,12 @@ set -eu
 ### Path to your CANDLEized model's main Python script###
 CANDLE_MODEL=/usr/local/Benchmarks/Pilot1/Uno/uno_baseline_keras2.py
 
-if (( $# < 2 )) ; then
-  echo "Uno/train.sh: Illegal number of parameters: given: ${#}"
+set -x
+
+ARGC=$#
+
+if (( $ARGC < 2 )) ; then
+  echo "Uno/train.sh: Illegal number of parameters: given: ${ARGC}"
   echo "CUDA_VISIBLE_DEVICES and CANDLE_DATA_DIR are required"
   exit -1
 fi
@@ -20,12 +24,11 @@ CUDA_VISIBLE_DEVICES=$1 ; shift
 CANDLE_DATA_DIR=$1 ; shift
 CANDLE_CONFIG=0
 
-set -x
-if (( $# == 2 )) ; then
+if (( $ARGC == 2 )) ; then
   CMD=( python ${CANDLE_MODEL} )
   echo "CMD2 = ${CMD[@]}"
-elif (( $# >= 3 )) ; then
-  # if original $3 is a file, set candle_config and passthrough $@
+elif (( $ARGC >= 3 )) ; then
+  # If original $3 is a file, set candle_config and passthrough $@
   CANDLE_CONFIG=$1
   if [[ -f $CANDLE_CONFIG ]] ; then
     echo "Uno/train.sh: found CANDLE_CONFIG=$CANDLE_CONFIG"

From a48bfb460c6d27e90ad9052b14a0cb9248526470 Mon Sep 17 00:00:00 2001
From: Justin Wozniak <woz@anl.gov>
Date: Wed, 12 Apr 2023 16:40:15 -0500
Subject: [PATCH 07/10] Clean up

---
 Pilot1/Uno/train.sh | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/Pilot1/Uno/train.sh b/Pilot1/Uno/train.sh
index e7c1cccc..1ced4b0c 100755
--- a/Pilot1/Uno/train.sh
+++ b/Pilot1/Uno/train.sh
@@ -10,8 +10,7 @@ set -eu
 ### Path to your CANDLEized model's main Python script###
 CANDLE_MODEL=/usr/local/Benchmarks/Pilot1/Uno/uno_baseline_keras2.py
 
-set -x
-
+# Make copy of $# before shifts
 ARGC=$#
 
 if (( $ARGC < 2 )) ; then
@@ -48,7 +47,6 @@ echo "using CANDLE_CONFIG ${CANDLE_CONFIG}"
 
 echo "train.sh: running command..."
 
-set -x
 # Set up environmental variables and execute the model!
 if env CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES} \
        CANDLE_DATA_DIR=${CANDLE_DATA_DIR} \

From a40951b7c0ce5a78ac12e0f3e0c70097db0e6b9e Mon Sep 17 00:00:00 2001
From: Justin Wozniak <woz@anl.gov>
Date: Thu, 13 Apr 2023 17:15:51 -0500
Subject: [PATCH 08/10] Clean up

---
 Pilot1/Uno/train.sh | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/Pilot1/Uno/train.sh b/Pilot1/Uno/train.sh
index 1ced4b0c..d1c60b7f 100755
--- a/Pilot1/Uno/train.sh
+++ b/Pilot1/Uno/train.sh
@@ -25,7 +25,6 @@ CANDLE_CONFIG=0
 
 if (( $ARGC == 2 )) ; then
   CMD=( python ${CANDLE_MODEL} )
-  echo "CMD2 = ${CMD[@]}"
 elif (( $ARGC >= 3 )) ; then
   # If original $3 is a file, set candle_config and passthrough $@
   CANDLE_CONFIG=$1
@@ -38,14 +37,14 @@ elif (( $ARGC >= 3 )) ; then
   fi
 fi
 
-echo "CMD = ${CMD[@]}"
-
 # Report runtime arguments
 echo "using CUDA_VISIBLE_DEVICES ${CUDA_VISIBLE_DEVICES}"
 echo "using CANDLE_DATA_DIR ${CANDLE_DATA_DIR}"
 echo "using CANDLE_CONFIG ${CANDLE_CONFIG}"
 
 echo "train.sh: running command..."
+echo "CMD = ${CMD[@]}"
+echo
 
 # Set up environmental variables and execute the model!
 if env CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES} \

From 84b2cb3788d502001de99d9893cad6c8f084c838 Mon Sep 17 00:00:00 2001
From: Justin Wozniak <woz@anl.gov>
Date: Thu, 13 Apr 2023 17:15:51 -0500
Subject: [PATCH 09/10] Clean up (closes #119)

---
 Pilot1/Uno/train.sh | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/Pilot1/Uno/train.sh b/Pilot1/Uno/train.sh
index 1ced4b0c..d1c60b7f 100755
--- a/Pilot1/Uno/train.sh
+++ b/Pilot1/Uno/train.sh
@@ -25,7 +25,6 @@ CANDLE_CONFIG=0
 
 if (( $ARGC == 2 )) ; then
   CMD=( python ${CANDLE_MODEL} )
-  echo "CMD2 = ${CMD[@]}"
 elif (( $ARGC >= 3 )) ; then
   # If original $3 is a file, set candle_config and passthrough $@
   CANDLE_CONFIG=$1
@@ -38,14 +37,14 @@ elif (( $ARGC >= 3 )) ; then
   fi
 fi
 
-echo "CMD = ${CMD[@]}"
-
 # Report runtime arguments
 echo "using CUDA_VISIBLE_DEVICES ${CUDA_VISIBLE_DEVICES}"
 echo "using CANDLE_DATA_DIR ${CANDLE_DATA_DIR}"
 echo "using CANDLE_CONFIG ${CANDLE_CONFIG}"
 
 echo "train.sh: running command..."
+echo "CMD = ${CMD[@]}"
+echo
 
 # Set up environmental variables and execute the model!
 if env CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES} \

From 1aa31bf6e76af36212b43799a4d6fe33fdbd4777 Mon Sep 17 00:00:00 2001
From: Justin Wozniak <woz@anl.gov>
Date: Fri, 15 Sep 2023 14:05:34 -0500
Subject: [PATCH 10/10] Support local non-containerized execution.  closes
 JDACS4C-IMPROVE/Singularity#64

---
 Pilot1/Uno/train.sh | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/Pilot1/Uno/train.sh b/Pilot1/Uno/train.sh
index d1c60b7f..20418b46 100755
--- a/Pilot1/Uno/train.sh
+++ b/Pilot1/Uno/train.sh
@@ -8,7 +8,14 @@ set -eu
 # arg 3 CANDLE_CONFIG
 
 ### Path to your CANDLEized model's main Python script###
-CANDLE_MODEL=/usr/local/Benchmarks/Pilot1/Uno/uno_baseline_keras2.py
+CANDLE_MODEL=uno_baseline_keras2.py
+IMPROVE_MODEL_DIR=${IMPROVE_MODEL_DIR:-$( dirname -- "$0" )}
+CANDLE_MODEL=${IMPROVE_MODEL_DIR}/${CANDLE_MODEL}
+
+if [ ! -f ${CANDLE_MODEL} ] ; then
+  echo No such file ${CANDLE_MODEL}
+  exit 404
+fi
 
 # Make copy of $# before shifts
 ARGC=$#