Move to contrib installation of spack-stack on Jet (NOAA-EMC#2878)

Migrates Global Workflow to use contrib installation of spack-stack on
Jet.
Following the failure of the storage /lfs4 on Jet, the installation of
spack spack moved to /contrib.
All softwares relying on spack-stack on Jet needs update.

Resolves NOAA-EMC#2841 
Refs NOAA-EMC/gfs-utils#78
Refs NOAA-EMC/GSI#786
Refs NOAA-EMC/GSI-Monitor#143
Refs NOAA-EMC/GSI-utils#51
Refs ufs-community/UFS_UTILS#977

docs/note_fixfield.txt

@@ -4,7 +4,7 @@ They are saved locally on all platforms
 
 Hera: /scratch1/NCEPDEV/global/glopara/fix
 Orion/Hercules: /work/noaa/global/glopara/fix
-Jet: /mnt/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix
+Jet: /mnt/lfs5/HFIP/hfv3gfs/glopara/FIX/fix
 S4: /data/prod/glopara/fix
 
 ------------------------------------------------------------------------------

docs/source/components.rst

@@ -59,7 +59,7 @@ Observation data, also known as dump data, is prepared in production and then ar
 
 * Hera: ``/scratch1/NCEPDEV/global/glopara/dump``
 * Orion/Hercules: ``/work/noaa/rstprod/dump``
-* Jet: ``/mnt/lfs4/HFIP/hfv3gfs/glopara/dump``
+* Jet: ``/mnt/lfs5/HFIP/hfv3gfs/glopara/dump``
 * WCOSS2: ``/lfs/h2/emc/global/noscrub/emc.global/dump``
 * S4: ``/data/prod/glopara/dump``
 

docs/source/init.rst

@@ -111,7 +111,7 @@ Warm-start cycled w/ coupled (S2S) model C48 atmosphere C48 enkf (80 members) 5
    Hera: /scratch1/NCEPDEV/global/glopara/data/ICSDIR/C48C48mx500
    Orion/Hercules: /work/noaa/global/glopara/data/ICSDIR/C48C48mx500
    WCOSS2: /lfs/h2/emc/global/noscrub/emc.global/data/ICSDIR/C48C48mx500
-   Jet: /lfs4/HFIP/hfv3gfs/glopara/data/ICSDIR/C48C48mx500
+   Jet: /lfs5/HFIP/hfv3gfs/glopara/data/ICSDIR/C48C48mx500
    AWS: https://noaa-nws-global-pds.s3.amazonaws.com/index.html#data/ICSDIR/C48C48mx500
 
 Start date = 2021032312
@@ -227,7 +227,7 @@ Forecast-only P8 prototype initial conditions are made available to users on sup
     WCOSS2: /lfs/h2/emc/global/noscrub/emc.global/IC/COUPLED
     HERA: /scratch1/NCEPDEV/climate/role.ufscpara/IC
     ORION/Hercules: /work/noaa/global/glopara/data/ICSDIR/prototype_ICs
-    JET: /mnt/lfs4/HFIP/hfv3gfs/glopara/data/ICSDIR/prototype_ICs
+    JET: /mnt/lfs5/HFIP/hfv3gfs/glopara/data/ICSDIR/prototype_ICs
     S4: /data/prod/glopara/coupled_ICs
 
 These locations are known within the workflow via paths set in ``parm/config/config.coupled_ic``.

modulefiles/module_base.jet.lua

@@ -52,7 +52,7 @@ setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None"))
 prepend_path("MODULEPATH", pathJoin("/lfs5/HFIP/hfv3gfs/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles"))
 load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None")))
 
-prepend_path("MODULEPATH", pathJoin("/lfs4/HFIP/hfv3gfs/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles"))
+prepend_path("MODULEPATH", pathJoin("/lfs5/HFIP/hfv3gfs/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles"))
 load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None")))
 
 whatis("Description: GFS run environment")

modulefiles/module_gwsetup.jet.lua

@@ -4,7 +4,7 @@ Load environment to run GFS workflow setup scripts on Jet
 
 load(pathJoin("rocoto"))
 
-prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev-rocky8/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.6.0/envs/gsi-addon-intel/install/modulefiles/Core")
 
 local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0"
 local python_ver=os.getenv("python_ver") or "3.11.6"

parm/config/gfs/config.aero

@@ -24,7 +24,7 @@ case ${machine} in
     AERO_INPUTS_DIR="/gpfs/f5/epic/proj-shared/global/glopara/data/gocart_emissions"
     ;;
   "JET")
-    AERO_INPUTS_DIR="/lfs4/HFIP/hfv3gfs/glopara/data/gocart_emissions"
+    AERO_INPUTS_DIR="/lfs5/HFIP/hfv3gfs/glopara/data/gocart_emissions"
     ;;
   *)
     echo "FATAL ERROR: Machine ${machine} unsupported for aerosols"

sorc/link_workflow.sh

@@ -73,7 +73,7 @@ case "${machine}" in
   "hera")     FIX_DIR="/scratch1/NCEPDEV/global/glopara/fix" ;;
   "orion")    FIX_DIR="/work/noaa/global/glopara/fix" ;;
   "hercules") FIX_DIR="/work/noaa/global/glopara/fix" ;;
-  "jet")      FIX_DIR="/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix" ;;
+  "jet")      FIX_DIR="/lfs5/HFIP/hfv3gfs/glopara/FIX/fix" ;;
   "s4")       FIX_DIR="/data/prod/glopara/fix" ;;
   "gaea")     FIX_DIR="/gpfs/f5/ufs-ard/world-shared/global/glopara/data/fix" ;;
   "noaacloud") FIX_DIR="/contrib/global-workflow-shared-data/fix" ;;

ush/detect_machine.sh

@@ -67,7 +67,7 @@ if [[ -d /lfs/h3 ]]; then
 elif [[ -d /lfs/h1 && ! -d /lfs/h3 ]]; then
   # We are on NOAA TDS Acorn
   MACHINE_ID=acorn
-elif [[ -d /mnt/lfs1 ]]; then
+elif [[ -d /mnt/lfs5 ]]; then
   # We are on NOAA Jet
   MACHINE_ID=jet
 elif [[ -d /scratch1 ]]; then

ush/forecast_predet.sh

@@ -548,7 +548,6 @@ FV3_predet(){
       ${NCP} "${PARMgfs}/post/gefs/postxconfig-NT-gefs-f00.txt"   "${DATA}/postxconfig-NT_FH00.txt"
     fi
   fi
-
 }
 
 # Disable variable not used warnings

versions/build.jet.ver

@@ -1,5 +1,5 @@
 export stack_intel_ver=2021.5.0
 export stack_impi_ver=2021.5.1
-export spack_env=gsi-addon-dev
+export spack_env=gsi-addon-intel
 source "${HOMEgfs:-}/versions/spack.ver"
-export spack_mod_path="/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core"
+export spack_mod_path="/contrib/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core"

versions/run.jet.ver

@@ -1,6 +1,6 @@
 export stack_intel_ver=2021.5.0
 export stack_impi_ver=2021.5.1
-export spack_env=gsi-addon-dev-rocky8
+export spack_env=gsi-addon-intel
 
 export hpss_ver=
 export ncl_ver=6.6.2
@@ -11,4 +11,8 @@ export gempak_ver=7.4.2
 export perl_ver=5.38.0
 
 source "${HOMEgfs:-}/versions/spack.ver"
-export spack_mod_path="/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core"
+export spack_mod_path="/contrib/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core"
+
+# Local version of TC_tracker
+export ens_tracker_ver=v1.1.15.7
+

workflow/hosts.py

@@ -42,7 +42,7 @@ def detect(cls):
             machine = 'HERA'
         elif os.path.exists('/work/noaa'):
             machine = socket.gethostname().split("-", 1)[0].upper()
-        elif os.path.exists('/lfs4/HFIP'):
+        elif os.path.exists('/lfs5/HFIP'):
             machine = 'JET'
         elif os.path.exists('/lfs/f1'):
             machine = 'WCOSS2'

workflow/hosts/jet.yaml

@@ -1,12 +1,12 @@
-BASE_GIT: '/lfs4/HFIP/hfv3gfs/glopara/git'
-DMPDIR: '/lfs4/HFIP/hfv3gfs/glopara/dump'
+BASE_GIT: '/lfs5/HFIP/hfv3gfs/glopara/git'
+DMPDIR: '/lfs5/HFIP/hfv3gfs/glopara/dump'
 BASE_DATA: '/lfs5/HFIP/hfv3gfs/glopara/data'
-BASE_IC: '/mnt/lfs4/HFIP/hfv3gfs/glopara/data/ICSDIR'
-PACKAGEROOT: '/lfs4/HFIP/hfv3gfs/glopara/nwpara'
-COMINsyn: '/lfs4/HFIP/hfv3gfs/glopara/com/gfs/prod/syndat'
-HOMEDIR: '/lfs4/HFIP/hfv3gfs/${USER}'
-STMP: '/lfs4/HFIP/hfv3gfs/${USER}/stmp'
-PTMP: '/lfs4/HFIP/hfv3gfs/${USER}/ptmp'
+BASE_IC: '/mnt/lfs5/HFIP/hfv3gfs/glopara/data/ICSDIR'
+PACKAGEROOT: '/lfs5/HFIP/hfv3gfs/glopara/nwpara'
+COMINsyn: '/lfs5/HFIP/hfv3gfs/glopara/com/gfs/prod/syndat'
+HOMEDIR: '/lfs5/HFIP/hfv3gfs/${USER}'
+STMP: '/lfs5/HFIP/hfv3gfs/${USER}/stmp'
+PTMP: '/lfs5/HFIP/hfv3gfs/${USER}/ptmp'
 NOSCRUB: $HOMEDIR
 ACCOUNT: hfv3gfs
 SCHEDULER: slurm
@@ -25,6 +25,6 @@ ATARDIR: '/NCEPDEV/${HPSS_PROJECT}/1year/${USER}/${machine}/scratch/${PSLOT}'
 MAKE_NSSTBUFR: 'NO'
 MAKE_ACFTBUFR: 'NO'
 SUPPORTED_RESOLUTIONS: ['C384', 'C192', 'C96', 'C48']
-COMINecmwf: /mnt/lfs4/HFIP/hfv3gfs/glopara/data/external_gempak/ecmwf
-COMINnam: /mnt/lfs4/HFIP/hfv3gfs/glopara/data/external_gempak/nam
-COMINukmet: /mnt/lfs4/HFIP/hfv3gfs/glopara/data/external_gempak/ukmet
+COMINecmwf: /mnt/lfs5/HFIP/hfv3gfs/glopara/data/external_gempak/ecmwf
+COMINnam: /mnt/lfs5/HFIP/hfv3gfs/glopara/data/external_gempak/nam
+COMINukmet: /mnt/lfs5/HFIP/hfv3gfs/glopara/data/external_gempak/ukmet