From 5b737b16e4e247950c515242b3d0f712aee47edd Mon Sep 17 00:00:00 2001
From: Benjamin Moore Wagman <bmwagma@sandia.gov>
Date: Thu, 14 Nov 2024 11:35:14 -0700
Subject: [PATCH] Merge pull request #1 from wagmanbe/jimsmods

Jimsmods
---
 .../machines/cmake_macros/intel_boca.cmake    |  24 +++
 .../machines/cmake_macros/intel_flight.cmake  |  24 +++
 cime_config/machines/config_batch.xml         |  20 +++
 cime_config/machines/config_machines.xml      | 160 ++++++++++++++++++
 .../eamxx/cmake/machine-files/flight.cmake    |  15 ++
 5 files changed, 243 insertions(+)
 create mode 100644 cime_config/machines/cmake_macros/intel_boca.cmake
 create mode 100644 cime_config/machines/cmake_macros/intel_flight.cmake
 create mode 100644 components/eamxx/cmake/machine-files/flight.cmake

diff --git a/cime_config/machines/cmake_macros/intel_boca.cmake b/cime_config/machines/cmake_macros/intel_boca.cmake
new file mode 100644
index 000000000000..b058430302c3
--- /dev/null
+++ b/cime_config/machines/cmake_macros/intel_boca.cmake
@@ -0,0 +1,24 @@
+set(ALBANY_PATH "/projects/ccsm/AlbanyTrilinos_20190904/albany-build/install")
+set(CLDERA_PATH "/projects/cldera/cldera-tools/install-master/intel/")
+if (COMP_NAME STREQUAL gptl)
+  string(APPEND CPPDEFS " -DHAVE_SLASHPROC")
+endif()
+set(ESMF_LIBDIR "/projects/ccsm/esmf-6.3.0rp1/lib/libO/Linux.intel.64.openmpi.default")
+if (MPILIB STREQUAL openmpi)
+  set(MPI_PATH "$ENV{MPIHOME}")
+endif()
+set(NETCDF_PATH "$ENV{NETCDFROOT}")
+if (MPILIB STREQUAL mpi-serial AND NOT compile_threaded)
+  set(PFUNIT_PATH "/projects/ccsm/pfunit/3.2.9/mpi-serial")
+endif()
+set(PIO_FILESYSTEM_HINTS "lustre")
+set(PNETCDF_PATH "$ENV{PNETCDFROOT}")
+execute_process(COMMAND ${NETCDF_PATH}/bin/nf-config --flibs OUTPUT_VARIABLE SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0 OUTPUT_STRIP_TRAILING_WHITESPACE)
+string(APPEND SLIBS " ${SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0} -L/projects/ccsm/BLAS-intel -lblas_LINUX -L$ENV{MKL_LIBS} -lmkl_rt")
+if (MPILIB STREQUAL openmpi)
+  string(APPEND SLIBS " -mkl=cluster")
+endif()
+if (MPILIB STREQUAL mpi-serial)
+  string(APPEND SLIBS " -mkl")
+endif()
+string(APPEND FFLAGS " -fpe3")
diff --git a/cime_config/machines/cmake_macros/intel_flight.cmake b/cime_config/machines/cmake_macros/intel_flight.cmake
new file mode 100644
index 000000000000..b058430302c3
--- /dev/null
+++ b/cime_config/machines/cmake_macros/intel_flight.cmake
@@ -0,0 +1,24 @@
+set(ALBANY_PATH "/projects/ccsm/AlbanyTrilinos_20190904/albany-build/install")
+set(CLDERA_PATH "/projects/cldera/cldera-tools/install-master/intel/")
+if (COMP_NAME STREQUAL gptl)
+  string(APPEND CPPDEFS " -DHAVE_SLASHPROC")
+endif()
+set(ESMF_LIBDIR "/projects/ccsm/esmf-6.3.0rp1/lib/libO/Linux.intel.64.openmpi.default")
+if (MPILIB STREQUAL openmpi)
+  set(MPI_PATH "$ENV{MPIHOME}")
+endif()
+set(NETCDF_PATH "$ENV{NETCDFROOT}")
+if (MPILIB STREQUAL mpi-serial AND NOT compile_threaded)
+  set(PFUNIT_PATH "/projects/ccsm/pfunit/3.2.9/mpi-serial")
+endif()
+set(PIO_FILESYSTEM_HINTS "lustre")
+set(PNETCDF_PATH "$ENV{PNETCDFROOT}")
+execute_process(COMMAND ${NETCDF_PATH}/bin/nf-config --flibs OUTPUT_VARIABLE SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0 OUTPUT_STRIP_TRAILING_WHITESPACE)
+string(APPEND SLIBS " ${SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0} -L/projects/ccsm/BLAS-intel -lblas_LINUX -L$ENV{MKL_LIBS} -lmkl_rt")
+if (MPILIB STREQUAL openmpi)
+  string(APPEND SLIBS " -mkl=cluster")
+endif()
+if (MPILIB STREQUAL mpi-serial)
+  string(APPEND SLIBS " -mkl")
+endif()
+string(APPEND FFLAGS " -fpe3")
diff --git a/cime_config/machines/config_batch.xml b/cime_config/machines/config_batch.xml
index 923e497367f7..d93e8b775087 100644
--- a/cime_config/machines/config_batch.xml
+++ b/cime_config/machines/config_batch.xml
@@ -620,6 +620,26 @@
     </queues>
   </batch_system>
 
+  <batch_system MACH="boca" type="slurm" >
+    <submit_args>
+      <arg flag="--reservation=boca-cldera"/>
+    </submit_args>
+    <queues>
+      <queue nodemax="20" walltimemax="04:00:00" strict="true" default="true">batch</queue>
+      <queue walltimemax="48:00:00">batch</queue>
+    </queues>
+  </batch_system>
+
+  <batch_system MACH="flight" type="slurm" >
+    <submit_args>
+      <arg flag="--reservation=flight-cldera"/>
+    </submit_args>
+    <queues>
+      <queue nodemax="20" walltimemax="04:00:00" strict="true" default="true">batch</queue>
+      <queue walltimemax="48:00:00">batch</queue>
+    </queues>
+  </batch_system>
+
   <batch_system MACH="ghost" type="slurm" >
     <queues>
       <queue nodemax="12" walltimemax="04:00:00" strict="true" default="true">short,batch</queue>
diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml
index 9142a5fb24ee..50698f7f664e 100644
--- a/cime_config/machines/config_machines.xml
+++ b/cime_config/machines/config_machines.xml
@@ -2121,6 +2121,166 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss
     </environment_variables>
   </machine>
 
+  <machine MACH="boca">
+    <DESC>SNL clust</DESC>
+    <NODENAME_REGEX>(boca)</NODENAME_REGEX>
+    <OS>LINUX</OS>
+    <PROXY>proxy.sandia.gov:80</PROXY>
+    <COMPILERS>intel</COMPILERS>
+    <MPILIBS>openmpi</MPILIBS>
+    <PROJECT>fy220022</PROJECT>
+    <SAVE_TIMING_DIR>/projects/ccsm/timings</SAVE_TIMING_DIR>
+    <SAVE_TIMING_DIR_PROJECTS>.*</SAVE_TIMING_DIR_PROJECTS>
+    <CIME_OUTPUT_ROOT>/gpfs/$USER/acme_scratch/boca</CIME_OUTPUT_ROOT>
+    <DIN_LOC_ROOT>/projects/ccsm/inputdata</DIN_LOC_ROOT>
+    <DIN_LOC_ROOT_CLMFORC>/projects/ccsm/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
+    <DOUT_S_ROOT>$CIME_OUTPUT_ROOT/archive/$CASE</DOUT_S_ROOT>
+    <BASELINE_ROOT>/projects/ccsm/ccsm_baselines/$COMPILER</BASELINE_ROOT>
+    <CCSM_CPRNC>/projects/ccsm/cprnc/build.toss3/cprnc</CCSM_CPRNC>
+    <GMAKE_J>32</GMAKE_J>
+    <TESTS>e3sm_integration</TESTS>
+    <BATCH_SYSTEM>slurm</BATCH_SYSTEM>
+    <SUPPORTED_BY>jgfouca at sandia dot gov</SUPPORTED_BY>
+    <MAX_TASKS_PER_NODE>36</MAX_TASKS_PER_NODE>
+    <MAX_MPITASKS_PER_NODE>36</MAX_MPITASKS_PER_NODE>
+    <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
+    <mpirun mpilib="default">
+      <executable>mpiexec</executable>
+      <arguments>
+        <arg name="num_tasks"> --n {{ total_tasks }}</arg>
+        <arg name="tasks_per_node"> --map-by ppr:{{ tasks_per_numa }}:socket:PE=$ENV{OMP_NUM_THREADS} --bind-to core</arg>
+      </arguments>
+    </mpirun>
+    <mpirun mpilib="mpi-serial">
+      <executable/>
+    </mpirun>
+    <module_system type="module">
+      <init_path lang="python">/usr/share/lmod/lmod/init/python.py</init_path>
+      <init_path lang="perl">/usr/share/lmod/lmod/init/perl.pm</init_path>
+      <init_path lang="sh">/usr/share/lmod/lmod/init/sh</init_path>
+      <init_path lang="csh">/usr/share/lmod/lmod/init/csh</init_path>
+      <cmd_path lang="python">/usr/share/lmod/lmod/libexec/lmod python</cmd_path>
+      <cmd_path lang="perl">/usr/share/lmod/lmod/libexec/lmod perl</cmd_path>
+      <cmd_path lang="csh">module</cmd_path>
+      <cmd_path lang="sh">module</cmd_path>
+      <modules>
+        <command name="purge"/>
+        <command name="load">sems-archive-env</command>
+        <command name="load">acme-env</command>
+        <command name="load">sems-archive-git</command>
+        <command name="load">sems-archive-cmake/3.19.1</command>
+        <command name="load">gnu/10.2</command>
+        <command name="load">sems-archive-intel/21.3.0</command>
+      </modules>
+      <modules mpilib="!mpi-serial">
+        <command name="load">sems-archive-openmpi/4.1.4</command>
+        <command name="load">acme-netcdf/4.7.4/acme</command>
+      </modules>
+      <modules mpilib="mpi-serial">
+        <command name="load">sems-archive-netcdf/4.4.1/exo</command>
+      </modules>
+    </module_system>
+    <RUNDIR>/nscratch/$USER/acme_scratch/boca/$CASE/run</RUNDIR>
+    <EXEROOT>$CIME_OUTPUT_ROOT/$CASE/bld</EXEROOT>
+    <!-- complete path to a short term archiving directory -->
+    <!-- path to the cprnc tool used to compare netcdf history files in testing -->
+    <TEST_TPUT_TOLERANCE>0.1</TEST_TPUT_TOLERANCE>
+
+    <environment_variables>
+      <env name="NETCDFROOT">$ENV{SEMS_NETCDF_ROOT}</env>
+      <env name="NETCDF_INCLUDES">$ENV{SEMS_NETCDF_ROOT}/include</env>
+      <env name="NETCDF_LIBS">$ENV{SEMS_NETCDF_ROOT}/lib</env>
+      <env name="OMP_STACKSIZE">64M</env>
+    </environment_variables>
+    <environment_variables mpilib="!mpi-serial">
+      <env name="PNETCDF_PATH">$ENV{SEMS_NETCDF_ROOT}</env>
+    </environment_variables>
+    <environment_variables compiler="intel">
+      <env name="CLDERA_PATH">/projects/cldera/cldera-tools/install-master/intel</env>
+    </environment_variables>
+  </machine>
+
+
+  <machine MACH="flight">
+    <DESC>SNL clust</DESC>
+    <NODENAME_REGEX>(flight)</NODENAME_REGEX>
+    <OS>LINUX</OS>
+    <PROXY>proxy.sandia.gov:80</PROXY>
+    <COMPILERS>intel</COMPILERS>
+    <MPILIBS>openmpi</MPILIBS>
+    <PROJECT>fy220022</PROJECT>
+    <SAVE_TIMING_DIR>/projects/ccsm/timings</SAVE_TIMING_DIR>
+    <SAVE_TIMING_DIR_PROJECTS>.*</SAVE_TIMING_DIR_PROJECTS>
+    <CIME_OUTPUT_ROOT>/gpfs/$USER/acme_scratch/flight</CIME_OUTPUT_ROOT>
+    <DIN_LOC_ROOT>/projects/ccsm/inputdata</DIN_LOC_ROOT>
+    <DIN_LOC_ROOT_CLMFORC>/projects/ccsm/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
+    <DOUT_S_ROOT>$CIME_OUTPUT_ROOT/archive/$CASE</DOUT_S_ROOT>
+    <BASELINE_ROOT>/projects/ccsm/ccsm_baselines/$COMPILER</BASELINE_ROOT>
+    <CCSM_CPRNC>/projects/ccsm/cprnc/build.toss3/cprnc</CCSM_CPRNC>
+    <GMAKE_J>56</GMAKE_J>
+    <TESTS>e3sm_integration</TESTS>
+    <BATCH_SYSTEM>slurm</BATCH_SYSTEM>
+    <SUPPORTED_BY>jgfouca at sandia dot gov</SUPPORTED_BY>
+    <MAX_TASKS_PER_NODE>112</MAX_TASKS_PER_NODE>
+    <MAX_MPITASKS_PER_NODE>112</MAX_MPITASKS_PER_NODE>
+    <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
+    <mpirun mpilib="default">
+      <executable>mpiexec</executable>
+      <arguments>
+        <arg name="num_tasks"> --n {{ total_tasks }}</arg>
+        <arg name="tasks_per_node"> --map-by ppr:{{ tasks_per_numa }}:socket:PE=$ENV{OMP_NUM_THREADS} --bind-to core</arg>
+      </arguments>
+    </mpirun>
+    <mpirun mpilib="mpi-serial">
+      <executable/>
+    </mpirun>
+    <module_system type="module">
+      <init_path lang="python">/usr/share/lmod/lmod/init/python.py</init_path>
+      <init_path lang="perl">/usr/share/lmod/lmod/init/perl.pm</init_path>
+      <init_path lang="sh">/usr/share/lmod/lmod/init/sh</init_path>
+      <init_path lang="csh">/usr/share/lmod/lmod/init/csh</init_path>
+      <cmd_path lang="python">/usr/share/lmod/lmod/libexec/lmod python</cmd_path>
+      <cmd_path lang="perl">/usr/share/lmod/lmod/libexec/lmod perl</cmd_path>
+      <cmd_path lang="csh">module</cmd_path>
+      <cmd_path lang="sh">module</cmd_path>
+      <modules>
+        <command name="purge"/>
+        <command name="use">/projects/sems/acme-boca-modulefiles/env-module/</command>
+        <command name="load">acme-boca-env</command>
+        <command name="load">sems-archive-git</command>
+        <command name="load">sems-archive-cmake/3.19.1</command>
+        <command name="load">gnu/10.3.1</command>
+        <command name="load">sems-archive-intel/21.3.0</command>
+      </modules>
+      <modules mpilib="!mpi-serial">
+        <command name="load">sems-archive-openmpi/4.1.4</command>
+        <command name="load">acme-netcdf/4.7.4/acme</command>
+      </modules>
+      <modules mpilib="mpi-serial">
+        <command name="load">sems-archive-netcdf/4.4.1/exo</command>
+      </modules>
+    </module_system>
+    <RUNDIR>/nscratch/$USER/acme_scratch/flight/$CASE/run</RUNDIR>
+    <EXEROOT>$CIME_OUTPUT_ROOT/$CASE/bld</EXEROOT>
+    <!-- complete path to a short term archiving directory -->
+    <!-- path to the cprnc tool used to compare netcdf history files in testing -->
+    <TEST_TPUT_TOLERANCE>0.1</TEST_TPUT_TOLERANCE>
+
+    <environment_variables>
+      <env name="NETCDFROOT">$ENV{SEMS_NETCDF_ROOT}</env>
+      <env name="NETCDF_INCLUDES">$ENV{SEMS_NETCDF_ROOT}/include</env>
+      <env name="NETCDF_LIBS">$ENV{SEMS_NETCDF_ROOT}/lib</env>
+      <env name="OMP_STACKSIZE">64M</env>
+    </environment_variables>
+    <environment_variables mpilib="!mpi-serial">
+      <env name="PNETCDF_PATH">$ENV{SEMS_NETCDF_ROOT}</env>
+    </environment_variables>
+    <environment_variables compiler="intel">
+      <env name="CLDERA_PATH">/projects/cldera/cldera-tools/install-master/intel</env>
+    </environment_variables>
+  </machine>
+
+
   <machine MACH="ghost">
     <DESC>SNL clust</DESC>
     <NODENAME_REGEX>ghost-login</NODENAME_REGEX>
diff --git a/components/eamxx/cmake/machine-files/flight.cmake b/components/eamxx/cmake/machine-files/flight.cmake
new file mode 100644
index 000000000000..8b587c1feed8
--- /dev/null
+++ b/components/eamxx/cmake/machine-files/flight.cmake
@@ -0,0 +1,15 @@
+include(${CMAKE_CURRENT_LIST_DIR}/common.cmake)
+common_setup()
+
+set(EKAT_MACH_FILES_PATH ${CMAKE_CURRENT_LIST_DIR}/../../../../externals/ekat/cmake/machine-files)
+
+# Get AMD arch settings
+include(${EKAT_MACH_FILES_PATH}/kokkos/intel-skx.cmake)
+
+# Add OpenMP settings in standalone mode OR e3sm with compile_threaded=ON
+if (NOT "${PROJECT_NAME}" STREQUAL "E3SM" OR compile_threaded)
+  include(${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake)
+endif()
+
+# Use srun for standalone testing
+include(${EKAT_MACH_FILES_PATH}/mpi/srun.cmake)