From 74fefaa793251d6a2e1ddbf9235a671f93396fdf Mon Sep 17 00:00:00 2001
From: Shane Canon <scanon@lbl.gov>
Date: Sat, 11 May 2024 14:23:39 +0000
Subject: [PATCH] add mpi scripts for spack and intel

---
 exercises/openfoam/mpirun_intel.sh | 37 +++++++++++++++++++++++++++
 exercises/openfoam/mpirun_spack.sh | 40 ++++++++++++++++++++++++++++++
 2 files changed, 77 insertions(+)
 create mode 100755 exercises/openfoam/mpirun_intel.sh
 create mode 100755 exercises/openfoam/mpirun_spack.sh

diff --git a/exercises/openfoam/mpirun_intel.sh b/exercises/openfoam/mpirun_intel.sh
new file mode 100755
index 0000000..158c197
--- /dev/null
+++ b/exercises/openfoam/mpirun_intel.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+NTASKS="2"
+image="library://marcodelapierre/beta/openfoam:v2012"
+
+# this configuration depends on the host
+PATH=/opt/intel/oneapi/mpi/latest/bin:/bin:/usr/bin:/usr/local/bin
+
+export MPICH_ROOT="/opt/intel/oneapi"
+export MPICH_RUN="$( which mpirun )"
+export MPICH_BASE="${MPICH_RUN%/bin/mpirun*}"
+
+export SINGULARITY_BINDPATH="$MPICH_ROOT"
+export SINGULARITYENV_LD_LIBRARY_PATH="$MPICH_BASE/lib/release:$MPICH_BASE/libfabric/lib:\$LD_LIBRARY_PATH"
+
+
+# pre-processing
+singularity exec $image \
+  blockMesh | tee log.blockMesh
+
+singularity exec $image \
+  topoSet | tee log.topoSet
+
+singularity exec $image \
+  decomposePar -fileHandler uncollated | tee log.decomposePar
+
+
+# run OpenFoam with MPI
+mpirun -n $NTASKS \
+  singularity exec $image \
+  simpleFoam -fileHandler uncollated -parallel | tee log.simpleFoam
+
+
+# post-processing
+singularity exec $image \
+  reconstructPar -latestTime -fileHandler uncollated | tee log.reconstructPar
+
diff --git a/exercises/openfoam/mpirun_spack.sh b/exercises/openfoam/mpirun_spack.sh
new file mode 100755
index 0000000..6d49945
--- /dev/null
+++ b/exercises/openfoam/mpirun_spack.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+
+PATH=/opt/spack/opt/spack/spack_path_placeholder/spack_path_placeholder/spack_path_placeholder/spack_path_placeholder/s/linux-amzn2-x86_64/gcc-7.3.1/mpich-3.3.2-q546vjf5teafqxsttapjokurnarsx5iz/bin/:$PATH
+
+NTASKS="2"
+image="library://marcodelapierre/beta/openfoam:v2012"
+
+# this configuration depends on the host
+export MPICH_ROOT="/opt/spack"
+#export MPICH_ROOT="/opt/intel/oneapi"
+export MPICH_LIBS="$( which mpirun )"
+export MPICH_LIBS="${MPICH_LIBS%/bin/mpirun*}/lib"
+
+export SINGULARITY_BINDPATH="$MPICH_ROOT"
+export SINGULARITYENV_LD_LIBRARY_PATH="$MPICH_LIBS:$MPICH_LIBS/libfabric:\$LD_LIBRARY_PATH"
+
+
+
+# pre-processing
+singularity exec $image \
+  blockMesh | tee log.blockMesh
+
+singularity exec $image \
+  topoSet | tee log.topoSet
+
+singularity exec $image \
+  decomposePar -fileHandler uncollated | tee log.decomposePar
+
+
+# run OpenFoam with MPI
+mpirun -n $NTASKS \
+  singularity exec $image \
+  simpleFoam -fileHandler uncollated -parallel | tee log.simpleFoam
+
+
+# post-processing
+singularity exec $image \
+  reconstructPar -latestTime -fileHandler uncollated | tee log.reconstructPar
+