add pennant with gpu/flux

Signed-off-by: vsoch <[email protected]>

pennant/Dockerfile.flux-gpu

@@ -0,0 +1,197 @@
+FROM nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Build command
+#  docker build -f Dockerfile.flux-gpu -t ghcr.io/converged-computing/metric-pennant:flux-gpu .
+#  docker push ghcr.io/converged-computing/metric-pennant:flux-gpu
+
+# Example command
+# pennant /opt/pennant/test/sedovsmall/sedovsmall.pnt 
+
+# Utilities
+RUN apt-get update && \
+    apt-get -qq install -y --no-install-recommends \
+        apt-utils \
+        locales \
+        ca-certificates \
+        wget \
+        man \
+        git \
+        flex \
+        ssh \
+        sudo \
+        vim \
+        luarocks \
+        munge \
+        lcov \
+        ccache \
+        lua5.2 \
+        mpich \
+        python3-dev \
+        python3-pip \
+        valgrind \
+        jq && \
+    rm -rf /var/lib/apt/lists/*
+
+# Compilers, autotools
+RUN apt-get update && \
+    apt-get -qq install -y --no-install-recommends \
+        build-essential \
+        pkg-config \
+        autotools-dev \
+        libtool \
+	libffi-dev \
+        autoconf \
+        automake \
+        make \
+        clang \
+        clang-tidy \
+        gcc \
+        g++ && \
+    rm -rf /var/lib/apt/lists/*
+
+# Python - instead of a system python we install anaconda
+# https://docs.conda.io/en/latest/miniconda.html#linux-installers
+RUN pip install --upgrade --ignore-installed \
+        "markupsafe==2.0.0" \
+        coverage cffi ply six pyyaml "jsonschema>=2.6,<4.0" \
+        sphinx sphinx-rtd-theme sphinxcontrib-spelling 
+
+RUN apt-get update && \
+    apt-get -qq install -y --no-install-recommends \
+        libsodium-dev \
+        libzmq3-dev \
+        libczmq-dev \
+        libjansson-dev \
+        libmunge-dev \
+        libncursesw5-dev \
+        liblua5.2-dev \
+        liblz4-dev \
+        libsqlite3-dev \
+        uuid-dev \
+        libhwloc-dev \
+        libmpich-dev \
+        libs3-dev \
+        libevent-dev \
+        libarchive-dev \
+        libpam-dev && \
+    rm -rf /var/lib/apt/lists/*
+
+# Testing utils and libs
+RUN apt-get update && \
+    apt-get -qq install -y --no-install-recommends \
+        faketime \
+        libfaketime \
+        pylint \
+        cppcheck \
+        enchant-2 \
+        aspell \
+        aspell-en && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN locale-gen en_US.UTF-8
+
+# NOTE: luaposix installed by rocks due to Ubuntu bug: #1752082 https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
+RUN luarocks install luaposix
+
+# Install openpmix, prrte
+WORKDIR /opt/prrte
+RUN git clone https://github.com/openpmix/openpmix.git && \
+    git clone https://github.com/openpmix/prrte.git && \
+    ls -l && \
+    set -x && \
+    cd openpmix && \
+    git checkout fefaed568f33bf86f28afb6e45237f1ec5e4de93 && \
+    ./autogen.pl && \
+    ./configure --prefix=/usr --disable-static && make -j 4 install && \
+    ldconfig && \
+    cd .. && \
+    cd prrte && \
+    git checkout 477894f4720d822b15cab56eee7665107832921c && \
+    ./autogen.pl && \
+    ./configure --prefix=/usr && make -j 4 install && \
+    cd ../.. && \
+    rm -rf prrte
+
+ENV LANG=C.UTF-8
+
+ARG FLUX_SECURITY_VERSION=0.11.0
+
+WORKDIR /opt
+RUN CCACHE_DISABLE=1 && \
+    V=$FLUX_SECURITY_VERSION && \
+    PKG=flux-security-$V && \
+    URL=https://github.com/flux-framework/flux-security/releases/download && \
+    wget ${URL}/v${V}/${PKG}.tar.gz && \
+    tar xvfz ${PKG}.tar.gz && \
+    cd ${PKG} && \
+    ./configure --prefix=/usr --sysconfdir=/etc || cat config.log && \
+    make -j 4 && \
+    make install && \
+    cd .. && \
+    rm -rf flux-security-*
+
+
+# Setup MUNGE directories & key
+RUN mkdir -p /var/run/munge && \
+    dd if=/dev/urandom bs=1 count=1024 > /etc/munge/munge.key && \
+    chown -R munge /etc/munge/munge.key /var/run/munge && \
+    chmod 600 /etc/munge/munge.key
+
+RUN wget https://github.com/flux-framework/flux-core/releases/download/v0.61.2/flux-core-0.61.2.tar.gz && \
+    tar xzvf flux-core-0.61.2.tar.gz && \
+    cd flux-core-0.61.2 && \
+    ./configure --prefix=/usr --sysconfdir=/etc && \
+    make clean && \
+    make && \
+    sudo make install
+
+RUN sudo apt-get update
+RUN sudo apt-get -qq install -y --no-install-recommends \
+	libboost-graph-dev \
+	libboost-system-dev \
+	libboost-filesystem-dev \
+	libboost-regex-dev \
+	libyaml-cpp-dev \
+	libedit-dev \
+        libboost-dev \
+        libyaml-cpp-dev \
+	curl
+
+ENV LD_LIBRARY_PATH=/opt/miniconda/lib:$LD_LIBRARY_PATH
+
+ENV CMAKE=3.23.1
+RUN curl -s -L https://github.com/Kitware/CMake/releases/download/v$CMAKE/cmake-$CMAKE-linux-x86_64.sh > cmake.sh && \
+    sudo sh cmake.sh --prefix=/usr/local --skip-license
+
+RUN wget https://github.com/flux-framework/flux-sched/releases/download/v0.33.1/flux-sched-0.33.1.tar.gz && \
+    tar -xzvf flux-sched-0.33.1.tar.gz && \
+    cd flux-sched-0.33.1 && \
+    ./configure --prefix=/usr --sysconfdir=/etc && \
+    make && \
+    sudo make install && \
+    ldconfig
+
+RUN apt-get update && \
+    apt-get install -y fftw3-dev fftw3 pdsh libfabric-dev libfabric1 \
+        openssh-client openssh-server \
+        dnsutils telnet strace git g++ \
+        mpich unzip bzip2
+
+
+WORKDIR /opt
+COPY ./PENNANT /opt/pennant
+WORKDIR /opt/pennant
+RUN make && mv ./build/pennant /usr/bin/pennant
+
+# Hack for ssh for now...
+RUN echo "        LogLevel ERROR" >> /etc/ssh/ssh_config && \
+    echo "        StrictHostKeyChecking no" >> /etc/ssh/ssh_config && \
+    echo "        UserKnownHostsFile=/dev/null" >> /etc/ssh/ssh_config && \
+    cd /root && \
+    mkdir -p /run/sshd && \
+    ssh-keygen -t rsa -N "" -f /root/.ssh/id_rsa && chmod og+rX . && \
+    cd .ssh && cat id_rsa.pub > authorized_keys && chmod 644 authorized_keys
+
+EXPOSE 22
+WORKDIR /opt/pennant/test

pennant/Dockerfile.gpu

@@ -0,0 +1,31 @@
+FROM nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Build command
+#  docker build -f Dockerfile.gpu -t ghcr.io/converged-computing/metrics-pennant:gpu .
+
+# Example command
+# pennant /opt/pennant/test/sedovsmall/sedovsmall.pnt 
+
+RUN apt-get update && \
+    apt-get install -y fftw3-dev fftw3 pdsh libfabric-dev libfabric1 \
+        openssh-client openssh-server \
+        dnsutils telnet strace cmake git g++ \
+        mpich unzip bzip2
+
+WORKDIR /opt
+COPY ./PENNANT /opt/pennant
+WORKDIR /opt/pennant
+RUN make && mv ./build/pennant /usr/bin/pennant
+
+# Hack for ssh for now...
+RUN echo "        LogLevel ERROR" >> /etc/ssh/ssh_config && \
+    echo "        StrictHostKeyChecking no" >> /etc/ssh/ssh_config && \
+    echo "        UserKnownHostsFile=/dev/null" >> /etc/ssh/ssh_config && \
+    cd /root && \
+    mkdir -p /run/sshd && \
+    ssh-keygen -t rsa -N "" -f /root/.ssh/id_rsa && chmod og+rX . && \
+    cd .ssh && cat id_rsa.pub > authorized_keys && chmod 644 authorized_keys
+
+EXPOSE 22
+WORKDIR /opt/pennant/test

pennant/PENNANT/LICENSE

@@ -0,0 +1,44 @@
+Copyright (c) 2012, Los Alamos National Security, LLC.
+All rights reserved.
+
+Copyright 2012. Los Alamos National Security, LLC.
+This software was produced under U.S. Government contract
+DE-AC52-06NA25396 for Los Alamos National Laboratory (LANL), which is
+operated by Los Alamos National Security, LLC for the U.S. Department
+of Energy. The U.S. Government has rights to use, reproduce, and
+distribute this software. NEITHER THE GOVERNMENT NOR LOS ALAMOS
+NATIONAL SECURITY, LLC MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR
+ASSUMES ANY LIABILITY FOR THE USE OF THIS SOFTWARE. If software is
+modified to produce derivative works, such modified software should be
+clearly marked, so as not to confuse it with the version available from
+LANL.
+
+Additionally, redistribution and use in source and binary forms, with
+or without modification, are permitted provided that the following
+conditions are met:
+
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above
+   copyright notice, this list of conditions and the following
+   disclaimer in the documentation and/or other materials provided
+   with the distribution.
+
+3. Neither the name of Los Alamos National Security, LLC, Los Alamos
+   National Laboratory, LANL, the U.S. Government, nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY LOS ALAMOS NATIONAL SECURITY, LLC AND
+CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,
+BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL LOS ALAMOS
+NATIONAL SECURITY, LLC OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.

pennant/PENNANT/Makefile

@@ -0,0 +1,97 @@
+BUILDDIR := build
+PRODUCT := pennant
+CUDA_INSTALL_PATH := /usr/local/cuda-12.4/targets/x86_64-linux
+
+SRCDIR := src
+
+HDRS := $(wildcard $(SRCDIR)/*.hh)
+SRCS := $(wildcard $(SRCDIR)/*.cc)
+OBJS := $(SRCS:$(SRCDIR)/%.cc=$(BUILDDIR)/%.o)
+DEPS := $(SRCS:$(SRCDIR)/%.cc=$(BUILDDIR)/%.d)
+
+HDRS += $(SRCDIR)/HydroGPU.hh
+SRCS += $(SRCDIR)/HydroGPU.cu
+OBJS += $(BUILDDIR)/HydroGPU.o
+DEPS += $(BUILDDIR)/HydroGPU.d
+
+BINARY := $(BUILDDIR)/$(PRODUCT)
+
+CPPFLAGS := -I.
+
+# begin compiler-dependent flags
+#
+# gcc flags:
+CXX := g++
+CXXFLAGS_DEBUG := -g
+CXXFLAGS_OPT := -O3
+CXXFLAGS_OPENMP := -fopenmp
+
+# intel flags:
+# CXX := icpc
+# CXXFLAGS_DEBUG := -g
+# CXXFLAGS_OPT := -O3 -fast -fno-alias
+# CXXFLAGS_OPENMP := -openmp
+
+# pgi flags:
+#CXX := pgCC
+#CXXFLAGS_DEBUG := -g
+#CXXFLAGS_OPT := -O3 -fastsse
+#CXXFLAGS_OPENMP := -mp
+
+# end compiler-dependent flags
+
+CUDAC := nvcc
+CUDACFLAGS := -arch=sm_80 --ptxas-options=-v
+CUDACFLAGS_DEBUG := -G -lineinfo
+CUDACFLAGS_OPT := -O3
+
+LD := $(CXX)
+LDFLAGS := -L$(CUDA_INSTALL_PATH)/lib -lcudart
+
+# select optimized or debug
+CXXFLAGS := $(CXXFLAGS_OPT) $(CPPFLAGS)
+CUDACFLAGS += $(CUDACFLAGS_OPT) $(CPPFLAGS)
+#CXXFLAGS := $(CXXFLAGS_DEBUG) $(CPPFLAGS)
+#CUDACFLAGS += $(CUDACFLAGS_DEBUG) $(CPPFLAGS)
+
+# add openmp flags (comment out for serial build)
+#CXXFLAGS += $(CXXFLAGS_OPENMP)
+#LDFLAGS += $(CXXFLAGS_OPENMP)
+
+all : $(BINARY)
+
+-include $(DEPS)
+
+$(BINARY) : $(OBJS)
+	@echo linking $@
+	$(maketargetdir)
+	$(LD) -o $@ $^ $(LDFLAGS)
+
+$(BUILDDIR)/%.o : $(SRCDIR)/%.cc
+	@echo compiling $<
+	$(maketargetdir)
+	$(CXX) $(CXXFLAGS) $(CXXINCLUDES) -c -o $@ $<
+
+$(BUILDDIR)/%.o : $(SRCDIR)/%.cu
+	@echo compiling $<
+	$(maketargetdir)
+	@# unsetting of CPATH is needed to make nvcc and icpc
+	@# play nicely together
+	(CPATH=;$(CUDAC) $(CUDACFLAGS) $(CUDACINCLUDES) -c -o $@ $<)
+
+$(BUILDDIR)/%.d : $(SRCDIR)/%.cc
+	@echo making depends for $<
+	$(maketargetdir)
+	@$(CXX) $(CXXFLAGS) $(CXXINCLUDES) -M $< | sed "1s![^ \t]\+\.o!$(@:.d=.o) $@!" >$@
+
+$(BUILDDIR)/%.d : $(SRCDIR)/%.cu
+	@echo making depends for $<
+	$(maketargetdir)
+	@$(CUDAC) $(CUDACFLAGS) $(CUDACINCLUDES) -M $< | sed "1s![^ \t]\+\.o!$(@:.d=.o) $@!" >$@
+
+define maketargetdir
+	-@mkdir -p $(dir $@) > /dev/null 2>&1
+endef
+
+clean :
+	rm -f $(BINARY) $(OBJS) $(DEPS)