From 848e50e6f15291958f4e4a77352c03cd2634f8c6 Mon Sep 17 00:00:00 2001 From: Matthew Whitlock Date: Wed, 13 Nov 2024 18:07:06 -0500 Subject: [PATCH] Swap to Spack and matrix-based GHA for more reliability --- .github/Dockerfile | 29 -------- .github/docker-compose.yml | 118 +++++++++++-------------------- .github/spack.yaml | 31 ++++++++ .github/workflows/ci_checks.yaml | 76 +++++++++++++++----- 4 files changed, 134 insertions(+), 120 deletions(-) delete mode 100644 .github/Dockerfile create mode 100644 .github/spack.yaml diff --git a/.github/Dockerfile b/.github/Dockerfile deleted file mode 100644 index 75e9f40..0000000 --- a/.github/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -#Built for testing, not designed for application use. - -FROM ubuntu:20.04 -#="open-mpi/ompi" for github.com/open-mpi/ompi -ARG OPENMPI_REPO="open-mpi/ompi" -#="tags" or ="heads", for tag or branch name -ARG OPENMPI_VERS_PREFIX="tags" -#="v5.0.0rc10" or ="v5.0.x", ie tag name or branch name. -ARG OPENMPI_VERS="v5.0.0rc10" -run echo Using https://github.com/${OPENMPI_REPO}/git/refs/${OPENMPI_VERS_PREFIX}/${OPENMPI_VERS} - -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential python3 m4 autoconf automake libtool flex git zlib1g-dev - -#Add files listing latest commit for this branch/tag, which invalidates the clone -#when a change has been pushed. -ADD https://api.github.com/repos/${OPENMPI_REPO}/git/refs/${OPENMPI_VERS_PREFIX}/${OPENMPI_VERS} commit_info -RUN git clone --recursive --branch ${OPENMPI_VERS} --depth 1 https://github.com/${OPENMPI_REPO}.git ompi_src && \ - mkdir ompi_build ompi_install && cd ompi_src && export AUTOMAKE_JOBS=8 && ./autogen.pl && cd ../ompi_build && ../ompi_src/configure --prefix=/ompi_install --disable-man-pages --with-ft=ulfm && make install -j8 && cd .. - - -#New build stage, tosses out src/build trees from openmpi -FROM ubuntu:20.04 -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential cmake ssh zlib1g-dev -COPY . ./fenix_src -COPY --from=0 ompi_install/ /ompi_install/ -ENV PATH="$PATH:/ompi_install/bin" -RUN mkdir fenix_build fenix_install && cd fenix_build && cmake ../fenix_src -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=/ompi_install/bin/mpicc \ - -DFENIX_EXAMPLES=ON -DFENIX_TESTS=ON -DCMAKE_INSTALL_PREFIX=../fenix_install -DMPIEXEC_PREFLAGS="--allow-run-as-root;--map-by;:OVERSUBSCRIBE" && make install -j8 -CMD ["sh", "-c", "cd fenix_build && ctest --verbose --timeout 60"] diff --git a/.github/docker-compose.yml b/.github/docker-compose.yml index b29e083..c8c9024 100644 --- a/.github/docker-compose.yml +++ b/.github/docker-compose.yml @@ -1,81 +1,49 @@ -version: "3.9" - -x-fenix: &fenix - build: &fenix-build - context: ./ - dockerfile: .github/Dockerfile - args: - OPENMPI_REPO: open-mpi/ompi - OPENMPI_VERS_PREFIX: tags - OPENMPI_VERS: v5.0.0rc10 - #Caches should be manually scoped, or they'll conflict. - x-bake: - cache-from: - - type=gha,scope=default - cache-to: - - type=gha,scope=default,mode=max - services: - #fenix_ompi_5rc10: - # <<: *fenix - # image: "fenix:ompi_5rc10" - # build: - # <<: *fenix-build - # x-bake: - # cache-from: - # - type=gha,scope=ompi_5rc10 - # cache-to: - # - type=gha,scope=ompi_5rc10,mode=max - - fenix_ompi_5: - <<: *fenix - image: "fenix:ompi_5" + bootstrap: + image: "bootstrap" build: - <<: *fenix-build + dockerfile_inline: | + FROM spack/ubuntu-jammy:0.22.2 + VOLUME /configs + ARG OMPI_VERSION + ENV OMPI_VERSION=$${OMPI_VERSION} + CMD cp /configs/spack.yaml . && \ + spack -e . add openmpi@$${OMPI_VERSION} && \ + spack -e . containerize >/configs/spack.Dockerfile args: - - OPENMPI_VERS_PREFIX=heads - - OPENMPI_VERS=v5.0.x - x-bake: - cache-from: - - type=gha,scope=ompi_5 - cache-to: - - type=gha,scope=ompi_5,mode=max - - fenix_ompi_main: - <<: *fenix - image: "fenix:ompi_main" + OMPI_VERSION: main + no_cache: true + pull_policy: build + volumes: + - .github/:/configs + + env: + image: "ghcr.io/sandialabs/fenix/env:main" build: - <<: *fenix-build - args: - - OPENMPI_VERS_PREFIX=heads - - OPENMPI_VERS=main - x-bake: - cache-from: - - type=gha,scope=ompi_main - cache-to: - - type=gha,scope=ompi_main,mode=max - - fenix_icldisco_latest: - <<: *fenix - image: "fenix:icldisco_latest" + # Generated by running the bootstrap image + dockerfile: .github/spack.Dockerfile + + fenix: + image: "fenix" build: - <<: *fenix-build + dockerfile_inline: | + ARG OMPI_VERSION main + FROM ghcr.io/sandialabs/fenix/env:$${OMPI_VERSION} + COPY . /fenix + RUN . /opt/spack-environment/activate.sh && \ + mkdir -p /fenix/build && \ + cd /fenix/build && \ + cmake /fenix \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=mpicc \ + -DFENIX_EXAMPLES=ON \ + -DFENIX_TESTS=ON \ + -DMPIEXEC_PREFLAGS="--allow-run-as-root;--map-by;:oversubscribe" && \ + make -j + + WORKDIR /fenix/build + ENTRYPOINT ["/entrypoint.sh"] + CMD ["ctest", "--output-on-failure", "--timeout", "60"] args: - - OPENMPI_REPO=icldisco/ompi - - OPENMPI_VERS_PREFIX=heads - - OPENMPI_VERS=ulfm/latest - x-bake: - cache-from: - - type=gha,scope=icldisco_latest - cache-to: - - type=gha,scope=icldisco_latest,mode=max - - #fenix_icldisco_experimental: - # <<: *fenix - # image: fenix/icldisco - # build: - # <<: *fenix-build - # args: - # - OPENMPI_REPO=icldisco/ompi - # - OPENMPI_VERS_PREFIX=heads - # - OPENMPI_VERS=ulfm/experimental + OMPI_VERSION: main + pull_policy: build diff --git a/.github/spack.yaml b/.github/spack.yaml new file mode 100644 index 0000000..c5d3611 --- /dev/null +++ b/.github/spack.yaml @@ -0,0 +1,31 @@ +spack: + packages: + openmpi: + variants: +internal-hwloc +internal-libevent +internal-pmix + concretizer: + unify: true + reuse: true + + container: + format: docker + strip: false + images: + os: ubuntu:22.04 + spack: 0.22.2 + os_packages: + build: + - build-essential + - autotools-dev + - pkg-config + - python3 + - m4 + - autoconf + - automake + - flex + - git + - zlib1g-dev + - libperl-dev + - numactl + final: + - build-essential + - cmake diff --git a/.github/workflows/ci_checks.yaml b/.github/workflows/ci_checks.yaml index ebeeef8..14c376c 100644 --- a/.github/workflows/ci_checks.yaml +++ b/.github/workflows/ci_checks.yaml @@ -11,21 +11,65 @@ on: jobs: test: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + ompi_version: + - main + - 5.0.3 + steps: - - uses: actions/checkout@v3 - - uses: docker/setup-buildx-action@v2 - - name: Build - uses: docker/bake-action@master + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Log in to GHCR container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Bake the bootstrap docker image + uses: docker/bake-action@v5 with: - files: | - .github/docker-compose.yml - load: true - - name: Test open-mpi v5.0.x - if: success() || failure() - run: docker run fenix:ompi_5 - - name: Test open-mpi main - if: success() || failure() - run: docker run fenix:ompi_main - - name: Test icldisco latest - if: success() || failure() - run: docker run fenix:icldisco_latest + files: .github/docker-compose.yml + targets: bootstrap + workdir: . + set: | + *.output=type=docker,name=bootstrap + *.cache-from=type=gha,scope=bootstrap/${{ matrix.ompi_version }} + *.cache-to=type=gha,mode=max,scope=bootstrap/${{ matrix.ompi_version }} + *.args.OMPI_VERSION=${{ matrix.ompi_version }} + + - name: Bootstrap the environment Dockerfile + run: docker run -v ${GITHUB_WORKSPACE}/.github:/configs bootstrap + + - name: Build the environment + uses: docker/bake-action@v5 + with: + files: .github/docker-compose.yml + targets: env + workdir: . + pull: true + set: | + *.cache-from=type=gha,scope=env/${{ matrix.ompi_version }} + *.cache-to=type=gha,mode=max,scope=env/${{ matrix.ompi_version }} + env.tags=ghcr.io/sandialabs/fenix/env:${{ matrix.ompi_version }} + env.output=type=registry,name=ghcr.io/sandialabs/fenix/env:${{ matrix.ompi_version }} + + - name: Build Fenix + uses: docker/bake-action@v5 + with: + source: . + files: .github/docker-compose.yml + targets: fenix + workdir: . + set: | + *.output=type=docker,name=fenix + *.args.OMPI_VERSION=${{ matrix.ompi_version }} + + - name: Test Fenix + run: docker run fenix