From 1e36c75f65de46f0876e4b5634aa8ec6a383b40f Mon Sep 17 00:00:00 2001 From: "Christopher H. Jordan" Date: Wed, 27 Sep 2023 10:10:31 +0800 Subject: [PATCH] Release v0.3.0 GitHub actions doesn't support ubuntu-18.04 any more, so binaries are now built with ubuntu-20.04. Update the CI scripts. --- .github/workflows/deploy-mdbook.yml | 2 +- .github/workflows/releases-readme.md | 14 ++--- .github/workflows/releases.yml | 87 ++++++++++++++++++++-------- .github/workflows/run-tests.yml | 37 +++++++----- CHANGELOG.md | 2 +- Cargo.lock | 2 +- Cargo.toml | 4 +- 7 files changed, 98 insertions(+), 50 deletions(-) diff --git a/.github/workflows/deploy-mdbook.yml b/.github/workflows/deploy-mdbook.yml index 96ea6478..db027e16 100644 --- a/.github/workflows/deploy-mdbook.yml +++ b/.github/workflows/deploy-mdbook.yml @@ -9,7 +9,7 @@ jobs: name: Build, Test and Deploy runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions-rs/toolchain@v1 with: toolchain: stable diff --git a/.github/workflows/releases-readme.md b/.github/workflows/releases-readme.md index 899aa6ef..cdbba248 100644 --- a/.github/workflows/releases-readme.md +++ b/.github/workflows/releases-readme.md @@ -31,23 +31,23 @@ compiled inside the `hyperdrive` binary. An NVIDIA license may also be included as, per the terms of the license, `hyperdrive` utilises code that was modified from an existing CUDA example. -# x86-64-v3? +# What are these different x86-64 versions? -This is a [microarchitecture -level](https://en.wikipedia.org/wiki/X86-64#Microarchitecture_levels). By +They are [microarchitecture +levels](https://en.wikipedia.org/wiki/X86-64#Microarchitecture_levels). By default, Rust compiles for all x86-64 CPUs; this allows maximum compatibility, but potentially limits the runtime performance because many modern CPU features can't be used. Compiling at different levels allows the code to be optimised for different classes of CPUs so users can get something that works best for them. -If your CPU does not support x86-64-v3, you will need to compile `hyperdrive` -from source. +If your CPU does not support the available microarchitecture levels on the +GitHub release page, you will need to compile `hyperdrive` from source. # CUDA? The releases with "CUDA" in the name are CUDA enabled. The `hyperdrive` binaries -have been dynamically linked against CUDA 11.2.0; to run them, a CUDA -installation on version 11 is required. +have been dynamically linked against CUDA 12.2.0; to run them, a CUDA +installation on version 12 is required. There is also a double- or single-precision version of `hyperdrive` provided. If you're running a desktop NVIDIA GPU (e.g. RTX 2070), then you probably want the diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index fae5853d..99f6cc1d 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -14,11 +14,11 @@ env: jobs: build-linux-release: name: Build release for Linux - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Checkout sources - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 @@ -32,14 +32,39 @@ jobs: cmake \ curl \ git \ + libz-dev \ + software-properties-common \ ; + + # This comes from software-properties-common, and is only necessary + # to install gcc-12. Remove this when we use a newer ubuntu image. + # https://stackoverflow.com/a/67453352 + sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + sudo apt update -y + sudo apt install -y gcc-11 g++-11 + sudo update-alternatives \ + --install /usr/bin/gcc gcc /usr/bin/gcc-11 110 \ + --slave /usr/bin/g++ g++ /usr/bin/g++-11 \ + --slave /usr/bin/gcov gcov /usr/bin/gcov-11 \ + --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-11 \ + --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-11 + gcc --version + g++ --version + gcov --version + + # Compile cfitsio from source to get its static library. + curl https://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/cfitsio-4.3.0.tar.gz -O + tar -xvf cfitsio-4.3.0.tar.gz && cd cfitsio-4.3.0 + CFLAGS="-fPIE -O3 -march=x86-64-v2" ./configure --enable-reentrant --disable-curl --enable-sse2 --enable-ssse3 --libdir=/usr/lib/x86_64-linux-gnu/ + sudo make -j install + env: DEBIAN_FRONTEND: noninteractive - name: Install CUDA - uses: Jimver/cuda-toolkit@v0.2.5 + uses: Jimver/cuda-toolkit@v0.2.11 with: - cuda: '11.2.0' + cuda: '12.2.0' - name: Install stable toolchain uses: actions-rs/toolchain@v1 @@ -55,25 +80,41 @@ jobs: curl https://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/License.txt -o LICENSE-cfitsio cp .github/workflows/releases-readme.md README.md - cargo build --profile production --locked --no-default-features --features=hdf5-static,cfitsio-static - mv target/production/hyperdrive . - tar -acvf mwa_hyperdrive-$(git describe --tags)-Linux-x86-64-v3.tar.gz \ - LICENSE COPYING-hdf5 LICENSE-erfa LICENSE-cfitsio README.md \ - hyperdrive - - cargo build --profile production --locked --no-default-features --features=hdf5-static,cfitsio-static,cuda - mv target/production/hyperdrive . - tar -acvf mwa_hyperdrive-$(git describe --tags)-Linux-x86-64-v3-CUDA-double.tar.gz \ - LICENSE COPYING-hdf5 LICENSE-erfa LICENSE-cfitsio LICENSE-NVIDIA README.md \ - hyperdrive - - cargo build --profile production --locked --no-default-features --features=hdf5-static,cfitsio-static,cuda,gpu-single - mv target/production/hyperdrive . - tar -acvf mwa_hyperdrive-$(git describe --tags)-Linux-x86-64-v3-CUDA-single.tar.gz \ - LICENSE COPYING-hdf5 LICENSE-erfa LICENSE-cfitsio LICENSE-NVIDIA README.md \ - hyperdrive - env: - RUSTFLAGS: "-C target-cpu=x86-64-v3" + # Build with our recently-built static cfitsio + export CFITSIO_STATIC=1 + # Don't build against the host CPU + rm .cargo/config.toml + # Don't make it look like the git tree is dirty because we aren't + # building against the host CPU + perl -0777 -pe 's/let dirty = .*\n.*\n.*\n.*/let dirty = "";/' src/cli/mod.rs + + for ARCH in {2,3}; do + echo "Building x86-64-v${ARCH}" + export CFLAGS="-march=x86-64-v${ARCH}" + export CXXFLAGS="-march=x86-64-v${ARCH}" + export RUSTFLAGS="-C target-cpu=x86-64-v${ARCH}" + + # Use --no-default-features to disable plotting; it's too hard to make this portable. + cargo build --profile production --locked --no-default-features --features=hdf5-static + mv target/production/hyperdrive . + tar -acvf mwa_hyperdrive-$(git describe --tags)-Linux-x86-64-v${ARCH}.tar.gz \ + LICENSE COPYING-hdf5 LICENSE-erfa LICENSE-cfitsio README.md \ + hyperdrive + + cargo build --profile production --locked --no-default-features --features=hdf5-static,cuda + mv target/production/hyperdrive . + tar -acvf mwa_hyperdrive-$(git describe --tags)-Linux-x86-64-v${ARCH}-CUDA-double.tar.gz \ + LICENSE COPYING-hdf5 LICENSE-erfa LICENSE-cfitsio LICENSE-NVIDIA README.md \ + hyperdrive + + cargo build --profile production --locked --no-default-features --features=hdf5-static,cuda,gpu-single + mv target/production/hyperdrive . + tar -acvf mwa_hyperdrive-$(git describe --tags)-Linux-x86-64-v${ARCH}-CUDA-single.tar.gz \ + LICENSE COPYING-hdf5 LICENSE-erfa LICENSE-cfitsio LICENSE-NVIDIA README.md \ + hyperdrive + + cargo clean + done - name: Upload tarball uses: actions/upload-artifact@v2 diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index c92bb76b..19a10b7a 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -10,9 +10,13 @@ on: branches: - '**' +# The CI jobs can use a lot of disk space; we take tips from this site to run +# the jobs faster and use less space. +# https://matklad.github.io/2021/09/04/fast-rust-builds.html env: CARGO_TERM_COLOR: always CARGO_INCREMENTAL: 0 + RUSTFLAGS: "-C debuginfo=0 --deny warnings" MWA_BEAM_FILE: /mwa_full_embedded_element_pattern.h5 jobs: @@ -20,7 +24,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout sources - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 @@ -44,9 +48,9 @@ jobs: DEBIAN_FRONTEND: noninteractive - name: Install CUDA - uses: Jimver/cuda-toolkit@v0.2.5 + uses: Jimver/cuda-toolkit@v0.2.11 with: - cuda: '11.5.1' + cuda: '12.2.0' - name: Install stable toolchain uses: actions-rs/toolchain@v1 @@ -58,29 +62,32 @@ jobs: - name: Get the beam HDF5 file run: sudo curl http://ws.mwatelescope.org/static/mwa_full_embedded_element_pattern.h5 -o $MWA_BEAM_FILE - # Test that building works. Build statically first so less compiling is - # needed when running tests. - - name: Debug build with static features + # Test that building works before running tests. Build statically first so + # less compiling is needed when running tests. + - name: Build with static features run: cargo test --locked --release --features=cuda,all-static --no-run - - name: Debug build + # Can't test with CUDA, no GPUs available in CI + - name: Build run: cargo test --locked --release --features=cuda --no-run - # Can't test with CUDA, no GPUs available in CI - name: Run tests run: cargo test --locked --release - name: Run ignored tests - run: cargo test --locked --release -- --ignored + run: | + # It seems that we use up a lot of space by this point. + cargo clean + cargo test --locked --release -- --ignored - name: Minimum-specified Rust version works run: | MIN_RUST=$(grep -m1 "rust-version" Cargo.toml | sed 's|.*\"\(.*\)\"|\1|') ~/.cargo/bin/rustup install $MIN_RUST --profile minimal cargo clean - cargo +${MIN_RUST} test --locked - cargo +${MIN_RUST} test --locked --features=all-static - # Can't test with --all-features, cuda and hip aren't allowed together - # hip is also difficult to install so ignore it - cargo +${MIN_RUST} test --locked --features=all-static,cuda --no-run - cargo +${MIN_RUST} test --locked --features=all-static,cuda,gpu-single --no-run + cargo +${MIN_RUST} test --locked --release + cargo +${MIN_RUST} test --locked --release --features=all-static + # Can't test with --all-features because cuda and hip aren't allowed + # together. hip is also difficult to install so ignore it. + cargo +${MIN_RUST} test --locked --release --features=all-static,cuda --no-run + cargo +${MIN_RUST} test --locked --release --features=all-static,cuda,gpu-single --no-run diff --git a/CHANGELOG.md b/CHANGELOG.md index 207b5895..4056e2a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.3.0] - Unreleased +## [0.3.0] - 2023-09-27 ### Added - Support for HIP, which allows AMD GPUs to be used instead of only NVIDIA GPUs via CUDA. diff --git a/Cargo.lock b/Cargo.lock index b7845b1b..85cf75af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1486,7 +1486,7 @@ dependencies = [ [[package]] name = "mwa_hyperdrive" -version = "0.2.1" +version = "0.3.0" dependencies = [ "approx", "assert_cmd", diff --git a/Cargo.toml b/Cargo.toml index 42ab3ed6..aef75e9a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,12 +1,12 @@ [package] name = "mwa_hyperdrive" -version = "0.2.1" +version = "0.3.0" authors = [ "Christopher H. Jordan ", "Dev Null ", ] edition = "2021" -rust-version = "1.65" +rust-version = "1.68" license = "MPL-2.0" readme = "README.md" description = "Calibration software for the Murchison Widefield Array (MWA) radio telescope"