From a80c25c431c4e8f303b488c5f5b4016ede3bb525 Mon Sep 17 00:00:00 2001 From: d3v-null Date: Wed, 25 Sep 2024 08:45:22 +0800 Subject: [PATCH] docker onto python3.11-bookworm, mwalib, test shim, clippy CI --- .dockerignore | 5 ++ .github/workflows/clippy.yml | 31 +++++++++ .github/workflows/docker_test.yml | 33 +--------- Dockerfile | 102 +++++++++++++++++++++++------- 4 files changed, 116 insertions(+), 55 deletions(-) create mode 100644 .github/workflows/clippy.yml diff --git a/.dockerignore b/.dockerignore index 4a5f340..86afc32 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,3 +6,8 @@ doc img scripts target +*.ms +*.uvfits +*.tar +*.metafits +*.fits diff --git a/.github/workflows/clippy.yml b/.github/workflows/clippy.yml new file mode 100644 index 0000000..11370b4 --- /dev/null +++ b/.github/workflows/clippy.yml @@ -0,0 +1,31 @@ +on: + push: + tags-ignore: + - "**" + branches: + - "**" + pull_request: + +name: clippy + +jobs: + coverage: + runs-on: ubuntu-latest + container: mwatelescope/birli:latest + steps: + - name: Checkout sources + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Install stable toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + profile: minimal + override: true + components: rustfmt, clippy + - name: Run cargo check, fmt, clippy + run: | + cargo check --all + cargo fmt --all -- --check + cargo clippy --all -- -D warnings \ No newline at end of file diff --git a/.github/workflows/docker_test.yml b/.github/workflows/docker_test.yml index 308afab..5237273 100644 --- a/.github/workflows/docker_test.yml +++ b/.github/workflows/docker_test.yml @@ -16,40 +16,10 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push - uses: docker/build-push-action@v6 - with: - platforms: linux/amd64,linux/arm64 - push: false - tags: mwatelescope/birli:latest - cache-from: type=gha - cache-to: type=gha,mode=max - check: - name: Docker Test and Release - needs: build - runs-on: ubuntu-latest - container: mwatelescope/birli:latest - steps: - name: Checkout sources uses: actions/checkout@v3 with: fetch-depth: 0 - - run: /opt/cargo/bin/cargo check - - run: /opt/cargo/bin/cargo fmt --all -- --check - - run: /opt/cargo/bin/cargo clippy --all-targets --all-features -- -D warnings - - run: /opt/cargo/bin/cargo test --release - - run: /opt/cargo/bin/cargo test --no-default-features --release --features=cli - push: - needs: build # TODO: needs: check - runs-on: ubuntu-latest - steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and push uses: docker/build-push-action@v6 with: @@ -57,4 +27,5 @@ jobs: push: true tags: mwatelescope/birli:latest cache-from: type=gha - cache-to: type=gha,mode=max \ No newline at end of file + cache-to: type=gha,mode=max + buildArgs: TEST_SHIM="cargo test --no-default-features; cargo test; cargo clean; rm -rf target" \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index af068e2..6482156 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,47 +1,101 @@ -FROM ubuntu:24.04 +FROM python:3.11-bookworm as base -ENV DEBIAN_FRONTEND=noninteractive +# suppress perl locale errors ENV LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8 -ARG DEBUG +# suppress apt-get prompts +ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update \ - && apt-get install -y \ - aoflagger-dev \ + && apt-get install -y --no-install-recommends \ + automake \ build-essential \ + casacore-data \ + casacore-dev \ clang \ + cmake \ curl \ - git \ jq \ lcov \ + libblas-dev \ + libboost-date-time-dev \ + libboost-system-dev \ libcfitsio-dev \ liberfa-dev \ + libfftw3-dev \ + libgsl-dev \ + libgtkmm-3.0-dev \ + libhdf5-serial-dev \ + liblapack-dev \ + liblua5.3-dev \ + libpng-dev \ libssl-dev \ + libtool \ pkg-config \ unzip \ zip \ - automake \ - libtool + 2>&1 | tee /birli-apt.txt \ + && \ + apt-get clean all && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ + apt-get -y autoremove -RUN test -z "$DEBUG" || ( \ - apt-get install -y vim gdb \ +# Get Rust +ARG RUST_VERSION=stable +ENV RUSTUP_HOME=/opt/rust CARGO_HOME=/opt/cargo +ENV PATH="${CARGO_HOME}/bin:${PATH}" +RUN mkdir -m755 $RUSTUP_HOME $CARGO_HOME && ( \ + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | env RUSTUP_HOME=$RUSTUP_HOME CARGO_HOME=$CARGO_HOME sh -s -- -y \ + --profile=minimal \ + --component llvm-tools \ + --default-toolchain=${RUST_VERSION} \ ) -RUN apt-get clean \ - && rm -rf /var/lib/apt/lists/* +# # Get cargo make, llvm-cov +RUN cargo install --force cargo-make cargo-llvm-cov && \ + rm -rf ${CARGO_HOME}/registry -# Get Rust -RUN mkdir -m755 /opt/rust /opt/cargo -ENV RUSTUP_HOME=/opt/rust CARGO_HOME=/opt/cargo PATH=/opt/cargo/bin:$PATH -# install latest stable rust toolchian, with llvm-tools-preview (for coverage) -ENV RUST_VERSION=stable -RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain=$RUST_VERSION -RUN rustup toolchain install $RUST_VERSION --component llvm-tools-preview -# Get cargo make, llvm-cov -RUN /opt/cargo/bin/cargo install --force cargo-make cargo-llvm-cov +RUN python -m pip install --no-cache-dir maturin[patchelf]==1.7.0 + +ARG MWALIB_BRANCH=v1.5.0 +RUN git clone --depth 1 --branch=${MWALIB_BRANCH} https://github.com/MWATelescope/mwalib.git /mwalib && \ + cd /mwalib && \ + maturin build --release --features=python && \ + python -m pip install $(ls -1 target/wheels/*.whl | tail -n 1) && \ + cd / && \ + rm -rf /mwalib ${CARGO_HOME}/registry + +# for example, CMAKE_ARGS="-D CMAKE_CXX_FLAGS='-march=native -mtune=native -O3 -fomit-frame-pointer'" +ARG CMAKE_ARGS="-D PORTABLE=ON" + +ARG AOFLAGGER_BRANCH=v3.4.0 +RUN git clone --depth 1 --branch=${AOFLAGGER_BRANCH} --recurse-submodules https://gitlab.com/aroffringa/aoflagger.git /aoflagger && \ + cd /aoflagger && \ + mkdir build && \ + cd build && \ + cmake $CMAKE_ARGS \ + -DENABLE_GUI=OFF \ + .. && \ + make install -j`nproc` && \ + ldconfig && \ + cd / && \ + rm -rf /aoflagger +# set up aoflagger python library +ENV PYTHONPATH="/usr/local/lib/:$PYTHONPATH" ADD . /app WORKDIR /app -RUN cargo clean \ - && cargo install --path . --features aoflagger --locked $(test -z "$DEBUG" || echo "--debug") \ - && cargo clean +ARG DEBUG +ARG BIRLI_FEATURES=cli,aoflagger +RUN cargo install --path . --no-default-features --features=${BIRLI_FEATURES} --locked $(test -z "$DEBUG" || echo "--debug") && \ + rm -rf /app/target ${CARGO_HOME}/registry + +RUN test -z "$DEBUG" || ( \ + apt-get install -y vim gdb && \ + apt-get clean all && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ + apt-get -y autoremove \ + ) + +ARG TEST_SHIM +RUN ${TEST_SHIM} ENTRYPOINT [ "/opt/cargo/bin/birli" ]