From c52aa0db52643a7995ca87033e606c396cd4702e Mon Sep 17 00:00:00 2001 From: Marcus Holland-Moritz Date: Mon, 12 Aug 2024 19:43:53 +0200 Subject: [PATCH] build: enable builds from source tarball in CI --- .docker/build-linux.sh | 220 +++++++++++++++++++++++------------- .github/workflows/build.yml | 102 ++++++++++++++++- 2 files changed, 242 insertions(+), 80 deletions(-) diff --git a/.docker/build-linux.sh b/.docker/build-linux.sh index 047e6ba63..1bb310902 100755 --- a/.docker/build-linux.sh +++ b/.docker/build-linux.sh @@ -6,10 +6,20 @@ export CCACHE_DIR=/ccache cd "$HOME" -git config --global --add safe.directory /workspace - -rm -f dwarfs -ln -s /workspace dwarfs +rm -rf dwarfs dwarfs-* + +if [[ "$BUILD_FROM_TARBALL" == "1" ]]; then + # delete all but the latest tarball + ls -1 -r --sort=time /workspace/dwarfs-*.tar.zst | tail -n +2 | while read -r tarball; do + echo "deleting old tarball: $tarball" + rm -f "$tarball" + done + tar xvf /workspace/dwarfs-*.tar.zst + ln -s dwarfs-* dwarfs +else + git config --global --add safe.directory /workspace + ln -s /workspace dwarfs +fi rm -rf build mkdir build @@ -132,106 +142,158 @@ if [[ "-$BUILD_TYPE-" == *-static-* ]]; then CMAKE_ARGS="${CMAKE_ARGS} -DSTATIC_BUILD_EXTRA_PREFIX=/opt/static-libs/$COMPILER" fi -# shellcheck disable=SC2086 -cmake ../dwarfs/ $CMAKE_ARGS +if [[ "$BUILD_FROM_TARBALL" == "1" ]]; then + INSTALLDIR="$HOME/install" + PREFIXPATH="$INSTALLDIR/usr/local" + rm -rf "$INSTALLDIR" -$BUILD_TOOL + if [[ "-$BUILD_TYPE-" == *-shared-* ]]; then + LDLIBPATH="$PREFIXPATH/lib" + if [[ ":$LD_LIBRARY_PATH:" != *":$LDLIBPATH:"* ]]; then + export "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}$LDLIBPATH" + fi + fi -if [[ "-$BUILD_TYPE-" != *-source-* ]]; then - ctest --output-on-failure -j$(nproc) -fi + case "-$BUILD_TYPE-" in + *-full-*) + cmake ../dwarfs/ $CMAKE_ARGS + $BUILD_TOOL + ctest --output-on-failure -j$(nproc) + DESTDIR="$INSTALLDIR" $BUILD_TOOL install + $BUILD_TOOL distclean + ;; -if [[ "-$BUILD_TYPE-" == *-coverage-* ]]; then - rm -f /tmp-runner/dwarfs-coverage.txt - llvm-profdata-$CLANG_VERSION merge -sparse profile/* -o dwarfs.profdata - llvm-cov-$CLANG_VERSION show -instr-profile=dwarfs.profdata \ - $(for i in mkdwarfs dwarfs dwarfsck dwarfsextract *_test ricepp/ricepp_test; do echo $i; done | sed -e's/^/-object=/') \ - >/tmp-runner/dwarfs-coverage.txt -fi + *-split-*) + # ==== libdwarfs ==== + cmake ../dwarfs/ $CMAKE_ARGS -DWITH_LIBDWARFS=ON -DWITH_TOOLS=OFF -DWITH_FUSE_DRIVER=OFF + $BUILD_TOOL + ctest --output-on-failure -j$(nproc) + DESTDIR="$INSTALLDIR" $BUILD_TOOL install + $BUILD_TOOL distclean + rm -rf * -if [[ "-$BUILD_TYPE-" == *-static-* ]]; then - if [[ "-$BUILD_TYPE-" == *-release-* ]]; then - # in the clang-release-static case, we also try to build from the source tarball - if [[ "-$BUILD_TYPE-" == *-clang-* ]] && [[ "-$BUILD_TYPE-" != *-O2-* ]]; then - $BUILD_TOOL package_source + # ==== dwarfs tools ==== + cmake ../dwarfs/ $CMAKE_ARGS -DWITH_LIBDWARFS=OFF -DWITH_TOOLS=ON -DWITH_FUSE_DRIVER=OFF -DCMAKE_PREFIX_PATH="$PREFIXPATH" + $BUILD_TOOL + ctest --output-on-failure -j$(nproc) + DESTDIR="$INSTALLDIR" $BUILD_TOOL install + $BUILD_TOOL distclean + rm -rf * - $BUILD_TOOL realclean + # ==== dwarfs fuse driver ==== + cmake ../dwarfs/ $CMAKE_ARGS -DWITH_LIBDWARFS=OFF -DWITH_TOOLS=OFF -DWITH_FUSE_DRIVER=ON -DCMAKE_PREFIX_PATH="$PREFIXPATH" + $BUILD_TOOL + ctest --output-on-failure -j$(nproc) + DESTDIR="$INSTALLDIR" $BUILD_TOOL install + $BUILD_TOOL distclean + ;; + + *) + echo "builds from source tarball must be 'full' or 'split'" + exit 1 + ;; + esac +else + # shellcheck disable=SC2086 + cmake ../dwarfs/ $CMAKE_ARGS - cd "$HOME" + $BUILD_TOOL - VERSION=$(git -C /workspace describe --tags --match "v*" --dirty --abbrev=10) - VERSION=${VERSION:1} + if [[ "-$BUILD_TYPE-" != *-source-* ]]; then + ctest --output-on-failure -j$(nproc) + fi - rm -rf dwarfs-* - rm -f dwarfs + if [[ "-$BUILD_TYPE-" == *-coverage-* ]]; then + rm -f /tmp-runner/dwarfs-coverage.txt + llvm-profdata-$CLANG_VERSION merge -sparse profile/* -o dwarfs.profdata + llvm-cov-$CLANG_VERSION show -instr-profile=dwarfs.profdata \ + $(for i in mkdwarfs dwarfs dwarfsck dwarfsextract *_test ricepp/ricepp_test; do echo $i; done | sed -e's/^/-object=/') \ + >/tmp-runner/dwarfs-coverage.txt + fi - mv "build/dwarfs-${VERSION}.tar.zst" . - rm -rf build + if [[ "-$BUILD_TYPE-" == *-static-* ]]; then + if [[ "-$BUILD_TYPE-" == *-release-* ]]; then + # in the clang-release-static case, we also try to build from the source tarball + if [[ "-$BUILD_TYPE-" == *-clang-* ]] && [[ "-$BUILD_TYPE-" != *-O2-* ]]; then + $BUILD_TOOL package_source - tar xvf "dwarfs-${VERSION}.tar.zst" - mv "dwarfs-${VERSION}" dwarfs + $BUILD_TOOL realclean - mkdir build - cd build + cd "$HOME" - # shellcheck disable=SC2086 - cmake ../dwarfs/ $CMAKE_ARGS + VERSION=$(git -C /workspace describe --tags --match "v*" --dirty --abbrev=10) + VERSION=${VERSION:1} - $BUILD_TOOL + rm -rf dwarfs-* + rm -f dwarfs - ctest --output-on-failure -j$(nproc) - fi + mv "build/dwarfs-${VERSION}.tar.zst" . + rm -rf build - $BUILD_TOOL strip - fi + tar xvf "dwarfs-${VERSION}.tar.zst" + mv "dwarfs-${VERSION}" dwarfs - $BUILD_TOOL package - $BUILD_TOOL universal_upx + mkdir build + cd build - $BUILD_TOOL copy_artifacts + # shellcheck disable=SC2086 + cmake ../dwarfs/ $CMAKE_ARGS - rm -rf /tmp-runner/artifacts - mkdir -p /tmp-runner/artifacts - cp artifacts.env /tmp-runner - cp dwarfs-universal-* /tmp-runner/artifacts - cp dwarfs-*-Linux*.tar.zst /tmp-runner/artifacts + $BUILD_TOOL - if [[ "$VERSION" != "" ]]; then - # also perform a non-static build based on the source tarball + ctest --output-on-failure -j$(nproc) + fi - $BUILD_TOOL distclean - unset LDFLAGS + $BUILD_TOOL strip + fi - # shellcheck disable=SC2086 - cmake ../dwarfs/ $CMAKE_ARGS_NONSTATIC + $BUILD_TOOL package + $BUILD_TOOL universal_upx - $BUILD_TOOL + $BUILD_TOOL copy_artifacts - ctest --output-on-failure -j$(nproc) + rm -rf /tmp-runner/artifacts + mkdir -p /tmp-runner/artifacts + cp artifacts.env /tmp-runner + cp dwarfs-universal-* /tmp-runner/artifacts + cp dwarfs-*-Linux*.tar.zst /tmp-runner/artifacts + + if [[ "$VERSION" != "" ]]; then + # also perform a non-static build based on the source tarball + + $BUILD_TOOL distclean + unset LDFLAGS + + # shellcheck disable=SC2086 + cmake ../dwarfs/ $CMAKE_ARGS_NONSTATIC + + $BUILD_TOOL + + ctest --output-on-failure -j$(nproc) + fi + elif [[ "-$BUILD_TYPE-" == *-source-* ]]; then + $BUILD_TOOL package_source + $BUILD_TOOL copy_source_artifacts + rm -rf /tmp-runner/artifacts + mkdir -p /tmp-runner/artifacts + cp source-artifacts.env /tmp-runner + cp dwarfs-*.tar.zst /tmp-runner/artifacts fi -elif [[ "-$BUILD_TYPE-" == *-source-* ]]; then - $BUILD_TOOL package_source - $BUILD_TOOL copy_source_artifacts - rm -rf /tmp-runner/artifacts - mkdir -p /tmp-runner/artifacts - cp source-artifacts.env /tmp-runner - cp dwarfs-*.tar.zst /tmp-runner/artifacts -fi -if [[ "-$BUILD_TYPE-" != *-[at]san-* ]] && \ - [[ "-$BUILD_TYPE-" != *-ubsan-* ]] && \ - [[ "-$BUILD_TYPE-" != *-source-* ]] && \ - ( [[ "-$BUILD_TYPE-" != *-static-* ]] || [[ "$VERSION" != "" ]] ); then - INSTALLDIR="$HOME/install" - rm -rf "$INSTALLDIR" - DESTDIR="$INSTALLDIR" $BUILD_TOOL install + if [[ "-$BUILD_TYPE-" != *-[at]san-* ]] && \ + [[ "-$BUILD_TYPE-" != *-ubsan-* ]] && \ + [[ "-$BUILD_TYPE-" != *-source-* ]] && \ + ( [[ "-$BUILD_TYPE-" != *-static-* ]] || [[ "$VERSION" != "" ]] ); then + INSTALLDIR="$HOME/install" + rm -rf "$INSTALLDIR" + DESTDIR="$INSTALLDIR" $BUILD_TOOL install - $BUILD_TOOL distclean + $BUILD_TOOL distclean - cmake ../dwarfs/example $CMAKE_TOOL_ARGS -DCMAKE_PREFIX_PATH="$INSTALLDIR/usr/local" - $BUILD_TOOL - $BUILD_TOOL clean -else - $BUILD_TOOL distclean + cmake ../dwarfs/example $CMAKE_TOOL_ARGS -DCMAKE_PREFIX_PATH="$INSTALLDIR/usr/local" + $BUILD_TOOL + $BUILD_TOOL clean + else + $BUILD_TOOL distclean + fi fi - diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5d3e5ff18..fc21f3292 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,6 +14,7 @@ permissions: jobs: windows: + if: false runs-on: - self-hosted - Windows @@ -146,7 +147,106 @@ jobs: if-no-files-found: error compression-level: 0 - linux: + linux-src: + needs: package-source + + strategy: + matrix: + arch: + - amd64 + - arm64v8 + dist: + - ubuntu + - fedora + - arch + build_type: + # - gcc-debug-ninja-full + - gcc-release-ninja-split + # - gcc-release-shared-ninja-split + # - clang-release-ninja-full + exclude: + - arch: arm64v8 + dist: arch + build_type: gcc-debug-ninja-full + - arch: arm64v8 + dist: arch + build_type: gcc-release-ninja-split + - arch: arm64v8 + dist: arch + build_type: gcc-release-shared-ninja-split + - arch: arm64v8 + dist: arch + build_type: clang-release-ninja-full + include: + # - arch: amd64 + # dist: ubuntu + # build_type: oldgcc-debug-make-split + # - arch: arm64v8 + # dist: ubuntu + # build_type: gcc-debug-noperfmon-ninja-split + # - arch: arm64v8 + # dist: ubuntu + # build_type: gcc-debug-nostacktrace-ninja-split + # - arch: arm64v8 + # dist: ubuntu + # build_type: oldclang-debug-make-split + - arch: amd64 + dist: ubuntu-2204 + build_type: gcc-release-ninja-split + - arch: amd64 + dist: ubuntu-2204 + build_type: gcc-debug-ninja-full + - arch: amd64 + dist: ubuntu-2204 + build_type: clang-release-ninja-split + + runs-on: + - self-hosted + - Linux + - ${{ matrix.arch }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: 'true' + fetch-depth: '0' + ref: ${{ github.ref }} + + - name: Download Source Tarball + uses: actions/download-artifact@v4 + with: + name: dwarfs-source + + - name: Build Docker Image + run: | + docker build \ + -f .docker/Dockerfile.${{ matrix.dist }} \ + -t dwarfs-linux-build-${{ matrix.dist }} \ + --build-arg ARCH=${{ matrix.arch }} \ + --build-arg SCRIPT=build-linux.sh \ + .docker + + - name: Run Build (${{ matrix.build_type }}) + run: | + docker run --rm \ + --cap-add SYS_ADMIN --device /dev/fuse --privileged \ + --mount type=bind,source=${GITHUB_WORKSPACE},target=/workspace,readonly \ + --mount type=bind,source=${{ runner.temp }},target=/tmp-runner \ + --mount type=bind,source=/home/mhx/github-ccache,target=/ccache \ + --env BUILD_FROM_TARBALL=1 \ + --env BUILD_TYPE=${{ matrix.build_type }} \ + --env BUILD_ARCH=${{ matrix.arch }} \ + --env BUILD_DIST=${{ matrix.dist }} \ + --env GITHUB_REF_NAME \ + --env GITHUB_REF_TYPE \ + --env GITHUB_RUN_ID \ + --env GITHUB_RUN_NUMBER \ + --env GITHUB_RUN_ATTEMPT \ + dwarfs-linux-build-${{ matrix.dist }} + + linux-git: + if: false strategy: matrix: arch: