build: enable builds from source tarball in CI

mhx · Aug 12, 2024 · 4af6b68 · 4af6b68
1 parent 22b5d51
commit 4af6b68
Show file tree

Hide file tree

Showing 2 changed files with 220 additions and 79 deletions.
diff --git a/.docker/build-linux.sh b/.docker/build-linux.sh
@@ -6,10 +6,15 @@ export CCACHE_DIR=/ccache
 
 cd "$HOME"
 
-git config --global --add safe.directory /workspace
+rm -rf dwarfs dwarfs-*
 
-rm -f dwarfs
-ln -s /workspace dwarfs
+if [[ "$BUILD_FROM_TARBALL" == "1" ]]; then
+  tar xvf /workspace/dwarfs-*.tar.zst
+  ln -s dwarfs-* dwarfs
+else
+  git config --global --add safe.directory /workspace
+  ln -s /workspace dwarfs
+fi
 
 rm -rf build
 mkdir build
@@ -132,106 +137,151 @@ if [[ "-$BUILD_TYPE-" == *-static-* ]]; then
   CMAKE_ARGS="${CMAKE_ARGS} -DSTATIC_BUILD_EXTRA_PREFIX=/opt/static-libs/$COMPILER"
 fi
 
-# shellcheck disable=SC2086
-cmake ../dwarfs/ $CMAKE_ARGS
+if [[ "$BUILD_FROM_TARBALL" == "1" ]]; then
+  INSTALLDIR="$HOME/install"
+  PREFIXPATH="$INSTALLDIR/usr/local"
+  rm -rf "$INSTALLDIR"
 
-$BUILD_TOOL
+  case "-$BUILD_TYPE-" in
+    *-full-*)
+      cmake ../dwarfs/ $CMAKE_ARGS
+      $BUILD_TOOL
+      ctest --output-on-failure -j$(nproc)
+      DESTDIR="$INSTALLDIR" $BUILD_TOOL install
+      $BUILD_TOOL distclean
+      ;;
 
-if [[ "-$BUILD_TYPE-" != *-source-* ]]; then
-  ctest --output-on-failure -j$(nproc)
-fi
+    *-split-*)
+      # ==== libdwarfs ====
+      cmake ../dwarfs/ $CMAKE_ARGS -DWITH_LIBDWARFS=ON -DWITH_TOOLS=OFF -DWITH_FUSE_DRIVER=OFF
+      $BUILD_TOOL
+      ctest --output-on-failure -j$(nproc)
+      DESTDIR="$INSTALLDIR" $BUILD_TOOL install
+      $BUILD_TOOL distclean
+      rm -rf * .*
 
-if [[ "-$BUILD_TYPE-" == *-coverage-* ]]; then
-  rm -f /tmp-runner/dwarfs-coverage.txt
-  llvm-profdata-$CLANG_VERSION merge -sparse profile/* -o dwarfs.profdata
-  llvm-cov-$CLANG_VERSION show -instr-profile=dwarfs.profdata \
-    $(for i in mkdwarfs dwarfs dwarfsck dwarfsextract *_test ricepp/ricepp_test; do echo $i; done | sed -e's/^/-object=/') \
-    >/tmp-runner/dwarfs-coverage.txt
-fi
+      # ==== dwarfs tools ====
+      cmake ../dwarfs/ $CMAKE_ARGS -DWITH_LIBDWARFS=OFF -DWITH_TOOLS=ON -DWITH_FUSE_DRIVER=OFF -DCMAKE_PREFIX_PATH="$PREFIXPATH"
+      $BUILD_TOOL
+      ctest --output-on-failure -j$(nproc)
+      DESTDIR="$INSTALLDIR" $BUILD_TOOL install
+      $BUILD_TOOL distclean
+      rm -rf * .*
 
-if [[ "-$BUILD_TYPE-" == *-static-* ]]; then
-  if [[ "-$BUILD_TYPE-" == *-release-* ]]; then
-    # in the clang-release-static case, we also try to build from the source tarball
-    if [[ "-$BUILD_TYPE-" == *-clang-* ]] && [[ "-$BUILD_TYPE-" != *-O2-* ]]; then
-      $BUILD_TOOL package_source
+      # ==== dwarfs fuse driver ====
+      cmake ../dwarfs/ $CMAKE_ARGS -DWITH_LIBDWARFS=OFF -DWITH_TOOLS=OFF -DWITH_FUSE_DRIVER=ON -DCMAKE_PREFIX_PATH="$PREFIXPATH"
+      $BUILD_TOOL
+      ctest --output-on-failure -j$(nproc)
+      DESTDIR="$INSTALLDIR" $BUILD_TOOL install
+      $BUILD_TOOL distclean
+      ;;
+
+    *)
+      echo "builds from source tarball must be 'full' or 'split'"
+      exit 1
+      ;;
+  esac
+else
+  # shellcheck disable=SC2086
+  cmake ../dwarfs/ $CMAKE_ARGS
 
-      $BUILD_TOOL realclean
+  $BUILD_TOOL
 
-      cd "$HOME"
+  if [[ "-$BUILD_TYPE-" != *-source-* ]]; then
+    ctest --output-on-failure -j$(nproc)
+  fi
 
-      VERSION=$(git -C /workspace describe --tags --match "v*" --dirty --abbrev=10)
-      VERSION=${VERSION:1}
+  if [[ "-$BUILD_TYPE-" == *-coverage-* ]]; then
+    rm -f /tmp-runner/dwarfs-coverage.txt
+    llvm-profdata-$CLANG_VERSION merge -sparse profile/* -o dwarfs.profdata
+    llvm-cov-$CLANG_VERSION show -instr-profile=dwarfs.profdata \
+      $(for i in mkdwarfs dwarfs dwarfsck dwarfsextract *_test ricepp/ricepp_test; do echo $i; done | sed -e's/^/-object=/') \
+      >/tmp-runner/dwarfs-coverage.txt
+  fi
 
-      rm -rf dwarfs-*
-      rm -f dwarfs
+  if [[ "-$BUILD_TYPE-" == *-static-* ]]; then
+    if [[ "-$BUILD_TYPE-" == *-release-* ]]; then
+      # in the clang-release-static case, we also try to build from the source tarball
+      if [[ "-$BUILD_TYPE-" == *-clang-* ]] && [[ "-$BUILD_TYPE-" != *-O2-* ]]; then
+        $BUILD_TOOL package_source
 
-      mv "build/dwarfs-${VERSION}.tar.zst" .
-      rm -rf build
+        $BUILD_TOOL realclean
 
-      tar xvf "dwarfs-${VERSION}.tar.zst"
-      mv "dwarfs-${VERSION}" dwarfs
+        cd "$HOME"
 
-      mkdir build
-      cd build
+        VERSION=$(git -C /workspace describe --tags --match "v*" --dirty --abbrev=10)
+        VERSION=${VERSION:1}
 
-      # shellcheck disable=SC2086
-      cmake ../dwarfs/ $CMAKE_ARGS
+        rm -rf dwarfs-*
+        rm -f dwarfs
 
-      $BUILD_TOOL
+        mv "build/dwarfs-${VERSION}.tar.zst" .
+        rm -rf build
 
-      ctest --output-on-failure -j$(nproc)
-    fi
+        tar xvf "dwarfs-${VERSION}.tar.zst"
+        mv "dwarfs-${VERSION}" dwarfs
 
-    $BUILD_TOOL strip
-  fi
+        mkdir build
+        cd build
 
-  $BUILD_TOOL package
-  $BUILD_TOOL universal_upx
+        # shellcheck disable=SC2086
+        cmake ../dwarfs/ $CMAKE_ARGS
 
-  $BUILD_TOOL copy_artifacts
+        $BUILD_TOOL
 
-  rm -rf /tmp-runner/artifacts
-  mkdir -p /tmp-runner/artifacts
-  cp artifacts.env /tmp-runner
-  cp dwarfs-universal-* /tmp-runner/artifacts
-  cp dwarfs-*-Linux*.tar.zst /tmp-runner/artifacts
+        ctest --output-on-failure -j$(nproc)
+      fi
 
-  if [[ "$VERSION" != "" ]]; then
-    # also perform a non-static build based on the source tarball
+      $BUILD_TOOL strip
+    fi
 
-    $BUILD_TOOL distclean
-    unset LDFLAGS
+    $BUILD_TOOL package
+    $BUILD_TOOL universal_upx
 
-    # shellcheck disable=SC2086
-    cmake ../dwarfs/ $CMAKE_ARGS_NONSTATIC
+    $BUILD_TOOL copy_artifacts
 
-    $BUILD_TOOL
+    rm -rf /tmp-runner/artifacts
+    mkdir -p /tmp-runner/artifacts
+    cp artifacts.env /tmp-runner
+    cp dwarfs-universal-* /tmp-runner/artifacts
+    cp dwarfs-*-Linux*.tar.zst /tmp-runner/artifacts
 
-    ctest --output-on-failure -j$(nproc)
+    if [[ "$VERSION" != "" ]]; then
+      # also perform a non-static build based on the source tarball
+
+      $BUILD_TOOL distclean
+      unset LDFLAGS
+
+      # shellcheck disable=SC2086
+      cmake ../dwarfs/ $CMAKE_ARGS_NONSTATIC
+
+      $BUILD_TOOL
+
+      ctest --output-on-failure -j$(nproc)
+    fi
+  elif [[ "-$BUILD_TYPE-" == *-source-* ]]; then
+    $BUILD_TOOL package_source
+    $BUILD_TOOL copy_source_artifacts
+    rm -rf /tmp-runner/artifacts
+    mkdir -p /tmp-runner/artifacts
+    cp source-artifacts.env /tmp-runner
+    cp dwarfs-*.tar.zst /tmp-runner/artifacts
   fi
-elif [[ "-$BUILD_TYPE-" == *-source-* ]]; then
-  $BUILD_TOOL package_source
-  $BUILD_TOOL copy_source_artifacts
-  rm -rf /tmp-runner/artifacts
-  mkdir -p /tmp-runner/artifacts
-  cp source-artifacts.env /tmp-runner
-  cp dwarfs-*.tar.zst /tmp-runner/artifacts
-fi
 
-if [[ "-$BUILD_TYPE-" != *-[at]san-* ]] && \
-   [[ "-$BUILD_TYPE-" != *-ubsan-* ]] && \
-   [[ "-$BUILD_TYPE-" != *-source-* ]] && \
-   ( [[ "-$BUILD_TYPE-" != *-static-* ]] || [[ "$VERSION" != "" ]] ); then
-  INSTALLDIR="$HOME/install"
-  rm -rf "$INSTALLDIR"
-  DESTDIR="$INSTALLDIR" $BUILD_TOOL install
+  if [[ "-$BUILD_TYPE-" != *-[at]san-* ]] && \
+     [[ "-$BUILD_TYPE-" != *-ubsan-* ]] && \
+     [[ "-$BUILD_TYPE-" != *-source-* ]] && \
+     ( [[ "-$BUILD_TYPE-" != *-static-* ]] || [[ "$VERSION" != "" ]] ); then
+    INSTALLDIR="$HOME/install"
+    rm -rf "$INSTALLDIR"
+    DESTDIR="$INSTALLDIR" $BUILD_TOOL install
 
-  $BUILD_TOOL distclean
+    $BUILD_TOOL distclean
 
-  cmake ../dwarfs/example $CMAKE_TOOL_ARGS -DCMAKE_PREFIX_PATH="$INSTALLDIR/usr/local"
-  $BUILD_TOOL
-  $BUILD_TOOL clean
-else
-  $BUILD_TOOL distclean
+    cmake ../dwarfs/example $CMAKE_TOOL_ARGS -DCMAKE_PREFIX_PATH="$INSTALLDIR/usr/local"
+    $BUILD_TOOL
+    $BUILD_TOOL clean
+  else
+    $BUILD_TOOL distclean
+  fi
 fi
-
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -146,7 +146,98 @@ jobs:
           if-no-files-found: error
           compression-level: 0
 
-  linux:
+  linux-src:
+    needs: package-source
+
+    strategy:
+      matrix:
+        arch:
+          - amd64
+          - arm64v8
+        dist:
+          - ubuntu
+          - fedora
+          - arch
+        build_type:
+          - gcc-debug-ninja-full
+          - gcc-release-ninja-split
+          - gcc-release-shared-ninja-split
+          - clang-release-ninja-full
+        exclude:
+          - arch: arm64v8
+            dist: arch
+            build_type: gcc-debug-ninja-full
+          - arch: arm64v8
+            dist: arch
+            build_type: gcc-release-ninja-split
+          - arch: arm64v8
+            dist: arch
+            build_type: gcc-release-shared-ninja-split
+          - arch: arm64v8
+            dist: arch
+            build_type: clang-release-ninja-full
+        include:
+          - arch: amd64
+            dist: ubuntu
+            build_type: oldgcc-debug-make-split
+          - arch: arm64v8
+            dist: ubuntu
+            build_type: gcc-debug-noperfmon-ninja-split
+          - arch: arm64v8
+            dist: ubuntu
+            build_type: gcc-debug-nostacktrace-ninja-split
+          - arch: arm64v8
+            dist: ubuntu
+            build_type: oldclang-debug-make-split
+          - arch: arm64v8
+            dist: ubuntu-2204
+            build_type: gcc-release-ninja-split
+          - arch: arm64v8
+            dist: ubuntu-2204
+            build_type: gcc-debug-ninja-full
+          - arch: arm64v8
+            dist: ubuntu-2204
+            build_type: clang-release-ninja-split
+
+    runs-on:
+      - self-hosted
+      - Linux
+      - ${{ matrix.arch }}
+
+    steps:
+      - name: Download Source Tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: dwarfs-source
+
+      - name: Build Docker Image
+        run: |
+          docker build \
+            -f .docker/Dockerfile.${{ matrix.dist }} \
+            -t dwarfs-linux-build-${{ matrix.dist }} \
+            --build-arg ARCH=${{ matrix.arch }} \
+            --build-arg SCRIPT=build-linux.sh \
+            .docker
+
+      - name: Run Build (${{ matrix.build_type }})
+        run: |
+          docker run --rm \
+            --cap-add SYS_ADMIN --device /dev/fuse --privileged \
+            --mount type=bind,source=${GITHUB_WORKSPACE},target=/workspace,readonly \
+            --mount type=bind,source=${{ runner.temp }},target=/tmp-runner \
+            --mount type=bind,source=/home/mhx/github-ccache,target=/ccache \
+            --env BUILD_FROM_TARBALL=1 \
+            --env BUILD_TYPE=${{ matrix.build_type }} \
+            --env BUILD_ARCH=${{ matrix.arch }} \
+            --env BUILD_DIST=${{ matrix.dist }} \
+            --env GITHUB_REF_NAME \
+            --env GITHUB_REF_TYPE \
+            --env GITHUB_RUN_ID \
+            --env GITHUB_RUN_NUMBER \
+            --env GITHUB_RUN_ATTEMPT \
+            dwarfs-linux-build-${{ matrix.dist }}
+
+  linux-git:
     strategy:
       matrix:
         arch: