Merge branch 'master' into jblanco/new-conductance-source

BlueBrain · Apr 26, 2024 · 0f80754 · 0f80754
2 parents 1c366b7 + 92cf118
commit 0f80754
Show file tree

Hide file tree

Showing 6 changed files with 55 additions and 126 deletions.
diff --git a/.github/workflows/publish-sdist-wheels.yml b/.github/workflows/publish-sdist-wheels.yml
@@ -8,6 +8,7 @@ env:
   CIBW_BUILD: 'cp*'
   CIBW_SKIP: 'cp35-* cp36-* cp37-* *-musllinux_* *-manylinux_i686'
   CIBW_TEST_COMMAND: ( cd {project}/python/tests; python -m unittest -v )
+  UNIXY_HDF5_VERSION: 1.14.3
 
 jobs:
   build_wheels:
@@ -47,38 +48,51 @@ jobs:
 
       - name: Build wheels on Linux
         if: runner.os == 'Linux'
+        env:
+          CIBW_ENVIRONMENT_PASS: "STATIC_HDF5 CMAKE_PREFIX_PATH"
+          CIBW_BEFORE_BUILD: |
+            # CMake complains if the dependencies come from within the same tree
+            # as the source, so we'll just pretend they are elsewhere
+            mkdir -p /host/home/runner/work/src-cache
+            ln -s /host/home/runner/work/src-cache /opt/hdf5-static
+            bash ci/hdf5-build.sh /opt/hdf5-static
+
         run: |
-          # note: config is stored in pyproject.toml now
+          # used by setup.py to decide if to set `FindHDF5` to use static hdf5 libraries
+          export STATIC_HDF5=True
+          export CMAKE_PREFIX_PATH=/opt/hdf5-static/install-/install/
+
           CIBW_MANYLINUX_X86_64_IMAGE=manylinux2014 python -m cibuildwheel --output-dir dist
           CIBW_MANYLINUX_X86_64_IMAGE=manylinux_2_28 python -m cibuildwheel --output-dir dist
 
       - name: Build wheels Mac OS
         if: runner.os == 'macOS'
 
         env:
-          CIBW_ENVIRONMENT_PASS: "HDF5_ROOT CMAKE_OSX_ARCHITECTURES MACOSX_DEPLOYMENT_TARGET"
+          CIBW_ENVIRONMENT_PASS: "STATIC_HDF5 CMAKE_PREFIX_PATH CMAKE_OSX_ARCHITECTURES MACOSX_DEPLOYMENT_TARGET"
           CIBW_BEFORE_BUILD: |
             # CMake complains if the dependencies come from within the same tree
             # as the source, so we'll just pretend they are elsewhere
             mkdir -p $PWD/src-cache
             ln -s $PWD/src-cache /Users/runner/work/src-cache
             bash ci/hdf5-build.sh /Users/runner/work/src-cache
-          UNIXY_AEC_VERSION: 1.0.4
-          UNIXY_HDF5_VERSION: 1.14.2
         run: |
+          # used by setup.py to decide if to set `FindHDF5` to use static hdf5 libraries
+          export STATIC_HDF5=True
+
           # x86_64 macOS allows for cross compilation; first we do arm64,
           # only for the 11.0 target; and store in the cache the compiled code...
           export CIBW_ARCHS_MACOS="arm64"
           export CMAKE_OSX_ARCHITECTURES="arm64"
           export MACOSX_DEPLOYMENT_TARGET="11.0"
-          export HDF5_ROOT=/Users/runner/work/src-cache/install-$CIBW_ARCHS_MACOS
+          export CMAKE_PREFIX_PATH=/Users/runner/work/src-cache/install-$CIBW_ARCHS_MACOS/install
           python -m cibuildwheel --output-dir dist
 
           # ...and now we do both targets for x86_64
           export CIBW_ARCHS_MACOS="x86_64"
           export CMAKE_OSX_ARCHITECTURES="x86_64"
           unset MACOSX_DEPLOYMENT_TARGET
-          export HDF5_ROOT=/Users/runner/work/src-cache/install-$CIBW_ARCHS_MACOS
+          export CMAKE_PREFIX_PATH=/Users/runner/work/src-cache/install-$CIBW_ARCHS_MACOS/install
           python -m cibuildwheel --output-dir dist
 
       - name: Store wheel as artifact

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
@@ -99,8 +99,8 @@ jobs:
     env:
       CIBW_ARCHS_MACOS: "x86_64"
       CMAKE_OSX_ARCHITECTURES: "x86_64"
-      UNIXY_AEC_VERSION: 1.0.4
-      UNIXY_HDF5_VERSION: 1.14.2
+      UNIXY_HDF5_VERSION: 1.14.3
+      MACOSX_DEPLOYMENT_TARGET: "11.0"
 
     steps:
       - name: Checkout repository
@@ -126,6 +126,8 @@ jobs:
 
       - name: Build and run unittests
         run: |
-            export HDF5_ROOT=/Users/runner/work/src-cache/install-$CIBW_ARCHS_MACOS
+            export CMAKE_PREFIX_PATH=/Users/runner/work/src-cache/install-$CIBW_ARCHS_MACOS/install
+            export STATIC_HDF5=True
+
             ./ci/cpp_test.sh
             ./ci/python_test.sh
diff --git a/ci/ci-buildwheel-prepare-linux.sh b/ci/ci-buildwheel-prepare-linux.sh
diff --git a/ci/hdf5-build.sh b/ci/hdf5-build.sh
@@ -5,46 +5,28 @@ set -e -x
 export INPUT=$(cd $(dirname "$1") && pwd -P)/$(basename "$1")
 export OUTPUT="$INPUT/install-$CIBW_ARCHS_MACOS"
 
-: "${ZLIB_VERSION:=1.3}"
-: "${UNIXY_AEC_VERSION:=1.0.4}"
-: "${UNIXY_HDF5_VERSION:=1.14.2}"
-
-function download_unpack_zlib {
-    pushd "$INPUT"
-    rm -rf "zlib-$ZLIB_VERSION"
-    echo "Downloading & unpacking ZLIB ${ZLIB_VERSION}"
-    curl -sLO https://zlib.net/fossils/zlib-$ZLIB_VERSION.tar.gz
-    tar xzf zlib-$ZLIB_VERSION.tar.gz
-    popd
-}
-
-function download_unpack_libaec {
-    pushd "$INPUT"
-    rm -rf "libaec-$UNIXY_AEC_VERSION"
-    echo "Downloading & unpacking aec ${UNIXY_AEC_VERSION}"
-    curl -fsSLO "https://gitlab.dkrz.de/k202009/libaec/uploads/ea0b7d197a950b0c110da8dfdecbb71f/libaec-${UNIXY_AEC_VERSION}.tar.gz"
-    tar zxf "libaec-$UNIXY_AEC_VERSION.tar.gz"
-    popd
-}
+: "${UNIXY_HDF5_VERSION:=1.14.3}"
 
 function download_unpack_hdf5 {
     pushd "$INPUT"
-    rm -rf hdf5-$UNIXY_HDF5_VERSION
-    echo "Downloading & unpacking HDF5 ${UNIXY_HDF5_VERSION}"
-    curl -fsSLO "https://www.hdfgroup.org/ftp/HDF5/releases/hdf5-${UNIXY_HDF5_VERSION%.*}/hdf5-$UNIXY_HDF5_VERSION/src/hdf5-$UNIXY_HDF5_VERSION.tar.gz"
-    tar xzf "hdf5-$UNIXY_HDF5_VERSION.tar.gz"
+    local name=CMake-hdf5-$UNIXY_HDF5_VERSION.tar.gz
+    if [[ ! -e $name ]]; then
+        echo "Downloading & unpacking HDF5 ${UNIXY_HDF5_VERSION}"
+        curl -fsSLO "https://www.hdfgroup.org/ftp/HDF5/releases/hdf5-${UNIXY_HDF5_VERSION%.*}/hdf5-$UNIXY_HDF5_VERSION/src/$name"
+    fi
+    tar xzf "$name"
     popd
 }
 
 if [[ "$OSTYPE" == "darwin"* ]]; then
-    lib_name=libhdf5.dylib
     NPROC=$(sysctl -n hw.ncpu)
 else
-    lib_name=libhdf5.so
     NPROC=$(nproc)
 fi
 
-if [[ -f "$OUTPUT/lib/$lib_name" ]]; then
+INSTALL="$OUTPUT/install"
+
+if [[ -f "$INSTALL/lib/libhdf5.a" ]]; then
     echo "using cached build"
 else
     if [[ "$OSTYPE" == "darwin"* ]]; then
@@ -53,60 +35,26 @@ else
         export CFLAGS="$CFLAGS -arch $CIBW_ARCHS_MACOS"
         export CPPFLAGS="$CPPFLAGS -arch $CIBW_ARCHS_MACOS"
         export CXXFLAGS="$CXXFLAGS -arch $CIBW_ARCHS_MACOS"
-
-        download_unpack_zlib
-
-        pushd "$INPUT/zlib-$ZLIB_VERSION"
-        ./configure                             \
-            --archs="-arch $CIBW_ARCHS_MACOS"   \
-            --prefix="$OUTPUT"
-        make
-        make install
-        popd
-
-        ZLIB_ARG="--with-zlib=$OUTPUT"
-
-        if [[ "$CIBW_ARCHS_MACOS" = "arm64" ]]; then
-            HOST_ARG="--host=aarch64-apple-darwin"
-        fi
     fi
 
-    echo "Building & installing libaec"
-    download_unpack_libaec
-    pushd "$INPUT/libaec-$UNIXY_AEC_VERSION"
-    ./configure            \
-        --prefix="$OUTPUT" \
-        ${HOST_ARG}        \
-        --enable-static=no
-
-    make -j "$NPROC"
-    make install
-    popd
-
     echo "Building & installing hdf5"
     download_unpack_hdf5
-    pushd "$INPUT/hdf5-$UNIXY_HDF5_VERSION"
-    ./configure                        \
-        --prefix="$OUTPUT"             \
-        --with-szlib="$OUTPUT"         \
-        ${HOST_ARG}                    \
-        ${ZLIB_ARG}                    \
-        --enable-build-mode=production \
-        --enable-tools=no              \
-        --enable-tests=no              \
-        --enable-static=no
 
-    # {
-    # work around for crosscompile getting: `./H5detect: Bad CPU type in executable`
-    # so build it on the host architecture instead
-    pushd src
-    ( export ARCHFLAGS= ; make H5detect H5make_libsettings CFLAGS= CCLDFLAGS= CXXFLAGS= CPPFLAGS= )
-    popd
-    # }
-
-    make -j "$NPROC"
-    make install
-    popd
-
-    file "$OUTPUT"/lib/*
+    cmake -B "$OUTPUT/build" -G'Unix Makefiles' \
+        -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+        -DBUILD_SHARED_LIBS=OFF \
+        -DHDF5_BUILD_UTILS=OFF \
+        -DHDF5_BUILD_HL_LIB=OFF \
+        -DHDF5_BUILD_EXAMPLES=OFF \
+        -DBUILD_TESTING=OFF \
+        -DHDF5_BUILD_TOOLS=OFF \
+        -DHDF5_ENABLE_SZIP_ENCODING=OFF \
+        -DHDF5_ENABLE_Z_LIB_SUPPORT=OFF \
+        -DCMAKE_INSTALL_PREFIX="$INSTALL" \
+        -S "$INPUT/CMake-hdf5-$UNIXY_HDF5_VERSION/hdf5-$UNIXY_HDF5_VERSION"
+
+    cmake --build "$OUTPUT/build" -j "$NPROC"
+    cmake --install "$OUTPUT/build"
 fi
+
+find "$OUTPUT"
diff --git a/pyproject.toml b/pyproject.toml
@@ -7,8 +7,5 @@ requires = [
 ]
 build-backend = "setuptools.build_meta"
 
-[tool.cibuildwheel.linux]
-before-all = "bash ci/ci-buildwheel-prepare-linux.sh"
-
 [tool.pytest.ini_options]
 testpaths = ["python/tests"]
diff --git a/setup.py b/setup.py
@@ -66,6 +66,9 @@ def build_extension(self, ext):
             '-DPYTHON_EXECUTABLE=' + sys.executable
         ]
 
+        if "STATIC_HDF5" in os.environ:
+            cmake_args += ["-DHDF5_USE_STATIC_LIBRARIES=True", ]
+
         build_args = ["--config", build_type,
                       "--target", self.target,
                       "--",