i3558 improve wheel repair + add minimal IDAKLU test for built wheels

.github/workflows/publish_pypi.yml

@@ -16,16 +16,37 @@ on:
         required: false
         default: false
 
+# Set options available for all jobs that use cibuildwheel
+env:
+  # Increase pip debugging output, equivalent to `pip -vv`
+  CIBW_BUILD_VERBOSITY: 2
+  # Disable build isolation to allow pre-installing build-time dependencies.
+  # Note: CIBW_BEFORE_BUILD must be present in all jobs using cibuildwheel.
+  CIBW_BUILD_FRONTEND: "pip; args: --no-build-isolation"
+  # Skip PyPy and MUSL builds in any and all jobs
+  CIBW_SKIP: "pp* *musllinux*"
+  FORCE_COLOR: 3
+
 jobs:
   build_windows_wheels:
-    name: Build wheels on windows-latest
+    name: Wheels (windows-latest)
     runs-on: windows-latest
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
           python-version: 3.8
 
+      - name: Get number of cores on Windows
+        id: get_num_cores
+        shell: python
+        run: |
+          from os import environ, cpu_count
+          num_cpus = cpu_count()
+          output_file = environ['GITHUB_OUTPUT']
+          with open(output_file, "a", encoding="utf-8") as output_stream:
+              output_stream.write(f"count={num_cpus}\n")
+
       - name: Clone pybind11 repo (no history)
         run: git clone --depth 1 --branch v2.11.1 https://github.com/pybind/pybind11.git
 
@@ -53,8 +74,17 @@ jobs:
       - name: Build 64-bit wheels on Windows
         run: pipx run cibuildwheel --output-dir wheelhouse
         env:
-          CIBW_ENVIRONMENT: 'PYBAMM_USE_VCPKG=ON VCPKG_ROOT_DIR=C:\vcpkg VCPKG_DEFAULT_TRIPLET=x64-windows-static-md VCPKG_FEATURE_FLAGS=manifests,registries CMAKE_GENERATOR="Visual Studio 17 2022" CMAKE_GENERATOR_PLATFORM=x64'
-          CIBW_ARCHS: "AMD64"
+          CIBW_ENVIRONMENT: >
+            PYBAMM_USE_VCPKG=ON
+            VCPKG_ROOT_DIR=C:\vcpkg
+            VCPKG_DEFAULT_TRIPLET=x64-windows-static-md
+            VCPKG_FEATURE_FLAGS=manifests,registries
+            CMAKE_GENERATOR="Visual Studio 17 2022"
+            CMAKE_GENERATOR_PLATFORM=x64
+            CMAKE_BUILD_PARALLEL_LEVEL=${{ steps.get_num_cores.outputs.count }}
+          CIBW_ARCHS: AMD64
+          CIBW_BEFORE_BUILD: python -m pip install setuptools wheel # skip CasADi and CMake
+          CIBW_TEST_COMMAND: python -c "import pybamm; pybamm.IDAKLUSolver()"
 
       - name: Upload Windows wheels
         uses: actions/upload-artifact@v4
@@ -63,16 +93,19 @@ jobs:
           path: ./wheelhouse/*.whl
           if-no-files-found: error
 
-  build_wheels:
-    name: Build wheels on ${{ matrix.os }}
+  build_macos_and_linux_wheels:
+    name: Wheels (${{ matrix.os }})
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest]
     steps:
       - uses: actions/checkout@v4
+        name: Check out PyBaMM repository
+
       - uses: actions/setup-python@v5
+        name: Set up Python
         with:
           python-version: 3.8
 
@@ -88,26 +121,26 @@ jobs:
           python -m pip install cmake wget
           python scripts/install_KLU_Sundials.py
 
-      - name: Build wheels on ${{ matrix.os }}
+      - name: Build wheels on Linux
         run: pipx run cibuildwheel --output-dir wheelhouse
+        if: matrix.os == 'ubuntu-latest'
         env:
           CIBW_ARCHS_LINUX: x86_64
           CIBW_BEFORE_ALL_LINUX: >
             yum -y install openblas-devel lapack-devel &&
             bash scripts/install_sundials.sh 6.0.3 6.5.0
-          CIBW_BEFORE_BUILD_LINUX: >
-            python -m pip install cmake casadi numpy
-          # override; point to casadi install path so that it can be found by the repair command
-          CIBW_REPAIR_WHEEL_COMMAND_LINUX: >
-            LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:$(python -c 'import casadi; print(casadi.__path__[0])')" auditwheel repair -w {dest_dir} {wheel}
+          CIBW_BEFORE_BUILD_LINUX: python -m pip install cmake casadi setuptools wheel
+          CIBW_REPAIR_WHEEL_COMMAND_LINUX: auditwheel repair -w {dest_dir} {wheel}
+          CIBW_TEST_COMMAND: python -c "import pybamm; pybamm.IDAKLUSolver()"
+
+      - name: Build wheels on macOS
+        if: matrix.os == 'macos-latest'
+        run: pipx run cibuildwheel --output-dir wheelhouse
+        env:
           CIBW_BEFORE_BUILD_MACOS: >
-            python -m pip
-            install cmake casadi numpy &&
-            python scripts/fix_casadi_rpath_mac.py && scripts/fix_suitesparse_rpath_mac.sh
-          CIBW_REPAIR_WHEEL_COMMAND_MACOS: >
-            delocate-listdeps {wheel} &&
-            delocate-wheel -v -w {dest_dir} {wheel}
-          CIBW_SKIP: "pp* *musllinux*"
+            python -m pip install --upgrade cmake casadi setuptools wheel
+          CIBW_REPAIR_WHEEL_COMMAND_MACOS: delocate-listdeps {wheel} && delocate-wheel -v -w {dest_dir} {wheel}
+          CIBW_TEST_COMMAND: python -c "import pybamm; pybamm.IDAKLUSolver()"
 
       - name: Upload wheels for Linux
         uses: actions/upload-artifact@v4
@@ -135,9 +168,6 @@ jobs:
         with:
           python-version: 3.12
 
-      - name: Install dependencies
-        run: pip install --upgrade pip setuptools wheel
-
       - name: Build SDist
         run: pipx run build --sdist
 
@@ -149,9 +179,10 @@ jobs:
           if-no-files-found: error
 
   publish_pypi:
-    if: github.event_name != 'schedule' && github.repository_owner == 'pybamm-team'
+    # This job is only of value to PyBaMM and would always be skipped in forks
+    if: github.event_name != 'schedule' && github.repository == 'pybamm-team/PyBaMM'
     name: Upload package to PyPI
-    needs: [build_wheels, build_windows_wheels, build_sdist]
+    needs: [build_macos_and_linux_wheels, build_windows_wheels, build_sdist]
     runs-on: ubuntu-latest
     steps:
       - name: Download all artifacts
@@ -180,7 +211,7 @@ jobs:
           repository-url: https://test.pypi.org/legacy/
 
   open_failure_issue:
-    needs: [build_windows_wheels, build_wheels, build_sdist]
+    needs: [build_windows_wheels, build_macos_and_linux_wheels, build_sdist]
     name: Open an issue if build fails
     if: ${{ always() && contains(needs.*.result, 'failure') }}
     runs-on: ubuntu-latest

.github/workflows/test_on_push.yml

@@ -106,8 +106,7 @@ jobs:
             # Headers and dynamic library files for SuiteSparse and SUNDIALS
             ${{ env.HOME }}/.local/lib/
             ${{ env.HOME }}/.local/include/
-            ${{ env.HOME }}/.local/examples/
-          key: nox-${{ matrix.os }}-pybamm-requires-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/install_KLU_Sundials.py', '**/noxfile.py') }}
+          key: nox-${{ matrix.os }}-pybamm-requires-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/install_KLU_Sundials.py', '**/noxfile.py', '**/test_on_push.yml') }}
 
       - name: Install SuiteSparse and SUNDIALS on GNU/Linux and macOS
         if: matrix.os != 'windows-latest'
@@ -163,8 +162,7 @@ jobs:
             # Headers and dynamic library files for SuiteSparse and SUNDIALS
             ${{ env.HOME }}/.local/lib/
             ${{ env.HOME }}/.local/include/
-            ${{ env.HOME }}/.local/examples/
-          key: nox-${{ matrix.os }}-pybamm-requires-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/install_KLU_Sundials.py', '**/noxfile.py') }}
+          key: nox-${{ matrix.os }}-pybamm-requires-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/install_KLU_Sundials.py', '**/noxfile.py', '**/test_on_push.yml') }}
 
       - name: Install SuiteSparse and SUNDIALS on GNU/Linux
         run: python -m nox -s pybamm-requires
@@ -245,8 +243,7 @@ jobs:
             # Headers and dynamic library files for SuiteSparse and SUNDIALS
             ${{ env.HOME }}/.local/lib/
             ${{ env.HOME }}/.local/include/
-            ${{ env.HOME }}/.local/examples/
-          key: nox-${{ matrix.os }}-pybamm-requires-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/install_KLU_Sundials.py', '**/noxfile.py') }}
+          key: nox-${{ matrix.os }}-pybamm-requires-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/install_KLU_Sundials.py', '**/noxfile.py', '**/test_on_push.yml') }}
 
       - name: Install SuiteSparse and SUNDIALS on GNU/Linux and macOS
         if: matrix.os != 'windows-latest'
@@ -344,8 +341,7 @@ jobs:
             # Headers and dynamic library files for SuiteSparse and SUNDIALS
             ${{ env.HOME }}/.local/lib/
             ${{ env.HOME }}/.local/include/
-            ${{ env.HOME }}/.local/examples/
-          key: nox-${{ matrix.os }}-pybamm-requires-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/install_KLU_Sundials.py', '**/noxfile.py') }}
+          key: nox-${{ matrix.os }}-pybamm-requires-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/install_KLU_Sundials.py', '**/noxfile.py', '**/test_on_push.yml') }}
 
       - name: Install SuiteSparse and SUNDIALS on GNU/Linux
         run: python -m nox -s pybamm-requires
@@ -399,8 +395,7 @@ jobs:
             # Headers and dynamic library files for SuiteSparse and SUNDIALS
             ${{ env.HOME }}/.local/lib/
             ${{ env.HOME }}/.local/include/
-            ${{ env.HOME }}/.local/examples/
-          key: nox-${{ matrix.os }}-pybamm-requires-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/install_KLU_Sundials.py', '**/noxfile.py') }}
+          key: nox-${{ matrix.os }}-pybamm-requires-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/install_KLU_Sundials.py', '**/noxfile.py', '**/test_on_push.yml') }}
 
       - name: Install SuiteSparse and SUNDIALS on GNU/Linux
         run: python -m nox -s pybamm-requires

CMakeLists.txt

@@ -64,26 +64,34 @@ if (NOT DEFINED USE_PYTHON_CASADI)
   set(USE_PYTHON_CASADI TRUE)
 endif()
 
+# Use importlib to find the casadi path without importing it. This is useful
+# to find the path for the build-time dependency, not the run-time dependency.
 execute_process(
     COMMAND "${PYTHON_EXECUTABLE}" -c
-    "import casadi as _; print(_.__path__[0])"
+    "import importlib.util; print(next(iter(importlib.util.find_spec('casadi').submodule_search_locations)))"
     OUTPUT_VARIABLE CASADI_DIR
     OUTPUT_STRIP_TRAILING_WHITESPACE)
 
 if (CASADI_DIR)
   file(TO_CMAKE_PATH ${CASADI_DIR} CASADI_DIR)
-  message("Found python casadi path: ${CASADI_DIR}")
+  message("Found Python casadi path: ${CASADI_DIR}")
 endif()
 
 if(${USE_PYTHON_CASADI})
-  message("Trying to link against python casadi package")
-  find_package(casadi CONFIG PATHS ${CASADI_DIR} REQUIRED)
+  message("Trying to link against Python casadi package")
+  find_package(casadi CONFIG PATHS ${CASADI_DIR} REQUIRED NO_DEFAULT_PATH)
 else()
-  message("Trying to link against any casadi package apart from the python one")
+  message("Trying to link against any casadi package apart from the Python one")
   set(CMAKE_IGNORE_PATH "${CASADI_DIR}/cmake")
   find_package(casadi CONFIG REQUIRED)
 endif()
 
+set_target_properties(
+  idaklu PROPERTIES
+  INSTALL_RPATH "${CASADI_DIR}"
+  INSTALL_RPATH_USE_LINK_PATH TRUE
+)
+
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${PROJECT_SOURCE_DIR})
 # Sundials
 find_package(SUNDIALS REQUIRED)

pybamm/install_odes.py

@@ -5,6 +5,7 @@
 import sys
 import logging
 import subprocess
+from multiprocessing import cpu_count
 
 from pybamm.util import root_dir
 
@@ -16,6 +17,9 @@
 except ModuleNotFoundError:
     NO_WGET = True
 
+# Build in parallel wherever possible
+os.environ["CMAKE_BUILD_PARALLEL_LEVEL"] = str(cpu_count())
+
 
 def download_extract_library(url, directory):
     # Download and extract archive at url