Isolate macOS wheel builds from Homebrew

.github/workflows/wheels-dependencies.sh

@@ -1,11 +1,34 @@
 #!/bin/bash
-# Define custom utilities
-# Test for macOS with [ -n "$IS_MACOS" ]
-if [ -z "$IS_MACOS" ]; then
-    export MB_ML_LIBC=${AUDITWHEEL_POLICY::9}
-    export MB_ML_VER=${AUDITWHEEL_POLICY:9}
+
+# Setup that needs to be done before multibuild utils are invoked
+PROJECTDIR=$(pwd)
+if [[ "$(uname -s)" == "Darwin" ]]; then
+    # Safety check - macOS builds require that CIBW_ARCHS is set, and that it
+    # only contains a single value (even though cibuildwheel) allows multiple
+    # values in CIBW_ARCHS.
+    if [[ -z "$CIBW_ARCHS" ]]; then
+        echo "ERROR: Pillow macOS builds require CIBW_ARCHS be defined."
+        exit 1
+    fi
+    if [[ "$CIBW_ARCHS" == *" "* ]]; then
+        echo "ERROR: Pillow macOS builds only support a single architecture in CIBW_ARCHS."
+        exit 1
+    fi
+
+    # Build macOS dependencies in `build/darwin`
+    # Install them into `build/deps/darwin`
+    WORKDIR=$(pwd)/build/darwin
+    BUILD_PREFIX=$(pwd)/build/deps/darwin
+    PLAT=$CIBW_ARCHS
+else
+    # Build prefix will default to /usr/local
+    WORKDIR=$(pwd)/build
+    PLAT=$CIBW_ARCHS
+    MB_ML_LIBC=${AUDITWHEEL_POLICY::9}
+    MB_ML_VER=${AUDITWHEEL_POLICY:9}
 fi
-export PLAT=$CIBW_ARCHS
+
+# Define custom utilities
 source wheels/multibuild/common_utils.sh
 source wheels/multibuild/library_builders.sh
 if [ -z "$IS_MACOS" ]; then
@@ -38,35 +61,44 @@ BZIP2_VERSION=1.0.8
 LIBXCB_VERSION=1.17.0
 BROTLI_VERSION=1.1.0
 
+function build_pkg_config {
+    if [ -e pkg-config-stamp ]; then return; fi
+    # This essentially duplicates the Homebrew recipe:
+    # https://github.com/Homebrew/homebrew-core/blob/master/Formula/p/pkg-config.rb
+    ORIGINAL_CFLAGS=$CFLAGS
+    CFLAGS="$CFLAGS -Wno-int-conversion"
+    build_simple pkg-config 0.29.2 https://pkg-config.freedesktop.org/releases tar.gz \
+        --disable-debug --disable-host-tool --with-internal-glib \
+        --with-pc-path=$BUILD_PREFIX/share/pkgconfig:$BUILD_PREFIX/lib/pkgconfig \
+        --with-system-include-path=$(xcrun --show-sdk-path --sdk macosx)/usr/include
+    CFLAGS=$ORIGINAL_CFLAGS
+    export PKG_CONFIG=$BUILD_PREFIX/bin/pkg-config
+    touch pkg-config-stamp
+}
+
 function build_brotli {
+    if [ -e brotli-stamp ]; then return; fi
     local cmake=$(get_modern_cmake)
     local out_dir=$(fetch_unpack https://github.com/google/brotli/archive/v$BROTLI_VERSION.tar.gz brotli-$BROTLI_VERSION.tar.gz)
     (cd $out_dir \
-        && $cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX -DCMAKE_INSTALL_NAME_DIR=$BUILD_PREFIX/lib . \
+        && $cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX -DCMAKE_INSTALL_LIBDIR=$BUILD_PREFIX/lib -DCMAKE_INSTALL_NAME_DIR=$BUILD_PREFIX/lib . \
         && make install)
-    if [[ "$MB_ML_LIBC" == "manylinux" ]]; then
-        cp /usr/local/lib64/libbrotli* /usr/local/lib
-        cp /usr/local/lib64/pkgconfig/libbrotli* /usr/local/lib/pkgconfig
-    fi
+    touch brotli-stamp
 }
 
 function build_harfbuzz {
+    if [ -e harfbuzz-stamp ]; then return; fi
     python3 -m pip install meson ninja
 
     local out_dir=$(fetch_unpack https://github.com/harfbuzz/harfbuzz/releases/download/$HARFBUZZ_VERSION/$HARFBUZZ_VERSION.tar.xz harfbuzz-$HARFBUZZ_VERSION.tar.xz)
     (cd $out_dir \
-        && meson setup build --buildtype=release -Dfreetype=enabled -Dglib=disabled)
+        && meson setup build --prefix=$BUILD_PREFIX --libdir=$BUILD_PREFIX/lib --buildtype=release -Dfreetype=enabled -Dglib=disabled)
     (cd $out_dir/build \
         && meson install)
-    if [[ "$MB_ML_LIBC" == "manylinux" ]]; then
-        cp /usr/local/lib64/libharfbuzz* /usr/local/lib
-    fi
+    touch harfbuzz-stamp
 }
 
 function build {
-    if [[ -n "$IS_MACOS" ]] && [[ "$CIBW_ARCHS" == "arm64" ]]; then
-        sudo chown -R runner /usr/local
-    fi
     build_xz
     if [ -z "$IS_ALPINE" ] && [ -z "$IS_MACOS" ]; then
         yum remove -y zlib-devel
@@ -77,67 +109,88 @@ function build {
     if [ -n "$IS_MACOS" ]; then
         build_simple xorgproto 2024.1 https://www.x.org/pub/individual/proto
         build_simple libXau 1.0.11 https://www.x.org/pub/individual/lib
+        build_simple libXdmcp 1.1.5 https://www.x.org/pub/individual/lib
         build_simple libpthread-stubs 0.5 https://xcb.freedesktop.org/dist
-        if [[ "$CIBW_ARCHS" == "arm64" ]]; then
-            cp /usr/local/share/pkgconfig/xcb-proto.pc /usr/local/lib/pkgconfig
-        fi
     else
-        sed s/\${pc_sysrootdir\}// /usr/local/share/pkgconfig/xcb-proto.pc > /usr/local/lib/pkgconfig/xcb-proto.pc
+        sed s/\${pc_sysrootdir\}// $BUILD_PREFIX/share/pkgconfig/xcb-proto.pc > $BUILD_PREFIX/lib/pkgconfig/xcb-proto.pc
     fi
     build_simple libxcb $LIBXCB_VERSION https://www.x.org/releases/individual/lib
 
     build_libjpeg_turbo
-    build_tiff
+    if [ -n "$IS_MACOS" ]; then
+        # Custom tiff build to include jpeg; by default, configure won't include
+        # headers/libs in the custom macOS prefix. Explicitly disable webp and
+        # zstd, because on x86_64 macs, it will pick up the Homebrew versions of
+        # webp and zstd from /usr/local.
+        build_simple tiff $TIFF_VERSION https://download.osgeo.org/libtiff tar.gz \
+            --with-jpeg-include-dir=$BUILD_PREFIX/include --with-jpeg-lib-dir=$BUILD_PREFIX/lib \
+            --disable-webp --disable-zstd --disable-libdeflate
+    else
+        build_tiff
+    fi
+
     build_libpng
     build_lcms2
     build_openjpeg
 
     ORIGINAL_CFLAGS=$CFLAGS
     CFLAGS="$CFLAGS -O3 -DNDEBUG"
-    if [[ -n "$IS_MACOS" ]]; then
-        CFLAGS="$CFLAGS -Wl,-headerpad_max_install_names"
-    fi
     build_libwebp
     CFLAGS=$ORIGINAL_CFLAGS
 
     build_brotli
 
     if [ -n "$IS_MACOS" ]; then
         # Custom freetype build
-        build_simple freetype $FREETYPE_VERSION https://download.savannah.gnu.org/releases/freetype tar.gz --with-harfbuzz=no
+        build_simple freetype $FREETYPE_VERSION https://download.savannah.gnu.org/releases/freetype tar.gz --without-harfbuzz
     else
         build_freetype
     fi
 
     build_harfbuzz
 }
 
+# Perform all dependency builds in the build subfolder.
+mkdir -p $WORKDIR
+pushd $WORKDIR > /dev/null
+
 # Any stuff that you need to do before you start building the wheels
 # Runs in the root directory of this repository.
-curl -fsSL -o pillow-depends-main.zip https://github.com/python-pillow/pillow-depends/archive/main.zip
-untar pillow-depends-main.zip
-
-if [[ -n "$IS_MACOS" ]]; then
-  # libdeflate may cause a minimum target error when repairing the wheel
-  # libtiff and libxcb cause a conflict with building libtiff and libxcb
-  # libxau and libxdmcp cause an issue on macOS < 11
-  # remove cairo to fix building harfbuzz on arm64
-  # remove lcms2 and libpng to fix building openjpeg on arm64
-  # remove jpeg-turbo to avoid inclusion on arm64
-  # remove webp and zstd to avoid inclusion on x86_64
-  # curl from brew requires zstd, use system curl
-  brew remove --ignore-dependencies libpng libtiff libxcb libxau libxdmcp curl cairo lcms2 zstd
-  if [[ "$CIBW_ARCHS" == "arm64" ]]; then
-    brew remove --ignore-dependencies jpeg-turbo
-  else
-    brew remove --ignore-dependencies libdeflate webp
+if [[ ! -d $WORKDIR/pillow-depends-main ]]; then
+  if [[ ! -f $PROJECTDIR/pillow-depends-main.zip ]]; then
+    echo "Download pillow dependency sources..."
+    curl -fSL -o $PROJECTDIR/pillow-depends-main.zip https://github.com/python-pillow/pillow-depends/archive/main.zip
   fi
+  echo "Unpacking pillow dependency sources..."
+  untar $PROJECTDIR/pillow-depends-main.zip
+fi
 
-  brew install pkg-config
+if [[ -n "$IS_MACOS" ]]; then
+    # Homebrew (or similar packaging environments) install can contain some of
+    # the libraries that we're going to build. However, they may be compiled
+    # with a MACOSX_DEPLOYMENT_TARGET that doesn't match what we want to use,
+    # and they may bring in other dependencies that we don't want. The same will
+    # be true of any other locations on the path. To avoid conflicts, strip the
+    # path down to the bare mimimum (which, on macOS, won't include any
+    # development dependencies).
+    export PATH="$BUILD_PREFIX/bin:$(dirname $(which python3)):/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin"
+    export CMAKE_PREFIX_PATH=$BUILD_PREFIX
+
+    # Link the brew command into our isolated build directory.
+    mkdir -p "$BUILD_PREFIX/bin"
+    mkdir -p "$BUILD_PREFIX/lib"
+
+    # Ensure pkg-config is available
+    build_pkg_config
+    # Ensure cmake is available
+    python3 -m pip install cmake
 fi
 
 wrap_wheel_builder build
 
+# Return to the project root to finish the build
+popd > /dev/null
+
 # Append licenses
 for filename in wheels/dependency_licenses/*; do
   echo -e "\n\n----\n\n$(basename $filename | cut -f 1 -d '.')\n" | cat >> LICENSE

.github/workflows/wheels-test.sh

@@ -1,12 +1,27 @@
 #!/bin/bash
 set -e
 
+# Ensure fribidi is installed by the system.
 if [[ "$OSTYPE" == "darwin"* ]]; then
-    brew install fribidi
-    export PKG_CONFIG_PATH="/usr/local/opt/openblas/lib/pkgconfig"
-    if [ -f /opt/homebrew/lib/libfribidi.dylib ]; then
-        sudo cp /opt/homebrew/lib/libfribidi.dylib /usr/local/lib
+    # If Homebrew is on the path during the build, it may leak into the wheels.
+    # However, we *do* need Homebrew to provide a copy of fribidi for
+    # testing purposes so that we can verify the fribidi shim works as expected.
+    if [[ "$(uname -m)" == "x86_64" ]]; then
+        # Use the "installed" location, rather than /usr/local, for two reasons:
+        # firstly, Homebrew allows libraries to be *installed*, but not linked;
+        # and secondly, we don't want any *other* leakage from /usr/local.
+        HOMEBREW_HOME=/usr/local/Homebrew
+    else
+        HOMEBREW_HOME=/opt/homebrew
     fi
+    $HOMEBREW_HOME/bin/brew install fribidi
+
+    # Add the lib folder for fribidi so that the vendored library can be found.
+    # Don't use /opt/homebrew/lib directly - use the lib folder where the
+    # installed copy of fribidi is cellared. This ensures we don't pick up the
+    # Homebrew version of any other library that we're dependent on (most notably,
+    # freetype).
+    export DYLD_LIBRARY_PATH=$(dirname $(realpath $HOMEBREW_HOME/lib/libfribidi.dylib))
 elif [ "${AUDITWHEEL_POLICY::9}" == "musllinux" ]; then
     apk add curl fribidi
 else

.gitignore

@@ -19,6 +19,7 @@ lib64/
 parts/
 sdist/
 var/
+wheelhouse/
 *.egg-info/
 .installed.cfg
 *.egg
@@ -90,5 +91,9 @@ Tests/images/msp
 Tests/images/picins
 Tests/images/sunraster
 
+# Test and dependency downloads
+pillow-depends-main.zip
+pillow-test-images.zip
+
 # pyinstaller
 *.spec

pyproject.toml

@@ -93,10 +93,17 @@ version = { attr = "PIL.__version__" }
 [tool.cibuildwheel]
 before-all = ".github/workflows/wheels-dependencies.sh"
 build-verbosity = 1
+
 config-settings = "raqm=enable raqm=vendor fribidi=vendor imagequant=disable"
+# Add an explicit dependencies prefix for macOS.
+macos.config-settings = "raqm=enable raqm=vendor fribidi=vendor imagequant=disable dependencies-prefix=./build/deps/darwin"
+
 test-command = "cd {project} && .github/workflows/wheels-test.sh"
 test-extras = "tests"
 
+[tool.cibuildwheel.macos.environment]
+PATH = "$(pwd)/build/deps/darwin/bin:$(dirname $(which python3)):/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin"
+
 [tool.black]
 exclude = "wheels/multibuild"