From 0fe55d611505fefaaa0d4b7b3a87b803714762de Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Fri, 25 Oct 2024 09:23:13 +0800 Subject: [PATCH 01/25] Isolate macOS build from Homebrew. --- .github/workflows/wheels-dependencies.sh | 98 +++++++++++++++++------- .github/workflows/wheels-test.sh | 15 ++-- .gitignore | 1 + pyproject.toml | 7 ++ setup.py | 90 +++++++++++++--------- wheels/multibuild | 2 +- 6 files changed, 139 insertions(+), 74 deletions(-) diff --git a/.github/workflows/wheels-dependencies.sh b/.github/workflows/wheels-dependencies.sh index 97c1adf098a..35b04c6d652 100755 --- a/.github/workflows/wheels-dependencies.sh +++ b/.github/workflows/wheels-dependencies.sh @@ -23,6 +23,8 @@ OPENJPEG_VERSION=2.5.2 XZ_VERSION=5.6.3 TIFF_VERSION=4.6.0 LCMS2_VERSION=2.16 +RAQM_VERSION=0.7.1 +FRIBIDI_VERSION=1.0.16 if [[ -n "$IS_MACOS" ]]; then GIFLIB_VERSION=5.2.2 else @@ -38,7 +40,23 @@ BZIP2_VERSION=1.0.8 LIBXCB_VERSION=1.17.0 BROTLI_VERSION=1.1.0 +function build_pkg_config { + if [ -e pkg-config-stamp ]; then return; fi + # This essentially duplicates the Homebrew recipe: + # https://github.com/Homebrew/homebrew-core/blob/master/Formula/p/pkg-config.rb + ORIGINAL_CFLAGS=$CFLAGS + CFLAGS="$CFLAGS -Wno-int-conversion" + build_simple pkg-config 0.29.2 https://pkg-config.freedesktop.org/releases tar.gz \ + --disable-debug --disable-host-tool --with-internal-glib \ + --with-pc-path=$BUILD_PREFIX/share/pkgconfig:$BUILD_PREFIX/lib/pkgconfig \ + --with-system-include-path=$(xcrun --show-sdk-path --sdk macosx)/usr/include + CFLAGS=$ORIGINAL_CFLAGS + export PKG_CONFIG=$BUILD_PREFIX/bin/pkg-config + touch pkg-config-stamp +} + function build_brotli { + if [ -e brotli-stamp ]; then return; fi local cmake=$(get_modern_cmake) local out_dir=$(fetch_unpack https://github.com/google/brotli/archive/v$BROTLI_VERSION.tar.gz brotli-$BROTLI_VERSION.tar.gz) (cd $out_dir \ @@ -48,25 +66,25 @@ function build_brotli { cp /usr/local/lib64/libbrotli* /usr/local/lib cp /usr/local/lib64/pkgconfig/libbrotli* /usr/local/lib/pkgconfig fi + touch brotli-stamp } function build_harfbuzz { - python3 -m pip install meson ninja + if [ -e harfbuzz-stamp ]; then return; fi + python -m pip install meson ninja local out_dir=$(fetch_unpack https://github.com/harfbuzz/harfbuzz/releases/download/$HARFBUZZ_VERSION/$HARFBUZZ_VERSION.tar.xz harfbuzz-$HARFBUZZ_VERSION.tar.xz) (cd $out_dir \ - && meson setup build --buildtype=release -Dfreetype=enabled -Dglib=disabled) + && meson setup build --prefix=$BUILD_PREFIX --buildtype=release -Dfreetype=enabled -Dglib=disabled) (cd $out_dir/build \ && meson install) if [[ "$MB_ML_LIBC" == "manylinux" ]]; then cp /usr/local/lib64/libharfbuzz* /usr/local/lib fi + touch harfbuzz-stamp } function build { - if [[ -n "$IS_MACOS" ]] && [[ "$CIBW_ARCHS" == "arm64" ]]; then - sudo chown -R runner /usr/local - fi build_xz if [ -z "$IS_ALPINE" ] && [ -z "$IS_MACOS" ]; then yum remove -y zlib-devel @@ -77,17 +95,23 @@ function build { if [ -n "$IS_MACOS" ]; then build_simple xorgproto 2024.1 https://www.x.org/pub/individual/proto build_simple libXau 1.0.11 https://www.x.org/pub/individual/lib + build_simple libXdmcp 1.1.5 https://www.x.org/pub/individual/lib build_simple libpthread-stubs 0.5 https://xcb.freedesktop.org/dist - if [[ "$CIBW_ARCHS" == "arm64" ]]; then - cp /usr/local/share/pkgconfig/xcb-proto.pc /usr/local/lib/pkgconfig - fi else sed s/\${pc_sysrootdir\}// /usr/local/share/pkgconfig/xcb-proto.pc > /usr/local/lib/pkgconfig/xcb-proto.pc fi build_simple libxcb $LIBXCB_VERSION https://www.x.org/releases/individual/lib build_libjpeg_turbo - build_tiff + if [ -n "$IS_MACOS" ]; then + # Custom tiff build to include jpeg; by default, configure won't include + # headers/libs in the custom macOS prefix + build_simple tiff $TIFF_VERSION https://download.osgeo.org/libtiff tar.gz \ + --with-jpeg-include-dir=$BUILD_PREFIX/include --with-jpeg-lib-dir=$BUILD_PREFIX/lib + else + build_tiff + fi + build_libpng build_lcms2 build_openjpeg @@ -113,33 +137,55 @@ function build { fi build_harfbuzz + + if [ -n "$IS_MACOS" ]; then + build_simple fribidi $FRIBIDI_VERSION https://github.com/fribidi/fribidi/releases/download/v$FRIBIDI_VERSION tar.xz --enable-shared + build_simple raqm $RAQM_VERSION https://github.com/Host_Oman/libraqm/releases/download/v$RAQM_VERSION tar.gz --enable-shared + fi } +# Perform all dependency builds in the build subfolder. +mkdir -p build +pushd build > /dev/null + # Any stuff that you need to do before you start building the wheels # Runs in the root directory of this repository. -curl -fsSL -o pillow-depends-main.zip https://github.com/python-pillow/pillow-depends/archive/main.zip -untar pillow-depends-main.zip - -if [[ -n "$IS_MACOS" ]]; then - # libtiff and libxcb cause a conflict with building libtiff and libxcb - # libxau and libxdmcp cause an issue on macOS < 11 - # remove cairo to fix building harfbuzz on arm64 - # remove lcms2 and libpng to fix building openjpeg on arm64 - # remove jpeg-turbo to avoid inclusion on arm64 - # remove webp and zstd to avoid inclusion on x86_64 - # curl from brew requires zstd, use system curl - brew remove --ignore-dependencies libpng libtiff libxcb libxau libxdmcp curl cairo lcms2 zstd - if [[ "$CIBW_ARCHS" == "arm64" ]]; then - brew remove --ignore-dependencies jpeg-turbo - else - brew remove --ignore-dependencies webp +if [[ ! -d pillow-depends-main ]]; then + if [[ ! -f pillow-depends-main.zip ]]; then + echo "Download pillow dependency sources..." + curl -fSL -o pillow-depends-main.zip https://github.com/python-pillow/pillow-depends/archive/main.zip fi + untar pillow-depends-main.zip +fi - brew install pkg-config +if [[ -n "$IS_MACOS" ]]; then + # Build and install into the `deps` folder. + BUILD_PREFIX=$(pwd)/deps + + # Homebrew (or similar packaging environments) install can contain some of + # the libraries that we're going to build. However, they may be compiled + # with a MACOSX_DEPLOYMENT_TARGET that doesn't match what we want to use, + # and they may bring in other dependencies that we don't want. The same will + # be true of any other locations on the path. To avoid conflicts, strip the + # path down to the bare mimimum (which, on macOS, won't include any + # development dependencies). + export PATH="$BUILD_PREFIX/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin:$(dirname $(which python))" + export CMAKE_PREFIX_PATH=$BUILD_PREFIX + + # Link the brew command into our isolated build directory. + mkdir -p "$BUILD_PREFIX/bin" + mkdir -p "$BUILD_PREFIX/lib" + + # Ensure pkg-confg and cmake are available + build_pkg_config + python -m pip install cmake fi wrap_wheel_builder build +# Return to the project root to finish the build +popd > /dev/null + # Append licenses for filename in wheels/dependency_licenses/*; do echo -e "\n\n----\n\n$(basename $filename | cut -f 1 -d '.')\n" | cat >> LICENSE diff --git a/.github/workflows/wheels-test.sh b/.github/workflows/wheels-test.sh index b30b1725f94..05f5162c78a 100755 --- a/.github/workflows/wheels-test.sh +++ b/.github/workflows/wheels-test.sh @@ -1,16 +1,13 @@ #!/bin/bash set -e -if [[ "$OSTYPE" == "darwin"* ]]; then - brew install fribidi - export PKG_CONFIG_PATH="/usr/local/opt/openblas/lib/pkgconfig" - if [ -f /opt/homebrew/lib/libfribidi.dylib ]; then - sudo cp /opt/homebrew/lib/libfribidi.dylib /usr/local/lib +# For Unix, Ensure fibidi is installed by the system. +if [[ "$OSTYPE" != "darwin"* ]]; then + if [ "${AUDITWHEEL_POLICY::9}" == "musllinux" ]; then + apk add curl fribidi + else + yum install -y fribidi fi -elif [ "${AUDITWHEEL_POLICY::9}" == "musllinux" ]; then - apk add curl fribidi -else - yum install -y fribidi fi python3 -m pip install numpy diff --git a/.gitignore b/.gitignore index 1dd6c917524..9b577325730 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ lib64/ parts/ sdist/ var/ +wheelhouse/ *.egg-info/ .installed.cfg *.egg diff --git a/pyproject.toml b/pyproject.toml index c55be769341..53546f0ab57 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,10 +93,17 @@ version = { attr = "PIL.__version__" } [tool.cibuildwheel] before-all = ".github/workflows/wheels-dependencies.sh" build-verbosity = 1 + config-settings = "raqm=enable raqm=vendor fribidi=vendor imagequant=disable" +# Add an explicit dependencies prefix for macOS, and don't request vendored libraries. +macos.config-settings = "raqm=enable imagequant=disable dependencies-prefix=./build/deps" + test-command = "cd {project} && .github/workflows/wheels-test.sh" test-extras = "tests" +[tool.cibuildwheel.macos.environment] +PATH = "$(pwd)/build/deps/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin:$(dirname $(which python))" + [tool.black] exclude = "wheels/multibuild" diff --git a/setup.py b/setup.py index 60707083f6e..cb1da7c73f3 100644 --- a/setup.py +++ b/setup.py @@ -344,6 +344,11 @@ def __iter__(self) -> Iterator[str]: for x in ("raqm", "fribidi") ] + [ + ( + "dependencies-prefix", + None, + "The prefix where build dependencies are located.", + ), ("disable-platform-guessing", None, "Disable platform guessing on Linux"), ("debug", None, "Debug logging"), ] @@ -355,6 +360,7 @@ def check_configuration(option: str, value: str) -> bool | None: return True if value in configuration.get(option, []) else None def initialize_options(self) -> None: + self.dependencies_prefix = configuration.get("dependencies-prefix", []) self.disable_platform_guessing = self.check_configuration( "platform-guessing", "disable" ) @@ -564,48 +570,56 @@ def build_extensions(self) -> None: ) elif sys.platform == "darwin": - # attempt to make sure we pick freetype2 over other versions - _add_directory(include_dirs, "/sw/include/freetype2") - _add_directory(include_dirs, "/sw/lib/freetype2/include") - # fink installation directories - _add_directory(library_dirs, "/sw/lib") - _add_directory(include_dirs, "/sw/include") - # darwin ports installation directories - _add_directory(library_dirs, "/opt/local/lib") - _add_directory(include_dirs, "/opt/local/include") - - # if Homebrew is installed, use its lib and include directories - try: - prefix = ( - subprocess.check_output(["brew", "--prefix"]) - .strip() - .decode("latin1") - ) - except Exception: - # Homebrew not installed - prefix = None + if self.dependencies_prefix: + # Use the explicitly provided prefixes for dependencies. + for prefix in self.dependencies_prefix: + _add_directory(library_dirs, os.path.join(prefix, "lib")) + _add_directory(include_dirs, os.path.join(prefix, "include")) + else: + # Guess the dependency locations based on homebrew/fink/macports + # attempt to make sure we pick freetype2 over other versions + _add_directory(include_dirs, "/sw/include/freetype2") + _add_directory(include_dirs, "/sw/lib/freetype2/include") + # fink installation directories + _add_directory(library_dirs, "/sw/lib") + _add_directory(include_dirs, "/sw/include") + # darwin ports installation directories + _add_directory(library_dirs, "/opt/local/lib") + _add_directory(include_dirs, "/opt/local/include") + + # if Homebrew is installed, use its lib and include directories + try: + prefix = ( + subprocess.check_output(["brew", "--prefix"]) + .strip() + .decode("latin1") + ) + except Exception: + # Homebrew not installed + prefix = None - ft_prefix = None + ft_prefix = None - if prefix: - # add Homebrew's include and lib directories - _add_directory(library_dirs, os.path.join(prefix, "lib")) - _add_directory(include_dirs, os.path.join(prefix, "include")) - _add_directory( - include_dirs, os.path.join(prefix, "opt", "zlib", "include") - ) - ft_prefix = os.path.join(prefix, "opt", "freetype") + if prefix: + # add Homebrew's include and lib directories + _add_directory(library_dirs, os.path.join(prefix, "lib")) + _add_directory(include_dirs, os.path.join(prefix, "include")) + _add_directory( + include_dirs, os.path.join(prefix, "opt", "zlib", "include") + ) + ft_prefix = os.path.join(prefix, "opt", "freetype") - if ft_prefix and os.path.isdir(ft_prefix): - # freetype might not be linked into Homebrew's prefix - _add_directory(library_dirs, os.path.join(ft_prefix, "lib")) - _add_directory(include_dirs, os.path.join(ft_prefix, "include")) - else: - # fall back to freetype from XQuartz if - # Homebrew's freetype is missing - _add_directory(library_dirs, "/usr/X11/lib") - _add_directory(include_dirs, "/usr/X11/include") + if ft_prefix and os.path.isdir(ft_prefix): + # freetype might not be linked into Homebrew's prefix + _add_directory(library_dirs, os.path.join(ft_prefix, "lib")) + _add_directory(include_dirs, os.path.join(ft_prefix, "include")) + else: + # fall back to freetype from XQuartz if + # Homebrew's freetype is missing + _add_directory(library_dirs, "/usr/X11/lib") + _add_directory(include_dirs, "/usr/X11/include") + # Add the macOS SDK path. sdk_path = self.get_macos_sdk_path() if sdk_path: _add_directory(library_dirs, os.path.join(sdk_path, "usr", "lib")) diff --git a/wheels/multibuild b/wheels/multibuild index 452dd2d1705..8a9333e307f 160000 --- a/wheels/multibuild +++ b/wheels/multibuild @@ -1 +1 @@ -Subproject commit 452dd2d1705f6b2375369a6570c415beb3163f70 +Subproject commit 8a9333e307fa3f24db2089a66f25e38a8272c261 From fc35fcc5ea844cd0184af6e999d0f35835a4254a Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Fri, 25 Oct 2024 10:55:30 +0800 Subject: [PATCH 02/25] Cleanups and typos identified by code review. --- .github/workflows/wheels-dependencies.sh | 8 ++++---- .github/workflows/wheels-test.sh | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/wheels-dependencies.sh b/.github/workflows/wheels-dependencies.sh index 35b04c6d652..2c0835e7e56 100755 --- a/.github/workflows/wheels-dependencies.sh +++ b/.github/workflows/wheels-dependencies.sh @@ -23,7 +23,7 @@ OPENJPEG_VERSION=2.5.2 XZ_VERSION=5.6.3 TIFF_VERSION=4.6.0 LCMS2_VERSION=2.16 -RAQM_VERSION=0.7.1 +RAQM_VERSION=0.10.2 FRIBIDI_VERSION=1.0.16 if [[ -n "$IS_MACOS" ]]; then GIFLIB_VERSION=5.2.2 @@ -71,7 +71,7 @@ function build_brotli { function build_harfbuzz { if [ -e harfbuzz-stamp ]; then return; fi - python -m pip install meson ninja + python3 -m pip install meson ninja local out_dir=$(fetch_unpack https://github.com/harfbuzz/harfbuzz/releases/download/$HARFBUZZ_VERSION/$HARFBUZZ_VERSION.tar.xz harfbuzz-$HARFBUZZ_VERSION.tar.xz) (cd $out_dir \ @@ -176,9 +176,9 @@ if [[ -n "$IS_MACOS" ]]; then mkdir -p "$BUILD_PREFIX/bin" mkdir -p "$BUILD_PREFIX/lib" - # Ensure pkg-confg and cmake are available + # Ensure pkg-config and cmake are available build_pkg_config - python -m pip install cmake + python3 -m pip install cmake fi wrap_wheel_builder build diff --git a/.github/workflows/wheels-test.sh b/.github/workflows/wheels-test.sh index 05f5162c78a..3a8fc87ba1e 100755 --- a/.github/workflows/wheels-test.sh +++ b/.github/workflows/wheels-test.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -# For Unix, Ensure fibidi is installed by the system. +# For Unix, ensure fribidi is installed by the system. if [[ "$OSTYPE" != "darwin"* ]]; then if [ "${AUDITWHEEL_POLICY::9}" == "musllinux" ]; then apk add curl fribidi From 00809a25516e1396e3e3ee38a760733efffbd1a9 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Fri, 25 Oct 2024 13:42:46 +0800 Subject: [PATCH 03/25] Tweaks to ensure isolation from Homebrew on x86_64. --- .github/workflows/wheels-dependencies.sh | 34 +++++++++++++++++------- pyproject.toml | 2 +- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/.github/workflows/wheels-dependencies.sh b/.github/workflows/wheels-dependencies.sh index 2c0835e7e56..dbabd4aef28 100755 --- a/.github/workflows/wheels-dependencies.sh +++ b/.github/workflows/wheels-dependencies.sh @@ -4,6 +4,9 @@ if [ -z "$IS_MACOS" ]; then export MB_ML_LIBC=${AUDITWHEEL_POLICY::9} export MB_ML_VER=${AUDITWHEEL_POLICY:9} + + # Build and install into the `build/deps` folder. + BUILD_PREFIX=$(pwd)/build/deps fi export PLAT=$CIBW_ARCHS source wheels/multibuild/common_utils.sh @@ -84,6 +87,18 @@ function build_harfbuzz { touch harfbuzz-stamp } +function build_raqm { + if [ -e raqm-stamp ]; then return; fi + python3 -m pip install meson ninja + + local out_dir=$(fetch_unpack https://github.com/HOST-Oman/libraqm/releases/download/v$RAQM_VERSION/raqm-$RAQM_VERSION.tar.xz raqm-$RAQM_VERSION.tar.xz) + (cd $out_dir \ + && meson setup build --prefix=$BUILD_PREFIX) + (cd $out_dir/build \ + && meson install) + touch raqm-stamp +} + function build { build_xz if [ -z "$IS_ALPINE" ] && [ -z "$IS_MACOS" ]; then @@ -105,9 +120,12 @@ function build { build_libjpeg_turbo if [ -n "$IS_MACOS" ]; then # Custom tiff build to include jpeg; by default, configure won't include - # headers/libs in the custom macOS prefix + # headers/libs in the custom macOS prefix. Explicitly disable webp and + # zstd, because on x86_64 macs, it will pick up the Homebrew versions of + # webp and zstd from /usr/local. build_simple tiff $TIFF_VERSION https://download.osgeo.org/libtiff tar.gz \ - --with-jpeg-include-dir=$BUILD_PREFIX/include --with-jpeg-lib-dir=$BUILD_PREFIX/lib + --with-jpeg-include-dir=$BUILD_PREFIX/include --with-jpeg-lib-dir=$BUILD_PREFIX/lib \ + --disable-webp --disable-zstd else build_tiff fi @@ -140,7 +158,7 @@ function build { if [ -n "$IS_MACOS" ]; then build_simple fribidi $FRIBIDI_VERSION https://github.com/fribidi/fribidi/releases/download/v$FRIBIDI_VERSION tar.xz --enable-shared - build_simple raqm $RAQM_VERSION https://github.com/Host_Oman/libraqm/releases/download/v$RAQM_VERSION tar.gz --enable-shared + build_raqm fi } @@ -159,9 +177,6 @@ if [[ ! -d pillow-depends-main ]]; then fi if [[ -n "$IS_MACOS" ]]; then - # Build and install into the `deps` folder. - BUILD_PREFIX=$(pwd)/deps - # Homebrew (or similar packaging environments) install can contain some of # the libraries that we're going to build. However, they may be compiled # with a MACOSX_DEPLOYMENT_TARGET that doesn't match what we want to use, @@ -169,16 +184,17 @@ if [[ -n "$IS_MACOS" ]]; then # be true of any other locations on the path. To avoid conflicts, strip the # path down to the bare mimimum (which, on macOS, won't include any # development dependencies). - export PATH="$BUILD_PREFIX/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin:$(dirname $(which python))" + export PATH="$BUILD_PREFIX/bin:$(dirname $(which python3)):/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin" export CMAKE_PREFIX_PATH=$BUILD_PREFIX # Link the brew command into our isolated build directory. mkdir -p "$BUILD_PREFIX/bin" mkdir -p "$BUILD_PREFIX/lib" - # Ensure pkg-config and cmake are available + # Ensure pkg-config is available build_pkg_config - python3 -m pip install cmake + # Ensure cmake is available + python3 -m pip cmake fi wrap_wheel_builder build diff --git a/pyproject.toml b/pyproject.toml index 53546f0ab57..3839cf8402a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -102,7 +102,7 @@ test-command = "cd {project} && .github/workflows/wheels-test.sh" test-extras = "tests" [tool.cibuildwheel.macos.environment] -PATH = "$(pwd)/build/deps/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin:$(dirname $(which python))" +PATH = "$(pwd)/build/deps/bin:$(dirname $(which python3)):/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin" [tool.black] exclude = "wheels/multibuild" From 06dbfedb0fbfcf38f6d6298ac4979e82fb653956 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Fri, 25 Oct 2024 13:51:37 +0800 Subject: [PATCH 04/25] Bump the multibuild version to fix jpeg-turbo issue. --- wheels/multibuild | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wheels/multibuild b/wheels/multibuild index 8a9333e307f..22e8ad0c69a 160000 --- a/wheels/multibuild +++ b/wheels/multibuild @@ -1 +1 @@ -Subproject commit 8a9333e307fa3f24db2089a66f25e38a8272c261 +Subproject commit 22e8ad0c69acdb0c94abab74da6e09f2810d50d5 From 5a8373e527bbea93cb9c3be87c206683f488ad8e Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Fri, 25 Oct 2024 14:47:51 +0800 Subject: [PATCH 05/25] Correct a dumb pip invocation error. --- .github/workflows/wheels-dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels-dependencies.sh b/.github/workflows/wheels-dependencies.sh index dbabd4aef28..0990943d32a 100755 --- a/.github/workflows/wheels-dependencies.sh +++ b/.github/workflows/wheels-dependencies.sh @@ -194,7 +194,7 @@ if [[ -n "$IS_MACOS" ]]; then # Ensure pkg-config is available build_pkg_config # Ensure cmake is available - python3 -m pip cmake + python3 -m pip install cmake fi wrap_wheel_builder build From 140a06e0fda60b7474dbdfde53b2ad2f1d344603 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Fri, 25 Oct 2024 16:04:56 +0800 Subject: [PATCH 06/25] Explicitly disable libdeflate on libtiff. --- .github/workflows/wheels-dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels-dependencies.sh b/.github/workflows/wheels-dependencies.sh index 0990943d32a..c6f57200f40 100755 --- a/.github/workflows/wheels-dependencies.sh +++ b/.github/workflows/wheels-dependencies.sh @@ -125,7 +125,7 @@ function build { # webp and zstd from /usr/local. build_simple tiff $TIFF_VERSION https://download.osgeo.org/libtiff tar.gz \ --with-jpeg-include-dir=$BUILD_PREFIX/include --with-jpeg-lib-dir=$BUILD_PREFIX/lib \ - --disable-webp --disable-zstd + --disable-webp --disable-zstd --disable-libdeflate else build_tiff fi From 0961d3d22bc966f19f68ce01db6d07373dfbe6bc Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Fri, 25 Oct 2024 16:10:29 +0800 Subject: [PATCH 07/25] Possible fix for linux build failures. --- .github/workflows/wheels-dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels-dependencies.sh b/.github/workflows/wheels-dependencies.sh index c6f57200f40..4d58bee901f 100755 --- a/.github/workflows/wheels-dependencies.sh +++ b/.github/workflows/wheels-dependencies.sh @@ -113,7 +113,7 @@ function build { build_simple libXdmcp 1.1.5 https://www.x.org/pub/individual/lib build_simple libpthread-stubs 0.5 https://xcb.freedesktop.org/dist else - sed s/\${pc_sysrootdir\}// /usr/local/share/pkgconfig/xcb-proto.pc > /usr/local/lib/pkgconfig/xcb-proto.pc + sed s/\${pc_sysrootdir\}// $BUILD_PREFIX/share/pkgconfig/xcb-proto.pc > /usr/local/lib/pkgconfig/xcb-proto.pc fi build_simple libxcb $LIBXCB_VERSION https://www.x.org/releases/individual/lib From 43c34fc0a5c22b1c00152a246fd39fe02925043d Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Fri, 25 Oct 2024 17:20:34 +0800 Subject: [PATCH 08/25] Copy manylinux lib64 files from the correct built prefix. --- .github/workflows/wheels-dependencies.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/wheels-dependencies.sh b/.github/workflows/wheels-dependencies.sh index 4d58bee901f..9dae287170a 100755 --- a/.github/workflows/wheels-dependencies.sh +++ b/.github/workflows/wheels-dependencies.sh @@ -66,8 +66,8 @@ function build_brotli { && $cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX -DCMAKE_INSTALL_NAME_DIR=$BUILD_PREFIX/lib . \ && make install) if [[ "$MB_ML_LIBC" == "manylinux" ]]; then - cp /usr/local/lib64/libbrotli* /usr/local/lib - cp /usr/local/lib64/pkgconfig/libbrotli* /usr/local/lib/pkgconfig + cp $BUILD_PREFIX/lib64/libbrotli* $BUILD_PREFIX/lib + cp $BUILD_PREFIX/lib64/pkgconfig/libbrotli* $BUILD_PREFIX/lib/pkgconfig fi touch brotli-stamp } @@ -82,7 +82,7 @@ function build_harfbuzz { (cd $out_dir/build \ && meson install) if [[ "$MB_ML_LIBC" == "manylinux" ]]; then - cp /usr/local/lib64/libharfbuzz* /usr/local/lib + cp $BUILD_PREFIX/lib64/libharfbuzz* $BUILD_PREFIX/lib fi touch harfbuzz-stamp } @@ -113,7 +113,7 @@ function build { build_simple libXdmcp 1.1.5 https://www.x.org/pub/individual/lib build_simple libpthread-stubs 0.5 https://xcb.freedesktop.org/dist else - sed s/\${pc_sysrootdir\}// $BUILD_PREFIX/share/pkgconfig/xcb-proto.pc > /usr/local/lib/pkgconfig/xcb-proto.pc + sed s/\${pc_sysrootdir\}// $BUILD_PREFIX/share/pkgconfig/xcb-proto.pc > $BUILD_PREFIX/lib/pkgconfig/xcb-proto.pc fi build_simple libxcb $LIBXCB_VERSION https://www.x.org/releases/individual/lib @@ -133,8 +133,8 @@ function build { build_libpng build_lcms2 build_openjpeg - if [ -f /usr/local/lib64/libopenjp2.so ]; then - cp /usr/local/lib64/libopenjp2.so /usr/local/lib + if [ -f $BUILD_PREFIX/lib64/libopenjp2.so ]; then + cp $BUILD_PREFIX/lib64/libopenjp2.so $BUILD_PREFIX/lib fi ORIGINAL_CFLAGS=$CFLAGS From 08554684b3f382f3d729db5e6cd1fe94126dab66 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Tue, 29 Oct 2024 07:46:59 +0800 Subject: [PATCH 09/25] Revert fribidi/raqm changes for macOS builds. --- .github/workflows/wheels-dependencies.sh | 5 ----- .github/workflows/wheels-test.sh | 21 ++++++++++++++++----- pyproject.toml | 4 ++-- setup.py | 2 +- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/.github/workflows/wheels-dependencies.sh b/.github/workflows/wheels-dependencies.sh index 9dae287170a..22084237118 100755 --- a/.github/workflows/wheels-dependencies.sh +++ b/.github/workflows/wheels-dependencies.sh @@ -155,11 +155,6 @@ function build { fi build_harfbuzz - - if [ -n "$IS_MACOS" ]; then - build_simple fribidi $FRIBIDI_VERSION https://github.com/fribidi/fribidi/releases/download/v$FRIBIDI_VERSION tar.xz --enable-shared - build_raqm - fi } # Perform all dependency builds in the build subfolder. diff --git a/.github/workflows/wheels-test.sh b/.github/workflows/wheels-test.sh index 3a8fc87ba1e..e671c8d45a8 100755 --- a/.github/workflows/wheels-test.sh +++ b/.github/workflows/wheels-test.sh @@ -1,13 +1,24 @@ #!/bin/bash set -e -# For Unix, ensure fribidi is installed by the system. -if [[ "$OSTYPE" != "darwin"* ]]; then - if [ "${AUDITWHEEL_POLICY::9}" == "musllinux" ]; then - apk add curl fribidi +# Ensure fribidi is installed by the system. +if [[ "$OSTYPE" == "darwin"* ]]; then + # If Homebrew is on the path during the build, it may leak into the wheels. + # However, we need a *do* need Homebrew to provide a copy of fribidi for + # testing purposes so that we can verify the fribidi shim works as expected. + if [[ "$(uname -m)" == "x86_64" ]]; then + HOMEBREW_HOME=/usr/local/homebrew else - yum install -y fribidi + HOMEBREW_HOME=/opt/homebrew fi + $HOMEBREW_HOME/bin/brew install fribidi + + # Add the Homebrew lib folder so that vendored libraries can be found. + export DYLD_LIBRARY_PATH=$HOMEBREW_HOME/lib +elif [ "${AUDITWHEEL_POLICY::9}" == "musllinux" ]; then + apk add curl fribidi +else + yum install -y fribidi fi python3 -m pip install numpy diff --git a/pyproject.toml b/pyproject.toml index 3839cf8402a..da8548dcbd3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -95,8 +95,8 @@ before-all = ".github/workflows/wheels-dependencies.sh" build-verbosity = 1 config-settings = "raqm=enable raqm=vendor fribidi=vendor imagequant=disable" -# Add an explicit dependencies prefix for macOS, and don't request vendored libraries. -macos.config-settings = "raqm=enable imagequant=disable dependencies-prefix=./build/deps" +# Add an explicit dependencies prefix for macOS. +macos.config-settings = "raqm=enable raqm=vendor fribidi=vendor imagequant=disable dependencies-prefix=./build/deps" test-command = "cd {project} && .github/workflows/wheels-test.sh" test-extras = "tests" diff --git a/setup.py b/setup.py index 16c25cbda80..38d793b1412 100644 --- a/setup.py +++ b/setup.py @@ -454,7 +454,7 @@ def _remove_extension(self, name: str) -> None: def get_macos_sdk_path(self) -> str | None: try: sdk_path = ( - subprocess.check_output(["xcrun", "--show-sdk-path"]) + subprocess.check_output(["xcrun", "--show-sdk-path", "--sdk", "macosx"]) .strip() .decode("latin1") ) From 8308bf361e7d4bac39b1710e4c3c0725c7ad07ac Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Tue, 29 Oct 2024 07:47:30 +0800 Subject: [PATCH 10/25] Bump multibuild to include more cmake changes. --- wheels/multibuild | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wheels/multibuild b/wheels/multibuild index 22e8ad0c69a..9a9d1275f02 160000 --- a/wheels/multibuild +++ b/wheels/multibuild @@ -1 +1 @@ -Subproject commit 22e8ad0c69acdb0c94abab74da6e09f2810d50d5 +Subproject commit 9a9d1275f025f737cdaa3c451ba07129dd95f361 From c74a5bdd85c3314fbdbbe2a72d17adad85fae5f0 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Tue, 29 Oct 2024 09:02:05 +0800 Subject: [PATCH 11/25] Correct paths used for Linux build. --- .github/workflows/wheels-dependencies.sh | 31 ++++-------------------- pyproject.toml | 4 +++ setup.py | 3 +-- 3 files changed, 10 insertions(+), 28 deletions(-) diff --git a/.github/workflows/wheels-dependencies.sh b/.github/workflows/wheels-dependencies.sh index 22084237118..0eb15d55315 100755 --- a/.github/workflows/wheels-dependencies.sh +++ b/.github/workflows/wheels-dependencies.sh @@ -26,8 +26,6 @@ OPENJPEG_VERSION=2.5.2 XZ_VERSION=5.6.3 TIFF_VERSION=4.6.0 LCMS2_VERSION=2.16 -RAQM_VERSION=0.10.2 -FRIBIDI_VERSION=1.0.16 if [[ -n "$IS_MACOS" ]]; then GIFLIB_VERSION=5.2.2 else @@ -63,12 +61,8 @@ function build_brotli { local cmake=$(get_modern_cmake) local out_dir=$(fetch_unpack https://github.com/google/brotli/archive/v$BROTLI_VERSION.tar.gz brotli-$BROTLI_VERSION.tar.gz) (cd $out_dir \ - && $cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX -DCMAKE_INSTALL_NAME_DIR=$BUILD_PREFIX/lib . \ + && $cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX -DCMAKE_INSTALL_LIBDIR=$BUILD_PREFIX/lib -DCMAKE_INSTALL_NAME_DIR=$BUILD_PREFIX/lib . \ && make install) - if [[ "$MB_ML_LIBC" == "manylinux" ]]; then - cp $BUILD_PREFIX/lib64/libbrotli* $BUILD_PREFIX/lib - cp $BUILD_PREFIX/lib64/pkgconfig/libbrotli* $BUILD_PREFIX/lib/pkgconfig - fi touch brotli-stamp } @@ -78,27 +72,12 @@ function build_harfbuzz { local out_dir=$(fetch_unpack https://github.com/harfbuzz/harfbuzz/releases/download/$HARFBUZZ_VERSION/$HARFBUZZ_VERSION.tar.xz harfbuzz-$HARFBUZZ_VERSION.tar.xz) (cd $out_dir \ - && meson setup build --prefix=$BUILD_PREFIX --buildtype=release -Dfreetype=enabled -Dglib=disabled) + && meson setup build --prefix=$BUILD_PREFIX --libdir=$BUILD_PREFIX/lib --buildtype=release -Dfreetype=enabled -Dglib=disabled) (cd $out_dir/build \ && meson install) - if [[ "$MB_ML_LIBC" == "manylinux" ]]; then - cp $BUILD_PREFIX/lib64/libharfbuzz* $BUILD_PREFIX/lib - fi touch harfbuzz-stamp } -function build_raqm { - if [ -e raqm-stamp ]; then return; fi - python3 -m pip install meson ninja - - local out_dir=$(fetch_unpack https://github.com/HOST-Oman/libraqm/releases/download/v$RAQM_VERSION/raqm-$RAQM_VERSION.tar.xz raqm-$RAQM_VERSION.tar.xz) - (cd $out_dir \ - && meson setup build --prefix=$BUILD_PREFIX) - (cd $out_dir/build \ - && meson install) - touch raqm-stamp -} - function build { build_xz if [ -z "$IS_ALPINE" ] && [ -z "$IS_MACOS" ]; then @@ -133,9 +112,6 @@ function build { build_libpng build_lcms2 build_openjpeg - if [ -f $BUILD_PREFIX/lib64/libopenjp2.so ]; then - cp $BUILD_PREFIX/lib64/libopenjp2.so $BUILD_PREFIX/lib - fi ORIGINAL_CFLAGS=$CFLAGS CFLAGS="$CFLAGS -O3 -DNDEBUG" @@ -190,6 +166,9 @@ if [[ -n "$IS_MACOS" ]]; then build_pkg_config # Ensure cmake is available python3 -m pip install cmake +else + # Ensure that any built libraries are on the linker path. + export LD_LIBRARY_PATH=$BUILD_PREFIX/lib fi wrap_wheel_builder build diff --git a/pyproject.toml b/pyproject.toml index da8548dcbd3..af3d0475750 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -104,6 +104,10 @@ test-extras = "tests" [tool.cibuildwheel.macos.environment] PATH = "$(pwd)/build/deps/bin:$(dirname $(which python3)):/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin" +[tool.cibuildwheel.linux.environment] +LD_LIBRARY_PATH = "$(pwd)/build/deps/lib" +PKG_CONFIG_PATH = "$(pwd)/build/deps/lib/pkgconfig:$(pwd)/build/deps/share/pkgconfig" + [tool.black] exclude = "wheels/multibuild" diff --git a/setup.py b/setup.py index 38d793b1412..d7e305fc0e1 100644 --- a/setup.py +++ b/setup.py @@ -131,7 +131,7 @@ def get_version() -> str: "codec_fd", ) -DEBUG = False +DEBUG = True class DependencyException(Exception): @@ -479,7 +479,6 @@ def build_extensions(self) -> None: pkg_config = None if _cmd_exists(os.environ.get("PKG_CONFIG", "pkg-config")): pkg_config = _pkg_config - # # add configured kits for root_name, lib_name in { From 72d81e24a81e6e765f21e5794218f080e7718ec7 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Tue, 29 Oct 2024 09:25:47 +0800 Subject: [PATCH 12/25] Simplify Linux config by correcting a logic error in macOS config. --- .github/workflows/wheels-dependencies.sh | 7 ++----- pyproject.toml | 4 ---- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/wheels-dependencies.sh b/.github/workflows/wheels-dependencies.sh index 0eb15d55315..2e915364abb 100755 --- a/.github/workflows/wheels-dependencies.sh +++ b/.github/workflows/wheels-dependencies.sh @@ -4,8 +4,8 @@ if [ -z "$IS_MACOS" ]; then export MB_ML_LIBC=${AUDITWHEEL_POLICY::9} export MB_ML_VER=${AUDITWHEEL_POLICY:9} - - # Build and install into the `build/deps` folder. +else + # Build and install macOS builds into the `build/deps` folder. BUILD_PREFIX=$(pwd)/build/deps fi export PLAT=$CIBW_ARCHS @@ -166,9 +166,6 @@ if [[ -n "$IS_MACOS" ]]; then build_pkg_config # Ensure cmake is available python3 -m pip install cmake -else - # Ensure that any built libraries are on the linker path. - export LD_LIBRARY_PATH=$BUILD_PREFIX/lib fi wrap_wheel_builder build diff --git a/pyproject.toml b/pyproject.toml index af3d0475750..da8548dcbd3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -104,10 +104,6 @@ test-extras = "tests" [tool.cibuildwheel.macos.environment] PATH = "$(pwd)/build/deps/bin:$(dirname $(which python3)):/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin" -[tool.cibuildwheel.linux.environment] -LD_LIBRARY_PATH = "$(pwd)/build/deps/lib" -PKG_CONFIG_PATH = "$(pwd)/build/deps/lib/pkgconfig:$(pwd)/build/deps/share/pkgconfig" - [tool.black] exclude = "wheels/multibuild" From ec214e4d7caa471c087e0d690b20d75b6dd929fc Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Tue, 29 Oct 2024 09:39:58 +0800 Subject: [PATCH 13/25] Can't check IS_MACOS until common_utils is invoked. --- .github/workflows/wheels-dependencies.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/wheels-dependencies.sh b/.github/workflows/wheels-dependencies.sh index 2e915364abb..9ce851f0f93 100755 --- a/.github/workflows/wheels-dependencies.sh +++ b/.github/workflows/wheels-dependencies.sh @@ -1,19 +1,21 @@ #!/bin/bash # Define custom utilities +export PLAT=$CIBW_ARCHS +source wheels/multibuild/common_utils.sh +source wheels/multibuild/library_builders.sh +if [ -z "$IS_MACOS" ]; then + source wheels/multibuild/manylinux_utils.sh +fi + # Test for macOS with [ -n "$IS_MACOS" ] +echo "IS MACOS: $IS_MACOS" if [ -z "$IS_MACOS" ]; then export MB_ML_LIBC=${AUDITWHEEL_POLICY::9} export MB_ML_VER=${AUDITWHEEL_POLICY:9} -else +elseac # Build and install macOS builds into the `build/deps` folder. BUILD_PREFIX=$(pwd)/build/deps fi -export PLAT=$CIBW_ARCHS -source wheels/multibuild/common_utils.sh -source wheels/multibuild/library_builders.sh -if [ -z "$IS_MACOS" ]; then - source wheels/multibuild/manylinux_utils.sh -fi ARCHIVE_SDIR=pillow-depends-main From d1a4f8029b2d89f19e4652c96e3b1909b90aa5df Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Tue, 29 Oct 2024 09:54:15 +0800 Subject: [PATCH 14/25] Don't use multibuild variables before invoking multibuild. --- .github/workflows/wheels-dependencies.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/wheels-dependencies.sh b/.github/workflows/wheels-dependencies.sh index 9ce851f0f93..fd13930db46 100755 --- a/.github/workflows/wheels-dependencies.sh +++ b/.github/workflows/wheels-dependencies.sh @@ -1,4 +1,14 @@ #!/bin/bash + +# Setup that needs to be done before multibuild utils are invoked +if [[ "$(uname -s)" == "Darwin" ]]; then + # Build and install macOS builds into the `build/deps` folder. + BUILD_PREFIX=$(pwd)/build/deps +else + export MB_ML_LIBC=${AUDITWHEEL_POLICY::9} + export MB_ML_VER=${AUDITWHEEL_POLICY:9} +fi + # Define custom utilities export PLAT=$CIBW_ARCHS source wheels/multibuild/common_utils.sh @@ -7,16 +17,6 @@ if [ -z "$IS_MACOS" ]; then source wheels/multibuild/manylinux_utils.sh fi -# Test for macOS with [ -n "$IS_MACOS" ] -echo "IS MACOS: $IS_MACOS" -if [ -z "$IS_MACOS" ]; then - export MB_ML_LIBC=${AUDITWHEEL_POLICY::9} - export MB_ML_VER=${AUDITWHEEL_POLICY:9} -elseac - # Build and install macOS builds into the `build/deps` folder. - BUILD_PREFIX=$(pwd)/build/deps -fi - ARCHIVE_SDIR=pillow-depends-main # Package versions for fresh source builds From 6d137043f712d6790eba835a8ec727e0a4fbb603 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Tue, 29 Oct 2024 10:01:46 +0800 Subject: [PATCH 15/25] Remove stray debug. --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d7e305fc0e1..38d793b1412 100644 --- a/setup.py +++ b/setup.py @@ -131,7 +131,7 @@ def get_version() -> str: "codec_fd", ) -DEBUG = True +DEBUG = False class DependencyException(Exception): @@ -479,6 +479,7 @@ def build_extensions(self) -> None: pkg_config = None if _cmd_exists(os.environ.get("PKG_CONFIG", "pkg-config")): pkg_config = _pkg_config + # # add configured kits for root_name, lib_name in { From c6912f81ff20211c2638be64bc198024aa9a5b40 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Wed, 30 Oct 2024 05:43:23 +0800 Subject: [PATCH 16/25] Corrected typo in code comment. Co-authored-by: Andrew Murray <3112309+radarhere@users.noreply.github.com> --- .github/workflows/wheels-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels-test.sh b/.github/workflows/wheels-test.sh index e671c8d45a8..9bcf42458cc 100755 --- a/.github/workflows/wheels-test.sh +++ b/.github/workflows/wheels-test.sh @@ -4,7 +4,7 @@ set -e # Ensure fribidi is installed by the system. if [[ "$OSTYPE" == "darwin"* ]]; then # If Homebrew is on the path during the build, it may leak into the wheels. - # However, we need a *do* need Homebrew to provide a copy of fribidi for + # However, we *do* need Homebrew to provide a copy of fribidi for # testing purposes so that we can verify the fribidi shim works as expected. if [[ "$(uname -m)" == "x86_64" ]]; then HOMEBREW_HOME=/usr/local/homebrew From 01270b5859800355b12fa4403c7684d9bf72c6f5 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Thu, 31 Oct 2024 06:27:54 +0800 Subject: [PATCH 17/25] Use the intended entry point for the x86_64 brew binary. Co-authored-by: Andrew Murray <3112309+radarhere@users.noreply.github.com> --- .github/workflows/wheels-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels-test.sh b/.github/workflows/wheels-test.sh index 9bcf42458cc..e2ef337119e 100755 --- a/.github/workflows/wheels-test.sh +++ b/.github/workflows/wheels-test.sh @@ -7,7 +7,7 @@ if [[ "$OSTYPE" == "darwin"* ]]; then # However, we *do* need Homebrew to provide a copy of fribidi for # testing purposes so that we can verify the fribidi shim works as expected. if [[ "$(uname -m)" == "x86_64" ]]; then - HOMEBREW_HOME=/usr/local/homebrew + HOMEBREW_HOME=/usr/local else HOMEBREW_HOME=/opt/homebrew fi From 51e36230660667df38c68379314b652b34da47e5 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Thu, 31 Oct 2024 09:50:38 +0800 Subject: [PATCH 18/25] Revert x86_64 homebrew location change (with explanation). --- .github/workflows/wheels-test.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/wheels-test.sh b/.github/workflows/wheels-test.sh index e2ef337119e..b697b740ec0 100755 --- a/.github/workflows/wheels-test.sh +++ b/.github/workflows/wheels-test.sh @@ -7,7 +7,10 @@ if [[ "$OSTYPE" == "darwin"* ]]; then # However, we *do* need Homebrew to provide a copy of fribidi for # testing purposes so that we can verify the fribidi shim works as expected. if [[ "$(uname -m)" == "x86_64" ]]; then - HOMEBREW_HOME=/usr/local + # Use the "installed" location, rather than /usr/local, for two reasons: + # firstly, Homebrew allows libraries to be *installed*, but not linked; + # and secondly, we don't want any *other* leakage from /usr/local. + HOMEBREW_HOME=/usr/local/Homebrew else HOMEBREW_HOME=/opt/homebrew fi From e82b5398e750c5dd37dfcc2edaf6b947371b53ef Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Wed, 6 Nov 2024 14:12:07 +0800 Subject: [PATCH 19/25] Correct handling of vendored fribidi. --- .github/workflows/wheels-dependencies.sh | 46 ++++++++++++++++-------- .github/workflows/wheels-test.sh | 8 +++-- .gitignore | 4 +++ pyproject.toml | 4 +-- 4 files changed, 43 insertions(+), 19 deletions(-) diff --git a/.github/workflows/wheels-dependencies.sh b/.github/workflows/wheels-dependencies.sh index fd13930db46..ff06e99d276 100755 --- a/.github/workflows/wheels-dependencies.sh +++ b/.github/workflows/wheels-dependencies.sh @@ -1,16 +1,34 @@ #!/bin/bash # Setup that needs to be done before multibuild utils are invoked +PROJECTDIR=$(pwd) if [[ "$(uname -s)" == "Darwin" ]]; then - # Build and install macOS builds into the `build/deps` folder. - BUILD_PREFIX=$(pwd)/build/deps + # Safety check - macOS builds require that CIBW_ARCHS is set, and that it + # only contains a single value (even though cwbuildwheel) allows multiple + # values in CIBW_ARCHS. + if [[ -z "$CIBW_ARCHS" ]]; then + echo "ERROR: Pillow macOS builds require CIBW_ARCHS be defined." + exit 1 + fi + if [[ "$CIBW_ARCHS" == *" "* ]]; then + echo "ERROR: Pillow macOS builds only support a single architecture in CIBW_ARCHS." + exit 1 + fi + + # Build macOS dependencies in `build/darwin` + # Install them into `build/deps/darwin` + WORKDIR=$(pwd)/build/darwin + BUILD_PREFIX=$(pwd)/build/deps/darwin + PLAT=$CIBW_ARCHS else - export MB_ML_LIBC=${AUDITWHEEL_POLICY::9} - export MB_ML_VER=${AUDITWHEEL_POLICY:9} + # Build prefix will default to /usr/local + WORKDIR=$(pwd)/build + PLAT=$CIBW_ARCHS + MB_ML_LIBC=${AUDITWHEEL_POLICY::9} + MB_ML_VER=${AUDITWHEEL_POLICY:9} fi # Define custom utilities -export PLAT=$CIBW_ARCHS source wheels/multibuild/common_utils.sh source wheels/multibuild/library_builders.sh if [ -z "$IS_MACOS" ]; then @@ -117,9 +135,6 @@ function build { ORIGINAL_CFLAGS=$CFLAGS CFLAGS="$CFLAGS -O3 -DNDEBUG" - if [[ -n "$IS_MACOS" ]]; then - CFLAGS="$CFLAGS -Wl,-headerpad_max_install_names" - fi build_libwebp CFLAGS=$ORIGINAL_CFLAGS @@ -127,7 +142,7 @@ function build { if [ -n "$IS_MACOS" ]; then # Custom freetype build - build_simple freetype $FREETYPE_VERSION https://download.savannah.gnu.org/releases/freetype tar.gz --with-harfbuzz=no + build_simple freetype $FREETYPE_VERSION https://download.savannah.gnu.org/releases/freetype tar.gz --without-harfbuzz else build_freetype fi @@ -136,17 +151,18 @@ function build { } # Perform all dependency builds in the build subfolder. -mkdir -p build -pushd build > /dev/null +mkdir -p $WORKDIR +pushd $WORKDIR > /dev/null # Any stuff that you need to do before you start building the wheels # Runs in the root directory of this repository. -if [[ ! -d pillow-depends-main ]]; then - if [[ ! -f pillow-depends-main.zip ]]; then +if [[ ! -d $WORKDIR/pillow-depends-main ]]; then + if [[ ! -f $PROJECTDIR/pillow-depends-main.zip ]]; then echo "Download pillow dependency sources..." - curl -fSL -o pillow-depends-main.zip https://github.com/python-pillow/pillow-depends/archive/main.zip + curl -fSL -o $PROJECTDIR/pillow-depends-main.zip https://github.com/python-pillow/pillow-depends/archive/main.zip fi - untar pillow-depends-main.zip + echo "Unpacking pillow dependency sources..." + untar $PROJECTDIR/pillow-depends-main.zip fi if [[ -n "$IS_MACOS" ]]; then diff --git a/.github/workflows/wheels-test.sh b/.github/workflows/wheels-test.sh index b697b740ec0..d1db8e8a5ec 100755 --- a/.github/workflows/wheels-test.sh +++ b/.github/workflows/wheels-test.sh @@ -16,8 +16,12 @@ if [[ "$OSTYPE" == "darwin"* ]]; then fi $HOMEBREW_HOME/bin/brew install fribidi - # Add the Homebrew lib folder so that vendored libraries can be found. - export DYLD_LIBRARY_PATH=$HOMEBREW_HOME/lib + # Add the lib folder for fribidi so that the vendored library can be found. + # Don't use /opt/homebrew/lib directly - use the lib folder where the + # installed copy of fribidi is cellared. This ensures we don't pick up the + # Homebrew version of any other library that we're dependent on (most notably, + # freetype). + export DYLD_LIBRARY_PATH=$(dirname $(realpath $HOMEBREW_HOME/lib/libfribidi.dylib)) elif [ "${AUDITWHEEL_POLICY::9}" == "musllinux" ]; then apk add curl fribidi else diff --git a/.gitignore b/.gitignore index 9b577325730..3033c2ea7ae 100644 --- a/.gitignore +++ b/.gitignore @@ -91,5 +91,9 @@ Tests/images/msp Tests/images/picins Tests/images/sunraster +# Test and dependency downloads +pillow-depends-main.zip +pillow-test-images.zip + # pyinstaller *.spec diff --git a/pyproject.toml b/pyproject.toml index da8548dcbd3..26fa4581706 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,13 +96,13 @@ build-verbosity = 1 config-settings = "raqm=enable raqm=vendor fribidi=vendor imagequant=disable" # Add an explicit dependencies prefix for macOS. -macos.config-settings = "raqm=enable raqm=vendor fribidi=vendor imagequant=disable dependencies-prefix=./build/deps" +macos.config-settings = "raqm=enable raqm=vendor fribidi=vendor imagequant=disable dependencies-prefix=./build/deps/darwin" test-command = "cd {project} && .github/workflows/wheels-test.sh" test-extras = "tests" [tool.cibuildwheel.macos.environment] -PATH = "$(pwd)/build/deps/bin:$(dirname $(which python3)):/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin" +PATH = "$(pwd)/build/deps/darwin/bin:$(dirname $(which python3)):/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin" [tool.black] exclude = "wheels/multibuild" From 4e35852ccc97f3e455f5130f1cd22f2f10292955 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Fri, 8 Nov 2024 05:34:35 +0800 Subject: [PATCH 20/25] Correct typo in comment. Co-authored-by: Andrew Murray <3112309+radarhere@users.noreply.github.com> --- .github/workflows/wheels-dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels-dependencies.sh b/.github/workflows/wheels-dependencies.sh index ff06e99d276..2dea8c62a95 100755 --- a/.github/workflows/wheels-dependencies.sh +++ b/.github/workflows/wheels-dependencies.sh @@ -4,7 +4,7 @@ PROJECTDIR=$(pwd) if [[ "$(uname -s)" == "Darwin" ]]; then # Safety check - macOS builds require that CIBW_ARCHS is set, and that it - # only contains a single value (even though cwbuildwheel) allows multiple + # only contains a single value (even though cibuildwheel) allows multiple # values in CIBW_ARCHS. if [[ -z "$CIBW_ARCHS" ]]; then echo "ERROR: Pillow macOS builds require CIBW_ARCHS be defined." From 681a03b1c38b26a36cddda812ead882b4af24eb4 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Sat, 9 Nov 2024 10:29:29 +0800 Subject: [PATCH 21/25] Apply suggestions from code review Co-authored-by: Andrew Murray <3112309+radarhere@users.noreply.github.com> --- .github/workflows/wheels-dependencies.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheels-dependencies.sh b/.github/workflows/wheels-dependencies.sh index 2dea8c62a95..f4106429d46 100755 --- a/.github/workflows/wheels-dependencies.sh +++ b/.github/workflows/wheels-dependencies.sh @@ -4,8 +4,8 @@ PROJECTDIR=$(pwd) if [[ "$(uname -s)" == "Darwin" ]]; then # Safety check - macOS builds require that CIBW_ARCHS is set, and that it - # only contains a single value (even though cibuildwheel) allows multiple - # values in CIBW_ARCHS. + # only contains a single value (even though cibuildwheel allows multiple + # values in CIBW_ARCHS). if [[ -z "$CIBW_ARCHS" ]]; then echo "ERROR: Pillow macOS builds require CIBW_ARCHS be defined." exit 1 @@ -171,7 +171,7 @@ if [[ -n "$IS_MACOS" ]]; then # with a MACOSX_DEPLOYMENT_TARGET that doesn't match what we want to use, # and they may bring in other dependencies that we don't want. The same will # be true of any other locations on the path. To avoid conflicts, strip the - # path down to the bare mimimum (which, on macOS, won't include any + # path down to the bare minimum (which, on macOS, won't include any # development dependencies). export PATH="$BUILD_PREFIX/bin:$(dirname $(which python3)):/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin" export CMAKE_PREFIX_PATH=$BUILD_PREFIX From 378df7a5b22e47a0652619bd2f3218674c579d91 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Tue, 12 Nov 2024 22:41:57 +1100 Subject: [PATCH 22/25] Disable platform guessing instead of adding dependencies-prefix --- pyproject.toml | 4 +-- setup.py | 89 +++++++++++++++++++++----------------------------- 2 files changed, 40 insertions(+), 53 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 26fa4581706..abb7be6458c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -95,8 +95,8 @@ before-all = ".github/workflows/wheels-dependencies.sh" build-verbosity = 1 config-settings = "raqm=enable raqm=vendor fribidi=vendor imagequant=disable" -# Add an explicit dependencies prefix for macOS. -macos.config-settings = "raqm=enable raqm=vendor fribidi=vendor imagequant=disable dependencies-prefix=./build/deps/darwin" +# Disable platform guessing on macOS +macos.config-settings = "raqm=enable raqm=vendor fribidi=vendor imagequant=disable platform-guessing=disable" test-command = "cd {project} && .github/workflows/wheels-test.sh" test-extras = "tests" diff --git a/setup.py b/setup.py index e4ce04883dd..4770c5eb78f 100644 --- a/setup.py +++ b/setup.py @@ -344,11 +344,6 @@ def __iter__(self) -> Iterator[str]: for x in ("raqm", "fribidi") ] + [ - ( - "dependencies-prefix", - None, - "The prefix where build dependencies are located.", - ), ("disable-platform-guessing", None, "Disable platform guessing on Linux"), ("debug", None, "Debug logging"), ] @@ -360,7 +355,6 @@ def check_configuration(option: str, value: str) -> bool | None: return True if value in configuration.get(option, []) else None def initialize_options(self) -> None: - self.dependencies_prefix = configuration.get("dependencies-prefix", []) self.disable_platform_guessing = self.check_configuration( "platform-guessing", "disable" ) @@ -570,54 +564,47 @@ def build_extensions(self) -> None: ) elif sys.platform == "darwin": - if self.dependencies_prefix: - # Use the explicitly provided prefixes for dependencies. - for prefix in self.dependencies_prefix: - _add_directory(library_dirs, os.path.join(prefix, "lib")) - _add_directory(include_dirs, os.path.join(prefix, "include")) - else: - # Guess the dependency locations based on homebrew/fink/macports - # attempt to make sure we pick freetype2 over other versions - _add_directory(include_dirs, "/sw/include/freetype2") - _add_directory(include_dirs, "/sw/lib/freetype2/include") - # fink installation directories - _add_directory(library_dirs, "/sw/lib") - _add_directory(include_dirs, "/sw/include") - # darwin ports installation directories - _add_directory(library_dirs, "/opt/local/lib") - _add_directory(include_dirs, "/opt/local/include") - - # if Homebrew is installed, use its lib and include directories - try: - prefix = ( - subprocess.check_output(["brew", "--prefix"]) - .strip() - .decode("latin1") - ) - except Exception: - # Homebrew not installed - prefix = None + # attempt to make sure we pick freetype2 over other versions + _add_directory(include_dirs, "/sw/include/freetype2") + _add_directory(include_dirs, "/sw/lib/freetype2/include") + # fink installation directories + _add_directory(library_dirs, "/sw/lib") + _add_directory(include_dirs, "/sw/include") + # darwin ports installation directories + _add_directory(library_dirs, "/opt/local/lib") + _add_directory(include_dirs, "/opt/local/include") - ft_prefix = None + # if Homebrew is installed, use its lib and include directories + try: + prefix = ( + subprocess.check_output(["brew", "--prefix"]) + .strip() + .decode("latin1") + ) + except Exception: + # Homebrew not installed + prefix = None - if prefix: - # add Homebrew's include and lib directories - _add_directory(library_dirs, os.path.join(prefix, "lib")) - _add_directory(include_dirs, os.path.join(prefix, "include")) - _add_directory( - include_dirs, os.path.join(prefix, "opt", "zlib", "include") - ) - ft_prefix = os.path.join(prefix, "opt", "freetype") + ft_prefix = None - if ft_prefix and os.path.isdir(ft_prefix): - # freetype might not be linked into Homebrew's prefix - _add_directory(library_dirs, os.path.join(ft_prefix, "lib")) - _add_directory(include_dirs, os.path.join(ft_prefix, "include")) - else: - # fall back to freetype from XQuartz if - # Homebrew's freetype is missing - _add_directory(library_dirs, "/usr/X11/lib") - _add_directory(include_dirs, "/usr/X11/include") + if prefix: + # add Homebrew's include and lib directories + _add_directory(library_dirs, os.path.join(prefix, "lib")) + _add_directory(include_dirs, os.path.join(prefix, "include")) + _add_directory( + include_dirs, os.path.join(prefix, "opt", "zlib", "include") + ) + ft_prefix = os.path.join(prefix, "opt", "freetype") + + if ft_prefix and os.path.isdir(ft_prefix): + # freetype might not be linked into Homebrew's prefix + _add_directory(library_dirs, os.path.join(ft_prefix, "lib")) + _add_directory(include_dirs, os.path.join(ft_prefix, "include")) + else: + # fall back to freetype from XQuartz if + # Homebrew's freetype is missing + _add_directory(library_dirs, "/usr/X11/lib") + _add_directory(include_dirs, "/usr/X11/include") # Add the macOS SDK path. sdk_path = self.get_macos_sdk_path() From 9dc690427f009f898ef74b8799569346eb0b3fd5 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Wed, 13 Nov 2024 11:03:26 +0800 Subject: [PATCH 23/25] Correct the lookup of libfribidi on x86 macOS installs. --- .github/workflows/wheels-test.sh | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/wheels-test.sh b/.github/workflows/wheels-test.sh index d1db8e8a5ec..ce83a4278cd 100755 --- a/.github/workflows/wheels-test.sh +++ b/.github/workflows/wheels-test.sh @@ -7,21 +7,18 @@ if [[ "$OSTYPE" == "darwin"* ]]; then # However, we *do* need Homebrew to provide a copy of fribidi for # testing purposes so that we can verify the fribidi shim works as expected. if [[ "$(uname -m)" == "x86_64" ]]; then - # Use the "installed" location, rather than /usr/local, for two reasons: - # firstly, Homebrew allows libraries to be *installed*, but not linked; - # and secondly, we don't want any *other* leakage from /usr/local. - HOMEBREW_HOME=/usr/local/Homebrew + HOMEBREW_PREFIX=/usr/local else - HOMEBREW_HOME=/opt/homebrew + HOMEBREW_PREFIX=/opt/homebrew fi - $HOMEBREW_HOME/bin/brew install fribidi + $HOMEBREW_PREFIX/bin/brew install fribidi # Add the lib folder for fribidi so that the vendored library can be found. - # Don't use /opt/homebrew/lib directly - use the lib folder where the + # Don't use $HOMEWBREW_PREFIX/lib directly - use the lib folder where the # installed copy of fribidi is cellared. This ensures we don't pick up the # Homebrew version of any other library that we're dependent on (most notably, # freetype). - export DYLD_LIBRARY_PATH=$(dirname $(realpath $HOMEBREW_HOME/lib/libfribidi.dylib)) + export DYLD_LIBRARY_PATH=$(dirname $(realpath $HOMEBREW_PREFIX/lib/libfribidi.dylib)) elif [ "${AUDITWHEEL_POLICY::9}" == "musllinux" ]; then apk add curl fribidi else From 54f23345d2fd1894c7cf89475e385fcbb3d3f16d Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Sat, 16 Nov 2024 09:58:29 +0800 Subject: [PATCH 24/25] More tweaks from code review. --- .github/workflows/wheels-dependencies.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/wheels-dependencies.sh b/.github/workflows/wheels-dependencies.sh index f4106429d46..ce2e40103d0 100755 --- a/.github/workflows/wheels-dependencies.sh +++ b/.github/workflows/wheels-dependencies.sh @@ -19,14 +19,13 @@ if [[ "$(uname -s)" == "Darwin" ]]; then # Install them into `build/deps/darwin` WORKDIR=$(pwd)/build/darwin BUILD_PREFIX=$(pwd)/build/deps/darwin - PLAT=$CIBW_ARCHS else # Build prefix will default to /usr/local WORKDIR=$(pwd)/build - PLAT=$CIBW_ARCHS MB_ML_LIBC=${AUDITWHEEL_POLICY::9} MB_ML_VER=${AUDITWHEEL_POLICY:9} fi +PLAT=$CIBW_ARCHS # Define custom utilities source wheels/multibuild/common_utils.sh @@ -119,9 +118,9 @@ function build { build_libjpeg_turbo if [ -n "$IS_MACOS" ]; then # Custom tiff build to include jpeg; by default, configure won't include - # headers/libs in the custom macOS prefix. Explicitly disable webp and - # zstd, because on x86_64 macs, it will pick up the Homebrew versions of - # webp and zstd from /usr/local. + # headers/libs in the custom macOS prefix. Explicitly disable webp, + # libdeflate and zstd, because on x86_64 macs, it will pick up the + # Homebrew versions of those libraries from /usr/local. build_simple tiff $TIFF_VERSION https://download.osgeo.org/libtiff tar.gz \ --with-jpeg-include-dir=$BUILD_PREFIX/include --with-jpeg-lib-dir=$BUILD_PREFIX/lib \ --disable-webp --disable-zstd --disable-libdeflate From 96b898c826982c5b99f2a9387e0d2011a73fe20a Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Mon, 18 Nov 2024 09:56:45 +0800 Subject: [PATCH 25/25] A couple more cleanups from code review. --- .github/workflows/wheels-dependencies.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/wheels-dependencies.sh b/.github/workflows/wheels-dependencies.sh index ce2e40103d0..f86099af16e 100755 --- a/.github/workflows/wheels-dependencies.sh +++ b/.github/workflows/wheels-dependencies.sh @@ -62,8 +62,7 @@ BROTLI_VERSION=1.1.0 function build_pkg_config { if [ -e pkg-config-stamp ]; then return; fi - # This essentially duplicates the Homebrew recipe: - # https://github.com/Homebrew/homebrew-core/blob/master/Formula/p/pkg-config.rb + # This essentially duplicates the Homebrew recipe ORIGINAL_CFLAGS=$CFLAGS CFLAGS="$CFLAGS -Wno-int-conversion" build_simple pkg-config 0.29.2 https://pkg-config.freedesktop.org/releases tar.gz \ @@ -108,7 +107,6 @@ function build { if [ -n "$IS_MACOS" ]; then build_simple xorgproto 2024.1 https://www.x.org/pub/individual/proto build_simple libXau 1.0.11 https://www.x.org/pub/individual/lib - build_simple libXdmcp 1.1.5 https://www.x.org/pub/individual/lib build_simple libpthread-stubs 0.5 https://xcb.freedesktop.org/dist else sed s/\${pc_sysrootdir\}// $BUILD_PREFIX/share/pkgconfig/xcb-proto.pc > $BUILD_PREFIX/lib/pkgconfig/xcb-proto.pc @@ -123,7 +121,7 @@ function build { # Homebrew versions of those libraries from /usr/local. build_simple tiff $TIFF_VERSION https://download.osgeo.org/libtiff tar.gz \ --with-jpeg-include-dir=$BUILD_PREFIX/include --with-jpeg-lib-dir=$BUILD_PREFIX/lib \ - --disable-webp --disable-zstd --disable-libdeflate + --disable-webp --disable-libdeflate --disable-zstd else build_tiff fi @@ -134,6 +132,9 @@ function build { ORIGINAL_CFLAGS=$CFLAGS CFLAGS="$CFLAGS -O3 -DNDEBUG" + if [[ -n "$IS_MACOS" ]]; then + CFLAGS="$CFLAGS -Wl,-headerpad_max_install_names" + fi build_libwebp CFLAGS=$ORIGINAL_CFLAGS @@ -141,7 +142,7 @@ function build { if [ -n "$IS_MACOS" ]; then # Custom freetype build - build_simple freetype $FREETYPE_VERSION https://download.savannah.gnu.org/releases/freetype tar.gz --without-harfbuzz + build_simple freetype $FREETYPE_VERSION https://download.savannah.gnu.org/releases/freetype tar.gz --with-harfbuzz=no else build_freetype fi @@ -175,7 +176,7 @@ if [[ -n "$IS_MACOS" ]]; then export PATH="$BUILD_PREFIX/bin:$(dirname $(which python3)):/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin" export CMAKE_PREFIX_PATH=$BUILD_PREFIX - # Link the brew command into our isolated build directory. + # Ensure the basic structure of the build prefix directory exists. mkdir -p "$BUILD_PREFIX/bin" mkdir -p "$BUILD_PREFIX/lib"