From efbd327322eb2485be4ea9ce58bfd2814b007efd Mon Sep 17 00:00:00 2001 From: ijl Date: Fri, 3 May 2024 11:59:31 +0000 Subject: [PATCH] Split non-amd64 manylinux build --- .github/workflows/artifact.yaml | 231 +++++++++++++++++++------------- 1 file changed, 139 insertions(+), 92 deletions(-) diff --git a/.github/workflows/artifact.yaml b/.github/workflows/artifact.yaml index 99cb3c76..208d58d0 100644 --- a/.github/workflows/artifact.yaml +++ b/.github/workflows/artifact.yaml @@ -62,11 +62,11 @@ jobs: fail-fast: false matrix: python: [ - { version: '3.12', abi: 'cp312-cp312' }, - { version: '3.11', abi: 'cp311-cp311' }, - { version: '3.10', abi: 'cp310-cp310' }, - { version: '3.9', abi: 'cp39-cp39' }, - { version: '3.8', abi: 'cp38-cp38' }, + { version: '3.12' }, + { version: '3.11' }, + { version: '3.10' }, + { version: '3.9' }, + { version: '3.8' }, ] env: CC: "clang" @@ -129,6 +129,139 @@ jobs: overwrite: true retention-days: 1 + manylinux_2_17_non_amd64_build: + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + python: [ + { version: '3.12', abi: 'cp312-cp312' }, + { version: '3.11', abi: 'cp311-cp311' }, + { version: '3.10', abi: 'cp310-cp310' }, + { version: '3.9', abi: 'cp39-cp39' }, + { version: '3.8', abi: 'cp38-cp38' }, + ] + target: [ + { + arch: 'aarch64', + cflags: '-Os -flto=full -fstrict-aliasing', + features: 'no-panic,unstable-simd,yyjson', + rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings', + target: 'aarch64-unknown-linux-gnu', + }, + { + arch: 'ppc64le', + cflags: '-Os -flto=full -fstrict-aliasing', + features: 'no-panic,unstable-simd,yyjson', + rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings', + target: 'powerpc64le-unknown-linux-gnu', + }, + { + arch: 's390x', + cflags: '-Os -flto=full -fstrict-aliasing -march=z10', + features: 'no-panic,yyjson', + rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings -C target-cpu=z10', + target: 's390x-unknown-linux-gnu', + }, + ] + env: + CC: "clang" + LDFLAGS: "-fuse-ld=lld -Wl,-plugin-opt=also-emit-llvm -Wl,--as-needed -Wl,-zrelro,-znow" + PATH: "/__w/orjson/orjson/.venv/bin:/github/home/.cargo/bin:/root/.local/bin:/root/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + container: + image: fedora:41 + options: --user 0 + steps: + - name: cpuinfo + run: cat /proc/cpuinfo + + - uses: actions/checkout@v4 + + - name: Build environment + run: | + dnf install -y rustup clang lld python${{ matrix.python.version }} + + rustup-init --default-toolchain "${RUST_TOOLCHAIN}-${{ matrix.target.target }}" --profile minimal --component rust-src -y + cargo fetch --target=${{ matrix.target.target }} & + + curl -LsSf https://astral.sh/uv/install.sh | sh + uv venv --python python${{ matrix.python.version }} + uv pip install --upgrade "maturin>=1,<2" -r test/requirements.txt -r integration/requirements.txt + + mkdir .cargo + cp ci/config.toml .cargo/config.toml + + - name: maturin + env: + PYO3_CROSS_LIB_DIR: "/opt/python/${{ matrix.python.abi }}" + CFLAGS: "${{ matrix.target.cflags }}" + LDFLAGS: "-fuse-ld=lld -Wl,-plugin-opt=also-emit-llvm -Wl,--as-needed -Wl,-zrelro,-znow" + RUSTFLAGS: "${{ matrix.target.rustflags }}" + run: | + echo $PYO3_CROSS_LIB_DIR + maturin build --release --strip \ + --features=${{ matrix.target.features }} \ + --compatibility manylinux_2_17 \ + --interpreter python${{ matrix.python.version }} \ + --target${{ matrix.target.target }} + + - name: Store wheels + uses: actions/upload-artifact@v4 + with: + name: orjson_manylinux_2_17_${{ matrix.target.arch }}_${{ matrix.python.version }} + path: target/wheels + overwrite: true + retention-days: 1 + + manylinux_2_17_non_amd64_test: + runs-on: ubuntu-22.04 + needs: manylinux_2_17_non_amd64_build + strategy: + fail-fast: false + matrix: + python: [ + { version: '3.12' }, + { version: '3.11' }, + { version: '3.10' }, + { version: '3.9' }, + { version: '3.8' }, + ] + platform: [ + { arch: 'aarch64', platform: 'linux/arm64' }, + { arch: 'ppc64le', platform: 'linux/ppc64le' }, + { arch: 's390x', platform: 'linux/s390x' }, + ] + steps: + - uses: actions/checkout@v4 + + - uses: actions/download-artifact@v4 + with: + pattern: orjson_manylinux_2_17_${{ matrix.platform.arch }}_${{ matrix.python.version }} + merge-multiple: true + + - run: ls -1 + + - name: QEMU + uses: docker/setup-qemu-action@v3 + with: + image: tonistiigi/binfmt:qemu-v8.1.5 + platforms: linux/${{ matrix.platform.platform }} + + - name: pytest + uses: addnab/docker-run-action@v3 + with: + image: quay.io/pypa/manylinux_1_17_${{ matrix.platform.arch }}:latest + options: -v ${{ github.workspace }}:/io -w /io + run: | + ls -1 + cd /io + ls -1 + python${{ matrix.python.version }} -m venv .venv + source .venv/bin/activate + pip install -U pip wheel pytest pytz + pip install --no-index ./orjson*.whl + pytest -s -rxX -v -n 4 test + musllinux_1_2: runs-on: ubuntu-22.04 strategy: @@ -202,92 +335,6 @@ jobs: overwrite: true retention-days: 1 - manylinux_2_17_non_amd64: - runs-on: ubuntu-22.04 - strategy: - fail-fast: false - matrix: - python: [ - { version: '3.12', abi: 'cp312-cp312' }, - { version: '3.11', abi: 'cp311-cp311' }, - { version: '3.10', abi: 'cp310-cp310' }, - { version: '3.9', abi: 'cp39-cp39' }, - { version: '3.8', abi: 'cp38-cp38' }, - ] - target: [ - { - arch: 'aarch64', - cflags: '-Os -flto=full -fstrict-aliasing', - features: 'no-panic,unstable-simd,yyjson', - rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings', - target: 'aarch64-unknown-linux-gnu', - }, - { - arch: 'ppc64le', - cflags: '-Os -flto=full -fstrict-aliasing', - features: 'no-panic,unstable-simd,yyjson', - rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings', - target: 'powerpc64le-unknown-linux-gnu', - }, - { - arch: 's390x', - cflags: '-Os -flto=full -fstrict-aliasing -march=z10', - features: 'no-panic,yyjson', - rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings -C target-cpu=z10', - target: 's390x-unknown-linux-gnu', - }, - ] - steps: - - uses: actions/checkout@v4 - - name: build-std - run: | - mkdir .cargo - cp ci/config.toml .cargo/config.toml - - - name: Build - uses: PyO3/maturin-action@v1 - env: - PYO3_CROSS_LIB_DIR: "/opt/python/${{ matrix.python.abi }}" - CFLAGS: "${{ matrix.target.cflags }}" - LDFLAGS: "-Wl,--as-needed" - RUSTFLAGS: "${{ matrix.target.rustflags }}" - with: - target: ${{ matrix.target.target }} - rust-toolchain: nightly-2024-04-30 - rustup-components: rust-src - manylinux: auto - args: --release --strip --out=dist --features=${{ matrix.target.features }} -i python${{ matrix.python.version }} - - - uses: uraimo/run-on-arch-action@v2 - name: Test - with: - arch: ${{ matrix.target.arch }} - distro: ubuntu22.04 - githubToken: ${{ github.token }} - install: | - export TZ=UTC - export DEBIAN_FRONTEND=noninteractive - apt-get update - apt-get install -y --no-install-recommends software-properties-common gpg gpg-agent curl - add-apt-repository ppa:deadsnakes/ppa - apt-get update - apt-get install -y python${{ matrix.python.version }}-dev python${{ matrix.python.version }}-venv - run: | - python${{ matrix.python.version }} -m venv venv - venv/bin/pip install -U pip wheel - venv/bin/pip install -r test/requirements.txt - venv/bin/pip install orjson --no-index --find-links dist/ --force-reinstall - venv/bin/python -m pytest -s -rxX -v -n 2 test - - - name: Store wheels - if: "startsWith(github.ref, 'refs/tags/')" - uses: actions/upload-artifact@v4 - with: - name: orjson_manylinux_2_17_${{ matrix.target.arch }}_${{ matrix.python.version }} - path: dist - overwrite: true - retention-days: 1 - macos_universal2_aarch64: runs-on: macos-14 strategy: @@ -437,7 +484,7 @@ jobs: sdist, manylinux_2_17_amd64, musllinux_1_2, - manylinux_2_17_non_amd64, + manylinux_2_17_non_amd64_test, macos_universal2_aarch64, macos_universal2_amd64, ]