Skip to content

Commit

Permalink
Split non-amd64 manylinux build
Browse files Browse the repository at this point in the history
  • Loading branch information
@ijl
ijl committed May 3, 2024
1 parent e3aa1b4 commit efbd327
Showing 1 changed file with 139 additions and 92 deletions.
231 changes: 139 additions & 92 deletions .github/workflows/artifact.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,11 +62,11 @@ jobs:
fail-fast: false
matrix:
python: [
{ version: '3.12', abi: 'cp312-cp312' },
{ version: '3.11', abi: 'cp311-cp311' },
{ version: '3.10', abi: 'cp310-cp310' },
{ version: '3.9', abi: 'cp39-cp39' },
{ version: '3.8', abi: 'cp38-cp38' },
{ version: '3.12' },
{ version: '3.11' },
{ version: '3.10' },
{ version: '3.9' },
{ version: '3.8' },
]
env:
CC: "clang"
Expand Down Expand Up @@ -129,6 +129,139 @@ jobs:
overwrite: true
retention-days: 1

manylinux_2_17_non_amd64_build:
runs-on: ubuntu-22.04
strategy:
fail-fast: false
matrix:
python: [
{ version: '3.12', abi: 'cp312-cp312' },
{ version: '3.11', abi: 'cp311-cp311' },
{ version: '3.10', abi: 'cp310-cp310' },
{ version: '3.9', abi: 'cp39-cp39' },
{ version: '3.8', abi: 'cp38-cp38' },
]
target: [
{
arch: 'aarch64',
cflags: '-Os -flto=full -fstrict-aliasing',
features: 'no-panic,unstable-simd,yyjson',
rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings',
target: 'aarch64-unknown-linux-gnu',
},
{
arch: 'ppc64le',
cflags: '-Os -flto=full -fstrict-aliasing',
features: 'no-panic,unstable-simd,yyjson',
rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings',
target: 'powerpc64le-unknown-linux-gnu',
},
{
arch: 's390x',
cflags: '-Os -flto=full -fstrict-aliasing -march=z10',
features: 'no-panic,yyjson',
rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings -C target-cpu=z10',
target: 's390x-unknown-linux-gnu',
},
]
env:
CC: "clang"
LDFLAGS: "-fuse-ld=lld -Wl,-plugin-opt=also-emit-llvm -Wl,--as-needed -Wl,-zrelro,-znow"
PATH: "/__w/orjson/orjson/.venv/bin:/github/home/.cargo/bin:/root/.local/bin:/root/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
container:
image: fedora:41
options: --user 0
steps:
- name: cpuinfo
run: cat /proc/cpuinfo

- uses: actions/checkout@v4

- name: Build environment
run: |
dnf install -y rustup clang lld python${{ matrix.python.version }}
rustup-init --default-toolchain "${RUST_TOOLCHAIN}-${{ matrix.target.target }}" --profile minimal --component rust-src -y
cargo fetch --target=${{ matrix.target.target }} &
curl -LsSf https://astral.sh/uv/install.sh | sh
uv venv --python python${{ matrix.python.version }}
uv pip install --upgrade "maturin>=1,<2" -r test/requirements.txt -r integration/requirements.txt
mkdir .cargo
cp ci/config.toml .cargo/config.toml
- name: maturin
env:
PYO3_CROSS_LIB_DIR: "/opt/python/${{ matrix.python.abi }}"
CFLAGS: "${{ matrix.target.cflags }}"
LDFLAGS: "-fuse-ld=lld -Wl,-plugin-opt=also-emit-llvm -Wl,--as-needed -Wl,-zrelro,-znow"
RUSTFLAGS: "${{ matrix.target.rustflags }}"
run: |
echo $PYO3_CROSS_LIB_DIR
maturin build --release --strip \
--features=${{ matrix.target.features }} \
--compatibility manylinux_2_17 \
--interpreter python${{ matrix.python.version }} \
--target${{ matrix.target.target }}
- name: Store wheels
uses: actions/upload-artifact@v4
with:
name: orjson_manylinux_2_17_${{ matrix.target.arch }}_${{ matrix.python.version }}
path: target/wheels
overwrite: true
retention-days: 1

manylinux_2_17_non_amd64_test:
runs-on: ubuntu-22.04
needs: manylinux_2_17_non_amd64_build
strategy:
fail-fast: false
matrix:
python: [
{ version: '3.12' },
{ version: '3.11' },
{ version: '3.10' },
{ version: '3.9' },
{ version: '3.8' },
]
platform: [
{ arch: 'aarch64', platform: 'linux/arm64' },
{ arch: 'ppc64le', platform: 'linux/ppc64le' },
{ arch: 's390x', platform: 'linux/s390x' },
]
steps:
- uses: actions/checkout@v4

- uses: actions/download-artifact@v4
with:
pattern: orjson_manylinux_2_17_${{ matrix.platform.arch }}_${{ matrix.python.version }}
merge-multiple: true

- run: ls -1

- name: QEMU
uses: docker/setup-qemu-action@v3
with:
image: tonistiigi/binfmt:qemu-v8.1.5
platforms: linux/${{ matrix.platform.platform }}

- name: pytest
uses: addnab/docker-run-action@v3
with:
image: quay.io/pypa/manylinux_1_17_${{ matrix.platform.arch }}:latest
options: -v ${{ github.workspace }}:/io -w /io
run: |
ls -1
cd /io
ls -1
python${{ matrix.python.version }} -m venv .venv
source .venv/bin/activate
pip install -U pip wheel pytest pytz
pip install --no-index ./orjson*.whl
pytest -s -rxX -v -n 4 test
musllinux_1_2:
runs-on: ubuntu-22.04
strategy:
Expand Down Expand Up @@ -202,92 +335,6 @@ jobs:
overwrite: true
retention-days: 1

manylinux_2_17_non_amd64:
runs-on: ubuntu-22.04
strategy:
fail-fast: false
matrix:
python: [
{ version: '3.12', abi: 'cp312-cp312' },
{ version: '3.11', abi: 'cp311-cp311' },
{ version: '3.10', abi: 'cp310-cp310' },
{ version: '3.9', abi: 'cp39-cp39' },
{ version: '3.8', abi: 'cp38-cp38' },
]
target: [
{
arch: 'aarch64',
cflags: '-Os -flto=full -fstrict-aliasing',
features: 'no-panic,unstable-simd,yyjson',
rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings',
target: 'aarch64-unknown-linux-gnu',
},
{
arch: 'ppc64le',
cflags: '-Os -flto=full -fstrict-aliasing',
features: 'no-panic,unstable-simd,yyjson',
rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings',
target: 'powerpc64le-unknown-linux-gnu',
},
{
arch: 's390x',
cflags: '-Os -flto=full -fstrict-aliasing -march=z10',
features: 'no-panic,yyjson',
rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings -C target-cpu=z10',
target: 's390x-unknown-linux-gnu',
},
]
steps:
- uses: actions/checkout@v4
- name: build-std
run: |
mkdir .cargo
cp ci/config.toml .cargo/config.toml
- name: Build
uses: PyO3/maturin-action@v1
env:
PYO3_CROSS_LIB_DIR: "/opt/python/${{ matrix.python.abi }}"
CFLAGS: "${{ matrix.target.cflags }}"
LDFLAGS: "-Wl,--as-needed"
RUSTFLAGS: "${{ matrix.target.rustflags }}"
with:
target: ${{ matrix.target.target }}
rust-toolchain: nightly-2024-04-30
rustup-components: rust-src
manylinux: auto
args: --release --strip --out=dist --features=${{ matrix.target.features }} -i python${{ matrix.python.version }}

- uses: uraimo/run-on-arch-action@v2
name: Test
with:
arch: ${{ matrix.target.arch }}
distro: ubuntu22.04
githubToken: ${{ github.token }}
install: |
export TZ=UTC
export DEBIAN_FRONTEND=noninteractive
apt-get update
apt-get install -y --no-install-recommends software-properties-common gpg gpg-agent curl
add-apt-repository ppa:deadsnakes/ppa
apt-get update
apt-get install -y python${{ matrix.python.version }}-dev python${{ matrix.python.version }}-venv
run: |
python${{ matrix.python.version }} -m venv venv
venv/bin/pip install -U pip wheel
venv/bin/pip install -r test/requirements.txt
venv/bin/pip install orjson --no-index --find-links dist/ --force-reinstall
venv/bin/python -m pytest -s -rxX -v -n 2 test
- name: Store wheels
if: "startsWith(github.ref, 'refs/tags/')"
uses: actions/upload-artifact@v4
with:
name: orjson_manylinux_2_17_${{ matrix.target.arch }}_${{ matrix.python.version }}
path: dist
overwrite: true
retention-days: 1

macos_universal2_aarch64:
runs-on: macos-14
strategy:
Expand Down Expand Up @@ -437,7 +484,7 @@ jobs:
sdist,
manylinux_2_17_amd64,
musllinux_1_2,
manylinux_2_17_non_amd64,
manylinux_2_17_non_amd64_test,
macos_universal2_aarch64,
macos_universal2_amd64,
]
Expand Down

0 comments on commit efbd327

Please sign in to comment.