From 6c749b6dfb18e70bd937e2059e23ab0f20753517 Mon Sep 17 00:00:00 2001 From: ijl Date: Tue, 22 Oct 2024 15:30:01 +0000 Subject: [PATCH] setup-qemu-container, restrict inline_int, cargo update --- .github/workflows/artifact.yaml | 167 ++++++++++++++++++++++++++------ Cargo.lock | 16 +-- build.rs | 5 +- script/install-fedora | 8 +- 4 files changed, 156 insertions(+), 40 deletions(-) diff --git a/.github/workflows/artifact.yaml b/.github/workflows/artifact.yaml index 16d6fa2b..32fd469c 100644 --- a/.github/workflows/artifact.yaml +++ b/.github/workflows/artifact.yaml @@ -40,7 +40,7 @@ jobs: - run: python3 -m pip install --user -r test/requirements.txt -r integration/requirements.txt mypy - - run: pytest -s -rxX -v -n 2 test + - run: pytest -s -rxX -v -n 4 test env: PYTHONMALLOC: "debug" @@ -60,9 +60,21 @@ jobs: manylinux_2_17_amd64: runs-on: ubuntu-24.04 + timeout-minutes: 10 strategy: fail-fast: false matrix: + arch: [ + { + cc: "clang", + cflags: "-Os -fstrict-aliasing -fno-plt -flto=full -emit-llvm", + features: "avx512,no-panic,unstable-simd,yyjson", + ldflags: "-fuse-ld=lld -Wl,-plugin-opt=also-emit-llvm -Wl,--as-needed -Wl,-zrelro,-znow", + rustflags: "-C linker=clang -C link-arg=-fuse-ld=lld -C linker-plugin-lto -C lto=fat -C link-arg=-Wl,-zrelro,-znow -Z mir-opt-level=4 -Z virtual-function-elimination -Z threads=4 -D warnings", + tag: null, + target: "x86_64-unknown-linux-gnu", + }, + ] python: [ { interpreter: 'python3.13', package: 'python3.13' }, { interpreter: 'python3.12', package: 'python3.12' }, @@ -74,48 +86,153 @@ jobs: env: PYTHON: "${{ matrix.python.interpreter }}" PYTHON_PACKAGE: "${{ matrix.python.package }}" - TARGET: "x86_64-unknown-linux-gnu" - CC: "clang" + TARGET: "${{ matrix.arch.target }}" + CC: "${{ matrix.arch.cc }}" VENV: ".venv" - CFLAGS: "-Os -fstrict-aliasing -fno-plt -flto=full -emit-llvm" - LDFLAGS: "-fuse-ld=lld -Wl,-plugin-opt=also-emit-llvm -Wl,--as-needed -Wl,-zrelro,-znow" - RUSTFLAGS: "-C linker=clang -C link-arg=-fuse-ld=lld -C linker-plugin-lto -C lto=fat -C link-arg=-Wl,-zrelro,-znow -Z mir-opt-level=4 -Z virtual-function-elimination -Z threads=2 -D warnings" - PATH: "/__w/orjson/orjson/.venv/bin:/github/home/.cargo/bin:/root/.local/bin:/root/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - container: - image: fedora:42 + FEATURES: "${{ matrix.arch.features }}" + CFLAGS: "${{ matrix.arch.cflags }}" + LDFLAGS: "${{ matrix.arch.ldflags }}" + RUSTFLAGS: "${{ matrix.arch.rustflags }}" + CARGO_TARGET_DIR: "/tmp/orjson" steps: - name: cpuinfo run: cat /proc/cpuinfo - - name: Build environment pre-clone - run: dnf install -y git - - uses: actions/checkout@v4 - - name: Build environment post-clone + - name: setup-qemu-container + uses: sandervocke/setup-qemu-container@v1 + with: + container: registry.fedoraproject.org/fedora:42 + arch: ${{ matrix.arch.tag }} + podman_args: "-v .:/orjson -v /tmp:/tmp --workdir /orjson" + + - name: setup-shell-wrapper + uses: sandervocke/setup-shell-wrapper@v1 + + - name: Build and test + shell: wrap-shell {0} + env: + WRAP_SHELL: run-in-container.sh run: | + set -eou pipefail + + mkdir dist + + export PATH="/root/.cargo/bin:/home/runner/work/orjson/orjson/.venv:/home/runner/.cargo/bin:$PATH" + ./script/install-fedora + source "${VENV}/bin/activate" + maturin build --release --strip \ - --features=avx512,no-panic,unstable-simd,yyjson \ + --features="${FEATURES}" \ --compatibility=manylinux_2_17 \ --interpreter="${PYTHON}" \ --target="${TARGET}" - uv pip install target/wheels/orjson*.whl + uv pip install ${CARGO_TARGET_DIR}/wheels/orjson*.whl - pytest -s -rxX -v -n 2 test + pytest -s -rxX -v -n 4 test ./integration/run thread ./integration/run http ./integration/run init + cp ${CARGO_TARGET_DIR}/wheels/orjson*.whl dist + - name: Store wheels if: "startsWith(github.ref, 'refs/tags/')" uses: actions/upload-artifact@v4 with: name: orjson_manylinux_2_17_amd64_${{ matrix.python.interpreter }} - path: target/wheels + path: dist + overwrite: true + retention-days: 1 + + + manylinux_2_17_aarch64: + runs-on: ubuntu-24.04 + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + arch: [ + { + cc: "clang", + cflags: "-Os -fstrict-aliasing -fno-plt -flto=full -emit-llvm", + features: "no-panic,unstable-simd,yyjson", + ldflags: "-fuse-ld=lld -Wl,-plugin-opt=also-emit-llvm -Wl,--as-needed -Wl,-zrelro,-znow", + rustflags: "-C linker=clang -C link-arg=-fuse-ld=lld -C linker-plugin-lto -C lto=fat -C link-arg=-Wl,-zrelro,-znow -Z mir-opt-level=4 -Z virtual-function-elimination -Z threads=4 -D warnings", + tag: "aarch64", + target: "aarch64-unknown-linux-gnu", + }, + ] + python: [ + { interpreter: 'python3.13', package: 'python3.13' }, + ] + env: + PYTHON: "${{ matrix.python.interpreter }}" + PYTHON_PACKAGE: "${{ matrix.python.package }}" + TARGET: "${{ matrix.arch.target }}" + CC: "${{ matrix.arch.cc }}" + VENV: ".venv" + FEATURES: "${{ matrix.arch.features }}" + CFLAGS: "${{ matrix.arch.cflags }}" + LDFLAGS: "${{ matrix.arch.ldflags }}" + RUSTFLAGS: "${{ matrix.arch.rustflags }}" + CARGO_TARGET_DIR: "/tmp/orjson" + steps: + + - name: cpuinfo + run: cat /proc/cpuinfo + + - uses: actions/checkout@v4 + + - name: setup-qemu-container + uses: sandervocke/setup-qemu-container@v1 + with: + container: registry.fedoraproject.org/fedora:42 + arch: ${{ matrix.arch.tag }} + podman_args: "-v .:/orjson -v /tmp:/tmp --workdir /orjson" + + - name: setup-shell-wrapper + uses: sandervocke/setup-shell-wrapper@v1 + + - name: Build and test + shell: wrap-shell {0} + env: + WRAP_SHELL: run-in-container.sh + run: | + set -eou pipefail + + mkdir dist + + export PATH="/root/.cargo/bin:/home/runner/work/orjson/orjson/.venv:/home/runner/.cargo/bin:$PATH" + + ./script/install-fedora + + source "${HOME}/.cargo/env" + source "${VENV}/bin/activate" + + maturin build --release --strip \ + --features="${FEATURES}" \ + --compatibility=manylinux_2_17 \ + --interpreter="${PYTHON}" \ + --target="${TARGET}" + + uv pip install ${CARGO_TARGET_DIR}/wheels/orjson*.whl + + pytest -s -rxX -v -n 2 test + + cp ${CARGO_TARGET_DIR}/wheels/orjson*.whl dist + + - name: Store wheels + if: "startsWith(github.ref, 'refs/tags/')" + uses: actions/upload-artifact@v4 + with: + name: orjson_manylinux_2_17_amd64_${{ matrix.python.interpreter }} + path: dist overwrite: true retention-days: 1 @@ -208,13 +325,6 @@ jobs: { version: '3.8', abi: 'cp38-cp38' }, ] target: [ - { - arch: 'aarch64', - cflags: '-Os -flto=full -fstrict-aliasing', - features: 'no-panic,unstable-simd,yyjson', - rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings', - target: 'aarch64-unknown-linux-gnu', - }, { arch: 'armv7', cflags: '-Os -flto=full -fstrict-aliasing', @@ -416,12 +526,13 @@ jobs: runs-on: ubuntu-24.04 if: "startsWith(github.ref, 'refs/tags/')" needs: [ - sdist, - manylinux_2_17_amd64, - musllinux_1_2, - manylinux_2_17_non_amd64, macos_universal2_aarch64, macos_universal2_amd64, + manylinux_2_17_aarch64, + manylinux_2_17_amd64, + manylinux_2_17_non_amd64, + musllinux_1_2, + sdist, ] steps: - uses: actions/download-artifact@v4 diff --git a/Cargo.lock b/Cargo.lock index 9be66c05..45b3f58a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -221,18 +221,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.210" +version = "1.0.211" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +checksum = "1ac55e59090389fb9f0dd9e0f3c09615afed1d19094284d0b200441f13550793" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.210" +version = "1.0.211" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +checksum = "54be4f245ce16bc58d57ef2716271d0d4519e0f6defa147f6e081005bcb278ff" dependencies = [ "proc-macro2", "quote", @@ -241,9 +241,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.131" +version = "1.0.132" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67d42a0bd4ac281beff598909bb56a86acaf979b84483e1c79c10dcaf98f8cf3" +checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" dependencies = [ "itoa", "memchr", @@ -277,9 +277,9 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "syn" -version = "2.0.79" +version = "2.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" +checksum = "83540f837a8afc019423a8edb95b52a8effe46957ee402287f4292fae35be021" dependencies = [ "proc-macro2", "quote", diff --git a/build.rs b/build.rs index cc2a140c..4eac1746 100644 --- a/build.rs +++ b/build.rs @@ -54,7 +54,10 @@ fn main() { } } - #[cfg(target_pointer_width = "64")] + #[cfg(all( + target_pointer_width = "64", + any(target_arch = "x86_64", target_arch = "aarch64") + ))] println!("cargo:rustc-cfg=feature=\"inline_int\""); if env::var("ORJSON_DISABLE_YYJSON").is_ok() { diff --git a/script/install-fedora b/script/install-fedora index c9606115..19a5a455 100755 --- a/script/install-fedora +++ b/script/install-fedora @@ -1,4 +1,6 @@ -#!/bin/sh -e +#!/usr/bin/env bash + +set -eou pipefail # export PYTHON=python3.11 # export PYTHON_PACKAGE=python3.11 @@ -10,9 +12,9 @@ export VENV="${VENV:-.venv}" export CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-target}" -sudo rm /etc/yum.repos.d/fedora-cisco-openh264.repo || true +rm /etc/yum.repos.d/fedora-cisco-openh264.repo || true -dnf install -y rustup clang lld "${PYTHON_PACKAGE}" +dnf install --setopt=install_weak_deps=false -y rustup clang lld "${PYTHON_PACKAGE}" rustup-init --default-toolchain "${RUST_TOOLCHAIN}-${TARGET}" --profile minimal --component rust-src -y source "${HOME}/.cargo/env"