manylinux_2_17, macos, windows CI

.github/workflows/artifact.yaml

@@ -0,0 +1,389 @@
+name: artifact
+on: push
+env:
+  RUST_TOOLCHAIN: "nightly-2024-03-27"
+jobs:
+
+  sdist:
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+    env:
+      RUST_TOOLCHAIN: "1.72" # MSRV
+    steps:
+    - name: rustup stable
+      run: |
+        curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain "${RUST_TOOLCHAIN}" -y
+        rustup default "${RUST_TOOLCHAIN}"
+
+    - uses: actions/checkout@v4
+
+    - run: python3 -m pip install --user --upgrade pip "maturin>=1,<2" wheel
+
+    - name: Vendor dependencies
+      run: |
+        maturin build
+        cargo fetch
+        mkdir .cargo
+        cp ci/sdist.toml .cargo/config.toml
+        cargo vendor include/cargo --versioned-dirs
+
+    - run: maturin sdist --out=dist
+
+    - run: python3 -m pip install --user dist/orjson*.tar.gz
+      env:
+        CARGO_NET_OFFLINE: "true"
+
+    - run: python3 -m pip install --user -r test/requirements.txt -r integration/requirements.txt mypy
+
+    - run: pytest -s -rxX -v -n 4 test
+      env:
+        PYTHONMALLOC: "debug"
+
+    - run: ./integration/run thread
+    - run: ./integration/run http
+    - run: ./integration/run init
+    - run: ./integration/run typestubs
+
+    - name: Store sdist
+      if: "startsWith(github.ref, 'refs/tags/')"
+      uses: actions/upload-artifact@v4
+      with:
+        name: orjson_sdist
+        path: dist
+        overwrite: true
+        retention-days: 1
+
+  manylinux_2_17_amd64:
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        python: [
+          { version: '3.12', abi: 'cp312-cp312' },
+          { version: '3.11', abi: 'cp311-cp311' },
+          { version: '3.10', abi: 'cp310-cp310' },
+          { version: '3.9', abi: 'cp39-cp39' },
+          { version: '3.8', abi: 'cp38-cp38' },
+        ]
+    env:
+      CC: "clang"
+      CFLAGS: "-Os -fstrict-aliasing -flto=full"
+      LDFLAGS: "-fuse-ld=lld -Wl,--as-needed"
+      RUSTFLAGS: "-C linker=clang -C lto=fat -C link-arg=-fuse-ld=lld -Z mir-opt-level=4 -Z virtual-function-elimination -Z threads=4 -D warnings"
+      CARGO_UNSTABLE_SPARSE_REGISTRY: "true"
+      UNSAFE_PYO3_SKIP_VERSION_CHECK: "1"
+      PATH: "/__w/orjson/orjson/.venv/bin:/github/home/.cargo/bin:/root/.local/bin:/root/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+    container:
+      image: fedora:41
+      options: --user 0
+    steps:
+
+    - uses: actions/checkout@v4
+
+    - name: Build environment
+      run: |
+        dnf install -y rustup clang lld python${{ matrix.python.version }}
+
+        rustup-init --default-toolchain "${RUST_TOOLCHAIN}-x86_64-unknown-linux-gnu" --profile minimal --component rust-src -y
+        cargo fetch --target=x86_64-unknown-linux-gnu &
+
+        curl -LsSf https://astral.sh/uv/install.sh | sh
+        uv venv --python python${{ matrix.python.version }}
+        uv pip install --upgrade "maturin>=1,<2" -r test/requirements.txt -r integration/requirements.txt
+
+        mkdir .cargo
+        cp ci/config.toml .cargo/config.toml
+
+    - name: maturin
+      run: |
+        maturin build --release --strip \
+          --features=no-panic,unstable-simd,yyjson \
+          --compatibility manylinux_2_17 \
+          --interpreter python${{ matrix.python.version }} \
+          --target=x86_64-unknown-linux-gnu
+        uv pip install target/wheels/orjson*.whl
+
+    - run: pytest -s -rxX -v -n 4 test
+      env:
+        PYTHONMALLOC: "debug"
+
+    - run: ./integration/run thread
+    - run: ./integration/run http
+    - run: ./integration/run init
+
+    - name: Store wheels
+      if: "startsWith(github.ref, 'refs/tags/')"
+      uses: actions/upload-artifact@v4
+      with:
+        name: orjson_manylinux_2_17_amd64_${{ matrix.python.version }}
+        path: target/wheels
+        overwrite: true
+        retention-days: 1
+
+  # musllinux_1_2:
+  #   runs-on: ubuntu-22.04
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       python: [
+  #         { version: '3.12' },
+  #         { version: '3.11' },
+  #         { version: '3.10' },
+  #         { version: '3.9' },
+  #         { version: '3.8' },
+  #       ]
+  #       platform:
+  #         - target: aarch64-unknown-linux-musl
+  #           arch: aarch64
+  #           platform: linux/arm64
+  #         - target: x86_64-unknown-linux-musl
+  #           arch: x86_64
+  #           platform: linux/amd64
+  #   steps:
+  #   - uses: actions/checkout@v4
+
+  #   - name: build-std
+  #     run: |
+  #       mkdir .cargo
+  #       cp ci/config.toml .cargo/config.toml
+
+  #   - name: Build
+  #     uses: PyO3/maturin-action@v1
+  #     env:
+  #       CC: "gcc"
+  #       CFLAGS: "-Os"
+  #       LDFLAGS: "-Wl,--as-needed"
+  #       RUSTFLAGS: "-Z mir-opt-level=4 -Z threads=4 -D warnings -C target-feature=-crt-static"
+  #       CARGO_UNSTABLE_SPARSE_REGISTRY: "true"
+  #       UNSAFE_PYO3_SKIP_VERSION_CHECK: "1"
+  #     with:
+  #       rust-toolchain: nightly-2024-03-27
+  #       rustup-components: rust-src
+  #       target: ${{ matrix.platform.target }}
+  #       manylinux: musllinux_1_2
+  #       args: --release --strip --out=dist --features=no-panic,unstable-simd,yyjson -i python${{ matrix.python.version }}
+
+  #   - name: QEMU
+  #     if: matrix.platform.arch != 'x86_64'
+  #     uses: docker/setup-qemu-action@v3
+  #     with:
+  #       image: tonistiigi/binfmt:qemu-v8.1.5
+  #       platforms: ${{ matrix.platform.platform }}
+
+  #   - name: Test
+  #     uses: addnab/docker-run-action@v3
+  #     with:
+  #       image: quay.io/pypa/musllinux_1_2_${{ matrix.platform.arch }}:latest
+  #       options: -v ${{ github.workspace }}:/io -w /io
+  #       run: |
+  #         apk add tzdata
+  #         sed -i '/^psutil/d' test/requirements.txt # missing 3.11, 3.12 wheels
+  #         sed -i '/^numpy/d' test/requirements.txt
+
+  #         python${{ matrix.python.version }} -m venv venv
+  #         venv/bin/pip install -U pip wheel
+  #         venv/bin/pip install -r test/requirements.txt
+  #         venv/bin/pip install orjson --no-index --find-links dist/ --force-reinstall
+  #         venv/bin/python -m pytest -s -rxX -v -n 4 test
+
+  #   - name: Store wheels
+  #     if: "startsWith(github.ref, 'refs/tags/')"
+  #     uses: actions/upload-artifact@v4
+  #     with:
+  #       name: orjson_musllinux_1_2_${{ matrix.platform.arch }}_${{ matrix.python.version }}
+  #       path: dist
+  #       overwrite: true
+  #       retention-days: 1
+
+  # manylinux_2_17_non_amd64:
+  #   runs-on: ubuntu-22.04
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       python: [
+  #         { version: '3.12', abi: 'cp312-cp312' },
+  #         { version: '3.11', abi: 'cp311-cp311' },
+  #         { version: '3.10', abi: 'cp310-cp310' },
+  #         { version: '3.9', abi: 'cp39-cp39' },
+  #         { version: '3.8', abi: 'cp38-cp38' },
+  #       ]
+  #       target: [
+  #         {
+  #           arch: 'aarch64',
+  #           cflags: '-Os -flto=full -fstrict-aliasing',
+  #           features: 'no-panic,unstable-simd,yyjson',
+  #           rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings',
+  #           target: 'aarch64-unknown-linux-gnu',
+  #         },
+  #         {
+  #           arch: 'armv7',
+  #           cflags: '-Os -flto=full -fstrict-aliasing',
+  #           features: 'no-panic,yyjson', # no SIMD
+  #           rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings -C opt-level=s',
+  #           target: 'armv7-unknown-linux-gnueabihf',
+  #         },
+  #         {
+  #           arch: 'ppc64le',
+  #           cflags: '-Os -flto=full -fstrict-aliasing',
+  #           features: 'no-panic,unstable-simd,yyjson',
+  #           rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings',
+  #           target: 'powerpc64le-unknown-linux-gnu',
+  #         },
+  #         {
+  #           arch: 's390x',
+  #           cflags: '-Os -flto=full -fstrict-aliasing -march=z10',
+  #           # no unstable-simd as rotate_elements_left() causes exit 125 in at least QEMU
+  #           features: 'no-panic,yyjson',
+  #           rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings -C target-cpu=z10',
+  #           target: 's390x-unknown-linux-gnu',
+  #         },
+  #       ]
+  #   steps:
+  #   - uses: actions/checkout@v4
+  #   - name: build-std
+  #     run: |
+  #       mkdir .cargo
+  #       cp ci/config.toml .cargo/config.toml
+
+  #   - name: Build
+  #     uses: PyO3/maturin-action@v1
+  #     env:
+  #       PYO3_CROSS_LIB_DIR: "/opt/python/${{ matrix.python.abi }}"
+  #       CFLAGS: "${{ matrix.target.cflags }}"
+  #       LDFLAGS: "-Wl,--as-needed"
+  #       CARGO_UNSTABLE_SPARSE_REGISTRY: "true"
+  #       RUSTFLAGS: "${{ matrix.target.rustflags }}"
+  #       UNSAFE_PYO3_SKIP_VERSION_CHECK: "1"
+  #     with:
+  #       target: ${{ matrix.target.target }}
+  #       rust-toolchain: nightly-2024-03-27
+  #       rustup-components: rust-src
+  #       manylinux: auto
+  #       args: --release --strip --out=dist --features=${{ matrix.target.features }} -i python${{ matrix.python.version }}
+
+  #   - uses: uraimo/run-on-arch-action@v2
+  #     name: Test
+  #     with:
+  #       arch: ${{ matrix.target.arch }}
+  #       distro: ubuntu22.04
+  #       githubToken: ${{ github.token }}
+  #       install: |
+  #         export TZ=UTC
+  #         export DEBIAN_FRONTEND=noninteractive
+  #         apt-get update
+  #         apt-get install -y --no-install-recommends software-properties-common gpg gpg-agent curl
+  #         add-apt-repository ppa:deadsnakes/ppa
+  #         apt-get update
+  #         apt-get install -y python${{ matrix.python.version }}-dev python${{ matrix.python.version }}-venv
+  #       run: |
+  #         python${{ matrix.python.version }} -m venv venv
+  #         venv/bin/pip install -U pip wheel
+  #         venv/bin/pip install -r test/requirements.txt
+  #         venv/bin/pip install orjson --no-index --find-links dist/ --force-reinstall
+  #         venv/bin/python -m pytest -s -rxX -v -n 2 test
+
+  #   - name: Store wheels
+  #     if: "startsWith(github.ref, 'refs/tags/')"
+  #     uses: actions/upload-artifact@v4
+  #     with:
+  #       name: orjson_manylinux_2_17_${{ matrix.target.arch }}_${{ matrix.python.version }}
+  #       path: dist
+  #       overwrite: true
+  #       retention-days: 1
+
+  macos_universal2:
+    runs-on: macos-14
+    strategy:
+      fail-fast: false
+      matrix:
+        python: [
+          { version: '3.12', macosx_target: "10.15" },
+          { version: '3.11', macosx_target: "10.15" },
+          { version: '3.10', macosx_target: "10.15" },
+        ]
+    env:
+      CC: "clang"
+      CFLAGS: "-Os -fstrict-aliasing -flto=full"
+      LDFLAGS: "-Wl,--as-needed"
+      CFLAGS_x86_64_apple_darwin: "-O2 -fstrict-aliasing -flto=full -march=x86-64-v2 -mtune=generic"
+      CFLAGS_aarch64_apple_darwin: "-O2 -fstrict-aliasing -flto=full -mcpu=apple-m1 -mtune=generic"
+      RUSTFLAGS: "-C lto=fat -Z mir-opt-level=4 -Z virtual-function-elimination -Z threads=3 -D warnings"
+      CARGO_UNSTABLE_SPARSE_REGISTRY: "true"
+      UNSAFE_PYO3_SKIP_VERSION_CHECK: "1"
+      PATH: "/Users/runner/work/orjson/orjson/.venv/bin:/Users/runner/.cargo/bin:/usr/local/opt/curl/bin:/usr/local/bin:/usr/local/sbin:/Users/runner/bin:/Library/Frameworks/Python.framework/Versions/Current/bin:/usr/bin:/bin:/usr/sbin:/sbin"
+    steps:
+
+    - uses: actions/checkout@v4
+
+    - uses: actions/setup-python@v5
+      with:
+        python-version: "${{ matrix.python.version }}"
+
+    - uses: dtolnay/rust-toolchain@master
+      with:
+        toolchain: "nightly-2024-03-27"
+        targets: "x86_64-apple-darwin, aarch64-apple-darwin"
+        components: "rust-src"
+
+    - name: Build environment
+      run: |
+        curl -LsSf https://astral.sh/uv/install.sh | sh
+        uv venv --python python${{ matrix.python.version }}
+        uv pip install --upgrade "maturin>=1,<2" -r test/requirements.txt -r integration/requirements.txt
+
+        mkdir .cargo
+        cp ci/config.toml .cargo/config.toml
+
+        echo $PATH
+
+    - name: maturin
+      run: |
+        PATH=$HOME/.cargo/bin:$PATH \
+        MACOSX_DEPLOYMENT_TARGET="${{ matrix.python.macosx_target }}" \
+        PYO3_CROSS_LIB_DIR=$(python -c "import sysconfig;print(sysconfig.get_config_var('LIBDIR'))") \
+        maturin build --release --strip \
+          --features=no-panic,unstable-simd,yyjson \
+          --interpreter python${{ matrix.python.version }} \
+          --target=universal2-apple-darwin
+        uv pip install target/wheels/orjson*.whl
+
+    - run: pytest -s -rxX -v -n 4 test
+      env:
+        PYTHONMALLOC: "debug"
+
+    - run: python${{ matrix.python.version }} ./integration/thread
+    - run: python${{ matrix.python.version }} ./integration/init
+
+    - name: Store wheels
+      if: "startsWith(github.ref, 'refs/tags/')"
+      uses: actions/upload-artifact@v4
+      with:
+        name: orjson_universal2_${{ matrix.python.version }}
+        path: target/wheels
+        overwrite: true
+        retention-days: 1
+
+  # pypi:
+  #   name: PyPI
+  #   runs-on: ubuntu-22.04
+  #   if: "startsWith(github.ref, 'refs/tags/')"
+  #   needs: [ manylinux_2_17_amd64, manylinux_2_17_non_amd64, musllinux_1_2, macos_universal2, sdist ]
+  #   steps:
+  #     - uses: actions/download-artifact@v4
+  #       with:
+  #         pattern: orjson_*
+  #         merge-multiple: true
+  #     - uses: actions/setup-python@v5
+  #       with:
+  #         python-version: "3.12"
+  #     - run: pip install pip "maturin>=1,<2"
+  #     - run: ls -1 .
+  #     - name: deploy wheel
+  #       run: maturin upload --skip-existing --username "$MATURIN_USERNAME" *.whl
+  #       env:
+  #         MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+  #     - name: deploy sdist
+  #       run: maturin upload --skip-existing --username "$MATURIN_USERNAME" *.tar.gz
+  #       env:
+  #         MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}

.github/workflows/lint.yaml

@@ -6,7 +6,7 @@ jobs:
     steps:
     - uses: actions/setup-python@v5
       with:
-        python-version: '3.11'
+        python-version: "3.12"
     - run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain=stable --profile=default -y
     - run: pip install -U autoflake isort black ruff mypy types-python-dateutil types-pytz types-simplejson types-ujson