From 2264ef7401155cb8e182f8221ab15046de6f12db Mon Sep 17 00:00:00 2001 From: ijl Date: Fri, 12 Apr 2024 22:46:07 +0000 Subject: [PATCH] manylinux_2_17, macos, windows CI --- .github/workflows/artifact.yaml | 391 ++++++++++++++++++++++++++++ .github/workflows/lint.yaml | 2 +- .github/workflows/linux.yaml | 306 ---------------------- Cargo.lock | 36 +-- bench/requirements.txt | 4 +- ci/azure-pipelines.yml | 44 ---- script/develop | 4 +- script/graph | 5 +- src/deserialize/cache.rs | 6 +- src/deserialize/yyjson.rs | 4 +- src/serialize/per_type/dataclass.rs | 4 +- src/serialize/per_type/dict.rs | 6 +- src/serialize/per_type/list.rs | 2 +- src/serialize/per_type/numpy.rs | 2 +- src/util.rs | 10 + 15 files changed, 438 insertions(+), 388 deletions(-) create mode 100644 .github/workflows/artifact.yaml delete mode 100644 .github/workflows/linux.yaml diff --git a/.github/workflows/artifact.yaml b/.github/workflows/artifact.yaml new file mode 100644 index 00000000..04ed8cfa --- /dev/null +++ b/.github/workflows/artifact.yaml @@ -0,0 +1,391 @@ +name: artifact +on: push +env: + RUST_TOOLCHAIN: "nightly-2024-03-27" +jobs: + + sdist: + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + env: + RUST_TOOLCHAIN: "1.72" # MSRV + steps: + - name: rustup stable + run: | + curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain "${RUST_TOOLCHAIN}" -y + rustup default "${RUST_TOOLCHAIN}" + + - uses: actions/checkout@v4 + + - run: python3 -m pip install --user --upgrade pip "maturin>=1,<2" wheel + + - name: Vendor dependencies + run: | + maturin build + cargo fetch + mkdir .cargo + cp ci/sdist.toml .cargo/config.toml + cargo vendor include/cargo --versioned-dirs + + - run: maturin sdist --out=dist + + - run: python3 -m pip install --user dist/orjson*.tar.gz + env: + CARGO_NET_OFFLINE: "true" + + - run: python3 -m pip install --user -r test/requirements.txt -r integration/requirements.txt mypy + + - run: pytest -s -rxX -v -n 4 test + env: + PYTHONMALLOC: "debug" + + - run: ./integration/run thread + - run: ./integration/run http + - run: ./integration/run init + - run: ./integration/run typestubs + + - name: Store sdist + if: "startsWith(github.ref, 'refs/tags/')" + uses: actions/upload-artifact@v4 + with: + name: orjson_sdist + path: dist + overwrite: true + retention-days: 1 + + manylinux_2_17_amd64: + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + python: [ + { version: '3.12', abi: 'cp312-cp312' }, + { version: '3.11', abi: 'cp311-cp311' }, + { version: '3.10', abi: 'cp310-cp310' }, + { version: '3.9', abi: 'cp39-cp39' }, + { version: '3.8', abi: 'cp38-cp38' }, + ] + env: + CC: "clang" + CFLAGS: "-Os -fstrict-aliasing -flto=full" + LDFLAGS: "-fuse-ld=lld -Wl,--as-needed" + RUSTFLAGS: "-C linker=clang -C lto=fat -C link-arg=-fuse-ld=lld -Z mir-opt-level=4 -Z virtual-function-elimination -Z threads=4 -D warnings" + CARGO_UNSTABLE_SPARSE_REGISTRY: "true" + UNSAFE_PYO3_SKIP_VERSION_CHECK: "1" + PATH: "/__w/orjson/orjson/.venv/bin:/github/home/.cargo/bin:/root/.local/bin:/root/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + container: + image: fedora:41 + options: --user 0 + steps: + + - uses: actions/checkout@v4 + + - name: Build environment + run: | + dnf install -y rustup clang lld python${{ matrix.python.version }} + + rustup-init --default-toolchain "${RUST_TOOLCHAIN}-x86_64-unknown-linux-gnu" --profile minimal --component rust-src -y + cargo fetch --target=x86_64-unknown-linux-gnu & + + curl -LsSf https://astral.sh/uv/install.sh | sh + uv venv --python python${{ matrix.python.version }} + uv pip install --upgrade "maturin>=1,<2" -r test/requirements.txt -r integration/requirements.txt + + mkdir .cargo + cp ci/config.toml .cargo/config.toml + + - name: maturin + run: | + maturin build --release --strip \ + --features=no-panic,unstable-simd,yyjson \ + --compatibility manylinux_2_17 \ + --interpreter python${{ matrix.python.version }} \ + --target=x86_64-unknown-linux-gnu + uv pip install target/wheels/orjson*.whl + + - run: pytest -s -rxX -v -n 4 test + env: + PYTHONMALLOC: "debug" + + - run: ./integration/run thread + - run: ./integration/run http + - run: ./integration/run init + + - name: Store wheels + if: "startsWith(github.ref, 'refs/tags/')" + uses: actions/upload-artifact@v4 + with: + name: orjson_manylinux_2_17_amd64_${{ matrix.python.version }} + path: target/wheels + overwrite: true + retention-days: 1 + + # musllinux_1_2: + # runs-on: ubuntu-22.04 + # strategy: + # fail-fast: false + # matrix: + # python: [ + # { version: '3.12' }, + # { version: '3.11' }, + # { version: '3.10' }, + # { version: '3.9' }, + # { version: '3.8' }, + # ] + # platform: + # - target: aarch64-unknown-linux-musl + # arch: aarch64 + # platform: linux/arm64 + # - target: x86_64-unknown-linux-musl + # arch: x86_64 + # platform: linux/amd64 + # steps: + # - uses: actions/checkout@v4 + + # - name: build-std + # run: | + # mkdir .cargo + # cp ci/config.toml .cargo/config.toml + + # - name: Build + # uses: PyO3/maturin-action@v1 + # env: + # CC: "gcc" + # CFLAGS: "-Os" + # LDFLAGS: "-Wl,--as-needed" + # RUSTFLAGS: "-Z mir-opt-level=4 -Z threads=4 -D warnings -C target-feature=-crt-static" + # CARGO_UNSTABLE_SPARSE_REGISTRY: "true" + # UNSAFE_PYO3_SKIP_VERSION_CHECK: "1" + # with: + # rust-toolchain: nightly-2024-03-27 + # rustup-components: rust-src + # target: ${{ matrix.platform.target }} + # manylinux: musllinux_1_2 + # args: --release --strip --out=dist --features=no-panic,unstable-simd,yyjson -i python${{ matrix.python.version }} + + # - name: QEMU + # if: matrix.platform.arch != 'x86_64' + # uses: docker/setup-qemu-action@v3 + # with: + # image: tonistiigi/binfmt:qemu-v8.1.5 + # platforms: ${{ matrix.platform.platform }} + + # - name: Test + # uses: addnab/docker-run-action@v3 + # with: + # image: quay.io/pypa/musllinux_1_2_${{ matrix.platform.arch }}:latest + # options: -v ${{ github.workspace }}:/io -w /io + # run: | + # apk add tzdata + # sed -i '/^psutil/d' test/requirements.txt # missing 3.11, 3.12 wheels + # sed -i '/^numpy/d' test/requirements.txt + + # python${{ matrix.python.version }} -m venv venv + # venv/bin/pip install -U pip wheel + # venv/bin/pip install -r test/requirements.txt + # venv/bin/pip install orjson --no-index --find-links dist/ --force-reinstall + # venv/bin/python -m pytest -s -rxX -v -n 4 test + + # - name: Store wheels + # if: "startsWith(github.ref, 'refs/tags/')" + # uses: actions/upload-artifact@v4 + # with: + # name: orjson_musllinux_1_2_${{ matrix.platform.arch }}_${{ matrix.python.version }} + # path: dist + # overwrite: true + # retention-days: 1 + + # manylinux_2_17_non_amd64: + # runs-on: ubuntu-22.04 + # strategy: + # fail-fast: false + # matrix: + # python: [ + # { version: '3.12', abi: 'cp312-cp312' }, + # { version: '3.11', abi: 'cp311-cp311' }, + # { version: '3.10', abi: 'cp310-cp310' }, + # { version: '3.9', abi: 'cp39-cp39' }, + # { version: '3.8', abi: 'cp38-cp38' }, + # ] + # target: [ + # { + # arch: 'aarch64', + # cflags: '-Os -flto=full -fstrict-aliasing', + # features: 'no-panic,unstable-simd,yyjson', + # rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings', + # target: 'aarch64-unknown-linux-gnu', + # }, + # { + # arch: 'armv7', + # cflags: '-Os -flto=full -fstrict-aliasing', + # features: 'no-panic,yyjson', # no SIMD + # rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings -C opt-level=s', + # target: 'armv7-unknown-linux-gnueabihf', + # }, + # { + # arch: 'ppc64le', + # cflags: '-Os -flto=full -fstrict-aliasing', + # features: 'no-panic,unstable-simd,yyjson', + # rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings', + # target: 'powerpc64le-unknown-linux-gnu', + # }, + # { + # arch: 's390x', + # cflags: '-Os -flto=full -fstrict-aliasing -march=z10', + # # no unstable-simd as rotate_elements_left() causes exit 125 in at least QEMU + # features: 'no-panic,yyjson', + # rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings -C target-cpu=z10', + # target: 's390x-unknown-linux-gnu', + # }, + # ] + # steps: + # - uses: actions/checkout@v4 + # - name: build-std + # run: | + # mkdir .cargo + # cp ci/config.toml .cargo/config.toml + + # - name: Build + # uses: PyO3/maturin-action@v1 + # env: + # PYO3_CROSS_LIB_DIR: "/opt/python/${{ matrix.python.abi }}" + # CFLAGS: "${{ matrix.target.cflags }}" + # LDFLAGS: "-Wl,--as-needed" + # CARGO_UNSTABLE_SPARSE_REGISTRY: "true" + # RUSTFLAGS: "${{ matrix.target.rustflags }}" + # UNSAFE_PYO3_SKIP_VERSION_CHECK: "1" + # with: + # target: ${{ matrix.target.target }} + # rust-toolchain: nightly-2024-03-27 + # rustup-components: rust-src + # manylinux: auto + # args: --release --strip --out=dist --features=${{ matrix.target.features }} -i python${{ matrix.python.version }} + + # - uses: uraimo/run-on-arch-action@v2 + # name: Test + # with: + # arch: ${{ matrix.target.arch }} + # distro: ubuntu22.04 + # githubToken: ${{ github.token }} + # install: | + # export TZ=UTC + # export DEBIAN_FRONTEND=noninteractive + # apt-get update + # apt-get install -y --no-install-recommends software-properties-common gpg gpg-agent curl + # add-apt-repository ppa:deadsnakes/ppa + # apt-get update + # apt-get install -y python${{ matrix.python.version }}-dev python${{ matrix.python.version }}-venv + # run: | + # python${{ matrix.python.version }} -m venv venv + # venv/bin/pip install -U pip wheel + # venv/bin/pip install -r test/requirements.txt + # venv/bin/pip install orjson --no-index --find-links dist/ --force-reinstall + # venv/bin/python -m pytest -s -rxX -v -n 2 test + + # - name: Store wheels + # if: "startsWith(github.ref, 'refs/tags/')" + # uses: actions/upload-artifact@v4 + # with: + # name: orjson_manylinux_2_17_${{ matrix.target.arch }}_${{ matrix.python.version }} + # path: dist + # overwrite: true + # retention-days: 1 + + macos_universal2: + runs-on: macos-14 + strategy: + fail-fast: false + matrix: + python: [ + { version: '3.12', macosx_target: "10.15" }, + { version: '3.11', macosx_target: "10.15" }, + { version: '3.10', macosx_target: "10.15" }, + ] + env: + CC: "clang" + CFLAGS: "-Os -fstrict-aliasing -flto=full" + LDFLAGS: "-Wl,--as-needed" + CFLAGS_x86_64_apple_darwin: "-O2 -fstrict-aliasing -flto=full -march=x86-64-v2 -mtune=generic" + CFLAGS_aarch64_apple_darwin: "-O2 -fstrict-aliasing -flto=full -mcpu=apple-m1 -mtune=generic" + RUSTFLAGS: "-C lto=fat -Z mir-opt-level=4 -Z virtual-function-elimination -Z threads=3 -D warnings" + CARGO_UNSTABLE_SPARSE_REGISTRY: "true" + UNSAFE_PYO3_SKIP_VERSION_CHECK: "1" + PATH: "/Users/runner/work/orjson/orjson/.venv/bin:/Users/runner/.cargo/bin:/usr/local/opt/curl/bin:/usr/local/bin:/usr/local/sbin:/Users/runner/bin:/Library/Frameworks/Python.framework/Versions/Current/bin:/usr/bin:/bin:/usr/sbin:/sbin" + steps: + + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "${{ matrix.python.version }}" + + - uses: dtolnay/rust-toolchain@master + with: + toolchain: "nightly-2024-03-27" + targets: "x86_64-apple-darwin, aarch64-apple-darwin" + components: "rust-src" + + - name: Build environment + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + uv venv --python python${{ matrix.python.version }} + uv pip install --upgrade "maturin>=1,<2" -r test/requirements.txt -r integration/requirements.txt + + cargo fetch & + mkdir .cargo + cp ci/config.toml .cargo/config.toml + + echo $PATH + + - name: maturin + run: | + PATH=$HOME/.cargo/bin:$PATH \ + MACOSX_DEPLOYMENT_TARGET="${{ matrix.python.macosx_target }}" \ + PYO3_CROSS_LIB_DIR=$(python -c "import sysconfig;print(sysconfig.get_config_var('LIBDIR'))") \ + maturin build --release --strip \ + --features=no-panic,unstable-simd,yyjson \ + --interpreter python${{ matrix.python.version }} \ + --target=universal2-apple-darwin + uv pip install target/wheels/orjson*.whl + + - run: pytest -s -rxX -v -n 4 test + env: + PYTHONMALLOC: "debug" + + - run: source .venv/bin/activate && ./integration/run thread + - run: source .venv/bin/activate && ./integration/run http + - run: source .venv/bin/activate && ./integration/run init + + - name: Store wheels + if: "startsWith(github.ref, 'refs/tags/')" + uses: actions/upload-artifact@v4 + with: + name: orjson_universal2_${{ matrix.python.version }} + path: target/wheels + overwrite: true + retention-days: 1 + + # pypi: + # name: PyPI + # runs-on: ubuntu-22.04 + # if: "startsWith(github.ref, 'refs/tags/')" + # needs: [ manylinux_2_17_amd64, manylinux_2_17_non_amd64, musllinux_1_2, macos_universal2, sdist ] + # steps: + # - uses: actions/download-artifact@v4 + # with: + # pattern: orjson_* + # merge-multiple: true + # - uses: actions/setup-python@v5 + # with: + # python-version: "3.12" + # - run: pip install pip "maturin>=1,<2" + # - run: ls -1 . + # - name: deploy wheel + # run: maturin upload --skip-existing --username "$MATURIN_USERNAME" *.whl + # env: + # MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} + # - name: deploy sdist + # run: maturin upload --skip-existing --username "$MATURIN_USERNAME" *.tar.gz + # env: + # MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index d7bbae4a..aaa5b77c 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -6,7 +6,7 @@ jobs: steps: - uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: "3.12" - run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain=stable --profile=default -y - run: pip install -U autoflake isort black ruff mypy types-python-dateutil types-pytz types-simplejson types-ujson diff --git a/.github/workflows/linux.yaml b/.github/workflows/linux.yaml deleted file mode 100644 index 1612e435..00000000 --- a/.github/workflows/linux.yaml +++ /dev/null @@ -1,306 +0,0 @@ -name: linux -on: push -jobs: - - sdist: - runs-on: ubuntu-22.04 - strategy: - fail-fast: false - env: - RUST_TOOLCHAIN: "1.72" # MSRV - steps: - - run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain $RUST_TOOLCHAIN -y - - run: rustup default $RUST_TOOLCHAIN - - - uses: actions/checkout@v4 - - - run: python3 -m pip install --user --upgrade pip "maturin>=1,<2" wheel - - - run: maturin build - - run: cargo fetch - - run: mkdir .cargo - - run: cp ci/sdist.toml .cargo/config.toml - - run: cargo vendor include/cargo --versioned-dirs - - - run: maturin sdist --out=dist - - - run: python3 -m pip install --user dist/orjson*.tar.gz - env: - CARGO_NET_OFFLINE: "true" - - - run: python3 -m pip install --user -r test/requirements.txt -r integration/requirements.txt mypy - - - run: pytest -s -rxX -v -n 4 test - env: - PYTHONMALLOC: "debug" - - - run: ./integration/run thread - - run: ./integration/run http - - run: ./integration/run init - - run: ./integration/run typestubs - - - name: Store sdist - if: "startsWith(github.ref, 'refs/tags/')" - uses: actions/upload-artifact@v4 - with: - name: orjson_sdist - path: dist - overwrite: true - retention-days: 1 - - manylinux_2_17_amd64: - runs-on: ubuntu-22.04 - strategy: - fail-fast: false - matrix: - python: [ - { version: '3.12', abi: 'cp312-cp312' }, - { version: '3.11', abi: 'cp311-cp311' }, - { version: '3.10', abi: 'cp310-cp310' }, - { version: '3.9', abi: 'cp39-cp39' }, - { version: '3.8', abi: 'cp38-cp38' }, - ] - env: - PATH: /github/home/.local/bin:/github/home/.cargo/bin:/opt/python/${{ matrix.python.abi }}/bin:/opt/rh/gcc-toolset-12/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - CC: "clang" - CFLAGS: "-Os -fstrict-aliasing -flto=full" - LDFLAGS: "-fuse-ld=lld -Wl,--as-needed" - RUSTFLAGS: "-C linker=clang -C lto=fat -C link-arg=-fuse-ld=lld -Z mir-opt-level=4 -Z virtual-function-elimination -D warnings" - CARGO_UNSTABLE_SPARSE_REGISTRY: "true" - UNSAFE_PYO3_SKIP_VERSION_CHECK: "1" - container: - image: quay.io/pypa/manylinux_2_28_x86_64:latest - options: --user 0 - steps: - - run: yum install -y clang lld - - run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2024-03-27 --profile minimal -y - - run: rustup component add rust-src --toolchain nightly-2024-03-27-x86_64-unknown-linux-gnu - - uses: actions/checkout@v4 - - - name: build-std - run: | - mkdir .cargo - cp ci/config.toml .cargo/config.toml - - - run: python3 -m pip install --user --upgrade pip "maturin>=1,<2" wheel - - run: | - maturin build --release --strip \ - --out=dist \ - --features=no-panic,unstable-simd,yyjson \ - --compatibility manylinux_2_17 \ - --interpreter python${{ matrix.python.version }} \ - --target=x86_64-unknown-linux-gnu - - run: python3 -m pip install --user dist/orjson*.whl - - run: python3 -m pip install --user -r test/requirements.txt -r integration/requirements.txt - - - run: pytest -s -rxX -v -n 4 test - env: - PYTHONMALLOC: "debug" - - - run: ./integration/run thread - - run: ./integration/run http - - run: ./integration/run init - - - name: Store wheels - if: "startsWith(github.ref, 'refs/tags/')" - uses: actions/upload-artifact@v4 - with: - name: orjson_manylinux_2_17_amd64_${{ matrix.python.version }} - path: dist - overwrite: true - retention-days: 1 - - musllinux_1_2: - runs-on: ubuntu-22.04 - strategy: - fail-fast: false - matrix: - python: [ - { version: '3.12' }, - { version: '3.11' }, - { version: '3.10' }, - { version: '3.9' }, - { version: '3.8' }, - ] - platform: - - target: aarch64-unknown-linux-musl - arch: aarch64 - platform: linux/arm64 - - target: x86_64-unknown-linux-musl - arch: x86_64 - platform: linux/amd64 - steps: - - uses: actions/checkout@v4 - - - name: build-std - run: | - mkdir .cargo - cp ci/config.toml .cargo/config.toml - - - name: Build - uses: PyO3/maturin-action@v1 - env: - CC: "gcc" - CFLAGS: "-Os" - LDFLAGS: "-Wl,--as-needed" - RUSTFLAGS: "-Z mir-opt-level=4 -D warnings -C target-feature=-crt-static" - CARGO_UNSTABLE_SPARSE_REGISTRY: "true" - UNSAFE_PYO3_SKIP_VERSION_CHECK: "1" - with: - rust-toolchain: nightly-2024-03-27 - rustup-components: rust-src - target: ${{ matrix.platform.target }} - manylinux: musllinux_1_2 - args: --release --strip --out=dist --features=no-panic,unstable-simd,yyjson -i python${{ matrix.python.version }} - - - name: Set up QEMU - if: matrix.platform.arch != 'x86_64' - uses: docker/setup-qemu-action@v3 - with: - image: tonistiigi/binfmt:qemu-v8.1.4 - platforms: ${{ matrix.platform.platform }} - - - name: Test - uses: addnab/docker-run-action@v3 - with: - image: quay.io/pypa/musllinux_1_2_${{ matrix.platform.arch }}:latest - options: -v ${{ github.workspace }}:/io -w /io - run: | - apk add tzdata - sed -i '/^psutil/d' test/requirements.txt # missing 3.11, 3.12 wheels - sed -i '/^numpy/d' test/requirements.txt - - python${{ matrix.python.version }} -m venv venv - venv/bin/pip install -U pip wheel - venv/bin/pip install -r test/requirements.txt - venv/bin/pip install orjson --no-index --find-links dist/ --force-reinstall - venv/bin/python -m pytest -s -rxX -v test - - - name: Store wheels - if: "startsWith(github.ref, 'refs/tags/')" - uses: actions/upload-artifact@v4 - with: - name: orjson_musllinux_1_2_${{ matrix.platform.arch }}_${{ matrix.python.version }} - path: dist - overwrite: true - retention-days: 1 - - manylinux_2_17_non_amd64: - runs-on: ubuntu-22.04 - strategy: - fail-fast: false - matrix: - python: [ - { version: '3.12', abi: 'cp312-cp312' }, - { version: '3.11', abi: 'cp311-cp311' }, - { version: '3.10', abi: 'cp310-cp310' }, - { version: '3.9', abi: 'cp39-cp39' }, - { version: '3.8', abi: 'cp38-cp38' }, - ] - target: [ - { - arch: 'aarch64', - cflags: '-Os -flto=full -fstrict-aliasing', - features: 'no-panic,unstable-simd,yyjson', - rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings', - target: 'aarch64-unknown-linux-gnu', - }, - { - arch: 'armv7', - cflags: '-Os -flto=full -fstrict-aliasing', - features: 'no-panic,yyjson', # no SIMD - rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings -C opt-level=s', - target: 'armv7-unknown-linux-gnueabihf', - }, - { - arch: 'ppc64le', - cflags: '-Os -flto=full -fstrict-aliasing', - features: 'no-panic,unstable-simd,yyjson', - rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings', - target: 'powerpc64le-unknown-linux-gnu', - }, - { - arch: 's390x', - cflags: '-Os -flto=full -fstrict-aliasing -march=z10', - # no unstable-simd as rotate_elements_left() causes exit 125 in at least QEMU - features: 'no-panic,yyjson', - rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings -C target-cpu=z10', - target: 's390x-unknown-linux-gnu', - }, - ] - steps: - - uses: actions/checkout@v4 - - name: build-std - run: | - mkdir .cargo - cp ci/config.toml .cargo/config.toml - - - name: Build - uses: PyO3/maturin-action@v1 - env: - PYO3_CROSS_LIB_DIR: "/opt/python/${{ matrix.python.abi }}" - CFLAGS: "${{ matrix.target.cflags }}" - LDFLAGS: "-Wl,--as-needed" - CARGO_UNSTABLE_SPARSE_REGISTRY: "true" - RUSTFLAGS: "${{ matrix.target.rustflags }}" - UNSAFE_PYO3_SKIP_VERSION_CHECK: "1" - with: - target: ${{ matrix.target.target }} - rust-toolchain: nightly-2024-03-27 - rustup-components: rust-src - manylinux: auto - args: --release --strip --out=dist --features=${{ matrix.target.features }} -i python${{ matrix.python.version }} - - - uses: uraimo/run-on-arch-action@v2 - name: Test - with: - arch: ${{ matrix.target.arch }} - distro: ubuntu22.04 - githubToken: ${{ github.token }} - install: | - export TZ=UTC - export DEBIAN_FRONTEND=noninteractive - apt-get update - apt-get install -y --no-install-recommends software-properties-common gpg gpg-agent curl - add-apt-repository ppa:deadsnakes/ppa - apt-get update - apt-get install -y python${{ matrix.python.version }}-dev python${{ matrix.python.version }}-venv - run: | - python${{ matrix.python.version }} -m venv venv - venv/bin/pip install -U pip wheel - venv/bin/pip install -r test/requirements.txt - venv/bin/pip install orjson --no-index --find-links dist/ --force-reinstall - venv/bin/python -m pytest -s -rxX -v -n 2 test - - - name: Store wheels - if: "startsWith(github.ref, 'refs/tags/')" - uses: actions/upload-artifact@v4 - with: - name: orjson_manylinux_2_17_${{ matrix.target.arch }}_${{ matrix.python.version }} - path: dist - overwrite: true - retention-days: 1 - - pypi: - name: PyPI - runs-on: ubuntu-22.04 - if: "startsWith(github.ref, 'refs/tags/')" - needs: [ manylinux_2_17_amd64, manylinux_2_17_non_amd64, musllinux_1_2, sdist ] - steps: - - uses: actions/download-artifact@v4 - with: - pattern: orjson_* - merge-multiple: true - - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - run: pip install pip "maturin>=1,<2" - - run: ls -1 . - - name: deploy wheel - run: maturin upload --skip-existing --username "$MATURIN_USERNAME" *.whl - env: - MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} - - name: deploy sdist - run: maturin upload --skip-existing --username "$MATURIN_USERNAME" *.tar.gz - env: - MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} diff --git a/Cargo.lock b/Cargo.lock index a396a873..7e85f5c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -62,9 +62,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.90" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5" +checksum = "2678b2e3449475e95b0aa6f9b506a28e61b3dc8996592b983695e8ebb58a8b41" [[package]] name = "cfg-if" @@ -123,18 +123,18 @@ checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" [[package]] name = "encoding_rs" -version = "0.8.33" +version = "0.8.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" dependencies = [ "cfg-if", ] [[package]] name = "getrandom" -version = "0.2.12" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" +checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" dependencies = [ "cfg-if", "libc", @@ -143,9 +143,9 @@ dependencies = [ [[package]] name = "half" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5eceaaeec696539ddaf7b333340f1af35a5aa87ae3e4f3ead0532f72affab2e" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" dependencies = [ "cfg-if", "crunchy", @@ -236,9 +236,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.21.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5eb0b6ecba38961f6f4bd6cd5906dfab3cd426ff37b2eed5771006aa31656f1" +checksum = "650dca34d463b6cdbdb02b1d71bfd6eb6b6816afc708faebb3bac1380ff4aef7" dependencies = [ "once_cell", "target-lexicon", @@ -246,9 +246,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.21.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba8a6e48a29b5d22e4fdaf132d8ba8d3203ee9f06362d48f244346902a594ec3" +checksum = "09a7da8fc04a8a2084909b59f29e1b8474decac98b951d77b80b26dc45f046ad" dependencies = [ "libc", "pyo3-build-config", @@ -256,18 +256,18 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.35" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] [[package]] name = "rustversion" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" +checksum = "80af6f9131f277a45a3fba6ce8e2258037bb0477a67e610d3c1fe046ab31de47" [[package]] name = "ryu" @@ -329,9 +329,9 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "syn" -version = "2.0.55" +version = "2.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "002a1b3dbf967edfafc32655d0f377ab0bb7b994aa1d32c8cc7e9b8bf3ebb8f0" +checksum = "44cfb93f38070beee36b3fef7d4f5a16f27751d94b187b666a5cc5e9b0d30687" dependencies = [ "proc-macro2", "quote", diff --git a/bench/requirements.txt b/bench/requirements.txt index 806f5b81..40d524ca 100644 --- a/bench/requirements.txt +++ b/bench/requirements.txt @@ -1,9 +1,9 @@ memory-profiler -pandas +pandas; python_version<"3.13" pytest-benchmark pytest-random-order python-rapidjson -seaborn +seaborn; python_version<"3.13" simplejson tabulate ujson diff --git a/ci/azure-pipelines.yml b/ci/azure-pipelines.yml index 69380b81..4e2ca6b7 100644 --- a/ci/azure-pipelines.yml +++ b/ci/azure-pipelines.yml @@ -3,50 +3,6 @@ variables: jobs: -- job: macos_python312_univeral2 - pool: - vmImage: macOS-11 - variables: - interpreter: python3.12 - macosx_deployment_target: '10.15' - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.12.0' - addToPath: true - allowUnstable: true - - checkout: self - - template: ./azure-macos.yml - -- job: macos_python311_univeral2 - pool: - vmImage: macOS-11 - variables: - interpreter: python3.11 - macosx_deployment_target: '10.15' - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.11.4' - addToPath: true - allowUnstable: true - - checkout: self - - template: ./azure-macos.yml - -- job: macos_python310_univeral2 - pool: - vmImage: macOS-11 - variables: - interpreter: python3.10 - macosx_deployment_target: '10.15' - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.10' - addToPath: true - - checkout: self - - template: ./azure-macos.yml - - job: macos_python39_univeral2 pool: vmImage: macOS-11 diff --git a/script/develop b/script/develop index 3a4042e4..a4207166 100755 --- a/script/develop +++ b/script/develop @@ -9,10 +9,10 @@ export LD="${LD:-lld}" echo "CC: ${CC}, LD: ${LD}, LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}" -export CFLAGS="-Os -fstrict-aliasing -flto=full" +export CFLAGS="-Os -fstrict-aliasing -fno-plt -flto=full" export LDFLAGS="-fuse-ld=${LD} -Wl,--as-needed" export RUSTFLAGS="-C linker=${CC} -C lto=fat -C link-arg=-fuse-ld=${LD} -Z mir-opt-level=4 -Z virtual-function-elimination -Z threads=8" maturin build "$@" -pip install --force target/wheels/*.whl +uv pip install target/wheels/*.whl diff --git a/script/graph b/script/graph index afd45ce9..72bcdd95 100755 --- a/script/graph +++ b/script/graph @@ -78,7 +78,10 @@ def tab(obj): if isinstance(each[1], float) else None ) - each[1] = "%.1f" % each[1] if isinstance(each[1], float) else None + if group.startswith("github"): + each[1] = "%.2f" % each[1] if isinstance(each[1], float) else None + else: + each[1] = "%.1f" % each[1] if isinstance(each[1], float) else None buf.write(tabulate(table, headers, tablefmt="github") + "\n") diff --git a/src/deserialize/cache.rs b/src/deserialize/cache.rs index 169712ad..610377d3 100644 --- a/src/deserialize/cache.rs +++ b/src/deserialize/cache.rs @@ -40,11 +40,7 @@ pub static mut KEY_MAP: OnceCell = OnceCell::new(); pub fn cache_hash(key: &[u8]) -> u64 { // try to omit code for >64 path in ahash - debug_assert!(key.len() <= 64); - #[cfg(feature = "intrinsics")] - unsafe { - core::intrinsics::assume(key.len() <= 64); - }; + assume!(key.len() <= 64); let mut hasher = ahash::AHasher::default(); hasher.write(key); hasher.finish() diff --git a/src/deserialize/yyjson.rs b/src/deserialize/yyjson.rs index a05fa37d..090da550 100644 --- a/src/deserialize/yyjson.rs +++ b/src/deserialize/yyjson.rs @@ -146,7 +146,7 @@ fn parse_yy_array(elem: *mut yyjson_val) -> NonNull { return nonnull!(list); } let mut cur = unsafe_yyjson_get_first(elem); - for idx in 0..=len - 1 { + for idx in 0..len { let next = unsafe_yyjson_get_next(cur); let val = parse_node(cur).as_ptr(); ffi!(PyList_SET_ITEM(list, idx as isize, val)); @@ -165,7 +165,7 @@ fn parse_yy_object(elem: *mut yyjson_val) -> NonNull { } let mut key = unsafe_yyjson_get_first(elem); let dict = ffi!(_PyDict_NewPresized(len as isize)); - for _ in 0..=len - 1 { + for _ in 0..len { let val = key.add(1); let key_str = str_from_slice!((*key).uni.str_ as *const u8, unsafe_yyjson_get_len(key)); let pykey = get_unicode_key(key_str); diff --git a/src/serialize/per_type/dataclass.rs b/src/serialize/per_type/dataclass.rs index 5acf0937..f3c434b9 100644 --- a/src/serialize/per_type/dataclass.rs +++ b/src/serialize/per_type/dataclass.rs @@ -98,7 +98,7 @@ impl Serialize for DataclassFastSerializer { let mut pos = 0; pydict_next!(self.ptr, &mut pos, &mut next_key, &mut next_value); - for _ in 0..=ffi!(Py_SIZE(self.ptr)) as usize - 1 { + for _ in 0..ffi!(Py_SIZE(self.ptr)) as usize { let key = next_key; let value = next_value; @@ -163,7 +163,7 @@ impl Serialize for DataclassFallbackSerializer { let mut pos = 0; pydict_next!(fields, &mut pos, &mut next_key, &mut next_value); - for _ in 0..=ffi!(Py_SIZE(fields)) as usize - 1 { + for _ in 0..ffi!(Py_SIZE(fields)) as usize { let attr = next_key; let field = next_value; diff --git a/src/serialize/per_type/dict.rs b/src/serialize/per_type/dict.rs index 0fb6a19c..f79cbf60 100644 --- a/src/serialize/per_type/dict.rs +++ b/src/serialize/per_type/dict.rs @@ -111,7 +111,7 @@ impl Serialize for Dict { let mut pos = 0; pydict_next!(self.ptr, &mut pos, &mut next_key, &mut next_value); - for _ in 0..=ffi!(Py_SIZE(self.ptr)) as usize - 1 { + for _ in 0..ffi!(Py_SIZE(self.ptr)) as usize { let key = next_key; let value = next_value; @@ -210,7 +210,7 @@ impl Serialize for DictSortedKey { let mut pos = 0; pydict_next!(self.ptr, &mut pos, &mut next_key, &mut next_value); - for _ in 0..=len as usize - 1 { + for _ in 0..len as usize { let key = next_key; let value = next_value; @@ -377,7 +377,7 @@ impl Serialize for DictNonStrKey { let mut pos = 0; pydict_next!(self.ptr, &mut pos, &mut next_key, &mut next_value); - for _ in 0..=len - 1 { + for _ in 0..len { let key = next_key; let value = next_value; diff --git a/src/serialize/per_type/list.rs b/src/serialize/per_type/list.rs index 1599a944..ae244b61 100644 --- a/src/serialize/per_type/list.rs +++ b/src/serialize/per_type/list.rs @@ -88,7 +88,7 @@ impl Serialize for ListTupleSerializer { } debug_assert!(self.len >= 1); let mut seq = serializer.serialize_seq(None).unwrap(); - for idx in 0..=self.len - 1 { + for idx in 0..self.len { let value = unsafe { *((self.data_ptr).add(idx)) }; let value_ob_type = ob_type!(value); if is_class_by_type!(value_ob_type, STR_TYPE) { diff --git a/src/serialize/per_type/numpy.rs b/src/serialize/per_type/numpy.rs index 2f86d560..dede7996 100644 --- a/src/serialize/per_type/numpy.rs +++ b/src/serialize/per_type/numpy.rs @@ -238,7 +238,7 @@ impl NumpyArray { #[cfg_attr(feature = "optimize", optimize(size))] fn build(&mut self) { if self.depth < self.dimensions() - 1 { - for i in 0..=self.shape()[self.depth] - 1 { + for i in 0..self.shape()[self.depth] { let mut position: Vec = self.position.to_vec(); position[self.depth] = i; let num_children: usize = if self.depth < self.dimensions() - 2 { diff --git a/src/util.rs b/src/util.rs index c7bbe721..787aa8aa 100644 --- a/src/util.rs +++ b/src/util.rs @@ -257,3 +257,13 @@ macro_rules! reserve_minimum { $writer.reserve(64); }; } + +macro_rules! assume { + ($expr:expr) => { + debug_assert!($expr); + #[cfg(feature = "intrinsics")] + unsafe { + core::intrinsics::assume($expr); + }; + }; +}