diff --git a/.github/workflows/artifact.yaml b/.github/workflows/artifact.yaml index eae5339d..a5249751 100644 --- a/.github/workflows/artifact.yaml +++ b/.github/workflows/artifact.yaml @@ -3,7 +3,7 @@ on: push env: CARGO_UNSTABLE_SPARSE_REGISTRY: "true" PIP_DISABLE_PIP_VERSION_CHECK: "1" - RUST_TOOLCHAIN: "nightly-2024-09-25" + RUST_TOOLCHAIN: "nightly-2024-11-12" UNSAFE_PYO3_BUILD_FREE_THREADED: "1" UNSAFE_PYO3_SKIP_VERSION_CHECK: "1" UV_LINK_MODE: "copy" @@ -77,7 +77,6 @@ jobs: { cc: "clang", cflags: "-Os -fstrict-aliasing -fno-plt -flto=full -emit-llvm", - features: "avx512,no-panic,unstable-simd,yyjson", ldflags: "-fuse-ld=lld -Wl,-plugin-opt=also-emit-llvm -Wl,--as-needed -Wl,-zrelro,-znow", rustflags: "-C linker=clang -C link-arg=-fuse-ld=lld -C linker-plugin-lto -C lto=fat -C link-arg=-Wl,-zrelro,-znow -Z mir-opt-level=4 -Z threads=4 -D warnings", tag: null, @@ -85,17 +84,16 @@ jobs: }, ] env: - PYTHON: "${{ matrix.python.interpreter }}" - PYTHON_PACKAGE: "${{ matrix.python.package }}" - TARGET: "${{ matrix.arch.target }}" + CARGO_TARGET_DIR: "/tmp/orjson" CC: "${{ matrix.arch.cc }}" - VENV: ".venv" - FEATURES: "${{ matrix.arch.features }}" CFLAGS: "${{ matrix.arch.cflags }}" + COMPATIBILITY: "${{ matrix.python.compatibility }}" LDFLAGS: "${{ matrix.arch.ldflags }}" + PYTHON: "${{ matrix.python.interpreter }}" + PYTHON_PACKAGE: "${{ matrix.python.package }}" RUSTFLAGS: "${{ matrix.arch.rustflags }}" - CARGO_TARGET_DIR: "/tmp/orjson" - COMPATIBILITY: "${{ matrix.python.compatibility }}" + TARGET: "${{ matrix.arch.target }}" + VENV: ".venv" steps: - name: cpuinfo @@ -130,7 +128,6 @@ jobs: source "${VENV}/bin/activate" maturin build --release --strip \ - --features="${FEATURES}" \ --compatibility="${COMPATIBILITY}" \ --interpreter="${PYTHON}" \ --target="${TARGET}" @@ -167,7 +164,6 @@ jobs: { cc: "clang", cflags: "-Os -fstrict-aliasing -fno-plt -flto=full -emit-llvm", - features: "no-panic,unstable-simd,yyjson", ldflags: "-fuse-ld=lld -Wl,-plugin-opt=also-emit-llvm -Wl,--as-needed -Wl,-zrelro,-znow", rustflags: "-C linker=clang -C link-arg=-fuse-ld=lld -C linker-plugin-lto -C lto=fat -C link-arg=-Wl,-zrelro,-znow -Z mir-opt-level=4 -Z threads=4 -D warnings", tag: "aarch64", @@ -175,17 +171,16 @@ jobs: }, ] env: - PYTHON: "${{ matrix.python.interpreter }}" - PYTHON_PACKAGE: "${{ matrix.python.package }}" - TARGET: "${{ matrix.arch.target }}" + CARGO_TARGET_DIR: "/tmp/orjson" CC: "${{ matrix.arch.cc }}" - VENV: ".venv" - FEATURES: "${{ matrix.arch.features }}" CFLAGS: "${{ matrix.arch.cflags }}" + COMPATIBILITY: "${{ matrix.python.compatibility }}" LDFLAGS: "${{ matrix.arch.ldflags }}" + PYTHON: "${{ matrix.python.interpreter }}" + PYTHON_PACKAGE: "${{ matrix.python.package }}" RUSTFLAGS: "${{ matrix.arch.rustflags }}" - CARGO_TARGET_DIR: "/tmp/orjson" - COMPATIBILITY: "${{ matrix.python.compatibility }}" + TARGET: "${{ matrix.arch.target }}" + VENV: ".venv" steps: - name: cpuinfo @@ -219,7 +214,6 @@ jobs: source "${VENV}/bin/activate" maturin build --release --strip \ - --features="${FEATURES}" \ --compatibility="${COMPATIBILITY}" \ --interpreter="${PYTHON}" \ --target="${TARGET}" @@ -256,11 +250,9 @@ jobs: - target: aarch64-unknown-linux-musl arch: aarch64 platform: linux/arm64 - features: no-panic,unstable-simd,unwind,yyjson - target: x86_64-unknown-linux-musl arch: x86_64 platform: linux/amd64 - features: avx512,no-panic,unstable-simd,unwind,yyjson steps: - uses: actions/checkout@v4 @@ -281,7 +273,7 @@ jobs: rustup-components: rust-src target: "${{ matrix.platform.target }}" manylinux: musllinux_1_2 - args: --release --strip --out=dist --features=${{ matrix.platform.features }} -i python${{ matrix.python.version }} + args: --release --strip --out=dist -i python${{ matrix.python.version }} - name: QEMU if: matrix.platform.arch != 'x86_64' @@ -331,28 +323,24 @@ jobs: { arch: 'aarch64', cflags: '-Os -flto=full -fstrict-aliasing', - features: 'no-panic,unstable-simd,yyjson', rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings', target: 'aarch64-unknown-linux-gnu', }, { arch: 'armv7', cflags: '-Os -flto=full -fstrict-aliasing', - features: 'no-panic,yyjson', # no SIMD rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings -C opt-level=s', target: 'armv7-unknown-linux-gnueabihf', }, { arch: 'ppc64le', cflags: '-Os -flto=full -fstrict-aliasing', - features: 'no-panic,unstable-simd,yyjson', rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings', target: 'powerpc64le-unknown-linux-gnu', }, { arch: 's390x', cflags: '-Os -flto=full -fstrict-aliasing -march=z10', - features: 'no-panic,yyjson', rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings -C target-cpu=z10', target: 's390x-unknown-linux-gnu', }, @@ -377,7 +365,7 @@ jobs: rust-toolchain: "${{ env.RUST_TOOLCHAIN }}" rustup-components: rust-src manylinux: auto - args: --release --strip --out=dist --features=${{ matrix.target.features }} -i python${{ matrix.python.version }} + args: --release --strip --out=dist -i python${{ matrix.python.version }} - name: Store wheels if: "startsWith(github.ref, 'refs/tags/')" @@ -439,7 +427,6 @@ jobs: MACOSX_DEPLOYMENT_TARGET="${{ matrix.python.macosx_target }}" \ PYO3_CROSS_LIB_DIR=$(python -c "import sysconfig;print(sysconfig.get_config_var('LIBDIR'))") \ maturin build --release --strip \ - --features=no-panic,unstable-simd,yyjson \ --interpreter python${{ matrix.python.version }} \ --target=universal2-apple-darwin uv pip install target/wheels/orjson*.whl @@ -509,7 +496,6 @@ jobs: MACOSX_DEPLOYMENT_TARGET="${{ matrix.python.macosx_target }}" \ PYO3_CROSS_LIB_DIR=$(python -c "import sysconfig;print(sysconfig.get_config_var('LIBDIR'))") \ maturin build --release --strip \ - --features=no-panic,unstable-simd,yyjson \ --interpreter python${{ matrix.python.version }} \ --target=universal2-apple-darwin uv pip install target/wheels/orjson*.whl diff --git a/.github/workflows/debug.yaml b/.github/workflows/debug.yaml index 863135ac..e9cda500 100644 --- a/.github/workflows/debug.yaml +++ b/.github/workflows/debug.yaml @@ -10,8 +10,7 @@ jobs: profile: [ { rust: "1.72", features: "" }, { rust: "1.72", features: "--features=yyjson" }, - { rust: "nightly-2024-09-25", features: "--features=yyjson,unstable-simd" }, - { rust: "nightly-2024-09-25", features: "--features=avx512,yyjson,unstable-simd" }, + { rust: "nightly-2024-11-12", features: "--features=yyjson,unstable-simd" }, ] python: [ { version: '3.13' }, diff --git a/Cargo.lock b/Cargo.lock index fde9db8d..5f9e8987 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -25,9 +25,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.31" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f" +checksum = "1aeb932158bd710538c73702db6945cb68a8fb08c519e6e12706b94263b36db8" dependencies = [ "shlex", ] @@ -70,9 +70,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.30.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e1d97fbe9722ba9bbd0c97051c2956e726562b61f86a25a4360398a40edfc9" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "half" @@ -107,9 +107,9 @@ checksum = "b9d9d414fc817d3e3d62b2598616733f76c4cc74fbac96069674739b881295c8" [[package]] name = "libc" -version = "0.2.161" +version = "0.2.162" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" +checksum = "18d287de67fe55fd7e1581fe933d965a5a9477b38e949cfa9f8574ef01506398" [[package]] name = "memchr" @@ -119,9 +119,9 @@ checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "no-panic" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8540b7d99a20166178b42a05776aef900cdbfec397f861dfc7819bf1d7760b3d" +checksum = "f83ab66434c11e561bb62671793397041e63d811f0123309eb34c3e61ea3295e" dependencies = [ "proc-macro2", "quote", @@ -212,18 +212,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.214" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5" +checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.214" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" +checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" dependencies = [ "proc-macro2", "quote", @@ -268,9 +268,9 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "syn" -version = "2.0.86" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89275301d38033efb81a6e60e3497e734dfcc62571f2854bf4b16690398824c" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", @@ -291,9 +291,9 @@ checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "unwinding" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc55842d0db6329a669d55a623c674b02d677b16bfb2d24857d4089d41eba882" +checksum = "637d511437df708cee34bdec7ba2f1548d256b7acf3ff20e0a1c559f9bf3a987" dependencies = [ "gimli", ] diff --git a/Cargo.toml b/Cargo.toml index 516f4d0e..7393c3fa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,28 +32,17 @@ crate-type = ["cdylib"] [features] default = [] -# Use SIMD intrinsics. This requires Rust on the nightly channel. -unstable-simd = [] - -# Include runtime-detected functions that use AVX512VL. Requires unstable-simd and amd64. -avx512 = [] - -no-panic = [ - "itoa/no-panic", - "ryu/no-panic", -] - -# Avoid bundling libgcc on musl. -unwind = ["unwinding"] - # Build yyjson as a backend and panic if it fails. The default is to attempt # to build and on failure fall back to another backend. yyjson = [] # Features detected by build.rs. Do not specify. +avx512 = [] inline_int = [] intrinsics = [] optimize = [] +unstable-simd = [] +unwind = [] [dependencies] associative-cache = { version = "2", default-features = false } @@ -61,17 +50,17 @@ bytecount = { version = "^0.6.7", default-features = false, features = ["runtime compact_str = { version = "0.8", default-features = false, features = ["serde"] } encoding_rs = { version = "0.8", default-features = false } half = { version = "2", default-features = false, features = ["std"] } -itoa = { version = "1", default-features = false } +itoa = { version = "1", default-features = false, features = ["no-panic"] } itoap = { version = "1", default-features = false, features = ["std", "simd"] } jiff = { version = "^0.1", default-features = false, features = ["alloc"] } once_cell = { version = "1", default-features = false, features = ["alloc", "race"] } pyo3-ffi = { path = "include/pyo3/pyo3-ffi", default-features = false, features = ["extension-module"]} -ryu = { version = "1", default-features = false } +ryu = { version = "1", default-features = false, features = ["no-panic"] } serde = { version = "1", default-features = false } serde_json = { version = "1", default-features = false, features = ["std", "float_roundtrip"] } simdutf8 = { version = "0.1", default-features = false, features = ["std", "public_imp", "aarch64_neon"] } smallvec = { version = "^1.11", default-features = false, features = ["union", "write"] } -unwinding = { version = "=0.2.2", default-features = false, features = ["unwinder"], optional = true } +unwinding = { version = "^0.2.3", default-features = false, features = ["unwinder"] } uuid = { version = "1", default-features = false } xxhash-rust = { version = "^0.8", default-features = false, features = ["xxh3"] } diff --git a/README.md b/README.md index 942597ef..e2648ac5 100644 --- a/README.md +++ b/README.md @@ -1224,9 +1224,10 @@ It benefits from also having a C build environment to compile a faster deserialization backend. See this project's `manylinux_2_28` builds for an example using clang and LTO. -The project's own CI tests against `nightly-2024-09-25` and stable 1.72. It +The project's own CI tests against `nightly-2024-11-12` and stable 1.72. It is prudent to pin the nightly version because that channel can introduce -breaking changes. +breaking changes. There is a significant performance benefit to using +nightly. orjson is tested for amd64 on Linux and cross-compiles for aarch64, arm7, ppc64le, and s390x. It is tested for either aarch64 or amd64 on macOS and diff --git a/build.rs b/build.rs index ad6d5b9c..9acd4c2a 100644 --- a/build.rs +++ b/build.rs @@ -8,12 +8,7 @@ fn main() { println!("cargo:rerun-if-env-changed=CC"); println!("cargo:rerun-if-env-changed=CFLAGS"); println!("cargo:rerun-if-env-changed=LDFLAGS"); - println!("cargo:rerun-if-env-changed=ORJSON_DISABLE_AVX512"); - println!("cargo:rerun-if-env-changed=ORJSON_DISABLE_SIMD"); - println!("cargo:rerun-if-env-changed=ORJSON_DISABLE_YYJSON"); println!("cargo:rerun-if-env-changed=RUSTFLAGS"); - println!("cargo:rustc-check-cfg=cfg(intrinsics)"); - println!("cargo:rustc-check-cfg=cfg(optimize)"); println!("cargo:rustc-check-cfg=cfg(Py_3_10)"); println!("cargo:rustc-check-cfg=cfg(Py_3_11)"); println!("cargo:rustc-check-cfg=cfg(Py_3_12)"); @@ -36,48 +31,46 @@ fn main() { println!("cargo:rustc-cfg=feature=\"optimize\""); } - #[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] - if env::var("ORJSON_DISABLE_SIMD").is_err() { - // auto build unstable SIMD on nightly - if let Some(true) = version_check::supports_feature("portable_simd") { - println!("cargo:rustc-cfg=feature=\"unstable-simd\""); - } - // auto build AVX512 on x86-64-v4 or supporting native targets - #[cfg(all(target_arch = "x86_64", target_feature = "avx512vl"))] - if let Some(true) = version_check::supports_feature("stdarch_x86_avx512") { - if env::var("ORJSON_DISABLE_AVX512").is_err() { - println!("cargo:rustc-cfg=feature=\"avx512\""); - } - } + // auto build unstable SIMD on nightly + #[cfg(any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "powerpc64" + ))] + if let Some(true) = version_check::supports_feature("portable_simd") { + println!("cargo:rustc-cfg=feature=\"unstable-simd\""); } + // auto build AVX512 on x86-64 + #[cfg(target_arch = "x86_64")] + if let Some(true) = version_check::supports_feature("stdarch_x86_avx512") { + println!("cargo:rustc-cfg=feature=\"avx512\""); + } + + #[cfg(target_env = "musl")] + println!("cargo:rustc-cfg=feature=\"unwind\""); + #[cfg(all( target_pointer_width = "64", any(target_arch = "x86_64", target_arch = "aarch64") ))] println!("cargo:rustc-cfg=feature=\"inline_int\""); - if env::var("ORJSON_DISABLE_YYJSON").is_ok() { - if env::var("CARGO_FEATURE_YYJSON").is_ok() { - panic!("ORJSON_DISABLE_YYJSON and --features=yyjson both enabled.") + match cc::Build::new() + .file("include/yyjson/yyjson.c") + .include("include/yyjson") + .define("YYJSON_DISABLE_NON_STANDARD", "1") + .define("YYJSON_DISABLE_UTF8_VALIDATION", "1") + .define("YYJSON_DISABLE_UTILS", "1") + .define("YYJSON_DISABLE_WRITER", "1") + .try_compile("yyjson") + { + Ok(_) => { + println!("cargo:rustc-cfg=feature=\"yyjson\""); } - } else { - match cc::Build::new() - .file("include/yyjson/yyjson.c") - .include("include/yyjson") - .define("YYJSON_DISABLE_NON_STANDARD", "1") - .define("YYJSON_DISABLE_UTF8_VALIDATION", "1") - .define("YYJSON_DISABLE_UTILS", "1") - .define("YYJSON_DISABLE_WRITER", "1") - .try_compile("yyjson") - { - Ok(_) => { - println!("cargo:rustc-cfg=feature=\"yyjson\""); - } - Err(_) => { - if env::var("CARGO_FEATURE_YYJSON").is_ok() { - panic!("yyjson was enabled but the build failed. To build with a different backend do not specify the feature.") - } + Err(_) => { + if env::var("CARGO_FEATURE_YYJSON").is_ok() { + panic!("yyjson was enabled but the build failed. To build with a different backend do not specify the feature.") } } } diff --git a/ci/azure-pipelines.yml b/ci/azure-pipelines.yml index 8382734e..30a6c71f 100644 --- a/ci/azure-pipelines.yml +++ b/ci/azure-pipelines.yml @@ -1,5 +1,5 @@ variables: - toolchain: nightly-2024-09-25 + toolchain: nightly-2024-11-12 jobs: diff --git a/ci/azure-win.yml b/ci/azure-win.yml index 655eab96..a52f7c17 100644 --- a/ci/azure-win.yml +++ b/ci/azure-win.yml @@ -19,7 +19,7 @@ steps: displayName: build dependencies - script: python.exe -m pip install -r test\requirements.txt -r integration\requirements.txt displayName: test dependencies -- script: maturin.exe build --release --features=no-panic,unstable-simd,yyjson --strip --interpreter $(interpreter) --target $(target) +- script: maturin.exe build --release --strip --interpreter $(interpreter) --target $(target) displayName: build env: CFLAGS: "-Os -flto" diff --git a/integration/requirements.txt b/integration/requirements.txt index e5b290de..2f47d3a7 100644 --- a/integration/requirements.txt +++ b/integration/requirements.txt @@ -1,3 +1,3 @@ flask;sys_platform!="win" gunicorn;sys_platform!="win" -httpx==0.24.1;sys_platform!="win" +httpx==0.27.2;sys_platform!="win" diff --git a/requirements.txt b/requirements.txt index 9981cc8c..d211c555 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,4 @@ -r test/requirements.txt maturin mypy==1.13.0 -ruff==0.7.1 +ruff==0.7.3 diff --git a/script/install-fedora b/script/install-fedora index 19a5a455..1862d28a 100755 --- a/script/install-fedora +++ b/script/install-fedora @@ -4,7 +4,7 @@ set -eou pipefail # export PYTHON=python3.11 # export PYTHON_PACKAGE=python3.11 -# export RUST_TOOLCHAIN=nightly-2024-09-25 +# export RUST_TOOLCHAIN=nightly-2024-11-12 # export TARGET=x86_64-unknown-linux-gnu # export VENV=.venv # export CARGO_TARGET_DIR=/tmp/orjson @@ -14,7 +14,7 @@ export CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-target}" rm /etc/yum.repos.d/fedora-cisco-openh264.repo || true -dnf install --setopt=install_weak_deps=false -y rustup clang lld "${PYTHON_PACKAGE}" +dnf install --setopt=install_weak_deps=false -y rustup clang lld "${PYTHON_PACKAGE}" python3-uv rustup-init --default-toolchain "${RUST_TOOLCHAIN}-${TARGET}" --profile minimal --component rust-src -y source "${HOME}/.cargo/env" @@ -24,7 +24,6 @@ cp ci/config.toml .cargo/config.toml cargo fetch --target="${TARGET}" & -curl -LsSf https://astral.sh/uv/install.sh | sh rm -rf "${VENV}" uv venv --python "${PYTHON}" "${VENV}" source "${VENV}/bin/activate"