diff --git a/.github/workflows/artifact.yaml b/.github/workflows/artifact.yaml index 6ff4e00a..08960ed7 100644 --- a/.github/workflows/artifact.yaml +++ b/.github/workflows/artifact.yaml @@ -1,8 +1,9 @@ name: artifact on: push env: - RUST_TOOLCHAIN: "nightly-2024-07-02" + RUST_TOOLCHAIN: "nightly-2024-08-05" CARGO_UNSTABLE_SPARSE_REGISTRY: "true" + UNSAFE_PYO3_BUILD_FREE_THREADED: "1" UNSAFE_PYO3_SKIP_VERSION_CHECK: "1" jobs: @@ -57,16 +58,17 @@ jobs: retention-days: 1 manylinux_2_17_amd64: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: python: [ - { version: '3.12' }, - { version: '3.11' }, - { version: '3.10' }, - { version: '3.9' }, - { version: '3.8' }, + { interpreter: 'python3.13', package: 'python3.13' }, + { interpreter: 'python3.12', package: 'python3.12' }, + { interpreter: 'python3.11', package: 'python3.11' }, + { interpreter: 'python3.10', package: 'python3.10' }, + { interpreter: 'python3.9', package: 'python3.9' }, + { interpreter: 'python3.8', package: 'python3.8' }, ] env: CC: "clang" @@ -83,7 +85,8 @@ jobs: - name: Build environment pre-clone run: | - dnf install -y rustup clang lld python${{ matrix.python.version }} git + dnf copr enable -y @fedora-llvm-team/llvm19 + dnf install -y rustup clang lld ${{ matrix.python.package }} git rustup-init --default-toolchain "${RUST_TOOLCHAIN}-x86_64-unknown-linux-gnu" --profile minimal --component rust-src -y - uses: actions/checkout@v4 @@ -96,7 +99,7 @@ jobs: cp ci/config.toml .cargo/config.toml curl -LsSf https://astral.sh/uv/install.sh | sh - uv venv --python python${{ matrix.python.version }} + uv venv --python ${{ matrix.python.interpreter }} uv pip install --upgrade "maturin>=1,<2" -r test/requirements.txt -r integration/requirements.txt - name: maturin @@ -105,7 +108,7 @@ jobs: maturin build --release --strip \ --features=avx512,no-panic,unstable-simd,yyjson \ --compatibility manylinux_2_17 \ - --interpreter python${{ matrix.python.version }} \ + --interpreter ${{ matrix.python.interpreter }} \ --target=x86_64-unknown-linux-gnu uv pip install target/wheels/orjson*.whl @@ -117,12 +120,6 @@ jobs: - run: source .venv/bin/activate && ./integration/run http - run: source .venv/bin/activate && ./integration/run init - - run: | - source .venv/bin/activate - uv pip install -U --pre "numpy>=2.0.0rc1" - pytest -s test/test_numpy.py - if: matrix.python.version != '3.8' - - name: Store wheels if: "startsWith(github.ref, 'refs/tags/')" uses: actions/upload-artifact@v4 @@ -133,11 +130,12 @@ jobs: retention-days: 1 musllinux_1_2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: python: [ + { version: '3.13' }, { version: '3.12' }, { version: '3.11' }, { version: '3.10' }, @@ -169,7 +167,7 @@ jobs: LDFLAGS: "-Wl,--as-needed" RUSTFLAGS: "-C lto=fat -Z mir-opt-level=4 -Z virtual-function-elimination -Z threads=2 -D warnings -C target-feature=-crt-static" with: - rust-toolchain: nightly-2024-07-02 + rust-toolchain: nightly-2024-08-05 rustup-components: rust-src target: ${{ matrix.platform.target }} manylinux: musllinux_1_2 @@ -208,11 +206,12 @@ jobs: retention-days: 1 manylinux_2_17_non_amd64: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: python: [ + { version: '3.13', abi: 'cp313-cp313g' }, { version: '3.12', abi: 'cp312-cp312' }, { version: '3.11', abi: 'cp311-cp311' }, { version: '3.10', abi: 'cp310-cp310' }, @@ -265,10 +264,11 @@ jobs: RUSTFLAGS: "${{ matrix.target.rustflags }}" with: target: ${{ matrix.target.target }} - rust-toolchain: nightly-2024-07-02 + rust-toolchain: nightly-2024-08-05 rustup-components: rust-src manylinux: auto args: --release --strip --out=dist --features=${{ matrix.target.features }} -i python${{ matrix.python.version }} + before-script-linux: apt-get update && apt-get install -y python${{ matrix.python.version }}-dev python${{ matrix.python.version }}-venv # 3.13 - uses: uraimo/run-on-arch-action@v2 name: Test @@ -306,6 +306,7 @@ jobs: fail-fast: false matrix: python: [ + { version: '3.13', macosx_target: "10.15" }, { version: '3.12', macosx_target: "10.15" }, { version: '3.11', macosx_target: "10.15" }, { version: '3.10', macosx_target: "10.15" }, @@ -325,10 +326,11 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "${{ matrix.python.version }}" + allow-preleases: true - uses: dtolnay/rust-toolchain@master with: - toolchain: "nightly-2024-07-02" + toolchain: "nightly-2024-08-05" targets: "aarch64-apple-darwin, x86_64-apple-darwin" components: "rust-src" @@ -398,7 +400,7 @@ jobs: - uses: dtolnay/rust-toolchain@master with: - toolchain: "nightly-2024-07-02" + toolchain: "nightly-2024-08-05" targets: "aarch64-apple-darwin, x86_64-apple-darwin" components: "rust-src" @@ -443,7 +445,7 @@ jobs: pypi: name: PyPI - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 if: "startsWith(github.ref, 'refs/tags/')" needs: [ sdist, diff --git a/.github/workflows/debug.yaml b/.github/workflows/debug.yaml index 46b0b6e2..f8564850 100644 --- a/.github/workflows/debug.yaml +++ b/.github/workflows/debug.yaml @@ -10,8 +10,8 @@ jobs: profile: [ { rust: "1.72", features: "" }, { rust: "1.72", features: "--features=yyjson" }, - { rust: "nightly-2024-07-02", features: "--features=yyjson,unstable-simd"}, - { rust: "nightly-2024-07-02", features: "--features=avx512,yyjson,unstable-simd"}, + { rust: "nightly-2024-08-05", features: "--features=yyjson,unstable-simd"}, + { rust: "nightly-2024-08-05", features: "--features=avx512,yyjson,unstable-simd"}, ] python: [ { version: '3.13' }, diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index aaa5b77c..df817411 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -2,7 +2,7 @@ name: lint on: push jobs: lint: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/setup-python@v5 with: diff --git a/Cargo.lock b/Cargo.lock index 487daf39..e6cea459 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -40,18 +40,18 @@ checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce" [[package]] name = "castaway" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a17ed5635fc8536268e5d4de1e22e81ac34419e5f052d4d51f4e01dcc263fcc" +checksum = "0abae9be0aaf9ea96a3b1b8b1b55c602ca751eba1b1500220cea4ecbafe7c0d5" dependencies = [ "rustversion", ] [[package]] name = "cc" -version = "1.0.104" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74b6a57f98764a267ff415d50a25e6e166f3831a5071af4995296ea97d210490" +checksum = "26a5c3fd7bfa1ce3897a3a3501d362b2d87b7f2583ebcb4a949ec25911025cbc" [[package]] name = "cfg-if" @@ -70,13 +70,14 @@ dependencies = [ [[package]] name = "compact_str" -version = "0.7.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f86b9c4c00838774a6d902ef931eff7470720c51d90c2e32cfe15dc304737b3f" +checksum = "6050c3a16ddab2e412160b31f2c871015704239bca62f72f6e5f0be631d3f644" dependencies = [ "castaway", "cfg-if", "itoa", + "rustversion", "ryu", "serde", "static_assertions", @@ -134,6 +135,12 @@ version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + [[package]] name = "no-panic" version = "0.1.30" @@ -199,9 +206,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.22.0" +version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab7164b2202753bd33afc7f90a10355a719aa973d1f94502c50d06f3488bc420" +checksum = "1e8730e591b14492a8945cdff32f089250b05f5accecf74aeddf9e8272ce1fa8" dependencies = [ "once_cell", "target-lexicon", @@ -209,9 +216,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.22.0" +version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6424906ca49013c0829c5c1ed405e20e2da2dc78b82d198564880a704e6a7b7" +checksum = "5e97e919d2df92eb88ca80a037969f44e5e70356559654962cbb3316d00300c6" dependencies = [ "libc", "pyo3-build-config", @@ -243,18 +250,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.203" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" +checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.203" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" +checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2", "quote", @@ -263,11 +270,12 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.120" +version = "1.0.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" +checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da" dependencies = [ "itoa", + "memchr", "ryu", "serde", ] @@ -292,9 +300,9 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "syn" -version = "2.0.68" +version = "2.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9" +checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af" dependencies = [ "proc-macro2", "quote", @@ -303,9 +311,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.14" +version = "0.12.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f" +checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "unicode-ident" @@ -325,12 +333,12 @@ dependencies = [ [[package]] name = "version_check" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "xxhash-rust" -version = "0.8.10" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927da81e25be1e1a2901d59b81b37dd2efd1fc9c9345a55007f09bf5a2d3ee03" +checksum = "6a5cbf750400958819fb6178eaa83bee5cd9c29a26a40cc241df8c70fdd46984" diff --git a/Cargo.toml b/Cargo.toml index edfdc6c0..e481d9ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,7 +61,7 @@ associative-cache = { version = "2", default-features = false } beef = { version = "0.5", default-features = false, features = ["impl_serde"] } bytecount = { version = "^0.6.7", default-features = false, features = ["runtime-dispatch-simd"] } chrono = { version = "=0.4.34", default-features = false } -compact_str = { version = "0.7", default-features = false, features = ["serde"] } +compact_str = { version = "0.8", default-features = false, features = ["serde"] } encoding_rs = { version = "0.8", default-features = false } half = { version = "2", default-features = false, features = ["std"] } itoa = { version = "1", default-features = false } diff --git a/README.md b/README.md index ceccea71..ad3297a6 100644 --- a/README.md +++ b/README.md @@ -1205,7 +1205,7 @@ It benefits from also having a C build environment to compile a faster deserialization backend. See this project's `manylinux_2_28` builds for an example using clang and LTO. -The project's own CI tests against `nightly-2024-07-02` and stable 1.72. It +The project's own CI tests against `nightly-2024-08-05` and stable 1.72. It is prudent to pin the nightly version because that channel can introduce breaking changes. diff --git a/ci/azure-pipelines.yml b/ci/azure-pipelines.yml index ee48ec0b..ca4b92da 100644 --- a/ci/azure-pipelines.yml +++ b/ci/azure-pipelines.yml @@ -1,5 +1,5 @@ variables: - toolchain: nightly-2024-07-02 + toolchain: nightly-2024-08-05 jobs: diff --git a/requirements.txt b/requirements.txt index ad527aec..a9d99bf5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ -r test/requirements.txt maturin mypy==1.10.0 -ruff==0.4.8 +ruff==0.5.5 types-python-dateutil types-pytz types-simplejson diff --git a/src/deserialize/json.rs b/src/deserialize/backend/json.rs similarity index 99% rename from src/deserialize/json.rs rename to src/deserialize/backend/json.rs index 9afd51e4..fd531f0d 100644 --- a/src/deserialize/json.rs +++ b/src/deserialize/backend/json.rs @@ -9,7 +9,7 @@ use smallvec::SmallVec; use std::borrow::Cow; use std::fmt; -pub fn deserialize_json( +pub(crate) fn deserialize( data: &'static str, ) -> Result, DeserializeError<'static>> { let mut deserializer = serde_json::Deserializer::from_str(data); diff --git a/src/deserialize/backend/mod.rs b/src/deserialize/backend/mod.rs new file mode 100644 index 00000000..d85ab163 --- /dev/null +++ b/src/deserialize/backend/mod.rs @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) + +#[cfg(not(feature = "yyjson"))] +mod json; + +#[cfg(feature = "yyjson")] +mod yyjson; + +#[cfg(feature = "yyjson")] +pub(crate) use yyjson::deserialize; + +#[cfg(not(feature = "yyjson"))] +pub(crate) use json::deserialize; diff --git a/src/deserialize/yyjson.rs b/src/deserialize/backend/yyjson.rs similarity index 99% rename from src/deserialize/yyjson.rs rename to src/deserialize/backend/yyjson.rs index 41519749..c5538b4b 100644 --- a/src/deserialize/yyjson.rs +++ b/src/deserialize/backend/yyjson.rs @@ -57,7 +57,7 @@ fn unsafe_yyjson_get_next_non_container(val: *mut yyjson_val) -> *mut yyjson_val unsafe { ((val as *mut u8).add(YYJSON_VAL_SIZE)) as *mut yyjson_val } } -pub fn deserialize_yyjson( +pub(crate) fn deserialize( data: &'static str, ) -> Result, DeserializeError<'static>> { let mut err = yyjson_read_err { diff --git a/src/deserialize/deserializer.rs b/src/deserialize/deserializer.rs index 2e7dd8cd..b5a94f3a 100644 --- a/src/deserialize/deserializer.rs +++ b/src/deserialize/deserializer.rs @@ -23,13 +23,5 @@ pub fn deserialize( let buffer_str = unsafe { std::str::from_utf8_unchecked(buffer) }; - #[cfg(feature = "yyjson")] - { - crate::deserialize::yyjson::deserialize_yyjson(buffer_str) - } - - #[cfg(not(feature = "yyjson"))] - { - crate::deserialize::json::deserialize_json(buffer_str) - } + crate::deserialize::backend::deserialize(buffer_str) } diff --git a/src/deserialize/mod.rs b/src/deserialize/mod.rs index 9ec01cfa..0f1484d5 100644 --- a/src/deserialize/mod.rs +++ b/src/deserialize/mod.rs @@ -1,17 +1,12 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) +mod backend; mod cache; mod deserializer; mod error; mod pyobject; mod utf8; -#[cfg(not(feature = "yyjson"))] -mod json; - -#[cfg(feature = "yyjson")] -mod yyjson; - pub use cache::{KeyMap, KEY_MAP}; pub use deserializer::deserialize; pub use error::DeserializeError; diff --git a/src/serialize/per_type/dict.rs b/src/serialize/per_type/dict.rs index 12c5e46a..20b8a064 100644 --- a/src/serialize/per_type/dict.rs +++ b/src/serialize/per_type/dict.rs @@ -381,7 +381,7 @@ fn non_str_uuid(key: *mut pyo3_ffi::PyObject) -> Result Result { let val = ffi!(PyFloat_AS_DOUBLE(key)); if !val.is_finite() { - Ok(CompactString::new_inline("null")) + Ok(CompactString::const_new("null")) } else { Ok(CompactString::from(ryu::Buffer::new().format_finite(val))) } @@ -419,12 +419,12 @@ impl DictNonStrKey { opts: crate::opt::Opt, ) -> Result { match pyobject_to_obtype(key, opts) { - ObType::None => Ok(CompactString::new_inline("null")), + ObType::None => Ok(CompactString::const_new("null")), ObType::Bool => { if unsafe { key == TRUE } { - Ok(CompactString::new_inline("true")) + Ok(CompactString::const_new("true")) } else { - Ok(CompactString::new_inline("false")) + Ok(CompactString::const_new("false")) } } ObType::Int => non_str_int(key), diff --git a/src/serialize/writer/byteswriter.rs b/src/serialize/writer/byteswriter.rs index 086d7202..dd121838 100644 --- a/src/serialize/writer/byteswriter.rs +++ b/src/serialize/writer/byteswriter.rs @@ -67,11 +67,7 @@ impl BytesWriter { fn grow(&mut self, len: usize) { let mut cap = self.cap; while len >= cap { - if len < 262144 { - cap *= 4; - } else { - cap *= 2; - } + cap *= 2; } self.resize(cap); } @@ -164,7 +160,7 @@ impl WriteExt for &mut BytesWriter { #[inline] fn has_capacity(&mut self, len: usize) -> bool { - return self.len + len <= self.cap; + self.len + len <= self.cap } #[inline(always)] diff --git a/test/requirements.txt b/test/requirements.txt index 2efb3e8a..1656dac3 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ arrow faker -numpy;(platform_machine=="x86_64" or (platform_machine=="aarch64" and sys_platform == "linux")) and python_version<"3.12" +numpy;(platform_machine=="x86_64" or (platform_machine=="aarch64" and sys_platform == "linux")) and python_version<"3.13" pendulum;sys_platform=="linux" and platform_machine=="x86_64" and python_version<"3.12" psutil;(sys_platform=="linux" or sys_platform == "macos") and platform_machine=="x86_64" pytest diff --git a/test/test_api.py b/test/test_api.py index 82208588..f4078b5b 100644 --- a/test/test_api.py +++ b/test/test_api.py @@ -49,7 +49,7 @@ def test_loads_type(self): """ loads() invalid type """ - for val in (1, 3.14, [], {}, None): + for val in (1, 3.14, [], {}, None): # type: ignore pytest.raises(orjson.JSONDecodeError, orjson.loads, val) def test_loads_recursion_partial(self):