diff --git a/.github/workflows/linux.yaml b/.github/workflows/artifact.yaml similarity index 88% rename from .github/workflows/linux.yaml rename to .github/workflows/artifact.yaml index 1612e435..2d34e06b 100644 --- a/.github/workflows/linux.yaml +++ b/.github/workflows/artifact.yaml @@ -1,5 +1,7 @@ -name: linux +name: artifact on: push +env: + RUST_TOOLCHAIN: "nightly-2024-03-27" jobs: sdist: @@ -9,18 +11,22 @@ jobs: env: RUST_TOOLCHAIN: "1.72" # MSRV steps: - - run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain $RUST_TOOLCHAIN -y - - run: rustup default $RUST_TOOLCHAIN + - name: rustup stable + run: | + curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain "${RUST_TOOLCHAIN}" -y + rustup default "${RUST_TOOLCHAIN}" - uses: actions/checkout@v4 - run: python3 -m pip install --user --upgrade pip "maturin>=1,<2" wheel - - run: maturin build - - run: cargo fetch - - run: mkdir .cargo - - run: cp ci/sdist.toml .cargo/config.toml - - run: cargo vendor include/cargo --versioned-dirs + - name: Vendor dependencies + run: | + maturin build + cargo fetch + mkdir .cargo + cp ci/sdist.toml .cargo/config.toml + cargo vendor include/cargo --versioned-dirs - run: maturin sdist --out=dist @@ -61,7 +67,6 @@ jobs: { version: '3.8', abi: 'cp38-cp38' }, ] env: - PATH: /github/home/.local/bin:/github/home/.cargo/bin:/opt/python/${{ matrix.python.abi }}/bin:/opt/rh/gcc-toolset-12/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin CC: "clang" CFLAGS: "-Os -fstrict-aliasing -flto=full" LDFLAGS: "-fuse-ld=lld -Wl,--as-needed" @@ -69,29 +74,35 @@ jobs: CARGO_UNSTABLE_SPARSE_REGISTRY: "true" UNSAFE_PYO3_SKIP_VERSION_CHECK: "1" container: - image: quay.io/pypa/manylinux_2_28_x86_64:latest + image: fedora:rawhide options: --user 0 steps: - - run: yum install -y clang lld - - run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2024-03-27 --profile minimal -y - - run: rustup component add rust-src --toolchain nightly-2024-03-27-x86_64-unknown-linux-gnu + - uses: actions/checkout@v4 - - name: build-std + - name: Build environment run: | + dnf install -y rustup clang lld python${{ matrix.python.version }} + + rustup-init --default-toolchain "${RUST_TOOLCHAIN}-x86_64-unknown-linux-gnu" --profile minimal --component rust-src -y + cargo fetch --target=x86_64-unknown-linux-gnu & + + curl -LsSf https://astral.sh/uv/install.sh | sh + uv venv --python python${{ matrix.python.version }} + source .venv/bin/activate + uv pip install --upgrade "maturin>=1,<2" -r test/requirements.txt -r integration/requirements.txt + mkdir .cargo cp ci/config.toml .cargo/config.toml - - run: python3 -m pip install --user --upgrade pip "maturin>=1,<2" wheel - - run: | + - name: maturin + run: | maturin build --release --strip \ - --out=dist \ --features=no-panic,unstable-simd,yyjson \ --compatibility manylinux_2_17 \ --interpreter python${{ matrix.python.version }} \ --target=x86_64-unknown-linux-gnu - - run: python3 -m pip install --user dist/orjson*.whl - - run: python3 -m pip install --user -r test/requirements.txt -r integration/requirements.txt + uv pip install target/wheels/orjson*.whl - run: pytest -s -rxX -v -n 4 test env: @@ -106,7 +117,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: orjson_manylinux_2_17_amd64_${{ matrix.python.version }} - path: dist + path: target/wheels overwrite: true retention-days: 1 @@ -153,11 +164,11 @@ jobs: manylinux: musllinux_1_2 args: --release --strip --out=dist --features=no-panic,unstable-simd,yyjson -i python${{ matrix.python.version }} - - name: Set up QEMU + - name: QEMU if: matrix.platform.arch != 'x86_64' uses: docker/setup-qemu-action@v3 with: - image: tonistiigi/binfmt:qemu-v8.1.4 + image: tonistiigi/binfmt:qemu-v8.1.5 platforms: ${{ matrix.platform.platform }} - name: Test @@ -293,7 +304,7 @@ jobs: merge-multiple: true - uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: "3.12" - run: pip install pip "maturin>=1,<2" - run: ls -1 . - name: deploy wheel diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index d7bbae4a..aaa5b77c 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -6,7 +6,7 @@ jobs: steps: - uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: "3.12" - run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain=stable --profile=default -y - run: pip install -U autoflake isort black ruff mypy types-python-dateutil types-pytz types-simplejson types-ujson diff --git a/Cargo.lock b/Cargo.lock index a396a873..8d0b0a90 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15,6 +15,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "any_all_workaround" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88fea40735f2cc320a5133ce772d39c571bd6c9b0d4c1a326926eecdd5af2e86" +dependencies = [ + "cfg-if", +] + [[package]] name = "arrayvec" version = "0.7.4" @@ -62,9 +71,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.90" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5" +checksum = "2678b2e3449475e95b0aa6f9b506a28e61b3dc8996592b983695e8ebb58a8b41" [[package]] name = "cfg-if" @@ -123,18 +132,19 @@ checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" [[package]] name = "encoding_rs" -version = "0.8.33" +version = "0.8.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" dependencies = [ + "any_all_workaround", "cfg-if", ] [[package]] name = "getrandom" -version = "0.2.12" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" +checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" dependencies = [ "cfg-if", "libc", @@ -143,9 +153,9 @@ dependencies = [ [[package]] name = "half" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5eceaaeec696539ddaf7b333340f1af35a5aa87ae3e4f3ead0532f72affab2e" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" dependencies = [ "cfg-if", "crunchy", @@ -236,9 +246,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.21.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5eb0b6ecba38961f6f4bd6cd5906dfab3cd426ff37b2eed5771006aa31656f1" +checksum = "650dca34d463b6cdbdb02b1d71bfd6eb6b6816afc708faebb3bac1380ff4aef7" dependencies = [ "once_cell", "target-lexicon", @@ -246,9 +256,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.21.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba8a6e48a29b5d22e4fdaf132d8ba8d3203ee9f06362d48f244346902a594ec3" +checksum = "09a7da8fc04a8a2084909b59f29e1b8474decac98b951d77b80b26dc45f046ad" dependencies = [ "libc", "pyo3-build-config", @@ -256,18 +266,18 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.35" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] [[package]] name = "rustversion" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" +checksum = "80af6f9131f277a45a3fba6ce8e2258037bb0477a67e610d3c1fe046ab31de47" [[package]] name = "ryu" @@ -329,9 +339,9 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "syn" -version = "2.0.55" +version = "2.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "002a1b3dbf967edfafc32655d0f377ab0bb7b994aa1d32c8cc7e9b8bf3ebb8f0" +checksum = "44cfb93f38070beee36b3fef7d4f5a16f27751d94b187b666a5cc5e9b0d30687" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 5095483d..8d742eb8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,7 +33,9 @@ crate-type = ["cdylib"] default = [] # Use SIMD intrinsics. This requires Rust on the nightly channel. -unstable-simd = [] +unstable-simd = [ + "encoding_rs/simd-accel", +] no-panic = [ "itoa/no-panic", @@ -52,7 +54,7 @@ beef = { version = "0.5", default_features = false, features = ["impl_serde"] } bytecount = { version = "^0.6.7", default_features = false, features = ["runtime-dispatch-simd"] } chrono = { version = "=0.4.34", default_features = false } compact_str = { version = "0.7", default_features = false, features = ["serde"] } -encoding_rs = { version = "0.8", default_features = false } +encoding_rs = { version = "^0.8.34", default_features = false } half = { version = "2", default_features = false, features = ["std"] } itoa = { version = "1", default_features = false } itoap = { version = "1", features = ["std", "simd"] } diff --git a/bench/requirements.txt b/bench/requirements.txt index 806f5b81..40d524ca 100644 --- a/bench/requirements.txt +++ b/bench/requirements.txt @@ -1,9 +1,9 @@ memory-profiler -pandas +pandas; python_version<"3.13" pytest-benchmark pytest-random-order python-rapidjson -seaborn +seaborn; python_version<"3.13" simplejson tabulate ujson diff --git a/script/develop b/script/develop index 3a4042e4..a4207166 100755 --- a/script/develop +++ b/script/develop @@ -9,10 +9,10 @@ export LD="${LD:-lld}" echo "CC: ${CC}, LD: ${LD}, LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}" -export CFLAGS="-Os -fstrict-aliasing -flto=full" +export CFLAGS="-Os -fstrict-aliasing -fno-plt -flto=full" export LDFLAGS="-fuse-ld=${LD} -Wl,--as-needed" export RUSTFLAGS="-C linker=${CC} -C lto=fat -C link-arg=-fuse-ld=${LD} -Z mir-opt-level=4 -Z virtual-function-elimination -Z threads=8" maturin build "$@" -pip install --force target/wheels/*.whl +uv pip install target/wheels/*.whl diff --git a/script/graph b/script/graph index afd45ce9..72bcdd95 100755 --- a/script/graph +++ b/script/graph @@ -78,7 +78,10 @@ def tab(obj): if isinstance(each[1], float) else None ) - each[1] = "%.1f" % each[1] if isinstance(each[1], float) else None + if group.startswith("github"): + each[1] = "%.2f" % each[1] if isinstance(each[1], float) else None + else: + each[1] = "%.1f" % each[1] if isinstance(each[1], float) else None buf.write(tabulate(table, headers, tablefmt="github") + "\n") diff --git a/src/deserialize/cache.rs b/src/deserialize/cache.rs index 169712ad..610377d3 100644 --- a/src/deserialize/cache.rs +++ b/src/deserialize/cache.rs @@ -40,11 +40,7 @@ pub static mut KEY_MAP: OnceCell = OnceCell::new(); pub fn cache_hash(key: &[u8]) -> u64 { // try to omit code for >64 path in ahash - debug_assert!(key.len() <= 64); - #[cfg(feature = "intrinsics")] - unsafe { - core::intrinsics::assume(key.len() <= 64); - }; + assume!(key.len() <= 64); let mut hasher = ahash::AHasher::default(); hasher.write(key); hasher.finish() diff --git a/src/deserialize/yyjson.rs b/src/deserialize/yyjson.rs index a05fa37d..090da550 100644 --- a/src/deserialize/yyjson.rs +++ b/src/deserialize/yyjson.rs @@ -146,7 +146,7 @@ fn parse_yy_array(elem: *mut yyjson_val) -> NonNull { return nonnull!(list); } let mut cur = unsafe_yyjson_get_first(elem); - for idx in 0..=len - 1 { + for idx in 0..len { let next = unsafe_yyjson_get_next(cur); let val = parse_node(cur).as_ptr(); ffi!(PyList_SET_ITEM(list, idx as isize, val)); @@ -165,7 +165,7 @@ fn parse_yy_object(elem: *mut yyjson_val) -> NonNull { } let mut key = unsafe_yyjson_get_first(elem); let dict = ffi!(_PyDict_NewPresized(len as isize)); - for _ in 0..=len - 1 { + for _ in 0..len { let val = key.add(1); let key_str = str_from_slice!((*key).uni.str_ as *const u8, unsafe_yyjson_get_len(key)); let pykey = get_unicode_key(key_str); diff --git a/src/serialize/per_type/dataclass.rs b/src/serialize/per_type/dataclass.rs index 5acf0937..f3c434b9 100644 --- a/src/serialize/per_type/dataclass.rs +++ b/src/serialize/per_type/dataclass.rs @@ -98,7 +98,7 @@ impl Serialize for DataclassFastSerializer { let mut pos = 0; pydict_next!(self.ptr, &mut pos, &mut next_key, &mut next_value); - for _ in 0..=ffi!(Py_SIZE(self.ptr)) as usize - 1 { + for _ in 0..ffi!(Py_SIZE(self.ptr)) as usize { let key = next_key; let value = next_value; @@ -163,7 +163,7 @@ impl Serialize for DataclassFallbackSerializer { let mut pos = 0; pydict_next!(fields, &mut pos, &mut next_key, &mut next_value); - for _ in 0..=ffi!(Py_SIZE(fields)) as usize - 1 { + for _ in 0..ffi!(Py_SIZE(fields)) as usize { let attr = next_key; let field = next_value; diff --git a/src/serialize/per_type/dict.rs b/src/serialize/per_type/dict.rs index 0fb6a19c..f79cbf60 100644 --- a/src/serialize/per_type/dict.rs +++ b/src/serialize/per_type/dict.rs @@ -111,7 +111,7 @@ impl Serialize for Dict { let mut pos = 0; pydict_next!(self.ptr, &mut pos, &mut next_key, &mut next_value); - for _ in 0..=ffi!(Py_SIZE(self.ptr)) as usize - 1 { + for _ in 0..ffi!(Py_SIZE(self.ptr)) as usize { let key = next_key; let value = next_value; @@ -210,7 +210,7 @@ impl Serialize for DictSortedKey { let mut pos = 0; pydict_next!(self.ptr, &mut pos, &mut next_key, &mut next_value); - for _ in 0..=len as usize - 1 { + for _ in 0..len as usize { let key = next_key; let value = next_value; @@ -377,7 +377,7 @@ impl Serialize for DictNonStrKey { let mut pos = 0; pydict_next!(self.ptr, &mut pos, &mut next_key, &mut next_value); - for _ in 0..=len - 1 { + for _ in 0..len { let key = next_key; let value = next_value; diff --git a/src/serialize/per_type/list.rs b/src/serialize/per_type/list.rs index 1599a944..ae244b61 100644 --- a/src/serialize/per_type/list.rs +++ b/src/serialize/per_type/list.rs @@ -88,7 +88,7 @@ impl Serialize for ListTupleSerializer { } debug_assert!(self.len >= 1); let mut seq = serializer.serialize_seq(None).unwrap(); - for idx in 0..=self.len - 1 { + for idx in 0..self.len { let value = unsafe { *((self.data_ptr).add(idx)) }; let value_ob_type = ob_type!(value); if is_class_by_type!(value_ob_type, STR_TYPE) { diff --git a/src/serialize/per_type/numpy.rs b/src/serialize/per_type/numpy.rs index 2f86d560..dede7996 100644 --- a/src/serialize/per_type/numpy.rs +++ b/src/serialize/per_type/numpy.rs @@ -238,7 +238,7 @@ impl NumpyArray { #[cfg_attr(feature = "optimize", optimize(size))] fn build(&mut self) { if self.depth < self.dimensions() - 1 { - for i in 0..=self.shape()[self.depth] - 1 { + for i in 0..self.shape()[self.depth] { let mut position: Vec = self.position.to_vec(); position[self.depth] = i; let num_children: usize = if self.depth < self.dimensions() - 2 { diff --git a/src/util.rs b/src/util.rs index c7bbe721..787aa8aa 100644 --- a/src/util.rs +++ b/src/util.rs @@ -257,3 +257,13 @@ macro_rules! reserve_minimum { $writer.reserve(64); }; } + +macro_rules! assume { + ($expr:expr) => { + debug_assert!($expr); + #[cfg(feature = "intrinsics")] + unsafe { + core::intrinsics::assume($expr); + }; + }; +}