From 834f7926870d69a582c7d8a4402a8edb4482883c Mon Sep 17 00:00:00 2001 From: ijl Date: Wed, 13 Dec 2023 23:43:30 +0000 Subject: [PATCH] cargo update, build and tooling misc --- .github/workflows/debug.yaml | 2 +- .github/workflows/linux.yaml | 34 ++++++++++--------- Cargo.lock | 52 ++++++++++++++--------------- README.md | 6 ++-- build.rs | 4 --- ci/azure-macos.yml | 5 +-- ci/azure-pipelines.yml | 2 +- ci/config.toml | 7 ---- integration/wsgi.py | 4 +-- pyproject.toml | 13 ++++---- requirements.txt | 3 -- script/develop | 4 +-- script/lint | 4 +-- script/pynumpy | 4 +-- src/deserialize/cache.rs | 2 +- src/deserialize/deserializer.rs | 2 +- src/deserialize/json.rs | 2 +- src/deserialize/pyobject.rs | 4 +-- src/deserialize/utf8.rs | 4 +-- src/deserialize/yyjson.rs | 7 ++-- src/ffi/list.rs | 44 ------------------------ src/ffi/mod.rs | 2 -- src/lib.rs | 8 ++--- src/serialize/json.rs | 8 +---- src/serialize/per_type/dataclass.rs | 6 ++-- src/serialize/per_type/datetime.rs | 6 ++-- src/serialize/per_type/default.rs | 4 +-- src/serialize/per_type/dict.rs | 4 +-- src/serialize/per_type/fragment.rs | 2 +- src/serialize/per_type/int.rs | 2 +- src/serialize/per_type/list.rs | 11 +++--- src/serialize/per_type/none.rs | 4 +-- src/serialize/per_type/numpy.rs | 2 +- src/serialize/per_type/pyenum.rs | 2 +- src/serialize/per_type/tuple.rs | 4 +-- src/serialize/per_type/unicode.rs | 4 +-- src/serialize/per_type/uuid.rs | 3 +- src/typeref.rs | 34 +++++++++---------- 38 files changed, 126 insertions(+), 189 deletions(-) delete mode 100644 src/ffi/list.rs diff --git a/.github/workflows/debug.yaml b/.github/workflows/debug.yaml index 06b6d8fe..19fc9dfa 100644 --- a/.github/workflows/debug.yaml +++ b/.github/workflows/debug.yaml @@ -9,7 +9,7 @@ jobs: matrix: rust: [ { version: "1.65" }, # MSRV - { version: "nightly-2023-10-10" }, + { version: "nightly-2023-11-15" }, ] python: [ { version: '3.12', abi: 'cp312-cp312' }, diff --git a/.github/workflows/linux.yaml b/.github/workflows/linux.yaml index d60d0c1e..71caa3c6 100644 --- a/.github/workflows/linux.yaml +++ b/.github/workflows/linux.yaml @@ -62,17 +62,17 @@ jobs: env: PATH: /github/home/.local/bin:/github/home/.cargo/bin:/opt/python/${{ matrix.python.abi }}/bin:/opt/rh/gcc-toolset-12/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin CC: "clang" - CFLAGS: "-O2 -fno-plt -flto=thin" - LDFLAGS: "-O2 -flto=thin -fuse-ld=lld -Wl,--as-needed" - RUSTFLAGS: "-C linker=clang -C link-arg=-fuse-ld=lld" + CFLAGS: "-O2 -fno-plt -foffload-lto=thin -fstrict-aliasing" + LDFLAGS: "-O2 -fno-plt -foffload-lto=thin -fstrict-aliasing -fuse-ld=lld -Wl,--as-needed" + RUSTFLAGS: "-C linker=clang -C linker-plugin-lto -C lto=thin -C link-arg=-fuse-ld=lld -Z mir-opt-level=4 -D warnings" CARGO_UNSTABLE_SPARSE_REGISTRY: "true" container: image: quay.io/pypa/manylinux_2_28_x86_64:latest options: --user 0 steps: - run: yum install -y clang lld - - run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2023-10-10 --profile minimal -y - - run: rustup component add rust-src --toolchain nightly-2023-10-10-x86_64-unknown-linux-gnu + - run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2023-11-15 --profile minimal -y + - run: rustup component add rust-src --toolchain nightly-2023-11-15-x86_64-unknown-linux-gnu - uses: actions/checkout@v4 - name: build-std @@ -140,10 +140,10 @@ jobs: CC: "gcc" CFLAGS: "-O2 -fno-plt" LDFLAGS: "-O2 -flto -Wl,--as-needed" - RUSTFLAGS: "-C target-feature=-crt-static" + RUSTFLAGS: "-C target-feature=-crt-static -Z mir-opt-level=4" CARGO_UNSTABLE_SPARSE_REGISTRY: "true" with: - rust-toolchain: nightly-2023-10-10 + rust-toolchain: nightly-2023-11-15 rustup-components: rust-src target: ${{ matrix.platform.target }} manylinux: musllinux_1_1 @@ -153,7 +153,7 @@ jobs: if: matrix.platform.arch != 'x86_64' uses: docker/setup-qemu-action@v3 with: - image: tonistiigi/binfmt:qemu-v7.0.0 + image: tonistiigi/binfmt:qemu-v8.0.4 platforms: ${{ matrix.platform.platform }} - name: Test @@ -195,27 +195,30 @@ jobs: { arch: 'aarch64', target: 'aarch64-unknown-linux-gnu', - cflags: '-O2', + cflags: '-O2 -flto', + rustflags: '-Z mir-opt-level=4 -D warnings', }, { arch: 'armv7', target: 'armv7-unknown-linux-gnueabihf', - cflags: '-Os -fstrict-aliasing', + cflags: '-Os -flto -fstrict-aliasing', + rustflags: '-C opt-level=s -Z mir-opt-level=4 -D warnings', }, { arch: 'ppc64le', target: 'powerpc64le-unknown-linux-gnu', - cflags: '-O2', + cflags: '-O2 -flto', + rustflags: '-Z mir-opt-level=4 -D warnings', }, { arch: 's390x', target: 's390x-unknown-linux-gnu', - cflags: '-O2 -march=z10', + cflags: '-O2 -flto -march=z10 -mtune=generic', + rustflags: '-Z mir-opt-level=4 -C target-cpu=z10 -Z tune-cpu=generic-D warnings', }, ] steps: - uses: actions/checkout@v4 - - name: build-std run: | mkdir .cargo @@ -226,11 +229,12 @@ jobs: env: PYO3_CROSS_LIB_DIR: "/opt/python/${{ matrix.python.abi }}" CFLAGS: "${{ matrix.target.cflags }}" - LDFLAGS: "${{ matrix.target.cflags }} -flto -Wl,--as-needed" + LDFLAGS: "${{ matrix.target.cflags }} -Wl,--as-needed" CARGO_UNSTABLE_SPARSE_REGISTRY: "true" + RUSTFLAGS: "${{ matrix.target.rustflags }}" with: target: ${{ matrix.target.target }} - rust-toolchain: nightly-2023-10-10 + rust-toolchain: nightly-2023-11-15 rustup-components: rust-src manylinux: auto args: --release --strip --out=dist --features=no-panic,yyjson -i python${{ matrix.python.version }} diff --git a/Cargo.lock b/Cargo.lock index 557acf5a..8be540f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -112,9 +112,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" dependencies = [ "no-panic", ] @@ -127,9 +127,9 @@ checksum = "9028f49264629065d057f340a86acb84867925865f73bbf8d47b4d149a7e88b8" [[package]] name = "libc" -version = "0.2.149" +version = "0.2.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" +checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" [[package]] name = "libm" @@ -139,9 +139,9 @@ checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" [[package]] name = "no-panic" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71a6d126424f5ce0bb4587ff4561421d44aeede520541cc66f1bb912506ae46a" +checksum = "bf98081e1af7145ec0f77baf9ec845903ccb6ae4a8b1e881265d5487f37f90a2" dependencies = [ "proc-macro2", "quote", @@ -160,9 +160,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "orjson" @@ -202,9 +202,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.69" +version = "1.0.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" +checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" dependencies = [ "unicode-ident", ] @@ -246,27 +246,27 @@ checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "ryu" -version = "1.0.15" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" dependencies = [ "no-panic", ] [[package]] name = "serde" -version = "1.0.190" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91d3c334ca1ee894a2c6f6ad698fe8c435b76d504b13d436f0685d648d6d96f7" +checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.190" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67c5609f394e5c2bd7fc51efda478004ea80ef42fee983d5c67a65e34f32c0e3" +checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ "proc-macro2", "quote", @@ -275,9 +275,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.107" +version = "1.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" dependencies = [ "itoa", "ryu", @@ -292,9 +292,9 @@ checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" [[package]] name = "smallvec" -version = "1.11.1" +version = "1.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" +checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" [[package]] name = "static_assertions" @@ -304,9 +304,9 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "syn" -version = "2.0.38" +version = "2.0.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" +checksum = "44c8b28c477cc3bf0e7966561e3460130e1255f7a1cf71931075f1c5e7a7e269" dependencies = [ "proc-macro2", "quote", @@ -333,18 +333,18 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "zerocopy" -version = "0.7.15" +version = "0.7.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81ba595b9f2772fbee2312de30eeb80ec773b4cb2f1e8098db024afadda6c06f" +checksum = "306dca4455518f1f31635ec308b6b3e4eb1b11758cefafc782827d0aa7acb5c7" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.15" +version = "0.7.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "772666c41fb6dceaf520b564b962d738a8e1a83b41bd48945f50837aed78bb1d" +checksum = "be912bf68235a88fbefd1b73415cb218405958d1655b2ece9035a19920bdf6ba" dependencies = [ "proc-macro2", "quote", diff --git a/README.md b/README.md index 532ca6d5..693b7450 100644 --- a/README.md +++ b/README.md @@ -985,9 +985,9 @@ library handles a combined 342 JSON fixtures from the | Library | Invalid JSON documents not rejected | Valid JSON documents not deserialized | |------------|---------------------------------------|-----------------------------------------| | orjson | 0 | 0 | -| ujson | 38 | 0 | +| ujson | 31 | 0 | | rapidjson | 6 | 0 | -| simplejson | 13 | 0 | +| simplejson | 10 | 0 | | json | 17 | 0 | This shows that all libraries deserialize valid JSON but only orjson @@ -1192,7 +1192,7 @@ It benefits from also having a C build environment to compile a faster deserialization backend. See this project's `manylinux_2_28` builds for an example using clang and LTO. -The project's own CI tests against `nightly-2023-10-10` and stable 1.65. It +The project's own CI tests against `nightly-2023-11-15` and stable 1.65. It is prudent to pin the nightly version because that channel can introduce breaking changes. diff --git a/build.rs b/build.rs index 86894e0a..a8a1515b 100644 --- a/build.rs +++ b/build.rs @@ -26,10 +26,6 @@ fn main() { println!("cargo:rustc-cfg=feature=\"strict_provenance\""); } - if let Some(true) = version_check::supports_feature("trusted_len") { - println!("cargo:rustc-cfg=feature=\"trusted_len\""); - } - if env::var("ORJSON_DISABLE_YYJSON").is_ok() { if env::var("CARGO_FEATURE_YYJSON").is_ok() { panic!("ORJSON_DISABLE_YYJSON and --features=yyjson both enabled.") diff --git a/ci/azure-macos.yml b/ci/azure-macos.yml index 7ada3141..df77140f 100644 --- a/ci/azure-macos.yml +++ b/ci/azure-macos.yml @@ -26,10 +26,11 @@ steps: maturin build --release --strip --features=no-panic,yyjson --interpreter $(interpreter) --target=universal2-apple-darwin env: CC: "clang" - LDFLAGS: "-O2 -flto=thin -fuse-ld=lld -Wl,--as-needed" - CFLAGS: "-O2 -fno-plt -flto=thin" + CFLAGS: "-O2 -fno-plt -foffload-lto=thin -fstrict-aliasing" + LDFLAGS: "-O2 -fno-plt -foffload-lto=thin -fstrict-aliasing -fuse-ld=lld -Wl,--as-needed" CFLAGS_x86_64_apple_darwin: "-O2 -fno-plt -flto=thin -march=x86-64-v2 -mtune=generic" CFLAGS_aarch64_apple_darwin: "-O2 -fno-plt -flto=thin -mcpu=apple-m1 -mtune=generic" + RUSTFLAGS: "-C linker=clang -C linker-plugin-lto -C lto=thin -C link-arg=-fuse-ld=lld -Z mir-opt-level=4 -D warnings" CARGO_UNSTABLE_SPARSE_REGISTRY: "true" displayName: build universal2 diff --git a/ci/azure-pipelines.yml b/ci/azure-pipelines.yml index c78da7a4..285aa085 100644 --- a/ci/azure-pipelines.yml +++ b/ci/azure-pipelines.yml @@ -1,5 +1,5 @@ variables: - toolchain: nightly-2023-10-10 + toolchain: nightly-2023-11-15 jobs: diff --git a/ci/config.toml b/ci/config.toml index a386c2ff..0823eb5a 100644 --- a/ci/config.toml +++ b/ci/config.toml @@ -9,10 +9,3 @@ rustflags = ["-C", "target-cpu=x86-64-v2", "-Z", "tune-cpu=generic"] [target.aarch64-apple-darwin] linker = "clang" rustflags = ["-C", "target-cpu=apple-m1"] - -[target.armv7-unknown-linux-gnueabihf] -rustflags = ["-C", "opt-level=s"] - -[target.s390x-unknown-linux-gnu] -linker = "clang" -rustflags = ["-C", "target-cpu=z10", "-Z", "tune-cpu=generic"] diff --git a/integration/wsgi.py b/integration/wsgi.py index 7e20e9b0..b5d4d4c4 100644 --- a/integration/wsgi.py +++ b/integration/wsgi.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) -from datetime import datetime +from datetime import datetime, timezone from uuid import uuid4 from flask import Flask @@ -9,7 +9,7 @@ app = Flask(__name__) -NOW = datetime.utcnow() +NOW = datetime.now(timezone.utc) @app.route("/") diff --git a/pyproject.toml b/pyproject.toml index 276703ed..58007e5f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,20 +39,21 @@ include = [ { format = "sdist", path = "include/**/*" }, ] - -[tool.black] -line-length = 88 -target-version = ["py38"] -include = ".pyi?$" - [tool.ruff] line-length = 88 target-version = "py38" +select = [ + "I", +] + ignore = [ "E501", # line too long "F601", # Dictionary key literal ... repeated ] +[tool.ruff.lint.isort] +known-first-party = ["orjson"] + [tool.mypy] python_version = "3.8" diff --git a/requirements.txt b/requirements.txt index c8d3e01a..9ef0c1fd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,6 @@ -r bench/requirements.txt -r integration/requirements.txt -r test/requirements.txt -autoflake -black -isort maturin mypy ruff diff --git a/script/develop b/script/develop index 6bad5db7..2ec26d1f 100755 --- a/script/develop +++ b/script/develop @@ -3,9 +3,9 @@ rm -f target/wheels/* export CC="clang" -export CFLAGS="-O2 -fno-plt -flto=thin" +export CFLAGS="-O2 -fno-plt -foffload-lto=thin -fstrict-aliasing" export LDFLAGS="${CFLAGS} -fuse-ld=lld -Wl,--as-needed" -export RUSTFLAGS="-C linker=clang -C link-arg=-fuse-ld=lld" +export RUSTFLAGS="-C linker=clang -C linker-plugin-lto -C lto=thin -C link-arg=-fuse-ld=lld -Z mir-opt-level=4" maturin build "$@" diff --git a/script/lint b/script/lint index c08a928f..15eed6ec 100755 --- a/script/lint +++ b/script/lint @@ -6,8 +6,6 @@ to_lint="./bench/*.py ./pysrc/orjson/__init__.pyi ./test/*.py script/pydataclass script/pysort script/pynumpy script/pynonstr script/pycorrectness script/graph integration/init integration/wsgi.py integration/typestubs.py integration/thread" -autoflake --in-place --recursive --remove-all-unused-imports --ignore-init-module-imports . -isort ${to_lint} ruff ${to_lint} --fix -black ${to_lint} +ruff format ${to_lint} mypy --ignore-missing-imports --check-untyped-defs ./bench/*.py ./pysrc/orjson/__init__.pyi ./test/*.py diff --git a/script/pynumpy b/script/pynumpy index df7a1194..102a395d 100755 --- a/script/pynumpy +++ b/script/pynumpy @@ -39,9 +39,7 @@ elif kind == "int32": elif kind == "uint8": array = numpy.random.randint(((2**8) - 1), size=(100000, 100), dtype=numpy.uint8) elif kind == "uint16": - array = numpy.random.randint( - ((2**16) - 1), size=(100000, 100), dtype=numpy.uint16 - ) + array = numpy.random.randint(((2**16) - 1), size=(100000, 100), dtype=numpy.uint16) else: print("usage: pynumpy (bool|int16|int32|float64|int8|uint8|uint16)") sys.exit(1) diff --git a/src/deserialize/cache.rs b/src/deserialize/cache.rs index 62142f77..87d9a888 100644 --- a/src/deserialize/cache.rs +++ b/src/deserialize/cache.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) -use crate::typeref::*; +use crate::typeref::HASH_BUILDER; use associative_cache::replacement::RoundRobinReplacement; use associative_cache::*; use once_cell::unsync::OnceCell; diff --git a/src/deserialize/deserializer.rs b/src/deserialize/deserializer.rs index 0675cce0..cd0d09bc 100644 --- a/src/deserialize/deserializer.rs +++ b/src/deserialize/deserializer.rs @@ -2,7 +2,7 @@ use crate::deserialize::utf8::read_input_to_buf; use crate::deserialize::DeserializeError; -use crate::typeref::*; +use crate::typeref::EMPTY_UNICODE; use std::ptr::NonNull; pub fn deserialize( diff --git a/src/deserialize/json.rs b/src/deserialize/json.rs index 9a6d7561..a9d36424 100644 --- a/src/deserialize/json.rs +++ b/src/deserialize/json.rs @@ -2,7 +2,7 @@ use crate::deserialize::pyobject::*; use crate::deserialize::DeserializeError; -use crate::str::*; +use crate::str::unicode_from_str; use serde::de::{self, DeserializeSeed, Deserializer, MapAccess, SeqAccess, Visitor}; use smallvec::SmallVec; use std::borrow::Cow; diff --git a/src/deserialize/pyobject.rs b/src/deserialize/pyobject.rs index 49766b14..584d3d2d 100644 --- a/src/deserialize/pyobject.rs +++ b/src/deserialize/pyobject.rs @@ -1,8 +1,8 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) use crate::deserialize::cache::*; -use crate::str::*; -use crate::typeref::*; +use crate::str::{hash_str, unicode_from_str}; +use crate::typeref::{EMPTY_UNICODE, FALSE, NONE, TRUE}; use std::ptr::NonNull; pub fn get_unicode_key(key_str: &str) -> *mut pyo3_ffi::PyObject { diff --git a/src/deserialize/utf8.rs b/src/deserialize/utf8.rs index ddb03563..d1cbb474 100644 --- a/src/deserialize/utf8.rs +++ b/src/deserialize/utf8.rs @@ -2,8 +2,8 @@ use crate::deserialize::DeserializeError; use crate::ffi::*; -use crate::str::*; -use crate::typeref::*; +use crate::str::unicode_to_str; +use crate::typeref::{BYTEARRAY_TYPE, BYTES_TYPE, MEMORYVIEW_TYPE, STR_TYPE}; use crate::util::INVALID_STR; use std::borrow::Cow; use std::os::raw::c_char; diff --git a/src/deserialize/yyjson.rs b/src/deserialize/yyjson.rs index c8c5b774..33ef4515 100644 --- a/src/deserialize/yyjson.rs +++ b/src/deserialize/yyjson.rs @@ -3,8 +3,8 @@ use crate::deserialize::pyobject::*; use crate::deserialize::DeserializeError; use crate::ffi::yyjson::*; -use crate::str::*; -use crate::typeref::*; +use crate::str::unicode_from_str; +use crate::typeref::{yyjson_init, YYJSON_ALLOC, YYJSON_BUFFER_SIZE}; use std::borrow::Cow; use std::os::raw::c_char; use std::ptr::{null, null_mut, NonNull}; @@ -164,10 +164,7 @@ fn parse_yy_object(elem: *mut yyjson_val) -> NonNull { return nonnull!(ffi!(PyDict_New())); } let mut key = unsafe_yyjson_get_first(elem); - #[cfg(not(Py_3_13))] let dict = ffi!(_PyDict_NewPresized(len as isize)); - #[cfg(Py_3_13)] - let dict = ffi!(PyDict_New()); for _ in 0..=len - 1 { let val = key.add(1); let key_str = str_from_slice!((*key).uni.str_ as *const u8, unsafe_yyjson_get_len(key)); diff --git a/src/ffi/list.rs b/src/ffi/list.rs deleted file mode 100644 index 44b386c4..00000000 --- a/src/ffi/list.rs +++ /dev/null @@ -1,44 +0,0 @@ -// SPDX-License-Identifier: (Apache-2.0 OR MIT) - -use std::ptr::NonNull; - -pub struct PyListIter { - obj: *mut pyo3_ffi::PyListObject, - len: usize, - pos: usize, -} - -impl PyListIter { - #[inline] - pub fn from_pyobject(obj: *mut pyo3_ffi::PyObject) -> Self { - unsafe { - PyListIter { - obj: obj as *mut pyo3_ffi::PyListObject, - len: ffi!(Py_SIZE(obj)) as usize, - pos: 0, - } - } - } -} - -impl Iterator for PyListIter { - type Item = NonNull; - - #[inline(always)] - fn next(&mut self) -> Option { - if self.pos == self.len { - None - } else { - let elem = unsafe { *((*self.obj).ob_item).add(self.pos) }; - self.pos += 1; - Some(nonnull!(elem)) - } - } - - fn size_hint(&self) -> (usize, Option) { - (self.len, Some(self.len)) - } -} - -#[cfg(feature = "trusted_len")] -unsafe impl std::iter::TrustedLen for PyListIter {} diff --git a/src/ffi/mod.rs b/src/ffi/mod.rs index afc7ef50..ba6dedcd 100644 --- a/src/ffi/mod.rs +++ b/src/ffi/mod.rs @@ -3,7 +3,6 @@ mod buffer; mod bytes; mod fragment; -mod list; mod long; #[cfg(feature = "yyjson")] pub mod yyjson; @@ -11,5 +10,4 @@ pub mod yyjson; pub use buffer::*; pub use bytes::*; pub use fragment::{orjson_fragmenttype_new, Fragment}; -pub use list::PyListIter; pub use long::{pylong_is_unsigned, pylong_is_zero}; diff --git a/src/lib.rs b/src/lib.rs index 63b9a59a..a4b157f4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,7 +4,7 @@ #![cfg_attr(feature = "optimize", feature(optimize_attribute))] #![cfg_attr(feature = "strict_provenance", feature(strict_provenance))] #![cfg_attr(feature = "strict_provenance", warn(fuzzy_provenance_casts))] -#![cfg_attr(feature = "trusted_len", feature(trusted_len))] +#![allow(internal_features)] // core_intrinsics #![allow(unused_unsafe)] #![allow(non_camel_case_types)] #![allow(clippy::explicit_auto_deref)] @@ -232,7 +232,7 @@ fn raise_dumps_exception_fixed(msg: &str) -> *mut PyObject { #[inline(never)] #[cfg_attr(feature = "optimize", optimize(size))] #[cfg(Py_3_12)] -fn raise_dumps_exception_dynamic(err: &String) -> *mut PyObject { +fn raise_dumps_exception_dynamic(err: &str) -> *mut PyObject { unsafe { let cause_exc: *mut PyObject = PyErr_GetRaisedException(); @@ -255,7 +255,7 @@ fn raise_dumps_exception_dynamic(err: &String) -> *mut PyObject { #[inline(never)] #[cfg_attr(feature = "optimize", optimize(size))] #[cfg(not(Py_3_12))] -fn raise_dumps_exception_dynamic(err: &String) -> *mut PyObject { +fn raise_dumps_exception_dynamic(err: &str) -> *mut PyObject { unsafe { let mut cause_tp: *mut PyObject = null_mut(); let mut cause_val: *mut PyObject = null_mut(); @@ -355,6 +355,6 @@ pub unsafe extern "C" fn dumps( match crate::serialize::serialize(*args, default, optsbits as opt::Opt) { Ok(val) => val.as_ptr(), - Err(err) => raise_dumps_exception_dynamic(&err), + Err(err) => raise_dumps_exception_dynamic(err.as_str()), } } diff --git a/src/serialize/json.rs b/src/serialize/json.rs index f4147c09..746845e6 100644 --- a/src/serialize/json.rs +++ b/src/serialize/json.rs @@ -1010,7 +1010,7 @@ pub struct PrettyFormatter { } impl PrettyFormatter { - pub fn new() -> Self { + pub const fn new() -> Self { PrettyFormatter { current_indent: 0, has_value: false, @@ -1018,12 +1018,6 @@ impl PrettyFormatter { } } -impl Default for PrettyFormatter { - fn default() -> Self { - PrettyFormatter::new() - } -} - impl Formatter for PrettyFormatter { #[inline] fn begin_array(&mut self, writer: &mut W) -> io::Result<()> diff --git a/src/serialize/per_type/dataclass.rs b/src/serialize/per_type/dataclass.rs index 6541d398..9d90d94d 100644 --- a/src/serialize/per_type/dataclass.rs +++ b/src/serialize/per_type/dataclass.rs @@ -1,9 +1,9 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) use crate::opt::*; -use crate::serialize::error::*; -use crate::serialize::serializer::*; -use crate::str::*; +use crate::serialize::error::SerializeError; +use crate::serialize::serializer::{PyObjectSerializer, RECURSION_LIMIT}; +use crate::str::unicode_to_str; use crate::typeref::*; use serde::ser::{Serialize, SerializeMap, Serializer}; diff --git a/src/serialize/per_type/datetime.rs b/src/serialize/per_type/datetime.rs index 47d0c20b..2ead1ef5 100644 --- a/src/serialize/per_type/datetime.rs +++ b/src/serialize/per_type/datetime.rs @@ -1,11 +1,13 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) use crate::opt::*; -use crate::serialize::error::*; +use crate::serialize::error::SerializeError; use crate::serialize::per_type::datetimelike::{ DateTimeBuffer, DateTimeError, DateTimeLike, Offset, }; -use crate::typeref::*; +#[cfg(Py_3_9)] +use crate::typeref::ZONEINFO_TYPE; +use crate::typeref::{CONVERT_METHOD_STR, DST_STR, NORMALIZE_METHOD_STR, UTCOFFSET_METHOD_STR}; use serde::ser::{Serialize, Serializer}; macro_rules! write_double_digit { diff --git a/src/serialize/per_type/default.rs b/src/serialize/per_type/default.rs index 6f2d0982..3c143b1b 100644 --- a/src/serialize/per_type/default.rs +++ b/src/serialize/per_type/default.rs @@ -1,8 +1,8 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) use crate::opt::*; -use crate::serialize::error::*; -use crate::serialize::serializer::*; +use crate::serialize::error::SerializeError; +use crate::serialize::serializer::{PyObjectSerializer, RECURSION_LIMIT}; use serde::ser::{Serialize, Serializer}; diff --git a/src/serialize/per_type/dict.rs b/src/serialize/per_type/dict.rs index a20eaf06..34b3b0b8 100644 --- a/src/serialize/per_type/dict.rs +++ b/src/serialize/per_type/dict.rs @@ -1,13 +1,13 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) use crate::opt::*; -use crate::serialize::error::*; +use crate::serialize::error::SerializeError; use crate::serialize::per_type::datetimelike::{DateTimeBuffer, DateTimeLike}; use crate::serialize::per_type::*; use crate::serialize::serializer::{ pyobject_to_obtype, ObType, PyObjectSerializer, RECURSION_LIMIT, }; -use crate::str::*; +use crate::str::{unicode_to_str, unicode_to_str_via_ffi}; use crate::typeref::*; use compact_str::CompactString; use serde::ser::{Serialize, SerializeMap, Serializer}; diff --git a/src/serialize/per_type/fragment.rs b/src/serialize/per_type/fragment.rs index 1b201bd9..969ad43f 100644 --- a/src/serialize/per_type/fragment.rs +++ b/src/serialize/per_type/fragment.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) use crate::ffi::{Fragment, PyBytes_AS_STRING, PyBytes_GET_SIZE}; -use crate::serialize::error::*; +use crate::serialize::error::SerializeError; use crate::str::unicode_to_str; use crate::typeref::{BYTES_TYPE, STR_TYPE}; diff --git a/src/serialize/per_type/int.rs b/src/serialize/per_type/int.rs index e92049ca..53a90e86 100644 --- a/src/serialize/per_type/int.rs +++ b/src/serialize/per_type/int.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) use crate::ffi::{pylong_is_unsigned, pylong_is_zero}; -use crate::serialize::error::*; +use crate::serialize::error::SerializeError; use serde::ser::{Serialize, Serializer}; // https://tools.ietf.org/html/rfc7159#section-6 diff --git a/src/serialize/per_type/list.rs b/src/serialize/per_type/list.rs index 9e51bfa9..327aea23 100644 --- a/src/serialize/per_type/list.rs +++ b/src/serialize/per_type/list.rs @@ -1,8 +1,7 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) -use crate::ffi::PyListIter; -use crate::opt::*; -use crate::serialize::error::*; +use crate::opt::Opt; +use crate::serialize::error::SerializeError; use crate::serialize::serializer::{PyObjectSerializer, RECURSION_LIMIT}; use serde::ser::{Serialize, SerializeSeq, Serializer}; @@ -46,9 +45,11 @@ impl Serialize for ListSerializer { serializer.serialize_seq(Some(0)).unwrap().end() } else { let mut seq = serializer.serialize_seq(None).unwrap(); - for elem in PyListIter::from_pyobject(self.ptr) { + for idx in 0..=ffi!(Py_SIZE(self.ptr)) - 1 { + let elem = + unsafe { *((*(self.ptr as *mut pyo3_ffi::PyListObject)).ob_item).offset(idx) }; let value = PyObjectSerializer::new( - elem.as_ptr(), + elem, self.opts, self.default_calls, self.recursion, diff --git a/src/serialize/per_type/none.rs b/src/serialize/per_type/none.rs index 173ce9f5..10240826 100644 --- a/src/serialize/per_type/none.rs +++ b/src/serialize/per_type/none.rs @@ -5,8 +5,8 @@ use serde::ser::{Serialize, Serializer}; pub struct NoneSerializer; impl NoneSerializer { - pub fn new() -> Self { - NoneSerializer {} + pub const fn new() -> Self { + Self {} } } diff --git a/src/serialize/per_type/numpy.rs b/src/serialize/per_type/numpy.rs index 0bfd60eb..3632f3bf 100644 --- a/src/serialize/per_type/numpy.rs +++ b/src/serialize/per_type/numpy.rs @@ -1,6 +1,6 @@ use crate::opt::*; -use crate::serialize::error::*; +use crate::serialize::error::SerializeError; use crate::serialize::per_type::datetimelike::{ DateTimeBuffer, DateTimeError, DateTimeLike, Offset, }; diff --git a/src/serialize/per_type/pyenum.rs b/src/serialize/per_type/pyenum.rs index 1972f8a2..06baaffd 100644 --- a/src/serialize/per_type/pyenum.rs +++ b/src/serialize/per_type/pyenum.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) use crate::opt::*; -use crate::serialize::serializer::*; +use crate::serialize::serializer::PyObjectSerializer; use crate::typeref::*; use serde::ser::{Serialize, Serializer}; use std::ptr::NonNull; diff --git a/src/serialize/per_type/tuple.rs b/src/serialize/per_type/tuple.rs index 7ad76e42..4f23abb6 100644 --- a/src/serialize/per_type/tuple.rs +++ b/src/serialize/per_type/tuple.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) -use crate::opt::*; -use crate::serialize::serializer::*; +use crate::opt::Opt; +use crate::serialize::serializer::PyObjectSerializer; use serde::ser::{Serialize, SerializeSeq, Serializer}; use std::ptr::NonNull; diff --git a/src/serialize/per_type/unicode.rs b/src/serialize/per_type/unicode.rs index 76994fd7..a44cda09 100644 --- a/src/serialize/per_type/unicode.rs +++ b/src/serialize/per_type/unicode.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) -use crate::serialize::error::*; -use crate::str::*; +use crate::serialize::error::SerializeError; +use crate::str::{unicode_to_str, unicode_to_str_via_ffi}; use serde::ser::{Serialize, Serializer}; diff --git a/src/serialize/per_type/uuid.rs b/src/serialize/per_type/uuid.rs index e4079475..badb759e 100644 --- a/src/serialize/per_type/uuid.rs +++ b/src/serialize/per_type/uuid.rs @@ -1,12 +1,13 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) -use crate::typeref::*; +use crate::typeref::INT_ATTR_STR; use serde::ser::{Serialize, Serializer}; use std::io::Write; use std::os::raw::c_uchar; pub type UUIDBuffer = arrayvec::ArrayVec; +#[repr(transparent)] pub struct UUID { ptr: *mut pyo3_ffi::PyObject, } diff --git a/src/typeref.rs b/src/typeref.rs index 55dad15c..242a8275 100644 --- a/src/typeref.rs +++ b/src/typeref.rs @@ -236,11 +236,9 @@ unsafe fn look_up_json_exc() -> *mut PyObject { #[cold] #[cfg_attr(feature = "optimize", optimize(size))] -unsafe fn look_up_numpy_type(numpy_module: *mut PyObject, np_type: &str) -> *mut PyTypeObject { - let mod_dict = PyObject_GenericGetDict(numpy_module, null_mut()); - let ptr = PyMapping_GetItemString(mod_dict, np_type.as_ptr() as *const c_char); +unsafe fn look_up_numpy_type(numpy_module_dict: *mut PyObject, np_type: &str) -> *mut PyTypeObject { + let ptr = PyMapping_GetItemString(numpy_module_dict, np_type.as_ptr() as *const c_char); Py_XDECREF(ptr); - Py_XDECREF(mod_dict); ptr as *mut PyTypeObject } @@ -253,21 +251,23 @@ pub fn load_numpy_types() -> Box>> { PyErr_Clear(); return Box::new(None); } + let numpy_module_dict = PyObject_GenericGetDict(numpy, null_mut()); let types = Box::new(NumpyTypes { - array: look_up_numpy_type(numpy, "ndarray\0"), - float32: look_up_numpy_type(numpy, "float32\0"), - float64: look_up_numpy_type(numpy, "float64\0"), - int8: look_up_numpy_type(numpy, "int8\0"), - int16: look_up_numpy_type(numpy, "int16\0"), - int32: look_up_numpy_type(numpy, "int32\0"), - int64: look_up_numpy_type(numpy, "int64\0"), - uint16: look_up_numpy_type(numpy, "uint16\0"), - uint32: look_up_numpy_type(numpy, "uint32\0"), - uint64: look_up_numpy_type(numpy, "uint64\0"), - uint8: look_up_numpy_type(numpy, "uint8\0"), - bool_: look_up_numpy_type(numpy, "bool_\0"), - datetime64: look_up_numpy_type(numpy, "datetime64\0"), + array: look_up_numpy_type(numpy_module_dict, "ndarray\0"), + float32: look_up_numpy_type(numpy_module_dict, "float32\0"), + float64: look_up_numpy_type(numpy_module_dict, "float64\0"), + int8: look_up_numpy_type(numpy_module_dict, "int8\0"), + int16: look_up_numpy_type(numpy_module_dict, "int16\0"), + int32: look_up_numpy_type(numpy_module_dict, "int32\0"), + int64: look_up_numpy_type(numpy_module_dict, "int64\0"), + uint16: look_up_numpy_type(numpy_module_dict, "uint16\0"), + uint32: look_up_numpy_type(numpy_module_dict, "uint32\0"), + uint64: look_up_numpy_type(numpy_module_dict, "uint64\0"), + uint8: look_up_numpy_type(numpy_module_dict, "uint8\0"), + bool_: look_up_numpy_type(numpy_module_dict, "bool_\0"), + datetime64: look_up_numpy_type(numpy_module_dict, "datetime64\0"), }); + Py_XDECREF(numpy_module_dict); Py_XDECREF(numpy); Box::new(Some(nonnull!(Box::::into_raw(types)))) }