From 4a05a588dbe0cbdef8ff207c0d3f2e5023714a96 Mon Sep 17 00:00:00 2001 From: ijl Date: Sat, 6 Jan 2024 16:55:00 +0000 Subject: [PATCH] cargo update, build and tooling misc --- .github/workflows/debug.yaml | 16 +++--- .github/workflows/lint.yaml | 2 +- .github/workflows/linux.yaml | 36 ++++++++------ Cargo.lock | 75 ++++++++++++++--------------- Cargo.toml | 5 +- README.md | 6 +-- build.rs | 9 ++-- ci/azure-macos.yml | 9 ++-- ci/azure-pipelines.yml | 2 +- ci/config.toml | 7 --- integration/wsgi.py | 4 +- pyproject.toml | 13 ++--- requirements.txt | 3 -- script/develop | 4 +- script/graph | 4 +- script/lint | 4 +- script/pynumpy | 4 +- src/deserialize/cache.rs | 4 +- src/deserialize/deserializer.rs | 2 +- src/deserialize/json.rs | 2 +- src/deserialize/pyobject.rs | 10 ++-- src/deserialize/utf8.rs | 4 +- src/deserialize/yyjson.rs | 9 ++-- src/ffi/list.rs | 44 ----------------- src/ffi/mod.rs | 2 - src/lib.rs | 15 +++--- src/serialize/json.rs | 8 +-- src/serialize/per_type/dataclass.rs | 34 +++---------- src/serialize/per_type/datetime.rs | 6 ++- src/serialize/per_type/default.rs | 4 +- src/serialize/per_type/dict.rs | 51 ++++---------------- src/serialize/per_type/fragment.rs | 2 +- src/serialize/per_type/int.rs | 2 +- src/serialize/per_type/list.rs | 11 +++-- src/serialize/per_type/none.rs | 4 +- src/serialize/per_type/numpy.rs | 69 +++++++++++++++----------- src/serialize/per_type/pyenum.rs | 2 +- src/serialize/per_type/tuple.rs | 4 +- src/serialize/per_type/unicode.rs | 4 +- src/serialize/per_type/uuid.rs | 3 +- src/typeref.rs | 41 +++++++++------- src/util.rs | 6 +++ 42 files changed, 227 insertions(+), 319 deletions(-) delete mode 100644 src/ffi/list.rs diff --git a/.github/workflows/debug.yaml b/.github/workflows/debug.yaml index 06b6d8fe..d2694d81 100644 --- a/.github/workflows/debug.yaml +++ b/.github/workflows/debug.yaml @@ -7,13 +7,13 @@ jobs: strategy: fail-fast: false matrix: - rust: [ - { version: "1.65" }, # MSRV - { version: "nightly-2023-10-10" }, + profile: [ + { rust: "1.65", features: "" }, + { rust: "1.65", features: "--features=yyjson" }, + { rust: "nightly-2023-01-06", features: "--features=yyjson,unstable-simd" }, ] python: [ { version: '3.12', abi: 'cp312-cp312' }, - { version: '3.11', abi: 'cp311-cp311' }, { version: '3.8', abi: 'cp38-cp38' }, ] env: @@ -22,9 +22,9 @@ jobs: LDFLAGS: "-Wl,--as-needed" CARGO_UNSTABLE_SPARSE_REGISTRY: "true" steps: - - run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain ${{ matrix.rust.version }} --profile minimal -y + - run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain ${{ matrix.profile.rust }} --profile minimal -y - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '${{ matrix.python.version }}' - run: python -m pip install --user --upgrade pip "maturin>=1,<2" wheel @@ -36,9 +36,9 @@ jobs: PATH="$HOME/.cargo/bin:$PATH" maturin build --release \ --out=dist \ --profile=dev \ - --features=yyjson \ --interpreter python${{ matrix.python.version }} \ - --target=x86_64-unknown-linux-gnu + --target=x86_64-unknown-linux-gnu \ + --features ${{ matrix.profile.features }} - run: python -m pip install --user dist/orjson*.whl - run: python -m pip install --user -r test/requirements.txt -r integration/requirements.txt diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index e9b6dc63..d7bbae4a 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -4,7 +4,7 @@ jobs: lint: runs-on: ubuntu-22.04 steps: - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.11' - run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain=stable --profile=default -y diff --git a/.github/workflows/linux.yaml b/.github/workflows/linux.yaml index d60d0c1e..78717801 100644 --- a/.github/workflows/linux.yaml +++ b/.github/workflows/linux.yaml @@ -62,17 +62,17 @@ jobs: env: PATH: /github/home/.local/bin:/github/home/.cargo/bin:/opt/python/${{ matrix.python.abi }}/bin:/opt/rh/gcc-toolset-12/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin CC: "clang" - CFLAGS: "-O2 -fno-plt -flto=thin" - LDFLAGS: "-O2 -flto=thin -fuse-ld=lld -Wl,--as-needed" - RUSTFLAGS: "-C linker=clang -C link-arg=-fuse-ld=lld" + CFLAGS: "-O2 -fno-plt -foffload-lto=full -fstrict-aliasing" + LDFLAGS: "-O2 -fno-plt -foffload-lto=full -fstrict-aliasing -fuse-ld=lld -Wl,--as-needed" + RUSTFLAGS: "-C linker=clang -C linker-plugin-lto -C lto=fat -C link-arg=-fuse-ld=lld -Z mir-opt-level=4 -Z virtual-function-elimination -D warnings" CARGO_UNSTABLE_SPARSE_REGISTRY: "true" container: image: quay.io/pypa/manylinux_2_28_x86_64:latest options: --user 0 steps: - run: yum install -y clang lld - - run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2023-10-10 --profile minimal -y - - run: rustup component add rust-src --toolchain nightly-2023-10-10-x86_64-unknown-linux-gnu + - run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2023-01-06 --profile minimal -y + - run: rustup component add rust-src --toolchain nightly-2023-01-06-x86_64-unknown-linux-gnu - uses: actions/checkout@v4 - name: build-std @@ -140,10 +140,10 @@ jobs: CC: "gcc" CFLAGS: "-O2 -fno-plt" LDFLAGS: "-O2 -flto -Wl,--as-needed" - RUSTFLAGS: "-C target-feature=-crt-static" + RUSTFLAGS: "-C target-feature=-crt-static -Z mir-opt-level=4" CARGO_UNSTABLE_SPARSE_REGISTRY: "true" with: - rust-toolchain: nightly-2023-10-10 + rust-toolchain: nightly-2023-01-06 rustup-components: rust-src target: ${{ matrix.platform.target }} manylinux: musllinux_1_1 @@ -153,7 +153,7 @@ jobs: if: matrix.platform.arch != 'x86_64' uses: docker/setup-qemu-action@v3 with: - image: tonistiigi/binfmt:qemu-v7.0.0 + image: tonistiigi/binfmt:qemu-v8.0.4 platforms: ${{ matrix.platform.platform }} - name: Test @@ -195,27 +195,30 @@ jobs: { arch: 'aarch64', target: 'aarch64-unknown-linux-gnu', - cflags: '-O2', + cflags: '-O2 -flto', + rustflags: '-Z mir-opt-level=4 -D warnings', }, { arch: 'armv7', target: 'armv7-unknown-linux-gnueabihf', - cflags: '-Os -fstrict-aliasing', + cflags: '-Os -flto -fstrict-aliasing', + rustflags: '-C opt-level=s -Z mir-opt-level=4 -D warnings', }, { arch: 'ppc64le', target: 'powerpc64le-unknown-linux-gnu', - cflags: '-O2', + cflags: '-O2 -flto', + rustflags: '-Z mir-opt-level=4 -D warnings', }, { arch: 's390x', target: 's390x-unknown-linux-gnu', - cflags: '-O2 -march=z10', + cflags: '-O2 -flto -march=z10', + rustflags: '-Z mir-opt-level=4 -C target-cpu=z10 -D warnings', }, ] steps: - uses: actions/checkout@v4 - - name: build-std run: | mkdir .cargo @@ -226,11 +229,12 @@ jobs: env: PYO3_CROSS_LIB_DIR: "/opt/python/${{ matrix.python.abi }}" CFLAGS: "${{ matrix.target.cflags }}" - LDFLAGS: "${{ matrix.target.cflags }} -flto -Wl,--as-needed" + LDFLAGS: "${{ matrix.target.cflags }} -Wl,--as-needed" CARGO_UNSTABLE_SPARSE_REGISTRY: "true" + RUSTFLAGS: "${{ matrix.target.rustflags }}" with: target: ${{ matrix.target.target }} - rust-toolchain: nightly-2023-10-10 + rust-toolchain: nightly-2023-01-06 rustup-components: rust-src manylinux: auto args: --release --strip --out=dist --features=no-panic,yyjson -i python${{ matrix.python.version }} @@ -277,7 +281,7 @@ jobs: - uses: actions/download-artifact@v3 with: name: wheels - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "3.11" - run: pip install pip "maturin>=1,<2" diff --git a/Cargo.lock b/Cargo.lock index 557acf5a..0b5d8fdd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "ahash" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" +checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" dependencies = [ "cfg-if", "once_cell", @@ -49,9 +49,6 @@ name = "bytecount" version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1e5f035d16fc623ae5f74981db80a439803888314e3a555fd6f04acd51a3205" -dependencies = [ - "packed_simd", -] [[package]] name = "castaway" @@ -112,9 +109,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" dependencies = [ "no-panic", ] @@ -127,9 +124,9 @@ checksum = "9028f49264629065d057f340a86acb84867925865f73bbf8d47b4d149a7e88b8" [[package]] name = "libc" -version = "0.2.149" +version = "0.2.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" +checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" [[package]] name = "libm" @@ -139,9 +136,9 @@ checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" [[package]] name = "no-panic" -version = "0.1.26" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71a6d126424f5ce0bb4587ff4561421d44aeede520541cc66f1bb912506ae46a" +checksum = "fc56831a2ae584dc43a8b0b33f496e71fb4d43cf8c1c0a3fd932e6340bea1f81" dependencies = [ "proc-macro2", "quote", @@ -160,9 +157,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "orjson" @@ -202,18 +199,18 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.69" +version = "1.0.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" +checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" dependencies = [ "unicode-ident", ] [[package]] name = "pyo3-build-config" -version = "0.20.0" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a96fe70b176a89cff78f2fa7b3c930081e163d5379b4dcdf993e3ae29ca662e5" +checksum = "07426f0d8fe5a601f26293f300afd1a7b1ed5e78b2a705870c5f30893c5163be" dependencies = [ "once_cell", "target-lexicon", @@ -221,9 +218,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.20.0" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "214929900fd25e6604661ed9cf349727c8920d47deff196c4e28165a6ef2a96b" +checksum = "dbb7dec17e17766b46bca4f1a4215a85006b4c2ecde122076c562dd058da6cf1" dependencies = [ "libc", "pyo3-build-config", @@ -231,9 +228,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -246,27 +243,27 @@ checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "ryu" -version = "1.0.15" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" dependencies = [ "no-panic", ] [[package]] name = "serde" -version = "1.0.190" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91d3c334ca1ee894a2c6f6ad698fe8c435b76d504b13d436f0685d648d6d96f7" +checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.190" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67c5609f394e5c2bd7fc51efda478004ea80ef42fee983d5c67a65e34f32c0e3" +checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" dependencies = [ "proc-macro2", "quote", @@ -275,9 +272,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.107" +version = "1.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" dependencies = [ "itoa", "ryu", @@ -292,9 +289,9 @@ checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" [[package]] name = "smallvec" -version = "1.11.1" +version = "1.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" +checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" [[package]] name = "static_assertions" @@ -304,9 +301,9 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "syn" -version = "2.0.38" +version = "2.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" dependencies = [ "proc-macro2", "quote", @@ -315,9 +312,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.12" +version = "0.12.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c39fd04924ca3a864207c66fc2cd7d22d7c016007f9ce846cbb9326331930a" +checksum = "69758bda2e78f098e4ccb393021a0963bb3442eac05f135c30f61b7370bbafae" [[package]] name = "unicode-ident" @@ -333,18 +330,18 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "zerocopy" -version = "0.7.15" +version = "0.7.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81ba595b9f2772fbee2312de30eeb80ec773b4cb2f1e8098db024afadda6c06f" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.15" +version = "0.7.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "772666c41fb6dceaf520b564b962d738a8e1a83b41bd48945f50837aed78bb1d" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 5a0dab31..ed5c699b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,7 +34,6 @@ default = [] # Use SIMD intrinsics. This requires Rust on the nightly channel. unstable-simd = [ - "bytecount/generic-simd", "encoding_rs/simd-accel", ] @@ -59,7 +58,7 @@ encoding_rs = { version = "0.8", default_features = false } itoa = { version = "1", default_features = false } itoap = { version = "1", features = ["std", "simd"] } once_cell = { version = "1", default_features = false, features = ["race"] } -pyo3-ffi = { version = "^0.20", default_features = false, features = ["extension-module"]} +pyo3-ffi = { version = "^0.20.2", default_features = false, features = ["extension-module"]} ryu = { version = "1", default_features = false } serde = { version = "1", default_features = false } serde_json = { version = "1", default_features = false, features = ["std", "float_roundtrip"] } @@ -68,7 +67,7 @@ smallvec = { version = "^1.11", default_features = false, features = ["union", " [build-dependencies] cc = { version = "1" } -pyo3-build-config = { version = "^0.20" } +pyo3-build-config = { version = "^0.20.2" } version_check = { version = "0.9" } [profile.dev] diff --git a/README.md b/README.md index 532ca6d5..2ee1b08f 100644 --- a/README.md +++ b/README.md @@ -985,9 +985,9 @@ library handles a combined 342 JSON fixtures from the | Library | Invalid JSON documents not rejected | Valid JSON documents not deserialized | |------------|---------------------------------------|-----------------------------------------| | orjson | 0 | 0 | -| ujson | 38 | 0 | +| ujson | 31 | 0 | | rapidjson | 6 | 0 | -| simplejson | 13 | 0 | +| simplejson | 10 | 0 | | json | 17 | 0 | This shows that all libraries deserialize valid JSON but only orjson @@ -1192,7 +1192,7 @@ It benefits from also having a C build environment to compile a faster deserialization backend. See this project's `manylinux_2_28` builds for an example using clang and LTO. -The project's own CI tests against `nightly-2023-10-10` and stable 1.65. It +The project's own CI tests against `nightly-2023-01-06` and stable 1.65. It is prudent to pin the nightly version because that channel can introduce breaking changes. diff --git a/build.rs b/build.rs index 86894e0a..66caf728 100644 --- a/build.rs +++ b/build.rs @@ -11,8 +11,9 @@ fn main() { println!("cargo:rerun-if-env-changed=RUSTFLAGS"); println!("cargo:rerun-if-env-changed=ORJSON_DISABLE_YYJSON"); - let py_cfg = pyo3_build_config::get(); - py_cfg.emit_pyo3_cfgs(); + for cfg in pyo3_build_config::get().build_script_outputs() { + println!("{cfg}"); + } if let Some(true) = version_check::supports_feature("core_intrinsics") { println!("cargo:rustc-cfg=feature=\"intrinsics\""); @@ -26,8 +27,8 @@ fn main() { println!("cargo:rustc-cfg=feature=\"strict_provenance\""); } - if let Some(true) = version_check::supports_feature("trusted_len") { - println!("cargo:rustc-cfg=feature=\"trusted_len\""); + if let Some(true) = version_check::supports_feature("build_hasher_simple_hash_one") { + println!("cargo:rustc-cfg=feature=\"build_hasher_simple_hash_one\""); } if env::var("ORJSON_DISABLE_YYJSON").is_ok() { diff --git a/ci/azure-macos.yml b/ci/azure-macos.yml index 7ada3141..e8ea91f0 100644 --- a/ci/azure-macos.yml +++ b/ci/azure-macos.yml @@ -26,10 +26,11 @@ steps: maturin build --release --strip --features=no-panic,yyjson --interpreter $(interpreter) --target=universal2-apple-darwin env: CC: "clang" - LDFLAGS: "-O2 -flto=thin -fuse-ld=lld -Wl,--as-needed" - CFLAGS: "-O2 -fno-plt -flto=thin" - CFLAGS_x86_64_apple_darwin: "-O2 -fno-plt -flto=thin -march=x86-64-v2 -mtune=generic" - CFLAGS_aarch64_apple_darwin: "-O2 -fno-plt -flto=thin -mcpu=apple-m1 -mtune=generic" + CFLAGS: "-O2 -fno-plt -flto=thin -fstrict-aliasing" + LDFLAGS: "-O2 -fno-plt -flto=thin -fstrict-aliasing -Wl,--as-needed" + CFLAGS_x86_64_apple_darwin: "-O2 -fno-plt -flto=thin -fstrict-aliasing -march=x86-64-v2 -mtune=generic" + CFLAGS_aarch64_apple_darwin: "-O2 -fno-plt -flto=thin -fstrict-aliasing -mcpu=apple-m1 -mtune=generic" + RUSTFLAGS: "-Z mir-opt-level=4 -D warnings" CARGO_UNSTABLE_SPARSE_REGISTRY: "true" displayName: build universal2 diff --git a/ci/azure-pipelines.yml b/ci/azure-pipelines.yml index c78da7a4..e1b38880 100644 --- a/ci/azure-pipelines.yml +++ b/ci/azure-pipelines.yml @@ -1,5 +1,5 @@ variables: - toolchain: nightly-2023-10-10 + toolchain: nightly-2023-01-06 jobs: diff --git a/ci/config.toml b/ci/config.toml index a386c2ff..0823eb5a 100644 --- a/ci/config.toml +++ b/ci/config.toml @@ -9,10 +9,3 @@ rustflags = ["-C", "target-cpu=x86-64-v2", "-Z", "tune-cpu=generic"] [target.aarch64-apple-darwin] linker = "clang" rustflags = ["-C", "target-cpu=apple-m1"] - -[target.armv7-unknown-linux-gnueabihf] -rustflags = ["-C", "opt-level=s"] - -[target.s390x-unknown-linux-gnu] -linker = "clang" -rustflags = ["-C", "target-cpu=z10", "-Z", "tune-cpu=generic"] diff --git a/integration/wsgi.py b/integration/wsgi.py index 7e20e9b0..b5d4d4c4 100644 --- a/integration/wsgi.py +++ b/integration/wsgi.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) -from datetime import datetime +from datetime import datetime, timezone from uuid import uuid4 from flask import Flask @@ -9,7 +9,7 @@ app = Flask(__name__) -NOW = datetime.utcnow() +NOW = datetime.now(timezone.utc) @app.route("/") diff --git a/pyproject.toml b/pyproject.toml index 276703ed..58007e5f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,20 +39,21 @@ include = [ { format = "sdist", path = "include/**/*" }, ] - -[tool.black] -line-length = 88 -target-version = ["py38"] -include = ".pyi?$" - [tool.ruff] line-length = 88 target-version = "py38" +select = [ + "I", +] + ignore = [ "E501", # line too long "F601", # Dictionary key literal ... repeated ] +[tool.ruff.lint.isort] +known-first-party = ["orjson"] + [tool.mypy] python_version = "3.8" diff --git a/requirements.txt b/requirements.txt index c8d3e01a..9ef0c1fd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,6 @@ -r bench/requirements.txt -r integration/requirements.txt -r test/requirements.txt -autoflake -black -isort maturin mypy ruff diff --git a/script/develop b/script/develop index 6bad5db7..5febd5ec 100755 --- a/script/develop +++ b/script/develop @@ -3,9 +3,9 @@ rm -f target/wheels/* export CC="clang" -export CFLAGS="-O2 -fno-plt -flto=thin" +export CFLAGS="-O2 -fno-plt -foffload-lto=full -fstrict-aliasing" export LDFLAGS="${CFLAGS} -fuse-ld=lld -Wl,--as-needed" -export RUSTFLAGS="-C linker=clang -C link-arg=-fuse-ld=lld" +export RUSTFLAGS="-C linker=clang -C linker-plugin-lto -C lto=fat -C link-arg=-fuse-ld=lld -Z mir-opt-level=4 -Z virtual-function-elimination" maturin build "$@" diff --git a/script/graph b/script/graph index 89c86303..4255a127 100755 --- a/script/graph +++ b/script/graph @@ -46,14 +46,14 @@ def tab(obj): [ lib, val[lib]["median"] if correct else None, - "%.1f" % val[lib]["ops"] if correct else None, + str(int(val[lib]["ops"])) if correct else None, 0, ] ) baseline = table[0][1] for each in table: each[3] = ( - "%.2f" % (each[1] / baseline) if isinstance(each[1], float) else None + "%.1f" % (each[1] / baseline) if isinstance(each[1], float) else None ) each[1] = "%.2f" % each[1] if isinstance(each[1], float) else None buf.write(tabulate(table, headers, tablefmt="github") + "\n") diff --git a/script/lint b/script/lint index c08a928f..15eed6ec 100755 --- a/script/lint +++ b/script/lint @@ -6,8 +6,6 @@ to_lint="./bench/*.py ./pysrc/orjson/__init__.pyi ./test/*.py script/pydataclass script/pysort script/pynumpy script/pynonstr script/pycorrectness script/graph integration/init integration/wsgi.py integration/typestubs.py integration/thread" -autoflake --in-place --recursive --remove-all-unused-imports --ignore-init-module-imports . -isort ${to_lint} ruff ${to_lint} --fix -black ${to_lint} +ruff format ${to_lint} mypy --ignore-missing-imports --check-untyped-defs ./bench/*.py ./pysrc/orjson/__init__.pyi ./test/*.py diff --git a/script/pynumpy b/script/pynumpy index df7a1194..102a395d 100755 --- a/script/pynumpy +++ b/script/pynumpy @@ -39,9 +39,7 @@ elif kind == "int32": elif kind == "uint8": array = numpy.random.randint(((2**8) - 1), size=(100000, 100), dtype=numpy.uint8) elif kind == "uint16": - array = numpy.random.randint( - ((2**16) - 1), size=(100000, 100), dtype=numpy.uint16 - ) + array = numpy.random.randint(((2**16) - 1), size=(100000, 100), dtype=numpy.uint16) else: print("usage: pynumpy (bool|int16|int32|float64|int8|uint8|uint16)") sys.exit(1) diff --git a/src/deserialize/cache.rs b/src/deserialize/cache.rs index 62142f77..4ba30fc8 100644 --- a/src/deserialize/cache.rs +++ b/src/deserialize/cache.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) -use crate::typeref::*; +use crate::typeref::HASH_BUILDER; use associative_cache::replacement::RoundRobinReplacement; use associative_cache::*; use once_cell::unsync::OnceCell; @@ -44,7 +44,7 @@ pub static mut KEY_MAP: OnceCell = OnceCell::new(); pub fn cache_hash(key: &[u8]) -> u64 { #[cfg(feature = "intrinsics")] unsafe { - std::intrinsics::assume(key.len() > 0); + std::intrinsics::assume(!key.is_empty()); std::intrinsics::assume(key.len() <= 64); } let mut hasher = unsafe { HASH_BUILDER.get().unwrap().build_hasher() }; diff --git a/src/deserialize/deserializer.rs b/src/deserialize/deserializer.rs index 0675cce0..cd0d09bc 100644 --- a/src/deserialize/deserializer.rs +++ b/src/deserialize/deserializer.rs @@ -2,7 +2,7 @@ use crate::deserialize::utf8::read_input_to_buf; use crate::deserialize::DeserializeError; -use crate::typeref::*; +use crate::typeref::EMPTY_UNICODE; use std::ptr::NonNull; pub fn deserialize( diff --git a/src/deserialize/json.rs b/src/deserialize/json.rs index 9a6d7561..a9d36424 100644 --- a/src/deserialize/json.rs +++ b/src/deserialize/json.rs @@ -2,7 +2,7 @@ use crate::deserialize::pyobject::*; use crate::deserialize::DeserializeError; -use crate::str::*; +use crate::str::unicode_from_str; use serde::de::{self, DeserializeSeed, Deserializer, MapAccess, SeqAccess, Visitor}; use smallvec::SmallVec; use std::borrow::Cow; diff --git a/src/deserialize/pyobject.rs b/src/deserialize/pyobject.rs index 49766b14..0da2f9fd 100644 --- a/src/deserialize/pyobject.rs +++ b/src/deserialize/pyobject.rs @@ -1,8 +1,8 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) use crate::deserialize::cache::*; -use crate::str::*; -use crate::typeref::*; +use crate::str::{hash_str, unicode_from_str}; +use crate::typeref::{EMPTY_UNICODE, FALSE, NONE, TRUE}; use std::ptr::NonNull; pub fn get_unicode_key(key_str: &str) -> *mut pyo3_ffi::PyObject { @@ -14,11 +14,7 @@ pub fn get_unicode_key(key_str: &str) -> *mut pyo3_ffi::PyObject { pykey = use_immortal!(EMPTY_UNICODE); } else { let hash = cache_hash(key_str.as_bytes()); - let map = unsafe { - KEY_MAP - .get_mut() - .unwrap_or_else(|| unsafe { std::hint::unreachable_unchecked() }) - }; + let map = unsafe { KEY_MAP.get_mut().unwrap_or_else(|| unreachable!()) }; let entry = map.entry(&hash).or_insert_with( || hash, || { diff --git a/src/deserialize/utf8.rs b/src/deserialize/utf8.rs index ddb03563..d1cbb474 100644 --- a/src/deserialize/utf8.rs +++ b/src/deserialize/utf8.rs @@ -2,8 +2,8 @@ use crate::deserialize::DeserializeError; use crate::ffi::*; -use crate::str::*; -use crate::typeref::*; +use crate::str::unicode_to_str; +use crate::typeref::{BYTEARRAY_TYPE, BYTES_TYPE, MEMORYVIEW_TYPE, STR_TYPE}; use crate::util::INVALID_STR; use std::borrow::Cow; use std::os::raw::c_char; diff --git a/src/deserialize/yyjson.rs b/src/deserialize/yyjson.rs index c8c5b774..968f7c39 100644 --- a/src/deserialize/yyjson.rs +++ b/src/deserialize/yyjson.rs @@ -3,8 +3,8 @@ use crate::deserialize::pyobject::*; use crate::deserialize::DeserializeError; use crate::ffi::yyjson::*; -use crate::str::*; -use crate::typeref::*; +use crate::str::unicode_from_str; +use crate::typeref::{yyjson_init, YYJSON_ALLOC, YYJSON_BUFFER_SIZE}; use std::borrow::Cow; use std::os::raw::c_char; use std::ptr::{null, null_mut, NonNull}; @@ -125,7 +125,7 @@ impl ElementType { TAG_FALSE => Self::False, TAG_ARRAY => Self::Array, TAG_OBJECT => Self::Object, - _ => unsafe { std::hint::unreachable_unchecked() }, + _ => unreachable!(), } } } @@ -164,10 +164,7 @@ fn parse_yy_object(elem: *mut yyjson_val) -> NonNull { return nonnull!(ffi!(PyDict_New())); } let mut key = unsafe_yyjson_get_first(elem); - #[cfg(not(Py_3_13))] let dict = ffi!(_PyDict_NewPresized(len as isize)); - #[cfg(Py_3_13)] - let dict = ffi!(PyDict_New()); for _ in 0..=len - 1 { let val = key.add(1); let key_str = str_from_slice!((*key).uni.str_ as *const u8, unsafe_yyjson_get_len(key)); diff --git a/src/ffi/list.rs b/src/ffi/list.rs deleted file mode 100644 index 44b386c4..00000000 --- a/src/ffi/list.rs +++ /dev/null @@ -1,44 +0,0 @@ -// SPDX-License-Identifier: (Apache-2.0 OR MIT) - -use std::ptr::NonNull; - -pub struct PyListIter { - obj: *mut pyo3_ffi::PyListObject, - len: usize, - pos: usize, -} - -impl PyListIter { - #[inline] - pub fn from_pyobject(obj: *mut pyo3_ffi::PyObject) -> Self { - unsafe { - PyListIter { - obj: obj as *mut pyo3_ffi::PyListObject, - len: ffi!(Py_SIZE(obj)) as usize, - pos: 0, - } - } - } -} - -impl Iterator for PyListIter { - type Item = NonNull; - - #[inline(always)] - fn next(&mut self) -> Option { - if self.pos == self.len { - None - } else { - let elem = unsafe { *((*self.obj).ob_item).add(self.pos) }; - self.pos += 1; - Some(nonnull!(elem)) - } - } - - fn size_hint(&self) -> (usize, Option) { - (self.len, Some(self.len)) - } -} - -#[cfg(feature = "trusted_len")] -unsafe impl std::iter::TrustedLen for PyListIter {} diff --git a/src/ffi/mod.rs b/src/ffi/mod.rs index afc7ef50..ba6dedcd 100644 --- a/src/ffi/mod.rs +++ b/src/ffi/mod.rs @@ -3,7 +3,6 @@ mod buffer; mod bytes; mod fragment; -mod list; mod long; #[cfg(feature = "yyjson")] pub mod yyjson; @@ -11,5 +10,4 @@ pub mod yyjson; pub use buffer::*; pub use bytes::*; pub use fragment::{orjson_fragmenttype_new, Fragment}; -pub use list::PyListIter; pub use long::{pylong_is_unsigned, pylong_is_zero}; diff --git a/src/lib.rs b/src/lib.rs index 63b9a59a..f3edd8a6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,14 +4,17 @@ #![cfg_attr(feature = "optimize", feature(optimize_attribute))] #![cfg_attr(feature = "strict_provenance", feature(strict_provenance))] #![cfg_attr(feature = "strict_provenance", warn(fuzzy_provenance_casts))] -#![cfg_attr(feature = "trusted_len", feature(trusted_len))] +#![cfg_attr( + feature = "build_hasher_simple_hash_one", + feature(build_hasher_simple_hash_one) +)] +#![allow(unknown_lints)] // internal_features +#![allow(internal_features)] // core_intrinsics #![allow(unused_unsafe)] #![allow(non_camel_case_types)] -#![allow(clippy::explicit_auto_deref)] #![allow(clippy::missing_safety_doc)] #![allow(clippy::redundant_field_names)] #![allow(clippy::uninlined_format_args)] // MSRV 1.66 -#![allow(clippy::unnecessary_unwrap)] #![allow(clippy::upper_case_acronyms)] #![allow(clippy::zero_prefixed_literal)] @@ -232,7 +235,7 @@ fn raise_dumps_exception_fixed(msg: &str) -> *mut PyObject { #[inline(never)] #[cfg_attr(feature = "optimize", optimize(size))] #[cfg(Py_3_12)] -fn raise_dumps_exception_dynamic(err: &String) -> *mut PyObject { +fn raise_dumps_exception_dynamic(err: &str) -> *mut PyObject { unsafe { let cause_exc: *mut PyObject = PyErr_GetRaisedException(); @@ -255,7 +258,7 @@ fn raise_dumps_exception_dynamic(err: &String) -> *mut PyObject { #[inline(never)] #[cfg_attr(feature = "optimize", optimize(size))] #[cfg(not(Py_3_12))] -fn raise_dumps_exception_dynamic(err: &String) -> *mut PyObject { +fn raise_dumps_exception_dynamic(err: &str) -> *mut PyObject { unsafe { let mut cause_tp: *mut PyObject = null_mut(); let mut cause_val: *mut PyObject = null_mut(); @@ -355,6 +358,6 @@ pub unsafe extern "C" fn dumps( match crate::serialize::serialize(*args, default, optsbits as opt::Opt) { Ok(val) => val.as_ptr(), - Err(err) => raise_dumps_exception_dynamic(&err), + Err(err) => raise_dumps_exception_dynamic(err.as_str()), } } diff --git a/src/serialize/json.rs b/src/serialize/json.rs index f4147c09..746845e6 100644 --- a/src/serialize/json.rs +++ b/src/serialize/json.rs @@ -1010,7 +1010,7 @@ pub struct PrettyFormatter { } impl PrettyFormatter { - pub fn new() -> Self { + pub const fn new() -> Self { PrettyFormatter { current_indent: 0, has_value: false, @@ -1018,12 +1018,6 @@ impl PrettyFormatter { } } -impl Default for PrettyFormatter { - fn default() -> Self { - PrettyFormatter::new() - } -} - impl Formatter for PrettyFormatter { #[inline] fn begin_array(&mut self, writer: &mut W) -> io::Result<()> diff --git a/src/serialize/per_type/dataclass.rs b/src/serialize/per_type/dataclass.rs index 6541d398..04f2878b 100644 --- a/src/serialize/per_type/dataclass.rs +++ b/src/serialize/per_type/dataclass.rs @@ -1,9 +1,9 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) use crate::opt::*; -use crate::serialize::error::*; -use crate::serialize::serializer::*; -use crate::str::*; +use crate::serialize::error::SerializeError; +use crate::serialize::serializer::{PyObjectSerializer, RECURSION_LIMIT}; +use crate::str::unicode_to_str; use crate::typeref::*; use serde::ser::{Serialize, SerializeMap, Serializer}; @@ -124,22 +124,12 @@ impl Serialize for DataclassFastSerializer { let mut pos = 0; - ffi!(PyDict_Next( - self.ptr, - &mut pos, - &mut next_key, - &mut next_value - )); + pydict_next!(self.ptr, &mut pos, &mut next_key, &mut next_value); for _ in 0..=ffi!(Py_SIZE(self.ptr)) as usize - 1 { let key = next_key; let value = next_value; - ffi!(PyDict_Next( - self.ptr, - &mut pos, - &mut next_key, - &mut next_value - )); + pydict_next!(self.ptr, &mut pos, &mut next_key, &mut next_value); if unlikely!(unsafe { ob_type!(key) != STR_TYPE }) { err!(SerializeError::KeyMustBeStr) @@ -211,22 +201,12 @@ impl Serialize for DataclassFallbackSerializer { let mut pos = 0; - ffi!(PyDict_Next( - fields, - &mut pos, - &mut next_key, - &mut next_value - )); + pydict_next!(fields, &mut pos, &mut next_key, &mut next_value); for _ in 0..=ffi!(Py_SIZE(fields)) as usize - 1 { let attr = next_key; let field = next_value; - ffi!(PyDict_Next( - fields, - &mut pos, - &mut next_key, - &mut next_value - )); + pydict_next!(fields, &mut pos, &mut next_key, &mut next_value); let field_type = ffi!(PyObject_GetAttr(field, FIELD_TYPE_STR)); debug_assert!(ffi!(Py_REFCNT(field_type)) >= 2); diff --git a/src/serialize/per_type/datetime.rs b/src/serialize/per_type/datetime.rs index 47d0c20b..2ead1ef5 100644 --- a/src/serialize/per_type/datetime.rs +++ b/src/serialize/per_type/datetime.rs @@ -1,11 +1,13 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) use crate::opt::*; -use crate::serialize::error::*; +use crate::serialize::error::SerializeError; use crate::serialize::per_type::datetimelike::{ DateTimeBuffer, DateTimeError, DateTimeLike, Offset, }; -use crate::typeref::*; +#[cfg(Py_3_9)] +use crate::typeref::ZONEINFO_TYPE; +use crate::typeref::{CONVERT_METHOD_STR, DST_STR, NORMALIZE_METHOD_STR, UTCOFFSET_METHOD_STR}; use serde::ser::{Serialize, Serializer}; macro_rules! write_double_digit { diff --git a/src/serialize/per_type/default.rs b/src/serialize/per_type/default.rs index 6f2d0982..3c143b1b 100644 --- a/src/serialize/per_type/default.rs +++ b/src/serialize/per_type/default.rs @@ -1,8 +1,8 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) use crate::opt::*; -use crate::serialize::error::*; -use crate::serialize::serializer::*; +use crate::serialize::error::SerializeError; +use crate::serialize::serializer::{PyObjectSerializer, RECURSION_LIMIT}; use serde::ser::{Serialize, Serializer}; diff --git a/src/serialize/per_type/dict.rs b/src/serialize/per_type/dict.rs index a20eaf06..229f8b58 100644 --- a/src/serialize/per_type/dict.rs +++ b/src/serialize/per_type/dict.rs @@ -1,13 +1,13 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) use crate::opt::*; -use crate::serialize::error::*; +use crate::serialize::error::SerializeError; use crate::serialize::per_type::datetimelike::{DateTimeBuffer, DateTimeLike}; use crate::serialize::per_type::*; use crate::serialize::serializer::{ pyobject_to_obtype, ObType, PyObjectSerializer, RECURSION_LIMIT, }; -use crate::str::*; +use crate::str::{unicode_to_str, unicode_to_str_via_ffi}; use crate::typeref::*; use compact_str::CompactString; use serde::ser::{Serialize, SerializeMap, Serializer}; @@ -119,22 +119,12 @@ impl Serialize for Dict { let mut pos = 0; - ffi!(PyDict_Next( - self.ptr, - &mut pos, - &mut next_key, - &mut next_value - )); + pydict_next!(self.ptr, &mut pos, &mut next_key, &mut next_value); for _ in 0..=ffi!(Py_SIZE(self.ptr)) as usize - 1 { let key = next_key; let value = next_value; - ffi!(PyDict_Next( - self.ptr, - &mut pos, - &mut next_key, - &mut next_value - )); + pydict_next!(self.ptr, &mut pos, &mut next_key, &mut next_value); if unlikely!(unsafe { ob_type!(key) != STR_TYPE }) { err!(SerializeError::KeyMustBeStr) @@ -198,22 +188,12 @@ impl Serialize for DictSortedKey { let mut pos = 0; - ffi!(PyDict_Next( - self.ptr, - &mut pos, - &mut next_key, - &mut next_value - )); + pydict_next!(self.ptr, &mut pos, &mut next_key, &mut next_value); for _ in 0..=len as usize - 1 { let key = next_key; let value = next_value; - ffi!(PyDict_Next( - self.ptr, - &mut pos, - &mut next_key, - &mut next_value - )); + pydict_next!(self.ptr, &mut pos, &mut next_key, &mut next_value); if unlikely!(unsafe { ob_type!(key) != STR_TYPE }) { err!(SerializeError::KeyMustBeStr) @@ -268,13 +248,12 @@ impl DictNonStrKey { } } - #[cfg_attr(feature = "optimize", optimize(size))] fn pyobject_to_string( key: *mut pyo3_ffi::PyObject, opts: crate::opt::Opt, ) -> Result { match pyobject_to_obtype(key, opts) { - ObType::None => Ok(CompactString::from("null")), + ObType::None => Ok(CompactString::new_inline("null")), ObType::Bool => { let key_as_str = if unsafe { key == TRUE } { "true" @@ -299,7 +278,7 @@ impl DictNonStrKey { ObType::Float => { let val = ffi!(PyFloat_AS_DOUBLE(key)); if !val.is_finite() { - Ok(CompactString::from("null")) + Ok(CompactString::new_inline("null")) } else { Ok(CompactString::from(ryu::Buffer::new().format_finite(val))) } @@ -386,22 +365,12 @@ impl Serialize for DictNonStrKey { let mut pos = 0; - ffi!(PyDict_Next( - self.ptr, - &mut pos, - &mut next_key, - &mut next_value - )); + pydict_next!(self.ptr, &mut pos, &mut next_key, &mut next_value); for _ in 0..=ffi!(Py_SIZE(self.ptr)) as usize - 1 { let key = next_key; let value = next_value; - ffi!(PyDict_Next( - self.ptr, - &mut pos, - &mut next_key, - &mut next_value - )); + pydict_next!(self.ptr, &mut pos, &mut next_key, &mut next_value); if is_type!(ob_type!(key), STR_TYPE) { let uni = unicode_to_str(key); diff --git a/src/serialize/per_type/fragment.rs b/src/serialize/per_type/fragment.rs index 1b201bd9..969ad43f 100644 --- a/src/serialize/per_type/fragment.rs +++ b/src/serialize/per_type/fragment.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) use crate::ffi::{Fragment, PyBytes_AS_STRING, PyBytes_GET_SIZE}; -use crate::serialize::error::*; +use crate::serialize::error::SerializeError; use crate::str::unicode_to_str; use crate::typeref::{BYTES_TYPE, STR_TYPE}; diff --git a/src/serialize/per_type/int.rs b/src/serialize/per_type/int.rs index e92049ca..53a90e86 100644 --- a/src/serialize/per_type/int.rs +++ b/src/serialize/per_type/int.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) use crate::ffi::{pylong_is_unsigned, pylong_is_zero}; -use crate::serialize::error::*; +use crate::serialize::error::SerializeError; use serde::ser::{Serialize, Serializer}; // https://tools.ietf.org/html/rfc7159#section-6 diff --git a/src/serialize/per_type/list.rs b/src/serialize/per_type/list.rs index 9e51bfa9..327aea23 100644 --- a/src/serialize/per_type/list.rs +++ b/src/serialize/per_type/list.rs @@ -1,8 +1,7 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) -use crate::ffi::PyListIter; -use crate::opt::*; -use crate::serialize::error::*; +use crate::opt::Opt; +use crate::serialize::error::SerializeError; use crate::serialize::serializer::{PyObjectSerializer, RECURSION_LIMIT}; use serde::ser::{Serialize, SerializeSeq, Serializer}; @@ -46,9 +45,11 @@ impl Serialize for ListSerializer { serializer.serialize_seq(Some(0)).unwrap().end() } else { let mut seq = serializer.serialize_seq(None).unwrap(); - for elem in PyListIter::from_pyobject(self.ptr) { + for idx in 0..=ffi!(Py_SIZE(self.ptr)) - 1 { + let elem = + unsafe { *((*(self.ptr as *mut pyo3_ffi::PyListObject)).ob_item).offset(idx) }; let value = PyObjectSerializer::new( - elem.as_ptr(), + elem, self.opts, self.default_calls, self.recursion, diff --git a/src/serialize/per_type/none.rs b/src/serialize/per_type/none.rs index 173ce9f5..10240826 100644 --- a/src/serialize/per_type/none.rs +++ b/src/serialize/per_type/none.rs @@ -5,8 +5,8 @@ use serde::ser::{Serialize, Serializer}; pub struct NoneSerializer; impl NoneSerializer { - pub fn new() -> Self { - NoneSerializer {} + pub const fn new() -> Self { + Self {} } } diff --git a/src/serialize/per_type/numpy.rs b/src/serialize/per_type/numpy.rs index 0bfd60eb..b83b6421 100644 --- a/src/serialize/per_type/numpy.rs +++ b/src/serialize/per_type/numpy.rs @@ -1,6 +1,6 @@ use crate::opt::*; -use crate::serialize::error::*; +use crate::serialize::error::SerializeError; use crate::serialize::per_type::datetimelike::{ DateTimeBuffer, DateTimeError, DateTimeLike, Offset, }; @@ -79,7 +79,7 @@ macro_rules! slice { }; } -#[cfg_attr(feature = "optimize", optimize(size))] +#[cold] pub fn is_numpy_scalar(ob_type: *mut PyTypeObject) -> bool { let numpy_types = unsafe { NUMPY_TYPES.get_or_init(load_numpy_types) }; if numpy_types.is_none() { @@ -101,7 +101,7 @@ pub fn is_numpy_scalar(ob_type: *mut PyTypeObject) -> bool { } } -#[cfg_attr(feature = "optimize", optimize(size))] +#[cold] pub fn is_numpy_array(ob_type: *mut PyTypeObject) -> bool { let numpy_types = unsafe { NUMPY_TYPES.get_or_init(load_numpy_types) }; if numpy_types.is_none() { @@ -395,7 +395,7 @@ impl<'a> NumpyF64Array<'a> { } impl<'a> Serialize for NumpyF64Array<'a> { - #[cfg_attr(feature = "optimize", optimize(size))] + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -414,6 +414,7 @@ pub struct DataTypeF64 { } impl Serialize for DataTypeF64 { + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -434,7 +435,7 @@ impl<'a> NumpyF32Array<'a> { } impl<'a> Serialize for NumpyF32Array<'a> { - #[cfg_attr(feature = "optimize", optimize(size))] + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -453,6 +454,7 @@ struct DataTypeF32 { } impl Serialize for DataTypeF32 { + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -473,7 +475,7 @@ impl<'a> NumpyU64Array<'a> { } impl<'a> Serialize for NumpyU64Array<'a> { - #[cfg_attr(feature = "optimize", optimize(size))] + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -492,6 +494,7 @@ pub struct DataTypeU64 { } impl Serialize for DataTypeU64 { + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -512,7 +515,7 @@ impl<'a> NumpyU32Array<'a> { } impl<'a> Serialize for NumpyU32Array<'a> { - #[cfg_attr(feature = "optimize", optimize(size))] + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -531,6 +534,7 @@ pub struct DataTypeU32 { } impl Serialize for DataTypeU32 { + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -551,7 +555,7 @@ impl<'a> NumpyU16Array<'a> { } impl<'a> Serialize for NumpyU16Array<'a> { - #[cfg_attr(feature = "optimize", optimize(size))] + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -570,6 +574,7 @@ pub struct DataTypeU16 { } impl Serialize for DataTypeU16 { + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -590,7 +595,7 @@ impl<'a> NumpyI64Array<'a> { } impl<'a> Serialize for NumpyI64Array<'a> { - #[cfg_attr(feature = "optimize", optimize(size))] + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -609,6 +614,7 @@ pub struct DataTypeI64 { } impl Serialize for DataTypeI64 { + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -629,7 +635,7 @@ impl<'a> NumpyI32Array<'a> { } impl<'a> Serialize for NumpyI32Array<'a> { - #[cfg_attr(feature = "optimize", optimize(size))] + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -648,6 +654,7 @@ pub struct DataTypeI32 { } impl Serialize for DataTypeI32 { + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -668,7 +675,7 @@ impl<'a> NumpyI16Array<'a> { } impl<'a> Serialize for NumpyI16Array<'a> { - #[cfg_attr(feature = "optimize", optimize(size))] + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -687,6 +694,7 @@ pub struct DataTypeI16 { } impl Serialize for DataTypeI16 { + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -707,7 +715,7 @@ impl<'a> NumpyI8Array<'a> { } impl<'a> Serialize for NumpyI8Array<'a> { - #[cfg_attr(feature = "optimize", optimize(size))] + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -726,6 +734,7 @@ pub struct DataTypeI8 { } impl Serialize for DataTypeI8 { + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -746,7 +755,7 @@ impl<'a> NumpyU8Array<'a> { } impl<'a> Serialize for NumpyU8Array<'a> { - #[cfg_attr(feature = "optimize", optimize(size))] + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -765,6 +774,7 @@ pub struct DataTypeU8 { } impl Serialize for DataTypeU8 { + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -785,7 +795,7 @@ impl<'a> NumpyBoolArray<'a> { } impl<'a> Serialize for NumpyBoolArray<'a> { - #[cfg_attr(feature = "optimize", optimize(size))] + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -804,6 +814,7 @@ pub struct DataTypeBool { } impl Serialize for DataTypeBool { + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -879,7 +890,7 @@ pub struct NumpyInt8 { } impl Serialize for NumpyInt8 { - #[cfg_attr(feature = "optimize", optimize(size))] + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -896,7 +907,7 @@ pub struct NumpyInt16 { } impl Serialize for NumpyInt16 { - #[cfg_attr(feature = "optimize", optimize(size))] + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -913,7 +924,7 @@ pub struct NumpyInt32 { } impl Serialize for NumpyInt32 { - #[cfg_attr(feature = "optimize", optimize(size))] + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -930,7 +941,7 @@ pub struct NumpyInt64 { } impl Serialize for NumpyInt64 { - #[cfg_attr(feature = "optimize", optimize(size))] + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -947,7 +958,7 @@ pub struct NumpyUint8 { } impl Serialize for NumpyUint8 { - #[cfg_attr(feature = "optimize", optimize(size))] + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -964,7 +975,7 @@ pub struct NumpyUint16 { } impl Serialize for NumpyUint16 { - #[cfg_attr(feature = "optimize", optimize(size))] + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -981,7 +992,7 @@ pub struct NumpyUint32 { } impl Serialize for NumpyUint32 { - #[cfg_attr(feature = "optimize", optimize(size))] + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -998,7 +1009,7 @@ pub struct NumpyUint64 { } impl Serialize for NumpyUint64 { - #[cfg_attr(feature = "optimize", optimize(size))] + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -1015,7 +1026,7 @@ pub struct NumpyFloat32 { } impl Serialize for NumpyFloat32 { - #[cfg_attr(feature = "optimize", optimize(size))] + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -1032,7 +1043,7 @@ pub struct NumpyFloat64 { } impl Serialize for NumpyFloat64 { - #[cfg_attr(feature = "optimize", optimize(size))] + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -1049,7 +1060,7 @@ pub struct NumpyBool { } impl Serialize for NumpyBool { - #[cfg_attr(feature = "optimize", optimize(size))] + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -1083,7 +1094,6 @@ pub enum NumpyDatetimeUnit { impl fmt::Display for NumpyDatetimeUnit { #[cold] - #[cfg_attr(feature = "optimize", optimize(size))] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let unit = match self { Self::NaT => "NaT", @@ -1114,7 +1124,6 @@ enum NumpyDateTimeError { impl NumpyDateTimeError { #[cold] - #[cfg_attr(feature = "optimize", optimize(size))] fn into_serde_err(self) -> T { let err = match self { Self::UnsupportedUnit(unit) => format!("unsupported numpy.datetime64 unit: {}", unit), @@ -1136,6 +1145,7 @@ impl NumpyDatetimeUnit { /// object rather than using the `descr` field of the `__array_struct__` /// because that field isn't populated for datetime64 arrays; see /// https://github.com/numpy/numpy/issues/5350. + #[cold] #[cfg_attr(feature = "optimize", optimize(size))] fn from_pyobject(ptr: *mut PyObject) -> Self { let dtype = ffi!(PyObject_GetAttr(ptr, DTYPE_STR)); @@ -1173,6 +1183,7 @@ impl NumpyDatetimeUnit { /// Return a `NumpyDatetime64Repr` for a value in array with this unit. /// /// Returns an `Err(NumpyDateTimeError)` if the value is invalid for this unit. + #[cold] #[cfg_attr(feature = "optimize", optimize(size))] fn datetime(&self, val: i64, opts: Opt) -> Result { match self { @@ -1245,7 +1256,7 @@ impl<'a> NumpyDatetime64Array<'a> { } impl<'a> Serialize for NumpyDatetime64Array<'a> { - #[cfg_attr(feature = "optimize", optimize(size))] + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -1313,7 +1324,7 @@ impl DateTimeLike for NumpyDatetime64Repr { } impl Serialize for NumpyDatetime64Repr { - #[cfg_attr(feature = "optimize", optimize(size))] + #[cold] fn serialize(&self, serializer: S) -> Result where S: Serializer, diff --git a/src/serialize/per_type/pyenum.rs b/src/serialize/per_type/pyenum.rs index 1972f8a2..06baaffd 100644 --- a/src/serialize/per_type/pyenum.rs +++ b/src/serialize/per_type/pyenum.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) use crate::opt::*; -use crate::serialize::serializer::*; +use crate::serialize::serializer::PyObjectSerializer; use crate::typeref::*; use serde::ser::{Serialize, Serializer}; use std::ptr::NonNull; diff --git a/src/serialize/per_type/tuple.rs b/src/serialize/per_type/tuple.rs index 7ad76e42..4f23abb6 100644 --- a/src/serialize/per_type/tuple.rs +++ b/src/serialize/per_type/tuple.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) -use crate::opt::*; -use crate::serialize::serializer::*; +use crate::opt::Opt; +use crate::serialize::serializer::PyObjectSerializer; use serde::ser::{Serialize, SerializeSeq, Serializer}; use std::ptr::NonNull; diff --git a/src/serialize/per_type/unicode.rs b/src/serialize/per_type/unicode.rs index 76994fd7..a44cda09 100644 --- a/src/serialize/per_type/unicode.rs +++ b/src/serialize/per_type/unicode.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) -use crate::serialize::error::*; -use crate::str::*; +use crate::serialize::error::SerializeError; +use crate::str::{unicode_to_str, unicode_to_str_via_ffi}; use serde::ser::{Serialize, Serializer}; diff --git a/src/serialize/per_type/uuid.rs b/src/serialize/per_type/uuid.rs index e4079475..badb759e 100644 --- a/src/serialize/per_type/uuid.rs +++ b/src/serialize/per_type/uuid.rs @@ -1,12 +1,13 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) -use crate::typeref::*; +use crate::typeref::INT_ATTR_STR; use serde::ser::{Serialize, Serializer}; use std::io::Write; use std::os::raw::c_uchar; pub type UUIDBuffer = arrayvec::ArrayVec; +#[repr(transparent)] pub struct UUID { ptr: *mut pyo3_ffi::PyObject, } diff --git a/src/typeref.rs b/src/typeref.rs index 55dad15c..dd1c2dbb 100644 --- a/src/typeref.rs +++ b/src/typeref.rs @@ -4,9 +4,13 @@ use crate::ffi::orjson_fragmenttype_new; use ahash::RandomState; use once_cell::race::{OnceBool, OnceBox}; use pyo3_ffi::*; +#[cfg(feature = "yyjson")] use std::cell::UnsafeCell; +#[cfg(feature = "yyjson")] use std::mem::MaybeUninit; -use std::os::raw::{c_char, c_void}; +use std::os::raw::c_char; +#[cfg(feature = "yyjson")] +use std::os::raw::c_void; use std::ptr::{null_mut, NonNull}; pub struct NumpyTypes { @@ -148,6 +152,7 @@ pub fn init_typerefs() { #[cfg_attr(feature = "optimize", optimize(size))] fn _init_typerefs_impl() -> bool { unsafe { + debug_assert!(crate::opt::MAX_OPT < u16::MAX as i32); assert!(crate::deserialize::KEY_MAP .set(crate::deserialize::KeyMap::default()) .is_ok()); @@ -236,11 +241,9 @@ unsafe fn look_up_json_exc() -> *mut PyObject { #[cold] #[cfg_attr(feature = "optimize", optimize(size))] -unsafe fn look_up_numpy_type(numpy_module: *mut PyObject, np_type: &str) -> *mut PyTypeObject { - let mod_dict = PyObject_GenericGetDict(numpy_module, null_mut()); - let ptr = PyMapping_GetItemString(mod_dict, np_type.as_ptr() as *const c_char); +unsafe fn look_up_numpy_type(numpy_module_dict: *mut PyObject, np_type: &str) -> *mut PyTypeObject { + let ptr = PyMapping_GetItemString(numpy_module_dict, np_type.as_ptr() as *const c_char); Py_XDECREF(ptr); - Py_XDECREF(mod_dict); ptr as *mut PyTypeObject } @@ -253,21 +256,23 @@ pub fn load_numpy_types() -> Box>> { PyErr_Clear(); return Box::new(None); } + let numpy_module_dict = PyObject_GenericGetDict(numpy, null_mut()); let types = Box::new(NumpyTypes { - array: look_up_numpy_type(numpy, "ndarray\0"), - float32: look_up_numpy_type(numpy, "float32\0"), - float64: look_up_numpy_type(numpy, "float64\0"), - int8: look_up_numpy_type(numpy, "int8\0"), - int16: look_up_numpy_type(numpy, "int16\0"), - int32: look_up_numpy_type(numpy, "int32\0"), - int64: look_up_numpy_type(numpy, "int64\0"), - uint16: look_up_numpy_type(numpy, "uint16\0"), - uint32: look_up_numpy_type(numpy, "uint32\0"), - uint64: look_up_numpy_type(numpy, "uint64\0"), - uint8: look_up_numpy_type(numpy, "uint8\0"), - bool_: look_up_numpy_type(numpy, "bool_\0"), - datetime64: look_up_numpy_type(numpy, "datetime64\0"), + array: look_up_numpy_type(numpy_module_dict, "ndarray\0"), + float32: look_up_numpy_type(numpy_module_dict, "float32\0"), + float64: look_up_numpy_type(numpy_module_dict, "float64\0"), + int8: look_up_numpy_type(numpy_module_dict, "int8\0"), + int16: look_up_numpy_type(numpy_module_dict, "int16\0"), + int32: look_up_numpy_type(numpy_module_dict, "int32\0"), + int64: look_up_numpy_type(numpy_module_dict, "int64\0"), + uint16: look_up_numpy_type(numpy_module_dict, "uint16\0"), + uint32: look_up_numpy_type(numpy_module_dict, "uint32\0"), + uint64: look_up_numpy_type(numpy_module_dict, "uint64\0"), + uint8: look_up_numpy_type(numpy_module_dict, "uint8\0"), + bool_: look_up_numpy_type(numpy_module_dict, "bool_\0"), + datetime64: look_up_numpy_type(numpy_module_dict, "datetime64\0"), }); + Py_XDECREF(numpy_module_dict); Py_XDECREF(numpy); Box::new(Some(nonnull!(Box::::into_raw(types)))) } diff --git a/src/util.rs b/src/util.rs index dd3b25bc..a5cd705a 100644 --- a/src/util.rs +++ b/src/util.rs @@ -221,3 +221,9 @@ macro_rules! use_immortal { } }; } + +macro_rules! pydict_next { + ($obj1:expr, $obj2:expr, $obj3:expr, $obj4:expr) => { + unsafe { pyo3_ffi::_PyDict_Next($obj1, $obj2, $obj3, $obj4, std::ptr::null_mut()) } + }; +}