Build 3.13 alpha 3 compatibility, misc

.github/workflows/debug.yaml

@@ -10,11 +10,12 @@ jobs:
         profile: [
           { rust: "1.65", features: "" },
           { rust: "1.65", features: "--features=yyjson" },
-          { rust: "nightly-2024-01-17", features: "--features=yyjson,unstable-simd" },
+          { rust: "nightly-2024-02-01", features: "--features=yyjson,unstable-simd" },
         ]
         python: [
-          { version: '3.12', abi: 'cp312-cp312' },
-          { version: '3.8', abi: 'cp38-cp38' },
+          { version: '3.13' },
+          { version: '3.12' },
+          { version: '3.8' },
         ]
     env:
       CC: "gcc"
@@ -26,7 +27,9 @@ jobs:
 
     - uses: actions/setup-python@v5
       with:
+        allow-prereleases: true
         python-version: '${{ matrix.python.version }}'
+
     - run: python -m pip install --user --upgrade pip "maturin>=1,<2" wheel
 
     - uses: actions/checkout@v4
@@ -43,9 +46,15 @@ jobs:
     - run: python -m pip install --user -r test/requirements.txt -r integration/requirements.txt
 
     - run: pytest -s -rxX -v test
+      timeout-minutes: 4
       env:
         PYTHONMALLOC: "debug"
 
     - run: ./integration/run thread
+      timeout-minutes: 2
+
     - run: ./integration/run http
+      timeout-minutes: 2
+
     - run: ./integration/run init
+      timeout-minutes: 2

.github/workflows/linux.yaml

@@ -71,8 +71,8 @@ jobs:
       options: --user 0
     steps:
     - run: yum install -y clang lld
-    - run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2024-01-17 --profile minimal -y
-    - run: rustup component add rust-src --toolchain nightly-2024-01-17-x86_64-unknown-linux-gnu
+    - run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2024-02-01 --profile minimal -y
+    - run: rustup component add rust-src --toolchain nightly-2024-02-01-x86_64-unknown-linux-gnu
     - uses: actions/checkout@v4
 
     - name: build-std
@@ -143,11 +143,11 @@ jobs:
         RUSTFLAGS: "-C target-feature=-crt-static"
         CARGO_UNSTABLE_SPARSE_REGISTRY: "true"
       with:
-        rust-toolchain: nightly-2024-01-17
+        rust-toolchain: nightly-2024-02-01
         rustup-components: rust-src
         target: ${{ matrix.platform.target }}
         manylinux: musllinux_1_1
-        args: --release --strip --out=dist --features=no-panic,yyjson -i python${{ matrix.python.version }}
+        args: --release --strip --out=dist --features=encoding_rs/simd-accel,no-panic,unstable-simd,yyjson -i python${{ matrix.python.version }}
 
     - name: Set up QEMU
       if: matrix.platform.arch != 'x86_64'
@@ -196,28 +196,28 @@ jobs:
             arch: 'aarch64',
             cflags: '-O2 -flto',
             features: 'encoding_rs/simd-accel,no-panic,unstable-simd,yyjson',
-            rustflags: '-Z mir-opt-level=4 -D warnings',
+            rustflags: '-D warnings',
             target: 'aarch64-unknown-linux-gnu',
           },
           {
             arch: 'armv7',
             cflags: '-Os -flto -fstrict-aliasing',
             features: 'no-panic,yyjson', # no SIMD
-            rustflags: '-C opt-level=s -Z mir-opt-level=4 -D warnings',
+            rustflags: '-C opt-level=s -D warnings',
             target: 'armv7-unknown-linux-gnueabihf',
           },
           {
             arch: 'ppc64le',
             cflags: '-O2 -flto',
             features: 'no-panic,unstable-simd,yyjson',
-            rustflags: '-Z mir-opt-level=4 -D warnings',
+            rustflags: '-D warnings',
             target: 'powerpc64le-unknown-linux-gnu',
           },
           {
             arch: 's390x',
             cflags: '-O2 -flto -march=z10',
             features: 'no-panic,unstable-simd,yyjson',
-            rustflags: '-Z mir-opt-level=4 -C target-cpu=z10 -D warnings',
+            rustflags: '-C target-cpu=z10 -D warnings',
             target: 's390x-unknown-linux-gnu',
           },
         ]
@@ -238,7 +238,7 @@ jobs:
         RUSTFLAGS: "${{ matrix.target.rustflags }}"
       with:
         target: ${{ matrix.target.target }}
-        rust-toolchain: nightly-2024-01-17
+        rust-toolchain: nightly-2024-02-01
         rustup-components: rust-src
         manylinux: auto
         args: --release --strip --out=dist --features=${{ matrix.target.features }} -i python${{ matrix.python.version }}

.gitignore

@@ -10,3 +10,5 @@
 /target
 /vendor
 __pycache__
+corpus
+data/yelp

Cargo.toml

@@ -14,13 +14,13 @@ keywords = ["fast", "json", "dataclass", "dataclasses", "datetime", "rfc", "8259
 include = [
     "Cargo.toml",
     "CHANGELOG.md",
-    "data/*",
+    "data",
     "include",
     "LICENSE-APACHE",
     "LICENSE-MIT",
     "pyproject.toml",
     "README.md",
-    "src/*",
+    "src",
     "test/*.py",
     "test/requirements.txt",
 ]

README.md

@@ -1196,7 +1196,7 @@ It benefits from also having a C build environment to compile a faster
 deserialization backend. See this project's `manylinux_2_28` builds for an
 example using clang and LTO.
 
-The project's own CI tests against `nightly-2024-01-17` and stable 1.65. It
+The project's own CI tests against `nightly-2024-02-01` and stable 1.65. It
 is prudent to pin the nightly version because that channel can introduce
 breaking changes.
 

ci/azure-macos.yml

@@ -30,7 +30,7 @@ steps:
     LDFLAGS: "-O2 -fstrict-aliasing -flto=full -Wl,--as-needed"
     CFLAGS_x86_64_apple_darwin: "-O2 -fstrict-aliasing -flto=full -march=x86-64-v2 -mtune=generic"
     CFLAGS_aarch64_apple_darwin: "-O2 -fstrict-aliasing -flto=full -mcpu=apple-m1 -mtune=generic"
-    RUSTFLAGS: "-C lto=fat -Z mir-opt-level=4 -Z virtual-function-elimination -D warnings"
+    RUSTFLAGS: "-C lto=fat -Z virtual-function-elimination -D warnings"
     CARGO_UNSTABLE_SPARSE_REGISTRY: "true"
   displayName: build universal2
 

ci/azure-pipelines.yml

@@ -1,5 +1,5 @@
 variables:
-  toolchain: nightly-2024-01-17
+  toolchain: nightly-2024-02-01
 
 jobs:
 

integration/run

@@ -29,3 +29,7 @@ fi
 if [[ $to_run == *"init"* ]]; then
 	"${_dir}"/init
 fi
+
+if [[ $to_run == *"yelp"* ]]; then
+	"${_dir}"/yelp.py
+fi

integration/yelp.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import sys
+from pathlib import Path
+
+import orjson
+
+# https://www.yelp.com/dataset, ~8.6GiB
+FILES = {
+    "yelp_academic_dataset_business.json",
+    "yelp_academic_dataset_checkin.json",
+    "yelp_academic_dataset_review.json",
+    "yelp_academic_dataset_tip.json",
+    "yelp_academic_dataset_user.json",
+}
+
+for filename in FILES:
+    message = f"Processing {filename} ..."
+    sys.stdout.buffer.write(f"{message}".encode("ascii"))
+    count = 0
+    data = Path(f"data/yelp/{filename}").read_bytes()
+    for line in data.split(b"\n"):
+        if not line:
+            continue
+        count += 1
+        deserialized = orjson.loads(line)
+        assert orjson.loads(orjson.dumps(deserialized)) == deserialized
+        if count % 100 == 0:
+            sys.stdout.buffer.write(f"\r{message} {count} entries".encode("ascii"))
+
+    sys.stdout.buffer.write(f"\r{message} {count} entries\n".encode("ascii"))

script/lint

@@ -4,7 +4,7 @@ set -eou pipefail
 
 to_lint="./bench/*.py ./pysrc/orjson/__init__.pyi ./test/*.py script/pydataclass script/pymem
 script/pysort script/pynumpy script/pynonstr script/pycorrectness script/graph integration/init
-integration/wsgi.py integration/typestubs.py integration/thread"
+integration/wsgi.py integration/typestubs.py integration/thread integration/yelp.py"
 
 ruff ${to_lint} --fix
 ruff format ${to_lint}

src/util.rs

@@ -238,12 +238,20 @@ macro_rules! use_immortal {
     };
 }
 
+#[cfg(not(Py_3_13))]
 macro_rules! pydict_next {
     ($obj1:expr, $obj2:expr, $obj3:expr, $obj4:expr) => {
         unsafe { pyo3_ffi::_PyDict_Next($obj1, $obj2, $obj3, $obj4, std::ptr::null_mut()) }
     };
 }
 
+#[cfg(Py_3_13)]
+macro_rules! pydict_next {
+    ($obj1:expr, $obj2:expr, $obj3:expr, $obj4:expr) => {
+        unsafe { pyo3_ffi::PyDict_Next($obj1, $obj2, $obj3, $obj4) }
+    };
+}
+
 macro_rules! reserve_minimum {
     ($writer:expr) => {
         $writer.reserve(64);