From 5b47d1734dd0869a021f8e46a8656221cb2ef391 Mon Sep 17 00:00:00 2001 From: ijl Date: Tue, 26 Mar 2024 12:59:16 +0000 Subject: [PATCH] cargo update, build misc --- .github/workflows/debug.yaml | 5 +-- .github/workflows/linux.yaml | 59 ++++++++++++++++++------------------ Cargo.lock | 44 +++++++++++++-------------- Cargo.toml | 6 ++-- README.md | 2 +- ci/azure-macos.yml | 5 +-- ci/azure-pipelines.yml | 2 +- ci/azure-win.yml | 6 +++- script/develop | 9 ++++-- script/lint | 2 +- script/pytest | 2 +- test/requirements.txt | 2 ++ test/test_type.py | 2 +- test/test_ujson.py | 2 +- 14 files changed, 79 insertions(+), 69 deletions(-) diff --git a/.github/workflows/debug.yaml b/.github/workflows/debug.yaml index 30deba90..20b71db6 100644 --- a/.github/workflows/debug.yaml +++ b/.github/workflows/debug.yaml @@ -10,7 +10,7 @@ jobs: profile: [ { rust: "1.72", features: "" }, { rust: "1.72", features: "--features=yyjson" }, - { rust: "nightly-2024-02-13", features: "--features=yyjson,unstable-simd" }, + { rust: "nightly-2024-03-25", features: "--features=yyjson,unstable-simd" }, ] python: [ { version: '3.13' }, @@ -19,9 +19,10 @@ jobs: ] env: CC: "gcc" - CFLAGS: "-O2" + CFLAGS: "-Os -fstrict-aliasing" LDFLAGS: "-Wl,--as-needed" CARGO_UNSTABLE_SPARSE_REGISTRY: "true" + UNSAFE_PYO3_SKIP_VERSION_CHECK: "1" steps: - run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain ${{ matrix.profile.rust }} --profile minimal -y diff --git a/.github/workflows/linux.yaml b/.github/workflows/linux.yaml index 4f7340a2..117b0b1d 100644 --- a/.github/workflows/linux.yaml +++ b/.github/workflows/linux.yaml @@ -30,7 +30,7 @@ jobs: - run: python3 -m pip install --user -r test/requirements.txt -r integration/requirements.txt mypy - - run: pytest -s -rxX -v test + - run: pytest -s -rxX -v -n 4 test env: PYTHONMALLOC: "debug" @@ -41,7 +41,7 @@ jobs: - name: Store sdist if: "startsWith(github.ref, 'refs/tags/')" - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: wheels path: dist @@ -62,17 +62,18 @@ jobs: env: PATH: /github/home/.local/bin:/github/home/.cargo/bin:/opt/python/${{ matrix.python.abi }}/bin:/opt/rh/gcc-toolset-12/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin CC: "clang" - CFLAGS: "-O2 -fstrict-aliasing -emit-llvm -flto=full" - LDFLAGS: "-O2 -fstrict-aliasing -emit-llvm -flto=full -fuse-ld=lld -Wl,--as-needed -Wl,-plugin-opt=also-emit-llvm" - RUSTFLAGS: "-C linker=clang -C linker-plugin-lto -C lto=fat -C link-arg=-fuse-ld=lld -Z mir-opt-level=4 -Z virtual-function-elimination -D warnings" + CFLAGS: "-Os -fstrict-aliasing -flto=full" + LDFLAGS: "-fuse-ld=lld -Wl,--as-needed" + RUSTFLAGS: "-C linker=clang -C lto=fat -C link-arg=-fuse-ld=lld -Z mir-opt-level=4 -Z virtual-function-elimination -D warnings" CARGO_UNSTABLE_SPARSE_REGISTRY: "true" + UNSAFE_PYO3_SKIP_VERSION_CHECK: "1" container: image: quay.io/pypa/manylinux_2_28_x86_64:latest options: --user 0 steps: - run: yum install -y clang lld - - run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2024-02-13 --profile minimal -y - - run: rustup component add rust-src --toolchain nightly-2024-02-13-x86_64-unknown-linux-gnu + - run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2024-03-25 --profile minimal -y + - run: rustup component add rust-src --toolchain nightly-2024-03-25-x86_64-unknown-linux-gnu - uses: actions/checkout@v4 - name: build-std @@ -91,7 +92,7 @@ jobs: - run: python3 -m pip install --user dist/orjson*.whl - run: python3 -m pip install --user -r test/requirements.txt -r integration/requirements.txt - - run: pytest -s -rxX -v test + - run: pytest -s -rxX -v -n 4 test env: PYTHONMALLOC: "debug" @@ -101,7 +102,7 @@ jobs: - name: Store wheels if: "startsWith(github.ref, 'refs/tags/')" - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: wheels path: dist @@ -138,12 +139,13 @@ jobs: uses: PyO3/maturin-action@v1 env: CC: "gcc" - CFLAGS: "-O2" - LDFLAGS: "-O2 -flto -Wl,--as-needed" - RUSTFLAGS: "-C target-feature=-crt-static -Z mir-opt-level=4" + CFLAGS: "-Os -flto=full" + LDFLAGS: "-Wl,--as-needed" + RUSTFLAGS: "-Z mir-opt-level=4 -C lto=fat -D warnings -C target-feature=-crt-static" CARGO_UNSTABLE_SPARSE_REGISTRY: "true" + UNSAFE_PYO3_SKIP_VERSION_CHECK: "1" with: - rust-toolchain: nightly-2024-02-13 + rust-toolchain: nightly-2024-03-25 rustup-components: rust-src target: ${{ matrix.platform.target }} manylinux: musllinux_1_2 @@ -174,7 +176,7 @@ jobs: - name: Store wheels if: "startsWith(github.ref, 'refs/tags/')" - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: wheels path: dist @@ -195,31 +197,31 @@ jobs: target: [ { arch: 'aarch64', - cflags: '-O2 -flto', + cflags: '-Os -flto=full -fstrict-aliasing', features: 'no-panic,unstable-simd,yyjson', - rustflags: '-Z mir-opt-level=4 -D warnings', + rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings', target: 'aarch64-unknown-linux-gnu', }, { arch: 'armv7', - cflags: '-Os -flto -fstrict-aliasing', + cflags: '-Os -flto=full -fstrict-aliasing', features: 'no-panic,yyjson', # no SIMD - rustflags: '-C opt-level=s -Z mir-opt-level=4 -D warnings', + rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings -C opt-level=s', target: 'armv7-unknown-linux-gnueabihf', }, { arch: 'ppc64le', - cflags: '-O2 -flto', + cflags: '-Os -flto=full -fstrict-aliasing', features: 'no-panic,unstable-simd,yyjson', - rustflags: '-Z mir-opt-level=4 -D warnings', + rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings', target: 'powerpc64le-unknown-linux-gnu', }, { arch: 's390x', - cflags: '-O2 -flto -march=z10', + cflags: '-Os -flto=full -fstrict-aliasing -march=z10', # no unstable-simd as rotate_elements_left() causes exit 125 in at least QEMU features: 'no-panic,yyjson', - rustflags: '-Z mir-opt-level=4 -C target-cpu=z10 -D warnings', + rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings -C target-cpu=z10', target: 's390x-unknown-linux-gnu', }, ] @@ -235,12 +237,13 @@ jobs: env: PYO3_CROSS_LIB_DIR: "/opt/python/${{ matrix.python.abi }}" CFLAGS: "${{ matrix.target.cflags }}" - LDFLAGS: "${{ matrix.target.cflags }} -Wl,--as-needed" + LDFLAGS: "-Wl,--as-needed" CARGO_UNSTABLE_SPARSE_REGISTRY: "true" RUSTFLAGS: "${{ matrix.target.rustflags }}" + UNSAFE_PYO3_SKIP_VERSION_CHECK: "1" with: target: ${{ matrix.target.target }} - rust-toolchain: nightly-2024-02-13 + rust-toolchain: nightly-2024-03-25 rustup-components: rust-src manylinux: auto args: --release --strip --out=dist --features=${{ matrix.target.features }} -i python${{ matrix.python.version }} @@ -268,7 +271,7 @@ jobs: - name: Store wheels if: "startsWith(github.ref, 'refs/tags/')" - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: wheels path: dist @@ -276,15 +279,11 @@ jobs: pypi: name: PyPI - permissions: - # trusted publisher - id-token: write - contents: write runs-on: ubuntu-22.04 if: "startsWith(github.ref, 'refs/tags/')" needs: [ manylinux_2_17_amd64, manylinux_2_17_non_amd64, musllinux_1_2, sdist ] steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: wheels - uses: actions/setup-python@v5 diff --git a/Cargo.lock b/Cargo.lock index 1d016a0e..0130e1f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "ahash" -version = "0.8.9" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d713b3834d76b85304d4d525563c1276e2e30dc97cc67bfb4585a4a29fc2c89f" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", "const-random", @@ -32,9 +32,9 @@ checksum = "b993cd767a2bc7307dd87622311ca22c44329cc7a21366206bfa0896827b2bad" [[package]] name = "autocfg" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" [[package]] name = "beef" @@ -62,9 +62,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.86" +version = "1.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f9fa1897e4325be0d68d48df6aa1a71ac2ed4d27723887e7754192705350730" +checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5" [[package]] name = "cfg-if" @@ -97,9 +97,9 @@ dependencies = [ [[package]] name = "const-random" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aaf16c9c2c612020bcfd042e170f6e32de9b9d75adb5277cdbbd2e2c8c8299a" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" dependencies = [ "const-random-macro", ] @@ -143,9 +143,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" dependencies = [ "no-panic", ] @@ -216,18 +216,18 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.78" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" +checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" dependencies = [ "unicode-ident", ] [[package]] name = "pyo3-build-config" -version = "0.20.2" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07426f0d8fe5a601f26293f300afd1a7b1ed5e78b2a705870c5f30893c5163be" +checksum = "a5eb0b6ecba38961f6f4bd6cd5906dfab3cd426ff37b2eed5771006aa31656f1" dependencies = [ "once_cell", "target-lexicon", @@ -235,9 +235,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.20.2" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb7dec17e17766b46bca4f1a4215a85006b4c2ecde122076c562dd058da6cf1" +checksum = "ba8a6e48a29b5d22e4fdaf132d8ba8d3203ee9f06362d48f244346902a594ec3" dependencies = [ "libc", "pyo3-build-config", @@ -289,9 +289,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.114" +version = "1.0.115" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" +checksum = "12dc5c46daa8e9fdf4f5e71b6cf9a53f2487da0e86e55808e2d35539666497dd" dependencies = [ "itoa", "ryu", @@ -306,9 +306,9 @@ checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" [[package]] name = "smallvec" -version = "1.13.1" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "static_assertions" @@ -318,9 +318,9 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "syn" -version = "2.0.50" +version = "2.0.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74f1bdc9872430ce9b75da68329d1c1746faf50ffac5f19e02b71e37ff881ffb" +checksum = "002a1b3dbf967edfafc32655d0f377ab0bb7b994aa1d32c8cc7e9b8bf3ebb8f0" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 4ff9c037..b001eb4c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,13 +50,13 @@ arrayvec = { version = "0.7", default_features = false, features = ["std", "serd associative-cache = { version = "2", default_features = false } beef = { version = "0.5", default_features = false, features = ["impl_serde"] } bytecount = { version = "^0.6.7", default_features = false, features = ["runtime-dispatch-simd"] } -chrono = { version = "^0.4.31", default_features = false } +chrono = { version = "=0.4.34", default_features = false } compact_str = { version = "0.7", default_features = false, features = ["serde"] } encoding_rs = { version = "0.8", default_features = false } itoa = { version = "1", default_features = false } itoap = { version = "1", features = ["std", "simd"] } once_cell = { version = "1", default_features = false, features = ["race"] } -pyo3-ffi = { version = "^0.20.2", default_features = false, features = ["extension-module"]} +pyo3-ffi = { version = "^0.21", default_features = false, features = ["extension-module"]} ryu = { version = "1", default_features = false } serde = { version = "1", default_features = false } serde_json = { version = "1", default_features = false, features = ["std", "float_roundtrip"] } @@ -65,7 +65,7 @@ smallvec = { version = "^1.11", default_features = false, features = ["union", " [build-dependencies] cc = { version = "1" } -pyo3-build-config = { version = "^0.20.2" } +pyo3-build-config = { version = "^0.21" } version_check = { version = "0.9" } [profile.dev] diff --git a/README.md b/README.md index 167518a9..6d1f0644 100644 --- a/README.md +++ b/README.md @@ -1199,7 +1199,7 @@ It benefits from also having a C build environment to compile a faster deserialization backend. See this project's `manylinux_2_28` builds for an example using clang and LTO. -The project's own CI tests against `nightly-2024-02-13` and stable 1.65. It +The project's own CI tests against `nightly-2024-03-25` and stable 1.65. It is prudent to pin the nightly version because that channel can introduce breaking changes. diff --git a/ci/azure-macos.yml b/ci/azure-macos.yml index 2e1b65d2..464c115b 100644 --- a/ci/azure-macos.yml +++ b/ci/azure-macos.yml @@ -26,12 +26,13 @@ steps: maturin build --release --strip --features=no-panic,unstable-simd,yyjson --interpreter $(interpreter) --target=universal2-apple-darwin env: CC: "clang" - CFLAGS: "-O2 -fstrict-aliasing -flto=full" - LDFLAGS: "-O2 -fstrict-aliasing -flto=full -Wl,--as-needed" + CFLAGS: "-Os -fstrict-aliasing -flto=full" + LDFLAGS: "-Wl,--as-needed" CFLAGS_x86_64_apple_darwin: "-O2 -fstrict-aliasing -flto=full -march=x86-64-v2 -mtune=generic" CFLAGS_aarch64_apple_darwin: "-O2 -fstrict-aliasing -flto=full -mcpu=apple-m1 -mtune=generic" RUSTFLAGS: "-C lto=fat -Z mir-opt-level=4 -Z virtual-function-elimination -D warnings" CARGO_UNSTABLE_SPARSE_REGISTRY: "true" + UNSAFE_PYO3_SKIP_VERSION_CHECK: "1" displayName: build universal2 - bash: pip install target/wheels/orjson*universal2.whl diff --git a/ci/azure-pipelines.yml b/ci/azure-pipelines.yml index 352d7569..440e3cd7 100644 --- a/ci/azure-pipelines.yml +++ b/ci/azure-pipelines.yml @@ -1,5 +1,5 @@ variables: - toolchain: nightly-2024-02-13 + toolchain: nightly-2024-03-25 jobs: diff --git a/ci/azure-win.yml b/ci/azure-win.yml index 19204451..218973fd 100644 --- a/ci/azure-win.yml +++ b/ci/azure-win.yml @@ -22,7 +22,11 @@ steps: - script: maturin.exe build --release --features=no-panic,unstable-simd,yyjson --strip --interpreter $(interpreter) --target $(target) displayName: build env: - RUSTFLAGS: "-Z mir-opt-level=4" + CFLAGS: "-Os -fstrict-aliasing -flto=full" + LDFLAGS: "-Wl,--as-needed" + RUSTFLAGS: "-C lto=fat -Z mir-opt-level=4 -Z virtual-function-elimination -D warnings" + CARGO_UNSTABLE_SPARSE_REGISTRY: "true" + UNSAFE_PYO3_SKIP_VERSION_CHECK: "1" - script: python.exe -m pip install orjson --no-index --find-links=D:\a\1\s\target\wheels displayName: install - script: python.exe -m pytest -s -rxX -v test diff --git a/script/develop b/script/develop index 33036719..3a4042e4 100755 --- a/script/develop +++ b/script/develop @@ -2,13 +2,16 @@ rm -f target/wheels/* +export UNSAFE_PYO3_SKIP_VERSION_CHECK=1 + export CC="${CC:-clang}" export LD="${LD:-lld}" echo "CC: ${CC}, LD: ${LD}, LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}" -export CFLAGS="-Os -fstrict-aliasing -emit-llvm -flto=full" -export LDFLAGS="${CFLAGS} -fuse-ld=${LD} -Wl,--as-needed -Wl,-plugin-opt=also-emit-llvm" -export RUSTFLAGS="-C linker=${CC} -C linker-plugin-lto -C lto=fat -C link-arg=-fuse-ld=${LD} -Z mir-opt-level=4 -Z virtual-function-elimination" + +export CFLAGS="-Os -fstrict-aliasing -flto=full" +export LDFLAGS="-fuse-ld=${LD} -Wl,--as-needed" +export RUSTFLAGS="-C linker=${CC} -C lto=fat -C link-arg=-fuse-ld=${LD} -Z mir-opt-level=4 -Z virtual-function-elimination -Z threads=8" maturin build "$@" diff --git a/script/lint b/script/lint index 15eed6ec..22f5742e 100755 --- a/script/lint +++ b/script/lint @@ -6,6 +6,6 @@ to_lint="./bench/*.py ./pysrc/orjson/__init__.pyi ./test/*.py script/pydataclass script/pysort script/pynumpy script/pynonstr script/pycorrectness script/graph integration/init integration/wsgi.py integration/typestubs.py integration/thread" -ruff ${to_lint} --fix +ruff check ${to_lint} --fix ruff format ${to_lint} mypy --ignore-missing-imports --check-untyped-defs ./bench/*.py ./pysrc/orjson/__init__.pyi ./test/*.py diff --git a/script/pytest b/script/pytest index 4d25dbe4..6b95014c 100755 --- a/script/pytest +++ b/script/pytest @@ -1,3 +1,3 @@ #!/bin/sh -e -PYTHONMALLOC="debug" pytest -s -rxX --random-order test +PYTHONMALLOC="debug" pytest -s -rxX --random-order -n 4 test diff --git a/test/requirements.txt b/test/requirements.txt index 90796a1d..2efb3e8a 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -4,6 +4,8 @@ numpy;(platform_machine=="x86_64" or (platform_machine=="aarch64" and sys_platfo pendulum;sys_platform=="linux" and platform_machine=="x86_64" and python_version<"3.12" psutil;(sys_platform=="linux" or sys_platform == "macos") and platform_machine=="x86_64" pytest +pytest-random-order +pytest-xdist pytz typing_extensions;python_version<"3.8" xxhash==1.4.3;sys_platform=="linux" and platform_machine=="x86_64" and python_version<"3.9" # creates non-compact ASCII for test_str_ascii diff --git a/test/test_type.py b/test/test_type.py index a2969ca8..12575297 100644 --- a/test/test_type.py +++ b/test/test_type.py @@ -75,7 +75,7 @@ def test_str_ascii_control(self): """ worst case format_escaped_str_with_escapes() allocation """ - ref = "\x01\x1F" * 1024 * 16 + ref = "\x01\x1f" * 1024 * 16 assert orjson.loads(orjson.dumps(ref)) == ref assert orjson.loads(orjson.dumps(ref, option=orjson.OPT_INDENT_2)) == ref diff --git a/test/test_ujson.py b/test/test_ujson.py index 81ba8d6b..4fdda0c7 100644 --- a/test/test_ujson.py +++ b/test/test_ujson.py @@ -128,7 +128,7 @@ def test_encodeUnicode4BytesUTF8Highest(self): assert dec == orjson.loads(enc) def testEncodeUnicodeBMP(self): - s = "\U0001f42e\U0001f42e\U0001F42D\U0001F42D" # 🐮🐮🐭🐭 + s = "\U0001f42e\U0001f42e\U0001f42d\U0001f42d" # 🐮🐮🐭🐭 orjson.dumps(s) json.dumps(s)