From b6b21b0366728f5d372ec68aa6f17755596f7351 Mon Sep 17 00:00:00 2001 From: ijl Date: Mon, 15 Apr 2024 21:28:58 +0000 Subject: [PATCH] manylinux_2_17, macos_universal2 CI --- .../workflows/{linux.yaml => artifact.yaml} | 226 +++++++++++++++--- .github/workflows/debug.yaml | 2 +- .github/workflows/lint.yaml | 2 +- Cargo.lock | 40 ++-- README.md | 2 +- bench/requirements.txt | 4 +- build.rs | 8 + ci/azure-macos.yml | 53 ---- ci/azure-pipelines.yml | 74 +----- script/develop | 4 +- script/graph | 5 +- src/deserialize/cache.rs | 6 +- src/deserialize/yyjson.rs | 4 +- src/serialize/per_type/dataclass.rs | 4 +- src/serialize/per_type/dict.rs | 6 +- src/serialize/per_type/list.rs | 2 +- src/serialize/per_type/numpy.rs | 2 +- src/util.rs | 10 + 18 files changed, 250 insertions(+), 204 deletions(-) rename .github/workflows/{linux.yaml => artifact.yaml} (54%) delete mode 100644 ci/azure-macos.yml diff --git a/.github/workflows/linux.yaml b/.github/workflows/artifact.yaml similarity index 54% rename from .github/workflows/linux.yaml rename to .github/workflows/artifact.yaml index 1612e435..9b0e9b08 100644 --- a/.github/workflows/linux.yaml +++ b/.github/workflows/artifact.yaml @@ -1,5 +1,9 @@ -name: linux +name: artifact on: push +env: + RUST_TOOLCHAIN: "nightly-2024-04-15" + CARGO_UNSTABLE_SPARSE_REGISTRY: "true" + UNSAFE_PYO3_SKIP_VERSION_CHECK: "1" jobs: sdist: @@ -9,18 +13,22 @@ jobs: env: RUST_TOOLCHAIN: "1.72" # MSRV steps: - - run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain $RUST_TOOLCHAIN -y - - run: rustup default $RUST_TOOLCHAIN + - name: rustup stable + run: | + curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain "${RUST_TOOLCHAIN}" -y + rustup default "${RUST_TOOLCHAIN}" - uses: actions/checkout@v4 - run: python3 -m pip install --user --upgrade pip "maturin>=1,<2" wheel - - run: maturin build - - run: cargo fetch - - run: mkdir .cargo - - run: cp ci/sdist.toml .cargo/config.toml - - run: cargo vendor include/cargo --versioned-dirs + - name: Vendor dependencies + run: | + maturin build + cargo fetch + mkdir .cargo + cp ci/sdist.toml .cargo/config.toml + cargo vendor include/cargo --versioned-dirs - run: maturin sdist --out=dist @@ -61,37 +69,40 @@ jobs: { version: '3.8', abi: 'cp38-cp38' }, ] env: - PATH: /github/home/.local/bin:/github/home/.cargo/bin:/opt/python/${{ matrix.python.abi }}/bin:/opt/rh/gcc-toolset-12/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin CC: "clang" CFLAGS: "-Os -fstrict-aliasing -flto=full" LDFLAGS: "-fuse-ld=lld -Wl,--as-needed" - RUSTFLAGS: "-C linker=clang -C lto=fat -C link-arg=-fuse-ld=lld -Z mir-opt-level=4 -Z virtual-function-elimination -D warnings" - CARGO_UNSTABLE_SPARSE_REGISTRY: "true" - UNSAFE_PYO3_SKIP_VERSION_CHECK: "1" + RUSTFLAGS: "-C linker=clang -C lto=fat -C link-arg=-fuse-ld=lld -Z mir-opt-level=4 -Z virtual-function-elimination -Z threads=4 -D warnings" + PATH: "/__w/orjson/orjson/.venv/bin:/github/home/.cargo/bin:/root/.local/bin:/root/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" container: - image: quay.io/pypa/manylinux_2_28_x86_64:latest + image: fedora:41 options: --user 0 steps: - - run: yum install -y clang lld - - run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2024-03-27 --profile minimal -y - - run: rustup component add rust-src --toolchain nightly-2024-03-27-x86_64-unknown-linux-gnu + - uses: actions/checkout@v4 - - name: build-std + - name: Build environment run: | + dnf install -y rustup clang lld python${{ matrix.python.version }} + + rustup-init --default-toolchain "${RUST_TOOLCHAIN}-x86_64-unknown-linux-gnu" --profile minimal --component rust-src -y + cargo fetch --target=x86_64-unknown-linux-gnu & + + curl -LsSf https://astral.sh/uv/install.sh | sh + uv venv --python python${{ matrix.python.version }} + uv pip install --upgrade "maturin>=1,<2" -r test/requirements.txt -r integration/requirements.txt + mkdir .cargo cp ci/config.toml .cargo/config.toml - - run: python3 -m pip install --user --upgrade pip "maturin>=1,<2" wheel - - run: | + - name: maturin + run: | maturin build --release --strip \ - --out=dist \ --features=no-panic,unstable-simd,yyjson \ --compatibility manylinux_2_17 \ --interpreter python${{ matrix.python.version }} \ --target=x86_64-unknown-linux-gnu - - run: python3 -m pip install --user dist/orjson*.whl - - run: python3 -m pip install --user -r test/requirements.txt -r integration/requirements.txt + uv pip install target/wheels/orjson*.whl - run: pytest -s -rxX -v -n 4 test env: @@ -106,7 +117,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: orjson_manylinux_2_17_amd64_${{ matrix.python.version }} - path: dist + path: target/wheels overwrite: true retention-days: 1 @@ -143,21 +154,19 @@ jobs: CC: "gcc" CFLAGS: "-Os" LDFLAGS: "-Wl,--as-needed" - RUSTFLAGS: "-Z mir-opt-level=4 -D warnings -C target-feature=-crt-static" - CARGO_UNSTABLE_SPARSE_REGISTRY: "true" - UNSAFE_PYO3_SKIP_VERSION_CHECK: "1" + RUSTFLAGS: "-Z mir-opt-level=4 -Z threads=4 -D warnings -C target-feature=-crt-static" with: - rust-toolchain: nightly-2024-03-27 + rust-toolchain: nightly-2024-04-15 rustup-components: rust-src target: ${{ matrix.platform.target }} manylinux: musllinux_1_2 args: --release --strip --out=dist --features=no-panic,unstable-simd,yyjson -i python${{ matrix.python.version }} - - name: Set up QEMU + - name: QEMU if: matrix.platform.arch != 'x86_64' uses: docker/setup-qemu-action@v3 with: - image: tonistiigi/binfmt:qemu-v8.1.4 + image: tonistiigi/binfmt:qemu-v8.1.5 platforms: ${{ matrix.platform.platform }} - name: Test @@ -174,7 +183,7 @@ jobs: venv/bin/pip install -U pip wheel venv/bin/pip install -r test/requirements.txt venv/bin/pip install orjson --no-index --find-links dist/ --force-reinstall - venv/bin/python -m pytest -s -rxX -v test + venv/bin/python -m pytest -s -rxX -v -n 4 test - name: Store wheels if: "startsWith(github.ref, 'refs/tags/')" @@ -222,7 +231,6 @@ jobs: { arch: 's390x', cflags: '-Os -flto=full -fstrict-aliasing -march=z10', - # no unstable-simd as rotate_elements_left() causes exit 125 in at least QEMU features: 'no-panic,yyjson', rustflags: '-Z mir-opt-level=4 -C lto=fat -D warnings -C target-cpu=z10', target: 's390x-unknown-linux-gnu', @@ -241,12 +249,10 @@ jobs: PYO3_CROSS_LIB_DIR: "/opt/python/${{ matrix.python.abi }}" CFLAGS: "${{ matrix.target.cflags }}" LDFLAGS: "-Wl,--as-needed" - CARGO_UNSTABLE_SPARSE_REGISTRY: "true" RUSTFLAGS: "${{ matrix.target.rustflags }}" - UNSAFE_PYO3_SKIP_VERSION_CHECK: "1" with: target: ${{ matrix.target.target }} - rust-toolchain: nightly-2024-03-27 + rust-toolchain: nightly-2024-04-15 rustup-components: rust-src manylinux: auto args: --release --strip --out=dist --features=${{ matrix.target.features }} -i python${{ matrix.python.version }} @@ -281,11 +287,159 @@ jobs: overwrite: true retention-days: 1 + macos_universal2_aarch64: + runs-on: macos-14 + strategy: + fail-fast: false + matrix: + python: [ + { version: '3.12', macosx_target: "10.15" }, + { version: '3.11', macosx_target: "10.15" }, + { version: '3.10', macosx_target: "10.15" }, + ] + env: + CC: "clang" + CFLAGS: "-Os -fstrict-aliasing -flto=full" + LDFLAGS: "-Wl,--as-needed" + CFLAGS_x86_64_apple_darwin: "-O2 -fstrict-aliasing -flto=full -march=x86-64-v2 -mtune=generic" + CFLAGS_aarch64_apple_darwin: "-O2 -fstrict-aliasing -flto=full -mcpu=apple-m1 -mtune=generic" + RUSTFLAGS: "-C lto=fat -Z mir-opt-level=4 -Z virtual-function-elimination -Z threads=3 -D warnings" + PATH: "/Users/runner/work/orjson/orjson/.venv/bin:/Users/runner/.cargo/bin:/usr/local/opt/curl/bin:/usr/local/bin:/usr/local/sbin:/Users/runner/bin:/Library/Frameworks/Python.framework/Versions/Current/bin:/usr/bin:/bin:/usr/sbin:/sbin" + steps: + + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "${{ matrix.python.version }}" + + - uses: dtolnay/rust-toolchain@master + with: + toolchain: "nightly-2024-04-15" + targets: "aarch64-apple-darwin, x86_64-apple-darwin" + components: "rust-src" + + - name: Build environment + run: | + cargo fetch --target aarch64-apple-darwin & + + curl -LsSf https://astral.sh/uv/install.sh | sh + uv venv --python python${{ matrix.python.version }} + uv pip install --upgrade "maturin>=1,<2" -r test/requirements.txt -r integration/requirements.txt + + mkdir .cargo + cp ci/config.toml .cargo/config.toml + + - name: maturin + run: | + PATH=$HOME/.cargo/bin:$PATH \ + MACOSX_DEPLOYMENT_TARGET="${{ matrix.python.macosx_target }}" \ + PYO3_CROSS_LIB_DIR=$(python -c "import sysconfig;print(sysconfig.get_config_var('LIBDIR'))") \ + maturin build --release --strip \ + --features=no-panic,unstable-simd,yyjson \ + --interpreter python${{ matrix.python.version }} \ + --target=universal2-apple-darwin + uv pip install target/wheels/orjson*.whl + + - run: pytest -s -rxX -v -n 3 test + env: + PYTHONMALLOC: "debug" + + - run: source .venv/bin/activate && ./integration/run thread + - run: source .venv/bin/activate && ./integration/run http + - run: source .venv/bin/activate && ./integration/run init + + - name: Store wheels + if: "startsWith(github.ref, 'refs/tags/')" + uses: actions/upload-artifact@v4 + with: + name: orjson_universal2_aarch64_${{ matrix.python.version }} + path: target/wheels + overwrite: true + retention-days: 1 + + macos_universal2_amd64: + runs-on: macos-13 + strategy: + fail-fast: false + matrix: + python: [ + { version: '3.9', macosx_target: "10.15" }, + { version: '3.8', macosx_target: "10.15" }, + ] + env: + CC: "clang" + CFLAGS: "-Os -fstrict-aliasing -flto=full" + LDFLAGS: "-Wl,--as-needed" + CFLAGS_x86_64_apple_darwin: "-O2 -fstrict-aliasing -flto=full -march=x86-64-v2 -mtune=generic" + CFLAGS_aarch64_apple_darwin: "-O2 -fstrict-aliasing -flto=full -mcpu=apple-m1 -mtune=generic" + RUSTFLAGS: "-C lto=fat -Z mir-opt-level=4 -Z virtual-function-elimination -Z threads=3 -D warnings" + PATH: "/Users/runner/work/orjson/orjson/.venv/bin:/Users/runner/.cargo/bin:/usr/local/opt/curl/bin:/usr/local/bin:/usr/local/sbin:/Users/runner/bin:/Library/Frameworks/Python.framework/Versions/Current/bin:/usr/bin:/bin:/usr/sbin:/sbin" + steps: + + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "${{ matrix.python.version }}" + + - uses: dtolnay/rust-toolchain@master + with: + toolchain: "nightly-2024-04-15" + targets: "aarch64-apple-darwin, x86_64-apple-darwin" + components: "rust-src" + + - name: Build environment + run: | + cargo fetch --target aarch64-apple-darwin & + + curl -LsSf https://astral.sh/uv/install.sh | sh + uv venv --python python${{ matrix.python.version }} + uv pip install --upgrade "maturin>=1,<2" -r test/requirements.txt -r integration/requirements.txt + + mkdir .cargo + cp ci/config.toml .cargo/config.toml + + - name: maturin + run: | + PATH=$HOME/.cargo/bin:$PATH \ + MACOSX_DEPLOYMENT_TARGET="${{ matrix.python.macosx_target }}" \ + PYO3_CROSS_LIB_DIR=$(python -c "import sysconfig;print(sysconfig.get_config_var('LIBDIR'))") \ + maturin build --release --strip \ + --features=no-panic,unstable-simd,yyjson \ + --interpreter python${{ matrix.python.version }} \ + --target=universal2-apple-darwin + uv pip install target/wheels/orjson*.whl + + - run: pytest -s -rxX -v -n 3 test + env: + PYTHONMALLOC: "debug" + + - run: source .venv/bin/activate && ./integration/run thread + - run: source .venv/bin/activate && ./integration/run http + - run: source .venv/bin/activate && ./integration/run init + + - name: Store wheels + if: "startsWith(github.ref, 'refs/tags/')" + uses: actions/upload-artifact@v4 + with: + name: orjson_universal2_amd64_${{ matrix.python.version }} + path: target/wheels + overwrite: true + retention-days: 1 + pypi: name: PyPI runs-on: ubuntu-22.04 if: "startsWith(github.ref, 'refs/tags/')" - needs: [ manylinux_2_17_amd64, manylinux_2_17_non_amd64, musllinux_1_2, sdist ] + needs: [ + sdist, + manylinux_2_17_amd64, + musllinux_1_2, + manylinux_2_17_non_amd64, + macos_universal2_aarch64, + macos_universal2_amd64, + ] steps: - uses: actions/download-artifact@v4 with: @@ -293,7 +447,7 @@ jobs: merge-multiple: true - uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: "3.12" - run: pip install pip "maturin>=1,<2" - run: ls -1 . - name: deploy wheel diff --git a/.github/workflows/debug.yaml b/.github/workflows/debug.yaml index 42cb053f..d42079ca 100644 --- a/.github/workflows/debug.yaml +++ b/.github/workflows/debug.yaml @@ -10,7 +10,7 @@ jobs: profile: [ { rust: "1.72", features: "" }, { rust: "1.72", features: "--features=yyjson" }, - { rust: "nightly-2024-03-27", features: "--features=yyjson,unstable-simd" }, + { rust: "nightly-2024-04-15", features: "--features=yyjson,unstable-simd" }, ] python: [ { version: '3.13' }, diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index d7bbae4a..aaa5b77c 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -6,7 +6,7 @@ jobs: steps: - uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: "3.12" - run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain=stable --profile=default -y - run: pip install -U autoflake isort black ruff mypy types-python-dateutil types-pytz types-simplejson types-ujson diff --git a/Cargo.lock b/Cargo.lock index a396a873..3a8d89e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -62,9 +62,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.90" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5" +checksum = "17f6e324229dc011159fcc089755d1e2e216a90d43a7dea6853ca740b84f35e7" [[package]] name = "cfg-if" @@ -123,18 +123,18 @@ checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" [[package]] name = "encoding_rs" -version = "0.8.33" +version = "0.8.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" dependencies = [ "cfg-if", ] [[package]] name = "getrandom" -version = "0.2.12" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" +checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" dependencies = [ "cfg-if", "libc", @@ -143,9 +143,9 @@ dependencies = [ [[package]] name = "half" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5eceaaeec696539ddaf7b333340f1af35a5aa87ae3e4f3ead0532f72affab2e" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" dependencies = [ "cfg-if", "crunchy", @@ -227,18 +227,18 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.79" +version = "1.0.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" +checksum = "a56dea16b0a29e94408b9aa5e2940a4eedbd128a1ba20e8f7ae60fd3d465af0e" dependencies = [ "unicode-ident", ] [[package]] name = "pyo3-build-config" -version = "0.21.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5eb0b6ecba38961f6f4bd6cd5906dfab3cd426ff37b2eed5771006aa31656f1" +checksum = "650dca34d463b6cdbdb02b1d71bfd6eb6b6816afc708faebb3bac1380ff4aef7" dependencies = [ "once_cell", "target-lexicon", @@ -246,9 +246,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.21.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba8a6e48a29b5d22e4fdaf132d8ba8d3203ee9f06362d48f244346902a594ec3" +checksum = "09a7da8fc04a8a2084909b59f29e1b8474decac98b951d77b80b26dc45f046ad" dependencies = [ "libc", "pyo3-build-config", @@ -256,18 +256,18 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.35" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] [[package]] name = "rustversion" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" +checksum = "80af6f9131f277a45a3fba6ce8e2258037bb0477a67e610d3c1fe046ab31de47" [[package]] name = "ryu" @@ -329,9 +329,9 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "syn" -version = "2.0.55" +version = "2.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "002a1b3dbf967edfafc32655d0f377ab0bb7b994aa1d32c8cc7e9b8bf3ebb8f0" +checksum = "4a6531ffc7b071655e4ce2e04bd464c4830bb585a61cabb96cf808f05172615a" dependencies = [ "proc-macro2", "quote", diff --git a/README.md b/README.md index c0906dd3..1bbcad6f 100644 --- a/README.md +++ b/README.md @@ -1199,7 +1199,7 @@ It benefits from also having a C build environment to compile a faster deserialization backend. See this project's `manylinux_2_28` builds for an example using clang and LTO. -The project's own CI tests against `nightly-2024-03-27` and stable 1.65. It +The project's own CI tests against `nightly-2024-04-15` and stable 1.72. It is prudent to pin the nightly version because that channel can introduce breaking changes. diff --git a/bench/requirements.txt b/bench/requirements.txt index 806f5b81..40d524ca 100644 --- a/bench/requirements.txt +++ b/bench/requirements.txt @@ -1,9 +1,9 @@ memory-profiler -pandas +pandas; python_version<"3.13" pytest-benchmark pytest-random-order python-rapidjson -seaborn +seaborn; python_version<"3.13" simplejson tabulate ujson diff --git a/build.rs b/build.rs index 329e2fbb..ee42945e 100644 --- a/build.rs +++ b/build.rs @@ -9,6 +9,7 @@ fn main() { println!("cargo:rerun-if-env-changed=CFLAGS"); println!("cargo:rerun-if-env-changed=LDFLAGS"); println!("cargo:rerun-if-env-changed=RUSTFLAGS"); + println!("cargo:rerun-if-env-changed=ORJSON_DISABLE_SIMD"); println!("cargo:rerun-if-env-changed=ORJSON_DISABLE_YYJSON"); for cfg in pyo3_build_config::get().build_script_outputs() { @@ -27,6 +28,13 @@ fn main() { println!("cargo:rustc-cfg=feature=\"strict_provenance\""); } + #[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] + if !env::var("ORJSON_DISABLE_SIMD").is_ok() { + if let Some(true) = version_check::supports_feature("portable_simd") { + println!("cargo:rustc-cfg=feature=\"unstable-simd\""); + } + } + if env::var("ORJSON_DISABLE_YYJSON").is_ok() { if env::var("CARGO_FEATURE_YYJSON").is_ok() { panic!("ORJSON_DISABLE_YYJSON and --features=yyjson both enabled.") diff --git a/ci/azure-macos.yml b/ci/azure-macos.yml deleted file mode 100644 index 464c115b..00000000 --- a/ci/azure-macos.yml +++ /dev/null @@ -1,53 +0,0 @@ -parameters: - interpreter: '' - macosx_deployment_target: '' - toolchain: '' - -steps: -- bash: | - curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain $(toolchain) --profile minimal -y - rustup component add rust-src --toolchain $(toolchain)-x86_64-apple-darwin - rustup target add aarch64-apple-darwin - rustup default $(toolchain) - -- bash: pip install --upgrade pip pip "maturin>=1,<2" wheel - displayName: build dependencies -- bash: pip install -r test/requirements.txt -r integration/requirements.txt - displayName: test dependencies - -- bash: | - mkdir .cargo - cp ci/config.toml .cargo/config - -- bash: | - PATH=$HOME/.cargo/bin:$PATH \ - MACOSX_DEPLOYMENT_TARGET=$(macosx_deployment_target) \ - PYO3_CROSS_LIB_DIR=$(python -c "import sysconfig;print(sysconfig.get_config_var('LIBDIR'))") \ - maturin build --release --strip --features=no-panic,unstable-simd,yyjson --interpreter $(interpreter) --target=universal2-apple-darwin - env: - CC: "clang" - CFLAGS: "-Os -fstrict-aliasing -flto=full" - LDFLAGS: "-Wl,--as-needed" - CFLAGS_x86_64_apple_darwin: "-O2 -fstrict-aliasing -flto=full -march=x86-64-v2 -mtune=generic" - CFLAGS_aarch64_apple_darwin: "-O2 -fstrict-aliasing -flto=full -mcpu=apple-m1 -mtune=generic" - RUSTFLAGS: "-C lto=fat -Z mir-opt-level=4 -Z virtual-function-elimination -D warnings" - CARGO_UNSTABLE_SPARSE_REGISTRY: "true" - UNSAFE_PYO3_SKIP_VERSION_CHECK: "1" - displayName: build universal2 - -- bash: pip install target/wheels/orjson*universal2.whl - displayName: install universal2 - -- bash: pytest -s -rxX -v test - env: - PYTHONMALLOC: "debug" - displayName: pytest -- bash: ./integration/run thread - displayName: thread -- bash: ./integration/run http - displayName: http -- bash: ./integration/run init - displayName: init - -- bash: ./ci/deploy target/wheels/*_universal2.whl - displayName: deploy universal2 diff --git a/ci/azure-pipelines.yml b/ci/azure-pipelines.yml index 69380b81..3ba78fe9 100644 --- a/ci/azure-pipelines.yml +++ b/ci/azure-pipelines.yml @@ -1,80 +1,8 @@ variables: - toolchain: nightly-2024-03-27 + toolchain: nightly-2024-04-15 jobs: -- job: macos_python312_univeral2 - pool: - vmImage: macOS-11 - variables: - interpreter: python3.12 - macosx_deployment_target: '10.15' - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.12.0' - addToPath: true - allowUnstable: true - - checkout: self - - template: ./azure-macos.yml - -- job: macos_python311_univeral2 - pool: - vmImage: macOS-11 - variables: - interpreter: python3.11 - macosx_deployment_target: '10.15' - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.11.4' - addToPath: true - allowUnstable: true - - checkout: self - - template: ./azure-macos.yml - -- job: macos_python310_univeral2 - pool: - vmImage: macOS-11 - variables: - interpreter: python3.10 - macosx_deployment_target: '10.15' - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.10' - addToPath: true - - checkout: self - - template: ./azure-macos.yml - -- job: macos_python39_univeral2 - pool: - vmImage: macOS-11 - variables: - interpreter: python3.9 - macosx_deployment_target: '10.15' - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.9' - addToPath: true - - checkout: self - - template: ./azure-macos.yml - -- job: macos_python38_univeral2 - pool: - vmImage: macOS-11 - variables: - interpreter: python3.8 - macosx_deployment_target: '10.15' - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.8' - addToPath: true - - checkout: self - - template: ./azure-macos.yml - - job: win_python312_amd64 pool: vmImage: windows-2022 diff --git a/script/develop b/script/develop index 3a4042e4..a4207166 100755 --- a/script/develop +++ b/script/develop @@ -9,10 +9,10 @@ export LD="${LD:-lld}" echo "CC: ${CC}, LD: ${LD}, LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}" -export CFLAGS="-Os -fstrict-aliasing -flto=full" +export CFLAGS="-Os -fstrict-aliasing -fno-plt -flto=full" export LDFLAGS="-fuse-ld=${LD} -Wl,--as-needed" export RUSTFLAGS="-C linker=${CC} -C lto=fat -C link-arg=-fuse-ld=${LD} -Z mir-opt-level=4 -Z virtual-function-elimination -Z threads=8" maturin build "$@" -pip install --force target/wheels/*.whl +uv pip install target/wheels/*.whl diff --git a/script/graph b/script/graph index afd45ce9..72bcdd95 100755 --- a/script/graph +++ b/script/graph @@ -78,7 +78,10 @@ def tab(obj): if isinstance(each[1], float) else None ) - each[1] = "%.1f" % each[1] if isinstance(each[1], float) else None + if group.startswith("github"): + each[1] = "%.2f" % each[1] if isinstance(each[1], float) else None + else: + each[1] = "%.1f" % each[1] if isinstance(each[1], float) else None buf.write(tabulate(table, headers, tablefmt="github") + "\n") diff --git a/src/deserialize/cache.rs b/src/deserialize/cache.rs index 169712ad..610377d3 100644 --- a/src/deserialize/cache.rs +++ b/src/deserialize/cache.rs @@ -40,11 +40,7 @@ pub static mut KEY_MAP: OnceCell = OnceCell::new(); pub fn cache_hash(key: &[u8]) -> u64 { // try to omit code for >64 path in ahash - debug_assert!(key.len() <= 64); - #[cfg(feature = "intrinsics")] - unsafe { - core::intrinsics::assume(key.len() <= 64); - }; + assume!(key.len() <= 64); let mut hasher = ahash::AHasher::default(); hasher.write(key); hasher.finish() diff --git a/src/deserialize/yyjson.rs b/src/deserialize/yyjson.rs index a05fa37d..090da550 100644 --- a/src/deserialize/yyjson.rs +++ b/src/deserialize/yyjson.rs @@ -146,7 +146,7 @@ fn parse_yy_array(elem: *mut yyjson_val) -> NonNull { return nonnull!(list); } let mut cur = unsafe_yyjson_get_first(elem); - for idx in 0..=len - 1 { + for idx in 0..len { let next = unsafe_yyjson_get_next(cur); let val = parse_node(cur).as_ptr(); ffi!(PyList_SET_ITEM(list, idx as isize, val)); @@ -165,7 +165,7 @@ fn parse_yy_object(elem: *mut yyjson_val) -> NonNull { } let mut key = unsafe_yyjson_get_first(elem); let dict = ffi!(_PyDict_NewPresized(len as isize)); - for _ in 0..=len - 1 { + for _ in 0..len { let val = key.add(1); let key_str = str_from_slice!((*key).uni.str_ as *const u8, unsafe_yyjson_get_len(key)); let pykey = get_unicode_key(key_str); diff --git a/src/serialize/per_type/dataclass.rs b/src/serialize/per_type/dataclass.rs index 5acf0937..f3c434b9 100644 --- a/src/serialize/per_type/dataclass.rs +++ b/src/serialize/per_type/dataclass.rs @@ -98,7 +98,7 @@ impl Serialize for DataclassFastSerializer { let mut pos = 0; pydict_next!(self.ptr, &mut pos, &mut next_key, &mut next_value); - for _ in 0..=ffi!(Py_SIZE(self.ptr)) as usize - 1 { + for _ in 0..ffi!(Py_SIZE(self.ptr)) as usize { let key = next_key; let value = next_value; @@ -163,7 +163,7 @@ impl Serialize for DataclassFallbackSerializer { let mut pos = 0; pydict_next!(fields, &mut pos, &mut next_key, &mut next_value); - for _ in 0..=ffi!(Py_SIZE(fields)) as usize - 1 { + for _ in 0..ffi!(Py_SIZE(fields)) as usize { let attr = next_key; let field = next_value; diff --git a/src/serialize/per_type/dict.rs b/src/serialize/per_type/dict.rs index 0fb6a19c..f79cbf60 100644 --- a/src/serialize/per_type/dict.rs +++ b/src/serialize/per_type/dict.rs @@ -111,7 +111,7 @@ impl Serialize for Dict { let mut pos = 0; pydict_next!(self.ptr, &mut pos, &mut next_key, &mut next_value); - for _ in 0..=ffi!(Py_SIZE(self.ptr)) as usize - 1 { + for _ in 0..ffi!(Py_SIZE(self.ptr)) as usize { let key = next_key; let value = next_value; @@ -210,7 +210,7 @@ impl Serialize for DictSortedKey { let mut pos = 0; pydict_next!(self.ptr, &mut pos, &mut next_key, &mut next_value); - for _ in 0..=len as usize - 1 { + for _ in 0..len as usize { let key = next_key; let value = next_value; @@ -377,7 +377,7 @@ impl Serialize for DictNonStrKey { let mut pos = 0; pydict_next!(self.ptr, &mut pos, &mut next_key, &mut next_value); - for _ in 0..=len - 1 { + for _ in 0..len { let key = next_key; let value = next_value; diff --git a/src/serialize/per_type/list.rs b/src/serialize/per_type/list.rs index 1599a944..ae244b61 100644 --- a/src/serialize/per_type/list.rs +++ b/src/serialize/per_type/list.rs @@ -88,7 +88,7 @@ impl Serialize for ListTupleSerializer { } debug_assert!(self.len >= 1); let mut seq = serializer.serialize_seq(None).unwrap(); - for idx in 0..=self.len - 1 { + for idx in 0..self.len { let value = unsafe { *((self.data_ptr).add(idx)) }; let value_ob_type = ob_type!(value); if is_class_by_type!(value_ob_type, STR_TYPE) { diff --git a/src/serialize/per_type/numpy.rs b/src/serialize/per_type/numpy.rs index 2f86d560..dede7996 100644 --- a/src/serialize/per_type/numpy.rs +++ b/src/serialize/per_type/numpy.rs @@ -238,7 +238,7 @@ impl NumpyArray { #[cfg_attr(feature = "optimize", optimize(size))] fn build(&mut self) { if self.depth < self.dimensions() - 1 { - for i in 0..=self.shape()[self.depth] - 1 { + for i in 0..self.shape()[self.depth] { let mut position: Vec = self.position.to_vec(); position[self.depth] = i; let num_children: usize = if self.depth < self.dimensions() - 2 { diff --git a/src/util.rs b/src/util.rs index c7bbe721..787aa8aa 100644 --- a/src/util.rs +++ b/src/util.rs @@ -257,3 +257,13 @@ macro_rules! reserve_minimum { $writer.reserve(64); }; } + +macro_rules! assume { + ($expr:expr) => { + debug_assert!($expr); + #[cfg(feature = "intrinsics")] + unsafe { + core::intrinsics::assume($expr); + }; + }; +}