diff --git a/.github/actions/setup-macos-aarch64-builder/action.yaml b/.github/actions/setup-macos-aarch64-builder/action.yaml index c4e14906ed10..6471e3a4741c 100644 --- a/.github/actions/setup-macos-aarch64-builder/action.yaml +++ b/.github/actions/setup-macos-aarch64-builder/action.yaml @@ -30,8 +30,8 @@ runs: run: | mkdir -p $HOME/d/protoc cd $HOME/d/protoc - export PROTO_ZIP="protoc-21.4-osx-aarch_64.zip" - curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP + export PROTO_ZIP="protoc-29.1-osx-aarch_64.zip" + curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v29.1/$PROTO_ZIP unzip $PROTO_ZIP echo "$HOME/d/protoc/bin" >> $GITHUB_PATH export PATH=$PATH:$HOME/d/protoc/bin diff --git a/.github/actions/setup-macos-builder/action.yaml b/.github/actions/setup-macos-builder/action.yaml index 02419f617942..fffdab160b04 100644 --- a/.github/actions/setup-macos-builder/action.yaml +++ b/.github/actions/setup-macos-builder/action.yaml @@ -30,8 +30,8 @@ runs: run: | mkdir -p $HOME/d/protoc cd $HOME/d/protoc - export PROTO_ZIP="protoc-21.4-osx-x86_64.zip" - curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP + export PROTO_ZIP="protoc-29.1-osx-x86_64.zip" + curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v29.1/$PROTO_ZIP unzip $PROTO_ZIP echo "$HOME/d/protoc/bin" >> $GITHUB_PATH export PATH=$PATH:$HOME/d/protoc/bin diff --git a/.github/actions/setup-rust-runtime/action.yaml b/.github/actions/setup-rust-runtime/action.yaml index cd18be989031..810062f22eac 100644 --- a/.github/actions/setup-rust-runtime/action.yaml +++ b/.github/actions/setup-rust-runtime/action.yaml @@ -22,6 +22,8 @@ runs: steps: - name: Run sccache-cache uses: mozilla-actions/sccache-action@v0.0.4 + - name: Run rust cache + uses: Swatinem/rust-cache@v2 - name: Configure runtime env shell: bash # do not produce debug symbols to keep memory usage down diff --git a/.github/actions/setup-windows-builder/action.yaml b/.github/actions/setup-windows-builder/action.yaml index 5e937358c7d7..a0304168c744 100644 --- a/.github/actions/setup-windows-builder/action.yaml +++ b/.github/actions/setup-windows-builder/action.yaml @@ -30,8 +30,8 @@ runs: run: | mkdir -p $HOME/d/protoc cd $HOME/d/protoc - export PROTO_ZIP="protoc-21.4-win64.zip" - curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP + export PROTO_ZIP="protoc-29.1-win64.zip" + curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v29.1/$PROTO_ZIP unzip $PROTO_ZIP export PATH=$PATH:$HOME/d/protoc/bin protoc.exe --version diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index ebc5bcf91c94..f87215565bb5 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -42,6 +42,7 @@ jobs: - uses: actions/checkout@v4 with: submodules: true + fetch-depth: 1 - name: Setup Rust toolchain uses: ./.github/actions/setup-builder with: diff --git a/.github/workflows/docs_pr.yaml b/.github/workflows/docs_pr.yaml index c2f3dd684a23..3fad08643aa2 100644 --- a/.github/workflows/docs_pr.yaml +++ b/.github/workflows/docs_pr.yaml @@ -43,6 +43,7 @@ jobs: - uses: actions/checkout@v4 with: submodules: true + fetch-depth: 1 - name: Setup Rust toolchain uses: ./.github/actions/setup-builder with: diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index f4e3d2fbf4d6..16e6bdd47c99 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -59,87 +59,74 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: stable - - - name: Cache Cargo - uses: actions/cache@v4 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - ./target/ - ./datafusion-cli/target/ - key: cargo-cache-${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }} - - name: Check datafusion without default features # Some of the test binaries require the parquet feature still #run: cargo check --all-targets --no-default-features -p datafusion - run: cargo check --no-default-features -p datafusion + run: cargo check --profile ci --no-default-features -p datafusion - name: Check datafusion-common without default features - run: cargo check --all-targets --no-default-features -p datafusion-common + run: cargo check --profile ci --all-targets --no-default-features -p datafusion-common - name: Check datafusion-functions without default features - run: cargo check --all-targets --no-default-features -p datafusion-functions + run: cargo check --profile ci --all-targets --no-default-features -p datafusion-functions - name: Check datafusion-substrait without default features - run: cargo check --all-targets --no-default-features -p datafusion-substrait + run: cargo check --profile ci --all-targets --no-default-features -p datafusion-substrait - name: Check workspace in debug mode - run: cargo check --all-targets --workspace + run: cargo check --profile ci --all-targets --workspace - name: Check workspace with avro,json features - run: cargo check --workspace --benches --features avro,json + run: cargo check --profile ci --workspace --benches --features avro,json - name: Check Cargo.lock for datafusion-cli run: | # If this test fails, try running `cargo update` in the `datafusion-cli` directory # and check in the updated Cargo.lock file. - cargo check --manifest-path datafusion-cli/Cargo.toml --locked + cargo check --profile ci --manifest-path datafusion-cli/Cargo.toml --locked # Ensure that the datafusion crate can be built with only a subset of the function # packages enabled. - name: Check datafusion (nested_expressions) - run: cargo check --no-default-features --features=nested_expressions -p datafusion + run: cargo check --profile ci --no-default-features --features=nested_expressions -p datafusion - name: Check datafusion (crypto) - run: cargo check --no-default-features --features=crypto_expressions -p datafusion + run: cargo check --profile ci --no-default-features --features=crypto_expressions -p datafusion - name: Check datafusion (datetime_expressions) - run: cargo check --no-default-features --features=datetime_expressions -p datafusion + run: cargo check --profile ci --no-default-features --features=datetime_expressions -p datafusion - name: Check datafusion (encoding_expressions) - run: cargo check --no-default-features --features=encoding_expressions -p datafusion + run: cargo check --profile ci --no-default-features --features=encoding_expressions -p datafusion - name: Check datafusion (math_expressions) - run: cargo check --no-default-features --features=math_expressions -p datafusion + run: cargo check --profile ci --no-default-features --features=math_expressions -p datafusion - name: Check datafusion (regex_expressions) - run: cargo check --no-default-features --features=regex_expressions -p datafusion + run: cargo check --profile ci --no-default-features --features=regex_expressions -p datafusion - name: Check datafusion (string_expressions) - run: cargo check --no-default-features --features=string_expressions -p datafusion + run: cargo check --profile ci --no-default-features --features=string_expressions -p datafusion # Ensure that the datafusion-functions crate can be built with only a subset of the function # packages enabled. - name: Check datafusion-functions (crypto) - run: cargo check --all-targets --no-default-features --features=crypto_expressions -p datafusion-functions + run: cargo check --profile ci --all-targets --no-default-features --features=crypto_expressions -p datafusion-functions - name: Check datafusion-functions (datetime_expressions) - run: cargo check --all-targets --no-default-features --features=datetime_expressions -p datafusion-functions + run: cargo check --profile ci --all-targets --no-default-features --features=datetime_expressions -p datafusion-functions - name: Check datafusion-functions (encoding_expressions) - run: cargo check --all-targets --no-default-features --features=encoding_expressions -p datafusion-functions + run: cargo check --profile ci --all-targets --no-default-features --features=encoding_expressions -p datafusion-functions - name: Check datafusion-functions (math_expressions) - run: cargo check --all-targets --no-default-features --features=math_expressions -p datafusion-functions + run: cargo check --profile ci --all-targets --no-default-features --features=math_expressions -p datafusion-functions - name: Check datafusion-functions (regex_expressions) - run: cargo check --all-targets --no-default-features --features=regex_expressions -p datafusion-functions + run: cargo check --profile ci --all-targets --no-default-features --features=regex_expressions -p datafusion-functions - name: Check datafusion-functions (string_expressions) - run: cargo check --all-targets --no-default-features --features=string_expressions -p datafusion-functions + run: cargo check --profile ci --all-targets --no-default-features --features=string_expressions -p datafusion-functions # Run tests linux-test: @@ -152,12 +139,17 @@ jobs: - uses: actions/checkout@v4 with: submodules: true + fetch-depth: 1 - name: Setup Rust toolchain uses: ./.github/actions/setup-builder with: - rust-version: stable + rust-version: stable + - name: Install nextest + uses: taiki-e/install-action@nextest - name: Run tests (excluding doctests) - run: cargo test --lib --tests --bins --features avro,json,backtrace + run: cargo nextest run --hide-progress-bar --no-fail-fast --cargo-profile ci --exclude datafusion-examples --exclude datafusion-benchmarks --exclude datafusion-sqllogictest --workspace --lib --tests --bins --features avro,json,backtrace + - name: Run sqllogictests + run: cargo test --profile ci --package datafusion-sqllogictest --tests sqllogictests - name: Verify Working Directory Clean run: git diff --exit-code @@ -171,14 +163,17 @@ jobs: - uses: actions/checkout@v4 with: submodules: true + fetch-depth: 1 - name: Setup Rust toolchain uses: ./.github/actions/setup-builder with: rust-version: stable + - name: Install nextest + uses: taiki-e/install-action@nextest - name: Run tests (excluding doctests) run: | cd datafusion-cli - cargo test --lib --tests --bins --all-features + cargo nextest run --hide-progress-bar --no-fail-fast --cargo-profile ci --lib --tests --bins --all-features - name: Verify Working Directory Clean run: git diff --exit-code @@ -192,6 +187,7 @@ jobs: - uses: actions/checkout@v4 with: submodules: true + fetch-depth: 1 - name: Setup Rust toolchain uses: ./.github/actions/setup-builder with: @@ -205,8 +201,6 @@ jobs: - name: Verify Working Directory Clean run: git diff --exit-code - - # Run `cargo test doc` (test documentation examples) linux-test-doc: name: cargo test doc (amd64) @@ -218,15 +212,16 @@ jobs: - uses: actions/checkout@v4 with: submodules: true + fetch-depth: 1 - name: Setup Rust toolchain uses: ./.github/actions/setup-builder with: rust-version: stable - name: Run doctests run: | - cargo test --doc --features avro,json + cargo test --profile ci --doc --features avro,json cd datafusion-cli - cargo test --doc --all-features + cargo test --profile ci --doc --all-features - name: Verify Working Directory Clean run: git diff --exit-code @@ -274,12 +269,13 @@ jobs: - uses: actions/checkout@v4 with: submodules: true + fetch-depth: 1 - name: Setup Rust toolchain uses: ./.github/actions/setup-builder with: rust-version: stable - name: Generate benchmark data and expected query results - run: | + -run: | mkdir -p datafusion/sqllogictest/test_files/tpch/data git clone https://github.com/databricks/tpch-dbgen.git cd tpch-dbgen @@ -289,9 +285,9 @@ jobs: - name: Verify that benchmark queries return expected results run: | export TPCH_DATA=`realpath datafusion/sqllogictest/test_files/tpch/data` - # use release build for plan verificaton because debug build causes stack overflow - cargo test plan_q --package datafusion-benchmarks --profile release-nonlto --features=ci -- --test-threads=1 - INCLUDE_TPCH=true cargo test --test sqllogictests + # use release build for plan verification because debug build causes stack overflow + cargo test plan_q --package datafusion-benchmarks --profile ci --features=ci -- --test-threads=1 + INCLUDE_TPCH=true cargo test --profile ci --package datafusion-sqllogictest --test sqllogictests - name: Verify Working Directory Clean run: git diff --exit-code @@ -317,12 +313,13 @@ jobs: - uses: actions/checkout@v4 with: submodules: true + fetch-depth: 1 - name: Setup toolchain run: | rustup toolchain install stable rustup default stable - name: Run sqllogictest - run: PG_COMPAT=true PG_URI="postgresql://postgres:postgres@localhost:$POSTGRES_PORT/db_test" cargo test --features=postgres --test sqllogictests + run: PG_COMPAT=true PG_URI="postgresql://postgres:postgres@localhost:$POSTGRES_PORT/db_test" cargo test --profile ci --package datafusion-sqllogictests --features=postgres --test sqllogictests env: POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }} @@ -333,15 +330,19 @@ jobs: - uses: actions/checkout@v4 with: submodules: true + fetch-depth: 1 - name: Setup Rust toolchain uses: ./.github/actions/setup-windows-builder + - name: Install nextest + uses: taiki-e/install-action@nextest - name: Run tests (excluding doctests) shell: bash run: | export PATH=$PATH:$HOME/d/protoc/bin - cargo test --lib --tests --bins --features avro,json,backtrace + cargo nextest run --hide-progress-bar --no-fail-fast --cargo-profile ci --exclude datafusion-examples --exclude datafusion-benchmarks --exclude datafusion-sqllogictest --workspace --lib --tests --bins --features avro,json,backtrace + cargo test --profile ci --package datafusion-sqllogictest --tests sqllogictests cd datafusion-cli - cargo test --lib --tests --bins --all-features + cargo nextest run --hide-progress-bar --no-fail-fast --cargo-profile ci --lib --tests --bins --all-features macos: name: cargo test (macos) @@ -349,15 +350,19 @@ jobs: steps: - uses: actions/checkout@v4 with: - submodules: true + submodules: true + fetch-depth: 1 - name: Setup Rust toolchain - uses: ./.github/actions/setup-macos-builder + uses: ./.github/actions/setup-macos-builder + - name: Install nextest + uses: taiki-e/install-action@nextest - name: Run tests (excluding doctests) shell: bash run: | - cargo test --lib --tests --bins --features avro,json,backtrace + cargo nextest run --hide-progress-bar --no-fail-fast --cargo-profile ci --exclude datafusion-examples --exclude datafusion-benchmarks --exclude datafusion-sqllogictest --workspace --lib --tests --bins --features avro,json,backtrace + cargo test --profile ci --package datafusion-sqllogictest --tests sqllogictests cd datafusion-cli - cargo test --lib --tests --bins --all-features + cargo nextest run --hide-progress-bar --no-fail-fast --cargo-profile ci --lib --tests --bins --all-features macos-aarch64: name: cargo test (macos-aarch64) @@ -366,14 +371,17 @@ jobs: - uses: actions/checkout@v4 with: submodules: true + fetch-depth: 1 - name: Setup Rust toolchain uses: ./.github/actions/setup-macos-aarch64-builder + - name: Install nextest + uses: taiki-e/install-action@nextest - name: Run tests (excluding doctests) shell: bash run: | - cargo test --lib --tests --bins --features avro,json,backtrace + cargo nextest run --hide-progress-bar --no-fail-fast --cargo-profile ci --exclude datafusion-examples --exclude datafusion-benchmarks --exclude datafusion-sqllogictest --workspace --lib --tests --bins --features avro,json,backtrace cd datafusion-cli - cargo test --lib --tests --bins --all-features + cargo nextest run --hide-progress-bar --no-fail-fast --cargo-profile ci --lib --tests --bins --all-features test-datafusion-pyarrow: name: cargo test pyarrow (amd64) @@ -385,6 +393,7 @@ jobs: - uses: actions/checkout@v4 with: submodules: true + fetch-depth: 1 - uses: actions/setup-python@v5 with: python-version: "3.8" @@ -396,8 +405,10 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: stable + - name: Install nextest + uses: taiki-e/install-action@nextest - name: Run datafusion-common tests - run: cargo test -p datafusion-common --features=pyarrow + run: cargo nextest run --hide-progress-bar --no-fail-fast --cargo-profile ci -p datafusion-common --features=pyarrow vendor: name: Verify Vendored Code @@ -482,6 +493,7 @@ jobs: - uses: actions/checkout@v4 with: submodules: true + fetch-depth: 1 - name: Setup Rust toolchain uses: ./.github/actions/setup-builder with: @@ -502,14 +514,17 @@ jobs: - uses: actions/checkout@v4 with: submodules: true + fetch-depth: 1 - name: Setup Rust toolchain uses: ./.github/actions/setup-builder with: rust-version: stable + - name: Install nextest + uses: taiki-e/install-action@nextest - name: Run tests run: | cd datafusion - cargo test --lib --tests --features=force_hash_collisions,avro + cargo nextest run --hide-progress-bar --no-fail-fast --cargo-profile ci --exclude datafusion-examples --exclude datafusion-benchmarks --exclude datafusion-sqllogictest --workspace --lib --tests --features=force_hash_collisions,avro cargo-toml-formatting-checks: name: check Cargo.toml formatting @@ -521,6 +536,7 @@ jobs: - uses: actions/checkout@v4 with: submodules: true + fetch-depth: 1 - name: Setup Rust toolchain uses: ./.github/actions/setup-builder with: @@ -541,6 +557,7 @@ jobs: - uses: actions/checkout@v4 with: submodules: true + fetch-depth: 1 - name: Setup Rust toolchain uses: ./.github/actions/setup-builder with: diff --git a/Cargo.toml b/Cargo.toml index cc94b4292a50..bf24d01a08f6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -170,6 +170,13 @@ overflow-checks = false panic = 'unwind' rpath = false +# ci turns off debug info to allow for smaller binaries making caching more effective +[profile.ci] +inherits = "dev" +debug = 0 +strip = "debuginfo" +incremental = false + [workspace.lints.clippy] # Detects large stack-allocated futures that may cause stack overflow crashes (see threshold in clippy.toml) large_futures = "warn" diff --git a/ci/scripts/rust_docs.sh b/ci/scripts/rust_docs.sh index 5c93711b6fb6..ed9ef18e5476 100755 --- a/ci/scripts/rust_docs.sh +++ b/ci/scripts/rust_docs.sh @@ -19,6 +19,6 @@ set -ex export RUSTDOCFLAGS="-D warnings" -cargo doc --document-private-items --no-deps --workspace +cargo doc --profile ci --document-private-items --no-deps --workspace cd datafusion-cli -cargo doc --document-private-items --no-deps +cargo doc --profile ci --document-private-items --no-deps diff --git a/ci/scripts/rust_example.sh b/ci/scripts/rust_example.sh index 1bb97c88106f..f107e00d7e65 100755 --- a/ci/scripts/rust_example.sh +++ b/ci/scripts/rust_example.sh @@ -19,7 +19,7 @@ set -ex cd datafusion-examples/examples/ -cargo check --examples +cargo check --profile ci --examples files=$(ls .) for filename in $files @@ -27,7 +27,7 @@ do example_name=`basename $filename ".rs"` # Skip tests that rely on external storage and flight if [ ! -d $filename ]; then - cargo run --example $example_name + cargo run --profile ci --example $example_name cargo clean -p datafusion-examples fi done diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml index 4cdc2120a029..83d4f0f28869 100644 --- a/datafusion-cli/Cargo.toml +++ b/datafusion-cli/Cargo.toml @@ -69,3 +69,10 @@ assert_cmd = "2.0" ctor = "0.2.0" predicates = "3.0" rstest = "0.22" + +# ci turns off debug info to allow for smaller binaries making caching more effective +[profile.ci] +inherits = "dev" +debug = 0 +strip = "debuginfo" +incremental = false \ No newline at end of file diff --git a/dev/depcheck/README.md b/dev/depcheck/README.md index 4c1571051f22..4033302c24e6 100644 --- a/dev/depcheck/README.md +++ b/dev/depcheck/README.md @@ -20,7 +20,7 @@ This directory contains a tool that ensures there are no circular dependencies in the DataFusion codebase. -Specifically, it checks that no create's tests depend on another crate which +Specifically, it checks that no crate's tests depend on another crate which depends on the first, which prevents publishing to crates.io, for example [issue 9272]: https://github.com/apache/datafusion/issues/9277: