From c3791c88dbcf2027403afa2650a0d07429896035 Mon Sep 17 00:00:00 2001 From: Luiz Irber Date: Thu, 9 Jan 2025 17:41:05 +0000 Subject: [PATCH] CI: github actions updates (#3476) Couple of CI updates - remove `actions-rs/*` actions, haven't been maintained for quite some time. Replace with `dtolnay/rust-toolchain`, `taiki-e/install-action` - Add rust caching via `Swatinem/rust-cache` - Use `nextest` for running tests, much faster - Use `cargo-llvm-cov` for coverage tracking - remove `cargo-wasi` and `wasm32-wasi` checks, should redo with `cargo-component` instead (and `wasm32-wasip2`): https://blog.rust-lang.org/2024/04/09/updates-to-rusts-wasi-targets.html - update to cbindgen `0.26.0`, which is not the latest version: https://github.com/mozilla/cbindgen/pull/959 broke `0.27.0` for us, will investigate how to fix --- .github/workflows/asv.yml | 7 +- .github/workflows/dev_envs.yml | 4 +- .github/workflows/python.yml | 3 + .github/workflows/rust.yml | 170 +++++++++++---------------------- flake.lock | 18 ++-- flake.nix | 9 +- 6 files changed, 78 insertions(+), 133 deletions(-) diff --git a/.github/workflows/asv.yml b/.github/workflows/asv.yml index aca239e935..3b3167a6c4 100644 --- a/.github/workflows/asv.yml +++ b/.github/workflows/asv.yml @@ -27,10 +27,9 @@ jobs: with: python-version: "3.10" - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true + - uses: dtolnay/rust-toolchain@stable + + - uses: Swatinem/rust-cache@v2 - name: Install dependencies run: | diff --git a/.github/workflows/dev_envs.yml b/.github/workflows/dev_envs.yml index 24ef7112e8..a6ee251e49 100644 --- a/.github/workflows/dev_envs.yml +++ b/.github/workflows/dev_envs.yml @@ -24,7 +24,7 @@ jobs: - run: nix run .# -- --version - - run: nix develop --command bash -c "tox -e py310" + - run: nix develop --command bash -c "tox -e py311" pixi: runs-on: ${{ matrix.os }} @@ -39,7 +39,7 @@ jobs: - name: set up pixi uses: prefix-dev/setup-pixi@v0.8.1 with: - pixi-version: v0.26.1 + pixi-version: v0.39.5 cache: true frozen: true diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 0805449e48..055e804da1 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -40,6 +40,9 @@ jobs: restore-keys: | ${{ runner.os }}-pip-v4- + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 459f29ece2..cf01cf3d3e 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -15,17 +15,13 @@ jobs: - name: Checkout sources uses: actions/checkout@v4 - - name: Install stable toolchain - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true + - uses: dtolnay/rust-toolchain@stable + + - uses: Swatinem/rust-cache@v2 - name: Run cargo check - uses: actions-rs/cargo@v1 - with: - command: check + run: | + cargo check test: runs-on: ${{ matrix.os }} @@ -53,32 +49,30 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions-rs/toolchain@v1 + - uses: dtolnay/rust-toolchain@master with: toolchain: ${{ matrix.rust }} - override: true + + - uses: Swatinem/rust-cache@v2 + + - uses: taiki-e/install-action@nextest - name: Run tests - uses: actions-rs/cargo@v1 - with: - command: test - args: --no-fail-fast + run: | + cargo nextest run test_all_features: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true + - uses: dtolnay/rust-toolchain@stable + + - uses: Swatinem/rust-cache@v2 - - uses: actions-rs/install@v0.1 + - uses: taiki-e/install-action@v2 with: - crate: cargo-all-features - version: latest - use-tool-cache: true + tool: cargo-all-features - name: Run tests for all feature combinations run: cargo test-all-features --no-fail-fast --all @@ -88,29 +82,23 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions-rs/toolchain@v1 + - uses: dtolnay/rust-toolchain@stable with: - toolchain: stable - override: true + components: llvm-tools-preview + - uses: Swatinem/rust-cache@v2 - - name: Install cargo-binstall - run: | - wget https://github.com/ryankurte/cargo-binstall/releases/latest/download/cargo-binstall-x86_64-unknown-linux-gnu.tgz - tar xf cargo-binstall-x86_64-unknown-linux-gnu.tgz - chmod +x cargo-binstall - mv cargo-binstall "$HOME/.cargo/bin" - - - name: Install cargo-tarpaulin - run: cargo binstall --no-confirm cargo-tarpaulin + - uses: taiki-e/install-action@cargo-llvm-cov + - uses: taiki-e/install-action@nextest - - name: Coverage with tarpaulin - run: cargo tarpaulin --all --all-features --timeout 600 --out xml -- --test-threads 1 + - name: Collect coverage data + run: cargo llvm-cov nextest --all-features --lcov --output-path lcov.info - name: Upload Rust coverage to codecov uses: codecov/codecov-action@v3 with: flags: rust fail_ci_if_error: true + files: lcov.info lints: name: Lints @@ -129,24 +117,20 @@ jobs: uses: actions/checkout@v4 - name: Install Rust toolchain - uses: actions-rs/toolchain@v1 + uses: dtolnay/rust-toolchain@master with: - profile: minimal toolchain: ${{ matrix.rust }} - override: true - components: rustfmt, clippy + components: "clippy, rustfmt" + + - uses: Swatinem/rust-cache@v2 - name: Run cargo fmt - uses: actions-rs/cargo@v1 - with: - command: fmt - args: --all -- --check + run: | + cargo fmt --all -- --check - name: Run cargo clippy - uses: actions-rs/cargo@v1 - with: - command: clippy - args: --all -- -D warnings + run: | + cargo clippy --all -- -D warnings wasm-pack: name: Check if wasm-pack builds a valid package for the sourmash crate @@ -154,11 +138,12 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions-rs/toolchain@v1 + - uses: dtolnay/rust-toolchain@stable with: - toolchain: stable target: wasm32-unknown-unknown + - uses: Swatinem/rust-cache@v2 + - name: Install wasm-pack run: "curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh" @@ -190,38 +175,6 @@ jobs: with: path: 'pkg/sourmash*.tgz' - wasm32-wasi: - name: Run tests under wasm32-wasi - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Install wasm32-wasi target - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - target: wasm32-wasi - - name: Install wasmtime - run: "curl https://wasmtime.dev/install.sh -sSf | bash" - - name: Add wasmtime to PATH - run: echo "$HOME/.wasmtime/bin" >> $GITHUB_PATH - - name: Install cargo-wasi command - uses: actions-rs/cargo@v1 - with: - command: install - args: --force cargo-wasi - - name: Build code with cargo-wasi - uses: actions-rs/cargo@v1 - continue-on-error: true ## TODO: remove this when build works... - with: - command: wasi - args: build - - name: Run tests under wasm32-wasi - uses: actions-rs/cargo@v1 - continue-on-error: true ## TODO: remove this when tests work... - with: - command: wasi - args: test - publish: name: Publish (on tags, dry-run otherwise) runs-on: ubuntu-latest @@ -229,12 +182,9 @@ jobs: - name: Checkout sources uses: actions/checkout@v4 - - name: Install stable toolchain - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true + - uses: dtolnay/rust-toolchain@stable + + - uses: Swatinem/rust-cache@v2 - name: Check semver uses: obi1kenobi/cargo-semver-checks-action@v2 @@ -245,45 +195,38 @@ jobs: features: branchwater - name: Make sure we can publish the sourmash crate - uses: actions-rs/cargo@v1 - with: - command: publish - args: --dry-run --manifest-path src/core/Cargo.toml + run: | + cargo publish --dry-run --manifest-path src/core/Cargo.toml # Login to crates.io on tags - name: login to crates.io - uses: actions-rs/cargo@v1 if: startsWith(github.ref, 'refs/tags/r') - with: - command: login - args: ${{ secrets.CRATES_IO_TOKEN }} + run: | + cargo login ${{ secrets.CRATES_IO_TOKEN }} # Publish to crates.io on tags - name: Publish to crates.io if: startsWith(github.ref, 'refs/tags/r') - uses: actions-rs/cargo@v1 - with: - command: publish - args: --manifest-path src/core/Cargo.toml + run: | + cargo publish --manifest-path src/core/Cargo.toml minimum_rust_version: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions-rs/toolchain@v1 + - uses: dtolnay/rust-toolchain@master with: toolchain: "1.66.0" - override: true + + - uses: Swatinem/rust-cache@v2 - name: check if README matches MSRV defined here run: grep '1.66.0' src/core/README.md - name: Check if it builds properly - uses: actions-rs/cargo@v1 - with: - command: build - args: --all-features + run: | + cargo build --all-features check_cbindgen: name: "Check if cbindgen runs cleanly for generating the C headers" @@ -291,16 +234,13 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true + - uses: dtolnay/rust-toolchain@stable + + - uses: Swatinem/rust-cache@v2 - - uses: actions-rs/install@v0.1 + - uses: taiki-e/install-action@v2 with: - crate: cbindgen - version: 0.20.0 - use-tool-cache: true + tool: cbindgen@0.26.0 - run: make include/sourmash.h diff --git a/flake.lock b/flake.lock index 41f929d8e7..eb0632cfe2 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1728979988, - "narHash": "sha256-GBJRnbFLDg0y7ridWJHAP4Nn7oss50/VNgqoXaf/RVk=", + "lastModified": 1736241350, + "narHash": "sha256-CHd7yhaDigUuJyDeX0SADbTM9FXfiWaeNyY34FL1wQU=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "7881fbfd2e3ed1dfa315fca889b2cfd94be39337", + "rev": "8c9fd3e564728e90829ee7dbac6edc972971cd0f", "type": "github" }, "original": { @@ -30,11 +30,11 @@ ] }, "locked": { - "lastModified": 1729184663, - "narHash": "sha256-uNyi5vQrzaLkt4jj6ZEOs4+4UqOAwP6jFG2s7LIDwIk=", + "lastModified": 1736303309, + "narHash": "sha256-IKrk7RL+Q/2NC6+Ql6dwwCNZI6T6JH2grTdJaVWHF0A=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "16fb78d443c1970dda9a0bbb93070c9d8598a925", + "rev": "a0b81d4fa349d9af1765b0f0b4a899c13776f706", "type": "github" }, "original": { @@ -63,11 +63,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1726560853, - "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=", + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", "owner": "numtide", "repo": "flake-utils", - "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 732f4b59de..aefe9a3468 100644 --- a/flake.nix +++ b/flake.nix @@ -21,8 +21,9 @@ }; rustVersion = pkgs.rust-bin.stable.latest.default.override { #extensions = [ "rust-src" ]; + extensions = [ "llvm-tools-preview" ]; #targets = [ "x86_64-unknown-linux-musl" ]; - targets = [ "wasm32-wasi" "wasm32-unknown-unknown" "wasm32-unknown-emscripten" ]; + targets = [ "wasm32-unknown-unknown" "wasm32-unknown-emscripten" ]; }; rustPlatform = pkgs.makeRustPlatform { cargo = rustVersion; @@ -106,7 +107,7 @@ (python311.withPackages (ps: with ps; [ virtualenv tox cffi ])) (python310.withPackages (ps: with ps; [ virtualenv ])) - rust-cbindgen + #rust-cbindgen maturin wasmtime @@ -122,7 +123,9 @@ cargo-outdated cargo-udeps cargo-deny - cargo-wasi + cargo-nextest + #cargo-llvm-cov + cargo-component cargo-codspeed cargo-semver-checks nixpkgs-fmt