Skip to content
Rust

feat: support remapping vector storage and flat index #8147

Sign in to view logs

feat: support remapping vector storage and flat index

feat: support remapping vector storage and flat index #8147

Workflow file for this run

.github/workflows/rust.yml at dfa1663
name: Rust
on:
push:
branches:
- main
pull_request:
paths:
- rust/**
- protos/**
- .github/workflows/rust.yml
- Cargo.toml
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
# This env var is used by Swatinem/rust-cache@v2 for the cache
# key, so we set it to make sure it is always consistent.
CARGO_TERM_COLOR: always
# Disable full debug symbol generation to speed up CI build and keep memory down
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
RUST_BACKTRACE: "1"
# according to: https://matklad.github.io/2021/09/04/fast-rust-builds.html
# CI builds are faster with incremental disabled.
CARGO_INCREMENTAL: "0"
CARGO_BUILD_JOBS: "1"
jobs:
format:
runs-on: ubuntu-24.04
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
- name: Check formatting
run: cargo fmt -- --check
clippy:
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v4
- uses: Swatinem/rust-cache@v2
- name: Install dependencies
run: |
sudo apt update
sudo apt install -y protobuf-compiler libssl-dev
- name: Run clippy
run: |
cargo clippy --version
cargo clippy --locked --all-features --tests --benches -- -D warnings
linux-build:
runs-on: "ubuntu-24.04"
timeout-minutes: 45
strategy:
matrix:
toolchain:
- stable
- nightly
env:
# Need up-to-date compilers for kernels
CC: clang
CXX: clang
steps:
- uses: actions/checkout@v4
# pin the toolchain version to avoid surprises
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
toolchain: ${{ matrix.toolchain }}
- uses: rui314/setup-mold@v1
- uses: Swatinem/rust-cache@v2
- name: Install dependencies
run: |
sudo apt update
sudo apt install -y protobuf-compiler libssl-dev
rustup update ${{ matrix.toolchain }} && rustup default ${{ matrix.toolchain }}
- name: Start DynamoDB local for tests
run: |
docker run -d -e AWS_ACCESS_KEY_ID=DUMMYKEY -e AWS_SECRET_ACCESS_KEY=DUMMYKEY -p 8000:8000 amazon/dynamodb-local
- name: Install cargo-llvm-cov
uses: taiki-e/install-action@cargo-llvm-cov
- name: Run tests
if: ${{ matrix.toolchain == 'stable' }}
run: |
cargo llvm-cov --locked --workspace --codecov --output-path coverage.codecov --all-features
- name: Build tests (nightly)
run: cargo test --locked --all-features --workspace --no-run
- name: Run tests (nightly)
if: ${{ matrix.toolchain != 'stable' }}
run: |
cargo test --all-features --workspace
- name: Upload coverage to Codecov
if: ${{ matrix.toolchain == 'stable' }}
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
codecov_yml_path: codecov.yml
files: coverage.codecov
flags: unittests
fail_ci_if_error: false
linux-arm:
runs-on: ubuntu-2404-4x-arm64
timeout-minutes: 45
steps:
- uses: actions/checkout@v4
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
toolchain: "stable"
- uses: rui314/setup-mold@v1
- uses: Swatinem/rust-cache@v2
- name: Install dependencies
run: |
sudo apt -y -qq update
sudo apt install -y protobuf-compiler libssl-dev pkg-config
- name: Build tests
run: |
cargo test --locked --all-features --no-run
- name: Start DynamoDB local for tests
run: |
docker run -d -e AWS_ACCESS_KEY_ID=DUMMYKEY -e AWS_SECRET_ACCESS_KEY=DUMMYKEY -p 8000:8000 amazon/dynamodb-local
- name: Run tests
run: |
cargo test --locked --all-features
build-no-lock:
runs-on: ubuntu-24.04
timeout-minutes: 30
env:
# Need up-to-date compilers for kernels
CC: clang
CXX: clang
steps:
- uses: actions/checkout@v4
# Remote cargo.lock to force a fresh build
- name: Remove Cargo.lock
run: rm -f Cargo.lock
- uses: rui314/setup-mold@v1
- uses: Swatinem/rust-cache@v2
- name: Install dependencies
run: |
sudo apt update
sudo apt install -y protobuf-compiler libssl-dev
- name: Build all
run: cargo build --benches --all-features --tests
mac-build:
runs-on: "macos-14"
timeout-minutes: 45
strategy:
matrix:
toolchain:
- stable
- nightly
defaults:
run:
working-directory: ./rust/lance
steps:
- uses: actions/checkout@v4
- uses: Swatinem/rust-cache@v2
- name: Select new xcode
# Default XCode right now is 15.0.1, which contains a bug that causes
# backtraces to not show properly. See:
# https://github.com/rust-lang/rust/issues/113783
run: sudo xcode-select -s /Applications/Xcode_15.4.app
- name: Install dependencies
run: brew install protobuf
- name: Set up Rust
run: |
rustup update ${{ matrix.toolchain }} && rustup default ${{ matrix.toolchain }}
- name: Build tests
run: cargo test --locked --all-features --no-run
- name: Run tests
run: cargo test --all-features
- name: Check benchmarks
run: cargo check --benches --all-features
windows-build:
runs-on: windows-latest
defaults:
run:
working-directory: rust/lance
steps:
- uses: actions/checkout@v4
- uses: Swatinem/rust-cache@v2
- name: Install Protoc v21.12
working-directory: C:\
run: |
New-Item -Path 'C:\protoc' -ItemType Directory
Set-Location C:\protoc
Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
7z x protoc.zip
Add-Content $env:GITHUB_PATH "C:\protoc\bin"
shell: powershell
- name: Build tests
run: cargo test --locked --no-run
- name: Run tests
run: cargo test
- name: Check benchmarks
run: cargo check --benches
msrv:
# Check the minimum supported Rust version
name: MSRV Check - Rust v${{ matrix.msrv }}
runs-on: ubuntu-24.04
strategy:
matrix:
msrv: ["1.78.0"] # This should match up with rust-version in Cargo.toml
env:
# Need up-to-date compilers for kernels
CC: clang
CXX: clang
steps:
- uses: actions/checkout@v4
with:
submodules: true
- uses: Swatinem/rust-cache@v2
- name: Install dependencies
run: |
sudo apt update
sudo apt install -y protobuf-compiler libssl-dev
- name: Install ${{ matrix.msrv }}
uses: dtolnay/rust-toolchain@master
with:
toolchain: ${{ matrix.msrv }}
- name: cargo +${{ matrix.msrv }} check
run: cargo check --workspace --tests --benches --all-features