From 36ff04aeb5d2f32e91f3347a46e6198f6cc70e08 Mon Sep 17 00:00:00 2001 From: Till Hartmann Date: Tue, 22 Oct 2024 12:14:20 +0200 Subject: [PATCH] chore: switch docker base image, use system rocksdb for faster compilation (#487) Co-authored-by: Manuel Holtgrewe --- .github/workflows/automerge.yml | 2 +- .github/workflows/conventional-prs.yml | 2 +- .github/workflows/docker-build.yml | 2 +- .github/workflows/protobuf.yml | 7 +++- .github/workflows/release-please.yml | 11 +++++-- .github/workflows/rust.yml | 44 ++++++++++++++++++-------- README.md | 36 +++++++++++++++++++++ utils/docker/Dockerfile | 36 ++++++++++----------- 8 files changed, 102 insertions(+), 38 deletions(-) diff --git a/.github/workflows/automerge.yml b/.github/workflows/automerge.yml index 05c540d5..20451975 100644 --- a/.github/workflows/automerge.yml +++ b/.github/workflows/automerge.yml @@ -7,7 +7,7 @@ permissions: jobs: dependabot: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: ${{ github.actor == 'dependabot[bot]' }} steps: - name: Enable auto-merge for Dependabot PRs diff --git a/.github/workflows/conventional-prs.yml b/.github/workflows/conventional-prs.yml index b6a5bca4..78f6611f 100644 --- a/.github/workflows/conventional-prs.yml +++ b/.github/workflows/conventional-prs.yml @@ -9,7 +9,7 @@ on: jobs: title-format: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: amannn/action-semantic-pull-request@v5.5.3 env: diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 7031aa65..c16c346c 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -18,7 +18,7 @@ env: jobs: build-and-push-image: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: contents: read packages: write diff --git a/.github/workflows/protobuf.yml b/.github/workflows/protobuf.yml index 6570c40d..976fab82 100644 --- a/.github/workflows/protobuf.yml +++ b/.github/workflows/protobuf.yml @@ -10,11 +10,16 @@ on: jobs: Linting: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout repository uses: actions/checkout@v4 + - name: Setup python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install protolint run: | pip install protolint-bin diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index fe21553d..f2a1ec22 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -17,7 +17,7 @@ env: jobs: release-please: if: github.repository_owner == 'varfish-org' - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: release_created: ${{ steps.release.outputs.release_created }} release_name: ${{ steps.release.outputs.major }}.${{ steps.release.outputs.minor }}.${{ steps.release.outputs.patch }} @@ -31,12 +31,17 @@ jobs: cargo-release: needs: release-please - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: ${{ needs.release-please.outputs.release_created }} steps: - uses: actions/checkout@v4 if: ${{ needs.release-please.outputs.release_created }} + - name: Install dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y librocksdb-dev libsnappy-dev libsqlite3-dev + - name: Install stable toolchain uses: actions-rs/toolchain@v1 if: ${{ needs.release-please.outputs.release_created }} @@ -64,7 +69,7 @@ jobs: container-release: needs: release-please - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: ${{ needs.release-please.outputs.release_created }} steps: - name: Checkout repository diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index c90431f5..83d38553 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -10,7 +10,7 @@ on: jobs: Formatting: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout repository uses: actions/checkout@v4 @@ -28,15 +28,20 @@ jobs: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Check format - run: | + run: cargo fmt -- --check Linting: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout repository uses: actions/checkout@v4 + - name: Install dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y librocksdb-dev libsnappy-dev libsqlite3-dev + - name: Install stable toolchain uses: actions-rs/toolchain@v1 with: @@ -49,20 +54,31 @@ jobs: with: repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: Swatinem/rust-cache@v2 + with: + cache-directories: | + ~/.cargo/registry/src/**/librocksdb-sys-* + - name: Lint with clippy uses: actions-rs/clippy-check@v1 with: token: ${{ secrets.GITHUB_TOKEN }} + args: --all-features Testing: needs: Formatting - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout repository uses: actions/checkout@v4 with: lfs: true + - name: Install dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y librocksdb-dev libsnappy-dev libsqlite3-dev + - name: Install stable toolchain uses: actions-rs/toolchain@v1 with: @@ -74,16 +90,18 @@ jobs: with: repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: Swatinem/rust-cache@v2.7.3 - - - name: Run cargo-tarpaulin - uses: actions-rs/tarpaulin@v0.1 + - uses: Swatinem/rust-cache@v2 with: - version: 0.21.0 - args: "--skip-clean -- --test-threads 1" + key: "covtest" + cache-directories: | + ~/.cargo/registry/src/**/librocksdb-sys-* - - name: Codecov - uses: codecov/codecov-action@v3 + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov + - name: Generate code coverage + run: cargo llvm-cov --all-features --workspace --lcov --output-path lcov.info + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 with: - verbose: true token: ${{ secrets.CODECOV_TOKEN }} + files: lcov.info diff --git a/README.md b/README.md index 496b593f..6a9e301e 100644 --- a/README.md +++ b/README.md @@ -359,3 +359,39 @@ For running protolint, install it as python package `protolint-bin`: # source /tmp/varfish-server-worker/bin/activate # pip install protolint-bin ``` + +### Building from scratch +To reduce compile times, we recommend using a pre-built version of `rocksdb`, either from the system package manager or e.g. via `conda`: + +```bash +# Ubuntu +sudo apt-get install librocksdb-dev + +# Conda +conda install -c conda-forge rocksdb +``` + +In either case, either add +```toml +[env] +ROCKSDB_LIB_DIR = "/usr/lib/" # in case of the system package manager, adjust the path accordingly for conda +SNAPPY_LIB_DIR = "/usr/lib/" # same as above +``` +to `.cargo/config.toml` or set the environment variables `ROCKSDB_LIB_DIR` and `SNAPPY_LIB_DIR` to the appropriate paths: + +```bash +export ROCKSDB_LIB_DIR=/usr/lib/ +export SNAPPY_LIB_DIR=/usr/lib/ +``` + +By default, the environment variables are defined in the `.cargo/config.toml` as described above, i.e. may need adjustments if not using the system package manager. + +To build the project, run: +```bash +cargo build --release +``` + +To install the project locally, run: +```bash +cargo install --path . +``` \ No newline at end of file diff --git a/utils/docker/Dockerfile b/utils/docker/Dockerfile index c4462498..b6f4e49c 100644 --- a/utils/docker/Dockerfile +++ b/utils/docker/Dockerfile @@ -13,18 +13,16 @@ # Builder # --------------------------------------------------------------------------- -# Pinning Rust version for now because of this issue: -# -# - https://github.com/rust-lang/rust/issues/95926 -FROM rust:1-bookworm AS builder +# Use ubuntu:noble as the base image +FROM ubuntu:noble AS builder -# Build dependencies first. -# -# Install dependencies for compilation of C code (e.g., rocksdb). +# Install Rust toolchain and dependencies for compilation of C code (e.g., rocksdb) RUN apt-get update && \ - apt-get install -y clang -# Add the needed Cargo components. -RUN rustup component add rustfmt + apt-get install -y unzip wget curl build-essential clang librocksdb-dev libsqlite3-dev && \ + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \ + . $HOME/.cargo/env && \ + rustup component add rustfmt + # Install build dependency `protoc`. COPY utils/install-protoc.sh /tmp RUN PREFIX=/usr/local bash /tmp/install-protoc.sh @@ -34,13 +32,13 @@ RUN PREFIX=/usr/local bash /tmp/install-protoc.sh # Set initial workdir. WORKDIR /usr/src # Create blank project. -RUN USER=root cargo new varfish-server-worker +RUN USER=root /root/.cargo/bin/cargo new varfish-server-worker # We want dependencies cached, so copy those first. COPY Cargo.toml Cargo.lock /usr/src/varfish-server-worker/ # Set the working directory. WORKDIR /usr/src/varfish-server-worker # This is a dummy build to get the dependencies cached. -RUN cargo build --release +RUN /root/.cargo/bin/cargo build --release # # Now copy in the rest of the sources. COPY build.rs Cargo.toml Cargo.lock /usr/src/varfish-server-worker/ @@ -49,20 +47,22 @@ COPY protos /usr/src/varfish-server-worker/protos/ # COPY utils/alpine-linker-script.sh /usr/src/varfish-server-worker/utils/ # RUN chmod a+rx /usr/src/varfish-server-worker/utils/alpine-linker-script.sh # COPY .cargo /usr/src/varfish-server-worker/.cargo/ -## Touch main.rs to prevent cached release build. + +# Touch main.rs to prevent cached release build. RUN touch /usr/src/varfish-server-worker/src/main.rs -# This is the actual application build. -RUN cargo build --release + +# Build the application +RUN /root/.cargo/bin/cargo build --release # --------------------------------------------------------------------------- # Runtime # --------------------------------------------------------------------------- -FROM debian:bookworm-slim AS runtime +FROM ubuntu:noble AS runtime -# Install dependencies (and cleanup afterwards) +# Install dependencies (and cleanup afterward) RUN apt-get update && \ - apt-get install -y libsqlite3-0 && \ + apt-get install -y librocksdb8.9 libsqlite3-0 && \ apt-get clean autoclean && \ apt-get autoremove --yes && \ rm -rf /var/lib/{apt,dpkg,cache,log}