chore: switch docker base image, use system rocksdb for faster compil…

…ation (#487)

Co-authored-by: Manuel Holtgrewe <[email protected]>

.github/workflows/automerge.yml

@@ -7,7 +7,7 @@ permissions:
 
 jobs:
   dependabot:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     if: ${{ github.actor == 'dependabot[bot]' }}
     steps:
       - name: Enable auto-merge for Dependabot PRs

.github/workflows/conventional-prs.yml

@@ -9,7 +9,7 @@ on:
 
 jobs:
   title-format:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - uses: amannn/[email protected]
         env:

.github/workflows/docker-build.yml

@@ -18,7 +18,7 @@ env:
 
 jobs:
   build-and-push-image:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     permissions:
       contents: read
       packages: write

.github/workflows/protobuf.yml

@@ -10,11 +10,16 @@ on:
 
 jobs:
   Linting:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
+      - name: Setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
       - name: Install protolint
         run: |
           pip install protolint-bin

.github/workflows/release-please.yml

@@ -17,7 +17,7 @@ env:
 jobs:
   release-please:
     if: github.repository_owner == 'varfish-org'
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     outputs:
       release_created: ${{ steps.release.outputs.release_created }}
       release_name: ${{ steps.release.outputs.major }}.${{ steps.release.outputs.minor }}.${{ steps.release.outputs.patch }}
@@ -31,12 +31,17 @@ jobs:
 
   cargo-release:
     needs: release-please
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     if: ${{ needs.release-please.outputs.release_created }}
     steps:
       - uses: actions/checkout@v4
         if: ${{ needs.release-please.outputs.release_created }}
 
+      - name: Install dependencies
+        run: |
+          sudo apt-get update -y
+          sudo apt-get install -y librocksdb-dev libsnappy-dev libsqlite3-dev
+
       - name: Install stable toolchain
         uses: actions-rs/toolchain@v1
         if: ${{ needs.release-please.outputs.release_created }}
@@ -64,7 +69,7 @@ jobs:
 
   container-release:
     needs: release-please
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     if: ${{ needs.release-please.outputs.release_created }}
     steps:
       - name: Checkout repository

.github/workflows/rust.yml

@@ -10,7 +10,7 @@ on:
 
 jobs:
   Formatting:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -28,15 +28,20 @@ jobs:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Check format
-        run: |
+        run:
           cargo fmt -- --check
 
   Linting:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
+      - name: Install dependencies
+        run: |
+          sudo apt-get update -y
+          sudo apt-get install -y librocksdb-dev libsnappy-dev libsqlite3-dev
+
       - name: Install stable toolchain
         uses: actions-rs/toolchain@v1
         with:
@@ -49,20 +54,31 @@ jobs:
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
+      - uses: Swatinem/rust-cache@v2
+        with:
+          cache-directories: |
+            ~/.cargo/registry/src/**/librocksdb-sys-*
+
       - name: Lint with clippy
         uses: actions-rs/clippy-check@v1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
+          args: --all-features
 
   Testing:
     needs: Formatting
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
           lfs: true
 
+      - name: Install dependencies
+        run: |
+          sudo apt-get update -y
+          sudo apt-get install -y librocksdb-dev libsnappy-dev libsqlite3-dev
+
       - name: Install stable toolchain
         uses: actions-rs/toolchain@v1
         with:
@@ -74,16 +90,18 @@ jobs:
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
-      - uses: Swatinem/[email protected]
-
-      - name: Run cargo-tarpaulin
-        uses: actions-rs/[email protected]
+      - uses: Swatinem/rust-cache@v2
         with:
-          version: 0.21.0
-          args: "--skip-clean -- --test-threads 1"
+          key: "covtest"
+          cache-directories: |
+            ~/.cargo/registry/src/**/librocksdb-sys-*
 
-      - name: Codecov
-        uses: codecov/codecov-action@v3
+      - name: Install cargo-llvm-cov
+        uses: taiki-e/install-action@cargo-llvm-cov
+      - name: Generate code coverage
+        run: cargo llvm-cov --all-features --workspace --lcov --output-path lcov.info
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v4
         with:
-          verbose: true
           token: ${{ secrets.CODECOV_TOKEN }}
+          files: lcov.info

README.md

@@ -359,3 +359,39 @@ For running protolint, install it as python package `protolint-bin`:
 # source /tmp/varfish-server-worker/bin/activate
 # pip install protolint-bin
 ```
+
+### Building from scratch
+To reduce compile times, we recommend using a pre-built version of `rocksdb`, either from the system package manager or e.g. via `conda`:
+
+```bash
+# Ubuntu
+sudo apt-get install librocksdb-dev
+
+# Conda
+conda install -c conda-forge rocksdb
+```
+
+In either case, either add
+```toml
+[env]
+ROCKSDB_LIB_DIR = "/usr/lib/" # in case of the system package manager, adjust the path accordingly for conda
+SNAPPY_LIB_DIR = "/usr/lib/"  # same as above
+```
+to `.cargo/config.toml` or set the environment variables `ROCKSDB_LIB_DIR` and `SNAPPY_LIB_DIR` to the appropriate paths:
+
+```bash
+export ROCKSDB_LIB_DIR=/usr/lib/
+export SNAPPY_LIB_DIR=/usr/lib/
+```
+
+By default, the environment variables are defined in the `.cargo/config.toml` as described above, i.e. may need adjustments if not using the system package manager.
+
+To build the project, run:
+```bash
+cargo build --release
+```
+
+To install the project locally, run:
+```bash
+cargo install --path .
+```

utils/docker/Dockerfile

@@ -13,18 +13,16 @@
 # Builder
 # ---------------------------------------------------------------------------
 
-# Pinning Rust version for now because of this issue:
-#
-# - https://github.com/rust-lang/rust/issues/95926
-FROM rust:1-bookworm AS builder
+# Use ubuntu:noble as the base image
+FROM ubuntu:noble AS builder
 
-# Build dependencies first.
-#
-# Install dependencies for compilation of C code (e.g., rocksdb).
+# Install Rust toolchain and dependencies for compilation of C code (e.g., rocksdb)
 RUN apt-get update && \
-    apt-get install -y clang
-# Add the needed Cargo components.
-RUN rustup component add rustfmt
+    apt-get install -y unzip wget curl build-essential clang librocksdb-dev libsqlite3-dev && \
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \
+    . $HOME/.cargo/env && \
+    rustup component add rustfmt
+
 # Install build dependency `protoc`.
 COPY utils/install-protoc.sh /tmp
 RUN PREFIX=/usr/local bash /tmp/install-protoc.sh
@@ -34,13 +32,13 @@ RUN PREFIX=/usr/local bash /tmp/install-protoc.sh
 # Set initial workdir.
 WORKDIR /usr/src
 # Create blank project.
-RUN USER=root cargo new varfish-server-worker
+RUN USER=root /root/.cargo/bin/cargo new varfish-server-worker
 # We want dependencies cached, so copy those first.
 COPY Cargo.toml Cargo.lock /usr/src/varfish-server-worker/
 # Set the working directory.
 WORKDIR /usr/src/varfish-server-worker
 # This is a dummy build to get the dependencies cached.
-RUN cargo build --release
+RUN /root/.cargo/bin/cargo build --release
 #
 # Now copy in the rest of the sources.
 COPY build.rs Cargo.toml Cargo.lock /usr/src/varfish-server-worker/
@@ -49,20 +47,22 @@ COPY protos /usr/src/varfish-server-worker/protos/
 # COPY utils/alpine-linker-script.sh /usr/src/varfish-server-worker/utils/
 # RUN chmod a+rx /usr/src/varfish-server-worker/utils/alpine-linker-script.sh
 # COPY .cargo /usr/src/varfish-server-worker/.cargo/
-## Touch main.rs to prevent cached release build.
+
+# Touch main.rs to prevent cached release build.
 RUN touch /usr/src/varfish-server-worker/src/main.rs
-# This is the actual application build.
-RUN cargo build --release
+
+# Build the application
+RUN /root/.cargo/bin/cargo build --release
 
 # ---------------------------------------------------------------------------
 # Runtime
 # ---------------------------------------------------------------------------
 
-FROM debian:bookworm-slim AS runtime
+FROM ubuntu:noble AS runtime
 
-# Install dependencies (and cleanup afterwards)
+# Install dependencies (and cleanup afterward)
 RUN apt-get update && \
-    apt-get install -y libsqlite3-0 && \
+    apt-get install -y librocksdb8.9 libsqlite3-0 && \
     apt-get clean autoclean && \
     apt-get autoremove --yes && \
     rm -rf /var/lib/{apt,dpkg,cache,log}