feat: faster builds by copying @corneliusroemer image caching for our…

… dependency images, which rarely change (#374)
GenSpectrum · Mar 28, 2024 · 7867bc7 · 7867bc7
1 parent dec0726
commit 7867bc7
Show file tree

Hide file tree

Showing 5 changed files with 186 additions and 87 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -3,14 +3,14 @@ name: LAPIS-SILO
 on:
   push:
 
-
 concurrency:
   group: ci-${{ github.ref }}
   cancel-in-progress: true
 
 env:
-  DOCKER_IMAGE_NAME: ghcr.io/GenSpectrum/LAPIS-SILO
-  DOCKER_TEST_IMAGE_NAME: silo/unittests
+  DOCKER_DEPENDENCY_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo-dependencies
+  DOCKER_LINTER_DEPENDENCY_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo-linter-dependencies
+  DOCKER_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo
 
 jobs:
   formatting-check:
@@ -31,56 +31,172 @@ jobs:
           check-path: ${{ matrix.path['check'] }}
           exclude-regex: ${{ matrix.path['exclude'] }}
 
-  linter:
-    name: Build And Run linter
+  linterDependencies:
+    name: Build linter dependencies
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
 
+      - name: Generate dependency files hash
+        id: files-hash
+        run: |
+          DIR_HASH=$(echo -n ${{ hashFiles('conanfile.py', 'conanprofile.docker', '.github/workflows/ci.yml', './Dockerfile_linter_dependencies') }})
+          echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV
+
+      - name: Docker metadata
+        id: dockerMetadata
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.DOCKER_LINTER_DEPENDENCY_IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
+            type=raw,value=${{ env.DIR_HASH }}
+            type=sha,prefix=commit-
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Check if image exists
+        id: check-image
+        run: |
+          EXISTS=$(docker manifest inspect ${{ env.DOCKER_LINTER_DEPENDENCY_IMAGE_NAME }}:${{ env.DIR_HASH }} > /dev/null 2>&1 && echo "true" || echo "false")
+          echo "CACHE_HIT=$EXISTS" >> $GITHUB_ENV
+
       - name: Set up Docker Buildx
+        if: env.CACHE_HIT == 'false'
         uses: docker/setup-buildx-action@v3
 
       - name: Build linter dependencies image
+        if: env.CACHE_HIT == 'false'
         uses: docker/build-push-action@v5
         with:
           context: .
-          target: linter_dependencies
-          tags: linter_dependencies
-          file: ./Dockerfile_linter
+          push: true
+          tags: ${{ steps.dockerMetadata.outputs.tags }}
+          file: ./Dockerfile_linter_dependencies
           cache-from: type=gha,ref=linter-dependencies-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile_linter') }}
           cache-to: type=gha,mode=min,ref=linter-dependencies-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile_linter') }}
 
+  linter:
+    name: Build And Run linter
+    needs: linterDependencies
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Generate dependency files hash
+        id: files-hash
+        run: |
+          DIR_HASH=$(echo -n ${{ hashFiles('conanfile.py', 'conanprofile.docker', '.github/workflows/ci.yml', './Dockerfile_linter_dependencies') }})
+          echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
       - name: Build linter image
         uses: docker/build-push-action@v5
         with:
           context: .
           file: ./Dockerfile_linter
-          target: linter
           cache-from: type=gha,ref=${{ github.ref_name }}-linter-image-cache
           cache-to: type=gha,mode=min,ref=${{ github.ref_name }}-linter-image-cache
           push: false
+          build-args: |
+            DEPENDENCY_IMAGE=ghcr.io/genspectrum/lapis-silo-linter-dependencies:${{ env.DIR_HASH }}
 
-  dockerImage:
-    name: Build And Run Unit Tests
+  dependencyImage:
+    name: Build Docker Image Dependencies
     runs-on: ubuntu-latest
     permissions:
       packages: write
     steps:
       - uses: actions/checkout@v4
 
+      - name: Generate dependency files hash
+        id: files-hash
+        run: |
+          DIR_HASH=$(echo -n ${{ hashFiles('conanfile.py', 'conanprofile.docker', '.github/workflows/ci.yml', './Dockerfile_dependencies') }})
+          echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV
+
+      - name: Docker metadata
+        id: dockerMetadata
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
+            type=raw,value=${{ env.DIR_HASH }}
+            type=sha,prefix=commit-
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Check if image exists
+        id: check-image
+        run: |
+          EXISTS=$(docker manifest inspect ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:${{ env.DIR_HASH }} > /dev/null 2>&1 && echo "true" || echo "false")
+          echo "CACHE_HIT=$EXISTS" >> $GITHUB_ENV
+
       - name: Set up Docker Buildx
+        if: env.CACHE_HIT == 'false'
         uses: docker/setup-buildx-action@v3
 
-      - name: Build builder image
+      - name: Build and push image if input files changed
+        if: env.CACHE_HIT == 'false'
         uses: docker/build-push-action@v5
         with:
           context: .
-          target: dep_builder
-          tags: dep_builder
-          load: true
-          file: ./Dockerfile
+          file: Dockerfile_dependencies
+          push: true
+          tags: ${{ steps.dockerMetadata.outputs.tags }}
           cache-from: type=gha,ref=builder-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile') }}
           cache-to: type=gha,mode=min,ref=builder-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile') }}
+          platforms: ${{ github.ref == 'refs/heads/main' && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
+
+  dockerImageUnitTests:
+    name: Build Docker Image and Run Unit Tests
+    runs-on: ubuntu-latest
+    needs: dependencyImage
+    permissions:
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Generate dependency files hash
+        id: files-hash
+        run: |
+          DIR_HASH=$(echo -n ${{ hashFiles('conanfile.py', 'conanprofile.docker', '.github/workflows/ci.yml', './Dockerfile_dependencies') }})
+          echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV
+
+      - name: Docker metadata
+        id: dockerMetadata
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.DOCKER_IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }}
+            type=sha,prefix=commit-
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
 
       - name: Build unit test image
         uses: docker/build-push-action@v5
@@ -91,29 +207,15 @@ jobs:
           load: true
           cache-from: type=gha,ref=${{ github.ref_name }}-image-cache
           cache-to: type=gha,mode=min,ref=${{ github.ref_name }}-image-cache
+          build-args: |
+            DEPENDENCY_IMAGE=ghcr.io/genspectrum/lapis-silo-dependencies:${{ env.DIR_HASH }}
 
       - name: Run tests
         uses: addnab/docker-run-action@v3
         with:
           image: builder
           run: ./silo_test
 
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Docker metadata
-        id: dockerMetadata
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.DOCKER_IMAGE_NAME }}
-          tags: |
-            type=ref,event=branch
-            type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }}
-
       - name: Build and push production image
         if: ${{ github.event_name == 'push' }}
         uses: docker/build-push-action@v5
@@ -122,9 +224,12 @@ jobs:
           push: true
           platforms: ${{ github.ref == 'refs/heads/main' && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
           tags: ${{ steps.dockerMetadata.outputs.tags }}
+          build-args: |
+            DEPENDENCY_IMAGE=ghcr.io/genspectrum/lapis-silo-dependencies:${{ env.DIR_HASH }}
 
   endToEndTests:
     name: Run End To End Tests
+    needs: dockerImageUnitTests
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -158,13 +263,6 @@ jobs:
           images: ${{ env.DOCKER_IMAGE_NAME }}
           tags: type=ref,event=branch
 
-      - name: Wait for Docker Image
-        uses: lewagon/[email protected]
-        with:
-          ref: ${{ github.ref }}
-          check-name: Build And Run Unit Tests
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-
       - name: Start Docker Container and preprocess data
         run: docker compose -f ${{ matrix.preprocessing-docker-compose }} up
         env:

diff --git a/Dockerfile b/Dockerfile
@@ -1,28 +1,6 @@
-FROM alpine:3.18 AS dep_builder
+ARG DEPENDENCY_IMAGE=ghcr.io/genspectrum/lapis-silo-dependencies:latest
 
-ARG TARGETPLATFORM
-
-RUN apk update && apk add --no-cache py3-pip \
-    build-base=0.5-r3 \
-    cmake=3.26.5-r0 \
-    bash=5.2.15-r5 \
-    linux-headers=6.3-r0 \
-    boost-build=1.82.0-r0 \
-    libtbb=2021.9.0-r0
-
-RUN pip install conan==2.0.17
-
-WORKDIR /src
-COPY conanfile.py conanprofile.docker conanprofile.docker_arm ./
-RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
-        mv conanprofile.docker_arm conanprofile; \
-    else \
-        mv conanprofile.docker conanprofile; \
-    fi
-
-RUN conan install . --build=missing --profile ./conanprofile --profile:build ./conanprofile --output-folder=build
-
-FROM dep_builder AS builder
+FROM $DEPENDENCY_IMAGE AS builder
 
 COPY . ./
 

diff --git a/Dockerfile_dependencies b/Dockerfile_dependencies
@@ -0,0 +1,23 @@
+FROM alpine:3.18
+
+ARG TARGETPLATFORM
+
+RUN apk update && apk add --no-cache py3-pip \
+    build-base=0.5-r3 \
+    cmake=3.26.5-r0 \
+    bash=5.2.15-r5 \
+    linux-headers=6.3-r0 \
+    boost-build=1.82.0-r0 \
+    libtbb=2021.9.0-r0
+
+RUN pip install conan==2.0.17
+
+WORKDIR /src
+COPY conanfile.py conanprofile.docker conanprofile.docker_arm ./
+RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
+        mv conanprofile.docker_arm conanprofile; \
+    else \
+        mv conanprofile.docker conanprofile; \
+    fi
+
+RUN conan install . --build=missing --profile ./conanprofile --profile:build ./conanprofile --output-folder=build
diff --git a/Dockerfile_linter b/Dockerfile_linter
@@ -1,28 +1,6 @@
-FROM ubuntu:22.04 AS linter_dependencies
+ARG DEPENDENCY_IMAGE=ghcr.io/genspectrum/lapis-silo-linter-dependencies:latest
 
-WORKDIR /src
-
-RUN apt update \
-    && apt install -y \
-    cmake=3.22.1-1ubuntu1.22.04.2 \
-    python3-pip=22.0.2+dfsg-1ubuntu0.4 \
-    software-properties-common=0.99.22.9 \
-    wget=1.21.2-2ubuntu1 \
-    gnupg=2.2.27-3ubuntu2.1 \
-    lsb-release=11.1.0ubuntu4 \
-    && wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc \
-    && add-apt-repository 'deb http://apt.llvm.org/jammy/  llvm-toolchain-jammy main' \
-    && apt install -y clang-tidy-19
-
-RUN pip install conan==2.0.17
-
-COPY conanfile.py conanprofile.docker ./
-RUN mv conanprofile.docker conanprofile
-
-RUN conan install . --build=missing --profile ./conanprofile --profile:build ./conanprofile --output-folder=build -s build_type=Debug
-
-
-FROM linter_dependencies AS linter
+FROM $DEPENDENCY_IMAGE AS linter
 
 WORKDIR /src
 

diff --git a/Dockerfile_linter_dependencies b/Dockerfile_linter_dependencies
@@ -0,0 +1,22 @@
+FROM ubuntu:22.04
+
+WORKDIR /src
+
+RUN apt update \
+    && apt install -y \
+    cmake=3.22.1-1ubuntu1.22.04.2 \
+    python3-pip=22.0.2+dfsg-1ubuntu0.4 \
+    software-properties-common=0.99.22.9 \
+    wget=1.21.2-2ubuntu1 \
+    gnupg=2.2.27-3ubuntu2.1 \
+    lsb-release=11.1.0ubuntu4 \
+    && wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc \
+    && add-apt-repository 'deb http://apt.llvm.org/jammy/  llvm-toolchain-jammy main' \
+    && apt install -y clang-tidy-19
+
+RUN pip install conan==2.0.17
+
+COPY conanfile.py conanprofile.docker ./
+RUN mv conanprofile.docker conanprofile
+
+RUN conan install . --build=missing --profile ./conanprofile --profile:build ./conanprofile --output-folder=build -s build_type=Debug