From 1509a60710ba3ac48d5b4fea97ecbbe8d7cf7197 Mon Sep 17 00:00:00 2001
From: Haowen Liu <lhw@lunacd.com>
Date: Sat, 23 Mar 2024 15:07:38 -0400
Subject: [PATCH] Re-enable ccache

** Changes **

Docker does not natively support exporting cache mounts, or saving it to
some external cache storage like the GHA cache.

This commit uses a workaround described in [1] where the content in the
cache mount is moved into a dumb container and then exported through its
file system. To load recovered cache into the cache mount, another dumb
container is built and its build process copies recovered cache into the
cache mount.

This is very much a hack and shouldn't be necessary once docker natively
supported by buildx.

** Justification for not caching more build steps **

dnf, apt, and pip can theoretically also be cached this way. However,
if the list of dependencies have not been changed, the entire docker
layer would be a cache hit and dnf/apt caches would not be beneficial.
Considering that dependency changes should be rare, not caching them
seems wise.

[1] https://github.com/moby/buildkit/issues/1512
---
 .github/workflows/test.yml         | 17 +++++++++++++++++
 tests/docker/rockylinux.Dockerfile |  9 +++++----
 tests/docker/ubuntu.Dockerfile     | 10 ++++++----
 3 files changed, 28 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 5cdd56f..e93e50b 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -32,6 +32,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
+
       # GitHub runners have updated the Ubuntu Linux Kernel to use strong ASLR,
       # but LLVM is not configured for this change, and thus the address
       # sanitizer breaks.
@@ -49,8 +50,24 @@ jobs:
           sudo sysctl -a | grep vm.mmap.rnd
           sudo sysctl -w vm.mmap_rnd_bits=28
         working-directory: .
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
+
+      - name: Cache ccache-${{ matrix.cfg.id }}
+        id: cache-ccache
+        uses: actions/cache@v3
+        with:
+          path: ccache-${{ matrix.cfg.id }}
+          key: ccache-${{ matrix.cfg.id }}-${{ hashFiles('**/*.cpp', '**/*.h', '**/meson.build', 'subprojects/*.wrap') }}
+          restore-keys: ccache-${{ matrix.cfg.id }}-
+      - name: Inject ccache into docker
+        uses: reproducible-containers/buildkit-cache-dance@v2.1.4
+        with:
+          cache-source: ccache-${{ matrix.cfg.id }}
+          cache-target: /ccache
+          skip-extraction: ${{ steps.cache-ccache.outputs.cache-hit }}
+
       - name: Build and push
         uses: docker/build-push-action@v5
         with:
diff --git a/tests/docker/rockylinux.Dockerfile b/tests/docker/rockylinux.Dockerfile
index ea623ac..621c96f 100644
--- a/tests/docker/rockylinux.Dockerfile
+++ b/tests/docker/rockylinux.Dockerfile
@@ -1,7 +1,6 @@
 FROM rockylinux:9
 
 # Enable EPEL
-RUN dnf update -y
 RUN dnf install -y 'dnf-command(config-manager)'
 RUN dnf config-manager --set-enabled crb -y
 RUN dnf install epel-release -y
@@ -9,7 +8,7 @@ RUN dnf install epel-release -y
 # Install dependencies
 RUN dnf install -y \
         python3.11 \
-        python3-pip \
+        python3.11-pip \
         pkgconf-pkg-config \
         ccache \
         clang \
@@ -23,8 +22,9 @@ RUN dnf install -y \
         cxxopts-devel
 RUN dnf clean all
 RUN update-alternatives --install /usr/local/bin/python python /usr/bin/python3.11 10
+
 # Install meson from pip
-RUN python3 -m pip install -U meson==0.64.1
+RUN python -m pip install -U meson==0.64.1
 
 # Copy code
 WORKDIR /workarea
@@ -38,4 +38,5 @@ ARG setup_options=
 ENV CC="ccache $cc" CXX="ccache $cxx"
 ENV CCACHE_DIR=/ccache
 RUN meson setup builddir $setup_options
-RUN --mount=type=cache,target=/ccache/ ninja -C builddir
+RUN --mount=type=cache,target=/ccache,sharing=locked \
+    ninja -C builddir
diff --git a/tests/docker/ubuntu.Dockerfile b/tests/docker/ubuntu.Dockerfile
index 0190226..bcb2036 100644
--- a/tests/docker/ubuntu.Dockerfile
+++ b/tests/docker/ubuntu.Dockerfile
@@ -1,8 +1,8 @@
 FROM ubuntu:22.04
 
 # Install dependencies
-RUN apt-get update
-RUN apt-get install -y \
+RUN apt-get update && \
+    apt-get install -y \
         python3.11 \
         python3-pip \
         pkg-config \
@@ -18,8 +18,9 @@ RUN apt-get install -y \
         libcxxopts-dev
 RUN apt-get clean
 RUN update-alternatives --install /usr/local/bin/python python /usr/bin/python3.11 10
+
 # Install meson from pip
-RUN python3 -m pip install -U meson==0.64.1
+RUN python -m pip install -U meson==0.64.1
 
 # Copy code
 WORKDIR /workarea
@@ -36,4 +37,5 @@ RUN sysctl vm.mmap_rnd_bits=28
 ENV CC="ccache $cc" CXX="ccache $cxx"
 ENV CCACHE_DIR=/ccache
 RUN meson setup builddir $setup_options
-RUN --mount=type=cache,target=/ccache/ ninja -C builddir
+RUN --mount=type=cache,target=/ccache,sharing=locked \
+    ninja -C builddir