From d848b7f9acc988db022f18b8516164689e66115b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Michalina=20Ciencia=C5=82a?=
 <michalina.cienciala@keep.network>
Date: Thu, 10 Jun 2021 13:09:11 +0200
Subject: [PATCH] Temp fix for the problem with failing jobs - move cache
 instead of copying

We've seen a number of workflows failing with `No space left on device`
error (e.g.
https://github.com/keep-network/keep-ecdsa/actions/runs/924123923).
The problem was a combination of a big cache size and significant memory
allocation after building Docker image.
According to https://github.com/moby/buildkit/issues/1850:
"At the moment caches are copied over the existing cache so it keeps
growing".
As a temporary fix (until issue gets fixed by GH), we introduce a step
that moves the cache (similarily to how it's described in
https://github.com/moby/buildkit/issues/1896).

The solution may not keep us from failures forever, as we still may
hit the problems with the full memory if the size of the Docker image
or cache increases (see this FD discussion:
https://www.flowdock.com/app/cardforcoin/tech/threads/Clml1_lI4HYRKrr3N_vcSInDn6A).
---
 .github/workflows/client-celo.yml     | 14 ++++++++++++--
 .github/workflows/client-ethereum.yml | 12 +++++++++++-
 .github/workflows/contracts.yml       | 12 +++++++++++-
 3 files changed, 34 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/client-celo.yml b/.github/workflows/client-celo.yml
index 6a903d129..21e739fb8 100644
--- a/.github/workflows/client-celo.yml
+++ b/.github/workflows/client-celo.yml
@@ -80,7 +80,7 @@ jobs:
             HOST_CHAIN=celo
           load: true # load image to local registry to use it in next steps
           cache-from: type=local,src=/tmp/.buildx-celo-cache
-          cache-to: type=local,dest=/tmp/.buildx-celo-cache
+          cache-to: type=local,dest=/tmp/.buildx-celo-cache-new
 
       - name: Run Go tests
         run: |
@@ -111,4 +111,14 @@ jobs:
             HOST_CHAIN=celo
             REVISION=${{ github.sha }}
           # VERSION= ? TODO: Configure version, sample: 1.7.6
-          push: ${{ github.event_name == 'workflow_dispatch' }}
\ No newline at end of file
+          push: ${{ github.event_name == 'workflow_dispatch' }}
+
+      - # Temp fix - move cache instead of copying (added below step and
+        # modified value of `cache-to`).
+        # https://github.com/docker/build-push-action/issues/252
+        # https://github.com/moby/buildkit/issues/1896
+        # Without the change some jobs were failing with `no space left on device`
+        name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-celo-cache
+          mv /tmp/.buildx-celo-cache-new /tmp/.buildx-celo-cache
\ No newline at end of file
diff --git a/.github/workflows/client-ethereum.yml b/.github/workflows/client-ethereum.yml
index 1adf5dc85..13ea2cff2 100644
--- a/.github/workflows/client-ethereum.yml
+++ b/.github/workflows/client-ethereum.yml
@@ -78,7 +78,7 @@ jobs:
           tags: go-build-env-ethereum
           load: true # load image to local registry to use it in next steps
           cache-from: type=local,src=/tmp/.buildx-ethereum-cache
-          cache-to: type=local,dest=/tmp/.buildx-ethereum-cache
+          cache-to: type=local,dest=/tmp/.buildx-ethereum-cache-new
           
       - name: Run Go tests
         run: |
@@ -110,6 +110,16 @@ jobs:
           # VERSION= ? TODO: Configure version, sample: 1.7.6
           push: ${{ github.event_name == 'workflow_dispatch' }}
 
+      - # Temp fix - move cache instead of copying (added below step and
+        # modified value of `cache-to`).
+        # https://github.com/docker/build-push-action/issues/252
+        # https://github.com/moby/buildkit/issues/1896
+        # Without the change some jobs were failing with `no space left on device`
+        name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-ethereum-cache
+          mv /tmp/.buildx-ethereum-cache-new /tmp/.buildx-ethereum-cache
+
       - name: Notify CI about completion of the workflow
         if: github.event_name == 'workflow_dispatch'
         uses: keep-network/notify-workflow-completed@v1
diff --git a/.github/workflows/contracts.yml b/.github/workflows/contracts.yml
index a751a2418..f6ecab25b 100644
--- a/.github/workflows/contracts.yml
+++ b/.github/workflows/contracts.yml
@@ -281,7 +281,17 @@ jobs:
             revision=${{ github.sha }}
           push: true
           cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
+
+      - # Temp fix - move cache instead of copying (added below step and
+        # modified value of `cache-to`).
+        # https://github.com/docker/build-push-action/issues/252
+        # https://github.com/moby/buildkit/issues/1896
+        # Without the change some jobs were failing with `no space left on device`
+        name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
       
       - name: Notify CI about completion of the workflow
         uses: keep-network/notify-workflow-completed@v1