Add caching to CI operations (#1696)

* Add basic caching * Fix syntax * Fix syntax * Load built image to docker images * This is a blank commit * Cache Go build and dependencies * Fix Go caching path and add temporary move fix for Docker layer caching * Try to build tester with docker compose before * Use cache for unit tests * Move caching for unit tests after repo checkout * Remove Go cache from integration tests * Always run unit tests; do not use cached results from previous runs with -count=1 * Always run unit tests; do not use cached results from previous runs with -count=1 * Add no cache Docker build for comparison * Test code change * Use cache in mode=min * Run integration tests with Go modules cached * Do not use buildx action * Change mount path * Run container with current user * Update docker compose * Echo user * Set permissions of files in bind mount * Clean up * Add Go build cache for local integration test runs. Avoid caching of integration test execution with count=1 * Use unique cache identifier to fall back to cache instead of getting cache hit for this commit when cache is not yet populated * Update cache keys * Adjust cache key * Enable autopeering integration test * Enable caching for all integration tests * Remove prints
iotaledger · Oct 11, 2021 · 1d816f6 · 1d816f6
1 parent 8103bcf
commit 1d816f6
Show file tree

Hide file tree

Showing 5 changed files with 191 additions and 42 deletions.
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -20,17 +20,32 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v2
 
+      - name: Cache Go modules for tester container
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/go
+          # make sure concurrent runs (not really supported) do not match the same key but instead fall back to a reasonable cache
+          key: ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-${{ github.job }}-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-
+            ${{ runner.os }}-it-go-
+
       - name: Build GoShimmer image
-        run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer .
+        run: |
+          docker build \
+            --build-arg DOWNLOAD_SNAPSHOT=0 \
+            -t iotaledger/goshimmer .
 
       - name: Pull additional Docker images
         run: |
-          docker pull angelocapossele/drand:v1.1.4
           docker pull gaiaadm/pumba:0.7.2
           docker pull gaiadocker/iproute2:latest
 
       - name: Run integration tests
-        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
+        run: |
+          export CURRENT_UID=$(id -u):$(id -g)
+          docker-compose -f tools/integration-tests/tester/docker-compose.ci.yml up --abort-on-container-exit --exit-code-from tester
 
       - name: Create logs from tester
         if: always()
@@ -44,7 +59,6 @@ jobs:
           name: ${{ env.TEST_NAME }}
           path: tools/integration-tests/logs
 
-
   common:
     name: common
     env:
@@ -55,16 +69,27 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v2
 
-      - name: Build GoShimmer image
-        run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer .
+      - name: Cache Go modules for tester container
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/go
+          # make sure concurrent runs (not really supported) do not match the same key but instead fall back to a reasonable cache
+          key: ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-${{ github.job }}-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-
+            ${{ runner.os }}-it-go-
 
-      - name: Pull additional Docker images
+      - name: Build GoShimmer image
         run: |
-          docker pull angelocapossele/drand:v1.1.4
-          docker pull gaiadocker/iproute2:latest
+          docker build \
+            --build-arg DOWNLOAD_SNAPSHOT=0 \
+            -t iotaledger/goshimmer .
 
       - name: Run integration tests
-        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
+        run: |
+          export CURRENT_UID=$(id -u):$(id -g)
+          docker-compose -f tools/integration-tests/tester/docker-compose.ci.yml up --abort-on-container-exit --exit-code-from tester
 
       - name: Create logs from tester
         if: always()
@@ -88,16 +113,31 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v2
 
+      - name: Cache Go modules for tester container
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/go
+          # make sure concurrent runs (not really supported) do not match the same key but instead fall back to a reasonable cache
+          key: ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-${{ github.job }}-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-
+            ${{ runner.os }}-it-go-
+
       - name: Build GoShimmer image
-        run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer .
+        run: |
+          docker build \
+            --build-arg DOWNLOAD_SNAPSHOT=0 \
+            -t iotaledger/goshimmer .
 
       - name: Pull additional Docker images
         run: |
           docker pull angelocapossele/drand:v1.1.4
-          docker pull gaiadocker/iproute2:latest
 
       - name: Run integration tests
-        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
+        run: |
+          export CURRENT_UID=$(id -u):$(id -g)
+          docker-compose -f tools/integration-tests/tester/docker-compose.ci.yml up --abort-on-container-exit --exit-code-from tester
 
       - name: Create logs from tester
         if: always()
@@ -122,16 +162,31 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v2
 
+      - name: Cache Go modules for tester container
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/go
+          # make sure concurrent runs (not really supported) do not match the same key but instead fall back to a reasonable cache
+          key: ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-${{ github.job }}-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-
+            ${{ runner.os }}-it-go-
+
       - name: Build GoShimmer image
-        run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer .
+        run: |
+          docker build \
+            --build-arg DOWNLOAD_SNAPSHOT=0 \
+            -t iotaledger/goshimmer .
 
       - name: Pull additional Docker images
         run: |
           docker pull angelocapossele/drand:v1.1.4
-          docker pull gaiadocker/iproute2:latest
 
       - name: Run integration tests
-        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
+        run: |
+          export CURRENT_UID=$(id -u):$(id -g)
+          docker-compose -f tools/integration-tests/tester/docker-compose.ci.yml up --abort-on-container-exit --exit-code-from tester
 
       - name: Create logs from tester
         if: always()
@@ -156,16 +211,27 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v2
 
-      - name: Build GoShimmer image
-        run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer .
+      - name: Cache Go modules for tester container
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/go
+          # make sure concurrent runs (not really supported) do not match the same key but instead fall back to a reasonable cache
+          key: ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-${{ github.job }}-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-
+            ${{ runner.os }}-it-go-
 
-      - name: Pull additional Docker images
+      - name: Build GoShimmer image
         run: |
-          docker pull angelocapossele/drand:v1.1.4
-          docker pull gaiadocker/iproute2:latest
+          docker build \
+            --build-arg DOWNLOAD_SNAPSHOT=0 \
+            -t iotaledger/goshimmer .
 
       - name: Run integration tests
-        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
+        run: |
+          export CURRENT_UID=$(id -u):$(id -g)
+          docker-compose -f tools/integration-tests/tester/docker-compose.ci.yml up --abort-on-container-exit --exit-code-from tester
 
       - name: Create logs from tester
         if: always()
@@ -189,16 +255,27 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v2
 
-      - name: Build GoShimmer image
-        run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer .
+      - name: Cache Go modules for tester container
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/go
+          # make sure concurrent runs (not really supported) do not match the same key but instead fall back to a reasonable cache
+          key: ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-${{ github.job }}-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-
+            ${{ runner.os }}-it-go-
 
-      - name: Pull additional Docker images
+      - name: Build GoShimmer image
         run: |
-          docker pull angelocapossele/drand:v1.1.4
-          docker pull gaiadocker/iproute2:latest
+          docker build \
+            --build-arg DOWNLOAD_SNAPSHOT=0 \
+            -t iotaledger/goshimmer .
 
       - name: Run integration tests
-        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
+        run: |
+          export CURRENT_UID=$(id -u):$(id -g)
+          docker-compose -f tools/integration-tests/tester/docker-compose.ci.yml up --abort-on-container-exit --exit-code-from tester
 
       - name: Create logs from tester
         if: always()
@@ -212,6 +289,7 @@ jobs:
           name: ${{ env.TEST_NAME }}
           path: tools/integration-tests/logs
 
+
   mana:
     name: mana
     env:
@@ -222,16 +300,27 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v2
 
-      - name: Build GoShimmer image
-        run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer .
+      - name: Cache Go modules for tester container
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/go
+          # make sure concurrent runs (not really supported) do not match the same key but instead fall back to a reasonable cache
+          key: ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-${{ github.job }}-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-
+            ${{ runner.os }}-it-go-
 
-      - name: Pull additional Docker images
+      - name: Build GoShimmer image
         run: |
-          docker pull angelocapossele/drand:1.1.3
-          docker pull gaiadocker/iproute2:latest
+          docker build \
+            --build-arg DOWNLOAD_SNAPSHOT=0 \
+            -t iotaledger/goshimmer .
 
       - name: Run integration tests
-        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
+        run: |
+          export CURRENT_UID=$(id -u):$(id -g)
+          docker-compose -f tools/integration-tests/tester/docker-compose.ci.yml up --abort-on-container-exit --exit-code-from tester
 
       - name: Create logs from tester
         if: always()
@@ -255,11 +344,27 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v2
 
+      - name: Cache Go modules for tester container
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/go
+          # make sure concurrent runs (not really supported) do not match the same key but instead fall back to a reasonable cache
+          key: ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-${{ github.job }}-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-
+            ${{ runner.os }}-it-go-
+
       - name: Build GoShimmer image
-        run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer .
+        run: |
+          docker build \
+            --build-arg DOWNLOAD_SNAPSHOT=0 \
+            -t iotaledger/goshimmer .
 
       - name: Run integration tests
-        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
+        run: |
+          export CURRENT_UID=$(id -u):$(id -g)
+          docker-compose -f tools/integration-tests/tester/docker-compose.ci.yml up --abort-on-container-exit --exit-code-from tester
 
       - name: Create logs from tester
         if: always()

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -26,5 +26,17 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v2
 
+      - name: Cache Go build and dependencies
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          # make sure concurrent runs (not really supported) do not match the same key but instead fall back to a reasonable cache
+          key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }}-${{ github.event.action }}${{ github.event.after }}
+          restore-keys: |
+            ${{ runner.os }}-go-${{ hashFiles('go.sum') }}
+            ${{ runner.os }}-go-
+
       - name: Run Tests
-        run: go test ./... -tags rocksdb
+        run: go test ./... -tags rocksdb -count=1
diff --git a/tools/integration-tests/assets/entrypoint.sh b/tools/integration-tests/assets/entrypoint.sh
@@ -6,4 +6,10 @@ chmod 777 /assets/*
 echo "assets:"
 ls /assets
 echo "running tests..."
-go test ./tests/"${TEST_NAME}" -v -timeout 30m
+go test ./tests/"${TEST_NAME}" -v -timeout 10m -count=1
+
+# if running in CI we need to set right permissions on the Go folder (within container) so that it can be exported to cache
+if [ ! -z ${CURRENT_UID} ]; then
+  echo "setting permissions on Go folder to '${CURRENT_UID}'..."
+  chown -R "${CURRENT_UID}" /go /root/.cache/go-build
+fi
diff --git a/tools/integration-tests/tester/docker-compose.ci.yml b/tools/integration-tests/tester/docker-compose.ci.yml
@@ -0,0 +1,23 @@
+version: "3.5"
+
+services:
+  tester:
+    container_name: tester
+    image: golang@sha256:cefedeae41e0bbbfa20bb1c37c3a43e0001fa541be9732f7bc6a28ecc154e9e4
+    working_dir: /tmp/goshimmer/tools/integration-tests/tester
+    command: /tmp/assets/entrypoint.sh
+    environment:
+      - TEST_NAME=${TEST_NAME}
+      - CURRENT_UID
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - ../../..:/tmp/goshimmer:rw
+      - ../logs:/tmp/logs
+      - ../assets:/tmp/assets
+      - ~/go/pkg:/go
+      - ~/go/cache:/root/.cache/go-build
+      - goshimmer-testing-assets:/assets
+
+volumes:
+  goshimmer-testing-assets:
+    name: goshimmer-testing-assets
diff --git a/tools/integration-tests/tester/docker-compose.yml b/tools/integration-tests/tester/docker-compose.yml
@@ -3,7 +3,7 @@ version: "3.5"
 services:
   tester:
     container_name: tester
-    image: golang:1.16.2
+    image: golang@sha256:cefedeae41e0bbbfa20bb1c37c3a43e0001fa541be9732f7bc6a28ecc154e9e4
     working_dir: /tmp/goshimmer/tools/integration-tests/tester
     command: /tmp/assets/entrypoint.sh
     environment:
@@ -13,11 +13,14 @@ services:
       - ../../..:/tmp/goshimmer:rw
       - ../logs:/tmp/logs
       - ../assets:/tmp/assets
-      - goshimmer-testing-cache:/go
+      - goshimmer-testing-package-cache:/go
+      - goshimmer-testing-build-cache:/root/.cache/go-build
       - goshimmer-testing-assets:/assets
 
 volumes:
-  goshimmer-testing-cache:
-    name: goshimmer-testing-cache
+  goshimmer-testing-package-cache:
+    name: goshimmer-testing-package-cache
+  goshimmer-testing-build-cache:
+    name: goshimmer-testing-build-cache
   goshimmer-testing-assets:
     name: goshimmer-testing-assets