From a03388537d15c35cd3e24e6b3785b33dba288a68 Mon Sep 17 00:00:00 2001
From: jonastheis <4181434+jonastheis@users.noreply.github.com>
Date: Fri, 17 Sep 2021 18:15:19 +0000
Subject: [PATCH 01/31] Add basic caching

---
 .github/workflows/integration-tests.yml | 416 +++++++++++++-----------
 1 file changed, 219 insertions(+), 197 deletions(-)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index d2ba568d4e..2f45ff6b15 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -10,39 +10,39 @@ env:
 
 jobs:
 
-  autopeering:
-    name: autopeering
-    env:
-      TEST_NAME: autopeering
-    runs-on: ubuntu-latest
-    steps:
-
-      - name: Checkout repository
-        uses: actions/checkout@v2
-
-      - name: Build GoShimmer image
-        run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer .
-
-      - name: Pull additional Docker images
-        run: |
-          docker pull angelocapossele/drand:v1.1.4
-          docker pull gaiaadm/pumba:0.7.2
-          docker pull gaiadocker/iproute2:latest
-
-      - name: Run integration tests
-        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
-
-      - name: Create logs from tester
-        if: always()
-        run: |
-          docker logs tester &> tools/integration-tests/logs/tester.log
-
-      - name: Save logs as artifacts
-        if: always()
-        uses: actions/upload-artifact@v1
-        with:
-          name: ${{ env.TEST_NAME }}
-          path: tools/integration-tests/logs
+#  autopeering:
+#    name: autopeering
+#    env:
+#      TEST_NAME: autopeering
+#    runs-on: ubuntu-latest
+#    steps:
+#
+#      - name: Checkout repository
+#        uses: actions/checkout@v2
+#
+#      - name: Build GoShimmer image
+#        run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer .
+#
+#      - name: Pull additional Docker images
+#        run: |
+#          docker pull angelocapossele/drand:v1.1.4
+#          docker pull gaiaadm/pumba:0.7.2
+#          docker pull gaiadocker/iproute2:latest
+#
+#      - name: Run integration tests
+#        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
+#
+#      - name: Create logs from tester
+#        if: always()
+#        run: |
+#          docker logs tester &> tools/integration-tests/logs/tester.log
+#
+#      - name: Save logs as artifacts
+#        if: always()
+#        uses: actions/upload-artifact@v1
+#        with:
+#          name: ${{ env.TEST_NAME }}
+#          path: tools/integration-tests/logs
 
 
   common:
@@ -55,75 +55,30 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v2
 
-      - name: Build GoShimmer image
-        run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer .
-
-      - name: Pull additional Docker images
-        run: |
-          docker pull angelocapossele/drand:v1.1.4
-          docker pull gaiadocker/iproute2:latest
-
-      - name: Run integration tests
-        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
-
-      - name: Create logs from tester
-        if: always()
-        run: |
-          docker logs tester &> tools/integration-tests/logs/tester.log
-
-      - name: Save logs as artifacts
-        if: always()
-        uses: actions/upload-artifact@v1
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
         with:
-          name: ${{ env.TEST_NAME }}
-          path: tools/integration-tests/logs
-
-  consensus:
-    name: consensus
-    env:
-      TEST_NAME: consensus
-    runs-on: ubuntu-latest
-    steps:
-
-      - name: Checkout repository
-        uses: actions/checkout@v2
-
-      - name: Build GoShimmer image
-        run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer .
-
-      - name: Pull additional Docker images
-        run: |
-          docker pull angelocapossele/drand:v1.1.4
-          docker pull gaiadocker/iproute2:latest
-
-      - name: Run integration tests
-        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
-
-      - name: Create logs from tester
-        if: always()
-        run: |
-          docker logs tester &> tools/integration-tests/logs/tester.log
-
-      - name: Save logs as artifacts
-        if: always()
-        uses: actions/upload-artifact@v1
+          install: true
+          buildkitd-flags: --debug
+          config-inline: |
+            [worker.oci]
+              max-parallelism = 4
+
+      - name: Cache Docker layers
+        uses: actions/cache@v2
         with:
-          name: ${{ env.TEST_NAME }}
-          path: tools/integration-tests/logs
-
-
-  drng:
-    name: drng
-    env:
-      TEST_NAME: drng
-    runs-on: ubuntu-latest
-    steps:
-
-      - name: Checkout repository
-        uses: actions/checkout@v2
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
 
       - name: Build GoShimmer image
-        run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer .
+        run: docker build \
+        --build-arg DOWNLOAD_SNAPSHOT=0 \
+        --cache-from=type=local,src=/tmp/.buildx-cache \
+        --cache-to=type=local,mode=max,dest=/tmp/.buildx-cache
+        -t iotaledger/goshimmer .
 
       - name: Pull additional Docker images
         run: |
@@ -145,105 +100,172 @@ jobs:
           name: ${{ env.TEST_NAME }}
           path: tools/integration-tests/logs
 
-
-  value:
-    name: value
-    env:
-      TEST_NAME: value
-    runs-on: ubuntu-latest
-    steps:
-
-      - name: Checkout repository
-        uses: actions/checkout@v2
-
-      - name: Build GoShimmer image
-        run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer .
-
-      - name: Pull additional Docker images
-        run: |
-          docker pull angelocapossele/drand:v1.1.4
-          docker pull gaiadocker/iproute2:latest
-
-      - name: Run integration tests
-        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
-
-      - name: Create logs from tester
-        if: always()
-        run: |
-          docker logs tester &> tools/integration-tests/logs/tester.log
-
-      - name: Save logs as artifacts
-        if: always()
-        uses: actions/upload-artifact@v1
-        with:
-          name: ${{ env.TEST_NAME }}
-          path: tools/integration-tests/logs
-
-  faucet:
-    name: faucet
-    env:
-      TEST_NAME: faucet
-    runs-on: ubuntu-latest
-    steps:
-
-      - name: Checkout repository
-        uses: actions/checkout@v2
-
-      - name: Build GoShimmer image
-        run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer .
-
-      - name: Pull additional Docker images
-        run: |
-          docker pull angelocapossele/drand:v1.1.4
-          docker pull gaiadocker/iproute2:latest
-
-      - name: Run integration tests
-        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
-
-      - name: Create logs from tester
-        if: always()
-        run: |
-          docker logs tester &> tools/integration-tests/logs/tester.log
-
-      - name: Save logs as artifacts
-        if: always()
-        uses: actions/upload-artifact@v1
-        with:
-          name: ${{ env.TEST_NAME }}
-          path: tools/integration-tests/logs
-
-  mana:
-    name: mana
-    env:
-      TEST_NAME: mana
-    runs-on: ubuntu-latest
-    steps:
-
-      - name: Check out code
-        uses: actions/checkout@v2
-
-      - name: Build GoShimmer image
-        run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer .
-
-      - name: Pull additional Docker images
-        run: |
-          docker pull angelocapossele/drand:1.1.3
-          docker pull gaiadocker/iproute2:latest
-
-      - name: Run integration tests
-        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
-
-      - name: Create logs from tester
-        if: always()
-        run: |
-          docker logs tester &> tools/integration-tests/logs/tester.log
-
-      - name: Save logs as artifacts
-        if: always()
-        uses: actions/upload-artifact@v1
-        with:
-          name: ${{ env.TEST_NAME }}
-          path: tools/integration-tests/logs
+#  consensus:
+#    name: consensus
+#    env:
+#      TEST_NAME: consensus
+#    runs-on: ubuntu-latest
+#    steps:
+#
+#      - name: Checkout repository
+#        uses: actions/checkout@v2
+#
+#      - name: Build GoShimmer image
+#        run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer .
+#
+#      - name: Pull additional Docker images
+#        run: |
+#          docker pull angelocapossele/drand:v1.1.4
+#          docker pull gaiadocker/iproute2:latest
+#
+#      - name: Run integration tests
+#        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
+#
+#      - name: Create logs from tester
+#        if: always()
+#        run: |
+#          docker logs tester &> tools/integration-tests/logs/tester.log
+#
+#      - name: Save logs as artifacts
+#        if: always()
+#        uses: actions/upload-artifact@v1
+#        with:
+#          name: ${{ env.TEST_NAME }}
+#          path: tools/integration-tests/logs
+#
+#
+#  drng:
+#    name: drng
+#    env:
+#      TEST_NAME: drng
+#    runs-on: ubuntu-latest
+#    steps:
+#
+#      - name: Checkout repository
+#        uses: actions/checkout@v2
+#
+#      - name: Build GoShimmer image
+#        run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer .
+#
+#      - name: Pull additional Docker images
+#        run: |
+#          docker pull angelocapossele/drand:v1.1.4
+#          docker pull gaiadocker/iproute2:latest
+#
+#      - name: Run integration tests
+#        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
+#
+#      - name: Create logs from tester
+#        if: always()
+#        run: |
+#          docker logs tester &> tools/integration-tests/logs/tester.log
+#
+#      - name: Save logs as artifacts
+#        if: always()
+#        uses: actions/upload-artifact@v1
+#        with:
+#          name: ${{ env.TEST_NAME }}
+#          path: tools/integration-tests/logs
+#
+#
+#  value:
+#    name: value
+#    env:
+#      TEST_NAME: value
+#    runs-on: ubuntu-latest
+#    steps:
+#
+#      - name: Checkout repository
+#        uses: actions/checkout@v2
+#
+#      - name: Build GoShimmer image
+#        run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer .
+#
+#      - name: Pull additional Docker images
+#        run: |
+#          docker pull angelocapossele/drand:v1.1.4
+#          docker pull gaiadocker/iproute2:latest
+#
+#      - name: Run integration tests
+#        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
+#
+#      - name: Create logs from tester
+#        if: always()
+#        run: |
+#          docker logs tester &> tools/integration-tests/logs/tester.log
+#
+#      - name: Save logs as artifacts
+#        if: always()
+#        uses: actions/upload-artifact@v1
+#        with:
+#          name: ${{ env.TEST_NAME }}
+#          path: tools/integration-tests/logs
+#
+#  faucet:
+#    name: faucet
+#    env:
+#      TEST_NAME: faucet
+#    runs-on: ubuntu-latest
+#    steps:
+#
+#      - name: Checkout repository
+#        uses: actions/checkout@v2
+#
+#      - name: Build GoShimmer image
+#        run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer .
+#
+#      - name: Pull additional Docker images
+#        run: |
+#          docker pull angelocapossele/drand:v1.1.4
+#          docker pull gaiadocker/iproute2:latest
+#
+#      - name: Run integration tests
+#        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
+#
+#      - name: Create logs from tester
+#        if: always()
+#        run: |
+#          docker logs tester &> tools/integration-tests/logs/tester.log
+#
+#      - name: Save logs as artifacts
+#        if: always()
+#        uses: actions/upload-artifact@v1
+#        with:
+#          name: ${{ env.TEST_NAME }}
+#          path: tools/integration-tests/logs
+#
+#  mana:
+#    name: mana
+#    env:
+#      TEST_NAME: mana
+#    runs-on: ubuntu-latest
+#    steps:
+#
+#      - name: Check out code
+#        uses: actions/checkout@v2
+#
+#      - name: Build GoShimmer image
+#        run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer .
+#
+#      - name: Pull additional Docker images
+#        run: |
+#          docker pull angelocapossele/drand:1.1.3
+#          docker pull gaiadocker/iproute2:latest
+#
+#      - name: Run integration tests
+#        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
+#
+#      - name: Create logs from tester
+#        if: always()
+#        run: |
+#          docker logs tester &> tools/integration-tests/logs/tester.log
+#
+#      - name: Save logs as artifacts
+#        if: always()
+#        uses: actions/upload-artifact@v1
+#        with:
+#          name: ${{ env.TEST_NAME }}
+#          path: tools/integration-tests/logs
 
   diagnostics:
     name: diagnostics

From b768a9bca57ff17e267d84ec587312ec98286ff1 Mon Sep 17 00:00:00 2001
From: jonastheis <4181434+jonastheis@users.noreply.github.com>
Date: Fri, 17 Sep 2021 18:19:46 +0000
Subject: [PATCH 02/31] Fix syntax

---
 .github/workflows/integration-tests.yml | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 2f45ff6b15..7f380b8163 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -74,11 +74,12 @@ jobs:
             ${{ runner.os }}-buildx-
 
       - name: Build GoShimmer image
-        run: docker build \
-        --build-arg DOWNLOAD_SNAPSHOT=0 \
-        --cache-from=type=local,src=/tmp/.buildx-cache \
-        --cache-to=type=local,mode=max,dest=/tmp/.buildx-cache
-        -t iotaledger/goshimmer .
+        run: |
+          docker build \
+            --build-arg DOWNLOAD_SNAPSHOT=0 \
+            --cache-from=type=local,src=/tmp/.buildx-cache \
+            --cache-to=type=local,mode=max,dest=/tmp/.buildx-cache
+            -t iotaledger/goshimmer .
 
       - name: Pull additional Docker images
         run: |

From 65aea384ae574a35e36291c7bfb6e989888b7a83 Mon Sep 17 00:00:00 2001
From: jonastheis <4181434+jonastheis@users.noreply.github.com>
Date: Fri, 17 Sep 2021 18:21:57 +0000
Subject: [PATCH 03/31] Fix syntax

---
 .github/workflows/integration-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 7f380b8163..cd38c8750c 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -78,7 +78,7 @@ jobs:
           docker build \
             --build-arg DOWNLOAD_SNAPSHOT=0 \
             --cache-from=type=local,src=/tmp/.buildx-cache \
-            --cache-to=type=local,mode=max,dest=/tmp/.buildx-cache
+            --cache-to=type=local,mode=max,dest=/tmp/.buildx-cache \
             -t iotaledger/goshimmer .
 
       - name: Pull additional Docker images

From 4e7c7228fc8e412f685d34b4b1b9001d727f0852 Mon Sep 17 00:00:00 2001
From: jonastheis <4181434+jonastheis@users.noreply.github.com>
Date: Fri, 17 Sep 2021 18:33:33 +0000
Subject: [PATCH 04/31] Load built image to docker images

---
 .github/workflows/integration-tests.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index cd38c8750c..3b45f6b182 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -79,6 +79,7 @@ jobs:
             --build-arg DOWNLOAD_SNAPSHOT=0 \
             --cache-from=type=local,src=/tmp/.buildx-cache \
             --cache-to=type=local,mode=max,dest=/tmp/.buildx-cache \
+            --load \
             -t iotaledger/goshimmer .
 
       - name: Pull additional Docker images

From 459e2804dbab7bdeeb097a39e154929d1ef8b78f Mon Sep 17 00:00:00 2001
From: jonastheis <4181434+jonastheis@users.noreply.github.com>
Date: Fri, 17 Sep 2021 18:43:17 +0000
Subject: [PATCH 05/31] This is a blank commit


From 7df9239434ac03ca73d6f52bcaccfa1270988a67 Mon Sep 17 00:00:00 2001
From: jonastheis <4181434+jonastheis@users.noreply.github.com>
Date: Fri, 17 Sep 2021 19:09:52 +0000
Subject: [PATCH 06/31] Cache Go build and dependencies

---
 .github/workflows/integration-tests.yml | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 3b45f6b182..527e14e67a 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -65,6 +65,17 @@ jobs:
             [worker.oci]
               max-parallelism = 4
 
+      - name: Cache Go build and dependencies
+        uses: actions/cache@v2
+        with:
+          path: |
+            /root/.cache/go-build
+            /root/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+
+# TODO: check which key to use
       - name: Cache Docker layers
         uses: actions/cache@v2
         with:
@@ -73,6 +84,7 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-buildx-
 
+# TODO: is mode=max required? maybe we can shrink the cache size
       - name: Build GoShimmer image
         run: |
           docker build \

From 7a23e71d244b94f778772b1d6c1280d5eceae4c6 Mon Sep 17 00:00:00 2001
From: jonastheis <4181434+jonastheis@users.noreply.github.com>
Date: Fri, 17 Sep 2021 19:26:29 +0000
Subject: [PATCH 07/31] Fix Go caching path and add temporary move fix for
 Docker layer caching

---
 .github/workflows/integration-tests.yml | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 527e14e67a..1611c34abe 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -69,8 +69,8 @@ jobs:
         uses: actions/cache@v2
         with:
           path: |
-            /root/.cache/go-build
-            /root/go/pkg/mod
+            ~/.cache/go-build
+            ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-go-
@@ -90,14 +90,17 @@ jobs:
           docker build \
             --build-arg DOWNLOAD_SNAPSHOT=0 \
             --cache-from=type=local,src=/tmp/.buildx-cache \
-            --cache-to=type=local,mode=max,dest=/tmp/.buildx-cache \
+            --cache-to=type=local,mode=max,dest=/tmp/.buildx-cache-new \
             --load \
             -t iotaledger/goshimmer .
 
-      - name: Pull additional Docker images
+      # Temporary fix for bloated cache size
+      # https://github.com/docker/build-push-action/issues/252
+      # https://github.com/moby/buildkit/issues/1896
+      - name: Move cache
         run: |
-          docker pull angelocapossele/drand:v1.1.4
-          docker pull gaiadocker/iproute2:latest
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
 
       - name: Run integration tests
         run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build

From 50263ed1b7a63194a01b59e66231a0c30f7d396b Mon Sep 17 00:00:00 2001
From: jonastheis <4181434+jonastheis@users.noreply.github.com>
Date: Fri, 17 Sep 2021 19:37:44 +0000
Subject: [PATCH 08/31] Try to build tester with docker compose before

---
 .github/workflows/integration-tests.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 1611c34abe..18ac175155 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -103,7 +103,9 @@ jobs:
           mv /tmp/.buildx-cache-new /tmp/.buildx-cache
 
       - name: Run integration tests
-        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
+        run: |
+          docker-compose build -f tools/integration-tests/tester/docker-compose.yml
+          docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester
 
       - name: Create logs from tester
         if: always()

From 3b10101ea5c80910aa172906d94b7ac7f09b1faf Mon Sep 17 00:00:00 2001
From: jonastheis <4181434+jonastheis@users.noreply.github.com>
Date: Fri, 17 Sep 2021 20:51:30 +0000
Subject: [PATCH 09/31] Use cache for unit tests

---
 .github/workflows/test.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 3d05f04901..42c5b54a05 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -23,6 +23,16 @@ jobs:
         with:
           go-version: 1.16.3
 
+      - name: Cache Go build and dependencies
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+
       - name: Checkout repository
         uses: actions/checkout@v2
 

From 933ec7d60af5c20c3f49c44fc8f9bbf3be747960 Mon Sep 17 00:00:00 2001
From: jonastheis <4181434+jonastheis@users.noreply.github.com>
Date: Fri, 17 Sep 2021 20:54:42 +0000
Subject: [PATCH 10/31] Move caching for unit tests after repo checkout

---
 .github/workflows/test.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 42c5b54a05..e755f03db2 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -23,6 +23,9 @@ jobs:
         with:
           go-version: 1.16.3
 
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
       - name: Cache Go build and dependencies
         uses: actions/cache@v2
         with:
@@ -33,8 +36,5 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-go-
 
-      - name: Checkout repository
-        uses: actions/checkout@v2
-
       - name: Run Tests
         run: go test ./... -tags rocksdb

From d5c593a229b2893d6f2afd31948638c2dbee135f Mon Sep 17 00:00:00 2001
From: jonastheis <4181434+jonastheis@users.noreply.github.com>
Date: Mon, 20 Sep 2021 17:48:51 +0000
Subject: [PATCH 11/31] Remove Go cache from integration tests

---
 .github/workflows/integration-tests.yml | 96 +++++++++++--------------
 1 file changed, 43 insertions(+), 53 deletions(-)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 18ac175155..5852ab9aca 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -65,16 +65,6 @@ jobs:
             [worker.oci]
               max-parallelism = 4
 
-      - name: Cache Go build and dependencies
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
-
 # TODO: check which key to use
       - name: Cache Docker layers
         uses: actions/cache@v2
@@ -102,22 +92,22 @@ jobs:
           rm -rf /tmp/.buildx-cache
           mv /tmp/.buildx-cache-new /tmp/.buildx-cache
 
-      - name: Run integration tests
-        run: |
-          docker-compose build -f tools/integration-tests/tester/docker-compose.yml
-          docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester
-
-      - name: Create logs from tester
-        if: always()
-        run: |
-          docker logs tester &> tools/integration-tests/logs/tester.log
-
-      - name: Save logs as artifacts
-        if: always()
-        uses: actions/upload-artifact@v1
-        with:
-          name: ${{ env.TEST_NAME }}
-          path: tools/integration-tests/logs
+#      - name: Run integration tests
+#        run: |
+#          docker-compose build -f tools/integration-tests/tester/docker-compose.yml
+#          docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester
+#
+#      - name: Create logs from tester
+#        if: always()
+#        run: |
+#          docker logs tester &> tools/integration-tests/logs/tester.log
+#
+#      - name: Save logs as artifacts
+#        if: always()
+#        uses: actions/upload-artifact@v1
+#        with:
+#          name: ${{ env.TEST_NAME }}
+#          path: tools/integration-tests/logs
 
 #  consensus:
 #    name: consensus
@@ -286,30 +276,30 @@ jobs:
 #          name: ${{ env.TEST_NAME }}
 #          path: tools/integration-tests/logs
 
-  diagnostics:
-    name: diagnostics
-    env:
-      TEST_NAME: diagnostics
-    runs-on: ubuntu-latest
-    steps:
-
-      - name: Check out code
-        uses: actions/checkout@v2
-
-      - name: Build GoShimmer image
-        run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer .
-
-      - name: Run integration tests
-        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
-
-      - name: Create logs from tester
-        if: always()
-        run: |
-          docker logs tester &> tools/integration-tests/logs/tester.log
-
-      - name: Save logs as artifacts
-        if: always()
-        uses: actions/upload-artifact@v1
-        with:
-          name: ${{ env.TEST_NAME }}
-          path: tools/integration-tests/logs
+#  diagnostics:
+#    name: diagnostics
+#    env:
+#      TEST_NAME: diagnostics
+#    runs-on: ubuntu-latest
+#    steps:
+#
+#      - name: Check out code
+#        uses: actions/checkout@v2
+#
+#      - name: Build GoShimmer image
+#        run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer .
+#
+#      - name: Run integration tests
+#        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
+#
+#      - name: Create logs from tester
+#        if: always()
+#        run: |
+#          docker logs tester &> tools/integration-tests/logs/tester.log
+#
+#      - name: Save logs as artifacts
+#        if: always()
+#        uses: actions/upload-artifact@v1
+#        with:
+#          name: ${{ env.TEST_NAME }}
+#          path: tools/integration-tests/logs

From 0a1c6b217664cff404b42cce86343372ac17af02 Mon Sep 17 00:00:00 2001
From: jonastheis <4181434+jonastheis@users.noreply.github.com>
Date: Tue, 21 Sep 2021 12:06:52 +0000
Subject: [PATCH 12/31] Always run unit tests; do not use cached results from
 previous runs with -count=1

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index e755f03db2..c999009401 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -37,4 +37,4 @@ jobs:
             ${{ runner.os }}-go-
 
       - name: Run Tests
-        run: go test ./... -tags rocksdb
+        run: go test ./... -tags rocksdb -count=1

From 5619f76fda1280b41ac46a6c6ab0516b5cc5c976 Mon Sep 17 00:00:00 2001
From: jonastheis <4181434+jonastheis@users.noreply.github.com>
Date: Tue, 21 Sep 2021 12:38:15 +0000
Subject: [PATCH 13/31] Always run unit tests; do not use cached results from
 previous runs with -count=1

---
 packages/tangle/booker.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/tangle/booker.go b/packages/tangle/booker.go
index 08f3ce82f0..56a009e2a2 100644
--- a/packages/tangle/booker.go
+++ b/packages/tangle/booker.go
@@ -96,6 +96,7 @@ func (b *Booker) run() {
 // BookMessage tries to book the given Message (and potentially its contained Transaction) into the LedgerState and the Tangle.
 // It fires a MessageBooked event if it succeeds.
 func (b *Booker) BookMessage(messageID MessageID) (err error) {
+	fmt.Println("Book message", messageID)
 	b.tangle.Storage.Message(messageID).Consume(func(message *Message) {
 		b.tangle.Storage.MessageMetadata(messageID).Consume(func(messageMetadata *MessageMetadata) {
 			// don't book the same message more than once!

From 853ff15999a6ab7cb4fdea282bb87e1d43e55dfb Mon Sep 17 00:00:00 2001
From: jonastheis <4181434+jonastheis@users.noreply.github.com>
Date: Tue, 21 Sep 2021 12:44:28 +0000
Subject: [PATCH 14/31] Add no cache Docker build for comparison

---
 .github/workflows/integration-tests.yml | 27 ++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 5852ab9aca..6495d8e21b 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -44,6 +44,32 @@ jobs:
 #          name: ${{ env.TEST_NAME }}
 #          path: tools/integration-tests/logs
 
+  common-no-cache:
+    name: common (no cache)
+    env:
+      TEST_NAME: common
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          install: true
+          buildkitd-flags: --debug
+          config-inline: |
+            [worker.oci]
+              max-parallelism = 4
+
+      - name: Build GoShimmer image
+        run: |
+          docker build \
+            --build-arg DOWNLOAD_SNAPSHOT=0 \
+            -t iotaledger/goshimmer .
+
 
   common:
     name: common
@@ -65,7 +91,6 @@ jobs:
             [worker.oci]
               max-parallelism = 4
 
-# TODO: check which key to use
       - name: Cache Docker layers
         uses: actions/cache@v2
         with:

From 2a8dbbb2cea0db399135352576d3b7d3b4895bae Mon Sep 17 00:00:00 2001
From: jonastheis <4181434+jonastheis@users.noreply.github.com>
Date: Tue, 21 Sep 2021 13:02:07 +0000
Subject: [PATCH 15/31] Test code change

---
 packages/tangle/booker.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/packages/tangle/booker.go b/packages/tangle/booker.go
index 56a009e2a2..ecd7e6e3c8 100644
--- a/packages/tangle/booker.go
+++ b/packages/tangle/booker.go
@@ -96,6 +96,9 @@ func (b *Booker) run() {
 // BookMessage tries to book the given Message (and potentially its contained Transaction) into the LedgerState and the Tangle.
 // It fires a MessageBooked event if it succeeds.
 func (b *Booker) BookMessage(messageID MessageID) (err error) {
+	fmt.Println("Book message", messageID)
+	fmt.Println("Book message", messageID)
+	fmt.Println("Book message", messageID)
 	fmt.Println("Book message", messageID)
 	b.tangle.Storage.Message(messageID).Consume(func(message *Message) {
 		b.tangle.Storage.MessageMetadata(messageID).Consume(func(messageMetadata *MessageMetadata) {

From 5c3d7ba36e16c0941c2edb02b3f62fef9a56f1f5 Mon Sep 17 00:00:00 2001
From: jonastheis <4181434+jonastheis@users.noreply.github.com>
Date: Tue, 21 Sep 2021 13:25:42 +0000
Subject: [PATCH 16/31] Use cache in mode=min

---
 .github/workflows/integration-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 6495d8e21b..0cec515dfc 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -105,7 +105,7 @@ jobs:
           docker build \
             --build-arg DOWNLOAD_SNAPSHOT=0 \
             --cache-from=type=local,src=/tmp/.buildx-cache \
-            --cache-to=type=local,mode=max,dest=/tmp/.buildx-cache-new \
+            --cache-to=type=local,mode=min,dest=/tmp/.buildx-cache-new \
             --load \
             -t iotaledger/goshimmer .
 

From 9d27313e075403ece0819e4850ea0b6dc00ceaac Mon Sep 17 00:00:00 2001
From: jonastheis <4181434+jonastheis@users.noreply.github.com>
Date: Tue, 21 Sep 2021 13:45:15 +0000
Subject: [PATCH 17/31] Run integration tests with Go modules cached

---
 .github/workflows/integration-tests.yml       | 31 +++++++------------
 .../tester/docker-compose.ci.yml              | 21 +++++++++++++
 .../tester/docker-compose.yml                 |  2 +-
 3 files changed, 33 insertions(+), 21 deletions(-)
 create mode 100644 tools/integration-tests/tester/docker-compose.ci.yml

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 0cec515dfc..1aa80d896e 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -70,9 +70,13 @@ jobs:
             --build-arg DOWNLOAD_SNAPSHOT=0 \
             -t iotaledger/goshimmer .
 
+      - name: Run integration tests
+        run: |
+          docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester
+
 
   common:
-    name: common
+    name: common (cached Go modules)
     env:
       TEST_NAME: common
     runs-on: ubuntu-latest
@@ -91,36 +95,23 @@ jobs:
             [worker.oci]
               max-parallelism = 4
 
-      - name: Cache Docker layers
+      - name: Cache Go modules for tester container
         uses: actions/cache@v2
         with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          path: /tmp/go
+          key: ${{ runner.os }}-it-go-${{ github.sha }}
           restore-keys: |
-            ${{ runner.os }}-buildx-
+            ${{ runner.os }}-it-go-
 
-# TODO: is mode=max required? maybe we can shrink the cache size
       - name: Build GoShimmer image
         run: |
           docker build \
             --build-arg DOWNLOAD_SNAPSHOT=0 \
-            --cache-from=type=local,src=/tmp/.buildx-cache \
-            --cache-to=type=local,mode=min,dest=/tmp/.buildx-cache-new \
-            --load \
             -t iotaledger/goshimmer .
 
-      # Temporary fix for bloated cache size
-      # https://github.com/docker/build-push-action/issues/252
-      # https://github.com/moby/buildkit/issues/1896
-      - name: Move cache
+      - name: Run integration tests
         run: |
-          rm -rf /tmp/.buildx-cache
-          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
-
-#      - name: Run integration tests
-#        run: |
-#          docker-compose build -f tools/integration-tests/tester/docker-compose.yml
-#          docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester
+          docker-compose -f tools/integration-tests/tester/docker-compose.ci.yml up --abort-on-container-exit --exit-code-from tester
 #
 #      - name: Create logs from tester
 #        if: always()
diff --git a/tools/integration-tests/tester/docker-compose.ci.yml b/tools/integration-tests/tester/docker-compose.ci.yml
new file mode 100644
index 0000000000..362ad3a195
--- /dev/null
+++ b/tools/integration-tests/tester/docker-compose.ci.yml
@@ -0,0 +1,21 @@
+version: "3.5"
+
+services:
+  tester:
+    container_name: tester
+    image: golang@sha256:cefedeae41e0bbbfa20bb1c37c3a43e0001fa541be9732f7bc6a28ecc154e9e4
+    working_dir: /tmp/goshimmer/tools/integration-tests/tester
+    command: /tmp/assets/entrypoint.sh
+    environment:
+      - TEST_NAME=${TEST_NAME}
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - ../../..:/tmp/goshimmer:rw
+      - ../logs:/tmp/logs
+      - ../assets:/tmp/assets
+      - /tmp/go:/go
+      - goshimmer-testing-assets:/assets
+
+volumes:
+  goshimmer-testing-assets:
+    name: goshimmer-testing-assets
diff --git a/tools/integration-tests/tester/docker-compose.yml b/tools/integration-tests/tester/docker-compose.yml
index 3db4017fc1..82641aa0da 100644
--- a/tools/integration-tests/tester/docker-compose.yml
+++ b/tools/integration-tests/tester/docker-compose.yml
@@ -3,7 +3,7 @@ version: "3.5"
 services:
   tester:
     container_name: tester
-    image: golang:1.16.2
+    image: golang@sha256:cefedeae41e0bbbfa20bb1c37c3a43e0001fa541be9732f7bc6a28ecc154e9e4
     working_dir: /tmp/goshimmer/tools/integration-tests/tester
     command: /tmp/assets/entrypoint.sh
     environment:

From c388a58d79bd6d73fd86fef8f71b7a0f62829885 Mon Sep 17 00:00:00 2001
From: jonastheis <4181434+jonastheis@users.noreply.github.com>
Date: Tue, 21 Sep 2021 13:52:20 +0000
Subject: [PATCH 18/31] Do not use buildx action

---
 .github/workflows/integration-tests.yml   | 20 --------------------
 tools/integration-tests/tester/Dockerfile | 17 +++++++++++++++++
 2 files changed, 17 insertions(+), 20 deletions(-)
 create mode 100644 tools/integration-tests/tester/Dockerfile

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 1aa80d896e..c7826ab09a 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -54,16 +54,6 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v2
 
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          install: true
-          buildkitd-flags: --debug
-          config-inline: |
-            [worker.oci]
-              max-parallelism = 4
-
       - name: Build GoShimmer image
         run: |
           docker build \
@@ -85,16 +75,6 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v2
 
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          install: true
-          buildkitd-flags: --debug
-          config-inline: |
-            [worker.oci]
-              max-parallelism = 4
-
       - name: Cache Go modules for tester container
         uses: actions/cache@v2
         with:
diff --git a/tools/integration-tests/tester/Dockerfile b/tools/integration-tests/tester/Dockerfile
new file mode 100644
index 0000000000..2de5bac364
--- /dev/null
+++ b/tools/integration-tests/tester/Dockerfile
@@ -0,0 +1,17 @@
+# syntax = docker/dockerfile:1.2.1
+
+############################
+# golang 1.17-buster amd64
+FROM golang@sha256:cefedeae41e0bbbfa20bb1c37c3a43e0001fa541be9732f7bc6a28ecc154e9e4
+
+# Set the current Working Directory inside the container
+#RUN mkdir /tmp/goshimmer/tools/integration-tests/tester
+WORKDIR /tmp/goshimmer/tools/integration-tests/tester
+
+# Use Go Modules
+COPY go.mod .
+COPY go.sum .
+
+ENV GO111MODULE=on
+RUN go mod download
+RUN go mod verify

From 1d5f9d81270a2cad1eebe0f584e9b9bdbecc7622 Mon Sep 17 00:00:00 2001
From: jonastheis <4181434+jonastheis@users.noreply.github.com>
Date: Tue, 21 Sep 2021 14:56:12 +0000
Subject: [PATCH 19/31] Change mount path

---
 .github/workflows/integration-tests.yml              | 3 ++-
 tools/integration-tests/tester/docker-compose.ci.yml | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index c7826ab09a..deec0d892f 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -78,7 +78,8 @@ jobs:
       - name: Cache Go modules for tester container
         uses: actions/cache@v2
         with:
-          path: /tmp/go
+          path: |
+            ~/go
           key: ${{ runner.os }}-it-go-${{ github.sha }}
           restore-keys: |
             ${{ runner.os }}-it-go-
diff --git a/tools/integration-tests/tester/docker-compose.ci.yml b/tools/integration-tests/tester/docker-compose.ci.yml
index 362ad3a195..53e572c682 100644
--- a/tools/integration-tests/tester/docker-compose.ci.yml
+++ b/tools/integration-tests/tester/docker-compose.ci.yml
@@ -13,7 +13,8 @@ services:
       - ../../..:/tmp/goshimmer:rw
       - ../logs:/tmp/logs
       - ../assets:/tmp/assets
-      - /tmp/go:/go
+      - ~/go/pkg:/go
+      - ~/go/cache:/root/.cache/go-build
       - goshimmer-testing-assets:/assets
 
 volumes:

From 3a378f3037f5b4017ac013984752881f9c64997c Mon Sep 17 00:00:00 2001
From: jonastheis <4181434+jonastheis@users.noreply.github.com>
Date: Tue, 21 Sep 2021 17:34:40 +0000
Subject: [PATCH 20/31] Run container with current user

---
 .github/workflows/integration-tests.yml           | 1 +
 tools/integration-tests/tester/docker-compose.yml | 1 +
 2 files changed, 2 insertions(+)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index deec0d892f..c0fd24daf9 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -92,6 +92,7 @@ jobs:
 
       - name: Run integration tests
         run: |
+          export CURRENT_UID=$(id -u):$(id -g)
           docker-compose -f tools/integration-tests/tester/docker-compose.ci.yml up --abort-on-container-exit --exit-code-from tester
 #
 #      - name: Create logs from tester
diff --git a/tools/integration-tests/tester/docker-compose.yml b/tools/integration-tests/tester/docker-compose.yml
index 82641aa0da..d9b158e307 100644
--- a/tools/integration-tests/tester/docker-compose.yml
+++ b/tools/integration-tests/tester/docker-compose.yml
@@ -6,6 +6,7 @@ services:
     image: golang@sha256:cefedeae41e0bbbfa20bb1c37c3a43e0001fa541be9732f7bc6a28ecc154e9e4
     working_dir: /tmp/goshimmer/tools/integration-tests/tester
     command: /tmp/assets/entrypoint.sh
+    user: ${CURRENT_UID}
     environment:
       - TEST_NAME=${TEST_NAME}
     volumes:

From 6aee8ca1a5648a7aa881cf71038c52d901c3629f Mon Sep 17 00:00:00 2001
From: jonastheis <4181434+jonastheis@users.noreply.github.com>
Date: Tue, 21 Sep 2021 17:40:16 +0000
Subject: [PATCH 21/31] Update docker compose

---
 tools/integration-tests/tester/docker-compose.ci.yml | 1 +
 tools/integration-tests/tester/docker-compose.yml    | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/integration-tests/tester/docker-compose.ci.yml b/tools/integration-tests/tester/docker-compose.ci.yml
index 53e572c682..4812025a67 100644
--- a/tools/integration-tests/tester/docker-compose.ci.yml
+++ b/tools/integration-tests/tester/docker-compose.ci.yml
@@ -6,6 +6,7 @@ services:
     image: golang@sha256:cefedeae41e0bbbfa20bb1c37c3a43e0001fa541be9732f7bc6a28ecc154e9e4
     working_dir: /tmp/goshimmer/tools/integration-tests/tester
     command: /tmp/assets/entrypoint.sh
+    user: ${CURRENT_UID}
     environment:
       - TEST_NAME=${TEST_NAME}
     volumes:
diff --git a/tools/integration-tests/tester/docker-compose.yml b/tools/integration-tests/tester/docker-compose.yml
index d9b158e307..82641aa0da 100644
--- a/tools/integration-tests/tester/docker-compose.yml
+++ b/tools/integration-tests/tester/docker-compose.yml
@@ -6,7 +6,6 @@ services:
     image: golang@sha256:cefedeae41e0bbbfa20bb1c37c3a43e0001fa541be9732f7bc6a28ecc154e9e4
     working_dir: /tmp/goshimmer/tools/integration-tests/tester
     command: /tmp/assets/entrypoint.sh
-    user: ${CURRENT_UID}
     environment:
       - TEST_NAME=${TEST_NAME}
     volumes:

From 4658c6ca627300d76bace195606619144154bc9d Mon Sep 17 00:00:00 2001
From: jonastheis <4181434+jonastheis@users.noreply.github.com>
Date: Tue, 21 Sep 2021 17:57:16 +0000
Subject: [PATCH 22/31] Echo user

---
 .github/workflows/integration-tests.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index c0fd24daf9..91c9e09afc 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -93,6 +93,7 @@ jobs:
       - name: Run integration tests
         run: |
           export CURRENT_UID=$(id -u):$(id -g)
+          echo $CURRENT_UID
           docker-compose -f tools/integration-tests/tester/docker-compose.ci.yml up --abort-on-container-exit --exit-code-from tester
 #
 #      - name: Create logs from tester

From 29dd737b46e726804d783640604b2c95df730d84 Mon Sep 17 00:00:00 2001
From: jonastheis <4181434+jonastheis@users.noreply.github.com>
Date: Wed, 22 Sep 2021 09:10:52 +0000
Subject: [PATCH 23/31] Set permissions of files in bind mount

---
 tools/integration-tests/assets/entrypoint.sh         | 9 ++++++++-
 tools/integration-tests/tester/docker-compose.ci.yml | 2 +-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/tools/integration-tests/assets/entrypoint.sh b/tools/integration-tests/assets/entrypoint.sh
index a5ec0407b8..3b06d5e261 100755
--- a/tools/integration-tests/assets/entrypoint.sh
+++ b/tools/integration-tests/assets/entrypoint.sh
@@ -6,4 +6,11 @@ chmod 777 /assets/*
 echo "assets:"
 ls /assets
 echo "running tests..."
-go test ./tests/"${TEST_NAME}" -v -timeout 30m
+go test ./tests/"${TEST_NAME}" -v -timeout 10m
+
+# if running in CI we need to set right permissions on the Go folder (within container)
+echo "CURRENT_UID ${CURRENT_UID}"
+if [ ! -z ${CURRENT_UID} ]; then
+  echo "setting permissions on Go folder... '${CURRENT_UID}'"
+  chown -R "${CURRENT_UID}" /go /root/.cache/go-build
+fi
diff --git a/tools/integration-tests/tester/docker-compose.ci.yml b/tools/integration-tests/tester/docker-compose.ci.yml
index 4812025a67..ecaa243a93 100644
--- a/tools/integration-tests/tester/docker-compose.ci.yml
+++ b/tools/integration-tests/tester/docker-compose.ci.yml
@@ -6,9 +6,9 @@ services:
     image: golang@sha256:cefedeae41e0bbbfa20bb1c37c3a43e0001fa541be9732f7bc6a28ecc154e9e4
     working_dir: /tmp/goshimmer/tools/integration-tests/tester
     command: /tmp/assets/entrypoint.sh
-    user: ${CURRENT_UID}
     environment:
       - TEST_NAME=${TEST_NAME}
+      - CURRENT_UID
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro
       - ../../..:/tmp/goshimmer:rw

From 9271fe44b5f8b164ef399d26d0f24a5cc6e034e9 Mon Sep 17 00:00:00 2001
From: jonastheis <4181434+jonastheis@users.noreply.github.com>
Date: Wed, 22 Sep 2021 09:23:55 +0000
Subject: [PATCH 24/31] Clean up

---
 .github/workflows/integration-tests.yml      | 1 -
 tools/integration-tests/assets/entrypoint.sh | 5 ++---
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 91c9e09afc..c0fd24daf9 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -93,7 +93,6 @@ jobs:
       - name: Run integration tests
         run: |
           export CURRENT_UID=$(id -u):$(id -g)
-          echo $CURRENT_UID
           docker-compose -f tools/integration-tests/tester/docker-compose.ci.yml up --abort-on-container-exit --exit-code-from tester
 #
 #      - name: Create logs from tester
diff --git a/tools/integration-tests/assets/entrypoint.sh b/tools/integration-tests/assets/entrypoint.sh
index 3b06d5e261..971e9865af 100755
--- a/tools/integration-tests/assets/entrypoint.sh
+++ b/tools/integration-tests/assets/entrypoint.sh
@@ -8,9 +8,8 @@ ls /assets
 echo "running tests..."
 go test ./tests/"${TEST_NAME}" -v -timeout 10m
 
-# if running in CI we need to set right permissions on the Go folder (within container)
-echo "CURRENT_UID ${CURRENT_UID}"
+# if running in CI we need to set right permissions on the Go folder (within container) so that it can be exported to cache
 if [ ! -z ${CURRENT_UID} ]; then
-  echo "setting permissions on Go folder... '${CURRENT_UID}'"
+  echo "setting permissions on Go folder to '${CURRENT_UID}'..."
   chown -R "${CURRENT_UID}" /go /root/.cache/go-build
 fi

From 3a3db8bac38d10e2c88771fce66345a0ad9ce535 Mon Sep 17 00:00:00 2001
From: jonastheis <4181434+jonastheis@users.noreply.github.com>
Date: Wed, 22 Sep 2021 09:33:22 +0000
Subject: [PATCH 25/31] Add Go build cache for local integration test runs.
 Avoid caching of integration test execution with count=1

---
 tools/integration-tests/assets/entrypoint.sh      | 2 +-
 tools/integration-tests/tester/docker-compose.yml | 9 ++++++---
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/tools/integration-tests/assets/entrypoint.sh b/tools/integration-tests/assets/entrypoint.sh
index 971e9865af..f86dbb179b 100755
--- a/tools/integration-tests/assets/entrypoint.sh
+++ b/tools/integration-tests/assets/entrypoint.sh
@@ -6,7 +6,7 @@ chmod 777 /assets/*
 echo "assets:"
 ls /assets
 echo "running tests..."
-go test ./tests/"${TEST_NAME}" -v -timeout 10m
+go test ./tests/"${TEST_NAME}" -v -timeout 10m -count=1
 
 # if running in CI we need to set right permissions on the Go folder (within container) so that it can be exported to cache
 if [ ! -z ${CURRENT_UID} ]; then
diff --git a/tools/integration-tests/tester/docker-compose.yml b/tools/integration-tests/tester/docker-compose.yml
index 82641aa0da..eaeaf281c4 100644
--- a/tools/integration-tests/tester/docker-compose.yml
+++ b/tools/integration-tests/tester/docker-compose.yml
@@ -13,11 +13,14 @@ services:
       - ../../..:/tmp/goshimmer:rw
       - ../logs:/tmp/logs
       - ../assets:/tmp/assets
-      - goshimmer-testing-cache:/go
+      - goshimmer-testing-package-cache:/go
+      - goshimmer-testing-build-cache:/root/.cache/go-build
       - goshimmer-testing-assets:/assets
 
 volumes:
-  goshimmer-testing-cache:
-    name: goshimmer-testing-cache
+  goshimmer-testing-package-cache:
+    name: goshimmer-testing-package-cache
+  goshimmer-testing-build-cache:
+    name: goshimmer-testing-build-cache
   goshimmer-testing-assets:
     name: goshimmer-testing-assets

From 6752a60d6c318664a71ba068f4099a267c0e70c1 Mon Sep 17 00:00:00 2001
From: jonastheis <4181434+jonastheis@users.noreply.github.com>
Date: Wed, 22 Sep 2021 10:02:25 +0000
Subject: [PATCH 26/31] Use unique cache identifier to fall back to cache
 instead of getting cache hit for this commit when cache is not yet populated

---
 .github/workflows/integration-tests.yml | 2 +-
 .github/workflows/test.yml              | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index c0fd24daf9..0c290550e9 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -80,7 +80,7 @@ jobs:
         with:
           path: |
             ~/go
-          key: ${{ runner.os }}-it-go-${{ github.sha }}
+          key: ${{ runner.os }}-it-go-${{ github.job }}-${{ github.sha }}
           restore-keys: |
             ${{ runner.os }}-it-go-
 
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index c999009401..f013091d16 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -32,8 +32,9 @@ jobs:
           path: |
             ~/.cache/go-build
             ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}-${{ github.action }}
           restore-keys: |
+            ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
             ${{ runner.os }}-go-
 
       - name: Run Tests

From 794ce61168b0b48cb0aac347e2a6d72be1dc5225 Mon Sep 17 00:00:00 2001
From: jonastheis <4181434+jonastheis@users.noreply.github.com>
Date: Wed, 22 Sep 2021 10:18:57 +0000
Subject: [PATCH 27/31] Update cache keys

---
 .github/workflows/integration-tests.yml   |  4 +++-
 .github/workflows/test.yml                |  5 +++--
 tools/integration-tests/tester/Dockerfile | 17 -----------------
 3 files changed, 6 insertions(+), 20 deletions(-)
 delete mode 100644 tools/integration-tests/tester/Dockerfile

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 0c290550e9..d4650bd335 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -80,8 +80,10 @@ jobs:
         with:
           path: |
             ~/go
-          key: ${{ runner.os }}-it-go-${{ github.job }}-${{ github.sha }}
+          # make sure concurrent runs (not really supported) do not match the same key but instead fall back to a reasonable cache
+          key: ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-${{ github.job }}-${{ github.sha }}
           restore-keys: |
+            ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-
             ${{ runner.os }}-it-go-
 
       - name: Build GoShimmer image
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index f013091d16..f9a035cc3e 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -32,9 +32,10 @@ jobs:
           path: |
             ~/.cache/go-build
             ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}-${{ github.action }}
+          # make sure concurrent runs (not really supported) do not match the same key but instead fall back to a reasonable cache
+          key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }}-${{ github.event.action }}
           restore-keys: |
-            ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+            ${{ runner.os }}-go-${{ hashFiles('go.sum') }}
             ${{ runner.os }}-go-
 
       - name: Run Tests
diff --git a/tools/integration-tests/tester/Dockerfile b/tools/integration-tests/tester/Dockerfile
deleted file mode 100644
index 2de5bac364..0000000000
--- a/tools/integration-tests/tester/Dockerfile
+++ /dev/null
@@ -1,17 +0,0 @@
-# syntax = docker/dockerfile:1.2.1
-
-############################
-# golang 1.17-buster amd64
-FROM golang@sha256:cefedeae41e0bbbfa20bb1c37c3a43e0001fa541be9732f7bc6a28ecc154e9e4
-
-# Set the current Working Directory inside the container
-#RUN mkdir /tmp/goshimmer/tools/integration-tests/tester
-WORKDIR /tmp/goshimmer/tools/integration-tests/tester
-
-# Use Go Modules
-COPY go.mod .
-COPY go.sum .
-
-ENV GO111MODULE=on
-RUN go mod download
-RUN go mod verify

From 9a640e286144e467e38af39105069cc51d39a1f9 Mon Sep 17 00:00:00 2001
From: jonastheis <4181434+jonastheis@users.noreply.github.com>
Date: Wed, 22 Sep 2021 10:27:01 +0000
Subject: [PATCH 28/31] Adjust cache key

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index f9a035cc3e..d042172cd2 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -33,7 +33,7 @@ jobs:
             ~/.cache/go-build
             ~/go/pkg/mod
           # make sure concurrent runs (not really supported) do not match the same key but instead fall back to a reasonable cache
-          key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }}-${{ github.event.action }}
+          key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }}-${{ github.event.action }}${{ github.event.after }}
           restore-keys: |
             ${{ runner.os }}-go-${{ hashFiles('go.sum') }}
             ${{ runner.os }}-go-

From ba4913308e568e771015d8cbdf9f0e1a3d25afcb Mon Sep 17 00:00:00 2001
From: jonastheis <4181434+jonastheis@users.noreply.github.com>
Date: Wed, 22 Sep 2021 10:33:26 +0000
Subject: [PATCH 29/31] Enable autopeering integration test

---
 .github/workflows/integration-tests.yml | 96 ++++++++++++-------------
 1 file changed, 45 insertions(+), 51 deletions(-)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index d4650bd335..5430d94481 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -10,63 +10,57 @@ env:
 
 jobs:
 
-#  autopeering:
-#    name: autopeering
-#    env:
-#      TEST_NAME: autopeering
-#    runs-on: ubuntu-latest
-#    steps:
-#
-#      - name: Checkout repository
-#        uses: actions/checkout@v2
-#
-#      - name: Build GoShimmer image
-#        run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer .
-#
-#      - name: Pull additional Docker images
-#        run: |
-#          docker pull angelocapossele/drand:v1.1.4
-#          docker pull gaiaadm/pumba:0.7.2
-#          docker pull gaiadocker/iproute2:latest
-#
-#      - name: Run integration tests
-#        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
-#
-#      - name: Create logs from tester
-#        if: always()
-#        run: |
-#          docker logs tester &> tools/integration-tests/logs/tester.log
-#
-#      - name: Save logs as artifacts
-#        if: always()
-#        uses: actions/upload-artifact@v1
-#        with:
-#          name: ${{ env.TEST_NAME }}
-#          path: tools/integration-tests/logs
-
-  common-no-cache:
-    name: common (no cache)
+  autopeering:
+    name: autopeering
     env:
-      TEST_NAME: common
+      TEST_NAME: autopeering
     runs-on: ubuntu-latest
     steps:
 
       - name: Checkout repository
         uses: actions/checkout@v2
 
+      - name: Cache Go modules for tester container
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/go
+          # make sure concurrent runs (not really supported) do not match the same key but instead fall back to a reasonable cache
+          key: ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-${{ github.job }}-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-
+            ${{ runner.os }}-it-go-
+
       - name: Build GoShimmer image
         run: |
           docker build \
             --build-arg DOWNLOAD_SNAPSHOT=0 \
             -t iotaledger/goshimmer .
 
+      - name: Pull additional Docker images
+        run: |
+          docker pull gaiaadm/pumba:0.7.2
+          docker pull gaiadocker/iproute2:latest
+
       - name: Run integration tests
         run: |
-          docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester
+          export CURRENT_UID=$(id -u):$(id -g)
+          docker-compose -f tools/integration-tests/tester/docker-compose.ci.yml up --abort-on-container-exit --exit-code-from tester
+
+      - name: Create logs from tester
+        if: always()
+        run: |
+          docker logs tester &> tools/integration-tests/logs/tester.log
 
+      - name: Save logs as artifacts
+        if: always()
+        uses: actions/upload-artifact@v1
+        with:
+          name: ${{ env.TEST_NAME }}
+          path: tools/integration-tests/logs
 
   common:
-    name: common (cached Go modules)
+    name: common
     env:
       TEST_NAME: common
     runs-on: ubuntu-latest
@@ -96,18 +90,18 @@ jobs:
         run: |
           export CURRENT_UID=$(id -u):$(id -g)
           docker-compose -f tools/integration-tests/tester/docker-compose.ci.yml up --abort-on-container-exit --exit-code-from tester
-#
-#      - name: Create logs from tester
-#        if: always()
-#        run: |
-#          docker logs tester &> tools/integration-tests/logs/tester.log
-#
-#      - name: Save logs as artifacts
-#        if: always()
-#        uses: actions/upload-artifact@v1
-#        with:
-#          name: ${{ env.TEST_NAME }}
-#          path: tools/integration-tests/logs
+
+      - name: Create logs from tester
+        if: always()
+        run: |
+          docker logs tester &> tools/integration-tests/logs/tester.log
+
+      - name: Save logs as artifacts
+        if: always()
+        uses: actions/upload-artifact@v1
+        with:
+          name: ${{ env.TEST_NAME }}
+          path: tools/integration-tests/logs
 
 #  consensus:
 #    name: consensus

From a61bdb175683431cdc76d6b8a6bd6a084f9f426c Mon Sep 17 00:00:00 2001
From: jonastheis <4181434+jonastheis@users.noreply.github.com>
Date: Wed, 22 Sep 2021 10:46:21 +0000
Subject: [PATCH 30/31] Enable caching for all integration tests

---
 .github/workflows/integration-tests.yml | 468 ++++++++++++++----------
 1 file changed, 274 insertions(+), 194 deletions(-)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 5430d94481..af8bc1c412 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -103,197 +103,277 @@ jobs:
           name: ${{ env.TEST_NAME }}
           path: tools/integration-tests/logs
 
-#  consensus:
-#    name: consensus
-#    env:
-#      TEST_NAME: consensus
-#    runs-on: ubuntu-latest
-#    steps:
-#
-#      - name: Checkout repository
-#        uses: actions/checkout@v2
-#
-#      - name: Build GoShimmer image
-#        run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer .
-#
-#      - name: Pull additional Docker images
-#        run: |
-#          docker pull angelocapossele/drand:v1.1.4
-#          docker pull gaiadocker/iproute2:latest
-#
-#      - name: Run integration tests
-#        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
-#
-#      - name: Create logs from tester
-#        if: always()
-#        run: |
-#          docker logs tester &> tools/integration-tests/logs/tester.log
-#
-#      - name: Save logs as artifacts
-#        if: always()
-#        uses: actions/upload-artifact@v1
-#        with:
-#          name: ${{ env.TEST_NAME }}
-#          path: tools/integration-tests/logs
-#
-#
-#  drng:
-#    name: drng
-#    env:
-#      TEST_NAME: drng
-#    runs-on: ubuntu-latest
-#    steps:
-#
-#      - name: Checkout repository
-#        uses: actions/checkout@v2
-#
-#      - name: Build GoShimmer image
-#        run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer .
-#
-#      - name: Pull additional Docker images
-#        run: |
-#          docker pull angelocapossele/drand:v1.1.4
-#          docker pull gaiadocker/iproute2:latest
-#
-#      - name: Run integration tests
-#        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
-#
-#      - name: Create logs from tester
-#        if: always()
-#        run: |
-#          docker logs tester &> tools/integration-tests/logs/tester.log
-#
-#      - name: Save logs as artifacts
-#        if: always()
-#        uses: actions/upload-artifact@v1
-#        with:
-#          name: ${{ env.TEST_NAME }}
-#          path: tools/integration-tests/logs
-#
-#
-#  value:
-#    name: value
-#    env:
-#      TEST_NAME: value
-#    runs-on: ubuntu-latest
-#    steps:
-#
-#      - name: Checkout repository
-#        uses: actions/checkout@v2
-#
-#      - name: Build GoShimmer image
-#        run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer .
-#
-#      - name: Pull additional Docker images
-#        run: |
-#          docker pull angelocapossele/drand:v1.1.4
-#          docker pull gaiadocker/iproute2:latest
-#
-#      - name: Run integration tests
-#        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
-#
-#      - name: Create logs from tester
-#        if: always()
-#        run: |
-#          docker logs tester &> tools/integration-tests/logs/tester.log
-#
-#      - name: Save logs as artifacts
-#        if: always()
-#        uses: actions/upload-artifact@v1
-#        with:
-#          name: ${{ env.TEST_NAME }}
-#          path: tools/integration-tests/logs
-#
-#  faucet:
-#    name: faucet
-#    env:
-#      TEST_NAME: faucet
-#    runs-on: ubuntu-latest
-#    steps:
-#
-#      - name: Checkout repository
-#        uses: actions/checkout@v2
-#
-#      - name: Build GoShimmer image
-#        run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer .
-#
-#      - name: Pull additional Docker images
-#        run: |
-#          docker pull angelocapossele/drand:v1.1.4
-#          docker pull gaiadocker/iproute2:latest
-#
-#      - name: Run integration tests
-#        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
-#
-#      - name: Create logs from tester
-#        if: always()
-#        run: |
-#          docker logs tester &> tools/integration-tests/logs/tester.log
-#
-#      - name: Save logs as artifacts
-#        if: always()
-#        uses: actions/upload-artifact@v1
-#        with:
-#          name: ${{ env.TEST_NAME }}
-#          path: tools/integration-tests/logs
-#
-#  mana:
-#    name: mana
-#    env:
-#      TEST_NAME: mana
-#    runs-on: ubuntu-latest
-#    steps:
-#
-#      - name: Check out code
-#        uses: actions/checkout@v2
-#
-#      - name: Build GoShimmer image
-#        run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer .
-#
-#      - name: Pull additional Docker images
-#        run: |
-#          docker pull angelocapossele/drand:1.1.3
-#          docker pull gaiadocker/iproute2:latest
-#
-#      - name: Run integration tests
-#        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
-#
-#      - name: Create logs from tester
-#        if: always()
-#        run: |
-#          docker logs tester &> tools/integration-tests/logs/tester.log
-#
-#      - name: Save logs as artifacts
-#        if: always()
-#        uses: actions/upload-artifact@v1
-#        with:
-#          name: ${{ env.TEST_NAME }}
-#          path: tools/integration-tests/logs
-
-#  diagnostics:
-#    name: diagnostics
-#    env:
-#      TEST_NAME: diagnostics
-#    runs-on: ubuntu-latest
-#    steps:
-#
-#      - name: Check out code
-#        uses: actions/checkout@v2
-#
-#      - name: Build GoShimmer image
-#        run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer .
-#
-#      - name: Run integration tests
-#        run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build
-#
-#      - name: Create logs from tester
-#        if: always()
-#        run: |
-#          docker logs tester &> tools/integration-tests/logs/tester.log
-#
-#      - name: Save logs as artifacts
-#        if: always()
-#        uses: actions/upload-artifact@v1
-#        with:
-#          name: ${{ env.TEST_NAME }}
-#          path: tools/integration-tests/logs
+  consensus:
+    name: consensus
+    env:
+      TEST_NAME: consensus
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Cache Go modules for tester container
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/go
+          # make sure concurrent runs (not really supported) do not match the same key but instead fall back to a reasonable cache
+          key: ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-${{ github.job }}-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-
+            ${{ runner.os }}-it-go-
+
+      - name: Build GoShimmer image
+        run: |
+          docker build \
+            --build-arg DOWNLOAD_SNAPSHOT=0 \
+            -t iotaledger/goshimmer .
+
+      - name: Pull additional Docker images
+        run: |
+          docker pull angelocapossele/drand:v1.1.4
+
+      - name: Run integration tests
+        run: |
+          export CURRENT_UID=$(id -u):$(id -g)
+          docker-compose -f tools/integration-tests/tester/docker-compose.ci.yml up --abort-on-container-exit --exit-code-from tester
+
+      - name: Create logs from tester
+        if: always()
+        run: |
+          docker logs tester &> tools/integration-tests/logs/tester.log
+
+      - name: Save logs as artifacts
+        if: always()
+        uses: actions/upload-artifact@v1
+        with:
+          name: ${{ env.TEST_NAME }}
+          path: tools/integration-tests/logs
+
+
+  drng:
+    name: drng
+    env:
+      TEST_NAME: drng
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Cache Go modules for tester container
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/go
+          # make sure concurrent runs (not really supported) do not match the same key but instead fall back to a reasonable cache
+          key: ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-${{ github.job }}-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-
+            ${{ runner.os }}-it-go-
+
+      - name: Build GoShimmer image
+        run: |
+          docker build \
+            --build-arg DOWNLOAD_SNAPSHOT=0 \
+            -t iotaledger/goshimmer .
+
+      - name: Pull additional Docker images
+        run: |
+          docker pull angelocapossele/drand:v1.1.4
+
+      - name: Run integration tests
+        run: |
+          export CURRENT_UID=$(id -u):$(id -g)
+          docker-compose -f tools/integration-tests/tester/docker-compose.ci.yml up --abort-on-container-exit --exit-code-from tester
+
+      - name: Create logs from tester
+        if: always()
+        run: |
+          docker logs tester &> tools/integration-tests/logs/tester.log
+
+      - name: Save logs as artifacts
+        if: always()
+        uses: actions/upload-artifact@v1
+        with:
+          name: ${{ env.TEST_NAME }}
+          path: tools/integration-tests/logs
+
+
+  value:
+    name: value
+    env:
+      TEST_NAME: value
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Cache Go modules for tester container
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/go
+          # make sure concurrent runs (not really supported) do not match the same key but instead fall back to a reasonable cache
+          key: ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-${{ github.job }}-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-
+            ${{ runner.os }}-it-go-
+
+      - name: Build GoShimmer image
+        run: |
+          docker build \
+            --build-arg DOWNLOAD_SNAPSHOT=0 \
+            -t iotaledger/goshimmer .
+
+      - name: Run integration tests
+        run: |
+          export CURRENT_UID=$(id -u):$(id -g)
+          docker-compose -f tools/integration-tests/tester/docker-compose.ci.yml up --abort-on-container-exit --exit-code-from tester
+
+      - name: Create logs from tester
+        if: always()
+        run: |
+          docker logs tester &> tools/integration-tests/logs/tester.log
+
+      - name: Save logs as artifacts
+        if: always()
+        uses: actions/upload-artifact@v1
+        with:
+          name: ${{ env.TEST_NAME }}
+          path: tools/integration-tests/logs
+
+  faucet:
+    name: faucet
+    env:
+      TEST_NAME: faucet
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Cache Go modules for tester container
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/go
+          # make sure concurrent runs (not really supported) do not match the same key but instead fall back to a reasonable cache
+          key: ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-${{ github.job }}-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-
+            ${{ runner.os }}-it-go-
+
+      - name: Build GoShimmer image
+        run: |
+          docker build \
+            --build-arg DOWNLOAD_SNAPSHOT=0 \
+            -t iotaledger/goshimmer .
+
+      - name: Run integration tests
+        run: |
+          export CURRENT_UID=$(id -u):$(id -g)
+          docker-compose -f tools/integration-tests/tester/docker-compose.ci.yml up --abort-on-container-exit --exit-code-from tester
+
+      - name: Create logs from tester
+        if: always()
+        run: |
+          docker logs tester &> tools/integration-tests/logs/tester.log
+
+      - name: Save logs as artifacts
+        if: always()
+        uses: actions/upload-artifact@v1
+        with:
+          name: ${{ env.TEST_NAME }}
+          path: tools/integration-tests/logs
+
+
+  mana:
+    name: mana
+    env:
+      TEST_NAME: mana
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Cache Go modules for tester container
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/go
+          # make sure concurrent runs (not really supported) do not match the same key but instead fall back to a reasonable cache
+          key: ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-${{ github.job }}-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-
+            ${{ runner.os }}-it-go-
+
+      - name: Build GoShimmer image
+        run: |
+          docker build \
+            --build-arg DOWNLOAD_SNAPSHOT=0 \
+            -t iotaledger/goshimmer .
+
+      - name: Run integration tests
+        run: |
+          export CURRENT_UID=$(id -u):$(id -g)
+          docker-compose -f tools/integration-tests/tester/docker-compose.ci.yml up --abort-on-container-exit --exit-code-from tester
+
+      - name: Create logs from tester
+        if: always()
+        run: |
+          docker logs tester &> tools/integration-tests/logs/tester.log
+
+      - name: Save logs as artifacts
+        if: always()
+        uses: actions/upload-artifact@v1
+        with:
+          name: ${{ env.TEST_NAME }}
+          path: tools/integration-tests/logs
+
+  diagnostics:
+    name: diagnostics
+    env:
+      TEST_NAME: diagnostics
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Cache Go modules for tester container
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/go
+          # make sure concurrent runs (not really supported) do not match the same key but instead fall back to a reasonable cache
+          key: ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-${{ github.job }}-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-
+            ${{ runner.os }}-it-go-
+
+      - name: Build GoShimmer image
+        run: |
+          docker build \
+            --build-arg DOWNLOAD_SNAPSHOT=0 \
+            -t iotaledger/goshimmer .
+
+      - name: Run integration tests
+        run: |
+          export CURRENT_UID=$(id -u):$(id -g)
+          docker-compose -f tools/integration-tests/tester/docker-compose.ci.yml up --abort-on-container-exit --exit-code-from tester
+
+      - name: Create logs from tester
+        if: always()
+        run: |
+          docker logs tester &> tools/integration-tests/logs/tester.log
+
+      - name: Save logs as artifacts
+        if: always()
+        uses: actions/upload-artifact@v1
+        with:
+          name: ${{ env.TEST_NAME }}
+          path: tools/integration-tests/logs

From 26d58e4674b5bec64f6f1b9a759f1a7a158d4602 Mon Sep 17 00:00:00 2001
From: jonastheis <4181434+jonastheis@users.noreply.github.com>
Date: Wed, 22 Sep 2021 10:48:41 +0000
Subject: [PATCH 31/31] Remove prints

---
 packages/tangle/booker.go | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/packages/tangle/booker.go b/packages/tangle/booker.go
index ecd7e6e3c8..08f3ce82f0 100644
--- a/packages/tangle/booker.go
+++ b/packages/tangle/booker.go
@@ -96,10 +96,6 @@ func (b *Booker) run() {
 // BookMessage tries to book the given Message (and potentially its contained Transaction) into the LedgerState and the Tangle.
 // It fires a MessageBooked event if it succeeds.
 func (b *Booker) BookMessage(messageID MessageID) (err error) {
-	fmt.Println("Book message", messageID)
-	fmt.Println("Book message", messageID)
-	fmt.Println("Book message", messageID)
-	fmt.Println("Book message", messageID)
 	b.tangle.Storage.Message(messageID).Consume(func(message *Message) {
 		b.tangle.Storage.MessageMetadata(messageID).Consume(func(messageMetadata *MessageMetadata) {
 			// don't book the same message more than once!