diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index d2ba568d4e..af8bc1c412 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -20,17 +20,32 @@ jobs: - name: Checkout repository uses: actions/checkout@v2 + - name: Cache Go modules for tester container + uses: actions/cache@v2 + with: + path: | + ~/go + # make sure concurrent runs (not really supported) do not match the same key but instead fall back to a reasonable cache + key: ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-${{ github.job }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}- + ${{ runner.os }}-it-go- + - name: Build GoShimmer image - run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer . + run: | + docker build \ + --build-arg DOWNLOAD_SNAPSHOT=0 \ + -t iotaledger/goshimmer . - name: Pull additional Docker images run: | - docker pull angelocapossele/drand:v1.1.4 docker pull gaiaadm/pumba:0.7.2 docker pull gaiadocker/iproute2:latest - name: Run integration tests - run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build + run: | + export CURRENT_UID=$(id -u):$(id -g) + docker-compose -f tools/integration-tests/tester/docker-compose.ci.yml up --abort-on-container-exit --exit-code-from tester - name: Create logs from tester if: always() @@ -44,7 +59,6 @@ jobs: name: ${{ env.TEST_NAME }} path: tools/integration-tests/logs - common: name: common env: @@ -55,16 +69,27 @@ jobs: - name: Checkout repository uses: actions/checkout@v2 - - name: Build GoShimmer image - run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer . + - name: Cache Go modules for tester container + uses: actions/cache@v2 + with: + path: | + ~/go + # make sure concurrent runs (not really supported) do not match the same key but instead fall back to a reasonable cache + key: ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-${{ github.job }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}- + ${{ runner.os }}-it-go- - - name: Pull additional Docker images + - name: Build GoShimmer image run: | - docker pull angelocapossele/drand:v1.1.4 - docker pull gaiadocker/iproute2:latest + docker build \ + --build-arg DOWNLOAD_SNAPSHOT=0 \ + -t iotaledger/goshimmer . - name: Run integration tests - run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build + run: | + export CURRENT_UID=$(id -u):$(id -g) + docker-compose -f tools/integration-tests/tester/docker-compose.ci.yml up --abort-on-container-exit --exit-code-from tester - name: Create logs from tester if: always() @@ -88,16 +113,31 @@ jobs: - name: Checkout repository uses: actions/checkout@v2 + - name: Cache Go modules for tester container + uses: actions/cache@v2 + with: + path: | + ~/go + # make sure concurrent runs (not really supported) do not match the same key but instead fall back to a reasonable cache + key: ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-${{ github.job }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}- + ${{ runner.os }}-it-go- + - name: Build GoShimmer image - run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer . + run: | + docker build \ + --build-arg DOWNLOAD_SNAPSHOT=0 \ + -t iotaledger/goshimmer . - name: Pull additional Docker images run: | docker pull angelocapossele/drand:v1.1.4 - docker pull gaiadocker/iproute2:latest - name: Run integration tests - run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build + run: | + export CURRENT_UID=$(id -u):$(id -g) + docker-compose -f tools/integration-tests/tester/docker-compose.ci.yml up --abort-on-container-exit --exit-code-from tester - name: Create logs from tester if: always() @@ -122,16 +162,31 @@ jobs: - name: Checkout repository uses: actions/checkout@v2 + - name: Cache Go modules for tester container + uses: actions/cache@v2 + with: + path: | + ~/go + # make sure concurrent runs (not really supported) do not match the same key but instead fall back to a reasonable cache + key: ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-${{ github.job }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}- + ${{ runner.os }}-it-go- + - name: Build GoShimmer image - run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer . + run: | + docker build \ + --build-arg DOWNLOAD_SNAPSHOT=0 \ + -t iotaledger/goshimmer . - name: Pull additional Docker images run: | docker pull angelocapossele/drand:v1.1.4 - docker pull gaiadocker/iproute2:latest - name: Run integration tests - run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build + run: | + export CURRENT_UID=$(id -u):$(id -g) + docker-compose -f tools/integration-tests/tester/docker-compose.ci.yml up --abort-on-container-exit --exit-code-from tester - name: Create logs from tester if: always() @@ -156,16 +211,27 @@ jobs: - name: Checkout repository uses: actions/checkout@v2 - - name: Build GoShimmer image - run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer . + - name: Cache Go modules for tester container + uses: actions/cache@v2 + with: + path: | + ~/go + # make sure concurrent runs (not really supported) do not match the same key but instead fall back to a reasonable cache + key: ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-${{ github.job }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}- + ${{ runner.os }}-it-go- - - name: Pull additional Docker images + - name: Build GoShimmer image run: | - docker pull angelocapossele/drand:v1.1.4 - docker pull gaiadocker/iproute2:latest + docker build \ + --build-arg DOWNLOAD_SNAPSHOT=0 \ + -t iotaledger/goshimmer . - name: Run integration tests - run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build + run: | + export CURRENT_UID=$(id -u):$(id -g) + docker-compose -f tools/integration-tests/tester/docker-compose.ci.yml up --abort-on-container-exit --exit-code-from tester - name: Create logs from tester if: always() @@ -189,16 +255,27 @@ jobs: - name: Checkout repository uses: actions/checkout@v2 - - name: Build GoShimmer image - run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer . + - name: Cache Go modules for tester container + uses: actions/cache@v2 + with: + path: | + ~/go + # make sure concurrent runs (not really supported) do not match the same key but instead fall back to a reasonable cache + key: ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-${{ github.job }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}- + ${{ runner.os }}-it-go- - - name: Pull additional Docker images + - name: Build GoShimmer image run: | - docker pull angelocapossele/drand:v1.1.4 - docker pull gaiadocker/iproute2:latest + docker build \ + --build-arg DOWNLOAD_SNAPSHOT=0 \ + -t iotaledger/goshimmer . - name: Run integration tests - run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build + run: | + export CURRENT_UID=$(id -u):$(id -g) + docker-compose -f tools/integration-tests/tester/docker-compose.ci.yml up --abort-on-container-exit --exit-code-from tester - name: Create logs from tester if: always() @@ -212,6 +289,7 @@ jobs: name: ${{ env.TEST_NAME }} path: tools/integration-tests/logs + mana: name: mana env: @@ -222,16 +300,27 @@ jobs: - name: Check out code uses: actions/checkout@v2 - - name: Build GoShimmer image - run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer . + - name: Cache Go modules for tester container + uses: actions/cache@v2 + with: + path: | + ~/go + # make sure concurrent runs (not really supported) do not match the same key but instead fall back to a reasonable cache + key: ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-${{ github.job }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}- + ${{ runner.os }}-it-go- - - name: Pull additional Docker images + - name: Build GoShimmer image run: | - docker pull angelocapossele/drand:1.1.3 - docker pull gaiadocker/iproute2:latest + docker build \ + --build-arg DOWNLOAD_SNAPSHOT=0 \ + -t iotaledger/goshimmer . - name: Run integration tests - run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build + run: | + export CURRENT_UID=$(id -u):$(id -g) + docker-compose -f tools/integration-tests/tester/docker-compose.ci.yml up --abort-on-container-exit --exit-code-from tester - name: Create logs from tester if: always() @@ -255,11 +344,27 @@ jobs: - name: Check out code uses: actions/checkout@v2 + - name: Cache Go modules for tester container + uses: actions/cache@v2 + with: + path: | + ~/go + # make sure concurrent runs (not really supported) do not match the same key but instead fall back to a reasonable cache + key: ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}-${{ github.job }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-it-go-${{ hashFiles('tools/integration-tests/tester/go.sum') }}- + ${{ runner.os }}-it-go- + - name: Build GoShimmer image - run: docker build --build-arg DOWNLOAD_SNAPSHOT=0 -t iotaledger/goshimmer . + run: | + docker build \ + --build-arg DOWNLOAD_SNAPSHOT=0 \ + -t iotaledger/goshimmer . - name: Run integration tests - run: docker-compose -f tools/integration-tests/tester/docker-compose.yml up --abort-on-container-exit --exit-code-from tester --build + run: | + export CURRENT_UID=$(id -u):$(id -g) + docker-compose -f tools/integration-tests/tester/docker-compose.ci.yml up --abort-on-container-exit --exit-code-from tester - name: Create logs from tester if: always() diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3d05f04901..d042172cd2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,5 +26,17 @@ jobs: - name: Checkout repository uses: actions/checkout@v2 + - name: Cache Go build and dependencies + uses: actions/cache@v2 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + # make sure concurrent runs (not really supported) do not match the same key but instead fall back to a reasonable cache + key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }}-${{ github.event.action }}${{ github.event.after }} + restore-keys: | + ${{ runner.os }}-go-${{ hashFiles('go.sum') }} + ${{ runner.os }}-go- + - name: Run Tests - run: go test ./... -tags rocksdb + run: go test ./... -tags rocksdb -count=1 diff --git a/tools/integration-tests/assets/entrypoint.sh b/tools/integration-tests/assets/entrypoint.sh index a5ec0407b8..f86dbb179b 100755 --- a/tools/integration-tests/assets/entrypoint.sh +++ b/tools/integration-tests/assets/entrypoint.sh @@ -6,4 +6,10 @@ chmod 777 /assets/* echo "assets:" ls /assets echo "running tests..." -go test ./tests/"${TEST_NAME}" -v -timeout 30m +go test ./tests/"${TEST_NAME}" -v -timeout 10m -count=1 + +# if running in CI we need to set right permissions on the Go folder (within container) so that it can be exported to cache +if [ ! -z ${CURRENT_UID} ]; then + echo "setting permissions on Go folder to '${CURRENT_UID}'..." + chown -R "${CURRENT_UID}" /go /root/.cache/go-build +fi diff --git a/tools/integration-tests/tester/docker-compose.ci.yml b/tools/integration-tests/tester/docker-compose.ci.yml new file mode 100644 index 0000000000..ecaa243a93 --- /dev/null +++ b/tools/integration-tests/tester/docker-compose.ci.yml @@ -0,0 +1,23 @@ +version: "3.5" + +services: + tester: + container_name: tester + image: golang@sha256:cefedeae41e0bbbfa20bb1c37c3a43e0001fa541be9732f7bc6a28ecc154e9e4 + working_dir: /tmp/goshimmer/tools/integration-tests/tester + command: /tmp/assets/entrypoint.sh + environment: + - TEST_NAME=${TEST_NAME} + - CURRENT_UID + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - ../../..:/tmp/goshimmer:rw + - ../logs:/tmp/logs + - ../assets:/tmp/assets + - ~/go/pkg:/go + - ~/go/cache:/root/.cache/go-build + - goshimmer-testing-assets:/assets + +volumes: + goshimmer-testing-assets: + name: goshimmer-testing-assets diff --git a/tools/integration-tests/tester/docker-compose.yml b/tools/integration-tests/tester/docker-compose.yml index 3db4017fc1..eaeaf281c4 100644 --- a/tools/integration-tests/tester/docker-compose.yml +++ b/tools/integration-tests/tester/docker-compose.yml @@ -3,7 +3,7 @@ version: "3.5" services: tester: container_name: tester - image: golang:1.16.2 + image: golang@sha256:cefedeae41e0bbbfa20bb1c37c3a43e0001fa541be9732f7bc6a28ecc154e9e4 working_dir: /tmp/goshimmer/tools/integration-tests/tester command: /tmp/assets/entrypoint.sh environment: @@ -13,11 +13,14 @@ services: - ../../..:/tmp/goshimmer:rw - ../logs:/tmp/logs - ../assets:/tmp/assets - - goshimmer-testing-cache:/go + - goshimmer-testing-package-cache:/go + - goshimmer-testing-build-cache:/root/.cache/go-build - goshimmer-testing-assets:/assets volumes: - goshimmer-testing-cache: - name: goshimmer-testing-cache + goshimmer-testing-package-cache: + name: goshimmer-testing-package-cache + goshimmer-testing-build-cache: + name: goshimmer-testing-build-cache goshimmer-testing-assets: name: goshimmer-testing-assets