From d90f4c920e8aad8124e8354c7bac0364c3b20091 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Mon, 16 Jan 2023 08:38:09 -0400 Subject: [PATCH 01/25] ci: add a test to validate Zebra's config file and path --- .../continous-integration-docker.yml | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.github/workflows/continous-integration-docker.yml b/.github/workflows/continous-integration-docker.yml index 2b42f717be3..4d89ccd12d4 100644 --- a/.github/workflows/continous-integration-docker.yml +++ b/.github/workflows/continous-integration-docker.yml @@ -299,6 +299,32 @@ jobs: env: ZEBRA_TEST_LIGHTWALLETD: '1' + # Test that Zebra works using the default config with the latest Zebra version + # + # Also check that $ZEBRA_CONF_PATH works + test-configuration-file: + name: Test Zebra configuration file + runs-on: ubuntu-latest + needs: build + if: ${{ github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' && github.event.inputs.run-lwd-send-tx != 'true' }} + steps: + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + with: + short-length: 7 + + - name: Run tests using the default config + run: | + docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} + docker run --rm --name default-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} | grep --max-count=1 'estimated progress to chain tip.*BeforeOverwinter' + + - name: Run tests using the $ZEBRA_CONF_PATH + run: | + docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} + docker run -e ZEBRA_CONF_PATH --rm --name variable-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} | grep --max-count=1 'v1.0.0-rc.2.toml' + env: + ZEBRA_CONF_PATH: 'zebra/zebrad/tests/common/configs/v1.0.0-rc.2.toml' + # zebrad cached checkpoint state tests # Regenerate mandatory checkpoint Zebra cached state disks. From eb99f64a48a41a66023a5c66c445d5c637cbecde Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Mon, 16 Jan 2023 08:56:41 -0400 Subject: [PATCH 02/25] fix: use `ZEBRA_CONF_PATH` as single variable locating the conf --- docker/Dockerfile | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index c3689b1f0e9..a002d648a63 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -140,12 +140,15 @@ ARG CHECKPOINT_SYNC=true ARG NETWORK=Mainnet # Use a configurable dir and file for the zebrad configuration file -ARG ZEBRA_CONF_PATH=/etc/zebra -ENV ZEBRA_CONF_PATH ${ZEBRA_CONF_PATH} +ARG ZEBRA_CONF_DIR=/etc/zebra +ENV ZEBRA_CONF_DIR ${ZEBRA_CONF_DIR} ARG ZEBRA_CONF_FILE=zebrad.toml ENV ZEBRA_CONF_FILE ${ZEBRA_CONF_FILE} +ARG ZEBRA_CONF_PATH=${ZEBRA_CONF_DIR}/${ZEBRA_CONF_FILE} +ENV ZEBRA_CONF_PATH ${ZEBRA_CONF_PATH} + # Build the `zebrad.toml` before starting the container, using the arguments from build # time, or using the default values set just above. And create the conf path and file if # it does not exist. @@ -160,8 +163,8 @@ ENV ZEBRA_CONF_FILE ${ZEBRA_CONF_FILE} # - move this file creation to an entrypoint as we can use default values at runtime, # and modify those as needed when starting the container (at runtime and not at build time) # - make `cache_dir`, `rpc.listen_addr`, `metrics.endpoint_addr`, and `tracing.endpoint_addr` into Docker arguments -RUN mkdir -p ${ZEBRA_CONF_PATH} \ - && touch ${ZEBRA_CONF_PATH}/${ZEBRA_CONF_FILE} +RUN mkdir -p ${ZEBRA_CONF_DIR} \ + && touch ${ZEBRA_CONF_PATH} RUN set -ex; \ { \ echo "[network]"; \ @@ -177,7 +180,7 @@ RUN set -ex; \ echo "#endpoint_addr = '127.0.0.1:9999'"; \ echo "[tracing]"; \ echo "#endpoint_addr = '127.0.0.1:3000'"; \ - } > "${ZEBRA_CONF_PATH}/${ZEBRA_CONF_FILE}" + } > "${ZEBRA_CONF_PATH}" EXPOSE 8233 18233 @@ -188,4 +191,4 @@ ARG SENTRY_DSN ENV SENTRY_DSN ${SENTRY_DSN} # TODO: remove the specified config file location and use the default expected by zebrad -CMD zebrad -c "${ZEBRA_CONF_PATH}/${ZEBRA_CONF_FILE}" start +CMD zebrad -c "${ZEBRA_CONF_PATH}" start From 20246fed436880e75e914291660ce85a817aaca9 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Mon, 16 Jan 2023 09:04:54 -0400 Subject: [PATCH 03/25] fix: do not remove the containers --- .github/workflows/continous-integration-docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/continous-integration-docker.yml b/.github/workflows/continous-integration-docker.yml index 4d89ccd12d4..3e35d11c5ca 100644 --- a/.github/workflows/continous-integration-docker.yml +++ b/.github/workflows/continous-integration-docker.yml @@ -316,12 +316,12 @@ jobs: - name: Run tests using the default config run: | docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} - docker run --rm --name default-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} | grep --max-count=1 'estimated progress to chain tip.*BeforeOverwinter' + docker run --name default-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} | grep --max-count=1 'estimated progress to chain tip.*BeforeOverwinter' - name: Run tests using the $ZEBRA_CONF_PATH run: | docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} - docker run -e ZEBRA_CONF_PATH --rm --name variable-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} | grep --max-count=1 'v1.0.0-rc.2.toml' + docker run -e ZEBRA_CONF_PATH --name variable-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} | grep --max-count=1 'v1.0.0-rc.2.toml' env: ZEBRA_CONF_PATH: 'zebra/zebrad/tests/common/configs/v1.0.0-rc.2.toml' From a56e16a63d21f9e717cdcad8a69295add6d1a3ae Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Mon, 16 Jan 2023 09:33:12 -0400 Subject: [PATCH 04/25] fix: use extended regex --- .github/workflows/continous-integration-docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/continous-integration-docker.yml b/.github/workflows/continous-integration-docker.yml index 3e35d11c5ca..cda299a4352 100644 --- a/.github/workflows/continous-integration-docker.yml +++ b/.github/workflows/continous-integration-docker.yml @@ -316,12 +316,12 @@ jobs: - name: Run tests using the default config run: | docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} - docker run --name default-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} | grep --max-count=1 'estimated progress to chain tip.*BeforeOverwinter' + docker run --name default-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} | grep --extended-regexp --max-count=1 -e 'estimated progress to chain tip.*BeforeOverwinter' - name: Run tests using the $ZEBRA_CONF_PATH run: | docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} - docker run -e ZEBRA_CONF_PATH --name variable-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} | grep --max-count=1 'v1.0.0-rc.2.toml' + docker run -e ZEBRA_CONF_PATH --name variable-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} | grep --extended-regexp --max-count=1 -e 'v1.0.0-rc.2.toml' env: ZEBRA_CONF_PATH: 'zebra/zebrad/tests/common/configs/v1.0.0-rc.2.toml' From d886b1fbd5b9ecf4f9cd6ee48f56c89641108bbc Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Mon, 16 Jan 2023 09:50:10 -0400 Subject: [PATCH 05/25] fix: use different steps to validate the conf tests --- .github/workflows/continous-integration-docker.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/continous-integration-docker.yml b/.github/workflows/continous-integration-docker.yml index cda299a4352..a19bc4ba7e2 100644 --- a/.github/workflows/continous-integration-docker.yml +++ b/.github/workflows/continous-integration-docker.yml @@ -316,12 +316,16 @@ jobs: - name: Run tests using the default config run: | docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} - docker run --name default-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} | grep --extended-regexp --max-count=1 -e 'estimated progress to chain tip.*BeforeOverwinter' + docker run --detach --name default-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} + docker logs --tail all --follow default-conf-tests | tee --output-error=exit /dev/stderr | grep --extended-regexp --max-count=1 -e 'estimated progress to chain tip.*BeforeOverwinter' + docker stop default-conf-tests - name: Run tests using the $ZEBRA_CONF_PATH run: | docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} - docker run -e ZEBRA_CONF_PATH --name variable-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} | grep --extended-regexp --max-count=1 -e 'v1.0.0-rc.2.toml' + docker run --detach -e ZEBRA_CONF_PATH --name variable-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} + docker logs --tail all --follow variable-conf-tests | tee --output-error=exit /dev/stderr | grep --extended-regexp --max-count=1 -e 'v1.0.0-rc.2.toml' + docker stop variable-conf-tests env: ZEBRA_CONF_PATH: 'zebra/zebrad/tests/common/configs/v1.0.0-rc.2.toml' From dce28b8c6a23d9157250757a7d73a42006713568 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Mon, 16 Jan 2023 10:09:18 -0400 Subject: [PATCH 06/25] fix: do not specify a default CMD for running Docker in test builds --- docker/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index a002d648a63..2c24d409936 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -110,7 +110,6 @@ RUN chmod u+x /entrypoint.sh # By default, runs the entrypoint tests specified by the environmental variables (if any are set) ENTRYPOINT [ "/entrypoint.sh" ] -CMD [ "cargo" ] # In this stage we build a release (generate the zebrad binary) # From 25a05f3036af4adab35f5312d4d8ba431c4d2a3c Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Mon, 16 Jan 2023 10:23:45 -0400 Subject: [PATCH 07/25] fix: use actual starting commands for entrypoint --- docker/entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index b7fa28f15c3..76415fddbb1 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -91,10 +91,10 @@ case "$1" in # # TODO: test that the following 3 cases actually work, or remove them else - exec "$@" + exec cargo "$@" fi ;; - zebrad) + -*) exec zebrad "$@" ;; *) From 761a402571ddab4e87322241730d60369c9f1b86 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Mon, 16 Jan 2023 11:46:11 -0400 Subject: [PATCH 08/25] fix: do not add cargo twice if cargo is in $1 --- docker/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 76415fddbb1..06a18a42b7e 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -91,7 +91,7 @@ case "$1" in # # TODO: test that the following 3 cases actually work, or remove them else - exec cargo "$@" + exec "$@" fi ;; -*) From 5ac8b4f6e59c6cd7ba12f425764d1b6437f20e35 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Mon, 16 Jan 2023 15:32:27 -0400 Subject: [PATCH 09/25] fix: allow to run `zebrad` in the `tests` stage of Dockerfile --- docker/entrypoint.sh | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 06a18a42b7e..768b1b9251c 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -15,7 +15,13 @@ echo "ZEBRA_CACHED_STATE_DIR=$ZEBRA_CACHED_STATE_DIR" echo "LIGHTWALLETD_DATA_DIR=$LIGHTWALLETD_DATA_DIR" case "$1" in - -- | cargo) + -- | zebrad) + exec zebrad "$@" + ;; + -*) + exec zebrad "$@" + ;; + *) # For these tests, we activate the gRPC feature to avoid recompiling `zebrad`, # but we might not actually run any gRPC tests. if [[ "$RUN_ALL_TESTS" -eq "1" ]]; then @@ -86,17 +92,8 @@ case "$1" in # Starting with a cached Zebra tip, test sending a block to Zebra's RPC port. ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory") cargo test --locked --release --features getblocktemplate-rpcs --package zebrad --test acceptance -- --nocapture --include-ignored submit_block - - # These command-lines are provided by the caller. - # - # TODO: test that the following 3 cases actually work, or remove them else exec "$@" fi ;; - -*) - exec zebrad "$@" - ;; - *) - exec "$@" esac From c5a1e7ecaa03e24595e45050394521bc8dee16f1 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Mon, 16 Jan 2023 16:31:50 -0400 Subject: [PATCH 10/25] fix: new entrypoint does not allow an empty CMD --- .github/workflows/continous-integration-docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/continous-integration-docker.yml b/.github/workflows/continous-integration-docker.yml index a19bc4ba7e2..e0d196e8c87 100644 --- a/.github/workflows/continous-integration-docker.yml +++ b/.github/workflows/continous-integration-docker.yml @@ -316,14 +316,14 @@ jobs: - name: Run tests using the default config run: | docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} - docker run --detach --name default-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} + docker run --detach --name default-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} zebrad start docker logs --tail all --follow default-conf-tests | tee --output-error=exit /dev/stderr | grep --extended-regexp --max-count=1 -e 'estimated progress to chain tip.*BeforeOverwinter' docker stop default-conf-tests - name: Run tests using the $ZEBRA_CONF_PATH run: | docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} - docker run --detach -e ZEBRA_CONF_PATH --name variable-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} + docker run --detach -e ZEBRA_CONF_PATH --name variable-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} zebrad -c $ZEBRA_CONF_PATH start docker logs --tail all --follow variable-conf-tests | tee --output-error=exit /dev/stderr | grep --extended-regexp --max-count=1 -e 'v1.0.0-rc.2.toml' docker stop variable-conf-tests env: From 823c56cd994f3d4746537dd789001c3e8f0d664e Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Mon, 16 Jan 2023 18:08:52 -0400 Subject: [PATCH 11/25] fix: do not duplicate the `zebrad` command --- .github/workflows/continous-integration-docker.yml | 4 ++-- docker/entrypoint.sh | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/continous-integration-docker.yml b/.github/workflows/continous-integration-docker.yml index e0d196e8c87..0d47ad17acc 100644 --- a/.github/workflows/continous-integration-docker.yml +++ b/.github/workflows/continous-integration-docker.yml @@ -316,14 +316,14 @@ jobs: - name: Run tests using the default config run: | docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} - docker run --detach --name default-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} zebrad start + docker run --detach --name default-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} start docker logs --tail all --follow default-conf-tests | tee --output-error=exit /dev/stderr | grep --extended-regexp --max-count=1 -e 'estimated progress to chain tip.*BeforeOverwinter' docker stop default-conf-tests - name: Run tests using the $ZEBRA_CONF_PATH run: | docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} - docker run --detach -e ZEBRA_CONF_PATH --name variable-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} zebrad -c $ZEBRA_CONF_PATH start + docker run --detach -e ZEBRA_CONF_PATH --name variable-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} -c $ZEBRA_CONF_PATH start docker logs --tail all --follow variable-conf-tests | tee --output-error=exit /dev/stderr | grep --extended-regexp --max-count=1 -e 'v1.0.0-rc.2.toml' docker stop variable-conf-tests env: diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 768b1b9251c..8782bb1fb88 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -95,5 +95,4 @@ case "$1" in else exec "$@" fi - ;; esac From ee4d714daed998d71964f46b07da6947a14c0c41 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Tue, 17 Jan 2023 07:19:25 -0400 Subject: [PATCH 12/25] fix: segregate configuration jobs --- .github/workflows/continous-integration-docker.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/continous-integration-docker.yml b/.github/workflows/continous-integration-docker.yml index 0d47ad17acc..41c3b04940c 100644 --- a/.github/workflows/continous-integration-docker.yml +++ b/.github/workflows/continous-integration-docker.yml @@ -320,6 +320,19 @@ jobs: docker logs --tail all --follow default-conf-tests | tee --output-error=exit /dev/stderr | grep --extended-regexp --max-count=1 -e 'estimated progress to chain tip.*BeforeOverwinter' docker stop default-conf-tests + + # Test that Zebra works using the $ZEBRA_CONF_PATH config + test-zebra-conf-path: + name: Test Zebra configuration file + runs-on: ubuntu-latest + needs: build + if: ${{ github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' && github.event.inputs.run-lwd-send-tx != 'true' }} + steps: + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + with: + short-length: 7 + - name: Run tests using the $ZEBRA_CONF_PATH run: | docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} From 0799b8a4d2bce750364b38e693e7ef48ef798e21 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Tue, 17 Jan 2023 17:40:25 -0400 Subject: [PATCH 13/25] refactor(entrypoint): handle better parameters conditions --- docker/Dockerfile | 2 +- docker/entrypoint.sh | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 2c24d409936..f1914715dea 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -103,7 +103,7 @@ COPY --from=us-docker.pkg.dev/zealous-zebra/zebra/lightwalletd /opt/lightwalletd RUN cargo chef cook --release --features sentry,lightwalletd-grpc-tests --workspace --recipe-path recipe.json COPY . . -RUN cargo test --locked --release --features lightwalletd-grpc-tests --workspace --no-run +RUN cargo test --locked --release --features lightwalletd-grpc-tests --workspace --no-run --bins COPY ./docker/entrypoint.sh / RUN chmod u+x /entrypoint.sh diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 8782bb1fb88..e17bfbcc163 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -15,10 +15,7 @@ echo "ZEBRA_CACHED_STATE_DIR=$ZEBRA_CACHED_STATE_DIR" echo "LIGHTWALLETD_DATA_DIR=$LIGHTWALLETD_DATA_DIR" case "$1" in - -- | zebrad) - exec zebrad "$@" - ;; - -*) + --* | -*) exec zebrad "$@" ;; *) From 56083e9e5dd54917242c744c3dbcb1afe9d6e8b2 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Wed, 18 Jan 2023 17:52:23 -0400 Subject: [PATCH 14/25] fix: make `zebrad` an executable command in `tests` stage --- .github/workflows/continous-integration-docker.yml | 3 +-- docker/Dockerfile | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/continous-integration-docker.yml b/.github/workflows/continous-integration-docker.yml index 41c3b04940c..340ab65a074 100644 --- a/.github/workflows/continous-integration-docker.yml +++ b/.github/workflows/continous-integration-docker.yml @@ -316,11 +316,10 @@ jobs: - name: Run tests using the default config run: | docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} - docker run --detach --name default-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} start + docker run --detach --name default-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} zebrad start docker logs --tail all --follow default-conf-tests | tee --output-error=exit /dev/stderr | grep --extended-regexp --max-count=1 -e 'estimated progress to chain tip.*BeforeOverwinter' docker stop default-conf-tests - # Test that Zebra works using the $ZEBRA_CONF_PATH config test-zebra-conf-path: name: Test Zebra configuration file diff --git a/docker/Dockerfile b/docker/Dockerfile index f1914715dea..c2c0c3d6e9c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -103,7 +103,8 @@ COPY --from=us-docker.pkg.dev/zealous-zebra/zebra/lightwalletd /opt/lightwalletd RUN cargo chef cook --release --features sentry,lightwalletd-grpc-tests --workspace --recipe-path recipe.json COPY . . -RUN cargo test --locked --release --features lightwalletd-grpc-tests --workspace --no-run --bins +RUN cargo test --locked --release --features lightwalletd-grpc-tests --workspace --no-run +RUN cp /opt/zebrad/target/release/zebrad /usr/local/bin COPY ./docker/entrypoint.sh / RUN chmod u+x /entrypoint.sh From 59850e583a253399faf86d67d2c678b0a41526bd Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 20 Jan 2023 09:30:02 +1000 Subject: [PATCH 15/25] Show the commands that are being executed in the new docker test --- .github/workflows/continous-integration-docker.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/continous-integration-docker.yml b/.github/workflows/continous-integration-docker.yml index 340ab65a074..9a7ecb4f073 100644 --- a/.github/workflows/continous-integration-docker.yml +++ b/.github/workflows/continous-integration-docker.yml @@ -315,6 +315,7 @@ jobs: - name: Run tests using the default config run: | + set -x docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} docker run --detach --name default-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} zebrad start docker logs --tail all --follow default-conf-tests | tee --output-error=exit /dev/stderr | grep --extended-regexp --max-count=1 -e 'estimated progress to chain tip.*BeforeOverwinter' @@ -334,6 +335,7 @@ jobs: - name: Run tests using the $ZEBRA_CONF_PATH run: | + set -x docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} docker run --detach -e ZEBRA_CONF_PATH --name variable-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} -c $ZEBRA_CONF_PATH start docker logs --tail all --follow variable-conf-tests | tee --output-error=exit /dev/stderr | grep --extended-regexp --max-count=1 -e 'v1.0.0-rc.2.toml' From a09e05bd4aa354eaf2e6e0f07a53a32a69319414 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 20 Jan 2023 11:00:58 +1000 Subject: [PATCH 16/25] Show full logs without tee or grep --- .github/workflows/continous-integration-docker.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/continous-integration-docker.yml b/.github/workflows/continous-integration-docker.yml index 9a7ecb4f073..c5c4b371563 100644 --- a/.github/workflows/continous-integration-docker.yml +++ b/.github/workflows/continous-integration-docker.yml @@ -320,6 +320,7 @@ jobs: docker run --detach --name default-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} zebrad start docker logs --tail all --follow default-conf-tests | tee --output-error=exit /dev/stderr | grep --extended-regexp --max-count=1 -e 'estimated progress to chain tip.*BeforeOverwinter' docker stop default-conf-tests + docker logs default-conf-tests # Test that Zebra works using the $ZEBRA_CONF_PATH config test-zebra-conf-path: @@ -340,6 +341,7 @@ jobs: docker run --detach -e ZEBRA_CONF_PATH --name variable-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} -c $ZEBRA_CONF_PATH start docker logs --tail all --follow variable-conf-tests | tee --output-error=exit /dev/stderr | grep --extended-regexp --max-count=1 -e 'v1.0.0-rc.2.toml' docker stop variable-conf-tests + docker logs variable-conf-tests env: ZEBRA_CONF_PATH: 'zebra/zebrad/tests/common/configs/v1.0.0-rc.2.toml' From 06a5a7b65976817199981132d906c0dabf39e773 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Sun, 22 Jan 2023 12:54:36 -0400 Subject: [PATCH 17/25] Apply suggestions from code review Co-authored-by: teor --- .github/workflows/continous-integration-docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/continous-integration-docker.yml b/.github/workflows/continous-integration-docker.yml index c5c4b371563..43ed56ef7f8 100644 --- a/.github/workflows/continous-integration-docker.yml +++ b/.github/workflows/continous-integration-docker.yml @@ -303,7 +303,7 @@ jobs: # # Also check that $ZEBRA_CONF_PATH works test-configuration-file: - name: Test Zebra configuration file + name: Test Zebra default Docker config file runs-on: ubuntu-latest needs: build if: ${{ github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' && github.event.inputs.run-lwd-send-tx != 'true' }} @@ -324,7 +324,7 @@ jobs: # Test that Zebra works using the $ZEBRA_CONF_PATH config test-zebra-conf-path: - name: Test Zebra configuration file + name: Test Zebra custom Docker config file runs-on: ubuntu-latest needs: build if: ${{ github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' && github.event.inputs.run-lwd-send-tx != 'true' }} From 49f50f5309a794c4aff42fc5f4f69897680bee15 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Sun, 22 Jan 2023 18:21:58 -0400 Subject: [PATCH 18/25] fix: use the actual path inside docker --- .github/workflows/continous-integration-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/continous-integration-docker.yml b/.github/workflows/continous-integration-docker.yml index 43ed56ef7f8..88750cafc54 100644 --- a/.github/workflows/continous-integration-docker.yml +++ b/.github/workflows/continous-integration-docker.yml @@ -343,7 +343,7 @@ jobs: docker stop variable-conf-tests docker logs variable-conf-tests env: - ZEBRA_CONF_PATH: 'zebra/zebrad/tests/common/configs/v1.0.0-rc.2.toml' + ZEBRA_CONF_PATH: 'zebrad/tests/common/configs/v1.0.0-rc.2.toml' # zebrad cached checkpoint state tests From 3b5fc86c10ffacf67db5da8cc8bcbe68a34299ab Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Sun, 22 Jan 2023 19:34:53 -0400 Subject: [PATCH 19/25] fix: use `grep` with exit code If the container is logging to stderr, piping works only for stdout, so we're adding `2>&1` --- .github/workflows/continous-integration-docker.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/continous-integration-docker.yml b/.github/workflows/continous-integration-docker.yml index 88750cafc54..adfe7ba42b6 100644 --- a/.github/workflows/continous-integration-docker.yml +++ b/.github/workflows/continous-integration-docker.yml @@ -318,9 +318,8 @@ jobs: set -x docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} docker run --detach --name default-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} zebrad start - docker logs --tail all --follow default-conf-tests | tee --output-error=exit /dev/stderr | grep --extended-regexp --max-count=1 -e 'estimated progress to chain tip.*BeforeOverwinter' + docker logs --tail all --follow default-conf-tests 2>&1 | grep --extended-regexp --max-count=1 -e 'estimated progress to chain tip.*BeforeOverwinter' || exit 1 docker stop default-conf-tests - docker logs default-conf-tests # Test that Zebra works using the $ZEBRA_CONF_PATH config test-zebra-conf-path: @@ -339,9 +338,8 @@ jobs: set -x docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} docker run --detach -e ZEBRA_CONF_PATH --name variable-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} -c $ZEBRA_CONF_PATH start - docker logs --tail all --follow variable-conf-tests | tee --output-error=exit /dev/stderr | grep --extended-regexp --max-count=1 -e 'v1.0.0-rc.2.toml' + docker logs --tail all --follow variable-conf-tests 2>&1 | grep --extended-regexp --max-count=1 -e 'v1.0.0-rc.2.toml' || exit 1 docker stop variable-conf-tests - docker logs variable-conf-tests env: ZEBRA_CONF_PATH: 'zebrad/tests/common/configs/v1.0.0-rc.2.toml' From 06f9152c9d8ea79f228750d2ecdb29a7215c8e83 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Sun, 22 Jan 2023 20:43:36 -0400 Subject: [PATCH 20/25] fix: use `grep -q` to get an exit code --- .github/workflows/continous-integration-docker.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/continous-integration-docker.yml b/.github/workflows/continous-integration-docker.yml index adfe7ba42b6..c5370b4dfb5 100644 --- a/.github/workflows/continous-integration-docker.yml +++ b/.github/workflows/continous-integration-docker.yml @@ -318,8 +318,10 @@ jobs: set -x docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} docker run --detach --name default-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} zebrad start - docker logs --tail all --follow default-conf-tests 2>&1 | grep --extended-regexp --max-count=1 -e 'estimated progress to chain tip.*BeforeOverwinter' || exit 1 + EXIT_STATUS=$(docker logs --tail all --follow default-conf-tests 2>&1 | grep -q --extended-regexp --max-count=1 -e 'estimated progress to chain tip.*BeforeOverwinter'; echo $?; ) docker stop default-conf-tests + docker logs default-conf-tests + exit "$EXIT_STATUS" # Test that Zebra works using the $ZEBRA_CONF_PATH config test-zebra-conf-path: @@ -338,8 +340,10 @@ jobs: set -x docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} docker run --detach -e ZEBRA_CONF_PATH --name variable-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} -c $ZEBRA_CONF_PATH start - docker logs --tail all --follow variable-conf-tests 2>&1 | grep --extended-regexp --max-count=1 -e 'v1.0.0-rc.2.toml' || exit 1 + EXIT_STATUS=$(docker logs --tail all --follow variable-conf-tests 2>&1 | grep -q --extended-regexp --max-count=1 -e 'v1.0.0-rc.2.toml'; echo $?; ) docker stop variable-conf-tests + docker logs variable-conf-tests + exit "$EXIT_STATUS" env: ZEBRA_CONF_PATH: 'zebrad/tests/common/configs/v1.0.0-rc.2.toml' From 96ec0717442a07c2368f020833effdea89967d05 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Sun, 22 Jan 2023 21:03:17 -0400 Subject: [PATCH 21/25] fix: fail if any error is detected --- .github/workflows/continous-integration-docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/continous-integration-docker.yml b/.github/workflows/continous-integration-docker.yml index c5370b4dfb5..295062fe7e8 100644 --- a/.github/workflows/continous-integration-docker.yml +++ b/.github/workflows/continous-integration-docker.yml @@ -315,7 +315,7 @@ jobs: - name: Run tests using the default config run: | - set -x + set -ex docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} docker run --detach --name default-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} zebrad start EXIT_STATUS=$(docker logs --tail all --follow default-conf-tests 2>&1 | grep -q --extended-regexp --max-count=1 -e 'estimated progress to chain tip.*BeforeOverwinter'; echo $?; ) @@ -337,7 +337,7 @@ jobs: - name: Run tests using the $ZEBRA_CONF_PATH run: | - set -x + set -ex docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} docker run --detach -e ZEBRA_CONF_PATH --name variable-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} -c $ZEBRA_CONF_PATH start EXIT_STATUS=$(docker logs --tail all --follow variable-conf-tests 2>&1 | grep -q --extended-regexp --max-count=1 -e 'v1.0.0-rc.2.toml'; echo $?; ) From 3ff322db3893c978b35cc9e06244508a1e3c719b Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Sun, 22 Jan 2023 21:04:01 -0400 Subject: [PATCH 22/25] fix: fail if this test takes more than 5 minutes --- .github/workflows/continous-integration-docker.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/continous-integration-docker.yml b/.github/workflows/continous-integration-docker.yml index 295062fe7e8..cdf23cdf0b9 100644 --- a/.github/workflows/continous-integration-docker.yml +++ b/.github/workflows/continous-integration-docker.yml @@ -304,6 +304,7 @@ jobs: # Also check that $ZEBRA_CONF_PATH works test-configuration-file: name: Test Zebra default Docker config file + timeout-minutes: 5 runs-on: ubuntu-latest needs: build if: ${{ github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' && github.event.inputs.run-lwd-send-tx != 'true' }} @@ -326,6 +327,7 @@ jobs: # Test that Zebra works using the $ZEBRA_CONF_PATH config test-zebra-conf-path: name: Test Zebra custom Docker config file + timeout-minutes: 5 runs-on: ubuntu-latest needs: build if: ${{ github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' && github.event.inputs.run-lwd-send-tx != 'true' }} From b32c447bb8ebf51a4ce73260aa6e82be216d5c05 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Sun, 22 Jan 2023 21:30:45 -0400 Subject: [PATCH 23/25] fix: update patch workflows --- .../workflows/continous-integration-docker.patch.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/continous-integration-docker.patch.yml b/.github/workflows/continous-integration-docker.patch.yml index c0a34842233..ae7f8f2a894 100644 --- a/.github/workflows/continous-integration-docker.patch.yml +++ b/.github/workflows/continous-integration-docker.patch.yml @@ -66,6 +66,18 @@ jobs: steps: - run: 'echo "No build required"' + test-configuration-file: + name: Test Zebra default Docker config file + runs-on: ubuntu-latest + steps: + - run: 'echo "No build required"' + + test-zebra-conf-path: + name: Test Zebra custom Docker config file + runs-on: ubuntu-latest + steps: + - run: 'echo "No build required"' + test-stateful-sync: name: Zebra checkpoint update / Run sync-past-checkpoint test runs-on: ubuntu-latest From 1beaebd875da34abf968cea6d24c271b7bf1fb3a Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Sun, 22 Jan 2023 21:39:58 -0400 Subject: [PATCH 24/25] feat: test Dockerfile `runtime` config --- .github/workflows/continous-delivery.yml | 22 +++++++++++++++++++ .../continous-integration-docker.yml | 2 -- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/.github/workflows/continous-delivery.yml b/.github/workflows/continous-delivery.yml index 1adb5297d70..f481b858d34 100644 --- a/.github/workflows/continous-delivery.yml +++ b/.github/workflows/continous-delivery.yml @@ -77,6 +77,28 @@ jobs: zebra_skip_ipv6_tests: '1' rust_log: info + # Test that Zebra works using the default config with the latest Zebra version + test-configuration-file: + name: Test Zebra default Docker config file + timeout-minutes: 5 + runs-on: ubuntu-latest + needs: build + steps: + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + with: + short-length: 7 + + - name: Run tests using the default config + run: | + set -ex + docker pull ${{ env.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }} + docker run --detach --name default-conf-tests -t ${{ env.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }} + EXIT_STATUS=$(docker logs --tail all --follow default-conf-tests 2>&1 | grep -q --extended-regexp --max-count=1 -e 'estimated progress to chain tip.*BeforeOverwinter'; echo $?; ) + docker stop default-conf-tests + docker logs default-conf-tests + exit "$EXIT_STATUS" + # This jobs handles the deployment of a Managed Instance Group (MiG) with 2 nodes in # the us-central1 region. Two different groups of MiGs are deployed one for pushes to # the main branch and another for version releases of Zebra diff --git a/.github/workflows/continous-integration-docker.yml b/.github/workflows/continous-integration-docker.yml index cdf23cdf0b9..233302e4096 100644 --- a/.github/workflows/continous-integration-docker.yml +++ b/.github/workflows/continous-integration-docker.yml @@ -300,8 +300,6 @@ jobs: ZEBRA_TEST_LIGHTWALLETD: '1' # Test that Zebra works using the default config with the latest Zebra version - # - # Also check that $ZEBRA_CONF_PATH works test-configuration-file: name: Test Zebra default Docker config file timeout-minutes: 5 From 5dbc5494225a131bac0e0015298b93a06f60f499 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Sun, 22 Jan 2023 21:47:07 -0400 Subject: [PATCH 25/25] fix: depend on the configuration test to continue --- .github/workflows/continous-delivery.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/continous-delivery.yml b/.github/workflows/continous-delivery.yml index f481b858d34..97a10ebf40d 100644 --- a/.github/workflows/continous-delivery.yml +++ b/.github/workflows/continous-delivery.yml @@ -112,7 +112,7 @@ jobs: # - on every release, when it's published deploy-nodes: name: Deploy ${{ inputs.network || 'Mainnet' }} nodes - needs: [ build, versioning ] + needs: [ build, test-configuration-file, versioning ] runs-on: ubuntu-latest timeout-minutes: 30 permissions: @@ -206,7 +206,7 @@ jobs: # Note: this instances are not automatically replaced or deleted deploy-instance: name: Deploy single instance - needs: build + needs: [ build, test-configuration-file ] runs-on: ubuntu-latest timeout-minutes: 30 permissions: