Merge of #6357

ZcashFoundation · Apr 13, 2023 · 40e68f7 · 40e68f7
2 parents 4667886 + 5534c74
commit 40e68f7
Show file tree

Hide file tree

Showing 12 changed files with 152 additions and 178 deletions.
diff --git a/.github/workflows/build-crates-individually.yml b/.github/workflows/build-crates-individually.yml
@@ -37,11 +37,11 @@ on:
       - '.github/workflows/build-crates-individually.yml'
 
 env:
-  CARGO_INCREMENTAL: 0
-  RUST_LOG: info
-  RUST_BACKTRACE: full
-  RUST_LIB_BACKTRACE: full
-  COLORBT_SHOW_HIDDEN: '1'
+  CARGO_INCREMENTAL: ${{ vars.CARGO_INCREMENTAL }}
+  RUST_LOG: ${{ vars.RUST_LOG }}
+  RUST_BACKTRACE: ${{ vars.RUST_BACKTRACE }}
+  RUST_LIB_BACKTRACE: ${{ vars.RUST_LIB_BACKTRACE }}
+  COLORBT_SHOW_HIDDEN: ${{ vars.COLORBT_SHOW_HIDDEN }}
 
 jobs:
   matrix:

diff --git a/.github/workflows/build-docker-image.yml b/.github/workflows/build-docker-image.yml
@@ -84,7 +84,7 @@ jobs:
         with:
           # list of Docker images to use as base name for tags
           images: |
-            us-docker.pkg.dev/zealous-zebra/zebra/${{ inputs.image_name }}
+            us-docker.pkg.dev/${{ vars.GCP_PROJECT }}/zebra/${{ inputs.image_name }}
             zfnd/zebra,enable=${{ github.event_name == 'release' && !github.event.release.prerelease }}
           # appends inputs.tag_suffix to image tags/names
           flavor: |
@@ -119,8 +119,8 @@ jobs:
         uses: google-github-actions/[email protected]
         with:
           retries: '3'
-          workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc'
-          service_account: '[email protected]'
+          workload_identity_provider: '${{ vars.GCP_WIF }}'
+          service_account: '${{ vars.GCP_ARTIFACTS_SA }}'
           token_format: 'access_token'
           # Some builds might take over an hour, and Google's default lifetime duration for
           # an access token is 1 hour (3600s). We increase this to 3 hours (10800s)
@@ -177,7 +177,7 @@ jobs:
           # The caches are tried in top-down order, the first available cache is used:
           # https://github.com/moby/moby/pull/26839#issuecomment-277383550
           cache-from: |
-            type=registry,ref=us-docker.pkg.dev/zealous-zebra/zebra-caching/${{ inputs.image_name }}${{ inputs.tag_suffix }}:${{ env.GITHUB_REF_SLUG_URL }}-cache
-            type=registry,ref=us-docker.pkg.dev/zealous-zebra/zebra-caching/${{ inputs.image_name }}${{ inputs.tag_suffix }}:main-cache
+            type=registry,ref=us-docker.pkg.dev/${{ vars.GCP_PROJECT }}/zebra-caching/${{ inputs.image_name }}${{ inputs.tag_suffix }}:${{ env.GITHUB_REF_SLUG_URL }}-cache
+            type=registry,ref=us-docker.pkg.dev/${{ vars.GCP_PROJECT }}/zebra-caching/${{ inputs.image_name }}${{ inputs.tag_suffix }}:main-cache
           cache-to: |
-            type=registry,ref=us-docker.pkg.dev/zealous-zebra/zebra-caching/${{ inputs.image_name }}${{ inputs.tag_suffix }}:${{ env.GITHUB_REF_SLUG_URL }}-cache,mode=max
+            type=registry,ref=us-docker.pkg.dev/${{ vars.GCP_PROJECT }}/zebra-caching/${{ inputs.image_name }}${{ inputs.tag_suffix }}:${{ env.GITHUB_REF_SLUG_URL }}-cache,mode=max
diff --git a/.github/workflows/continous-delivery.yml b/.github/workflows/continous-delivery.yml
@@ -25,12 +25,6 @@ on:
     types:
       - published
 
-env:
-  GAR_BASE: us-docker.pkg.dev/zealous-zebra/zebra
-  REGION: us-central1
-  ZONE: us-central1-a
-  MACHINE_TYPE: c2-standard-4
-
 jobs:
   # If a release was made we want to extract the first part of the semver from the
   # tag_name
@@ -92,8 +86,8 @@ jobs:
       - name: Run tests using the default config
         run: |
           set -ex
-          docker pull ${{ env.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }}
-          docker run --detach --name default-conf-tests -t ${{ env.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }}
+          docker pull ${{ vars.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }}
+          docker run --detach --name default-conf-tests -t ${{ vars.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }}
           EXIT_STATUS=$(docker logs --tail all --follow default-conf-tests 2>&1 | grep -q --extended-regexp --max-count=1 -e 'estimated progress to chain tip.*BeforeOverwinter'; echo $?; )
           docker stop default-conf-tests
           docker logs default-conf-tests
@@ -147,8 +141,8 @@ jobs:
         uses: google-github-actions/[email protected]
         with:
           retries: '3'
-          workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc'
-          service_account: '[email protected]'
+          workload_identity_provider: '${{ vars.GCP_WIF }}'
+          service_account: '${{ vars.GCP_DEPLOYMENTS_SA }}'
 
       - name: Set up Cloud SDK
         uses: google-github-actions/[email protected]
@@ -161,10 +155,10 @@ jobs:
           --boot-disk-type=pd-ssd \
           --image-project=cos-cloud \
           --image-family=cos-stable \
-          --container-image ${{ env.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }} \
+          --container-image ${{ vars.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }} \
           --create-disk=name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }},auto-delete=yes,size=300GB,type=pd-ssd \
           --container-mount-disk=mount-path="/zebrad-cache",name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }} \
-          --machine-type ${{ env.MACHINE_TYPE }} \
+          --machine-type ${{ vars.GCP_SMALL_MACHINE }} \
           --scopes cloud-platform \
           --labels=app=zebrad,environment=prod,network=${NETWORK},github_ref=${{ env.GITHUB_REF_SLUG_URL }} \
           --tags zebrad
@@ -174,7 +168,7 @@ jobs:
         id: does-group-exist
         continue-on-error: true
         run: |
-          gcloud compute instance-groups list | grep "zebrad-${{ needs.versioning.outputs.major_version || env.GITHUB_REF_SLUG_URL }}" | grep "${{ env.REGION }}"
+          gcloud compute instance-groups list | grep "zebrad-${{ needs.versioning.outputs.major_version || env.GITHUB_REF_SLUG_URL }}" | grep "${{ vars.GCP_REGION }}"
 
       # Deploy new managed instance group using the new instance template
       - name: Create managed instance group
@@ -185,7 +179,7 @@ jobs:
           --template "zebrad-${{ needs.versioning.outputs.major_version || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \
           --health-check zebrad-tracing-filter \
           --initial-delay 30 \
-          --region "${{ env.REGION }}" \
+          --region "${{ vars.GCP_REGION }}" \
           --size 1
 
       # Rolls out update to existing group using the new instance template
@@ -195,7 +189,7 @@ jobs:
           gcloud compute instance-groups managed rolling-action start-update \
           "zebrad-${{ needs.versioning.outputs.major_version || env.GITHUB_REF_SLUG_URL }}" \
           --version template="zebrad-${{ needs.versioning.outputs.major_version || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \
-          --region "${{ env.REGION }}"
+          --region "${{ vars.GCP_REGION }}"
 
   # This jobs handles the deployment of a single node (1) in the us-central1-a zone
   # when an instance is required to test a specific commit
@@ -230,8 +224,8 @@ jobs:
         uses: google-github-actions/[email protected]
         with:
           retries: '3'
-          workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc'
-          service_account: '[email protected]'
+          workload_identity_provider: '${{ vars.GCP_WIF }}'
+          service_account: '${{ vars.GCP_DEPLOYMENTS_SA }}'
 
       - name: Set up Cloud SDK
         uses: google-github-actions/[email protected]
@@ -246,11 +240,11 @@ jobs:
           --image-family=cos-stable \
           --container-stdin \
           --container-tty \
-          --container-image ${{ env.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }} \
+          --container-image ${{ vars.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }} \
           --create-disk=auto-delete=yes,size=300GB,type=pd-ssd \
           --create-disk=name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }},auto-delete=yes,size=300GB,type=pd-ssd \
           --container-mount-disk=mount-path='/zebrad-cache',name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }} \
-          --machine-type ${{ env.MACHINE_TYPE }} \
-          --zone ${{ env.ZONE }} \
+          --machine-type ${{ vars.GCP_SMALL_MACHINE }} \
+          --zone ${{ vars.GCP_ZONE }} \
           --labels=app=zebrad,environment=qa,network=${NETWORK},github_ref=${{ env.GITHUB_REF_SLUG_URL }} \
           --tags zebrad
diff --git a/.github/workflows/continous-integration-docker.yml b/.github/workflows/continous-integration-docker.yml
@@ -76,13 +76,6 @@ on:
       - '.github/workflows/deploy-gcp-tests.yml'
       - '.github/workflows/build-docker-image.yml'
 
-env:
-  # TODO: use the output from ./.github/workflows/build-docker-image.yml
-  IMAGE_NAME: zebrad-test
-  GAR_BASE: us-docker.pkg.dev/zealous-zebra/zebra
-  # TODO: use environmental secrets for dynamic values
-  NETWORK: Mainnet
-
 jobs:
   get-available-disks:
     runs-on: ubuntu-latest
@@ -106,19 +99,19 @@ jobs:
         uses: google-github-actions/[email protected]
         with:
           retries: '3'
-          workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc'
-          service_account: '[email protected]'
+          workload_identity_provider: '${{ vars.GCP_WIF }}'
+          service_account: '${{ vars.GCP_DEPLOYMENTS_SA }}'
 
       - name: Set up Cloud SDK
         uses: google-github-actions/[email protected]
 
       # Disk images in GCP are required to be in lowercase, but the blockchain network
-      # uses sentence case, so we need to downcase ${{ env.NETWORK or github.event.inputs.network }}
+      # uses sentence case, so we need to downcase ${{ vars.ZCASH_NETWORK or github.event.inputs.network }}
       #
       # Passes a lowercase Network name to subsequent steps using $NETWORK env variable
       - name: Downcase network name for disks
         run: |
-          NETWORK_CAPS=${{ env.NETWORK || github.event.inputs.network }}
+          NETWORK_CAPS=${{ github.event.inputs.network || vars.ZCASH_NETWORK }}
           echo "NETWORK=${NETWORK_CAPS,,}" >> $GITHUB_ENV
 
       # Find a cached state disk for subsequent jobs needing a cached state without
@@ -169,9 +162,9 @@ jobs:
     with:
       dockerfile_path: ./docker/Dockerfile
       dockerfile_target: tests
-      image_name: zebrad-test
-      # TODO: validate how to use variable/conditional values for Testnet
-      network: Mainnet
+      image_name: ${{ vars.CI_IMAGE_NAME }}
+      # The default network in the image is based on the default network variable in GitHub
+      network: ${{ vars.ZCASH_NETWORK }}
       checkpoint_sync: true
       rust_backtrace: full
       rust_lib_backtrace: full
@@ -204,8 +197,8 @@ jobs:
       # If some tests hang, add "-- --nocapture" for just that test, or for all the tests.
       - name: Run zebrad tests
         run: |
-          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
-          docker run --name zebrad-tests --tty ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features "lightwalletd-grpc-tests" --workspace -- --include-ignored
+          docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
+          docker run --name zebrad-tests --tty ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features "lightwalletd-grpc-tests" --workspace -- --include-ignored
 
   # zebrad tests without cached state with `getblocktemplate-rpcs` feature
   #
@@ -223,8 +216,8 @@ jobs:
 
       - name: Run zebrad tests
         run: |
-          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
-          docker run --name zebrad-tests --tty ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features "lightwalletd-grpc-tests getblocktemplate-rpcs" --workspace -- --include-ignored
+          docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
+          docker run --name zebrad-tests --tty ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features "lightwalletd-grpc-tests getblocktemplate-rpcs" --workspace -- --include-ignored
 
   # Run state tests with fake activation heights.
   #
@@ -248,8 +241,8 @@ jobs:
 
       - name: Run tests with fake activation heights
         run: |
-          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
-          docker run -e TEST_FAKE_ACTIVATION_HEIGHTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --package zebra-state --lib -- --nocapture --include-ignored with_fake_activation_heights
+          docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
+          docker run -e TEST_FAKE_ACTIVATION_HEIGHTS --name zebrad-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --package zebra-state --lib -- --nocapture --include-ignored with_fake_activation_heights
         env:
           TEST_FAKE_ACTIVATION_HEIGHTS: '1'
 
@@ -269,8 +262,8 @@ jobs:
 
       - name: Run zebrad large sync tests
         run: |
-          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
-          docker run --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored sync_large_checkpoints_
+          docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
+          docker run --name zebrad-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored sync_large_checkpoints_
 
   # Test launching lightwalletd with an empty lightwalletd and Zebra state.
   #
@@ -288,8 +281,8 @@ jobs:
 
       - name: Run tests with empty lightwalletd launch
         run: |
-          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
-          docker run -e ZEBRA_TEST_LIGHTWALLETD --name lightwalletd-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored lightwalletd_integration
+          docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
+          docker run -e ZEBRA_TEST_LIGHTWALLETD --name lightwalletd-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored lightwalletd_integration
         env:
           ZEBRA_TEST_LIGHTWALLETD: '1'
 
@@ -309,8 +302,8 @@ jobs:
       - name: Run tests using the default config
         run: |
           set -ex
-          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
-          docker run --detach --name default-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} zebrad start
+          docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
+          docker run --detach --name default-conf-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} zebrad start
           EXIT_STATUS=$(docker logs --tail all --follow default-conf-tests 2>&1 | grep -q --extended-regexp --max-count=1 -e 'estimated progress to chain tip.*BeforeOverwinter'; echo $?; )
           docker stop default-conf-tests
           docker logs default-conf-tests
@@ -332,8 +325,8 @@ jobs:
       - name: Run tests using the $ZEBRA_CONF_PATH
         run: |
           set -ex
-          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
-          docker run --detach -e ZEBRA_CONF_PATH --name variable-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} -c $ZEBRA_CONF_PATH start
+          docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
+          docker run --detach -e ZEBRA_CONF_PATH --name variable-conf-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} -c $ZEBRA_CONF_PATH start
           EXIT_STATUS=$(docker logs --tail all --follow variable-conf-tests 2>&1 | grep -q --extended-regexp --max-count=1 -e 'v1.0.0-rc.2.toml'; echo $?; )
           docker stop variable-conf-tests
           docker logs variable-conf-tests

diff --git a/.github/workflows/continous-integration-os.yml b/.github/workflows/continous-integration-os.yml
@@ -51,11 +51,11 @@ on:
       - '.github/workflows/continous-integration-os.yml'
 
 env:
-  CARGO_INCREMENTAL: 0
-  RUST_LOG: info
-  RUST_BACKTRACE: full
-  RUST_LIB_BACKTRACE: full
-  COLORBT_SHOW_HIDDEN: '1'
+  CARGO_INCREMENTAL: ${{ vars.CARGO_INCREMENTAL }}
+  RUST_LOG: ${{ vars.RUST_LOG }}
+  RUST_BACKTRACE: ${{ vars.RUST_BACKTRACE }}
+  RUST_LIB_BACKTRACE: ${{ vars.RUST_LIB_BACKTRACE }}
+  COLORBT_SHOW_HIDDEN: ${{ vars.COLORBT_SHOW_HIDDEN }}
 
 jobs:
   test:

diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
@@ -41,11 +41,11 @@ on:
       - '.github/workflows/coverage.yml'
 
 env:
-  CARGO_INCREMENTAL: 0
-  RUST_LOG: info
-  RUST_BACKTRACE: full
-  RUST_LIB_BACKTRACE: full
-  COLORBT_SHOW_HIDDEN: '1'
+  CARGO_INCREMENTAL: ${{ vars.CARGO_INCREMENTAL }}
+  RUST_LOG: ${{ vars.RUST_LOG }}
+  RUST_BACKTRACE: ${{ vars.RUST_BACKTRACE }}
+  RUST_LIB_BACKTRACE: ${{ vars.RUST_LIB_BACKTRACE }}
+  COLORBT_SHOW_HIDDEN: ${{ vars.COLORBT_SHOW_HIDDEN }}
 
 jobs:
   coverage:

diff --git a/.github/workflows/delete-gcp-resources.yml b/.github/workflows/delete-gcp-resources.yml
@@ -40,8 +40,8 @@ jobs:
         uses: google-github-actions/[email protected]
         with:
           retries: '3'
-          workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc'
-          service_account: '[email protected]'
+          workload_identity_provider: '${{ vars.GCP_WIF }}'
+          service_account: '${{ vars.GCP_DEPLOYMENTS_SA }}'
 
       - name: Set up Cloud SDK
         uses: google-github-actions/[email protected]
@@ -192,8 +192,8 @@ jobs:
         uses: google-github-actions/[email protected]
         with:
           retries: '3'
-          workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc'
-          service_account: '[email protected]'
+          workload_identity_provider: '${{ vars.GCP_WIF }}'
+          service_account: '${{ vars.GCP_DEPLOYMENTS_SA }}'
           token_format: 'access_token'
 
       - name: Login to Google Artifact Registry
@@ -210,7 +210,7 @@ jobs:
         # https://github.com/GoogleCloudPlatform/gcr-cleaner
         with:
           args: >-
-            -repo=us-docker.pkg.dev/zealous-zebra/zebra/zebrad-test
-            -repo=us-docker.pkg.dev/zealous-zebra/zebra/lightwalletd
+            -repo=us-docker.pkg.dev/${{ vars.GCP_PROJECT }}/zebra/zebrad-test
+            -repo=us-docker.pkg.dev/${{ vars.GCP_PROJECT }}/zebra/lightwalletd
             -grace=${{ env.DELETE_IMAGE_HOURS }}
             -keep=${{ env.KEEP_LATEST_IMAGE_COUNT }}