Merge branch 'main' of https://github.com/ZcashFoundation/zebra into …

…update-zcash

.github/mergify.yml

@@ -5,7 +5,7 @@ queue_rules:
     allow_inplace_checks: True
     allow_checks_interruption: False
     speculative_checks: 1
-    batch_size: 5
+    batch_size: 8
     # Wait a short time to embark hotfixes together in a merge train
     batch_max_wait_time: "2 minutes"
     conditions:
@@ -18,19 +18,19 @@ queue_rules:
     allow_inplace_checks: True
     allow_checks_interruption: True
     speculative_checks: 1
-    batch_size: 5
+    batch_size: 20
     # Wait for a few minutes to embark high priority tickets together in a merge train
-    batch_max_wait_time: "5 minutes"
+    batch_max_wait_time: "10 minutes"
     conditions:
       - base=main
 
   - name: low
     allow_inplace_checks: True
     allow_checks_interruption: True
     speculative_checks: 1
-    batch_size: 5
+    batch_size: 20
     # Wait a bit longer to embark low priority tickets together in a merge train
-    batch_max_wait_time: "10 minutes"
+    batch_max_wait_time: "20 minutes"
     conditions:
       - base=main
 
@@ -85,4 +85,4 @@ pull_request_rules:
     actions:
       queue:
         name: low
-        method: squash
+        method: squash

.github/workflows/build-crates-individually.patch.yml

@@ -10,6 +10,9 @@ on:
       # dependencies
       - '**/Cargo.toml'
       - '**/Cargo.lock'
+      # configuration files
+      - '.cargo/config.toml'
+      - '**/clippy.toml'
       # workflow definitions
       - '.github/workflows/build-crates-individually.yml'
 

.github/workflows/build-crates-individually.yml

@@ -1,5 +1,12 @@
 name: Build crates individually
 
+# Ensures that only one workflow task will run at a time. Previous builds, if
+# already in process, will get cancelled. Only the latest commit will be allowed
+# to run, cancelling any workflows in between
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 on:
   workflow_dispatch:
   push:
@@ -11,6 +18,9 @@ on:
       # dependencies
       - '**/Cargo.toml'
       - '**/Cargo.lock'
+      # configuration files
+      - '.cargo/config.toml'
+      - '**/clippy.toml'
       # workflow definitions
       - '.github/workflows/build-crates-individually.yml'
   pull_request:
@@ -20,11 +30,15 @@ on:
       # dependencies
       - '**/Cargo.toml'
       - '**/Cargo.lock'
+      # configuration files
+      - '.cargo/config.toml'
+      - '**/clippy.toml'
       # workflow definitions
       - '.github/workflows/build-crates-individually.yml'
 
 env:
   CARGO_INCREMENTAL: 0
+  RUST_LOG: info
   RUST_BACKTRACE: full
   RUST_LIB_BACKTRACE: full
   COLORBT_SHOW_HIDDEN: '1'
@@ -48,11 +62,11 @@ jobs:
       # This step is meant to dynamically create a JSON containing the values of each crate
       # available in this repo in the root directory. We use `cargo tree` to accomplish this task.
       #
-      # The result from `cargo tree` is then transform to JSON values between double quotes, 
+      # The result from `cargo tree` is then transform to JSON values between double quotes,
       # and separated by commas, then added to a `crates.txt` and assigned to a $JSON_CRATES variable.
       #
       # A JSON object is created and assigned to a $MATRIX variable, which is use to create an output
-      # named `matrix`, which is then used as the input in following steps, 
+      # named `matrix`, which is then used as the input in following steps,
       # using ` ${{ fromJson(needs.matrix.outputs.matrix) }}`
       - id: set-matrix
         name: Dynamically build crates JSON
@@ -104,7 +118,7 @@ jobs:
 
         # We could use `features: ['', '--all-features', '--no-default-features']` as a matrix argument,
         # but it's faster to run these commands sequentially, so they can re-use the local cargo cache.
-        # 
+        #
         # Some Zebra crates do not have any features, and most don't have any default features.
       - name: Build ${{ matrix.crate }} crate with no default features
         uses: actions-rs/[email protected]

.github/workflows/build-docker-image.yml

@@ -36,12 +36,20 @@ on:
       rust_log:
         required: false
         type: string
+        default: info
+    outputs:
+      image_digest:
+        description: 'The image digest to be used on a caller workflow'
+        value: ${{ jobs.build.outputs.image_digest }}
 
 jobs:
   build:
     name: Build images
     timeout-minutes: 210
     runs-on: ubuntu-latest
+    outputs:
+      image_digest: ${{ steps.docker_build.outputs.digest }}
+      image_name: ${{ fromJSON(steps.docker_build.outputs.metadata)['image.name'] }}
     permissions:
       contents: 'read'
       id-token: 'write'
@@ -66,12 +74,12 @@ jobs:
           # generate Docker tags based on the following events/attributes
           tags: |
             type=schedule
+            type=sha
             type=ref,event=branch
             type=ref,event=pr
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
             type=semver,pattern={{major}}
-            type=sha
 
       # Setup Docker Buildx to allow use of docker cache layers from GH
       - name: Set up Docker Buildx
@@ -82,6 +90,7 @@ jobs:
         id: auth
         uses: google-github-actions/[email protected]
         with:
+          retries: '3'
           workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc'
           service_account: '[email protected]'
           token_format: 'access_token'

.github/workflows/continous-delivery.yml

@@ -1,5 +1,12 @@
 name: CD
 
+# Ensures that only one workflow task will run at a time. Previous deployments, if
+# already in process, won't get cancelled. Instead, we let the first to complete
+# then queue the latest pending workflow, cancelling any workflows in between
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
+
 on:
   workflow_dispatch:
     inputs:
@@ -14,6 +21,9 @@ on:
   push:
     branches:
       - main
+  release:
+    types:
+      - published
 
 env:
   NETWORK: Mainnet
@@ -23,6 +33,37 @@ env:
   MACHINE_TYPE: c2-standard-4
 
 jobs:
+  # If a release was made we want to extract the first part of the semver from the
+  # tag_name
+  #
+  # Generate the following output to pass to subsequent jobs
+  # - If our semver is `v1.3.0` the resulting output from this job would be `v1`
+  #
+  # Note: We just use the first part of the version to replace old instances, and change
+  # it when a major version is released, to keep a segregation between new and old 
+  # versions.
+  versioning:
+    name: Versioning
+    runs-on: ubuntu-latest
+    outputs:
+      major_version: ${{ steps.set.outputs.major_version }}
+    if: ${{ github.event_name == 'release' }}
+    steps:
+      - name: Getting Zebrad Version
+        id: get
+        uses: actions/[email protected]
+        with:
+          result-encoding: string
+          script: |
+            return context.payload.release.tag_name.substring(0,2)
+      - name: Setting API Version
+        id: set
+        run: echo "::set-output name=major_version::${{ steps.get.outputs.result }}"
+
+  # Each time this workflow is executed, a build will be triggered to create a new image
+  # with the corresponding tags using information from Git
+  #
+  # The image will be commonly named `zebrad:<short-hash | github-ref | semver>`
   build:
     uses: ./.github/workflows/build-docker-image.yml
     with:
@@ -35,15 +76,26 @@ jobs:
       zebra_skip_ipv6_tests: '1'
       rust_log: info
 
+  # This jobs handles the deployment of a Managed Instance Group (MiG) with 2 nodes in
+  # the us-central1 region. Two different groups of MiGs are deployed one for pushes to
+  # the main branch and another for version releases of Zebra
+  #
+  # Once this workflow is triggered the previous MiG is replaced, on pushes to main its
+  # always replaced, and with releases its only replaced if the same major version is
+  # being deployed, otherwise a new major version is deployed
+  #
+  # Runs:
+  # - on every push/merge to the `main` branch
+  # - on every release, when it's published
   deploy-nodes:
     name: Deploy Mainnet nodes
-    needs: build
+    needs: [ build, versioning ]
     runs-on: ubuntu-latest
     timeout-minutes: 30
     permissions:
       contents: 'read'
       id-token: 'write'
-    if: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
+    if: ${{ !cancelled() && !failure() && ((github.event_name == 'push' && github.ref_name == 'main') || github.event_name == 'release') }}
 
     steps:
       - name: Inject slug/short variables
@@ -56,15 +108,16 @@ jobs:
         id: auth
         uses: google-github-actions/[email protected]
         with:
+          retries: '3'
           workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc'
           service_account: '[email protected]'
           token_format: 'access_token'
 
       - name: Create instance template
         run: |
-          gcloud compute instance-templates create-with-container zebrad-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
+          gcloud compute instance-templates create-with-container zebrad-${{ needs.versioning.outputs.major_version || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
           --boot-disk-type=pd-ssd \
-          --container-image ${{ env.GAR_BASE }}/${{ env.GITHUB_REF_SLUG_URL }}:${{ env.GITHUB_SHA_SHORT }} \
+          --container-image ${{ env.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }} \
           --create-disk name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }},auto-delete=yes,size=100GB,type=pd-ssd \
           --container-mount-disk mount-path="/zebrad-cache",name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }} \
           --machine-type ${{ env.MACHINE_TYPE }} \
@@ -76,15 +129,15 @@ jobs:
         id: does-group-exist
         continue-on-error: true
         run: |
-          gcloud compute instance-groups list | grep "zebrad-${{ env.GITHUB_REF_SLUG_URL }}" | grep "${{ env.REGION }}"
+          gcloud compute instance-groups list | grep "zebrad-${{ needs.versioning.outputs.major_version || env.GITHUB_REF_SLUG_URL }}" | grep "${{ env.REGION }}"
 
       # Deploy new managed instance group using the new instance template
       - name: Create managed instance group
         if: steps.does-group-exist.outcome == 'failure'
         run: |
           gcloud compute instance-groups managed create \
-          "zebrad-${{ env.GITHUB_REF_SLUG_URL }}" \
-          --template "zebrad-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \
+          "zebrad-${{ needs.versioning.outputs.major_version || env.GITHUB_REF_SLUG_URL }}" \
+          --template "zebrad-${{ needs.versioning.outputs.major_version || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \
           --health-check zebrad-tracing-filter \
           --initial-delay 30 \
           --region "${{ env.REGION }}" \
@@ -95,10 +148,17 @@ jobs:
         if: steps.does-group-exist.outcome == 'success'
         run: |
           gcloud compute instance-groups managed rolling-action start-update \
-          "zebrad-${{ env.GITHUB_REF_SLUG_URL }}" \
-          --version template="zebrad-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \
+          "zebrad-${{ needs.versioning.outputs.major_version || env.GITHUB_REF_SLUG_URL }}" \
+          --version template="zebrad-${{ needs.versioning.outputs.major_version || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \
           --region "${{ env.REGION }}"
 
+  # This jobs handles the deployment of a single node (1) in the us-central1-a zone
+  # when an instance is required to test a specific commit
+  #
+  # Runs:
+  # - on request, using workflow_dispatch with regenerate-disks
+  #
+  # Note: this instances are not automatically replaced or deleted
   deploy-instance:
     name: Deploy single instance
     needs: build
@@ -120,6 +180,7 @@ jobs:
         id: auth
         uses: google-github-actions/[email protected]
         with:
+          retries: '3'
           workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc'
           service_account: '[email protected]'
           token_format: 'access_token'
@@ -132,7 +193,7 @@ jobs:
           --boot-disk-type=pd-ssd \
           --container-stdin \
           --container-tty \
-          --container-image ${{ env.GAR_BASE }}/${{ env.GITHUB_REF_SLUG_URL }}:${{ env.GITHUB_SHA_SHORT }} \
+          --container-image ${{ env.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }} \
           --create-disk name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }},auto-delete=yes,size=100GB,type=pd-ssd \
           --container-mount-disk mount-path='/zebrad-cache',name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }} \
           --machine-type ${{ env.MACHINE_TYPE }} \

.github/workflows/continous-integration-docker.patch.yml

@@ -14,6 +14,9 @@ on:
       # dependencies
       - '**/Cargo.toml'
       - '**/Cargo.lock'
+      # configuration files
+      - '.cargo/config.toml'
+      - '**/clippy.toml'
       # workflow definitions
       - 'docker/**'
       - '.github/workflows/continous-integration-docker.yml'