From 5f812db94c1ba40f6a12ef8144e337509df45dce Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Thu, 27 Jan 2022 22:43:31 -0400
Subject: [PATCH 001/124] refactor (cd): overall pipeline improvement

- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
---
 .github/workflows/cd.yml            | 111 +++++++++++++++++++++++-----
 .github/workflows/manual-deploy.yml |  53 -------------
 cloudbuild.yaml                     |  25 -------
 docker/Dockerfile.build             |  62 ++++++++++------
 4 files changed, 133 insertions(+), 118 deletions(-)
 delete mode 100644 .github/workflows/manual-deploy.yml
 delete mode 100644 cloudbuild.yaml

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index 0ed6421bfa6..2c657ca8e11 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -2,22 +2,30 @@ name: CD
 
 on:
   workflow_dispatch:
+    inputs:
+      network:
+        default: 'Mainnet'
+      checkpoint_sync:
+        default: true
   push:
     branches:
       - main
 
 env:
+  NETWORK: Mainnet
   PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
-  REGION: us-east1
-  ZONE: us-east1-b
-  MACHINE_TYPE: n2d-standard-4
-  DEPLOY_SA: cos-vm@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com
+  GAR_BASE: us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/zebra
+  REGION: us-central1
+  ZONE: us-central1-a
+  MACHINE_TYPE: c2d-standard-4
 
 jobs:
   build:
     name: Build images
     timeout-minutes: 60
     runs-on: ubuntu-latest
+    if: github.event_name == 'push'
+
     steps:
     - uses: actions/checkout@v2.4.0
       with:
@@ -33,19 +41,54 @@ jobs:
         project_id: ${{ env.PROJECT_ID }}
         service_account_key: ${{ secrets.GCLOUD_AUTH }}
 
-    # Build and push image to Google Container Registry
-    - name: Build
-      # Tagging w/ the commit SHA blocks the :latest tag on GCR
-      run: |
-        gcloud builds submit \
-          --config cloudbuild.yaml \
-          --substitutions SHORT_SHA="${{ env.GITHUB_SHA_SHORT }}",BRANCH_NAME="${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}"
+      # Setup Docker Buildx to allow use of docker cache layers from GH
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          driver-opts: network=host
 
-  deploy:
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: |
+            /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}
+
+    - name: Login to GAR
+      uses: docker/login-action@v1.12.0
+      with:
+        registry: us-docker.pkg.dev
+        username: _json_key
+        password: ${{ secrets.GCLOUD_AUTH }}
+
+    # Build and push image to Google Artifact Registry
+    - name: Build & push
+      id: docker_build
+      uses: docker/build-push-action@v2.8.0
+      with:
+        target: zebrad-release
+        context: .
+        file: ./docker/Dockerfile.build
+        tags: ${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:${{ env.GITHUB_SHA_SHORT }}
+        build-args: |
+          NETWORK=${{ env.NETWORK }}
+          SHORT_SHA=${{ env.GITHUB_SHA_SHORT }}
+          RUST_BACKTRACE=full
+          ZEBRA_SKIP_NETWORK_TESTS="-1"
+          CHECKPOINT_SYNC=true
+          RUST_LOG=debug
+          SENTRY_DSN=${{ secrets.SENTRY_ENDPOINT }}
+        push: true
+        cache-from: type=registry,ref=${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}
+        cache-to: type=inline
+
+  deploy-nodes:
     name: Deploy mainnet nodes
     needs: build
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    if: github.event_name == 'push'
+
     steps:
       - uses: actions/checkout@v2.4.0
         with:
@@ -61,17 +104,14 @@ jobs:
           project_id: ${{ env.PROJECT_ID }}
           service_account_key: ${{ secrets.GCLOUD_AUTH }}
 
-      # Create instance template from container image
       - name: Create instance template
         run: |
-          gcloud compute instance-templates create-with-container "zebrad-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
-          --container-image "gcr.io/${{ env.PROJECT_ID}}/${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:${{ env.GITHUB_SHA_SHORT }}" \
-          --create-disk name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }},auto-delete=yes,size=100GB,type=pd-balanced \
+          gcloud compute instance-templates create-with-container zebrad-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} \
+          --container-image ${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }} \
+          --create-disk name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }},auto-delete=yes,size=100GB,type=pd-ssd \
           --container-mount-disk mount-path="/zebrad-cache",name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }} \
           --machine-type ${{ env.MACHINE_TYPE }} \
-          --service-account ${{ env.DEPLOY_SA }} \
-          --scopes cloud-platform \
-          --tags zebrad \
+          --tags zebrad
 
       # Check if our destination instance group exists already
       - name: Check if instance group exists
@@ -100,3 +140,36 @@ jobs:
           "zebrad-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}" \
           --version template="zebrad-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
           --region "${{ env.REGION }}"
+
+  deploy-instance:
+    name: Deploy mainnet nodes
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    if: github.event_name == 'workflow_dispatch'
+
+    steps:
+      - uses: actions/checkout@v2.4.0
+        with:
+          persist-credentials: false
+
+      - name: Inject slug/short variables
+        uses: rlespinasse/github-slug-action@v4
+
+      # Setup gcloud CLI
+      - name: Set up gcloud SDK environment
+        uses: google-github-actions/setup-gcloud@v0.4.0
+        with:
+          project_id: ${{ env.PROJECT_ID }}
+          service_account_key: ${{ secrets.GCLOUD_AUTH }}
+
+      # Create instance template from container image
+      - name: Manual deploy of a single instance running zebrad
+        run: |
+          gcloud compute instances create-with-container "zebrad-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
+          --container-image ${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }} \
+          --create-disk name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }},auto-delete=yes,size=100GB,type=pd-ssd \
+          --container-mount-disk mount-path='/zebrad-cache',name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }} \
+          --machine-type ${{ env.MACHINE_TYPE }} \
+          --zone ${{ env.ZONE }} \
+          --tags zebrad
diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml
deleted file mode 100644
index 47087393c09..00000000000
--- a/.github/workflows/manual-deploy.yml
+++ /dev/null
@@ -1,53 +0,0 @@
-name: Manual Deploy
-
-on:
-  workflow_dispatch:
-    inputs:
-      network:
-        default: 'Mainnet'
-      checkpoint_sync:
-        default: true
-
-env:
-  PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
-  ZONE: us-central1-a
-  MACHINE_TYPE: n2-standard-4
-  DEPLOY_SA: cos-vm@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com
-
-jobs:
-  deploy:
-    name: Deploy one zebrad node
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2.4.0
-      with:
-        persist-credentials: false
-
-    - name: Inject slug/short variables
-      uses: rlespinasse/github-slug-action@v4
-
-    - name: Set up gcloud
-      uses: google-github-actions/setup-gcloud@v0.4.0
-      with:
-        project_id: ${{ env.PROJECT_ID }}
-        service_account_key: ${{ secrets.GCLOUD_AUTH }}
-
-    # Build and push image to Google Container Registry
-    - name: Build
-      # Tagging w/ the commit SHA blocks the :latest tag on GCR
-      run: |
-        gcloud builds submit \
-          --config cloudbuild.yaml \
-          --substitutions SHORT_SHA="${{ env.GITHUB_SHA_SHORT }}",BRANCH_NAME="${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}",_CHECKPOINT_SYNC="${{ github.event.inputs.checkpoint_sync }}",_NETWORK="${{ github.event.inputs.network }}"
-
-    # Creates Compute Engine virtual machine instance w/ zebrad container and disks
-    - name: Create instance running zebrad container image
-      run: |
-        gcloud compute instances create-with-container "zebrad-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
-        --container-image "gcr.io/${{ env.PROJECT_ID }}/${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:${{ env.GITHUB_SHA_SHORT }}" \
-        --container-mount-disk mount-path='/zebrad-cache',name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }} \
-        --create-disk name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }},auto-delete=yes,size=100GB,type=pd-balanced \
-        --machine-type ${{ env.MACHINE_TYPE }} \
-        --service-account ${{ env.DEPLOY_SA }} \
-        --tags zebrad \
-        --zone ${{ env.ZONE }}
diff --git a/cloudbuild.yaml b/cloudbuild.yaml
deleted file mode 100644
index f09a10e0292..00000000000
--- a/cloudbuild.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-steps:
-- name: 'gcr.io/cloud-builders/docker'
-  args: ['build',
-        '--build-arg',
-         'SHORT_SHA=$SHORT_SHA',
-         '--build-arg',
-         'checkpoint_sync=${_CHECKPOINT_SYNC}',
-         '--build-arg',
-         'network=${_NETWORK}',
-         '--target',
-         'zebrad-release',
-         '-t',
-         'gcr.io/$PROJECT_ID/zcashfoundation/zebra/$BRANCH_NAME:$SHORT_SHA',
-         '-f',
-         'docker/Dockerfile.build',
-         '.']
-
-images:
-- 'gcr.io/$PROJECT_ID/zcashfoundation/zebra/$BRANCH_NAME:$SHORT_SHA'
-
-options:
-    machineType: 'N1_HIGHCPU_32'
-    substitution_option: 'ALLOW_LOOSE'
-
-timeout: 3600s # 60 mins
diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index 31e1d1c04a2..7fb37765c55 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -1,9 +1,9 @@
-# Builder image
-FROM rust:buster as builder
+FROM rust:1.57-buster as builder
 
 RUN apt-get update && \
-	apt-get install -y --no-install-recommends \
-	make cmake g++ gcc llvm libclang-dev clang ca-certificates
+    apt-get install -y --no-install-recommends \
+    make cmake g++ gcc llvm libclang-dev clang ca-certificates && \
+    rm -Rf /var/lib/apt/lists/* /tmp/*
 
 RUN mkdir /zebra
 WORKDIR /zebra
@@ -11,37 +11,54 @@ WORKDIR /zebra
 ARG SHORT_SHA
 ENV SHORT_SHA $SHORT_SHA
 
-ENV RUST_BACKTRACE full
+ARG RUST_BACKTRACE
+ENV RUST_BACKTRACE ${RUST_BACKTRACE:-full}
+
+# Skip test on debian based OS by default as compilation fails
+ARG ZEBRA_SKIP_NETWORK_TESTS
+ENV ZEBRA_SKIP_NETWORK_TESTS ${ZEBRA_SKIP_NETWORK_TESTS:-1}
+
+# Optimize builds. In particular, regenerate-stateful-test-disks.yml was reaching the
+# GitHub Actions time limit (6 hours), so we needed to make it faster.
+ENV RUSTFLAGS -O
+
 ENV CARGO_HOME /zebra/.cargo/
 
 RUN rustc -V; cargo -V; rustup -V
 
 COPY . .
 
-RUN cd zebrad/; cargo build --release --features enable-sentry
+# Build and then compile tests
+# Compile, but don't run tests (--no-run). Add verbosity and colors
+RUN cargo build --release --features enable-sentry
+RUN cargo test --workspace --no-run
 
 # Runner image
 FROM debian:buster-slim AS zebrad-release
 
 RUN apt-get update && \
-	apt-get install -y --no-install-recommends \
-	ca-certificates
+    apt-get install -y --no-install-recommends \
+    ca-certificates
 
 COPY --from=builder /zebra/target/release/zebrad /
 
 ARG CHECKPOINT_SYNC=true
 ARG NETWORK=Mainnet
 
-RUN printf "[consensus]\n" >> /zebrad.toml
-RUN printf "checkpoint_sync = ${CHECKPOINT_SYNC}\n" >> /zebrad.toml
-RUN printf "[metrics]\n" >> /zebrad.toml
-RUN printf "endpoint_addr = '0.0.0.0:9999'\n" >> /zebrad.toml
-RUN printf "[network]\n" >> /zebrad.toml
-RUN printf "network = '${NETWORK}'\n" >> /zebrad.toml
-RUN printf "[state]\n" >> /zebrad.toml
-RUN printf "cache_dir = '/zebrad-cache'\n" >> /zebrad.toml
-RUN printf "[tracing]\n" >> /zebrad.toml
-RUN printf "endpoint_addr = '0.0.0.0:3000'\n" >> /zebrad.toml
+RUN set -ex; \
+  { \
+    echo "[consensus]"; \
+    echo "checkpoint_sync = ${CHECKPOINT_SYNC}"; \
+    echo "[metrics]"; \
+    echo "endpoint_addr = '0.0.0.0:9999'"; \
+    echo "[network]"; \
+    echo "network = '${NETWORK}'"; \
+    echo "[state]"; \
+    echo "cache_dir = '/zebrad-cache'"; \
+    echo "[tracing]"; \
+    echo "endpoint_addr = '0.0.0.0:3000'"; \
+  } > "/zebrad.toml"
+
 RUN cat /zebrad.toml
 
 # Pre-download Zcash Sprout and Sapling parameters
@@ -49,8 +66,11 @@ RUN /zebrad download
 
 EXPOSE 3000 8233 18233
 
-ENV RUST_LOG debug
-ENV RUST_BACKTRACE full
-ENV SENTRY_DSN https://94059ee72a44420286310990b7c614b5@o485484.ingest.sentry.io/5540918
+ARG RUST_LOG
+ENV RUST_LOG ${RUST_LOG:-debug}
+ARG RUST_BACKTRACE
+ENV RUST_BACKTRACE ${RUST_BACKTRACE:-full}
+ARG SENTRY_DSN
+ENV SENTRY_DSN ${SENTRY_DSN}
 
 CMD [ "/zebrad", "-c", "/zebrad.toml", "start" ]

From 3824e7d49764dba3b73143102ef75e9524aeca2a Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Thu, 27 Jan 2022 22:43:31 -0400
Subject: [PATCH 002/124] refactor (cd): overall pipeline improvement

- Use a more ENV configurable Dockerfile
- Remove cloudbuild dependency
- Use compute optimized machine types
- Use SSD instead of normal hard drives
- Move Sentry endpoint to secrets
- Use a single yml for auto & manual deploy
- Migrate to Google Artifact Registry
---
 .github/workflows/cd.yml            | 111 +++++++++++++++++++++++-----
 .github/workflows/manual-deploy.yml |  53 -------------
 cloudbuild.yaml                     |  25 -------
 docker/Dockerfile.build             |  62 ++++++++++------
 4 files changed, 133 insertions(+), 118 deletions(-)
 delete mode 100644 .github/workflows/manual-deploy.yml
 delete mode 100644 cloudbuild.yaml

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index 0ed6421bfa6..a360c086ba5 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -2,22 +2,30 @@ name: CD
 
 on:
   workflow_dispatch:
+    inputs:
+      network:
+        default: 'Mainnet'
+      checkpoint_sync:
+        default: true
   push:
     branches:
       - main
 
 env:
+  NETWORK: Mainnet
   PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
-  REGION: us-east1
-  ZONE: us-east1-b
-  MACHINE_TYPE: n2d-standard-4
-  DEPLOY_SA: cos-vm@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com
+  GAR_BASE: us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/zebra
+  REGION: us-central1
+  ZONE: us-central1-a
+  MACHINE_TYPE: c2d-standard-4
 
 jobs:
   build:
     name: Build images
     timeout-minutes: 60
     runs-on: ubuntu-latest
+    if: github.event_name == 'push'
+
     steps:
     - uses: actions/checkout@v2.4.0
       with:
@@ -33,19 +41,54 @@ jobs:
         project_id: ${{ env.PROJECT_ID }}
         service_account_key: ${{ secrets.GCLOUD_AUTH }}
 
-    # Build and push image to Google Container Registry
-    - name: Build
-      # Tagging w/ the commit SHA blocks the :latest tag on GCR
-      run: |
-        gcloud builds submit \
-          --config cloudbuild.yaml \
-          --substitutions SHORT_SHA="${{ env.GITHUB_SHA_SHORT }}",BRANCH_NAME="${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}"
+    # Setup Docker Buildx to allow use of docker cache layers from GH
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+      with:
+        driver-opts: network=host
 
-  deploy:
+    - name: Cache Docker layers
+      uses: actions/cache@v2
+      with:
+        path: |
+          /tmp/.buildx-cache
+        key: ${{ runner.os }}-buildx-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}
+
+    - name: Login to GAR
+      uses: docker/login-action@v1.12.0
+      with:
+        registry: us-docker.pkg.dev
+        username: _json_key
+        password: ${{ secrets.GCLOUD_AUTH }}
+
+    # Build and push image to Google Artifact Registry
+    - name: Build & push
+      id: docker_build
+      uses: docker/build-push-action@v2.8.0
+      with:
+        target: zebrad-release
+        context: .
+        file: ./docker/Dockerfile.build
+        tags: ${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:${{ env.GITHUB_SHA_SHORT }}
+        build-args: |
+          NETWORK=${{ env.NETWORK }}
+          SHORT_SHA=${{ env.GITHUB_SHA_SHORT }}
+          RUST_BACKTRACE=full
+          ZEBRA_SKIP_NETWORK_TESTS="-1"
+          CHECKPOINT_SYNC=true
+          RUST_LOG=debug
+          SENTRY_DSN=${{ secrets.SENTRY_ENDPOINT }}
+        push: true
+        cache-from: type=registry,ref=${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}
+        cache-to: type=inline
+
+  deploy-nodes:
     name: Deploy mainnet nodes
     needs: build
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    if: github.event_name == 'push'
+
     steps:
       - uses: actions/checkout@v2.4.0
         with:
@@ -61,17 +104,14 @@ jobs:
           project_id: ${{ env.PROJECT_ID }}
           service_account_key: ${{ secrets.GCLOUD_AUTH }}
 
-      # Create instance template from container image
       - name: Create instance template
         run: |
-          gcloud compute instance-templates create-with-container "zebrad-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
-          --container-image "gcr.io/${{ env.PROJECT_ID}}/${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:${{ env.GITHUB_SHA_SHORT }}" \
-          --create-disk name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }},auto-delete=yes,size=100GB,type=pd-balanced \
+          gcloud compute instance-templates create-with-container zebrad-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} \
+          --container-image ${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }} \
+          --create-disk name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }},auto-delete=yes,size=100GB,type=pd-ssd \
           --container-mount-disk mount-path="/zebrad-cache",name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }} \
           --machine-type ${{ env.MACHINE_TYPE }} \
-          --service-account ${{ env.DEPLOY_SA }} \
-          --scopes cloud-platform \
-          --tags zebrad \
+          --tags zebrad
 
       # Check if our destination instance group exists already
       - name: Check if instance group exists
@@ -100,3 +140,36 @@ jobs:
           "zebrad-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}" \
           --version template="zebrad-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
           --region "${{ env.REGION }}"
+
+  deploy-instance:
+    name: Deploy mainnet nodes
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    if: github.event_name == 'workflow_dispatch'
+
+    steps:
+      - uses: actions/checkout@v2.4.0
+        with:
+          persist-credentials: false
+
+      - name: Inject slug/short variables
+        uses: rlespinasse/github-slug-action@v4
+
+      # Setup gcloud CLI
+      - name: Set up gcloud SDK environment
+        uses: google-github-actions/setup-gcloud@v0.4.0
+        with:
+          project_id: ${{ env.PROJECT_ID }}
+          service_account_key: ${{ secrets.GCLOUD_AUTH }}
+
+      # Create instance template from container image
+      - name: Manual deploy of a single instance running zebrad
+        run: |
+          gcloud compute instances create-with-container "zebrad-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
+          --container-image ${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }} \
+          --create-disk name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }},auto-delete=yes,size=100GB,type=pd-ssd \
+          --container-mount-disk mount-path='/zebrad-cache',name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }} \
+          --machine-type ${{ env.MACHINE_TYPE }} \
+          --zone ${{ env.ZONE }} \
+          --tags zebrad
diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml
deleted file mode 100644
index 47087393c09..00000000000
--- a/.github/workflows/manual-deploy.yml
+++ /dev/null
@@ -1,53 +0,0 @@
-name: Manual Deploy
-
-on:
-  workflow_dispatch:
-    inputs:
-      network:
-        default: 'Mainnet'
-      checkpoint_sync:
-        default: true
-
-env:
-  PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
-  ZONE: us-central1-a
-  MACHINE_TYPE: n2-standard-4
-  DEPLOY_SA: cos-vm@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com
-
-jobs:
-  deploy:
-    name: Deploy one zebrad node
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2.4.0
-      with:
-        persist-credentials: false
-
-    - name: Inject slug/short variables
-      uses: rlespinasse/github-slug-action@v4
-
-    - name: Set up gcloud
-      uses: google-github-actions/setup-gcloud@v0.4.0
-      with:
-        project_id: ${{ env.PROJECT_ID }}
-        service_account_key: ${{ secrets.GCLOUD_AUTH }}
-
-    # Build and push image to Google Container Registry
-    - name: Build
-      # Tagging w/ the commit SHA blocks the :latest tag on GCR
-      run: |
-        gcloud builds submit \
-          --config cloudbuild.yaml \
-          --substitutions SHORT_SHA="${{ env.GITHUB_SHA_SHORT }}",BRANCH_NAME="${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}",_CHECKPOINT_SYNC="${{ github.event.inputs.checkpoint_sync }}",_NETWORK="${{ github.event.inputs.network }}"
-
-    # Creates Compute Engine virtual machine instance w/ zebrad container and disks
-    - name: Create instance running zebrad container image
-      run: |
-        gcloud compute instances create-with-container "zebrad-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
-        --container-image "gcr.io/${{ env.PROJECT_ID }}/${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:${{ env.GITHUB_SHA_SHORT }}" \
-        --container-mount-disk mount-path='/zebrad-cache',name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }} \
-        --create-disk name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }},auto-delete=yes,size=100GB,type=pd-balanced \
-        --machine-type ${{ env.MACHINE_TYPE }} \
-        --service-account ${{ env.DEPLOY_SA }} \
-        --tags zebrad \
-        --zone ${{ env.ZONE }}
diff --git a/cloudbuild.yaml b/cloudbuild.yaml
deleted file mode 100644
index f09a10e0292..00000000000
--- a/cloudbuild.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-steps:
-- name: 'gcr.io/cloud-builders/docker'
-  args: ['build',
-        '--build-arg',
-         'SHORT_SHA=$SHORT_SHA',
-         '--build-arg',
-         'checkpoint_sync=${_CHECKPOINT_SYNC}',
-         '--build-arg',
-         'network=${_NETWORK}',
-         '--target',
-         'zebrad-release',
-         '-t',
-         'gcr.io/$PROJECT_ID/zcashfoundation/zebra/$BRANCH_NAME:$SHORT_SHA',
-         '-f',
-         'docker/Dockerfile.build',
-         '.']
-
-images:
-- 'gcr.io/$PROJECT_ID/zcashfoundation/zebra/$BRANCH_NAME:$SHORT_SHA'
-
-options:
-    machineType: 'N1_HIGHCPU_32'
-    substitution_option: 'ALLOW_LOOSE'
-
-timeout: 3600s # 60 mins
diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index 31e1d1c04a2..7fb37765c55 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -1,9 +1,9 @@
-# Builder image
-FROM rust:buster as builder
+FROM rust:1.57-buster as builder
 
 RUN apt-get update && \
-	apt-get install -y --no-install-recommends \
-	make cmake g++ gcc llvm libclang-dev clang ca-certificates
+    apt-get install -y --no-install-recommends \
+    make cmake g++ gcc llvm libclang-dev clang ca-certificates && \
+    rm -Rf /var/lib/apt/lists/* /tmp/*
 
 RUN mkdir /zebra
 WORKDIR /zebra
@@ -11,37 +11,54 @@ WORKDIR /zebra
 ARG SHORT_SHA
 ENV SHORT_SHA $SHORT_SHA
 
-ENV RUST_BACKTRACE full
+ARG RUST_BACKTRACE
+ENV RUST_BACKTRACE ${RUST_BACKTRACE:-full}
+
+# Skip test on debian based OS by default as compilation fails
+ARG ZEBRA_SKIP_NETWORK_TESTS
+ENV ZEBRA_SKIP_NETWORK_TESTS ${ZEBRA_SKIP_NETWORK_TESTS:-1}
+
+# Optimize builds. In particular, regenerate-stateful-test-disks.yml was reaching the
+# GitHub Actions time limit (6 hours), so we needed to make it faster.
+ENV RUSTFLAGS -O
+
 ENV CARGO_HOME /zebra/.cargo/
 
 RUN rustc -V; cargo -V; rustup -V
 
 COPY . .
 
-RUN cd zebrad/; cargo build --release --features enable-sentry
+# Build and then compile tests
+# Compile, but don't run tests (--no-run). Add verbosity and colors
+RUN cargo build --release --features enable-sentry
+RUN cargo test --workspace --no-run
 
 # Runner image
 FROM debian:buster-slim AS zebrad-release
 
 RUN apt-get update && \
-	apt-get install -y --no-install-recommends \
-	ca-certificates
+    apt-get install -y --no-install-recommends \
+    ca-certificates
 
 COPY --from=builder /zebra/target/release/zebrad /
 
 ARG CHECKPOINT_SYNC=true
 ARG NETWORK=Mainnet
 
-RUN printf "[consensus]\n" >> /zebrad.toml
-RUN printf "checkpoint_sync = ${CHECKPOINT_SYNC}\n" >> /zebrad.toml
-RUN printf "[metrics]\n" >> /zebrad.toml
-RUN printf "endpoint_addr = '0.0.0.0:9999'\n" >> /zebrad.toml
-RUN printf "[network]\n" >> /zebrad.toml
-RUN printf "network = '${NETWORK}'\n" >> /zebrad.toml
-RUN printf "[state]\n" >> /zebrad.toml
-RUN printf "cache_dir = '/zebrad-cache'\n" >> /zebrad.toml
-RUN printf "[tracing]\n" >> /zebrad.toml
-RUN printf "endpoint_addr = '0.0.0.0:3000'\n" >> /zebrad.toml
+RUN set -ex; \
+  { \
+    echo "[consensus]"; \
+    echo "checkpoint_sync = ${CHECKPOINT_SYNC}"; \
+    echo "[metrics]"; \
+    echo "endpoint_addr = '0.0.0.0:9999'"; \
+    echo "[network]"; \
+    echo "network = '${NETWORK}'"; \
+    echo "[state]"; \
+    echo "cache_dir = '/zebrad-cache'"; \
+    echo "[tracing]"; \
+    echo "endpoint_addr = '0.0.0.0:3000'"; \
+  } > "/zebrad.toml"
+
 RUN cat /zebrad.toml
 
 # Pre-download Zcash Sprout and Sapling parameters
@@ -49,8 +66,11 @@ RUN /zebrad download
 
 EXPOSE 3000 8233 18233
 
-ENV RUST_LOG debug
-ENV RUST_BACKTRACE full
-ENV SENTRY_DSN https://94059ee72a44420286310990b7c614b5@o485484.ingest.sentry.io/5540918
+ARG RUST_LOG
+ENV RUST_LOG ${RUST_LOG:-debug}
+ARG RUST_BACKTRACE
+ENV RUST_BACKTRACE ${RUST_BACKTRACE:-full}
+ARG SENTRY_DSN
+ENV SENTRY_DSN ${SENTRY_DSN}
 
 CMD [ "/zebrad", "-c", "/zebrad.toml", "start" ]

From 9fd3c228e0bb2c396f8a2cf50f0e994cd3b38cd0 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Thu, 27 Jan 2022 23:38:33 -0400
Subject: [PATCH 003/124] refactor (cd): use newer google auth action

---
 .github/workflows/cd.yml                      | 22 ++++++++++++++-----
 .../regenerate-stateful-test-disks.yml        | 12 +++++++---
 .github/workflows/test.yml                    | 10 +++++++--
 .github/workflows/zcashd-manual-deploy.yml    |  7 +++++-
 4 files changed, 39 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index 42f6a6e163a..3acc4f0c561 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -34,11 +34,11 @@ jobs:
       uses: rlespinasse/github-slug-action@v4
 
     # Setup gcloud CLI
-    - name: Set up gcloud SDK environment
-      uses: google-github-actions/setup-gcloud@v0.4.0
+    - name: Authenticate to Google Cloud
+      id: auth
+      uses: google-github-actions/auth@v0.5.0
       with:
-        project_id: ${{ env.PROJECT_ID }}
-        service_account_key: ${{ secrets.GCLOUD_AUTH }}
+        credentials_json: ${{ secrets.GCLOUD_AUTH }}
 
     # Setup Docker Buildx to allow use of docker cache layers from GH
     - name: Set up Docker Buildx
@@ -97,11 +97,16 @@ jobs:
         uses: rlespinasse/github-slug-action@v4
 
       # Setup gcloud CLI
+      - name: Authenticate to Google Cloud
+        id: auth
+        uses: google-github-actions/auth@v0.5.0
+        with:
+          credentials_json: ${{ secrets.GCLOUD_AUTH }}
+
       - name: Set up gcloud SDK environment
         uses: google-github-actions/setup-gcloud@v0.4.0
         with:
           project_id: ${{ env.PROJECT_ID }}
-          service_account_key: ${{ secrets.GCLOUD_AUTH }}
 
       - name: Create instance template
         run: |
@@ -156,11 +161,16 @@ jobs:
         uses: rlespinasse/github-slug-action@v4
 
       # Setup gcloud CLI
+      - name: Authenticate to Google Cloud
+        id: auth
+        uses: google-github-actions/auth@v0.5.0
+        with:
+          credentials_json: ${{ secrets.GCLOUD_AUTH }}
+
       - name: Set up gcloud SDK environment
         uses: google-github-actions/setup-gcloud@v0.4.0
         with:
           project_id: ${{ env.PROJECT_ID }}
-          service_account_key: ${{ secrets.GCLOUD_AUTH }}
 
       # Create instance template from container image
       - name: Manual deploy of a single instance running zebrad
diff --git a/.github/workflows/regenerate-stateful-test-disks.yml b/.github/workflows/regenerate-stateful-test-disks.yml
index e81bcdaae9e..90def9df418 100644
--- a/.github/workflows/regenerate-stateful-test-disks.yml
+++ b/.github/workflows/regenerate-stateful-test-disks.yml
@@ -25,11 +25,17 @@ jobs:
       - name: Inject slug/short variables
         uses: rlespinasse/github-slug-action@v4
 
-      - name: Set up gcloud
+      # Setup gcloud CLI
+      - name: Authenticate to Google Cloud
+        id: auth
+        uses: google-github-actions/auth@v0.5.0
+        with:
+          credentials_json: ${{ secrets.GCLOUD_AUTH }}
+
+      - name: Set up gcloud SDK environment
         uses: google-github-actions/setup-gcloud@v0.4.0
         with:
           project_id: ${{ env.PROJECT_ID }}
-          service_account_key: ${{ secrets.GCLOUD_AUTH }}
 
       # Creates Compute Engine virtual machine instance w/ disks
       - name: Create instance
@@ -40,7 +46,7 @@ jobs:
           --container-image rust:buster \
           --container-mount-disk mount-path='/${{ github.event.inputs.network }}',name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy" \
           --container-restart-policy never \
-          --create-disk name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy",size=100GB,type=pd-balanced \
+          --create-disk name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy",size=100GB,type=pd-ssd \
           --machine-type ${{ env.MACHINE_TYPE }} \
           --service-account ${{ env.DEPLOY_SA }} \
           --scopes cloud-platform \
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 39dca79928f..7bc937f7b21 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -32,11 +32,17 @@ jobs:
       - name: Inject slug/short variables
         uses: rlespinasse/github-slug-action@v4
 
-      - name: Set up gcloud
+      # Setup gcloud CLI
+      - name: Authenticate to Google Cloud
+        id: auth
+        uses: google-github-actions/auth@v0.5.0
+        with:
+          credentials_json: ${{ secrets.GCLOUD_AUTH }}
+
+      - name: Set up gcloud SDK environment
         uses: google-github-actions/setup-gcloud@v0.4.0
         with:
           project_id: ${{ env.PROJECT_ID }}
-          service_account_key: ${{ secrets.GCLOUD_AUTH }}
 
       # Creates Compute Engine virtual machine instance w/ disks
       - name: Create instance
diff --git a/.github/workflows/zcashd-manual-deploy.yml b/.github/workflows/zcashd-manual-deploy.yml
index 73e1973c948..1697386473f 100644
--- a/.github/workflows/zcashd-manual-deploy.yml
+++ b/.github/workflows/zcashd-manual-deploy.yml
@@ -29,11 +29,16 @@ jobs:
         uses: rlespinasse/github-slug-action@v4
 
       # Setup gcloud CLI
+      - name: Authenticate to Google Cloud
+        id: auth
+        uses: google-github-actions/auth@v0.5.0
+        with:
+          credentials_json: ${{ secrets.GCLOUD_AUTH }}
+
       - name: Set up gcloud SDK environment
         uses: google-github-actions/setup-gcloud@v0.4.0
         with:
           project_id: ${{ env.PROJECT_ID }}
-          service_account_key: ${{ secrets.GCLOUD_AUTH }}
 
       # Create instance template from container image
       - name: Create instance template

From af417261248654335f1c33314d5d1eb5f4a282e8 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Thu, 27 Jan 2022 23:46:16 -0400
Subject: [PATCH 004/124] fix (cd): use newer secret as gcp credential

---
 .github/workflows/cd.yml                             | 8 ++++----
 .github/workflows/regenerate-stateful-test-disks.yml | 2 +-
 .github/workflows/test.yml                           | 2 +-
 .github/workflows/zcashd-manual-deploy.yml           | 2 +-
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index 3acc4f0c561..1d7c7a13981 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -38,7 +38,7 @@ jobs:
       id: auth
       uses: google-github-actions/auth@v0.5.0
       with:
-        credentials_json: ${{ secrets.GCLOUD_AUTH }}
+        credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
 
     # Setup Docker Buildx to allow use of docker cache layers from GH
     - name: Set up Docker Buildx
@@ -58,7 +58,7 @@ jobs:
       with:
         registry: us-docker.pkg.dev
         username: _json_key
-        password: ${{ secrets.GCLOUD_AUTH }}
+        password: ${{ secrets.GOOGLE_CREDENTIALS }}
 
     # Build and push image to Google Artifact Registry
     - name: Build & push
@@ -101,7 +101,7 @@ jobs:
         id: auth
         uses: google-github-actions/auth@v0.5.0
         with:
-          credentials_json: ${{ secrets.GCLOUD_AUTH }}
+          credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
 
       - name: Set up gcloud SDK environment
         uses: google-github-actions/setup-gcloud@v0.4.0
@@ -165,7 +165,7 @@ jobs:
         id: auth
         uses: google-github-actions/auth@v0.5.0
         with:
-          credentials_json: ${{ secrets.GCLOUD_AUTH }}
+          credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
 
       - name: Set up gcloud SDK environment
         uses: google-github-actions/setup-gcloud@v0.4.0
diff --git a/.github/workflows/regenerate-stateful-test-disks.yml b/.github/workflows/regenerate-stateful-test-disks.yml
index 90def9df418..01652218ab7 100644
--- a/.github/workflows/regenerate-stateful-test-disks.yml
+++ b/.github/workflows/regenerate-stateful-test-disks.yml
@@ -30,7 +30,7 @@ jobs:
         id: auth
         uses: google-github-actions/auth@v0.5.0
         with:
-          credentials_json: ${{ secrets.GCLOUD_AUTH }}
+          credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
 
       - name: Set up gcloud SDK environment
         uses: google-github-actions/setup-gcloud@v0.4.0
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 7bc937f7b21..21610a2c397 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -37,7 +37,7 @@ jobs:
         id: auth
         uses: google-github-actions/auth@v0.5.0
         with:
-          credentials_json: ${{ secrets.GCLOUD_AUTH }}
+          credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
 
       - name: Set up gcloud SDK environment
         uses: google-github-actions/setup-gcloud@v0.4.0
diff --git a/.github/workflows/zcashd-manual-deploy.yml b/.github/workflows/zcashd-manual-deploy.yml
index 1697386473f..0586bad796c 100644
--- a/.github/workflows/zcashd-manual-deploy.yml
+++ b/.github/workflows/zcashd-manual-deploy.yml
@@ -33,7 +33,7 @@ jobs:
         id: auth
         uses: google-github-actions/auth@v0.5.0
         with:
-          credentials_json: ${{ secrets.GCLOUD_AUTH }}
+          credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
 
       - name: Set up gcloud SDK environment
         uses: google-github-actions/setup-gcloud@v0.4.0

From fb8b91f50c4769cdb90815842b74aa51a7fbccf9 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Fri, 28 Jan 2022 00:25:28 -0400
Subject: [PATCH 005/124] fix (docker): do not create extra directories

---
 docker/Dockerfile.build | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index 7fb37765c55..873c0b9097a 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -5,9 +5,6 @@ RUN apt-get update && \
     make cmake g++ gcc llvm libclang-dev clang ca-certificates && \
     rm -Rf /var/lib/apt/lists/* /tmp/*
 
-RUN mkdir /zebra
-WORKDIR /zebra
-
 ARG SHORT_SHA
 ENV SHORT_SHA $SHORT_SHA
 
@@ -28,6 +25,8 @@ RUN rustc -V; cargo -V; rustup -V
 
 COPY . .
 
+WORKDIR /zebrad
+
 # Build and then compile tests
 # Compile, but don't run tests (--no-run). Add verbosity and colors
 RUN cargo build --release --features enable-sentry

From f55cf9ba02b72cdd857ac19678e0421e710956f7 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Fri, 28 Jan 2022 00:26:31 -0400
Subject: [PATCH 006/124] fix (docker): ignore .github for caching purposes

---
 docker/.dockerignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker/.dockerignore b/docker/.dockerignore
index 3645101174c..fae64da0fe6 100644
--- a/docker/.dockerignore
+++ b/docker/.dockerignore
@@ -2,4 +2,5 @@ target
 Dockerfile
 .dockerignore
 .git
+.github
 .gitignore

From 850f5a41009bab6964a0fd888428106f402f17df Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Fri, 28 Jan 2022 01:04:17 -0400
Subject: [PATCH 007/124] fix (docker): use latest rust

---
 docker/Dockerfile.build | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index 873c0b9097a..13eeeb9b9a8 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -1,10 +1,13 @@
-FROM rust:1.57-buster as builder
+FROM rust:buster as builder
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     make cmake g++ gcc llvm libclang-dev clang ca-certificates && \
     rm -Rf /var/lib/apt/lists/* /tmp/*
 
+RUN mkdir /zebra
+WORKDIR /zebra
+
 ARG SHORT_SHA
 ENV SHORT_SHA $SHORT_SHA
 
@@ -25,11 +28,9 @@ RUN rustc -V; cargo -V; rustup -V
 
 COPY . .
 
-WORKDIR /zebrad
-
 # Build and then compile tests
 # Compile, but don't run tests (--no-run). Add verbosity and colors
-RUN cargo build --release --features enable-sentry
+RUN cd zebrad/; cargo build --release --features enable-sentry
 RUN cargo test --workspace --no-run
 
 # Runner image

From 4bf58f202b06132ca2b8cf0a58a62ab81bc84669 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Fri, 28 Jan 2022 07:09:39 -0400
Subject: [PATCH 008/124] fix (cd): bump build timeout

---
 .github/workflows/cd.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index 1d7c7a13981..d55bb3ff387 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -22,7 +22,7 @@ env:
 jobs:
   build:
     name: Build images
-    timeout-minutes: 60
+    timeout-minutes: 115
     runs-on: ubuntu-latest
 
     steps:

From 9cf620a8615c351f15a9d6c2dbd9ca9975960328 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Fri, 28 Jan 2022 13:20:13 -0400
Subject: [PATCH 009/124] fix: use a better name for manual deployment

---
 .github/workflows/cd.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index d55bb3ff387..246bd2424b1 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -146,7 +146,7 @@ jobs:
           --region "${{ env.REGION }}"
 
   deploy-instance:
-    name: Deploy mainnet nodes
+    name: Deploy single instance
     needs: build
     runs-on: ubuntu-latest
     timeout-minutes: 30

From ea2b5bd0692267b9709a35511632da4a480ae234 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Sun, 30 Jan 2022 09:14:11 -0400
Subject: [PATCH 010/124] refactor (docker): use standard directories for
 executable

---
 docker/Dockerfile.build | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index 13eeeb9b9a8..5bb4566ceb9 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -1,13 +1,11 @@
 FROM rust:buster as builder
+WORKDIR /app
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     make cmake g++ gcc llvm libclang-dev clang ca-certificates && \
     rm -Rf /var/lib/apt/lists/* /tmp/*
 
-RUN mkdir /zebra
-WORKDIR /zebra
-
 ARG SHORT_SHA
 ENV SHORT_SHA $SHORT_SHA
 
@@ -22,7 +20,7 @@ ENV ZEBRA_SKIP_NETWORK_TESTS ${ZEBRA_SKIP_NETWORK_TESTS:-1}
 # GitHub Actions time limit (6 hours), so we needed to make it faster.
 ENV RUSTFLAGS -O
 
-ENV CARGO_HOME /zebra/.cargo/
+ENV CARGO_HOME /app/.cargo/
 
 RUN rustc -V; cargo -V; rustup -V
 
@@ -40,7 +38,7 @@ RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     ca-certificates
 
-COPY --from=builder /zebra/target/release/zebrad /
+COPY --from=builder /app/target/release/zebrad /usr/local/bin
 
 ARG CHECKPOINT_SYNC=true
 ARG NETWORK=Mainnet
@@ -57,12 +55,12 @@ RUN set -ex; \
     echo "cache_dir = '/zebrad-cache'"; \
     echo "[tracing]"; \
     echo "endpoint_addr = '0.0.0.0:3000'"; \
-  } > "/zebrad.toml"
+  } > "zebrad.toml"
 
-RUN cat /zebrad.toml
+RUN cat zebrad.toml
 
 # Pre-download Zcash Sprout and Sapling parameters
-RUN /zebrad download
+RUN /usr/local/bin/zebrad download
 
 EXPOSE 3000 8233 18233
 
@@ -73,4 +71,4 @@ ENV RUST_BACKTRACE ${RUST_BACKTRACE:-full}
 ARG SENTRY_DSN
 ENV SENTRY_DSN ${SENTRY_DSN}
 
-CMD [ "/zebrad", "-c", "/zebrad.toml", "start" ]
+CMD [ "/usr/local/bin/zebrad", "-c", "zebrad.toml", "start" ]

From 823508555eb1fb0c7aaf8f80547b03818a872846 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Sun, 30 Jan 2022 10:11:13 -0400
Subject: [PATCH 011/124] fix (cd): most systems expect a "latest" tag

Caching from the latest image is one of the main reasons to add this extra tag. Before this commit, the inline cache was not being used.
---
 .github/workflows/cd.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index 246bd2424b1..1a1fabd08dd 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -68,7 +68,9 @@ jobs:
         target: zebrad-release
         context: .
         file: ./docker/Dockerfile.build
-        tags: ${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:${{ env.GITHUB_SHA_SHORT }}
+        tags: |
+          ${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:latest
+          ${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:${{ env.GITHUB_SHA_SHORT }}
         build-args: |
           NETWORK=${{ github.event.inputs.network || env.NETWORK }}
           SHORT_SHA=${{ env.GITHUB_SHA_SHORT }}

From 358fbd378d2b44cc77f509da88a3b47020315cd1 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Sun, 30 Jan 2022 10:53:15 -0400
Subject: [PATCH 012/124] fix (cd): push the build image and the cache
 separately

The inline cache exporter only supports `min` cache mode. To enable `max` cache mode, push the image and the cache separately by using the registry cache exporter.

This also allows for smaller release images.
---
 .github/workflows/cd.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index 1a1fabd08dd..14433247052 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -80,8 +80,8 @@ jobs:
           RUST_LOG=debug
           SENTRY_DSN=${{ secrets.SENTRY_ENDPOINT }}
         push: true
-        cache-from: type=registry,ref=${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}
-        cache-to: type=inline
+        cache-from: type=registry,ref=${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:buildcache
+        cache-to: type=registry,ref=${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:buildcache,mode=max
 
   deploy-nodes:
     name: Deploy mainnet nodes

From 6b5eeecd71a9a8a50cbebe496aedd20470dd35cc Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Sun, 30 Jan 2022 11:23:05 -0400
Subject: [PATCH 013/124] fix (cd): remove unused GHA cache

We're leveraging the registry to cache the actions, instead of using the 10GB limits from Github Actions cache storage
---
 .github/workflows/cd.yml | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index 14433247052..2bae0a3345f 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -46,14 +46,7 @@ jobs:
       with:
         driver-opts: network=host
 
-    - name: Cache Docker layers
-      uses: actions/cache@v2
-      with:
-        path: |
-          /tmp/.buildx-cache
-        key: ${{ runner.os }}-buildx-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}
-
-    - name: Login to GAR
+    - name: Login to Google Artifact Registry
       uses: docker/login-action@v1.12.0
       with:
         registry: us-docker.pkg.dev

From 0702d79b2ff00c84fa2e9cc52bb6376469da0abd Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Sun, 30 Jan 2022 12:20:59 -0400
Subject: [PATCH 014/124] refactor (cd): use cargo-chef for caching rust deps

---
 .github/workflows/cd.yml |  2 +-
 docker/Dockerfile.build  | 34 +++++++++++++++++++++++-----------
 2 files changed, 24 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index 2bae0a3345f..6ad10732162 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -58,7 +58,7 @@ jobs:
       id: docker_build
       uses: docker/build-push-action@v2.8.0
       with:
-        target: zebrad-release
+        target: runtime
         context: .
         file: ./docker/Dockerfile.build
         tags: |
diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index 5bb4566ceb9..00e234738ad 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -1,38 +1,50 @@
-FROM rust:buster as builder
+# This steps implement cargo-chef for docker layer caching
+# We are using four stages:
+# - chef: installs cargo-chef
+# - planner: computes the recipe file
+# - builder: caches our dependencies and builds the binary
+# - runtime: is our runtime environment
+FROM rust:1.58.1-buster as chef
+RUN cargo install cargo-chef
 WORKDIR /app
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-    make cmake g++ gcc llvm libclang-dev clang ca-certificates && \
-    rm -Rf /var/lib/apt/lists/* /tmp/*
+FROM chef AS planner
+COPY . .
+RUN cargo chef prepare  --recipe-path recipe.json
+
+FROM chef AS builder
+COPY --from=planner /app/recipe.json recipe.json
+# Build dependencies - this is the caching Docker layer!
+RUN cargo chef cook --release --recipe-path recipe.json
 
+# Build zebra
 ARG SHORT_SHA
 ENV SHORT_SHA $SHORT_SHA
-
 ARG RUST_BACKTRACE
 ENV RUST_BACKTRACE ${RUST_BACKTRACE:-full}
-
 # Skip test on debian based OS by default as compilation fails
 ARG ZEBRA_SKIP_NETWORK_TESTS
 ENV ZEBRA_SKIP_NETWORK_TESTS ${ZEBRA_SKIP_NETWORK_TESTS:-1}
-
 # Optimize builds. In particular, regenerate-stateful-test-disks.yml was reaching the
 # GitHub Actions time limit (6 hours), so we needed to make it faster.
 ENV RUSTFLAGS -O
-
 ENV CARGO_HOME /app/.cargo/
 
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    make cmake g++ gcc llvm libclang-dev clang ca-certificates && \
+    rm -Rf /var/lib/apt/lists/* /tmp/*
+
 RUN rustc -V; cargo -V; rustup -V
 
 COPY . .
-
 # Build and then compile tests
 # Compile, but don't run tests (--no-run). Add verbosity and colors
 RUN cd zebrad/; cargo build --release --features enable-sentry
 RUN cargo test --workspace --no-run
 
 # Runner image
-FROM debian:buster-slim AS zebrad-release
+FROM debian:buster-slim AS runtime
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \

From 08b9ba95bd92c4f7cfbbda03ab4ee148c875c2c7 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Sun, 30 Jan 2022 12:37:33 -0400
Subject: [PATCH 015/124] fix: move build system deps before cargo cheg cook

---
 docker/Dockerfile.build | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index 00e234738ad..d18487f343a 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -14,6 +14,10 @@ RUN cargo chef prepare  --recipe-path recipe.json
 
 FROM chef AS builder
 COPY --from=planner /app/recipe.json recipe.json
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    make cmake g++ gcc llvm libclang-dev clang ca-certificates && \
+    rm -Rf /var/lib/apt/lists/* /tmp/*
 # Build dependencies - this is the caching Docker layer!
 RUN cargo chef cook --release --recipe-path recipe.json
 
@@ -30,11 +34,6 @@ ENV ZEBRA_SKIP_NETWORK_TESTS ${ZEBRA_SKIP_NETWORK_TESTS:-1}
 ENV RUSTFLAGS -O
 ENV CARGO_HOME /app/.cargo/
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-    make cmake g++ gcc llvm libclang-dev clang ca-certificates && \
-    rm -Rf /var/lib/apt/lists/* /tmp/*
-
 RUN rustc -V; cargo -V; rustup -V
 
 COPY . .

From 212231990b3b5d4cd9ba36b073c91c0d8acca430 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Sun, 30 Jan 2022 19:20:54 -0400
Subject: [PATCH 016/124] fix (release): use newer debian to reduce
 vulnerabilities

---
 docker/Dockerfile.build | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index d18487f343a..5f6722db0ea 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -4,7 +4,7 @@
 # - planner: computes the recipe file
 # - builder: caches our dependencies and builds the binary
 # - runtime: is our runtime environment
-FROM rust:1.58.1-buster as chef
+FROM rust:1.58.1-bullseye as chef
 RUN cargo install cargo-chef
 WORKDIR /app
 
@@ -43,7 +43,7 @@ RUN cd zebrad/; cargo build --release --features enable-sentry
 RUN cargo test --workspace --no-run
 
 # Runner image
-FROM debian:buster-slim AS runtime
+FROM debian:bullseye-slim AS runtime
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \

From d7dc0623cc710dcea441e25e90003d57311753ae Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Sun, 30 Jan 2022 22:10:08 -0400
Subject: [PATCH 017/124] fix (cd): use same zone, region and service accounts

---
 .github/workflows/cd.yml                             | 2 ++
 .github/workflows/regenerate-stateful-test-disks.yml | 5 +++--
 .github/workflows/test.yml                           | 5 +++--
 .github/workflows/zcashd-manual-deploy.yml           | 3 ++-
 4 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index 6ad10732162..0f6064b421b 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -110,6 +110,8 @@ jobs:
           --create-disk name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }},auto-delete=yes,size=100GB,type=pd-ssd \
           --container-mount-disk mount-path="/zebrad-cache",name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }} \
           --machine-type ${{ env.MACHINE_TYPE }} \
+          --service-account ${{ env.DEPLOY_SA }} \
+          --scopes cloud-platform \
           --tags zebrad
 
       # Check if our destination instance group exists already
diff --git a/.github/workflows/regenerate-stateful-test-disks.yml b/.github/workflows/regenerate-stateful-test-disks.yml
index 01652218ab7..bff8d622452 100644
--- a/.github/workflows/regenerate-stateful-test-disks.yml
+++ b/.github/workflows/regenerate-stateful-test-disks.yml
@@ -8,8 +8,9 @@ on:
 
 env:
   PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
-  ZONE: europe-west1-b
-  MACHINE_TYPE: n2-standard-4
+  REGION: us-central1
+  ZONE: us-central1-a
+  MACHINE_TYPE: c2d-standard-4
   DEPLOY_SA: cos-vm@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com
 
 jobs:
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 21610a2c397..bdd0b3d7dd4 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -15,8 +15,9 @@ on:
 
 env:
   PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
-  ZONE: europe-west1-b
-  MACHINE_TYPE: n2-standard-8
+  REGION: us-central1
+  ZONE: us-central1-a
+  MACHINE_TYPE: c2d-standard-8
   DEPLOY_SA: cos-vm@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com
 
 jobs:
diff --git a/.github/workflows/zcashd-manual-deploy.yml b/.github/workflows/zcashd-manual-deploy.yml
index 0586bad796c..67555d824a5 100644
--- a/.github/workflows/zcashd-manual-deploy.yml
+++ b/.github/workflows/zcashd-manual-deploy.yml
@@ -11,7 +11,8 @@ on:
 env:
   PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
   REGION: us-central1
-  MACHINE_TYPE: n2-standard-4
+  ZONE: us-central1-a
+  MACHINE_TYPE: c2d-standard-4
   DEPLOY_SA: cos-vm@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com
 
 jobs:

From 0e66323b6ee1cf62277a9acb32558d03c65220c1 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Sun, 30 Jan 2022 22:22:53 -0400
Subject: [PATCH 018/124] fix (cd): use same disk size and type for all
 deployments

---
 .github/workflows/cd.yml                   | 2 ++
 .github/workflows/zcashd-manual-deploy.yml | 1 +
 2 files changed, 3 insertions(+)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index 0f6064b421b..a6c6f3bba26 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -173,6 +173,8 @@ jobs:
       - name: Manual deploy of a single instance running zebrad
         run: |
           gcloud compute instances create-with-container "zebrad-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
+          --boot-disk-size 100GB \
+          --boot-disk-type=pd-ssd \
           --container-image ${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }} \
           --create-disk name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }},auto-delete=yes,size=100GB,type=pd-ssd \
           --container-mount-disk mount-path='/zebrad-cache',name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }} \
diff --git a/.github/workflows/zcashd-manual-deploy.yml b/.github/workflows/zcashd-manual-deploy.yml
index 67555d824a5..f71f0246ee9 100644
--- a/.github/workflows/zcashd-manual-deploy.yml
+++ b/.github/workflows/zcashd-manual-deploy.yml
@@ -46,6 +46,7 @@ jobs:
         run: |
           gcloud compute instance-templates create-with-container "zcashd-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
           --boot-disk-size 100GB \
+          --boot-disk-type=pd-ssd \
           --container-image "electriccoinco/zcashd" \
           --container-env ZCASHD_NETWORK="${{ github.event.inputs.network }}" \
           --machine-type ${{ env.MACHINE_TYPE }} \

From c4a6e889beb27be4eee1d3e4068f01187e992229 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Sun, 30 Jan 2022 22:27:44 -0400
Subject: [PATCH 019/124] refactor (cd): activate interactive shells

Use interactive shells for manual and test deployments. This allow greater flexibility if troubleshooting is needed inside the machines
---
 .github/workflows/cd.yml                   | 3 +++
 .github/workflows/test.yml                 | 2 ++
 .github/workflows/zcashd-manual-deploy.yml | 2 ++
 3 files changed, 7 insertions(+)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index a6c6f3bba26..74c81570bef 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -106,6 +106,7 @@ jobs:
       - name: Create instance template
         run: |
           gcloud compute instance-templates create-with-container zebrad-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} \
+          --boot-disk-type=pd-ssd \
           --container-image ${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }} \
           --create-disk name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }},auto-delete=yes,size=100GB,type=pd-ssd \
           --container-mount-disk mount-path="/zebrad-cache",name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }} \
@@ -175,6 +176,8 @@ jobs:
           gcloud compute instances create-with-container "zebrad-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
           --boot-disk-size 100GB \
           --boot-disk-type=pd-ssd \
+          --container-stdin \
+          --container-tty \
           --container-image ${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }} \
           --create-disk name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }},auto-delete=yes,size=100GB,type=pd-ssd \
           --container-mount-disk mount-path='/zebrad-cache',name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }} \
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index bdd0b3d7dd4..7f4040a3e9f 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -51,6 +51,8 @@ jobs:
           gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
           --boot-disk-size 100GB \
           --boot-disk-type pd-ssd \
+          --container-stdin \
+          --container-tty \
           --container-image rust:buster \
           --container-mount-disk mount-path='/mainnet',name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-mainnet-canopy" \
           --container-restart-policy never \
diff --git a/.github/workflows/zcashd-manual-deploy.yml b/.github/workflows/zcashd-manual-deploy.yml
index f71f0246ee9..17b98841b27 100644
--- a/.github/workflows/zcashd-manual-deploy.yml
+++ b/.github/workflows/zcashd-manual-deploy.yml
@@ -47,6 +47,8 @@ jobs:
           gcloud compute instance-templates create-with-container "zcashd-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
           --boot-disk-size 100GB \
           --boot-disk-type=pd-ssd \
+          --container-stdin \
+          --container-tty \
           --container-image "electriccoinco/zcashd" \
           --container-env ZCASHD_NETWORK="${{ github.event.inputs.network }}" \
           --machine-type ${{ env.MACHINE_TYPE }} \

From 104b4f8017d2d490f3c282364ce25174498519f0 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Mon, 31 Jan 2022 00:18:32 -0400
Subject: [PATCH 020/124] refactor (test): use docker artifact from registry

Instead of using a VM to SSH into in to build and test. Build in GHA (to have the logs available), run the workspace tests in GHA, and just run the sync tests in GCP

Use a cintainer VM with zebra's image directly on it, and pass the needed parameters to run the Sync past mandatory checkpoint.
---
 .github/workflows/test.yml | 89 ++++++++++++++++++++++++++++++--------
 1 file changed, 71 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 7f4040a3e9f..7c21a62dc99 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -19,11 +19,12 @@ env:
   ZONE: us-central1-a
   MACHINE_TYPE: c2d-standard-8
   DEPLOY_SA: cos-vm@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com
+  IMAGE_NAME: zebrad-test
 
 jobs:
 
-  test:
-    name: Run all tests
+  build:
+    name: Build
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2.4.0
@@ -40,39 +41,91 @@ jobs:
         with:
           credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
 
-      - name: Set up gcloud SDK environment
-        uses: google-github-actions/setup-gcloud@v0.4.0
+      # Setup Docker Buildx to allow use of docker cache layers from GH
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          driver-opts: network=host
+
+      - name: Login to Google Artifact Registry
+        uses: docker/login-action@v1.12.0
         with:
-          project_id: ${{ env.PROJECT_ID }}
+          registry: us-docker.pkg.dev
+          username: _json_key
+          password: ${{ secrets.GOOGLE_CREDENTIALS }}
+
+      # Build and push image to Google Artifact Registry
+      - name: Build & push
+        id: docker_build
+        uses: docker/build-push-action@v2.8.0
+        with:
+          target: builder
+          context: .
+          file: ./docker/Dockerfile.build
+          tags: |
+            ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:latest
+            ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
+          build-args: |
+            NETWORK=${{ github.event.inputs.network || env.NETWORK }}
+            SHORT_SHA=${{ env.GITHUB_SHA_SHORT }}
+            RUST_BACKTRACE=full
+            ZEBRA_SKIP_NETWORK_TESTS="1"
+            CHECKPOINT_SYNC=${{ github.event.inputs.checkpoint_sync || true }}
+            RUST_LOG=debug
+            SENTRY_DSN=${{ secrets.SENTRY_ENDPOINT }}
+          push: true
+          cache-from: type=registry,ref=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:buildcache
+          cache-to: type=registry,ref=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:buildcache,mode=max
+
+
+    name: Run all tests
+    runs-on: ubuntu-latest
+    steps:
+    needs: build
+      - uses: actions/checkout@v2.4.0
+        with:
+          persist-credentials: false
+
+      - name: Inject slug/short variables
+        uses: rlespinasse/github-slug-action@v4
+
+      - name: Login to Google Artifact Registry
+        uses: docker/login-action@v1.12.0
+        with:
+          registry: us-docker.pkg.dev
+          username: _json_key
+          password: ${{ secrets.GOOGLE_CREDENTIALS }}
+
+      - name: Run workspace tests with Zunstable-options
+        run: |
+          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
+          docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} cargo test --workspace -- -Zunstable-options --include-ignored
+        env:
+          ZEBRA_SKIP_IPV6_TESTS: "1"
 
       # Creates Compute Engine virtual machine instance w/ disks
-      - name: Create instance
+      # TODO: Remove hardcoded value zebrad-cache-1558f3378-mainnet-canopy
+      - name: Sync past mandatory checkpoint
         run: |
           gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
           --boot-disk-size 100GB \
           --boot-disk-type pd-ssd \
           --container-stdin \
           --container-tty \
-          --container-image rust:buster \
+          --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} \
+          --container-command "/bin/bash" \
+          --container-arg="-c" \
+          --container-arg="cargo test --verbose --features test_sync_past_mandatory_checkpoint_mainnet --manifest-path zebrad/Cargo.toml sync_past_mandatory_checkpoint_mainnet" \
           --container-mount-disk mount-path='/mainnet',name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-mainnet-canopy" \
           --container-restart-policy never \
-          --create-disk name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-mainnet-canopy",image=zebrad-cache-1558f3378-mainnet-canopy \
+          --create-disk name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-mainnet-canopy,auto-delete=yes,size=100GB,type=pd-ssd \
+          --container-mount-disk mount-path="/zebrad-cache",name=zebrad-cache-1558f3378-mainnet-canopy \
           --machine-type ${{ env.MACHINE_TYPE }} \
           --service-account ${{ env.DEPLOY_SA }} \
           --scopes cloud-platform \
           --tags zebrad \
           --zone "${{ env.ZONE }}"
 
-      # Build and run test container
-      - name: Run all tests
-        run: |
-          gcloud compute ssh "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --ssh-flag="-o ServerAliveInterval=5" --zone "${{ env.ZONE }}" --command \
-          "git clone -b ${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }} https://github.com/ZcashFoundation/zebra.git &&
-          cd zebra/ &&
-          docker build --build-arg SHORT_SHA=${{ env.GITHUB_SHA_SHORT }} -f docker/Dockerfile.test -t zebrad-test . &&
-          docker run -t -e ZEBRA_SKIP_IPV6_TESTS=1 zebrad-test:latest cargo test --workspace --no-fail-fast -- -Zunstable-options --include-ignored &&
-          docker run -t -e ZEBRA_SKIP_IPV6_TESTS=1 --mount type=bind,source=/mnt/disks/gce-containers-mounts/gce-persistent-disks/zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-mainnet-canopy,target=/zebrad-cache zebrad-test:latest cargo test --verbose --features test_sync_past_mandatory_checkpoint_mainnet --manifest-path zebrad/Cargo.toml sync_past_mandatory_checkpoint_mainnet
-          "
       # Clean up
       - name: Delete test instance
         # Always run even if the earlier step fails

From 7a2bbdd25d67c7f3240914652f4c402f27dd16fd Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Mon, 31 Jan 2022 00:21:59 -0400
Subject: [PATCH 021/124] tmp (cd): bump timeout for building from scratch

---
 .github/workflows/cd.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index 74c81570bef..1ceebbf806f 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -22,7 +22,7 @@ env:
 jobs:
   build:
     name: Build images
-    timeout-minutes: 115
+    timeout-minutes: 180
     runs-on: ubuntu-latest
 
     steps:

From 25d5e36872a964d0e7fc957c3b8adbaddf0d8eaf Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Mon, 31 Jan 2022 00:23:32 -0400
Subject: [PATCH 022/124] tmp (test): bump build time

---
 .github/workflows/test.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 7c21a62dc99..e0fe96f32d2 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -24,7 +24,8 @@ env:
 jobs:
 
   build:
-    name: Build
+    name: Build images
+    timeout-minutes: 180
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2.4.0

From 237ab322d4942d7d6983e4e2909a6abe85080b29 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Mon, 31 Jan 2022 16:00:57 -0400
Subject: [PATCH 023/124] fix (cd, test): bump build time-out to 210 minutes

---
 .github/workflows/cd.yml   | 2 +-
 .github/workflows/test.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index 74c81570bef..82b1412652c 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -22,7 +22,7 @@ env:
 jobs:
   build:
     name: Build images
-    timeout-minutes: 115
+    timeout-minutes: 210
     runs-on: ubuntu-latest
 
     steps:
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index e0fe96f32d2..ea8bc3c15a1 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -25,7 +25,7 @@ jobs:
 
   build:
     name: Build images
-    timeout-minutes: 180
+    timeout-minutes: 210
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2.4.0

From 556c0f84757a94c77f297510436c68dc7b61d0ed Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Mon, 31 Jan 2022 21:21:37 -0400
Subject: [PATCH 024/124] fix (docker): do not build with different settings

Compiling might be slow because different steps are compiling the same code 2-4 times because of the variations
---
 docker/Dockerfile.build | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index 5f6722db0ea..39b08b0e19b 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -19,7 +19,7 @@ RUN apt-get update && \
     make cmake g++ gcc llvm libclang-dev clang ca-certificates && \
     rm -Rf /var/lib/apt/lists/* /tmp/*
 # Build dependencies - this is the caching Docker layer!
-RUN cargo chef cook --release --recipe-path recipe.json
+RUN cargo chef cook --release --features enable-sentry --recipe-path recipe.json
 
 # Build zebra
 ARG SHORT_SHA
@@ -40,7 +40,7 @@ COPY . .
 # Build and then compile tests
 # Compile, but don't run tests (--no-run). Add verbosity and colors
 RUN cd zebrad/; cargo build --release --features enable-sentry
-RUN cargo test --workspace --no-run
+RUN cargo test --release --features enable-sentry --workspace --no-run
 
 # Runner image
 FROM debian:bullseye-slim AS runtime

From 607f4eac52a4555150a8a9c1ba7d17932ecf8ade Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Mon, 31 Jan 2022 21:25:33 -0400
Subject: [PATCH 025/124] revert (docker): do not fix the rust version

---
 docker/Dockerfile.build | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index 39b08b0e19b..33430a6c364 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -4,7 +4,7 @@
 # - planner: computes the recipe file
 # - builder: caches our dependencies and builds the binary
 # - runtime: is our runtime environment
-FROM rust:1.58.1-bullseye as chef
+FROM rust:bullseye as chef
 RUN cargo install cargo-chef
 WORKDIR /app
 
@@ -14,10 +14,15 @@ RUN cargo chef prepare  --recipe-path recipe.json
 
 FROM chef AS builder
 COPY --from=planner /app/recipe.json recipe.json
+
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     make cmake g++ gcc llvm libclang-dev clang ca-certificates && \
     rm -Rf /var/lib/apt/lists/* /tmp/*
+
+# Optimize builds. In particular, regenerate-stateful-test-disks.yml was reaching the
+# GitHub Actions time limit (6 hours), so we needed to make it faster.
+ENV RUSTFLAGS -O
 # Build dependencies - this is the caching Docker layer!
 RUN cargo chef cook --release --features enable-sentry --recipe-path recipe.json
 
@@ -29,9 +34,7 @@ ENV RUST_BACKTRACE ${RUST_BACKTRACE:-full}
 # Skip test on debian based OS by default as compilation fails
 ARG ZEBRA_SKIP_NETWORK_TESTS
 ENV ZEBRA_SKIP_NETWORK_TESTS ${ZEBRA_SKIP_NETWORK_TESTS:-1}
-# Optimize builds. In particular, regenerate-stateful-test-disks.yml was reaching the
-# GitHub Actions time limit (6 hours), so we needed to make it faster.
-ENV RUSTFLAGS -O
+
 ENV CARGO_HOME /app/.cargo/
 
 RUN rustc -V; cargo -V; rustup -V
@@ -39,7 +42,7 @@ RUN rustc -V; cargo -V; rustup -V
 COPY . .
 # Build and then compile tests
 # Compile, but don't run tests (--no-run). Add verbosity and colors
-RUN cd zebrad/; cargo build --release --features enable-sentry
+RUN cd zebrad/; cargo build --release --features enable-sentry --bin zebrad
 RUN cargo test --release --features enable-sentry --workspace --no-run
 
 # Runner image

From 7ad399aa3213d723a7dde971af50c3aca336ae80 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Mon, 31 Jan 2022 22:36:14 -0400
Subject: [PATCH 026/124] fix (docker): build on the root directory

---
 docker/Dockerfile.build | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index 33430a6c364..05d8f719325 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -42,7 +42,7 @@ RUN rustc -V; cargo -V; rustup -V
 COPY . .
 # Build and then compile tests
 # Compile, but don't run tests (--no-run). Add verbosity and colors
-RUN cd zebrad/; cargo build --release --features enable-sentry --bin zebrad
+RUN cargo build --release --features enable-sentry --bin zebrad
 RUN cargo test --release --features enable-sentry --workspace --no-run
 
 # Runner image

From 3f0604c50566b6ce973bf543cca78710f84a682e Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 1 Feb 2022 07:01:39 -0400
Subject: [PATCH 027/124] refactor(docker): Use base image commands and tools

---
 docker/Dockerfile.build | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index 05d8f719325..299b46f9b49 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -17,7 +17,7 @@ COPY --from=planner /app/recipe.json recipe.json
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
-    make cmake g++ gcc llvm libclang-dev clang ca-certificates && \
+    llvm libclang-dev clang ca-certificates && \
     rm -Rf /var/lib/apt/lists/* /tmp/*
 
 # Optimize builds. In particular, regenerate-stateful-test-disks.yml was reaching the
@@ -36,12 +36,9 @@ ARG ZEBRA_SKIP_NETWORK_TESTS
 ENV ZEBRA_SKIP_NETWORK_TESTS ${ZEBRA_SKIP_NETWORK_TESTS:-1}
 
 ENV CARGO_HOME /app/.cargo/
-
-RUN rustc -V; cargo -V; rustup -V
-
-COPY . .
 # Build and then compile tests
 # Compile, but don't run tests (--no-run). Add verbosity and colors
+COPY . .
 RUN cargo build --release --features enable-sentry --bin zebrad
 RUN cargo test --release --features enable-sentry --workspace --no-run
 
@@ -79,9 +76,9 @@ RUN /usr/local/bin/zebrad download
 EXPOSE 3000 8233 18233
 
 ARG RUST_LOG
-ENV RUST_LOG ${RUST_LOG:-debug}
+ENV RUST_LOG ${RUST_LOG:-info}
 ARG RUST_BACKTRACE
-ENV RUST_BACKTRACE ${RUST_BACKTRACE:-full}
+ENV RUST_BACKTRACE ${RUST_BACKTRACE:-1}
 ARG SENTRY_DSN
 ENV SENTRY_DSN ${SENTRY_DSN}
 

From a118ee6943fc1eae800b78e967c4ad380dc8f4dd Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 1 Feb 2022 08:23:19 -0400
Subject: [PATCH 028/124] fix (cd): use correct variables & values, add build
 concurrency

---
 .github/workflows/cd.yml |  6 +++---
 docker/Dockerfile.build  | 16 ++++++++--------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index 1ceebbf806f..cf507a1cdb1 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -67,10 +67,10 @@ jobs:
         build-args: |
           NETWORK=${{ github.event.inputs.network || env.NETWORK }}
           SHORT_SHA=${{ env.GITHUB_SHA_SHORT }}
-          RUST_BACKTRACE=full
-          ZEBRA_SKIP_NETWORK_TESTS="1"
+          RUST_BACKTRACE=1
+          ZEBRA_SKIP_IPV6_TESTS="1"
           CHECKPOINT_SYNC=${{ github.event.inputs.checkpoint_sync || true }}
-          RUST_LOG=debug
+          RUST_LOG=info
           SENTRY_DSN=${{ secrets.SENTRY_ENDPOINT }}
         push: true
         cache-from: type=registry,ref=${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:buildcache
diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index 299b46f9b49..e6374201a77 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -10,7 +10,7 @@ WORKDIR /app
 
 FROM chef AS planner
 COPY . .
-RUN cargo chef prepare  --recipe-path recipe.json
+RUN cargo chef prepare --recipe-path recipe.json
 
 FROM chef AS builder
 COPY --from=planner /app/recipe.json recipe.json
@@ -18,7 +18,7 @@ COPY --from=planner /app/recipe.json recipe.json
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     llvm libclang-dev clang ca-certificates && \
-    rm -Rf /var/lib/apt/lists/* /tmp/*
+    rm -rf /var/lib/apt/lists/* /tmp/*
 
 # Optimize builds. In particular, regenerate-stateful-test-disks.yml was reaching the
 # GitHub Actions time limit (6 hours), so we needed to make it faster.
@@ -30,17 +30,17 @@ RUN cargo chef cook --release --features enable-sentry --recipe-path recipe.json
 ARG SHORT_SHA
 ENV SHORT_SHA $SHORT_SHA
 ARG RUST_BACKTRACE
-ENV RUST_BACKTRACE ${RUST_BACKTRACE:-full}
-# Skip test on debian based OS by default as compilation fails
-ARG ZEBRA_SKIP_NETWORK_TESTS
-ENV ZEBRA_SKIP_NETWORK_TESTS ${ZEBRA_SKIP_NETWORK_TESTS:-1}
+ENV RUST_BACKTRACE ${RUST_BACKTRACE:-1}
+# Skip IPv6 tests by default, as some CI environment don't have IPv6 available
+ARG ZEBRA_SKIP_IPV6_TESTS
+ENV ZEBRA_SKIP_IPV6_TESTS ${ZEBRA_SKIP_IPV6_TESTS:-1}
 
 ENV CARGO_HOME /app/.cargo/
 # Build and then compile tests
 # Compile, but don't run tests (--no-run). Add verbosity and colors
 COPY . .
-RUN cargo build --release --features enable-sentry --bin zebrad
-RUN cargo test --release --features enable-sentry --workspace --no-run
+RUN cargo build --jobs 2 --release --features enable-sentry --bin zebrad
+RUN cargo test --jobs 2 --release --features enable-sentry --workspace --no-run
 
 # Runner image
 FROM debian:bullseye-slim AS runtime

From 5301086d9625c6b6e46cec8c19d70001f1181dbc Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 1 Feb 2022 08:44:36 -0400
Subject: [PATCH 029/124] fix(cd): use Mainnet instead of mainnet

---
 .github/workflows/cd.yml                             | 2 +-
 .github/workflows/regenerate-stateful-test-disks.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index cf507a1cdb1..0b3ff3843c3 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -77,7 +77,7 @@ jobs:
         cache-to: type=registry,ref=${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:buildcache,mode=max
 
   deploy-nodes:
-    name: Deploy mainnet nodes
+    name: Deploy Mainnet nodes
     needs: build
     runs-on: ubuntu-latest
     timeout-minutes: 30
diff --git a/.github/workflows/regenerate-stateful-test-disks.yml b/.github/workflows/regenerate-stateful-test-disks.yml
index bff8d622452..5395050265a 100644
--- a/.github/workflows/regenerate-stateful-test-disks.yml
+++ b/.github/workflows/regenerate-stateful-test-disks.yml
@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
     inputs:
       network:
-        default: 'mainnet'
+        default: 'Mainnet'
 
 env:
   PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}

From 0b1ff6eb8d5ad131a3dfa3f64eca7e1a5fa2cb8c Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 1 Feb 2022 09:02:44 -0400
Subject: [PATCH 030/124] imp: remove checkout as Buildkit uses the git context

---
 .github/workflows/cd.yml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index 0b3ff3843c3..eb3a2849706 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -26,10 +26,6 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2.4.0
-      with:
-        persist-credentials: false
-
     - name: Inject slug/short variables
       uses: rlespinasse/github-slug-action@v4
 

From 41635f9309efa49caef3a19659e3765073acdfd6 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 1 Feb 2022 09:11:26 -0400
Subject: [PATCH 031/124] fix (docker): just Buildkit uses a .dockerignore in a
 path

---
 docker/.dockerignore => .dockerignore | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename docker/.dockerignore => .dockerignore (100%)

diff --git a/docker/.dockerignore b/.dockerignore
similarity index 100%
rename from docker/.dockerignore
rename to .dockerignore

From e6fa407ae13d2a18cfc2b61b1e1f7a28b9f98860 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 1 Feb 2022 09:23:21 -0400
Subject: [PATCH 032/124] imp (cd): just use needed variables in the right
 place

---
 .github/workflows/cd.yml | 1 -
 docker/Dockerfile.build  | 4 ----
 2 files changed, 5 deletions(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index eb3a2849706..2d5164dc61d 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -66,7 +66,6 @@ jobs:
           RUST_BACKTRACE=1
           ZEBRA_SKIP_IPV6_TESTS="1"
           CHECKPOINT_SYNC=${{ github.event.inputs.checkpoint_sync || true }}
-          RUST_LOG=info
           SENTRY_DSN=${{ secrets.SENTRY_ENDPOINT }}
         push: true
         cache-from: type=registry,ref=${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:buildcache
diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index e6374201a77..18f2bd8877c 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -75,10 +75,6 @@ RUN /usr/local/bin/zebrad download
 
 EXPOSE 3000 8233 18233
 
-ARG RUST_LOG
-ENV RUST_LOG ${RUST_LOG:-info}
-ARG RUST_BACKTRACE
-ENV RUST_BACKTRACE ${RUST_BACKTRACE:-1}
 ARG SENTRY_DSN
 ENV SENTRY_DSN ${SENTRY_DSN}
 

From 5d4318da60731178ea7b89840059208400dba0f5 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 1 Feb 2022 09:33:05 -0400
Subject: [PATCH 033/124] imp (cd): do not checkout if not needed

---
 .github/workflows/cd.yml | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index 2d5164dc61d..34f700aeb4e 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -12,6 +12,7 @@ on:
       - main
 
 env:
+  CARGO_INCREMENTAL: '1'
   NETWORK: Mainnet
   PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
   GAR_BASE: us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/zebra
@@ -79,10 +80,6 @@ jobs:
     if: github.event_name == 'push'
 
     steps:
-      - uses: actions/checkout@v2.4.0
-        with:
-          persist-credentials: false
-
       - name: Inject slug/short variables
         uses: rlespinasse/github-slug-action@v4
 
@@ -146,10 +143,6 @@ jobs:
     if: github.event_name == 'workflow_dispatch'
 
     steps:
-      - uses: actions/checkout@v2.4.0
-        with:
-          persist-credentials: false
-
       - name: Inject slug/short variables
         uses: rlespinasse/github-slug-action@v4
 

From a89dd2f467ad25d1e23b42dab457ca0fa5a0001e Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 1 Feb 2022 09:53:14 -0400
Subject: [PATCH 034/124] test: run on push

---
 .github/workflows/cd.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index 34f700aeb4e..a937a2bb241 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -8,8 +8,8 @@ on:
       checkpoint_sync:
         default: true
   push:
-    branches:
-      - main
+    # branches:
+    #   - main
 
 env:
   CARGO_INCREMENTAL: '1'

From 06269d7517ec585ec39aa9bf51147b54d6a41bc7 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 1 Feb 2022 10:19:08 -0400
Subject: [PATCH 035/124] refactor(docker): reduce build changes

---
 docker/Dockerfile.build | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index 18f2bd8877c..b10ce35dccc 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -5,7 +5,7 @@
 # - builder: caches our dependencies and builds the binary
 # - runtime: is our runtime environment
 FROM rust:bullseye as chef
-RUN cargo install cargo-chef
+RUN cargo install cargo-chef --locked
 WORKDIR /app
 
 FROM chef AS planner
@@ -23,24 +23,22 @@ RUN apt-get update && \
 # Optimize builds. In particular, regenerate-stateful-test-disks.yml was reaching the
 # GitHub Actions time limit (6 hours), so we needed to make it faster.
 ENV RUSTFLAGS -O
+ENV CARGO_HOME /app/.cargo/
 # Build dependencies - this is the caching Docker layer!
 RUN cargo chef cook --release --features enable-sentry --recipe-path recipe.json
 
 # Build zebra
-ARG SHORT_SHA
-ENV SHORT_SHA $SHORT_SHA
-ARG RUST_BACKTRACE
-ENV RUST_BACKTRACE ${RUST_BACKTRACE:-1}
+ARG RUST_BACKTRACE=1
+ENV RUST_BACKTRACE ${RUST_BACKTRACE}
 # Skip IPv6 tests by default, as some CI environment don't have IPv6 available
-ARG ZEBRA_SKIP_IPV6_TESTS
-ENV ZEBRA_SKIP_IPV6_TESTS ${ZEBRA_SKIP_IPV6_TESTS:-1}
+ARG ZEBRA_SKIP_IPV6_TESTS=1
+ENV ZEBRA_SKIP_IPV6_TESTS ${ZEBRA_SKIP_IPV6_TESTS}
 
-ENV CARGO_HOME /app/.cargo/
 # Build and then compile tests
 # Compile, but don't run tests (--no-run). Add verbosity and colors
 COPY . .
-RUN cargo build --jobs 2 --release --features enable-sentry --bin zebrad
-RUN cargo test --jobs 2 --release --features enable-sentry --workspace --no-run
+RUN cargo build --locked --jobs 2 --release --features enable-sentry --bin zebrad
+RUN cargo test --locked --jobs 2 --release --features enable-sentry --workspace --no-run
 
 # Runner image
 FROM debian:bullseye-slim AS runtime
@@ -75,6 +73,9 @@ RUN /usr/local/bin/zebrad download
 
 EXPOSE 3000 8233 18233
 
+ARG SHORT_SHA
+ENV SHORT_SHA $SHORT_SHA
+
 ARG SENTRY_DSN
 ENV SENTRY_DSN ${SENTRY_DSN}
 

From 67eda8c6bda96d8d1c39408f51c5c2451532532f Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 1 Feb 2022 10:28:29 -0400
Subject: [PATCH 036/124] fix(cd): not checking out was limiting some variables

---
 .github/workflows/cd.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index a937a2bb241..6618186fd94 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -27,6 +27,10 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+    - uses: actions/checkout@v2.4.0
+      with:
+        persist-credentials: false
+
     - name: Inject slug/short variables
       uses: rlespinasse/github-slug-action@v4
 

From f30601d6dbf0205b9b38bc30b0b8282d0eec6321 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 1 Feb 2022 11:11:35 -0400
Subject: [PATCH 037/124] refactor(test): add an multistage exclusive for
 testing

---
 .github/workflows/test.yml |  2 +-
 docker/Dockerfile.build    | 12 ++++++++----
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index ea8bc3c15a1..386a52b8202 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -60,7 +60,7 @@ jobs:
         id: docker_build
         uses: docker/build-push-action@v2.8.0
         with:
-          target: builder
+          target: tester
           context: .
           file: ./docker/Dockerfile.build
           tags: |
diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index b10ce35dccc..46d9c70997f 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -30,15 +30,19 @@ RUN cargo chef cook --release --features enable-sentry --recipe-path recipe.json
 # Build zebra
 ARG RUST_BACKTRACE=1
 ENV RUST_BACKTRACE ${RUST_BACKTRACE}
+
+COPY . .
+RUN cargo build --locked --jobs 2 --release --features enable-sentry --bin zebrad
+
+FROM builder AS tester
 # Skip IPv6 tests by default, as some CI environment don't have IPv6 available
 ARG ZEBRA_SKIP_IPV6_TESTS=1
 ENV ZEBRA_SKIP_IPV6_TESTS ${ZEBRA_SKIP_IPV6_TESTS}
 
-# Build and then compile tests
-# Compile, but don't run tests (--no-run). Add verbosity and colors
-COPY . .
-RUN cargo build --locked --jobs 2 --release --features enable-sentry --bin zebrad
 RUN cargo test --locked --jobs 2 --release --features enable-sentry --workspace --no-run
+RUN cargo run --bin zebrad download
+
+CMD ["cargo" "test" "--workspace"]
 
 # Runner image
 FROM debian:bullseye-slim AS runtime

From bb892076ecf7ea5a8a581e9cfbb306d101e67044 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 1 Feb 2022 11:21:29 -0400
Subject: [PATCH 038/124] fix(cd): remove tests as a runtime dependency

---
 docker/Dockerfile.build | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index b10ce35dccc..1d51a2b51f7 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -3,6 +3,7 @@
 # - chef: installs cargo-chef
 # - planner: computes the recipe file
 # - builder: caches our dependencies and builds the binary
+# - tester: builds and run tests
 # - runtime: is our runtime environment
 FROM rust:bullseye as chef
 RUN cargo install cargo-chef --locked
@@ -30,15 +31,19 @@ RUN cargo chef cook --release --features enable-sentry --recipe-path recipe.json
 # Build zebra
 ARG RUST_BACKTRACE=1
 ENV RUST_BACKTRACE ${RUST_BACKTRACE}
+
+COPY . .
+RUN cargo build --locked --jobs 2 --release --features enable-sentry --bin zebrad
+
+FROM builder AS tester
 # Skip IPv6 tests by default, as some CI environment don't have IPv6 available
 ARG ZEBRA_SKIP_IPV6_TESTS=1
 ENV ZEBRA_SKIP_IPV6_TESTS ${ZEBRA_SKIP_IPV6_TESTS}
 
-# Build and then compile tests
-# Compile, but don't run tests (--no-run). Add verbosity and colors
-COPY . .
-RUN cargo build --locked --jobs 2 --release --features enable-sentry --bin zebrad
 RUN cargo test --locked --jobs 2 --release --features enable-sentry --workspace --no-run
+RUN cargo run --bin zebrad download
+
+CMD ["cargo" "test" "--workspace"]
 
 # Runner image
 FROM debian:bullseye-slim AS runtime

From aa9191bf72f1ad149d9c6c2098be9a207f702bcc Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 1 Feb 2022 13:07:11 -0400
Subject: [PATCH 039/124] fix(cd): use default service account with
 cloud-platform scope

---
 .github/workflows/cd.yml                   | 1 -
 .github/workflows/test.yml                 | 1 -
 .github/workflows/zcashd-manual-deploy.yml | 1 -
 3 files changed, 3 deletions(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index 6618186fd94..c8d78bd81e2 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -107,7 +107,6 @@ jobs:
           --create-disk name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }},auto-delete=yes,size=100GB,type=pd-ssd \
           --container-mount-disk mount-path="/zebrad-cache",name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }} \
           --machine-type ${{ env.MACHINE_TYPE }} \
-          --service-account ${{ env.DEPLOY_SA }} \
           --scopes cloud-platform \
           --tags zebrad
 
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 7f4040a3e9f..61186ef2624 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -58,7 +58,6 @@ jobs:
           --container-restart-policy never \
           --create-disk name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-mainnet-canopy",image=zebrad-cache-1558f3378-mainnet-canopy \
           --machine-type ${{ env.MACHINE_TYPE }} \
-          --service-account ${{ env.DEPLOY_SA }} \
           --scopes cloud-platform \
           --tags zebrad \
           --zone "${{ env.ZONE }}"
diff --git a/.github/workflows/zcashd-manual-deploy.yml b/.github/workflows/zcashd-manual-deploy.yml
index 17b98841b27..4ade742d3e2 100644
--- a/.github/workflows/zcashd-manual-deploy.yml
+++ b/.github/workflows/zcashd-manual-deploy.yml
@@ -52,7 +52,6 @@ jobs:
           --container-image "electriccoinco/zcashd" \
           --container-env ZCASHD_NETWORK="${{ github.event.inputs.network }}" \
           --machine-type ${{ env.MACHINE_TYPE }} \
-          --service-account ${{ env.DEPLOY_SA }} \
           --scopes cloud-platform \
           --tags zcashd \
 

From 2d422926e62848f3fa7ad9d7526fca7093dcc633 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 1 Feb 2022 13:21:35 -0400
Subject: [PATCH 040/124] fix(cd): revert checkout actions

---
 .github/workflows/cd.yml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index c8d78bd81e2..e427fc3357b 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -84,6 +84,10 @@ jobs:
     if: github.event_name == 'push'
 
     steps:
+      - uses: actions/checkout@v2.4.0
+        with:
+          persist-credentials: false
+
       - name: Inject slug/short variables
         uses: rlespinasse/github-slug-action@v4
 
@@ -146,6 +150,10 @@ jobs:
     if: github.event_name == 'workflow_dispatch'
 
     steps:
+      - uses: actions/checkout@v2.4.0
+        with:
+          persist-credentials: false
+
       - name: Inject slug/short variables
         uses: rlespinasse/github-slug-action@v4
 

From d4c81bb14c159874247e48148924fe6219fd9615 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 1 Feb 2022 13:36:33 -0400
Subject: [PATCH 041/124] fix: use GA c2 instead of Preview c2d machine types

---
 .github/workflows/cd.yml                             | 2 +-
 .github/workflows/regenerate-stateful-test-disks.yml | 2 +-
 .github/workflows/test.yml                           | 2 +-
 .github/workflows/zcashd-manual-deploy.yml           | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index e427fc3357b..88650a5a63b 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -18,7 +18,7 @@ env:
   GAR_BASE: us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/zebra
   REGION: us-central1
   ZONE: us-central1-a
-  MACHINE_TYPE: c2d-standard-4
+  MACHINE_TYPE: c2-standard-4
 
 jobs:
   build:
diff --git a/.github/workflows/regenerate-stateful-test-disks.yml b/.github/workflows/regenerate-stateful-test-disks.yml
index 5395050265a..67c8b693a1a 100644
--- a/.github/workflows/regenerate-stateful-test-disks.yml
+++ b/.github/workflows/regenerate-stateful-test-disks.yml
@@ -10,7 +10,7 @@ env:
   PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
   REGION: us-central1
   ZONE: us-central1-a
-  MACHINE_TYPE: c2d-standard-4
+  MACHINE_TYPE: c2-standard-4
   DEPLOY_SA: cos-vm@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com
 
 jobs:
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 61186ef2624..85725cdd8ce 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -17,7 +17,7 @@ env:
   PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
   REGION: us-central1
   ZONE: us-central1-a
-  MACHINE_TYPE: c2d-standard-8
+  MACHINE_TYPE: c2-standard-4
   DEPLOY_SA: cos-vm@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com
 
 jobs:
diff --git a/.github/workflows/zcashd-manual-deploy.yml b/.github/workflows/zcashd-manual-deploy.yml
index 4ade742d3e2..627dd4233e2 100644
--- a/.github/workflows/zcashd-manual-deploy.yml
+++ b/.github/workflows/zcashd-manual-deploy.yml
@@ -12,7 +12,7 @@ env:
   PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
   REGION: us-central1
   ZONE: us-central1-a
-  MACHINE_TYPE: c2d-standard-4
+  MACHINE_TYPE: c2-standard-4
   DEPLOY_SA: cos-vm@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com
 
 jobs:

From 7f67d2b4f5a45e533cecf1e023bb7ca5fbb4943a Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 1 Feb 2022 14:25:42 -0400
Subject: [PATCH 042/124] fix(actions): remove workflow_dispatch from patched
 actions

This causes GitHub confusion as it can't determined which of the actions using workflow_dispatch is the right one
---
 .github/workflows/cd.yml             | 4 ++--
 .github/workflows/coverage.patch.yml | 1 -
 .github/workflows/docs.patch.yml     | 1 -
 .github/workflows/test.patch.yml     | 1 -
 4 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index ec7c14a5cd5..e70343c0285 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -8,8 +8,8 @@ on:
       checkpoint_sync:
         default: true
   push:
-    # branches:
-    #   - main
+    branches:
+      - main
 
 env:
   CARGO_INCREMENTAL: '1'
diff --git a/.github/workflows/coverage.patch.yml b/.github/workflows/coverage.patch.yml
index 16e78662142..3677ea94786 100644
--- a/.github/workflows/coverage.patch.yml
+++ b/.github/workflows/coverage.patch.yml
@@ -1,7 +1,6 @@
 name: Coverage
 
 on:
-  workflow_dispatch:
   pull_request:
     path-ignore:
       - '**/*.rs'
diff --git a/.github/workflows/docs.patch.yml b/.github/workflows/docs.patch.yml
index d5197b510d7..36c921d5eaf 100644
--- a/.github/workflows/docs.patch.yml
+++ b/.github/workflows/docs.patch.yml
@@ -1,7 +1,6 @@
 name: Docs
 
 on:
-  workflow_dispatch:
   push:
     branches:
       - main
diff --git a/.github/workflows/test.patch.yml b/.github/workflows/test.patch.yml
index 820ef6161ef..cbaae05bc07 100644
--- a/.github/workflows/test.patch.yml
+++ b/.github/workflows/test.patch.yml
@@ -1,7 +1,6 @@
 name: Test
 
 on:
-  workflow_dispatch:
   push:
     branches:
       - main

From 6fc062d04e9c9f940ce461a0de5081473665bf7a Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 1 Feb 2022 14:29:11 -0400
Subject: [PATCH 043/124] fix(actions): remove patches from push actions

---
 .github/workflows/docs.patch.yml | 20 --------------------
 .github/workflows/test.patch.yml | 21 ---------------------
 2 files changed, 41 deletions(-)
 delete mode 100644 .github/workflows/docs.patch.yml
 delete mode 100644 .github/workflows/test.patch.yml

diff --git a/.github/workflows/docs.patch.yml b/.github/workflows/docs.patch.yml
deleted file mode 100644
index 36c921d5eaf..00000000000
--- a/.github/workflows/docs.patch.yml
+++ /dev/null
@@ -1,20 +0,0 @@
-name: Docs
-
-on:
-  push:
-    branches:
-      - main
-    path-ignore:
-      - 'book/**'
-      - '**/firebase.json'
-      - 'katex-header.html'
-      - '.github/workflows/docs.yml'
-
-jobs:
-  build:
-    name: Build and Deploy Docs (+beta)
-    timeout-minutes: 30
-    runs-on: ubuntu-latest
-
-    steps:
-      - run: 'echo "No build required"'
diff --git a/.github/workflows/test.patch.yml b/.github/workflows/test.patch.yml
deleted file mode 100644
index cbaae05bc07..00000000000
--- a/.github/workflows/test.patch.yml
+++ /dev/null
@@ -1,21 +0,0 @@
-name: Test
-
-on:
-  push:
-    branches:
-      - main
-    path-ignore:
-      - '**/*.rs'
-      - '**/*.txt'
-      - '**/Cargo.toml'
-      - '**/Cargo.lock'
-      - 'docker/**'
-      - '.github/workflows/test.yml'
-
-jobs:
-  test:
-    name: Run all tests
-    runs-on: ubuntu-latest
-
-    steps:
-      - run: 'echo "No build required"'

From 3d9aaca87ecd7eedfad2e19e2d47a029acf4b560 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 1 Feb 2022 14:31:37 -0400
Subject: [PATCH 044/124] test: validate changes on each push

---
 .github/workflows/test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index f66af365ab7..ece6e5abd67 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -3,8 +3,8 @@ name: Test
 on:
   workflow_dispatch:
   push:
-    branches:
-      - main
+    # branches:
+    #   - main
     path:
       - '**/*.rs'
       - '**/*.txt'

From ed48fa14eef65e57e07c0adb5d7b830415ba5953 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 1 Feb 2022 14:36:52 -0400
Subject: [PATCH 045/124] fix(test): wrong file syntax on test job

---
 .github/workflows/test.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index ece6e5abd67..60fa7cef30d 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -78,11 +78,12 @@ jobs:
           cache-from: type=registry,ref=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:buildcache
           cache-to: type=registry,ref=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:buildcache,mode=max
 
-
+  test:
     name: Run all tests
     runs-on: ubuntu-latest
-    steps:
     needs: build
+
+    steps:
       - uses: actions/checkout@v2.4.0
         with:
           persist-credentials: false

From 28867cd7bc00c93f6ba7d7c22f869c0d681c9e30 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 1 Feb 2022 14:47:28 -0400
Subject: [PATCH 046/124] fix(test): add missing env parameters

---
 .github/workflows/test.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 60fa7cef30d..2f3ea4046e5 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -14,11 +14,13 @@ on:
       - '.github/workflows/test.yml'
 
 env:
+  CARGO_INCREMENTAL: '1'
+  NETWORK: Testnet
   PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
+  GAR_BASE: us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/zebra
   REGION: us-central1
   ZONE: us-central1-a
   MACHINE_TYPE: c2-standard-4
-  DEPLOY_SA: cos-vm@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com
   IMAGE_NAME: zebrad-test
 
 jobs:

From 9e522e1cf8e6720b951a191c042e86b5fbd54964 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 1 Feb 2022 16:23:32 -0400
Subject: [PATCH 047/124] fix(docker): Do not rebuild to download params and
 run tests

---
 docker/Dockerfile.build | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index 1d51a2b51f7..2771a9a7031 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -41,9 +41,10 @@ ARG ZEBRA_SKIP_IPV6_TESTS=1
 ENV ZEBRA_SKIP_IPV6_TESTS ${ZEBRA_SKIP_IPV6_TESTS}
 
 RUN cargo test --locked --jobs 2 --release --features enable-sentry --workspace --no-run
-RUN cargo run --bin zebrad download
+# Pre-download Zcash Sprout and Sapling parameters
+RUN cargo run --locked --jobs 2 --release --features enable-sentry --bin zebrad download
 
-CMD ["cargo" "test" "--workspace"]
+CMD ["cargo" "test" "--locked" "--release" "--features" "enable-sentry" "--workspace"]
 
 # Runner image
 FROM debian:bullseye-slim AS runtime

From 341b34049b3a9877210c0cae8b39146cef7c667c Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 1 Feb 2022 18:07:54 -0400
Subject: [PATCH 048/124] fix(test): setup gcloud and loginto artifact just
 when needed

Try not to rebuild the tests
---
 .github/workflows/test.yml | 19 ++++++-------------
 1 file changed, 6 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 2f3ea4046e5..5b8366ce11e 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -37,13 +37,6 @@ jobs:
       - name: Inject slug/short variables
         uses: rlespinasse/github-slug-action@v4
 
-      # Setup gcloud CLI
-      - name: Authenticate to Google Cloud
-        id: auth
-        uses: google-github-actions/auth@v0.5.0
-        with:
-          credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
-
       # Setup Docker Buildx to allow use of docker cache layers from GH
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
@@ -93,17 +86,17 @@ jobs:
       - name: Inject slug/short variables
         uses: rlespinasse/github-slug-action@v4
 
-      - name: Login to Google Artifact Registry
-        uses: docker/login-action@v1.12.0
+      # Setup gcloud CLI
+      - name: Authenticate to Google Cloud
+        id: auth
+        uses: google-github-actions/auth@v0.5.0
         with:
-          registry: us-docker.pkg.dev
-          username: _json_key
-          password: ${{ secrets.GOOGLE_CREDENTIALS }}
+          credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
 
       - name: Run workspace tests with Zunstable-options
         run: |
           docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
-          docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} cargo test --workspace -- -Zunstable-options --include-ignored
+          docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --workspace -- -Zunstable-options --include-ignored
         env:
           ZEBRA_SKIP_IPV6_TESTS: "1"
 

From 2089904a8a9107bfd5a6c808044d838b8b373455 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 1 Feb 2022 21:59:11 -0400
Subject: [PATCH 049/124] fix(test): use GCP container to sync past mandatory
 checkpoint

---
 .github/workflows/test.yml | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 5b8366ce11e..516df613322 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -105,18 +105,16 @@ jobs:
       - name: Sync past mandatory checkpoint
         run: |
           gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
-          --boot-disk-size 100GB \
+          --boot-disk-size 10GB \
           --boot-disk-type pd-ssd \
+          --disk=boot=no,device-name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-mainnet-canopy,mode=rw,name=zebrad-cache-1558f3378-mainnet-canopy
+          --container-mount-disk=mount-path=/zebrad-cache,name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-mainnet-canopy
+          --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} \
+          --container-restart-policy=never
           --container-stdin \
           --container-tty \
-          --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} \
-          --container-command "/bin/bash" \
-          --container-arg="-c" \
-          --container-arg="cargo test --verbose --features test_sync_past_mandatory_checkpoint_mainnet --manifest-path zebrad/Cargo.toml sync_past_mandatory_checkpoint_mainnet" \
-          --container-mount-disk mount-path='/mainnet',name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-mainnet-canopy" \
-          --container-restart-policy never \
-          --create-disk name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-mainnet-canopy,auto-delete=yes,size=100GB,type=pd-ssd \
-          --container-mount-disk mount-path="/zebrad-cache",name=zebrad-cache-1558f3378-mainnet-canopy \
+          --container-command "cargo" \
+          --container-arg="test --locked --release --features enable-sentry,test_sync_past_mandatory_checkpoint_mainnet --manifest-path zebrad/Cargo.toml sync_past_mandatory_checkpoint_mainnet" \
           --machine-type ${{ env.MACHINE_TYPE }} \
           --scopes cloud-platform \
           --tags zebrad \
@@ -127,4 +125,4 @@ jobs:
         # Always run even if the earlier step fails
         if: ${{ always() }}
         run: |
-          gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"
+          gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --zone "${{ env.ZONE }}"

From c1b028c0b12a4d33bdd732de7b5220d5ad1c001d Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 1 Feb 2022 22:02:18 -0400
Subject: [PATCH 050/124] fix(test): missing separators

---
 .github/workflows/test.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 516df613322..826207175e3 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -107,10 +107,10 @@ jobs:
           gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
           --boot-disk-size 10GB \
           --boot-disk-type pd-ssd \
-          --disk=boot=no,device-name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-mainnet-canopy,mode=rw,name=zebrad-cache-1558f3378-mainnet-canopy
-          --container-mount-disk=mount-path=/zebrad-cache,name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-mainnet-canopy
+          --disk=boot=no,device-name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-mainnet-canopy,mode=rw,name=zebrad-cache-1558f3378-mainnet-canopy \
+          --container-mount-disk=mount-path=/zebrad-cache,name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-mainnet-canopy \
           --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} \
-          --container-restart-policy=never
+          --container-restart-policy=never \
           --container-stdin \
           --container-tty \
           --container-command "cargo" \

From 10dd7de6be49dee796dbcc5bcc54097f169c12b8 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 1 Feb 2022 22:11:43 -0400
Subject: [PATCH 051/124] test

---
 docker/Dockerfile.build | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index 2771a9a7031..0b98b2ff522 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -46,6 +46,7 @@ RUN cargo run --locked --jobs 2 --release --features enable-sentry --bin zebrad
 
 CMD ["cargo" "test" "--locked" "--release" "--features" "enable-sentry" "--workspace"]
 
+
 # Runner image
 FROM debian:bullseye-slim AS runtime
 

From bb38ff56d0af931cf53bfe7716da9888405fa994 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 1 Feb 2022 23:02:42 -0400
Subject: [PATCH 052/124] fix(test): mount the available disk

---
 .github/workflows/test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 826207175e3..8eb49e2e3d6 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -107,8 +107,8 @@ jobs:
           gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
           --boot-disk-size 10GB \
           --boot-disk-type pd-ssd \
-          --disk=boot=no,device-name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-mainnet-canopy,mode=rw,name=zebrad-cache-1558f3378-mainnet-canopy \
-          --container-mount-disk=mount-path=/zebrad-cache,name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-mainnet-canopy \
+          --disk=boot=no,mode=rw,name=zebrad-cache-1558f3378-mainnet-canopy \
+          --container-mount-disk=mount-path=/zebrad-cache,name=zebrad-cache-1558f3378-mainnet-canopy \
           --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} \
           --container-restart-policy=never \
           --container-stdin \

From 545499250abd808e8ed7f3a2c16d435ff96c9869 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 1 Feb 2022 23:10:00 -0400
Subject: [PATCH 053/124] push

---
 docker/Dockerfile.build | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index 0b98b2ff522..2771a9a7031 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -46,7 +46,6 @@ RUN cargo run --locked --jobs 2 --release --features enable-sentry --bin zebrad
 
 CMD ["cargo" "test" "--locked" "--release" "--features" "enable-sentry" "--workspace"]
 
-
 # Runner image
 FROM debian:bullseye-slim AS runtime
 

From d0fc3bb342813d6af90f2f16468ed2ebfb8dceff Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Wed, 2 Feb 2022 01:09:19 -0400
Subject: [PATCH 054/124] refactor(test): merge disk regeneration into test.yml

---
 .../regenerate-stateful-test-disks.yml        | 77 -------------------
 .github/workflows/test.yml                    | 55 ++++++++++++-
 2 files changed, 52 insertions(+), 80 deletions(-)
 delete mode 100644 .github/workflows/regenerate-stateful-test-disks.yml

diff --git a/.github/workflows/regenerate-stateful-test-disks.yml b/.github/workflows/regenerate-stateful-test-disks.yml
deleted file mode 100644
index 67c8b693a1a..00000000000
--- a/.github/workflows/regenerate-stateful-test-disks.yml
+++ /dev/null
@@ -1,77 +0,0 @@
-name: Regenerate test state
-
-on:
-  workflow_dispatch:
-    inputs:
-      network:
-        default: 'Mainnet'
-
-env:
-  PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
-  REGION: us-central1
-  ZONE: us-central1-a
-  MACHINE_TYPE: c2-standard-4
-  DEPLOY_SA: cos-vm@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com
-
-jobs:
-
-  regenerate:
-    name: Regenerate test state
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2.4.0
-        with:
-          persist-credentials: false
-
-      - name: Inject slug/short variables
-        uses: rlespinasse/github-slug-action@v4
-
-      # Setup gcloud CLI
-      - name: Authenticate to Google Cloud
-        id: auth
-        uses: google-github-actions/auth@v0.5.0
-        with:
-          credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
-
-      - name: Set up gcloud SDK environment
-        uses: google-github-actions/setup-gcloud@v0.4.0
-        with:
-          project_id: ${{ env.PROJECT_ID }}
-
-      # Creates Compute Engine virtual machine instance w/ disks
-      - name: Create instance
-        run: |
-          gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
-          --boot-disk-size 100GB \
-          --boot-disk-type pd-ssd \
-          --container-image rust:buster \
-          --container-mount-disk mount-path='/${{ github.event.inputs.network }}',name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy" \
-          --container-restart-policy never \
-          --create-disk name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy",size=100GB,type=pd-ssd \
-          --machine-type ${{ env.MACHINE_TYPE }} \
-          --service-account ${{ env.DEPLOY_SA }} \
-          --scopes cloud-platform \
-          --tags zebrad \
-          --zone "${{ env.ZONE }}"
-      # Build and run test container to sync up to activation and no further
-      - name: Regenerate state for tests
-        id: regenerate-state
-        run: |
-          gcloud compute ssh "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --zone "${{ env.ZONE }}" --command \
-          "git clone -b ${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }} https://github.com/ZcashFoundation/zebra.git &&
-          cd zebra/ &&
-          docker build --build-arg SHORT_SHA=${{ env.GITHUB_SHA_SHORT }} -f docker/Dockerfile.test -t zebrad-test . &&
-          docker run -i -e "ZEBRA_SKIP_IPV6_TESTS=1" --mount type=bind,source=/mnt/disks/gce-containers-mounts/gce-persistent-disks/zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy,target=/zebrad-cache zebrad-test:latest cargo test --verbose --features test_sync_to_mandatory_checkpoint_${{ github.event.inputs.network }} --manifest-path zebrad/Cargo.toml sync_to_mandatory_checkpoint_${{ github.event.inputs.network }};
-          "
-      # Create image from disk that will be used in test.yml workflow
-      - name: Create image from state disk
-        # Only run if the earlier step succeeds
-        if: steps.regenerate-state.outcome == 'success'
-        run: |
-          gcloud compute images create "zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy" --source-disk="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy" --source-disk-zone="${{ env.ZONE }}"
-      # Clean up
-      - name: Delete test instance
-        # Always run even if the earlier step fails
-        if: ${{ always() }}
-        run: |
-          gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 8eb49e2e3d6..c6ff1f84939 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -2,6 +2,9 @@ name: Test
 
 on:
   workflow_dispatch:
+    inputs:
+      network:
+        default: 'Mainnet'
   push:
     # branches:
     #   - main
@@ -100,9 +103,50 @@ jobs:
         env:
           ZEBRA_SKIP_IPV6_TESTS: "1"
 
+      - name: Regenerate state test disk
+        id: regenerate-state
+        if: github.event_name == 'workflow_dispatch'
+        run: |
+          gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
+          --boot-disk-size 10GB \
+          --boot-disk-type pd-ssd \
+          --create-disk name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy",size=100GB,type=pd-ssd \
+          --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} \
+          --container-restart-policy=never \
+          --container-stdin \
+          --container-tty \
+          --container-mount-disk mount-path='/zebrad-cache',name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy" \
+          --container-command "cargo" \
+          --container-arg="test --verbose --features test_sync_to_mandatory_checkpoint_mainnet --manifest-path zebrad/Cargo.toml sync_to_mandatory_checkpoint_mainnet" \
+          --container-env=ZEBRA_SKIP_IPV6_TESTS=1 \
+          --machine-type ${{ env.MACHINE_TYPE }} \
+          --scopes cloud-platform \
+          --tags zebrad \
+          --zone "${{ env.ZONE }}"
+
+      - name: Wait regeneration for 2 hours
+        run: sleep 7200s
+        shell: bash
+
+      # Create image from disk that will be used to sync past mandatory checkpoint test
+      - name: Create image from state disk
+        # Only run if the earlier step succeeds
+        if: steps.regenerate-state.outcome == 'success'
+        run: |
+          gcloud compute images create zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy --source-disk=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy --source-disk-zone=${{ env.ZONE }} --storage-location=us
+
+      - name: Delete test instance
+        # Always run even if the earlier step fails
+        if: github.event_name == 'workflow_dispatch'
+        continue-on-error: true
+        run: |
+          gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"
+
       # Creates Compute Engine virtual machine instance w/ disks
       # TODO: Remove hardcoded value zebrad-cache-1558f3378-mainnet-canopy
       - name: Sync past mandatory checkpoint
+        id: sync-past-checkpoint
+        if: github.event_name == 'push'
         run: |
           gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
           --boot-disk-size 10GB \
@@ -115,14 +159,19 @@ jobs:
           --container-tty \
           --container-command "cargo" \
           --container-arg="test --locked --release --features enable-sentry,test_sync_past_mandatory_checkpoint_mainnet --manifest-path zebrad/Cargo.toml sync_past_mandatory_checkpoint_mainnet" \
+          --container-env=ZEBRA_SKIP_IPV6_TESTS=1 \
           --machine-type ${{ env.MACHINE_TYPE }} \
           --scopes cloud-platform \
           --tags zebrad \
           --zone "${{ env.ZONE }}"
 
-      # Clean up
+      - name: Wait sync for 2 hours
+        run: sleep 7200s
+        shell: bash
+
       - name: Delete test instance
         # Always run even if the earlier step fails
-        if: ${{ always() }}
+        if: steps.sync-past-checkpoint.outcome == 'success'
+        continue-on-error: true
         run: |
-          gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --zone "${{ env.ZONE }}"
+          gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"

From 0d0dde0b681c0c12a9cfc7e5d9e8e62636784c63 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Wed, 2 Feb 2022 01:20:57 -0400
Subject: [PATCH 055/124] fix(cd): minor typo fixes

---
 .github/workflows/zcashd-manual-deploy.yml | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/zcashd-manual-deploy.yml b/.github/workflows/zcashd-manual-deploy.yml
index 627dd4233e2..ae6ec3c3422 100644
--- a/.github/workflows/zcashd-manual-deploy.yml
+++ b/.github/workflows/zcashd-manual-deploy.yml
@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
     inputs:
       network:
-        default: 'testnet'
+        default: 'Testnet'
       size:
         default: 10
 
@@ -13,14 +13,13 @@ env:
   REGION: us-central1
   ZONE: us-central1-a
   MACHINE_TYPE: c2-standard-4
-  DEPLOY_SA: cos-vm@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com
 
 jobs:
-
   deploy:
     name: Deploy zcashd nodes
     runs-on: ubuntu-latest
     timeout-minutes: 30
+
     steps:
       - uses: actions/checkout@v2.4.0
         with:
@@ -44,16 +43,16 @@ jobs:
       # Create instance template from container image
       - name: Create instance template
         run: |
-          gcloud compute instance-templates create-with-container "zcashd-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
-          --boot-disk-size 100GB \
+          gcloud compute instance-templates create-with-container zcashd-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} \
+          --boot-disk-size 10GB \
           --boot-disk-type=pd-ssd \
           --container-stdin \
           --container-tty \
-          --container-image "electriccoinco/zcashd" \
+          --container-image electriccoinco/zcashd \
           --container-env ZCASHD_NETWORK="${{ github.event.inputs.network }}" \
           --machine-type ${{ env.MACHINE_TYPE }} \
           --scopes cloud-platform \
-          --tags zcashd \
+          --tags zcashd
 
       # Check if our destination instance group exists already
       - name: Check if instance group exists

From 73d103db20ab6f9bd906eebb4a7e157964214872 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Wed, 2 Feb 2022 01:22:02 -0400
Subject: [PATCH 056/124] fix(docker): rebuild on .github changes

---
 .dockerignore | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.dockerignore b/.dockerignore
index fae64da0fe6..3645101174c 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -2,5 +2,4 @@ target
 Dockerfile
 .dockerignore
 .git
-.github
 .gitignore

From 8a67e1d91740b3e5a2b3e90fd3e7a17c431145d4 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Wed, 2 Feb 2022 13:42:33 -0400
Subject: [PATCH 057/124] fix(cd): keep compatibility with gcr.io

To prevent conflicts between registries, and migrate when the time is right, we'll keep pushing to both registries and use github actions cache to prevent conflicts between artifacts.
---
 .github/workflows/cd.yml | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index 88650a5a63b..a66600e873f 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -16,6 +16,7 @@ env:
   NETWORK: Mainnet
   PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
   GAR_BASE: us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/zebra
+  GCR_BASE: gcr.io/${{ secrets.GCP_PROJECT_ID }}
   REGION: us-central1
   ZONE: us-central1-a
   MACHINE_TYPE: c2-standard-4
@@ -44,8 +45,6 @@ jobs:
     # Setup Docker Buildx to allow use of docker cache layers from GH
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v1
-      with:
-        driver-opts: network=host
 
     - name: Login to Google Artifact Registry
       uses: docker/login-action@v1.12.0
@@ -54,6 +53,13 @@ jobs:
         username: _json_key
         password: ${{ secrets.GOOGLE_CREDENTIALS }}
 
+    - name: Login to Google Container Registry
+      uses: docker/login-action@v1.12.0
+      with:
+        registry: gcr.io
+        username: _json_key
+        password: ${{ secrets.GOOGLE_CREDENTIALS }}
+
     # Build and push image to Google Artifact Registry
     - name: Build & push
       id: docker_build
@@ -65,6 +71,8 @@ jobs:
         tags: |
           ${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:latest
           ${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:${{ env.GITHUB_SHA_SHORT }}
+          ${{ env.GCR_BASE }}/$${{ env.GITHUB_REPOSITORY_SLUG_URL }}/{{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:latest
+          ${{ env.GCR_BASE }}/$${{ env.GITHUB_REPOSITORY_SLUG_URL }}/{{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:${{ env.GITHUB_SHA_SHORT }}
         build-args: |
           NETWORK=${{ github.event.inputs.network || env.NETWORK }}
           SHORT_SHA=${{ env.GITHUB_SHA_SHORT }}
@@ -73,8 +81,8 @@ jobs:
           CHECKPOINT_SYNC=${{ github.event.inputs.checkpoint_sync || true }}
           SENTRY_DSN=${{ secrets.SENTRY_ENDPOINT }}
         push: true
-        cache-from: type=registry,ref=${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:buildcache
-        cache-to: type=registry,ref=${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:buildcache,mode=max
+        cache-from: type=gha,scope=CD
+        cache-to: type=gha,mode=max,scope=CD
 
   deploy-nodes:
     name: Deploy Mainnet nodes

From 3ba0359e26ca4054e7e8a17ee2ea241057522f96 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Wed, 2 Feb 2022 14:02:09 -0400
Subject: [PATCH 058/124] fix(cd): typo and scope

---
 .github/workflows/cd.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index a66600e873f..b95bb13efa7 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -71,8 +71,8 @@ jobs:
         tags: |
           ${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:latest
           ${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:${{ env.GITHUB_SHA_SHORT }}
-          ${{ env.GCR_BASE }}/$${{ env.GITHUB_REPOSITORY_SLUG_URL }}/{{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:latest
-          ${{ env.GCR_BASE }}/$${{ env.GITHUB_REPOSITORY_SLUG_URL }}/{{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:${{ env.GITHUB_SHA_SHORT }}
+          ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/{{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:latest
+          ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/{{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:${{ env.GITHUB_SHA_SHORT }}
         build-args: |
           NETWORK=${{ github.event.inputs.network || env.NETWORK }}
           SHORT_SHA=${{ env.GITHUB_SHA_SHORT }}
@@ -81,8 +81,8 @@ jobs:
           CHECKPOINT_SYNC=${{ github.event.inputs.checkpoint_sync || true }}
           SENTRY_DSN=${{ secrets.SENTRY_ENDPOINT }}
         push: true
-        cache-from: type=gha,scope=CD
-        cache-to: type=gha,mode=max,scope=CD
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
 
   deploy-nodes:
     name: Deploy Mainnet nodes

From b8c43e335cbcc18b67887bf34c593cad662fe29f Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Wed, 2 Feb 2022 14:08:18 -0400
Subject: [PATCH 059/124] fix(cd): typos everywhere

---
 .github/workflows/cd.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index b95bb13efa7..ac0eb706d40 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -71,8 +71,8 @@ jobs:
         tags: |
           ${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:latest
           ${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:${{ env.GITHUB_SHA_SHORT }}
-          ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/{{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:latest
-          ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/{{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:${{ env.GITHUB_SHA_SHORT }}
+          ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:latest
+          ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:${{ env.GITHUB_SHA_SHORT }}
         build-args: |
           NETWORK=${{ github.event.inputs.network || env.NETWORK }}
           SHORT_SHA=${{ env.GITHUB_SHA_SHORT }}

From 3be6ba6e0d1478d4c7001b9810c84b677c4ee4f8 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Wed, 2 Feb 2022 18:37:00 -0400
Subject: [PATCH 060/124] refactor(test): use smarter docker wait and keep old
 registry

---
 .github/workflows/test.yml | 32 +++++++++++++++++++++-----------
 1 file changed, 21 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index c6ff1f84939..acdb52e1715 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -21,6 +21,7 @@ env:
   NETWORK: Testnet
   PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
   GAR_BASE: us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/zebra
+  GCR_BASE: gcr.io/${{ secrets.GCP_PROJECT_ID }}
   REGION: us-central1
   ZONE: us-central1-a
   MACHINE_TYPE: c2-standard-4
@@ -53,6 +54,13 @@ jobs:
           username: _json_key
           password: ${{ secrets.GOOGLE_CREDENTIALS }}
 
+      - name: Login to Google Container Registry
+        uses: docker/login-action@v1.12.0
+        with:
+          registry: gcr.io
+          username: _json_key
+          password: ${{ secrets.GOOGLE_CREDENTIALS }}
+
       # Build and push image to Google Artifact Registry
       - name: Build & push
         id: docker_build
@@ -64,6 +72,8 @@ jobs:
           tags: |
             ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:latest
             ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
+            ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.IMAGE_NAME }}:latest
+            ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
           build-args: |
             NETWORK=${{ github.event.inputs.network || env.NETWORK }}
             SHORT_SHA=${{ env.GITHUB_SHA_SHORT }}
@@ -73,14 +83,13 @@ jobs:
             RUST_LOG=debug
             SENTRY_DSN=${{ secrets.SENTRY_ENDPOINT }}
           push: true
-          cache-from: type=registry,ref=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:buildcache
-          cache-to: type=registry,ref=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:buildcache,mode=max
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
 
   test:
     name: Run all tests
     runs-on: ubuntu-latest
     needs: build
-
     steps:
       - uses: actions/checkout@v2.4.0
         with:
@@ -124,14 +133,15 @@ jobs:
           --tags zebrad \
           --zone "${{ env.ZONE }}"
 
-      - name: Wait regeneration for 2 hours
-        run: sleep 7200s
-        shell: bash
+      - name: Wait for sync process to end
+        run: |
+          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --ssh-flag="-o ServerAliveInterval=5" --zone "${{ env.ZONE }}" --command \
+          docker wait $(docker ps -aqf ancestor=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }})
 
       # Create image from disk that will be used to sync past mandatory checkpoint test
       - name: Create image from state disk
         # Only run if the earlier step succeeds
-        if: steps.regenerate-state.outcome == 'success'
+        if: github.event_name == 'workflow_dispatch' && steps.regenerate-state.outcome == 'success'
         run: |
           gcloud compute images create zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy --source-disk=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy --source-disk-zone=${{ env.ZONE }} --storage-location=us
 
@@ -146,7 +156,6 @@ jobs:
       # TODO: Remove hardcoded value zebrad-cache-1558f3378-mainnet-canopy
       - name: Sync past mandatory checkpoint
         id: sync-past-checkpoint
-        if: github.event_name == 'push'
         run: |
           gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
           --boot-disk-size 10GB \
@@ -165,9 +174,10 @@ jobs:
           --tags zebrad \
           --zone "${{ env.ZONE }}"
 
-      - name: Wait sync for 2 hours
-        run: sleep 7200s
-        shell: bash
+      - name: Wait for sync process to end
+        run: |
+          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --ssh-flag="-o ServerAliveInterval=5" --zone "${{ env.ZONE }}" --command \
+          docker wait $(docker ps -aqf ancestor=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }})
 
       - name: Delete test instance
         # Always run even if the earlier step fails

From 5989bb992537fc6b63f8cad7101ce956266e5bd3 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Wed, 2 Feb 2022 18:38:49 -0400
Subject: [PATCH 061/124] fix(cd): do not constraint the CPUs for bigger
 machines

---
 docker/Dockerfile.build | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index 2771a9a7031..890fcc822cb 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -33,16 +33,16 @@ ARG RUST_BACKTRACE=1
 ENV RUST_BACKTRACE ${RUST_BACKTRACE}
 
 COPY . .
-RUN cargo build --locked --jobs 2 --release --features enable-sentry --bin zebrad
+RUN cargo build --locked --release --features enable-sentry --bin zebrad
 
 FROM builder AS tester
 # Skip IPv6 tests by default, as some CI environment don't have IPv6 available
 ARG ZEBRA_SKIP_IPV6_TESTS=1
 ENV ZEBRA_SKIP_IPV6_TESTS ${ZEBRA_SKIP_IPV6_TESTS}
 
-RUN cargo test --locked --jobs 2 --release --features enable-sentry --workspace --no-run
+RUN cargo test --locked --release --features enable-sentry --workspace --no-run
 # Pre-download Zcash Sprout and Sapling parameters
-RUN cargo run --locked --jobs 2 --release --features enable-sentry --bin zebrad download
+RUN cargo run --locked --release --features enable-sentry --bin zebrad download
 
 CMD ["cargo" "test" "--locked" "--release" "--features" "enable-sentry" "--workspace"]
 

From de70c6fa21d38fa2d763ecc62705a2422decfd4d Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Wed, 2 Feb 2022 20:48:46 -0400
Subject: [PATCH 062/124] revert(cd): reduce PR diff as there's a separate one
 for tests

---
 .github/workflows/cd.yml                   |  6 +++---
 .github/workflows/test.yml                 | 20 ++++++--------------
 .github/workflows/zcashd-manual-deploy.yml | 16 ++++------------
 3 files changed, 13 insertions(+), 29 deletions(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index ac0eb706d40..a95a5452638 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -8,8 +8,8 @@ on:
       checkpoint_sync:
         default: true
   push:
-    # branches:
-    #   - main
+    branches:
+      - main
 
 env:
   CARGO_INCREMENTAL: '1'
@@ -24,7 +24,7 @@ env:
 jobs:
   build:
     name: Build images
-    timeout-minutes: 180
+    timeout-minutes: 60
     runs-on: ubuntu-latest
 
     steps:
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 85725cdd8ce..64e1712ff93 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -15,9 +15,8 @@ on:
 
 env:
   PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
-  REGION: us-central1
-  ZONE: us-central1-a
-  MACHINE_TYPE: c2-standard-4
+  ZONE: europe-west1-b
+  MACHINE_TYPE: n2-standard-8
   DEPLOY_SA: cos-vm@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com
 
 jobs:
@@ -33,17 +32,11 @@ jobs:
       - name: Inject slug/short variables
         uses: rlespinasse/github-slug-action@v4
 
-      # Setup gcloud CLI
-      - name: Authenticate to Google Cloud
-        id: auth
-        uses: google-github-actions/auth@v0.5.0
-        with:
-          credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
-
-      - name: Set up gcloud SDK environment
+      - name: Set up gcloud
         uses: google-github-actions/setup-gcloud@v0.4.0
         with:
           project_id: ${{ env.PROJECT_ID }}
+          service_account_key: ${{ secrets.GCLOUD_AUTH }}
 
       # Creates Compute Engine virtual machine instance w/ disks
       - name: Create instance
@@ -51,13 +44,12 @@ jobs:
           gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
           --boot-disk-size 100GB \
           --boot-disk-type pd-ssd \
-          --container-stdin \
-          --container-tty \
           --container-image rust:buster \
           --container-mount-disk mount-path='/mainnet',name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-mainnet-canopy" \
           --container-restart-policy never \
           --create-disk name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-mainnet-canopy",image=zebrad-cache-1558f3378-mainnet-canopy \
           --machine-type ${{ env.MACHINE_TYPE }} \
+          --service-account ${{ env.DEPLOY_SA }} \
           --scopes cloud-platform \
           --tags zebrad \
           --zone "${{ env.ZONE }}"
@@ -77,4 +69,4 @@ jobs:
         # Always run even if the earlier step fails
         if: ${{ always() }}
         run: |
-          gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"
+          gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"
\ No newline at end of file
diff --git a/.github/workflows/zcashd-manual-deploy.yml b/.github/workflows/zcashd-manual-deploy.yml
index 627dd4233e2..ba546212dc7 100644
--- a/.github/workflows/zcashd-manual-deploy.yml
+++ b/.github/workflows/zcashd-manual-deploy.yml
@@ -11,8 +11,7 @@ on:
 env:
   PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
   REGION: us-central1
-  ZONE: us-central1-a
-  MACHINE_TYPE: c2-standard-4
+  MACHINE_TYPE: n2-standard-4
   DEPLOY_SA: cos-vm@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com
 
 jobs:
@@ -30,28 +29,21 @@ jobs:
         uses: rlespinasse/github-slug-action@v4
 
       # Setup gcloud CLI
-      - name: Authenticate to Google Cloud
-        id: auth
-        uses: google-github-actions/auth@v0.5.0
-        with:
-          credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
-
       - name: Set up gcloud SDK environment
         uses: google-github-actions/setup-gcloud@v0.4.0
         with:
           project_id: ${{ env.PROJECT_ID }}
+          service_account_key: ${{ secrets.GCLOUD_AUTH }}
 
       # Create instance template from container image
       - name: Create instance template
         run: |
           gcloud compute instance-templates create-with-container "zcashd-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
           --boot-disk-size 100GB \
-          --boot-disk-type=pd-ssd \
-          --container-stdin \
-          --container-tty \
           --container-image "electriccoinco/zcashd" \
           --container-env ZCASHD_NETWORK="${{ github.event.inputs.network }}" \
           --machine-type ${{ env.MACHINE_TYPE }} \
+          --service-account ${{ env.DEPLOY_SA }} \
           --scopes cloud-platform \
           --tags zcashd \
 
@@ -79,4 +71,4 @@ jobs:
           gcloud compute instance-groups managed rolling-action start-update \
           "zcashd-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ github.event.inputs.network }}" \
           --version template="zcashd-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
-          --region "${{ env.REGION }}"
+          --region "${{ env.REGION }}"
\ No newline at end of file

From 38d8567c2af594d7c77cfb19aca3b691efc3c867 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Wed, 2 Feb 2022 22:41:23 -0400
Subject: [PATCH 063/124] fix(docker): add .github as it has no impact on
 caching

---
 .dockerignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.dockerignore b/.dockerignore
index 3645101174c..fae64da0fe6 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -2,4 +2,5 @@ target
 Dockerfile
 .dockerignore
 .git
+.github
 .gitignore

From a5c28556d16be4e20b0140cadfe2a5fdaae3fdda Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Wed, 2 Feb 2022 22:45:37 -0400
Subject: [PATCH 064/124] fix(test): run command correctly

---
 .github/workflows/test.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index acdb52e1715..34a1dcbe0af 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -135,8 +135,8 @@ jobs:
 
       - name: Wait for sync process to end
         run: |
-          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --ssh-flag="-o ServerAliveInterval=5" --zone "${{ env.ZONE }}" --command \
-          docker wait $(docker ps -aqf ancestor=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }})
+          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --ssh-flag="-o ServerAliveInterval=5" --zone ${{ env.ZONE }} 
+          --command="docker wait $(docker ps -aqf ancestor=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }})"
 
       # Create image from disk that will be used to sync past mandatory checkpoint test
       - name: Create image from state disk
@@ -176,8 +176,8 @@ jobs:
 
       - name: Wait for sync process to end
         run: |
-          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --ssh-flag="-o ServerAliveInterval=5" --zone "${{ env.ZONE }}" --command \
-          docker wait $(docker ps -aqf ancestor=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }})
+          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --ssh-flag="-o ServerAliveInterval=5" --zone "${{ env.ZONE }}" \
+          --command="docker wait $(docker ps -aqf ancestor=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }})"
 
       - name: Delete test instance
         # Always run even if the earlier step fails

From d851cf389d206c6619eefd7144c59e9eb4032778 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Wed, 2 Feb 2022 23:43:43 -0400
Subject: [PATCH 065/124] fix(test): wiat and create image if previous step
 succeded

---
 .github/workflows/test.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 34a1dcbe0af..89eb84e6f74 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -134,6 +134,8 @@ jobs:
           --zone "${{ env.ZONE }}"
 
       - name: Wait for sync process to end
+        id: wait-regenerate
+        if: github.event_name == 'workflow_dispatch' && steps.regenerate-state.outcome == 'success'
         run: |
           gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --ssh-flag="-o ServerAliveInterval=5" --zone ${{ env.ZONE }} 
           --command="docker wait $(docker ps -aqf ancestor=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }})"
@@ -141,7 +143,7 @@ jobs:
       # Create image from disk that will be used to sync past mandatory checkpoint test
       - name: Create image from state disk
         # Only run if the earlier step succeeds
-        if: github.event_name == 'workflow_dispatch' && steps.regenerate-state.outcome == 'success'
+        if: github.event_name == 'workflow_dispatch' && steps.wait-regenerate.outcome == 'success'
         run: |
           gcloud compute images create zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy --source-disk=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy --source-disk-zone=${{ env.ZONE }} --storage-location=us
 

From bfff02419fd1c6a4d615465e3146ff850300e439 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Thu, 3 Feb 2022 00:03:43 -0400
Subject: [PATCH 066/124] force rebuild

---
 docker/Dockerfile.build | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index 890fcc822cb..8a39c72c0a8 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -73,7 +73,6 @@ RUN set -ex; \
   } > "zebrad.toml"
 
 RUN cat zebrad.toml
-
 # Pre-download Zcash Sprout and Sapling parameters
 RUN /usr/local/bin/zebrad download
 

From 70a6b057dce3ca0152ade05bda3a578ed11828a3 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Thu, 3 Feb 2022 08:26:34 -0400
Subject: [PATCH 067/124] fix(test): do not restrict interdependant steps based
 on event

---
 .github/workflows/test.yml | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 89eb84e6f74..c380a5ce237 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -18,7 +18,7 @@ on:
 
 env:
   CARGO_INCREMENTAL: '1'
-  NETWORK: Testnet
+  NETWORK: Mainnet
   PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
   GAR_BASE: us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/zebra
   GCR_BASE: gcr.io/${{ secrets.GCP_PROJECT_ID }}
@@ -114,7 +114,6 @@ jobs:
 
       - name: Regenerate state test disk
         id: regenerate-state
-        if: github.event_name == 'workflow_dispatch'
         run: |
           gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
           --boot-disk-size 10GB \
@@ -135,7 +134,7 @@ jobs:
 
       - name: Wait for sync process to end
         id: wait-regenerate
-        if: github.event_name == 'workflow_dispatch' && steps.regenerate-state.outcome == 'success'
+        if: steps.regenerate-state.outcome == 'success'
         run: |
           gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --ssh-flag="-o ServerAliveInterval=5" --zone ${{ env.ZONE }} 
           --command="docker wait $(docker ps -aqf ancestor=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }})"
@@ -143,13 +142,12 @@ jobs:
       # Create image from disk that will be used to sync past mandatory checkpoint test
       - name: Create image from state disk
         # Only run if the earlier step succeeds
-        if: github.event_name == 'workflow_dispatch' && steps.wait-regenerate.outcome == 'success'
+        if: steps.wait-regenerate.outcome == 'success'
         run: |
           gcloud compute images create zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy --source-disk=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy --source-disk-zone=${{ env.ZONE }} --storage-location=us
 
       - name: Delete test instance
         # Always run even if the earlier step fails
-        if: github.event_name == 'workflow_dispatch'
         continue-on-error: true
         run: |
           gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"

From e4a2b09dc3e60edd1b934e0abd16108cb425f26b Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Thu, 3 Feb 2022 08:31:02 -0400
Subject: [PATCH 068/124] force push

---
 docker/Dockerfile.build | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index 8a39c72c0a8..b516a737fb6 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -84,4 +84,4 @@ ENV SHORT_SHA $SHORT_SHA
 ARG SENTRY_DSN
 ENV SENTRY_DSN ${SENTRY_DSN}
 
-CMD [ "/usr/local/bin/zebrad", "-c", "zebrad.toml", "start" ]
+CMD [ "/usr/local/bin/zebrad", "-c", "zebrad.toml", "start" ]
\ No newline at end of file

From 2f3f0fdbefeb4199b2eee0ee4b0e270cf3aa70ee Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Thu, 3 Feb 2022 11:43:16 -0400
Subject: [PATCH 069/124] feat(docker): add google OS Config agent

Use a separate step to have better flexibility in case a better approach is available
---
 docker/Dockerfile.build | 23 ++++++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index b516a737fb6..2d3d69d13e6 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -16,9 +16,26 @@ RUN cargo chef prepare --recipe-path recipe.json
 FROM chef AS builder
 COPY --from=planner /app/recipe.json recipe.json
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-    llvm libclang-dev clang ca-certificates && \
+# Install zebra build deps
+RUN apt-get -qq update && \
+    apt-get -qq install -y --no-install-recommends \
+    llvm \
+    libclang-dev \
+    clang \
+    ca-certificates \
+    ; \
+    rm -rf /var/lib/apt/lists/* /tmp/*
+
+# Install google OS Config agent
+RUN apt-get -qq update && \
+    apt-get -qq install -y --no-install-recommends \
+    curl \
+    lsb-release \
+    ; \
+    echo "deb http://packages.cloud.google.com/apt google-compute-engine-$(lsb_release -cs)-stable main" > /etc/apt/sources.list.d/google-compute-engine.list && \
+    curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \
+    apt-get -qq update  && \
+    apt-get -qq install -y --no-install-recommends google-osconfig-agent && \
     rm -rf /var/lib/apt/lists/* /tmp/*
 
 # Optimize builds. In particular, regenerate-stateful-test-disks.yml was reaching the

From 621d1f4ca9ac59e7fbd9ae7e649c765e8d8cbf13 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Thu, 3 Feb 2022 20:10:56 -0400
Subject: [PATCH 070/124] fix(test): remove all hardoced values and increase
 disks

---
 .dockerignore              |  2 +-
 .github/workflows/test.yml | 28 +++++++++++++---------------
 2 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/.dockerignore b/.dockerignore
index fae64da0fe6..797ef17b32f 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -2,5 +2,5 @@ target
 Dockerfile
 .dockerignore
 .git
-.github
+# .github
 .gitignore
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index c380a5ce237..73fcf0c2864 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
     inputs:
       network:
-        default: 'Mainnet'
+        default: 'mainnet'
   push:
     # branches:
     #   - main
@@ -18,7 +18,7 @@ on:
 
 env:
   CARGO_INCREMENTAL: '1'
-  NETWORK: Mainnet
+  NETWORK: mainnet
   PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
   GAR_BASE: us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/zebra
   GCR_BASE: gcr.io/${{ secrets.GCP_PROJECT_ID }}
@@ -28,7 +28,6 @@ env:
   IMAGE_NAME: zebrad-test
 
 jobs:
-
   build:
     name: Build images
     timeout-minutes: 210
@@ -86,8 +85,8 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
-  test:
-    name: Run all tests
+  regenerate-test:
+    name: Renerate disks and test
     runs-on: ubuntu-latest
     needs: build
     steps:
@@ -116,16 +115,16 @@ jobs:
         id: regenerate-state
         run: |
           gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
-          --boot-disk-size 10GB \
+          --boot-disk-size 100GB \
           --boot-disk-type pd-ssd \
-          --create-disk name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy",size=100GB,type=pd-ssd \
+          --create-disk name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network || env.NETWORK }}-canopy",size=100GB,type=pd-ssd \
+          --container-mount-disk mount-path='/${{ github.event.inputs.network || env.NETWORK }}',name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network || env.NETWORK }}-canopy" \
           --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} \
           --container-restart-policy=never \
           --container-stdin \
           --container-tty \
-          --container-mount-disk mount-path='/zebrad-cache',name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy" \
           --container-command "cargo" \
-          --container-arg="test --verbose --features test_sync_to_mandatory_checkpoint_mainnet --manifest-path zebrad/Cargo.toml sync_to_mandatory_checkpoint_mainnet" \
+          --container-arg="test --verbose --features test_sync_to_mandatory_checkpoint_${{ github.event.inputs.network || env.NETWORK }} --manifest-path zebrad/Cargo.toml sync_to_mandatory_checkpoint_${{ github.event.inputs.network || env.NETWORK }}" \
           --container-env=ZEBRA_SKIP_IPV6_TESTS=1 \
           --machine-type ${{ env.MACHINE_TYPE }} \
           --scopes cloud-platform \
@@ -144,7 +143,7 @@ jobs:
         # Only run if the earlier step succeeds
         if: steps.wait-regenerate.outcome == 'success'
         run: |
-          gcloud compute images create zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy --source-disk=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy --source-disk-zone=${{ env.ZONE }} --storage-location=us
+          gcloud compute images create zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network || env.NETWORK }}-canopy --source-disk=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network || env.NETWORK }}-canopy --source-disk-zone=${{ env.ZONE }} --storage-location=us
 
       - name: Delete test instance
         # Always run even if the earlier step fails
@@ -153,21 +152,20 @@ jobs:
           gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"
 
       # Creates Compute Engine virtual machine instance w/ disks
-      # TODO: Remove hardcoded value zebrad-cache-1558f3378-mainnet-canopy
       - name: Sync past mandatory checkpoint
         id: sync-past-checkpoint
         run: |
           gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
-          --boot-disk-size 10GB \
+          --boot-disk-size 100GB \
           --boot-disk-type pd-ssd \
-          --disk=boot=no,mode=rw,name=zebrad-cache-1558f3378-mainnet-canopy \
-          --container-mount-disk=mount-path=/zebrad-cache,name=zebrad-cache-1558f3378-mainnet-canopy \
+          --disk=boot=no,mode=rw,name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network || env.NETWORK }}-canopy \
+          --container-mount-disk=mount-path=/${{ github.event.inputs.network || env.NETWORK }},name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network || env.NETWORK }}-canopy \
           --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} \
           --container-restart-policy=never \
           --container-stdin \
           --container-tty \
           --container-command "cargo" \
-          --container-arg="test --locked --release --features enable-sentry,test_sync_past_mandatory_checkpoint_mainnet --manifest-path zebrad/Cargo.toml sync_past_mandatory_checkpoint_mainnet" \
+          --container-arg="test --locked --release --features enable-sentry,test_sync_past_mandatory_checkpoint_${{ github.event.inputs.network || env.NETWORK }} --manifest-path zebrad/Cargo.toml sync_past_mandatory_checkpoint_${{ github.event.inputs.network || env.NETWORK }}" \
           --container-env=ZEBRA_SKIP_IPV6_TESTS=1 \
           --machine-type ${{ env.MACHINE_TYPE }} \
           --scopes cloud-platform \

From d8c0a12edd27e8f0536b7b87bcb6a4aed60a7c02 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Thu, 3 Feb 2022 21:16:57 -0400
Subject: [PATCH 071/124] fix(test): use correct commands on deploy

---
 .github/workflows/test.yml | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 73fcf0c2864..5ce3f8a9794 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -123,8 +123,7 @@ jobs:
           --container-restart-policy=never \
           --container-stdin \
           --container-tty \
-          --container-command "cargo" \
-          --container-arg="test --verbose --features test_sync_to_mandatory_checkpoint_${{ github.event.inputs.network || env.NETWORK }} --manifest-path zebrad/Cargo.toml sync_to_mandatory_checkpoint_${{ github.event.inputs.network || env.NETWORK }}" \
+          --container-command "cargo test --verbose --features test_sync_to_mandatory_checkpoint_${{ github.event.inputs.network || env.NETWORK }} --manifest-path zebrad/Cargo.toml sync_to_mandatory_checkpoint_${{ github.event.inputs.network || env.NETWORK }}" \
           --container-env=ZEBRA_SKIP_IPV6_TESTS=1 \
           --machine-type ${{ env.MACHINE_TYPE }} \
           --scopes cloud-platform \
@@ -136,7 +135,7 @@ jobs:
         if: steps.regenerate-state.outcome == 'success'
         run: |
           gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --ssh-flag="-o ServerAliveInterval=5" --zone ${{ env.ZONE }} 
-          --command="docker wait $(docker ps -aqf ancestor=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }})"
+          --command='docker wait $(docker ps -aqf ancestor=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }})'
 
       # Create image from disk that will be used to sync past mandatory checkpoint test
       - name: Create image from state disk
@@ -164,8 +163,7 @@ jobs:
           --container-restart-policy=never \
           --container-stdin \
           --container-tty \
-          --container-command "cargo" \
-          --container-arg="test --locked --release --features enable-sentry,test_sync_past_mandatory_checkpoint_${{ github.event.inputs.network || env.NETWORK }} --manifest-path zebrad/Cargo.toml sync_past_mandatory_checkpoint_${{ github.event.inputs.network || env.NETWORK }}" \
+          --container-command "cargo test --locked --release --features enable-sentry,test_sync_past_mandatory_checkpoint_${{ github.event.inputs.network || env.NETWORK }} --manifest-path zebrad/Cargo.toml sync_past_mandatory_checkpoint_${{ github.event.inputs.network || env.NETWORK }}" \
           --container-env=ZEBRA_SKIP_IPV6_TESTS=1 \
           --machine-type ${{ env.MACHINE_TYPE }} \
           --scopes cloud-platform \
@@ -175,7 +173,7 @@ jobs:
       - name: Wait for sync process to end
         run: |
           gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --ssh-flag="-o ServerAliveInterval=5" --zone "${{ env.ZONE }}" \
-          --command="docker wait $(docker ps -aqf ancestor=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }})"
+          --command='docker wait $(docker ps -aqf ancestor=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }})'
 
       - name: Delete test instance
         # Always run even if the earlier step fails

From 9a4c23fb0b61f066d3858f0c4c3e41ea1fe0724a Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Fri, 4 Feb 2022 03:10:51 -0400
Subject: [PATCH 072/124] fix(test): use args as required by google

---
 .github/workflows/test.yml | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 5ce3f8a9794..87c938bb73b 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -118,12 +118,19 @@ jobs:
           --boot-disk-size 100GB \
           --boot-disk-type pd-ssd \
           --create-disk name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network || env.NETWORK }}-canopy",size=100GB,type=pd-ssd \
-          --container-mount-disk mount-path='/${{ github.event.inputs.network || env.NETWORK }}',name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network || env.NETWORK }}-canopy" \
+          --container-mount-disk mount-path='/zebrad-cache,name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network || env.NETWORK }}-canopy" \
           --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} \
           --container-restart-policy=never \
           --container-stdin \
           --container-tty \
-          --container-command "cargo test --verbose --features test_sync_to_mandatory_checkpoint_${{ github.event.inputs.network || env.NETWORK }} --manifest-path zebrad/Cargo.toml sync_to_mandatory_checkpoint_${{ github.event.inputs.network || env.NETWORK }}" \
+          --container-command="cargo" \
+          --container-arg="test" \
+          --container-arg="--verbose" \
+          --container-arg="--features" \
+          --container-arg="enable-sentry,test_sync_to_mandatory_checkpoint_${{ github.event.inputs.network || env.NETWORK }}" \
+          --container-arg="--manifest-path" \
+          --container-arg="zebrad/Cargo.toml" \
+          --container-arg="sync_to_mandatory_checkpoint_${{ github.event.inputs.network || env.NETWORK }}" \
           --container-env=ZEBRA_SKIP_IPV6_TESTS=1 \
           --machine-type ${{ env.MACHINE_TYPE }} \
           --scopes cloud-platform \
@@ -134,7 +141,7 @@ jobs:
         id: wait-regenerate
         if: steps.regenerate-state.outcome == 'success'
         run: |
-          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --ssh-flag="-o ServerAliveInterval=5" --zone ${{ env.ZONE }} 
+          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --ssh-flag="-o ServerAliveInterval=5" --zone ${{ env.ZONE }} \
           --command='docker wait $(docker ps -aqf ancestor=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }})'
 
       # Create image from disk that will be used to sync past mandatory checkpoint test
@@ -158,12 +165,20 @@ jobs:
           --boot-disk-size 100GB \
           --boot-disk-type pd-ssd \
           --disk=boot=no,mode=rw,name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network || env.NETWORK }}-canopy \
-          --container-mount-disk=mount-path=/${{ github.event.inputs.network || env.NETWORK }},name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network || env.NETWORK }}-canopy \
+          --container-mount-disk=mount-path='/zebrad-cache',name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network || env.NETWORK }}-canopy \
           --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} \
           --container-restart-policy=never \
           --container-stdin \
           --container-tty \
-          --container-command "cargo test --locked --release --features enable-sentry,test_sync_past_mandatory_checkpoint_${{ github.event.inputs.network || env.NETWORK }} --manifest-path zebrad/Cargo.toml sync_past_mandatory_checkpoint_${{ github.event.inputs.network || env.NETWORK }}" \
+          --container-command="cargo" \
+          --container-arg="test" \
+          --container-arg="--locked" \
+          --container-arg="--release" \
+          --container-arg="--features" \
+          --container-arg="enable-sentry,test_sync_past_mandatory_checkpoint_${{ github.event.inputs.network || env.NETWORK }}" \
+          --container-arg="--manifest-path" \
+          --container-arg="zebrad/Cargo.toml" \
+          --container-arg="sync_past_mandatory_checkpoint_${{ github.event.inputs.network || env.NETWORK }}" \
           --container-env=ZEBRA_SKIP_IPV6_TESTS=1 \
           --machine-type ${{ env.MACHINE_TYPE }} \
           --scopes cloud-platform \

From a7bb2d706cad08c610afa407dc0437d7fbe4439b Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Fri, 4 Feb 2022 11:11:44 -0400
Subject: [PATCH 073/124] fix(docker): try not to invalidate zebrad download
 cache

---
 .dockerignore           | 2 +-
 docker/Dockerfile.build | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.dockerignore b/.dockerignore
index 797ef17b32f..fae64da0fe6 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -2,5 +2,5 @@ target
 Dockerfile
 .dockerignore
 .git
-# .github
+.github
 .gitignore
diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index 2d3d69d13e6..ab2991034fa 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -45,11 +45,13 @@ ENV CARGO_HOME /app/.cargo/
 # Build dependencies - this is the caching Docker layer!
 RUN cargo chef cook --release --features enable-sentry --recipe-path recipe.json
 
-# Build zebra
 ARG RUST_BACKTRACE=1
 ENV RUST_BACKTRACE ${RUST_BACKTRACE}
 
 COPY . .
+# Pre-download Zcash Sprout and Sapling parameters
+RUN cargo run --locked --release --features enable-sentry --bin zebrad download
+# Build zebra
 RUN cargo build --locked --release --features enable-sentry --bin zebrad
 
 FROM builder AS tester
@@ -58,8 +60,6 @@ ARG ZEBRA_SKIP_IPV6_TESTS=1
 ENV ZEBRA_SKIP_IPV6_TESTS ${ZEBRA_SKIP_IPV6_TESTS}
 
 RUN cargo test --locked --release --features enable-sentry --workspace --no-run
-# Pre-download Zcash Sprout and Sapling parameters
-RUN cargo run --locked --release --features enable-sentry --bin zebrad download
 
 CMD ["cargo" "test" "--locked" "--release" "--features" "enable-sentry" "--workspace"]
 

From ad468dbddea5a8ab67aeaeba27d349d7b7c4bda4 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Fri, 4 Feb 2022 11:12:04 -0400
Subject: [PATCH 074/124] fix(test): minor typo

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 87c938bb73b..9fdafb35437 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -118,7 +118,7 @@ jobs:
           --boot-disk-size 100GB \
           --boot-disk-type pd-ssd \
           --create-disk name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network || env.NETWORK }}-canopy",size=100GB,type=pd-ssd \
-          --container-mount-disk mount-path='/zebrad-cache,name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network || env.NETWORK }}-canopy" \
+          --container-mount-disk mount-path='/zebrad-cache',name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network || env.NETWORK }}-canopy" \
           --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} \
           --container-restart-policy=never \
           --container-stdin \

From 40ebfebba9da2cd61ffbe1a9d6141cbf7eac4597 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Fri, 4 Feb 2022 12:23:10 -0400
Subject: [PATCH 075/124] refactor(test): decouple jobs for better modularity

This also allows faster tests as testing Zunstable won't be a dependency and it can't stop already started jobs if it fails.
---
 .github/workflows/test.yml | 54 ++++++++++++++++++++++++++++++--------
 1 file changed, 43 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 9fdafb35437..1cf81b17cde 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -85,7 +85,26 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
-  regenerate-test:
+  test-zunstable:
+    name: Test with Zunstable options
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - uses: actions/checkout@v2.4.0
+        with:
+          persist-credentials: false
+
+      - name: Inject slug/short variables
+        uses: rlespinasse/github-slug-action@v4
+
+      - name: Run Zunstable-options tests
+        run: |
+          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
+          docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --workspace -- -Zunstable-options --include-ignored
+        env:
+          ZEBRA_SKIP_IPV6_TESTS: "1"
+
+  regenerate-disks:
     name: Renerate disks and test
     runs-on: ubuntu-latest
     needs: build
@@ -104,13 +123,6 @@ jobs:
         with:
           credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
 
-      - name: Run workspace tests with Zunstable-options
-        run: |
-          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
-          docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --workspace -- -Zunstable-options --include-ignored
-        env:
-          ZEBRA_SKIP_IPV6_TESTS: "1"
-
       - name: Regenerate state test disk
         id: regenerate-state
         run: |
@@ -137,7 +149,7 @@ jobs:
           --tags zebrad \
           --zone "${{ env.ZONE }}"
 
-      - name: Wait for sync process to end
+      - name: Wait regeneration process to end
         id: wait-regenerate
         if: steps.regenerate-state.outcome == 'success'
         run: |
@@ -153,10 +165,30 @@ jobs:
 
       - name: Delete test instance
         # Always run even if the earlier step fails
+        if: ${{ always() }}
         continue-on-error: true
         run: |
           gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"
 
+  test-sync:
+    name: Test blocks sync
+    runs-on: ubuntu-latest
+    needs: [ build, regenerate-disks]
+    steps:
+      - uses: actions/checkout@v2.4.0
+        with:
+          persist-credentials: false
+
+      - name: Inject slug/short variables
+        uses: rlespinasse/github-slug-action@v4
+
+      # Setup gcloud CLI
+      - name: Authenticate to Google Cloud
+        id: auth
+        uses: google-github-actions/auth@v0.5.0
+        with:
+          credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
+
       # Creates Compute Engine virtual machine instance w/ disks
       - name: Sync past mandatory checkpoint
         id: sync-past-checkpoint
@@ -185,14 +217,14 @@ jobs:
           --tags zebrad \
           --zone "${{ env.ZONE }}"
 
-      - name: Wait for sync process to end
+      - name: Wait sync process to end
         run: |
           gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --ssh-flag="-o ServerAliveInterval=5" --zone "${{ env.ZONE }}" \
           --command='docker wait $(docker ps -aqf ancestor=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }})'
 
       - name: Delete test instance
         # Always run even if the earlier step fails
-        if: steps.sync-past-checkpoint.outcome == 'success'
+        if: ${{ always() }}
         continue-on-error: true
         run: |
           gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"

From c97fc7562f088a05700ccb2036bba87bda1cb5a1 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Fri, 4 Feb 2022 12:32:41 -0400
Subject: [PATCH 076/124] fix(test): Do not try to execute ss and commands in
 one line

---
 .github/workflows/test.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 1cf81b17cde..de5cfa460c6 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -153,8 +153,8 @@ jobs:
         id: wait-regenerate
         if: steps.regenerate-state.outcome == 'success'
         run: |
-          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --ssh-flag="-o ServerAliveInterval=5" --zone ${{ env.ZONE }} \
-          --command='docker wait $(docker ps -aqf ancestor=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }})'
+          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --ssh-flag="-o ServerAliveInterval=5" --zone ${{ env.ZONE }}
+          docker wait $(docker ps -aqf ancestor=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }})
 
       # Create image from disk that will be used to sync past mandatory checkpoint test
       - name: Create image from state disk
@@ -219,8 +219,8 @@ jobs:
 
       - name: Wait sync process to end
         run: |
-          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --ssh-flag="-o ServerAliveInterval=5" --zone "${{ env.ZONE }}" \
-          --command='docker wait $(docker ps -aqf ancestor=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }})'
+          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --ssh-flag="-o ServerAliveInterval=5" --zone "${{ env.ZONE }}"
+          docker wait $(docker ps -aqf ancestor=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }})
 
       - name: Delete test instance
         # Always run even if the earlier step fails

From e4681e0840d5044a0bebcb48ff1a01b951ae0b75 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Fri, 4 Feb 2022 12:47:44 -0400
Subject: [PATCH 077/124] fix(test): do not show undeeded information in the
 terminal

---
 .github/workflows/test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index de5cfa460c6..ae51dcb73ac 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -153,7 +153,7 @@ jobs:
         id: wait-regenerate
         if: steps.regenerate-state.outcome == 'success'
         run: |
-          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --ssh-flag="-o ServerAliveInterval=5" --zone ${{ env.ZONE }}
+          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --quiet --force-key-file-overwrite --ssh-flag="-o ServerAliveInterval=5" --zone ${{ env.ZONE }}
           docker wait $(docker ps -aqf ancestor=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }})
 
       # Create image from disk that will be used to sync past mandatory checkpoint test
@@ -219,7 +219,7 @@ jobs:
 
       - name: Wait sync process to end
         run: |
-          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --ssh-flag="-o ServerAliveInterval=5" --zone "${{ env.ZONE }}"
+          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --quiet --force-key-file-overwrite --ssh-flag="-o ServerAliveInterval=5" --zone ${{ env.ZONE }}
           docker wait $(docker ps -aqf ancestor=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }})
 
       - name: Delete test instance

From a1e11d288c788c2c2e6fc99e5485346ce6587197 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Fri, 4 Feb 2022 13:46:23 -0400
Subject: [PATCH 078/124] fix(test): sleep befor/after machine
 creation/deletion

---
 .github/workflows/test.yml | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index ae51dcb73ac..b986ca15cf2 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -105,7 +105,7 @@ jobs:
           ZEBRA_SKIP_IPV6_TESTS: "1"
 
   regenerate-disks:
-    name: Renerate disks and test
+    name: Renerate state disks
     runs-on: ubuntu-latest
     needs: build
     steps:
@@ -123,7 +123,7 @@ jobs:
         with:
           credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
 
-      - name: Regenerate state test disk
+      - name: Run state disk regeneration
         id: regenerate-state
         run: |
           gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
@@ -153,7 +153,8 @@ jobs:
         id: wait-regenerate
         if: steps.regenerate-state.outcome == 'success'
         run: |
-          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --quiet --force-key-file-overwrite --ssh-flag="-o ServerAliveInterval=5" --zone ${{ env.ZONE }}
+          sleep 30s
+          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --quiet --ssh-flag="-t" --ssh-flag="-o ServerAliveInterval=5"
           docker wait $(docker ps -aqf ancestor=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }})
 
       # Create image from disk that will be used to sync past mandatory checkpoint test
@@ -169,6 +170,7 @@ jobs:
         continue-on-error: true
         run: |
           gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"
+          sleep 60s
 
   test-sync:
     name: Test blocks sync
@@ -219,7 +221,8 @@ jobs:
 
       - name: Wait sync process to end
         run: |
-          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --quiet --force-key-file-overwrite --ssh-flag="-o ServerAliveInterval=5" --zone ${{ env.ZONE }}
+          sleep 30s
+          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --quiet --ssh-flag="-t" --ssh-flag="-o ServerAliveInterval=5"
           docker wait $(docker ps -aqf ancestor=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }})
 
       - name: Delete test instance

From b7e2f103291502e83949f07605bed298053555bd Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Fri, 4 Feb 2022 15:02:09 -0400
Subject: [PATCH 079/124] fix(docker): do not download zcash params twice

---
 docker/Dockerfile.build | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index 890fcc822cb..a111d336eab 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -16,9 +16,14 @@ RUN cargo chef prepare --recipe-path recipe.json
 FROM chef AS builder
 COPY --from=planner /app/recipe.json recipe.json
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-    llvm libclang-dev clang ca-certificates && \
+# Install zebra build deps
+RUN apt-get -qq update && \
+    apt-get -qq install -y --no-install-recommends \
+    llvm \
+    libclang-dev \
+    clang \
+    ca-certificates \
+    ; \
     rm -rf /var/lib/apt/lists/* /tmp/*
 
 # Optimize builds. In particular, regenerate-stateful-test-disks.yml was reaching the
@@ -28,11 +33,13 @@ ENV CARGO_HOME /app/.cargo/
 # Build dependencies - this is the caching Docker layer!
 RUN cargo chef cook --release --features enable-sentry --recipe-path recipe.json
 
-# Build zebra
 ARG RUST_BACKTRACE=1
 ENV RUST_BACKTRACE ${RUST_BACKTRACE}
 
 COPY . .
+# Pre-download Zcash Sprout and Sapling parameters
+RUN cargo run --locked --release --features enable-sentry --bin zebrad download
+# Build zebra
 RUN cargo build --locked --release --features enable-sentry --bin zebrad
 
 FROM builder AS tester
@@ -41,20 +48,18 @@ ARG ZEBRA_SKIP_IPV6_TESTS=1
 ENV ZEBRA_SKIP_IPV6_TESTS ${ZEBRA_SKIP_IPV6_TESTS}
 
 RUN cargo test --locked --release --features enable-sentry --workspace --no-run
-# Pre-download Zcash Sprout and Sapling parameters
-RUN cargo run --locked --release --features enable-sentry --bin zebrad download
 
 CMD ["cargo" "test" "--locked" "--release" "--features" "enable-sentry" "--workspace"]
 
 # Runner image
 FROM debian:bullseye-slim AS runtime
+COPY --from=builder /app/target/release/zebrad /usr/local/bin
+COPY --from=builder $HOME/.zcash-params $HOME/.zcash-params
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     ca-certificates
 
-COPY --from=builder /app/target/release/zebrad /usr/local/bin
-
 ARG CHECKPOINT_SYNC=true
 ARG NETWORK=Mainnet
 
@@ -72,11 +77,6 @@ RUN set -ex; \
     echo "endpoint_addr = '0.0.0.0:3000'"; \
   } > "zebrad.toml"
 
-RUN cat zebrad.toml
-
-# Pre-download Zcash Sprout and Sapling parameters
-RUN /usr/local/bin/zebrad download
-
 EXPOSE 3000 8233 18233
 
 ARG SHORT_SHA

From bb5623e3932e969d73659c49887f1e0b6767cf58 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Fri, 4 Feb 2022 15:05:03 -0400
Subject: [PATCH 080/124] feat(docker): add google OS Config agent

Use a separate step to have better flexibility in case a better approach is available
---
 docker/Dockerfile.build | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index a111d336eab..bbb870076ea 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -26,6 +26,18 @@ RUN apt-get -qq update && \
     ; \
     rm -rf /var/lib/apt/lists/* /tmp/*
 
+# Install google OS Config agent
+RUN apt-get -qq update && \
+    apt-get -qq install -y --no-install-recommends \
+    curl \
+    lsb-release \
+    ; \
+    echo "deb http://packages.cloud.google.com/apt google-compute-engine-$(lsb_release -cs)-stable main" > /etc/apt/sources.list.d/google-compute-engine.list && \
+    curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \
+    apt-get -qq update  && \
+    apt-get -qq install -y --no-install-recommends google-osconfig-agent && \
+    rm -rf /var/lib/apt/lists/* /tmp/*
+
 # Optimize builds. In particular, regenerate-stateful-test-disks.yml was reaching the
 # GitHub Actions time limit (6 hours), so we needed to make it faster.
 ENV RUSTFLAGS -O

From f8bcbc4323f4347e8b6721ee5f0a1d6b81154b96 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Fri, 4 Feb 2022 15:07:19 -0400
Subject: [PATCH 081/124] merge: docker-actions-refactor into
 docker-test-refactor

---
 docker/Dockerfile.build | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index ab2991034fa..bbb870076ea 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -65,13 +65,13 @@ CMD ["cargo" "test" "--locked" "--release" "--features" "enable-sentry" "--works
 
 # Runner image
 FROM debian:bullseye-slim AS runtime
+COPY --from=builder /app/target/release/zebrad /usr/local/bin
+COPY --from=builder $HOME/.zcash-params $HOME/.zcash-params
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     ca-certificates
 
-COPY --from=builder /app/target/release/zebrad /usr/local/bin
-
 ARG CHECKPOINT_SYNC=true
 ARG NETWORK=Mainnet
 
@@ -89,10 +89,6 @@ RUN set -ex; \
     echo "endpoint_addr = '0.0.0.0:3000'"; \
   } > "zebrad.toml"
 
-RUN cat zebrad.toml
-# Pre-download Zcash Sprout and Sapling parameters
-RUN /usr/local/bin/zebrad download
-
 EXPOSE 3000 8233 18233
 
 ARG SHORT_SHA
@@ -101,4 +97,4 @@ ENV SHORT_SHA $SHORT_SHA
 ARG SENTRY_DSN
 ENV SENTRY_DSN ${SENTRY_DSN}
 
-CMD [ "/usr/local/bin/zebrad", "-c", "zebrad.toml", "start" ]
\ No newline at end of file
+CMD [ "/usr/local/bin/zebrad", "-c", "zebrad.toml", "start" ]

From 8a0925b82a8dd03637a57b29291f012cf9fe7c14 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Fri, 4 Feb 2022 15:29:11 -0400
Subject: [PATCH 082/124] test docker wait scenarios

---
 .github/workflows/test.yml | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index b986ca15cf2..3dfbdb5e5c4 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -155,7 +155,13 @@ jobs:
         run: |
           sleep 30s
           gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --quiet --ssh-flag="-t" --ssh-flag="-o ServerAliveInterval=5"
-          docker wait $(docker ps -aqf ancestor=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }})
+          echo $CONTAINER_IMAGE
+          echo $(docker ps -aqf ancestor=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }})
+          docker ps -aqf ancestor=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
+          docker ps -aqf ancestor=$CONTAINER_IMAGE | tee output.txt
+          docker wait $(cat output.txt)
+        env:
+          CONTAINER_IMAGE: ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
 
       # Create image from disk that will be used to sync past mandatory checkpoint test
       - name: Create image from state disk

From 9f6bc796bbc514ddda0a30a639f1e400d810e3af Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Fri, 4 Feb 2022 16:08:26 -0400
Subject: [PATCH 083/124] fix(docker): $HOME variables is not being expanded

---
 docker/Dockerfile.build | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index bbb870076ea..98aecb35317 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -66,7 +66,7 @@ CMD ["cargo" "test" "--locked" "--release" "--features" "enable-sentry" "--works
 # Runner image
 FROM debian:bullseye-slim AS runtime
 COPY --from=builder /app/target/release/zebrad /usr/local/bin
-COPY --from=builder $HOME/.zcash-params $HOME/.zcash-params
+COPY --from=builder ~/.zcash-params ~/.zcash-params
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \

From 7227fe76aecb898d242e4ede293efd976efa56c8 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Fri, 4 Feb 2022 16:45:26 -0400
Subject: [PATCH 084/124] fix(test): allow docker wait to work correctly

---
 .github/workflows/test.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 3dfbdb5e5c4..69ea41e80b9 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -155,9 +155,6 @@ jobs:
         run: |
           sleep 30s
           gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --quiet --ssh-flag="-t" --ssh-flag="-o ServerAliveInterval=5"
-          echo $CONTAINER_IMAGE
-          echo $(docker ps -aqf ancestor=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }})
-          docker ps -aqf ancestor=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
           docker ps -aqf ancestor=$CONTAINER_IMAGE | tee output.txt
           docker wait $(cat output.txt)
         env:
@@ -229,7 +226,10 @@ jobs:
         run: |
           sleep 30s
           gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --quiet --ssh-flag="-t" --ssh-flag="-o ServerAliveInterval=5"
-          docker wait $(docker ps -aqf ancestor=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }})
+          docker ps -aqf ancestor=$CONTAINER_IMAGE | tee output.txt
+          docker wait $(cat output.txt)
+        env:
+          CONTAINER_IMAGE: ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
 
       - name: Delete test instance
         # Always run even if the earlier step fails

From 66a77fad61ef79807bfce94a3c607557f6c9fc49 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Fri, 4 Feb 2022 17:23:52 -0400
Subject: [PATCH 085/124] fix(docker): do not use variables while using COPY

---
 docker/Dockerfile.build | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index 98aecb35317..610421c16e3 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -66,7 +66,8 @@ CMD ["cargo" "test" "--locked" "--release" "--features" "enable-sentry" "--works
 # Runner image
 FROM debian:bullseye-slim AS runtime
 COPY --from=builder /app/target/release/zebrad /usr/local/bin
-COPY --from=builder ~/.zcash-params ~/.zcash-params
+# TODO: do not hardcode the user /root/ even though is a safe assumption
+COPY --from=builder /root/.zcash-params /root/.zcash-params
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \

From aa48d2c66786461315e5f76c6beb87111e402e5a Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Fri, 4 Feb 2022 17:24:22 -0400
Subject: [PATCH 086/124] fix(docker): allow to use zebrad as a command

---
 docker/Dockerfile.build | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index 610421c16e3..af26b7fa428 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -98,4 +98,4 @@ ENV SHORT_SHA $SHORT_SHA
 ARG SENTRY_DSN
 ENV SENTRY_DSN ${SENTRY_DSN}
 
-CMD [ "/usr/local/bin/zebrad", "-c", "zebrad.toml", "start" ]
+CMD [ "zebrad", "-c", "zebrad.toml", "start" ]

From c89c9662ccaaaac609008156ece6ef821fc5f50f Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Fri, 4 Feb 2022 17:36:03 -0400
Subject: [PATCH 087/124] fix(cd): use test .yml from main

---
 .../regenerate-stateful-test-disks.yml        | 19 ++++++-------------
 .github/workflows/test.yml                    |  2 +-
 2 files changed, 7 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/regenerate-stateful-test-disks.yml b/.github/workflows/regenerate-stateful-test-disks.yml
index 67c8b693a1a..e81bcdaae9e 100644
--- a/.github/workflows/regenerate-stateful-test-disks.yml
+++ b/.github/workflows/regenerate-stateful-test-disks.yml
@@ -4,13 +4,12 @@ on:
   workflow_dispatch:
     inputs:
       network:
-        default: 'Mainnet'
+        default: 'mainnet'
 
 env:
   PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
-  REGION: us-central1
-  ZONE: us-central1-a
-  MACHINE_TYPE: c2-standard-4
+  ZONE: europe-west1-b
+  MACHINE_TYPE: n2-standard-4
   DEPLOY_SA: cos-vm@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com
 
 jobs:
@@ -26,17 +25,11 @@ jobs:
       - name: Inject slug/short variables
         uses: rlespinasse/github-slug-action@v4
 
-      # Setup gcloud CLI
-      - name: Authenticate to Google Cloud
-        id: auth
-        uses: google-github-actions/auth@v0.5.0
-        with:
-          credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
-
-      - name: Set up gcloud SDK environment
+      - name: Set up gcloud
         uses: google-github-actions/setup-gcloud@v0.4.0
         with:
           project_id: ${{ env.PROJECT_ID }}
+          service_account_key: ${{ secrets.GCLOUD_AUTH }}
 
       # Creates Compute Engine virtual machine instance w/ disks
       - name: Create instance
@@ -47,7 +40,7 @@ jobs:
           --container-image rust:buster \
           --container-mount-disk mount-path='/${{ github.event.inputs.network }}',name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy" \
           --container-restart-policy never \
-          --create-disk name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy",size=100GB,type=pd-ssd \
+          --create-disk name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy",size=100GB,type=pd-balanced \
           --machine-type ${{ env.MACHINE_TYPE }} \
           --service-account ${{ env.DEPLOY_SA }} \
           --scopes cloud-platform \
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 64e1712ff93..39dca79928f 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -69,4 +69,4 @@ jobs:
         # Always run even if the earlier step fails
         if: ${{ always() }}
         run: |
-          gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"
\ No newline at end of file
+          gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"

From 8bbc8bce016f970ff74ae6739d2f638de1ae8c24 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Fri, 4 Feb 2022 17:46:22 -0400
Subject: [PATCH 088/124] fix(cd): Do not duplicate network values

The Dockerfile has an ARG with a default value of 'Mainnet', if this value is changed it will be done manually on a workflow_dispatch, making the ENV option a uneeded duplicate in this workflow
---
 .github/workflows/cd.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index 23f89443457..64d071b030c 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -13,7 +13,6 @@ on:
 
 env:
   CARGO_INCREMENTAL: '1'
-  NETWORK: Mainnet
   PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
   GAR_BASE: us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/zebra
   GCR_BASE: gcr.io/${{ secrets.GCP_PROJECT_ID }}
@@ -74,7 +73,7 @@ jobs:
           ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:latest
           ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:${{ env.GITHUB_SHA_SHORT }}
         build-args: |
-          NETWORK=${{ github.event.inputs.network || env.NETWORK }}
+          NETWORK=${{ github.event.inputs.network }}
           SHORT_SHA=${{ env.GITHUB_SHA_SHORT }}
           RUST_BACKTRACE=1
           ZEBRA_SKIP_IPV6_TESTS="1"

From 758d7e2878b795e39622c4149b3041b0da542e28 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Sat, 5 Feb 2022 11:31:03 -0400
Subject: [PATCH 089/124] fix(test): use bigger machine type for compute
 intensive tasks

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index aac56c1c958..a195a763512 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -24,7 +24,7 @@ env:
   GCR_BASE: gcr.io/${{ secrets.GCP_PROJECT_ID }}
   REGION: us-central1
   ZONE: us-central1-a
-  MACHINE_TYPE: c2-standard-4
+  MACHINE_TYPE: c2-standard-16
   IMAGE_NAME: zebrad-test
 
 jobs:

From dc82af9d3038c7648564104938082186b409d9d2 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Sat, 5 Feb 2022 12:47:37 -0400
Subject: [PATCH 090/124] refactor(test): add tests in CI file

---
 .github/workflows/test.yml | 60 ++++++++++++++++++++++++++++++++++----
 1 file changed, 55 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index a195a763512..a75310a2df1 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -26,6 +26,7 @@ env:
   ZONE: us-central1-a
   MACHINE_TYPE: c2-standard-16
   IMAGE_NAME: zebrad-test
+  ZEBRA_SKIP_IPV6_TESTS: "1"
 
 jobs:
   build:
@@ -85,6 +86,57 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
+  test-all:
+    name: Test all
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - uses: actions/checkout@v2.4.0
+        with:
+          persist-credentials: false
+
+      - name: Inject slug/short variables
+        uses: rlespinasse/github-slug-action@v4
+
+      - name: Run all zebrad tests
+        run: |
+          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
+          docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --workspace
+
+  test-large-sync:
+    name: Test large sync
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - uses: actions/checkout@v2.4.0
+        with:
+          persist-credentials: false
+
+      - name: Inject slug/short variables
+        uses: rlespinasse/github-slug-action@v4
+
+      - name: Run zebrad large sync tests
+        run: |
+          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
+          docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --test acceptance sync_large_checkpoints_ -- --ignored
+
+  test-large-sync:
+    name: Test large sync
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - uses: actions/checkout@v2.4.0
+        with:
+          persist-credentials: false
+
+      - name: Inject slug/short variables
+        uses: rlespinasse/github-slug-action@v4
+
+      - name: Run zebrad large sync tests
+        run: |
+          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
+          docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --package zebra-state --lib -- with_fake_activation_heights
+
   test-zunstable:
     name: Test with Zunstable options
     runs-on: ubuntu-latest
@@ -101,10 +153,8 @@ jobs:
         run: |
           docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
           docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --workspace -- -Zunstable-options --include-ignored
-        env:
-          ZEBRA_SKIP_IPV6_TESTS: "1"
 
-  regenerate-disks:
+  regenerate-stateful-disks:
     name: Renerate state disks
     runs-on: ubuntu-latest
     needs: build
@@ -175,10 +225,10 @@ jobs:
           gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"
           sleep 60s
 
-  test-sync:
+  test-stateful-sync:
     name: Test blocks sync
     runs-on: ubuntu-latest
-    needs: [ build, regenerate-disks]
+    needs: [ build, rregenerate-stateful-disks]
     steps:
       - uses: actions/checkout@v2.4.0
         with:

From 027fbb2bfa91a5a4056066cafa4aa1a0c7e9c9a3 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Sat, 5 Feb 2022 12:54:27 -0400
Subject: [PATCH 091/124] fix(test): remove duplicated tests

---
 .github/workflows/test.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index a75310a2df1..277ba93a34a 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -103,8 +103,8 @@ jobs:
           docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
           docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --workspace
 
-  test-large-sync:
-    name: Test large sync
+  test-fake-activation-heights:
+    name: Test with fake activation heights
     runs-on: ubuntu-latest
     needs: build
     steps:
@@ -115,10 +115,10 @@ jobs:
       - name: Inject slug/short variables
         uses: rlespinasse/github-slug-action@v4
 
-      - name: Run zebrad large sync tests
+      - name: Run tests with fake activation heights
         run: |
           docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
-          docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --test acceptance sync_large_checkpoints_ -- --ignored
+          docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --package zebra-state --lib -- with_fake_activation_heights
 
   test-large-sync:
     name: Test large sync
@@ -135,7 +135,7 @@ jobs:
       - name: Run zebrad large sync tests
         run: |
           docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
-          docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --package zebra-state --lib -- with_fake_activation_heights
+          docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --test acceptance sync_large_checkpoints_ -- --ignored
 
   test-zunstable:
     name: Test with Zunstable options

From d5be58746f8b8617df72b3af52c81a3f5925c306 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Sat, 5 Feb 2022 12:56:16 -0400
Subject: [PATCH 092/124] fix(test): typo

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 277ba93a34a..e6ed2d96273 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -228,7 +228,7 @@ jobs:
   test-stateful-sync:
     name: Test blocks sync
     runs-on: ubuntu-latest
-    needs: [ build, rregenerate-stateful-disks]
+    needs: [ build, regenerate-stateful-disks]
     steps:
       - uses: actions/checkout@v2.4.0
         with:

From 10c5b500f42a3b341cf350efa2be1ef00991190a Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Sat, 5 Feb 2022 13:00:14 -0400
Subject: [PATCH 093/124] test: build on .github changes temporarily

---
 .dockerignore | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.dockerignore b/.dockerignore
index fae64da0fe6..797ef17b32f 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -2,5 +2,5 @@ target
 Dockerfile
 .dockerignore
 .git
-.github
+# .github
 .gitignore

From a0794624e16ba7914b35d8f701d46bbf0cc3fa71 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Sat, 5 Feb 2022 23:12:14 -0400
Subject: [PATCH 094/124] fix(test): bigger machines have no effect on sync
 times

---
 .github/workflows/test.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index e6ed2d96273..2b8bbb2b23d 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -18,15 +18,15 @@ on:
 
 env:
   CARGO_INCREMENTAL: '1'
+  ZEBRA_SKIP_IPV6_TESTS: "1"
   NETWORK: mainnet
   PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
   GAR_BASE: us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/zebra
   GCR_BASE: gcr.io/${{ secrets.GCP_PROJECT_ID }}
   REGION: us-central1
   ZONE: us-central1-a
-  MACHINE_TYPE: c2-standard-16
+  MACHINE_TYPE: c2-standard-4
   IMAGE_NAME: zebrad-test
-  ZEBRA_SKIP_IPV6_TESTS: "1"
 
 jobs:
   build:
@@ -118,7 +118,7 @@ jobs:
       - name: Run tests with fake activation heights
         run: |
           docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
-          docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --package zebra-state --lib -- with_fake_activation_heights
+          docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --package zebra-state --lib -- with_fake_activation_heights
 
   test-large-sync:
     name: Test large sync

From 7677517a3f7e9c5f408ab34b9726174fa7b03d91 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 8 Feb 2022 10:44:45 -0400
Subject: [PATCH 095/124] feat: add an image to inherit from with zcash params

---
 .github/workflows/zcash-params.yml | 83 ++++++++++++++++++++++++++++++
 docker/Dockerfile.params           | 38 ++++++++++++++
 2 files changed, 121 insertions(+)
 create mode 100644 .github/workflows/zcash-params.yml
 create mode 100644 docker/Dockerfile.params

diff --git a/.github/workflows/zcash-params.yml b/.github/workflows/zcash-params.yml
new file mode 100644
index 00000000000..2611ce269aa
--- /dev/null
+++ b/.github/workflows/zcash-params.yml
@@ -0,0 +1,83 @@
+name: zcash-params
+
+on:
+  workflow_dispatch:
+    inputs:
+      network:
+        default: 'Mainnet'
+      checkpoint_sync:
+        default: true
+  pull_request:
+    branches:
+      - main
+    path:
+      - 'zebra-consensus/src/primitives/groth16/params.rs'
+      - 'zebra-consensus/src/chain.rs'
+      - 'zebrad/src/commands/start.rs'
+      - '.github/workflows/zcash-params.yml'
+
+env:
+  CARGO_INCREMENTAL: '1'
+  PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
+  GAR_BASE: us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/zebra
+  GCR_BASE: gcr.io/${{ secrets.GCP_PROJECT_ID }}
+
+jobs:
+  build:
+    name: Build images
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2.4.0
+      with:
+        persist-credentials: false
+
+    - name: Inject slug/short variables
+      uses: rlespinasse/github-slug-action@v4
+
+    # Setup gcloud CLI
+    - name: Authenticate to Google Cloud
+      id: auth
+      uses: google-github-actions/auth@v0.5.0
+      with:
+        credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
+
+    # Setup Docker Buildx to allow use of docker cache layers from GH
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+
+    - name: Login to Google Artifact Registry
+      uses: docker/login-action@v1.12.0
+      with:
+        registry: us-docker.pkg.dev
+        username: _json_key
+        password: ${{ secrets.GOOGLE_CREDENTIALS }}
+
+    - name: Login to Google Container Registry
+      uses: docker/login-action@v1.12.0
+      with:
+        registry: gcr.io
+        username: _json_key
+        password: ${{ secrets.GOOGLE_CREDENTIALS }}
+
+    # Build and push image to Google Artifact Registry
+    - name: Build & push
+      id: docker_build
+      uses: docker/build-push-action@v2.8.0
+      with:
+        target: runtime
+        context: .
+        file: ./docker/Dockerfile.params
+        tags: |
+          ${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:latest
+          ${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:${{ env.GITHUB_SHA_SHORT }}
+          ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:latest
+          ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:${{ env.GITHUB_SHA_SHORT }}
+        build-args: |
+          SHORT_SHA=${{ env.GITHUB_SHA_SHORT }}
+          ZEBRA_SKIP_IPV6_TESTS="1"
+          SENTRY_DSN=${{ secrets.SENTRY_ENDPOINT }}
+        push: true
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
diff --git a/docker/Dockerfile.params b/docker/Dockerfile.params
new file mode 100644
index 00000000000..95bba52203c
--- /dev/null
+++ b/docker/Dockerfile.params
@@ -0,0 +1,38 @@
+# This steps implement cargo-chef for docker layer caching
+# This image is for caching Zcash Sprout and Sapling parameters
+FROM rust:bullseye as chef
+RUN cargo install cargo-chef --locked
+WORKDIR /app
+
+FROM chef AS planner
+COPY . .
+RUN cargo chef prepare --recipe-path recipe.json
+
+FROM chef AS builder
+COPY --from=planner /app/recipe.json recipe.json
+
+# Install zebra build deps
+RUN apt-get -qq update && \
+    apt-get -qq install -y --no-install-recommends \
+    llvm \
+    libclang-dev \
+    clang \
+    ca-certificates \
+    ; \
+    rm -rf /var/lib/apt/lists/* /tmp/*
+
+# Optimize builds. In particular, regenerate-stateful-test-disks.yml was reaching the
+# GitHub Actions time limit (6 hours), so we needed to make it faster.
+ENV RUSTFLAGS -O
+ENV CARGO_HOME /app/.cargo/
+# Build dependencies - this is the caching Docker layer!
+RUN cargo chef cook --release --features enable-sentry --recipe-path recipe.json
+
+ARG RUST_BACKTRACE=1
+ENV RUST_BACKTRACE ${RUST_BACKTRACE}
+
+COPY . .
+# Pre-download Zcash Sprout and Sapling parameters
+RUN cargo run --locked --release --features enable-sentry --bin zebrad download
+# Build zebra
+RUN cargo build --locked --release --features enable-sentry --bin zebrad

From 476b48c7b6a44c0a6af04861d3c44c2bb040d725 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 8 Feb 2022 12:02:57 -0400
Subject: [PATCH 096/124] fix(cd): use the right image name and allow push to
 test

---
 .github/workflows/zcash-params.yml | 20 +++++++-------------
 1 file changed, 7 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/zcash-params.yml b/.github/workflows/zcash-params.yml
index 2611ce269aa..a97f2fb37fc 100644
--- a/.github/workflows/zcash-params.yml
+++ b/.github/workflows/zcash-params.yml
@@ -8,19 +8,22 @@ on:
       checkpoint_sync:
         default: true
   pull_request:
-    branches:
-      - main
     path:
       - 'zebra-consensus/src/primitives/groth16/params.rs'
       - 'zebra-consensus/src/chain.rs'
       - 'zebrad/src/commands/start.rs'
       - '.github/workflows/zcash-params.yml'
+  # TODO: remove this before merging
+  push:
+    branches:
+      - '**'
 
 env:
   CARGO_INCREMENTAL: '1'
   PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
   GAR_BASE: us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/zebra
   GCR_BASE: gcr.io/${{ secrets.GCP_PROJECT_ID }}
+  IMAGE_NAME: zcash-params
 
 jobs:
   build:
@@ -54,13 +57,6 @@ jobs:
         username: _json_key
         password: ${{ secrets.GOOGLE_CREDENTIALS }}
 
-    - name: Login to Google Container Registry
-      uses: docker/login-action@v1.12.0
-      with:
-        registry: gcr.io
-        username: _json_key
-        password: ${{ secrets.GOOGLE_CREDENTIALS }}
-
     # Build and push image to Google Artifact Registry
     - name: Build & push
       id: docker_build
@@ -70,10 +66,8 @@ jobs:
         context: .
         file: ./docker/Dockerfile.params
         tags: |
-          ${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:latest
-          ${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:${{ env.GITHUB_SHA_SHORT }}
-          ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:latest
-          ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:${{ env.GITHUB_SHA_SHORT }}
+          ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:latest
+          ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
         build-args: |
           SHORT_SHA=${{ env.GITHUB_SHA_SHORT }}
           ZEBRA_SKIP_IPV6_TESTS="1"

From 87cfba99b43ee95a88878918b06b8aef653ed72c Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 8 Feb 2022 12:09:41 -0400
Subject: [PATCH 097/124] fix(cd): use the right docker target and remove extra
 builds

---
 .github/workflows/zcash-params.yml | 2 +-
 docker/Dockerfile.params           | 2 --
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/zcash-params.yml b/.github/workflows/zcash-params.yml
index a97f2fb37fc..a030efdde55 100644
--- a/.github/workflows/zcash-params.yml
+++ b/.github/workflows/zcash-params.yml
@@ -62,7 +62,7 @@ jobs:
       id: docker_build
       uses: docker/build-push-action@v2.8.0
       with:
-        target: runtime
+        target: builder
         context: .
         file: ./docker/Dockerfile.params
         tags: |
diff --git a/docker/Dockerfile.params b/docker/Dockerfile.params
index 95bba52203c..13d86691f68 100644
--- a/docker/Dockerfile.params
+++ b/docker/Dockerfile.params
@@ -34,5 +34,3 @@ ENV RUST_BACKTRACE ${RUST_BACKTRACE}
 COPY . .
 # Pre-download Zcash Sprout and Sapling parameters
 RUN cargo run --locked --release --features enable-sentry --bin zebrad download
-# Build zebra
-RUN cargo build --locked --release --features enable-sentry --bin zebrad

From acc38fb6b00a908d6549e887f4797c82aa5af940 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 8 Feb 2022 13:28:15 -0400
Subject: [PATCH 098/124] refactor(docker): use cached zcash params from
 previous build

---
 docker/Dockerfile.build | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index af26b7fa428..b599474afe5 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -49,12 +49,14 @@ ARG RUST_BACKTRACE=1
 ENV RUST_BACKTRACE ${RUST_BACKTRACE}
 
 COPY . .
-# Pre-download Zcash Sprout and Sapling parameters
-RUN cargo run --locked --release --features enable-sentry --bin zebrad download
 # Build zebra
 RUN cargo build --locked --release --features enable-sentry --bin zebrad
 
 FROM builder AS tester
+# Pre-download Zcash Sprout and Sapling parameters
+# TODO: do not hardcode the user /root/ even though is a safe assumption
+COPY --from=us-docker.pkg.dev/zealous-zebra/zebra/zcash-params /root/.zcash-params /root/.zcash-params
+
 # Skip IPv6 tests by default, as some CI environment don't have IPv6 available
 ARG ZEBRA_SKIP_IPV6_TESTS=1
 ENV ZEBRA_SKIP_IPV6_TESTS ${ZEBRA_SKIP_IPV6_TESTS}
@@ -66,8 +68,7 @@ CMD ["cargo" "test" "--locked" "--release" "--features" "enable-sentry" "--works
 # Runner image
 FROM debian:bullseye-slim AS runtime
 COPY --from=builder /app/target/release/zebrad /usr/local/bin
-# TODO: do not hardcode the user /root/ even though is a safe assumption
-COPY --from=builder /root/.zcash-params /root/.zcash-params
+COPY --from=us-docker.pkg.dev/zealous-zebra/zebra/zcash-params /root/.zcash-params /root/.zcash-params
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \

From 3a1c4a7fcfc276cab7750d1486b9bce91106a229 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 8 Feb 2022 14:47:22 -0400
Subject: [PATCH 099/124] fix(cd): finalize for merging

---
 .github/workflows/zcash-params.yml | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/.github/workflows/zcash-params.yml b/.github/workflows/zcash-params.yml
index a030efdde55..fd41437bfe7 100644
--- a/.github/workflows/zcash-params.yml
+++ b/.github/workflows/zcash-params.yml
@@ -2,21 +2,12 @@ name: zcash-params
 
 on:
   workflow_dispatch:
-    inputs:
-      network:
-        default: 'Mainnet'
-      checkpoint_sync:
-        default: true
   pull_request:
     path:
       - 'zebra-consensus/src/primitives/groth16/params.rs'
       - 'zebra-consensus/src/chain.rs'
       - 'zebrad/src/commands/start.rs'
       - '.github/workflows/zcash-params.yml'
-  # TODO: remove this before merging
-  push:
-    branches:
-      - '**'
 
 env:
   CARGO_INCREMENTAL: '1'

From 22005a6ed18b90be52731ca4a439e3e3ccf0478a Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 8 Feb 2022 14:54:58 -0400
Subject: [PATCH 100/124] imp(cd): add double safety measure for production

---
 .github/workflows/cd.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index a95a5452638..6a48527d0bc 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -89,7 +89,7 @@ jobs:
     needs: build
     runs-on: ubuntu-latest
     timeout-minutes: 30
-    if: github.event_name == 'push'
+    if: github.event_name == 'push' && github.ref == 'ref/head/main'
 
     steps:
       - uses: actions/checkout@v2.4.0

From e88f443ec0f4f965c1178609916e8fd2dc503e0a Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 8 Feb 2022 14:57:14 -0400
Subject: [PATCH 101/124] fix(cd): use specific SHA for containers

---
 .github/workflows/cd.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index 6a48527d0bc..36a5831ebb4 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -115,7 +115,7 @@ jobs:
         run: |
           gcloud compute instance-templates create-with-container zebrad-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} \
           --boot-disk-type=pd-ssd \
-          --container-image ${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }} \
+          --container-image ${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:${{ env.GITHUB_SHA_SHORT }} \
           --create-disk name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }},auto-delete=yes,size=100GB,type=pd-ssd \
           --container-mount-disk mount-path="/zebrad-cache",name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }} \
           --machine-type ${{ env.MACHINE_TYPE }} \
@@ -185,7 +185,7 @@ jobs:
           --boot-disk-type=pd-ssd \
           --container-stdin \
           --container-tty \
-          --container-image ${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }} \
+          --container-image ${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}:${{ env.GITHUB_SHA_SHORT }} \
           --create-disk name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }},auto-delete=yes,size=100GB,type=pd-ssd \
           --container-mount-disk mount-path='/zebrad-cache',name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }} \
           --machine-type ${{ env.MACHINE_TYPE }} \

From ef8269985fc11e65ad37ea2be7cd911e4558f94f Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 8 Feb 2022 15:02:11 -0400
Subject: [PATCH 102/124] fix(cd): use latest gcloud action version

---
 .github/workflows/cd.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index 5fab2b22e45..65dfc8f488a 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -173,7 +173,7 @@ jobs:
           credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
 
       - name: Set up gcloud SDK environment
-        uses: google-github-actions/setup-gcloud@v0.4.0
+        uses: google-github-actions/setup-gcloud@v0.5.0
         with:
           project_id: ${{ env.PROJECT_ID }}
 

From df5407c530d4027f655ffaf3e86469b549a5466d Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 8 Feb 2022 15:44:57 -0400
Subject: [PATCH 103/124] fix(test): use the network as Mainnet and remove the
 uppercase from tests

---
 .github/workflows/test.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 2b8bbb2b23d..1510db2da62 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
     inputs:
       network:
-        default: 'mainnet'
+        default: 'Mainnet'
   push:
     # branches:
     #   - main
@@ -19,7 +19,7 @@ on:
 env:
   CARGO_INCREMENTAL: '1'
   ZEBRA_SKIP_IPV6_TESTS: "1"
-  NETWORK: mainnet
+  NETWORK: Mainnet
   PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
   GAR_BASE: us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/zebra
   GCR_BASE: gcr.io/${{ secrets.GCP_PROJECT_ID }}
@@ -189,10 +189,10 @@ jobs:
           --container-arg="test" \
           --container-arg="--verbose" \
           --container-arg="--features" \
-          --container-arg="enable-sentry,test_sync_to_mandatory_checkpoint_${{ github.event.inputs.network || env.NETWORK }}" \
+          --container-arg="enable-sentry,test_sync_to_mandatory_checkpoint_mainnet" \
           --container-arg="--manifest-path" \
           --container-arg="zebrad/Cargo.toml" \
-          --container-arg="sync_to_mandatory_checkpoint_${{ github.event.inputs.network || env.NETWORK }}" \
+          --container-arg="sync_to_mandatory_checkpoint_mainnet" \
           --container-env=ZEBRA_SKIP_IPV6_TESTS=1 \
           --machine-type ${{ env.MACHINE_TYPE }} \
           --scopes cloud-platform \

From e00b2c5b0838500b66fbc26128e7909ba4fd9f95 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 8 Feb 2022 16:20:46 -0400
Subject: [PATCH 104/124] fix(test): run disk regeneration on specific file
 change

Just run this regeneration when changing the following files:

https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state/disk_format.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/service/finalized_state.rs
https://github.com/ZcashFoundation/zebra/blob/main/zebra-state/src/constants.rs
---
 .github/workflows/test.yml | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 1510db2da62..6ab1a183d2a 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -162,6 +162,16 @@ jobs:
       - uses: actions/checkout@v2.4.0
         with:
           persist-credentials: false
+          fetch-depth: '2'
+
+      - name: Get specific changed files
+        id: changed-files-specific
+        uses: tj-actions/changed-files@v14.4
+        with:
+          files: |
+            /zebra-state/**/disk_format.rs
+            /zebra-state/**/finalized_state.rs
+            /zebra-state/**/constants.rs
 
       - name: Inject slug/short variables
         uses: rlespinasse/github-slug-action@v4
@@ -175,6 +185,7 @@ jobs:
 
       - name: Run state disk regeneration
         id: regenerate-state
+        if: steps.changed-files-specific.outputs.any_changed == 'true'
         run: |
           gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
           --boot-disk-size 100GB \
@@ -219,7 +230,7 @@ jobs:
 
       - name: Delete test instance
         # Always run even if the earlier step fails
-        if: ${{ always() }}
+        if: steps.changed-files-specific.outputs.any_changed == 'true'
         continue-on-error: true
         run: |
           gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"

From cf2c3d17e2ac321c5de179f3388cf6a0945d9907 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 8 Feb 2022 20:43:13 -0400
Subject: [PATCH 105/124] refactor(test): seggregate disks regeneration from
 tests

Allow to regenerate disks without running tests, and to run tests from previous disk regeneration.

Disk will be regenerated just if specific files were changed, or triggered manually.

Tests will run just if a disk regeneration was not manually triggered.
---
 .github/workflows/test.yml | 53 +++++++++++++++++++++++++++++++++-----
 1 file changed, 46 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 6ab1a183d2a..0c856c35bae 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -5,9 +5,13 @@ on:
     inputs:
       network:
         default: 'Mainnet'
-  push:
-    # branches:
-    #   - main
+      regenerate-disks:
+        type: boolean
+        default: false
+        description: Just update stateful disks
+  pull_request:
+    branches:
+      - main
     path:
       - '**/*.rs'
       - '**/*.txt'
@@ -90,6 +94,7 @@ jobs:
     name: Test all
     runs-on: ubuntu-latest
     needs: build
+    if: ${{ github.event.inputs.regenerate-disks != 'true' }}
     steps:
       - uses: actions/checkout@v2.4.0
         with:
@@ -107,6 +112,7 @@ jobs:
     name: Test with fake activation heights
     runs-on: ubuntu-latest
     needs: build
+    if: ${{ github.event.inputs.regenerate-disks != 'true' }}
     steps:
       - uses: actions/checkout@v2.4.0
         with:
@@ -124,6 +130,7 @@ jobs:
     name: Test large sync
     runs-on: ubuntu-latest
     needs: build
+    if: ${{ github.event.inputs.regenerate-disks != 'true' }}
     steps:
       - uses: actions/checkout@v2.4.0
         with:
@@ -141,6 +148,7 @@ jobs:
     name: Test with Zunstable options
     runs-on: ubuntu-latest
     needs: build
+    if: ${{ github.event.inputs.regenerate-disks != 'true' }}
     steps:
       - uses: actions/checkout@v2.4.0
         with:
@@ -185,7 +193,7 @@ jobs:
 
       - name: Run state disk regeneration
         id: regenerate-state
-        if: steps.changed-files-specific.outputs.any_changed == 'true'
+        if: ${{ steps.changed-files-specific.outputs.any_changed == 'true' }} || ${{ github.event.inputs.regenerate-disks == 'true' }}
         run: |
           gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
           --boot-disk-size 100GB \
@@ -228,9 +236,22 @@ jobs:
         run: |
           gcloud compute images create zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network || env.NETWORK }}-canopy --source-disk=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network || env.NETWORK }}-canopy --source-disk-zone=${{ env.ZONE }} --storage-location=us
 
+      - name: Write the disk SHORT_SHA to a txt
+        if: steps.regenerate-state.outcome == 'success'
+        run: |
+          short_sha=$(echo "${{ env.GITHUB_SHA_SHORT }}")
+          echo "$short_sha" > latest-disk-state-sha.txt
+
+      - name: Upload the disk state txt
+        uses: actions/upload-artifact@v2
+        with:
+          name: latest-disk-state-sha
+          path: latest-disk-state-sha.txt
+          retention-days: 1095
+
       - name: Delete test instance
         # Always run even if the earlier step fails
-        if: steps.changed-files-specific.outputs.any_changed == 'true'
+        if: steps.changed-files-specific.outputs.any_changed == 'true' || ${{ github.event.inputs.regenerate-disks == 'true' }}
         continue-on-error: true
         run: |
           gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"
@@ -240,6 +261,7 @@ jobs:
     name: Test blocks sync
     runs-on: ubuntu-latest
     needs: [ build, regenerate-stateful-disks]
+    if: ${{ github.event.inputs.regenerate-disks != 'true' }}
     steps:
       - uses: actions/checkout@v2.4.0
         with:
@@ -248,6 +270,21 @@ jobs:
       - name: Inject slug/short variables
         uses: rlespinasse/github-slug-action@v4
 
+      # Get the latest uploaded txt with the disk SHORT_SHA from this workflow
+      - name: Download latest disk state SHORT_SHA
+        uses: dawidd6/action-download-artifact@v2.17.0
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          workflow: test.yml
+          name: latest-disk-state-sha
+          check_artifacts:  true
+
+      - name: Get disk state SHA from txt
+        id: get_disk_sha
+        run: |
+          output=$(cat latest-disk-state-sha.txt)
+          echo "::set-output name=sha::$output"
+
       # Setup gcloud CLI
       - name: Authenticate to Google Cloud
         id: auth
@@ -262,8 +299,8 @@ jobs:
           gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
           --boot-disk-size 100GB \
           --boot-disk-type pd-ssd \
-          --disk=boot=no,mode=rw,name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network || env.NETWORK }}-canopy \
-          --container-mount-disk=mount-path='/zebrad-cache',name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network || env.NETWORK }}-canopy \
+          --disk=boot=no,mode=rw,name=zebrad-cache-${{ env.DISK_SHORT_SHA }}-${{ github.event.inputs.network || env.NETWORK }}-canopy \
+          --container-mount-disk=mount-path='/zebrad-cache',name=zebrad-cache-${{ env.DISK_SHORT_SHA }}-${{ github.event.inputs.network || env.NETWORK }}-canopy \
           --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} \
           --container-restart-policy=never \
           --container-stdin \
@@ -283,6 +320,8 @@ jobs:
           --scopes cloud-platform \
           --tags zebrad \
           --zone "${{ env.ZONE }}"
+        env:
+          DISK_SHORT_SHA: ${{ steps.get_disk_sha.outputs.sha }}
 
       - name: Wait sync process to end
         run: |

From e342d8723e374dc18e577e38b5be93a67246db51 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Wed, 9 Feb 2022 18:42:49 -0400
Subject: [PATCH 106/124] fix(test): gcp disks require lower case conventions

---
 .github/workflows/test.yml | 28 ++++++++++++++++++----------
 1 file changed, 18 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 0c856c35bae..a0d7033ad61 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -184,6 +184,10 @@ jobs:
       - name: Inject slug/short variables
         uses: rlespinasse/github-slug-action@v4
 
+      - name: Downcase network name for disks
+        run: |
+          echo LOWER_NET_NAME="${{ github.event.inputs.network || env.NETWORK }}" | awk '{print tolower($0)}' >> $GITHUB_ENV
+
       # Setup gcloud CLI
       - name: Authenticate to Google Cloud
         id: auth
@@ -198,8 +202,8 @@ jobs:
           gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
           --boot-disk-size 100GB \
           --boot-disk-type pd-ssd \
-          --create-disk name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network || env.NETWORK }}-canopy",size=100GB,type=pd-ssd \
-          --container-mount-disk mount-path='/zebrad-cache',name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network || env.NETWORK }}-canopy" \
+          --create-disk name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.LOWER_NET_NAME }}-canopy",size=100GB,type=pd-ssd \
+          --container-mount-disk mount-path='/zebrad-cache',name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.LOWER_NET_NAME }}-canopy" \
           --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} \
           --container-restart-policy=never \
           --container-stdin \
@@ -208,10 +212,10 @@ jobs:
           --container-arg="test" \
           --container-arg="--verbose" \
           --container-arg="--features" \
-          --container-arg="enable-sentry,test_sync_to_mandatory_checkpoint_mainnet" \
+          --container-arg="enable-sentry,test_sync_to_mandatory_checkpoint_${{ env.LOWER_NET_NAME }}" \
           --container-arg="--manifest-path" \
           --container-arg="zebrad/Cargo.toml" \
-          --container-arg="sync_to_mandatory_checkpoint_mainnet" \
+          --container-arg="sync_to_mandatory_checkpoint_${{ env.LOWER_NET_NAME }}" \
           --container-env=ZEBRA_SKIP_IPV6_TESTS=1 \
           --machine-type ${{ env.MACHINE_TYPE }} \
           --scopes cloud-platform \
@@ -234,7 +238,7 @@ jobs:
         # Only run if the earlier step succeeds
         if: steps.wait-regenerate.outcome == 'success'
         run: |
-          gcloud compute images create zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network || env.NETWORK }}-canopy --source-disk=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network || env.NETWORK }}-canopy --source-disk-zone=${{ env.ZONE }} --storage-location=us
+          gcloud compute images create zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.LOWER_NET_NAME }}-canopy --source-disk=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.LOWER_NET_NAME }}-canopy --source-disk-zone=${{ env.ZONE }} --storage-location=us
 
       - name: Write the disk SHORT_SHA to a txt
         if: steps.regenerate-state.outcome == 'success'
@@ -243,7 +247,7 @@ jobs:
           echo "$short_sha" > latest-disk-state-sha.txt
 
       - name: Upload the disk state txt
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v2.3.1
         with:
           name: latest-disk-state-sha
           path: latest-disk-state-sha.txt
@@ -270,6 +274,10 @@ jobs:
       - name: Inject slug/short variables
         uses: rlespinasse/github-slug-action@v4
 
+      - name: Downcase network name for disks
+        run: |
+          echo LOWER_NET_NAME="${{ github.event.inputs.network || env.NETWORK }}" | awk '{print tolower($0)}' >> $GITHUB_ENV
+
       # Get the latest uploaded txt with the disk SHORT_SHA from this workflow
       - name: Download latest disk state SHORT_SHA
         uses: dawidd6/action-download-artifact@v2.17.0
@@ -299,8 +307,8 @@ jobs:
           gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
           --boot-disk-size 100GB \
           --boot-disk-type pd-ssd \
-          --disk=boot=no,mode=rw,name=zebrad-cache-${{ env.DISK_SHORT_SHA }}-${{ github.event.inputs.network || env.NETWORK }}-canopy \
-          --container-mount-disk=mount-path='/zebrad-cache',name=zebrad-cache-${{ env.DISK_SHORT_SHA }}-${{ github.event.inputs.network || env.NETWORK }}-canopy \
+          --disk=boot=no,mode=rw,name=zebrad-cache-${{ env.DISK_SHORT_SHA }}-${{ env.LOWER_NET_NAME }}-canopy \
+          --container-mount-disk=mount-path='/zebrad-cache',name=zebrad-cache-${{ env.DISK_SHORT_SHA }}-${{ env.LOWER_NET_NAME }}-canopy \
           --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} \
           --container-restart-policy=never \
           --container-stdin \
@@ -310,10 +318,10 @@ jobs:
           --container-arg="--locked" \
           --container-arg="--release" \
           --container-arg="--features" \
-          --container-arg="enable-sentry,test_sync_past_mandatory_checkpoint_${{ github.event.inputs.network || env.NETWORK }}" \
+          --container-arg="enable-sentry,test_sync_past_mandatory_checkpoint_${{ env.LOWER_NET_NAME }}" \
           --container-arg="--manifest-path" \
           --container-arg="zebrad/Cargo.toml" \
-          --container-arg="sync_past_mandatory_checkpoint_${{ github.event.inputs.network || env.NETWORK }}" \
+          --container-arg="sync_past_mandatory_checkpoint_${{ env.LOWER_NET_NAME }}" \
           --container-env=ZEBRA_SKIP_IPV6_TESTS=1 \
           --machine-type ${{ env.MACHINE_TYPE }} \
           --service-account ${{ env.DEPLOY_SA }} \

From f7dbed45f257384e4e91945d0b2aec5b84e71237 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Thu, 10 Feb 2022 08:08:48 -0400
Subject: [PATCH 107/124] fix(test): validate logs being emmited by docker

GHA is transforming is somehow transforwing the variable to lowercase also, so we're changint it to adapt to it
---
 .github/workflows/test.yml | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index a0d7033ad61..dd366ba6475 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -202,8 +202,8 @@ jobs:
           gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
           --boot-disk-size 100GB \
           --boot-disk-type pd-ssd \
-          --create-disk name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.LOWER_NET_NAME }}-canopy",size=100GB,type=pd-ssd \
-          --container-mount-disk mount-path='/zebrad-cache',name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.LOWER_NET_NAME }}-canopy" \
+          --create-disk name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy",size=100GB,type=pd-ssd \
+          --container-mount-disk mount-path='/zebrad-cache',name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy" \
           --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} \
           --container-restart-policy=never \
           --container-stdin \
@@ -212,10 +212,10 @@ jobs:
           --container-arg="test" \
           --container-arg="--verbose" \
           --container-arg="--features" \
-          --container-arg="enable-sentry,test_sync_to_mandatory_checkpoint_${{ env.LOWER_NET_NAME }}" \
+          --container-arg="enable-sentry,test_sync_to_mandatory_checkpoint_${{ env.lower_net_name }}" \
           --container-arg="--manifest-path" \
           --container-arg="zebrad/Cargo.toml" \
-          --container-arg="sync_to_mandatory_checkpoint_${{ env.LOWER_NET_NAME }}" \
+          --container-arg="sync_to_mandatory_checkpoint_${{ env.lower_net_name }}" \
           --container-env=ZEBRA_SKIP_IPV6_TESTS=1 \
           --machine-type ${{ env.MACHINE_TYPE }} \
           --scopes cloud-platform \
@@ -229,7 +229,7 @@ jobs:
           sleep 30s
           gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --quiet --ssh-flag="-t" --ssh-flag="-o ServerAliveInterval=5"
           docker ps -aqf ancestor=$CONTAINER_IMAGE | tee output.txt
-          docker wait $(cat output.txt)
+          docker logs $(cat output.txt) --follow
         env:
           CONTAINER_IMAGE: ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
 
@@ -238,7 +238,7 @@ jobs:
         # Only run if the earlier step succeeds
         if: steps.wait-regenerate.outcome == 'success'
         run: |
-          gcloud compute images create zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.LOWER_NET_NAME }}-canopy --source-disk=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.LOWER_NET_NAME }}-canopy --source-disk-zone=${{ env.ZONE }} --storage-location=us
+          gcloud compute images create zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy --source-disk=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy --source-disk-zone=${{ env.ZONE }} --storage-location=us
 
       - name: Write the disk SHORT_SHA to a txt
         if: steps.regenerate-state.outcome == 'success'
@@ -307,8 +307,8 @@ jobs:
           gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
           --boot-disk-size 100GB \
           --boot-disk-type pd-ssd \
-          --disk=boot=no,mode=rw,name=zebrad-cache-${{ env.DISK_SHORT_SHA }}-${{ env.LOWER_NET_NAME }}-canopy \
-          --container-mount-disk=mount-path='/zebrad-cache',name=zebrad-cache-${{ env.DISK_SHORT_SHA }}-${{ env.LOWER_NET_NAME }}-canopy \
+          --disk=boot=no,mode=rw,name=zebrad-cache-${{ env.DISK_SHORT_SHA }}-${{ env.lower_net_name }}-canopy \
+          --container-mount-disk=mount-path='/zebrad-cache',name=zebrad-cache-${{ env.DISK_SHORT_SHA }}-${{ env.lower_net_name }}-canopy \
           --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} \
           --container-restart-policy=never \
           --container-stdin \
@@ -318,10 +318,10 @@ jobs:
           --container-arg="--locked" \
           --container-arg="--release" \
           --container-arg="--features" \
-          --container-arg="enable-sentry,test_sync_past_mandatory_checkpoint_${{ env.LOWER_NET_NAME }}" \
+          --container-arg="enable-sentry,test_sync_past_mandatory_checkpoint_${{ env.lower_net_name }}" \
           --container-arg="--manifest-path" \
           --container-arg="zebrad/Cargo.toml" \
-          --container-arg="sync_past_mandatory_checkpoint_${{ env.LOWER_NET_NAME }}" \
+          --container-arg="sync_past_mandatory_checkpoint_${{ env.lower_net_name }}" \
           --container-env=ZEBRA_SKIP_IPV6_TESTS=1 \
           --machine-type ${{ env.MACHINE_TYPE }} \
           --service-account ${{ env.DEPLOY_SA }} \
@@ -336,7 +336,7 @@ jobs:
           sleep 30s
           gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --quiet --ssh-flag="-t" --ssh-flag="-o ServerAliveInterval=5"
           docker ps -aqf ancestor=$CONTAINER_IMAGE | tee output.txt
-          docker wait $(cat output.txt)
+          docker logs $(cat output.txt) --follow
         env:
           CONTAINER_IMAGE: ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
 

From 43e731b4a913a95e7b156a785d0321e7b086ca35 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Thu, 10 Feb 2022 08:33:59 -0400
Subject: [PATCH 108/124] test

---
 .dockerignore | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.dockerignore b/.dockerignore
index fae64da0fe6..797ef17b32f 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -2,5 +2,5 @@ target
 Dockerfile
 .dockerignore
 .git
-.github
+# .github
 .gitignore

From c1c2babbb82ec72ecbd2080847d973c9bb08c699 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Thu, 10 Feb 2022 10:26:29 -0400
Subject: [PATCH 109/124] fix(test): force tty terminal

---
 .github/workflows/test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index dd366ba6475..7ff8e7892c9 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -227,7 +227,7 @@ jobs:
         if: steps.regenerate-state.outcome == 'success'
         run: |
           sleep 30s
-          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --quiet --ssh-flag="-t" --ssh-flag="-o ServerAliveInterval=5"
+          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --quiet --ssh-flag="-tt" --ssh-flag="-o ServerAliveInterval=5"
           docker ps -aqf ancestor=$CONTAINER_IMAGE | tee output.txt
           docker logs $(cat output.txt) --follow
         env:
@@ -334,7 +334,7 @@ jobs:
       - name: Wait sync process to end
         run: |
           sleep 30s
-          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --quiet --ssh-flag="-t" --ssh-flag="-o ServerAliveInterval=5"
+          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --quiet --ssh-flag="-tt" --ssh-flag="-o ServerAliveInterval=5"
           docker ps -aqf ancestor=$CONTAINER_IMAGE | tee output.txt
           docker logs $(cat output.txt) --follow
         env:

From 6f83d647dc875a560f52de99044ee94b807357f4 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Thu, 10 Feb 2022 15:32:16 -0400
Subject: [PATCH 110/124] fix(test): use a one line command to test terminal
 output

---
 .github/workflows/test.yml | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 7ff8e7892c9..3fc5644291e 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -227,9 +227,8 @@ jobs:
         if: steps.regenerate-state.outcome == 'success'
         run: |
           sleep 30s
-          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --quiet --ssh-flag="-tt" --ssh-flag="-o ServerAliveInterval=5"
-          docker ps -aqf ancestor=$CONTAINER_IMAGE | tee output.txt
-          docker logs $(cat output.txt) --follow
+          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --quiet --ssh-flag="-tt" --ssh-flag="-o ServerAliveInterval=5" \
+          --command="docker logs $(docker ps -aqf ancestor=$CONTAINER_IMAGE) --follow"
         env:
           CONTAINER_IMAGE: ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
 
@@ -334,9 +333,8 @@ jobs:
       - name: Wait sync process to end
         run: |
           sleep 30s
-          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --quiet --ssh-flag="-tt" --ssh-flag="-o ServerAliveInterval=5"
-          docker ps -aqf ancestor=$CONTAINER_IMAGE | tee output.txt
-          docker logs $(cat output.txt) --follow
+          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --quiet --ssh-flag="-tt" --ssh-flag="-o ServerAliveInterval=5" \
+          --command="docker logs $(docker ps -aqf ancestor=$CONTAINER_IMAGE) --follow"
         env:
           CONTAINER_IMAGE: ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
 

From 246a8cd6cb30ad307f5ca0b711fcbb277047b282 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Thu, 10 Feb 2022 20:33:18 -0400
Subject: [PATCH 111/124] fix(test): always delete test instance

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 3fc5644291e..994ee33d539 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -254,7 +254,7 @@ jobs:
 
       - name: Delete test instance
         # Always run even if the earlier step fails
-        if: steps.changed-files-specific.outputs.any_changed == 'true' || ${{ github.event.inputs.regenerate-disks == 'true' }}
+        if: ${{ always() }}
         continue-on-error: true
         run: |
           gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"

From cce709ab8509fa94ebf8950b477cf6ecba4aa91c Mon Sep 17 00:00:00 2001
From: Alfredo Garcia <oxarbitrage@gmail.com>
Date: Sat, 12 Feb 2022 16:30:23 -0300
Subject: [PATCH 112/124] ignore non-verack message in handshake

---
 zebra-network/src/peer/handshake.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/zebra-network/src/peer/handshake.rs b/zebra-network/src/peer/handshake.rs
index d4131b12c4b..2c70395ac7e 100644
--- a/zebra-network/src/peer/handshake.rs
+++ b/zebra-network/src/peer/handshake.rs
@@ -705,9 +705,9 @@ where
         .await
         .ok_or(HandshakeError::ConnectionClosed)??;
     if let Message::Verack = remote_msg {
-        debug!("got verack from remote peer");
+        debug!("got verack message from remote peer");
     } else {
-        Err(HandshakeError::UnexpectedMessage(Box::new(remote_msg)))?;
+        debug!("ignoring non-verack message from remote peer");
     }
 
     Ok((remote_version, remote_services, remote_canonical_addr))

From 222dfdd69eb7f4ed82dd06d376f0adb2f6ed537c Mon Sep 17 00:00:00 2001
From: Alfredo Garcia <oxarbitrage@gmail.com>
Date: Sat, 12 Feb 2022 16:52:50 -0300
Subject: [PATCH 113/124] ignore non-version message in handshake

---
 zebra-network/src/peer/handshake.rs | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/zebra-network/src/peer/handshake.rs b/zebra-network/src/peer/handshake.rs
index 2c70395ac7e..f79296397b5 100644
--- a/zebra-network/src/peer/handshake.rs
+++ b/zebra-network/src/peer/handshake.rs
@@ -588,13 +588,25 @@ where
     debug!(?our_version, "sending initial version message");
     peer_conn.send(our_version).await?;
 
-    let remote_msg = peer_conn
+    let mut remote_msg = peer_conn
         .next()
         .await
         .ok_or(HandshakeError::ConnectionClosed)??;
-
-    // Check that we got a Version and destructure its fields into the local scope.
     debug!(?remote_msg, "got message from remote peer");
+
+    // Wait for next message if the one we have is not Version
+    match remote_msg {
+        Message::Version { .. } => (),
+        _ => {
+            remote_msg = peer_conn
+                .next()
+                .await
+                .ok_or(HandshakeError::ConnectionClosed)??;
+            debug!(?remote_msg, "got message from remote peer");
+        }
+    }
+
+    // If we got a Version message, destructure its fields into the local scope.
     let (remote_nonce, remote_services, remote_version, remote_canonical_addr, user_agent) =
         if let Message::Version {
             version,

From da85ca303750df002a83922aead186fb71917dcd Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Mon, 14 Feb 2022 00:12:52 -0400
Subject: [PATCH 114/124] fix(test): use short SHA from the PR head

Using the SHA from the base, creates confusion and it's not accurate with the SHA being shown and used on GitHub.

We have to keep both as manual runs with `workflow_dispatch` does not have a PR SHA
---
 .github/workflows/test.yml | 50 +++++++++++++++++++-------------------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 27b6910e685..02f3a6a44fe 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -75,12 +75,12 @@ jobs:
           file: ./docker/Dockerfile.build
           tags: |
             ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:latest
-            ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
+            ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
             ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.IMAGE_NAME }}:latest
-            ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
+            ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
           build-args: |
             NETWORK=${{ github.event.inputs.network || env.NETWORK }}
-            SHORT_SHA=${{ env.GITHUB_SHA_SHORT }}
+            SHORT_SHA=${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
             RUST_BACKTRACE=full
             ZEBRA_SKIP_NETWORK_TESTS="1"
             CHECKPOINT_SYNC=${{ github.event.inputs.checkpoint_sync || true }}
@@ -105,8 +105,8 @@ jobs:
 
       - name: Run all zebrad tests
         run: |
-          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
-          docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --workspace
+          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
+          docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --workspace
 
   test-fake-activation-heights:
     name: Test with fake activation heights
@@ -123,8 +123,8 @@ jobs:
 
       - name: Run tests with fake activation heights
         run: |
-          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
-          docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --package zebra-state --lib -- with_fake_activation_heights
+          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
+          docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} cargo test --locked --release --package zebra-state --lib -- with_fake_activation_heights
 
   test-large-sync:
     name: Test large sync
@@ -141,8 +141,8 @@ jobs:
 
       - name: Run zebrad large sync tests
         run: |
-          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
-          docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --test acceptance sync_large_checkpoints_ -- --ignored
+          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
+          docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --test acceptance sync_large_checkpoints_ -- --ignored
 
   test-zunstable:
     name: Test with Zunstable options
@@ -159,8 +159,8 @@ jobs:
 
       - name: Run Zunstable-options tests
         run: |
-          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
-          docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --workspace -- -Zunstable-options --include-ignored
+          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
+          docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --workspace -- -Zunstable-options --include-ignored
 
   regenerate-stateful-disks:
     name: Renerate state disks
@@ -199,12 +199,12 @@ jobs:
         id: regenerate-state
         if: ${{ steps.changed-files-specific.outputs.any_changed == 'true' }} || ${{ github.event.inputs.regenerate-disks == 'true' }}
         run: |
-          gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
+          gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" \
           --boot-disk-size 100GB \
           --boot-disk-type pd-ssd \
-          --create-disk name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy",size=100GB,type=pd-ssd \
-          --container-mount-disk mount-path='/zebrad-cache',name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy" \
-          --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} \
+          --create-disk name="zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy",size=100GB,type=pd-ssd \
+          --container-mount-disk mount-path='/zebrad-cache',name="zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy" \
+          --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \
           --container-restart-policy=never \
           --container-stdin \
           --container-tty \
@@ -227,22 +227,22 @@ jobs:
         if: steps.regenerate-state.outcome == 'success'
         run: |
           sleep 30s
-          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --quiet --ssh-flag="-tt" --ssh-flag="-o ServerAliveInterval=5" \
+          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --quiet --ssh-flag="-tt" --ssh-flag="-o ServerAliveInterval=5" \
           --command="docker logs $(docker ps -aqf ancestor=$CONTAINER_IMAGE) --follow"
         env:
-          CONTAINER_IMAGE: ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
+          CONTAINER_IMAGE: ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
 
       # Create image from disk that will be used to sync past mandatory checkpoint test
       - name: Create image from state disk
         # Only run if the earlier step succeeds
         if: steps.wait-regenerate.outcome == 'success'
         run: |
-          gcloud compute images create zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy --source-disk=zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy --source-disk-zone=${{ env.ZONE }} --storage-location=us
+          gcloud compute images create zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy --source-disk=zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy --source-disk-zone=${{ env.ZONE }} --storage-location=us
 
       - name: Write the disk SHORT_SHA to a txt
         if: steps.regenerate-state.outcome == 'success'
         run: |
-          short_sha=$(echo "${{ env.GITHUB_SHA_SHORT }}")
+          short_sha=$(echo "${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}")
           echo "$short_sha" > latest-disk-state-sha.txt
 
       - name: Upload the disk state txt
@@ -257,7 +257,7 @@ jobs:
         if: ${{ always() }}
         continue-on-error: true
         run: |
-          gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"
+          gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"
           sleep 60s
 
   test-stateful-sync:
@@ -303,12 +303,12 @@ jobs:
       - name: Sync past mandatory checkpoint
         id: sync-past-checkpoint
         run: |
-          gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \
+          gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" \
           --boot-disk-size 100GB \
           --boot-disk-type pd-ssd \
           --disk=boot=no,mode=rw,name=zebrad-cache-${{ env.DISK_SHORT_SHA }}-${{ env.lower_net_name }}-canopy \
           --container-mount-disk=mount-path='/zebrad-cache',name=zebrad-cache-${{ env.DISK_SHORT_SHA }}-${{ env.lower_net_name }}-canopy \
-          --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} \
+          --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \
           --container-restart-policy=never \
           --container-stdin \
           --container-tty \
@@ -333,14 +333,14 @@ jobs:
       - name: Wait sync process to end
         run: |
           sleep 30s
-          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --quiet --ssh-flag="-tt" --ssh-flag="-o ServerAliveInterval=5" \
+          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --quiet --ssh-flag="-tt" --ssh-flag="-o ServerAliveInterval=5" \
           --command="docker logs $(docker ps -aqf ancestor=$CONTAINER_IMAGE) --follow"
         env:
-          CONTAINER_IMAGE: ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
+          CONTAINER_IMAGE: ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
 
       - name: Delete test instance
         # Always run even if the earlier step fails
         if: ${{ always() }}
         continue-on-error: true
         run: |
-          gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"
+          gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"

From 85d977fa4909f9f296b8c04b2b5cefb002c92057 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Mon, 14 Feb 2022 00:27:42 -0400
Subject: [PATCH 115/124] fix(ci): do not trigger CI on docker changes

There's no impact in this workflow when a change is done in the dockerfile
---
 .github/workflows/ci.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 9fe0c628da5..0dcbf60c453 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -9,7 +9,6 @@ on:
       - '**/Cargo.toml'
       - '**/Cargo.lock'
       - '**/deny.toml'
-      - 'docker/**'
       - '.github/workflows/ci.yml'
 
 env:

From c882606c733f67efe759ee9919bdebb45dfceb16 Mon Sep 17 00:00:00 2001
From: Alfredo Garcia <oxarbitrage@gmail.com>
Date: Mon, 14 Feb 2022 08:46:20 -0300
Subject: [PATCH 116/124] add loops for waiting

---
 zebra-network/src/peer/handshake.rs | 46 +++++++++++++++++++----------
 1 file changed, 31 insertions(+), 15 deletions(-)

diff --git a/zebra-network/src/peer/handshake.rs b/zebra-network/src/peer/handshake.rs
index f79296397b5..8fe1bf0642a 100644
--- a/zebra-network/src/peer/handshake.rs
+++ b/zebra-network/src/peer/handshake.rs
@@ -592,17 +592,21 @@ where
         .next()
         .await
         .ok_or(HandshakeError::ConnectionClosed)??;
-    debug!(?remote_msg, "got message from remote peer");
 
-    // Wait for next message if the one we have is not Version
-    match remote_msg {
-        Message::Version { .. } => (),
-        _ => {
-            remote_msg = peer_conn
-                .next()
-                .await
-                .ok_or(HandshakeError::ConnectionClosed)??;
-            debug!(?remote_msg, "got message from remote peer");
+    // Wait for next message if the one we got is not Version
+    loop {
+        match remote_msg {
+            Message::Version { .. } => {
+                debug!(?remote_msg, "got version message from remote peer");
+                break;
+            }
+            _ => {
+                remote_msg = peer_conn
+                    .next()
+                    .await
+                    .ok_or(HandshakeError::ConnectionClosed)??;
+                debug!("ignoring non-version message from remote peer");
+            }
         }
     }
 
@@ -712,14 +716,26 @@ where
 
     peer_conn.send(Message::Verack).await?;
 
-    let remote_msg = peer_conn
+    let mut remote_msg = peer_conn
         .next()
         .await
         .ok_or(HandshakeError::ConnectionClosed)??;
-    if let Message::Verack = remote_msg {
-        debug!("got verack message from remote peer");
-    } else {
-        debug!("ignoring non-verack message from remote peer");
+
+    // Wait for next message if the one we got is not Verack
+    loop {
+        match remote_msg {
+            Message::Verack => {
+                debug!(?remote_msg, "got verack message from remote peer");
+                break;
+            }
+            _ => {
+                remote_msg = peer_conn
+                    .next()
+                    .await
+                    .ok_or(HandshakeError::ConnectionClosed)??;
+                debug!("ignoring non-verack message from remote peer");
+            }
+        }
     }
 
     Ok((remote_version, remote_services, remote_canonical_addr))

From 3c0908b2b4d6baee876b8eced5e08e877fc72fc1 Mon Sep 17 00:00:00 2001
From: Marek <mail@marek.onl>
Date: Mon, 14 Feb 2022 19:05:05 +0100
Subject: [PATCH 117/124] Check the length of coinbase data

---
 zebra-chain/src/transparent/serialize.rs | 41 +++++++++++++++++-------
 1 file changed, 30 insertions(+), 11 deletions(-)

diff --git a/zebra-chain/src/transparent/serialize.rs b/zebra-chain/src/transparent/serialize.rs
index 0a9d01915c4..11cd98964e1 100644
--- a/zebra-chain/src/transparent/serialize.rs
+++ b/zebra-chain/src/transparent/serialize.rs
@@ -1,3 +1,5 @@
+//! Serializes and deserializes transparent data.
+
 use std::io;
 
 use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
@@ -14,13 +16,24 @@ use crate::{
 use super::{CoinbaseData, Input, OutPoint, Output, Script};
 
 /// The maximum length of the coinbase data.
+///
 /// Includes the encoded coinbase height, if any.
 ///
 /// > The number of bytes in the coinbase script, up to a maximum of 100 bytes.
 ///
-/// https://developer.bitcoin.org/reference/transactions.html#coinbase-input-the-input-of-the-first-transaction-in-a-block
+/// <https://developer.bitcoin.org/reference/transactions.html#coinbase-input-the-input-of-the-first-transaction-in-a-block>
 pub const MAX_COINBASE_DATA_LEN: usize = 100;
 
+/// The minimum length of the coinbase data.
+///
+/// Includes the encoded coinbase height, if any.
+///
+// TODO: Update the link below once the constant is documented in the
+// protocol.
+///
+/// <https://github.com/zcash/zips/issues/589>
+pub const MIN_COINBASE_DATA_LEN: usize = 2;
+
 /// The coinbase data for a genesis block.
 ///
 /// Zcash uses the same coinbase data for the Mainnet, Testnet, and Regtest
@@ -68,18 +81,19 @@ impl ZcashDeserialize for OutPoint {
 ///
 /// # Consensus
 ///
-/// > A coinbase transaction for a block at block height greater than 0 MUST have
-/// > a script that, as its first item, encodes the block height height as follows.
-/// > For height in the range {1..16}, the encoding is a single byte of value
-/// > 0x50 height. Otherwise, let heightBytes be the signed little-endian
-/// > representation of height, using the minimum nonzero number of bytes such that
-/// > the most significant byte is < 0x80.
-/// > The length of heightBytes MUST be in the range {1..5}.
-/// > Then the encoding is the length of heightBytes encoded as one byte,
-/// > followed by heightBytes itself. This matches the encoding used by Bitcoin in the
+/// > A coinbase transaction for a *block* at *block height* greater than 0 MUST have
+/// > a script that, as its first item, encodes the *block height* `height` as follows.
+/// > For `height` in the range {1..16}, the encoding is a single byte of value
+/// > `0x50` + `height`. Otherwise, let `heightBytes` be the signed little-endian
+/// > representation of `height`, using the minimum nonzero number of bytes such that
+/// > the most significant byte is < `0x80`.
+/// > The length of `heightBytes` MUST be in the range {1..5}.
+/// > Then the encoding is the length of `heightBytes` encoded as one byte,
+/// > followed by `heightBytes` itself. This matches the encoding used by Bitcoin in the
 /// > implementation of [BIP-34] (but the description here is to be considered normative).
 ///
 /// <https://zips.z.cash/protocol/protocol.pdf#txnconsensus>
+/// <https://github.com/bitcoin/bips/blob/master/bip-0034.mediawiki>
 pub(crate) fn parse_coinbase_height(
     mut data: Vec<u8>,
 ) -> Result<(block::Height, CoinbaseData), SerializationError> {
@@ -262,9 +276,14 @@ impl ZcashDeserialize for Input {
             }
 
             let data: Vec<u8> = (&mut reader).zcash_deserialize_into()?;
+
+            // Check the coinbase data length.
             if data.len() > MAX_COINBASE_DATA_LEN {
-                return Err(SerializationError::Parse("coinbase has too much data"));
+                return Err(SerializationError::Parse("coinbase data is too long"));
+            } else if data.len() < MIN_COINBASE_DATA_LEN {
+                return Err(SerializationError::Parse("coinbase data is too short"));
             }
+
             let (height, data) = parse_coinbase_height(data)?;
 
             let sequence = reader.read_u32::<LittleEndian>()?;

From ce3a2697d9545bce11d95a81b3f871aa210841b7 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Mon, 14 Feb 2022 18:18:01 -0400
Subject: [PATCH 118/124] Instead of runing cargo test when the instance gets
 created, run this commands afterwards in a different step.

As GHA TTY is not working as expected, and workarounds does not play nicely with `gcloud compute ssh` actions/runner#241 (comment) we decided to get the container name from the logs, log directly to the container and run the cargo command from there.
---
 .github/workflows/test.yml | 92 ++++++++++++++++++++++++++------------
 1 file changed, 63 insertions(+), 29 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 02f3a6a44fe..9831d7ade0e 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -195,8 +195,8 @@ jobs:
         with:
           credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
 
-      - name: Run state disk regeneration
-        id: regenerate-state
+      - name: Create GCP compute instance
+        id: create-instance
         if: ${{ steps.changed-files-specific.outputs.any_changed == 'true' }} || ${{ github.event.inputs.regenerate-disks == 'true' }}
         run: |
           gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" \
@@ -212,35 +212,52 @@ jobs:
           --container-arg="test" \
           --container-arg="--verbose" \
           --container-arg="--features" \
-          --container-arg="enable-sentry,test_sync_to_mandatory_checkpoint_${{ env.lower_net_name }}" \
-          --container-arg="--manifest-path" \
-          --container-arg="zebrad/Cargo.toml" \
-          --container-arg="sync_to_mandatory_checkpoint_${{ env.lower_net_name }}" \
+          --container-arg="enable-sentry" \
+          --container-arg="--test" \
+          --container-arg="acceptance" \
+          --container-arg="sync_large_checkpoints_" \
           --container-env=ZEBRA_SKIP_IPV6_TESTS=1 \
           --machine-type ${{ env.MACHINE_TYPE }} \
           --scopes cloud-platform \
+          --metadata=google-monitoring-enabled=true,google-logging-enabled=true \
           --tags zebrad \
           --zone "${{ env.ZONE }}"
 
-      - name: Wait regeneration process to end
-        id: wait-regenerate
-        if: steps.regenerate-state.outcome == 'success'
+      - name: Get container name from logs
+        id: get-container-name
+        if: steps.create-instance.outcome == 'success'
         run: |
-          sleep 30s
-          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --quiet --ssh-flag="-tt" --ssh-flag="-o ServerAliveInterval=5" \
-          --command="docker logs $(docker ps -aqf ancestor=$CONTAINER_IMAGE) --follow"
+          INSTANCE_ID=$(gcloud compute instances describe zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --format='value(id)')
+          echo "Using instance: $INSTANCE_ID"
+          while [[ ${CONTAINER_NAME} != *"zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}"* ]]; do
+              CONTAINER_NAME=$(gcloud logging read 'logName=projects/${{ env.PROJECT_ID }}/logs/cos_containers AND jsonPayload."cos.googleapis.com/container_name":zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format="value(jsonPayload.'cos.googleapis.com/container_name')" --limit=1)
+              echo "Using container: ${CONTAINER_NAME}"
+          done
+          CONTAINER_NAME=$(gcloud logging read 'logName=projects/${{ env.PROJECT_ID }}/logs/cos_containers AND jsonPayload."cos.googleapis.com/container_name":zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format="value(jsonPayload.'cos.googleapis.com/container_name')" --limit=1)
+          echo "::set-output name=zebra_container::$CONTAINER_NAME"
+
+      - name: Regenerate stateful disks
+        id: sync-to-checkpoint
+        if: steps.create-instance.outcome == 'success'
+        run: |
+          gcloud compute ssh \
+          zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \
+          --zone ${{ env.ZONE }} \
+          --quiet \
+          --container=${{ env.ZEBRA_CONTAINER }} \
+          --command="cargo test --locked --release --verbose --features enable-sentry,test_sync_to_mandatory_checkpoint_${{ env.lower_net_name }} --manifest-path zebrad/Cargo.toml sync_to_mandatory_checkpoint_${{ env.lower_net_name }}"
         env:
-          CONTAINER_IMAGE: ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
+          ZEBRA_CONTAINER: ${{ steps.get-container-name.outputs.zebra_container }}
 
       # Create image from disk that will be used to sync past mandatory checkpoint test
       - name: Create image from state disk
         # Only run if the earlier step succeeds
-        if: steps.wait-regenerate.outcome == 'success'
+        if: steps.sync-to-checkpoint.outcome == 'success'
         run: |
           gcloud compute images create zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy --source-disk=zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy --source-disk-zone=${{ env.ZONE }} --storage-location=us
 
       - name: Write the disk SHORT_SHA to a txt
-        if: steps.regenerate-state.outcome == 'success'
+        if: steps.create-instance.outcome == 'success'
         run: |
           short_sha=$(echo "${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}")
           echo "$short_sha" > latest-disk-state-sha.txt
@@ -287,7 +304,7 @@ jobs:
           check_artifacts:  true
 
       - name: Get disk state SHA from txt
-        id: get_disk_sha
+        id: get-disk-sha
         run: |
           output=$(cat latest-disk-state-sha.txt)
           echo "::set-output name=sha::$output"
@@ -314,29 +331,46 @@ jobs:
           --container-tty \
           --container-command="cargo" \
           --container-arg="test" \
-          --container-arg="--locked" \
-          --container-arg="--release" \
+          --container-arg="--verbose" \
           --container-arg="--features" \
-          --container-arg="enable-sentry,test_sync_past_mandatory_checkpoint_${{ env.lower_net_name }}" \
-          --container-arg="--manifest-path" \
-          --container-arg="zebrad/Cargo.toml" \
-          --container-arg="sync_past_mandatory_checkpoint_${{ env.lower_net_name }}" \
+          --container-arg="enable-sentry" \
+          --container-arg="--test" \
+          --container-arg="acceptance" \
+          --container-arg="sync_large_checkpoints_" \
           --container-env=ZEBRA_SKIP_IPV6_TESTS=1 \
           --machine-type ${{ env.MACHINE_TYPE }} \
-          --service-account ${{ env.DEPLOY_SA }} \
           --scopes cloud-platform \
+          --metadata=google-monitoring-enabled=true,google-logging-enabled=true \
           --tags zebrad \
           --zone "${{ env.ZONE }}"
         env:
-          DISK_SHORT_SHA: ${{ steps.get_disk_sha.outputs.sha }}
+          DISK_SHORT_SHA: ${{ steps.get-disk-sha.outputs.sha }}
 
-      - name: Wait sync process to end
+      - name: Get container name from logs
+        id: get-container-name
+        if: steps.create-instance.outcome == 'success'
+        run: |
+          INSTANCE_ID=$(gcloud compute instances describe zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --format='value(id)')
+          echo "Using instance: $INSTANCE_ID"
+          while [[ ${CONTAINER_NAME} != *"zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}"* ]]; do
+              CONTAINER_NAME=$(gcloud logging read 'logName=projects/${{ env.PROJECT_ID }}/logs/cos_containers AND jsonPayload."cos.googleapis.com/container_name":zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format="value(jsonPayload.'cos.googleapis.com/container_name')" --limit=1)
+              echo "Using container: ${CONTAINER_NAME}"
+          done
+          CONTAINER_NAME=$(gcloud logging read 'logName=projects/${{ env.PROJECT_ID }}/logs/cos_containers AND jsonPayload."cos.googleapis.com/container_name":zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format="value(jsonPayload.'cos.googleapis.com/container_name')" --limit=1)
+          echo "::set-output name=zebra_container::$CONTAINER_NAME"
+
+      - name: Regenerate stateful disks
+        id: sync-past-checkpoint
+        if: steps.create-instance.outcome == 'success'
         run: |
-          sleep 30s
-          gcloud compute ssh zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --quiet --ssh-flag="-tt" --ssh-flag="-o ServerAliveInterval=5" \
-          --command="docker logs $(docker ps -aqf ancestor=$CONTAINER_IMAGE) --follow"
+          gcloud compute ssh \
+          zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \
+          --zone ${{ env.ZONE }} \
+          --quiet \
+          --container=${{ env.ZEBRA_CONTAINER }} \
+          --command="cargo test --locked --release --verbose --features enable-sentry,test_sync_past_mandatory_checkpoint_${{ env.lower_net_name }} --manifest-path zebrad/Cargo.toml sync_past_mandatory_checkpoint_${{ env.lower_net_name }}"
         env:
-          CONTAINER_IMAGE: ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
+          ZEBRA_CONTAINER: ${{ steps.get-container-name.outputs.zebra_container }}
 
       - name: Delete test instance
         # Always run even if the earlier step fails

From bc0615486077c90c48d4fa2b88f1ba42b971d359 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Mon, 14 Feb 2022 18:28:48 -0400
Subject: [PATCH 119/124] doc(test): document reasoning for new steps

---
 .github/workflows/test.yml | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 9831d7ade0e..c8645e2d2c7 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -223,6 +223,10 @@ jobs:
           --tags zebrad \
           --zone "${{ env.ZONE }}"
 
+      # TODO: this approach is very mesy, but getting the just created container name is very error prone and GCP don't have a workaround for this without requiring a TTY
+      # This TODO relates to the following issues:
+      # https://github.com/actions/runner/issues/241
+      # https://www.googlecloudcommunity.com/gc/Infrastructure-Compute-Storage/SSH-into-Compute-Container-not-easily-possible/td-p/170915
       - name: Get container name from logs
         id: get-container-name
         if: steps.create-instance.outcome == 'success'
@@ -238,7 +242,6 @@ jobs:
 
       - name: Regenerate stateful disks
         id: sync-to-checkpoint
-        if: steps.create-instance.outcome == 'success'
         run: |
           gcloud compute ssh \
           zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \
@@ -317,8 +320,8 @@ jobs:
           credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
 
       # Creates Compute Engine virtual machine instance w/ disks
-      - name: Sync past mandatory checkpoint
-        id: sync-past-checkpoint
+      - name: Create GCP compute instance
+        id: create-instance
         run: |
           gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" \
           --boot-disk-size 100GB \
@@ -346,6 +349,10 @@ jobs:
         env:
           DISK_SHORT_SHA: ${{ steps.get-disk-sha.outputs.sha }}
 
+      # TODO: this approach is very mesy, but getting the just created container name is very error prone and GCP don't have a workaround for this without requiring a TTY
+      # This TODO relates to the following issues:
+      # https://github.com/actions/runner/issues/241
+      # https://www.googlecloudcommunity.com/gc/Infrastructure-Compute-Storage/SSH-into-Compute-Container-not-easily-possible/td-p/170915
       - name: Get container name from logs
         id: get-container-name
         if: steps.create-instance.outcome == 'success'
@@ -359,9 +366,8 @@ jobs:
           CONTAINER_NAME=$(gcloud logging read 'logName=projects/${{ env.PROJECT_ID }}/logs/cos_containers AND jsonPayload."cos.googleapis.com/container_name":zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format="value(jsonPayload.'cos.googleapis.com/container_name')" --limit=1)
           echo "::set-output name=zebra_container::$CONTAINER_NAME"
 
-      - name: Regenerate stateful disks
+      - name: Sync past mandatory checkpoint
         id: sync-past-checkpoint
-        if: steps.create-instance.outcome == 'success'
         run: |
           gcloud compute ssh \
           zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \

From 0eb854606c4114675b0afaad5a783e9937c376cb Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Mon, 14 Feb 2022 23:32:32 -0400
Subject: [PATCH 120/124] fix(test): increase machine type and ssh timeout

---
 .github/workflows/test.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index c8645e2d2c7..f8acb0897e5 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -29,7 +29,7 @@ env:
   GCR_BASE: gcr.io/${{ secrets.GCP_PROJECT_ID }}
   REGION: us-central1
   ZONE: us-central1-a
-  MACHINE_TYPE: c2-standard-4
+  MACHINE_TYPE: c2-standard-8
   IMAGE_NAME: zebrad-test
 
 jobs:
@@ -247,6 +247,7 @@ jobs:
           zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \
           --zone ${{ env.ZONE }} \
           --quiet \
+          --ssh-flag="-o ServerAliveInterval=5" \
           --container=${{ env.ZEBRA_CONTAINER }} \
           --command="cargo test --locked --release --verbose --features enable-sentry,test_sync_to_mandatory_checkpoint_${{ env.lower_net_name }} --manifest-path zebrad/Cargo.toml sync_to_mandatory_checkpoint_${{ env.lower_net_name }}"
         env:
@@ -373,6 +374,7 @@ jobs:
           zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \
           --zone ${{ env.ZONE }} \
           --quiet \
+          --ssh-flag="-o ServerAliveInterval=5" \
           --container=${{ env.ZEBRA_CONTAINER }} \
           --command="cargo test --locked --release --verbose --features enable-sentry,test_sync_past_mandatory_checkpoint_${{ env.lower_net_name }} --manifest-path zebrad/Cargo.toml sync_past_mandatory_checkpoint_${{ env.lower_net_name }}"
         env:

From dc6679b35413fcb02069f54b10b0cfd6a2d8ea8e Mon Sep 17 00:00:00 2001
From: Alfredo Garcia <oxarbitrage@gmail.com>
Date: Tue, 15 Feb 2022 09:56:25 -0300
Subject: [PATCH 121/124] improve logging

Co-authored-by: teor <teor@riseup.net>
---
 zebra-network/src/peer/handshake.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/zebra-network/src/peer/handshake.rs b/zebra-network/src/peer/handshake.rs
index 8fe1bf0642a..e7107678cce 100644
--- a/zebra-network/src/peer/handshake.rs
+++ b/zebra-network/src/peer/handshake.rs
@@ -605,7 +605,7 @@ where
                     .next()
                     .await
                     .ok_or(HandshakeError::ConnectionClosed)??;
-                debug!("ignoring non-version message from remote peer");
+                debug!(?remote_msg, "ignoring non-version message from remote peer");
             }
         }
     }
@@ -733,7 +733,7 @@ where
                     .next()
                     .await
                     .ok_or(HandshakeError::ConnectionClosed)??;
-                debug!("ignoring non-verack message from remote peer");
+                debug!(?remote_msg, "ignoring non-verack message from remote peer");
             }
         }
     }

From 5b5594905e2d5a8f1edf377396e4cffcbd6df7f0 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 15 Feb 2022 12:53:01 -0400
Subject: [PATCH 122/124] fix(test): run tests on creation and follow container
 logs

This allows to follow logs in Github Actions terminal, while the GCP container is still running.

Just delete the instance when following the logs ends successfully or fails
---
 .github/workflows/test.yml | 58 ++++++++++++++++++++------------------
 1 file changed, 30 insertions(+), 28 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index f8acb0897e5..109576008b6 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -210,12 +210,13 @@ jobs:
           --container-tty \
           --container-command="cargo" \
           --container-arg="test" \
-          --container-arg="--verbose" \
+          --container-arg="--locked" \
+          --container-arg="--release" \
           --container-arg="--features" \
-          --container-arg="enable-sentry" \
-          --container-arg="--test" \
-          --container-arg="acceptance" \
-          --container-arg="sync_large_checkpoints_" \
+          --container-arg="enable-sentry,test_sync_to_mandatory_checkpoint_${{ env.lower_net_name }}" \
+          --container-arg="--manifest-path" \
+          --container-arg="zebrad/Cargo.toml" \
+          --container-arg="sync_to_mandatory_checkpoint_${{ env.lower_net_name }}" \
           --container-env=ZEBRA_SKIP_IPV6_TESTS=1 \
           --machine-type ${{ env.MACHINE_TYPE }} \
           --scopes cloud-platform \
@@ -223,7 +224,7 @@ jobs:
           --tags zebrad \
           --zone "${{ env.ZONE }}"
 
-      # TODO: this approach is very mesy, but getting the just created container name is very error prone and GCP don't have a workaround for this without requiring a TTY
+      # TODO: this approach is very mesy, but getting the just created container name is very error prone and GCP doesn't have a workaround for this without requiring a TTY
       # This TODO relates to the following issues:
       # https://github.com/actions/runner/issues/241
       # https://www.googlecloudcommunity.com/gc/Infrastructure-Compute-Storage/SSH-into-Compute-Container-not-easily-possible/td-p/170915
@@ -234,13 +235,14 @@ jobs:
           INSTANCE_ID=$(gcloud compute instances describe zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --format='value(id)')
           echo "Using instance: $INSTANCE_ID"
           while [[ ${CONTAINER_NAME} != *"zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}"* ]]; do
-              CONTAINER_NAME=$(gcloud logging read 'logName=projects/${{ env.PROJECT_ID }}/logs/cos_containers AND jsonPayload."cos.googleapis.com/container_name":zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format="value(jsonPayload.'cos.googleapis.com/container_name')" --limit=1)
-              echo "Using container: ${CONTAINER_NAME}"
+              CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-....' | tr -d "'.")
+              echo "Using container: ${CONTAINER_NAME} from instance: ${INSTANCE_ID}"
+              sleep 10
           done
-          CONTAINER_NAME=$(gcloud logging read 'logName=projects/${{ env.PROJECT_ID }}/logs/cos_containers AND jsonPayload."cos.googleapis.com/container_name":zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format="value(jsonPayload.'cos.googleapis.com/container_name')" --limit=1)
+          CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-....' | tr -d "'.")
           echo "::set-output name=zebra_container::$CONTAINER_NAME"
 
-      - name: Regenerate stateful disks
+      - name: Regenerate stateful disks logs
         id: sync-to-checkpoint
         run: |
           gcloud compute ssh \
@@ -248,8 +250,7 @@ jobs:
           --zone ${{ env.ZONE }} \
           --quiet \
           --ssh-flag="-o ServerAliveInterval=5" \
-          --container=${{ env.ZEBRA_CONTAINER }} \
-          --command="cargo test --locked --release --verbose --features enable-sentry,test_sync_to_mandatory_checkpoint_${{ env.lower_net_name }} --manifest-path zebrad/Cargo.toml sync_to_mandatory_checkpoint_${{ env.lower_net_name }}"
+          --command="docker logs --follow ${{ env.ZEBRA_CONTAINER }}"
         env:
           ZEBRA_CONTAINER: ${{ steps.get-container-name.outputs.zebra_container }}
 
@@ -274,8 +275,8 @@ jobs:
           retention-days: 1095
 
       - name: Delete test instance
-        # Always run even if the earlier step fails
-        if: ${{ always() }}
+        # Do not delete the instance if the sync timeouts in GitHub
+        if: ${{ steps.sync-to-checkpoint.outcome == 'success' }} || ${{ steps.sync-to-checkpoint.outcome == 'failure' }}
         continue-on-error: true
         run: |
           gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"
@@ -335,12 +336,13 @@ jobs:
           --container-tty \
           --container-command="cargo" \
           --container-arg="test" \
-          --container-arg="--verbose" \
+          --container-arg="--locked" \
+          --container-arg="--release" \
           --container-arg="--features" \
-          --container-arg="enable-sentry" \
-          --container-arg="--test" \
-          --container-arg="acceptance" \
-          --container-arg="sync_large_checkpoints_" \
+          --container-arg="enable-sentry,test_sync_past_mandatory_checkpoint_${{ env.lower_net_name }}" \
+          --container-arg="--manifest-path" \
+          --container-arg="zebrad/Cargo.toml" \
+          --container-arg="sync_past_mandatory_checkpoint_${{ env.lower_net_name }}" \
           --container-env=ZEBRA_SKIP_IPV6_TESTS=1 \
           --machine-type ${{ env.MACHINE_TYPE }} \
           --scopes cloud-platform \
@@ -350,7 +352,7 @@ jobs:
         env:
           DISK_SHORT_SHA: ${{ steps.get-disk-sha.outputs.sha }}
 
-      # TODO: this approach is very mesy, but getting the just created container name is very error prone and GCP don't have a workaround for this without requiring a TTY
+      # TODO: this approach is very mesy, but getting the just created container name is very error prone and GCP doesn't have a workaround for this without requiring a TTY
       # This TODO relates to the following issues:
       # https://github.com/actions/runner/issues/241
       # https://www.googlecloudcommunity.com/gc/Infrastructure-Compute-Storage/SSH-into-Compute-Container-not-easily-possible/td-p/170915
@@ -361,13 +363,14 @@ jobs:
           INSTANCE_ID=$(gcloud compute instances describe zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --format='value(id)')
           echo "Using instance: $INSTANCE_ID"
           while [[ ${CONTAINER_NAME} != *"zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}"* ]]; do
-              CONTAINER_NAME=$(gcloud logging read 'logName=projects/${{ env.PROJECT_ID }}/logs/cos_containers AND jsonPayload."cos.googleapis.com/container_name":zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format="value(jsonPayload.'cos.googleapis.com/container_name')" --limit=1)
-              echo "Using container: ${CONTAINER_NAME}"
+              CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-....' | tr -d "'.")
+              echo "Using container: ${CONTAINER_NAME} from instance: ${INSTANCE_ID}"
+              sleep 10
           done
-          CONTAINER_NAME=$(gcloud logging read 'logName=projects/${{ env.PROJECT_ID }}/logs/cos_containers AND jsonPayload."cos.googleapis.com/container_name":zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format="value(jsonPayload.'cos.googleapis.com/container_name')" --limit=1)
+          CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-....' | tr -d "'.")
           echo "::set-output name=zebra_container::$CONTAINER_NAME"
 
-      - name: Sync past mandatory checkpoint
+      - name: Sync past mandatory checkpoint logs
         id: sync-past-checkpoint
         run: |
           gcloud compute ssh \
@@ -375,14 +378,13 @@ jobs:
           --zone ${{ env.ZONE }} \
           --quiet \
           --ssh-flag="-o ServerAliveInterval=5" \
-          --container=${{ env.ZEBRA_CONTAINER }} \
-          --command="cargo test --locked --release --verbose --features enable-sentry,test_sync_past_mandatory_checkpoint_${{ env.lower_net_name }} --manifest-path zebrad/Cargo.toml sync_past_mandatory_checkpoint_${{ env.lower_net_name }}"
+          --command="docker logs --follow ${{ env.ZEBRA_CONTAINER }}"
         env:
           ZEBRA_CONTAINER: ${{ steps.get-container-name.outputs.zebra_container }}
 
       - name: Delete test instance
-        # Always run even if the earlier step fails
-        if: ${{ always() }}
+        # Do not delete the instance if the sync timeouts in GitHub
+        if: ${{ steps.sync-past-checkpoint.outcome == 'success' }} || ${{ steps.sync-past-checkpoint.outcome == 'failure' }}
         continue-on-error: true
         run: |
           gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"

From 35d74d3dfef74c285fcbd563ddc2df2498f4f890 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 15 Feb 2022 12:57:07 -0400
Subject: [PATCH 123/124] finalize(test): do not rebuild image when changing
 actions

---
 .dockerignore | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.dockerignore b/.dockerignore
index 797ef17b32f..fae64da0fe6 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -2,5 +2,5 @@ target
 Dockerfile
 .dockerignore
 .git
-# .github
+.github
 .gitignore

From 4fb6420ad77bfabe63d0ddeee17104dfb5385049 Mon Sep 17 00:00:00 2001
From: Gustavo Valverde <gustavo@iterativo.do>
Date: Tue, 15 Feb 2022 17:01:30 -0400
Subject: [PATCH 124/124] fix(test): run tests on creation and follow container
 logs

This allows to follow logs in Github Actions terminal, while the GCP container is still running.

Just delete the instance when following the logs ends successfully or fails
---
 .github/workflows/test.yml | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 109576008b6..fca0dc727ce 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -255,14 +255,20 @@ jobs:
           ZEBRA_CONTAINER: ${{ steps.get-container-name.outputs.zebra_container }}
 
       # Create image from disk that will be used to sync past mandatory checkpoint test
+      # Force the image creation as the disk is still attached even though is not being used by the container
       - name: Create image from state disk
         # Only run if the earlier step succeeds
         if: steps.sync-to-checkpoint.outcome == 'success'
         run: |
-          gcloud compute images create zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy --source-disk=zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy --source-disk-zone=${{ env.ZONE }} --storage-location=us
+          gcloud compute images create zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy \
+          --force \
+          --source-disk=zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy \
+          --source-disk-zone=${{ env.ZONE }} \
+          --storage-location=us \
+          --description="Created from head branch ${{ env.GITHUB_HEAD_REF_SLUG }} targeting ${{ env.GITHUB_BASE_REF_SLUG }} from PR ${{ env.GITHUB_REF_SLUG }} with commit ${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA }}"
 
       - name: Write the disk SHORT_SHA to a txt
-        if: steps.create-instance.outcome == 'success'
+        if: steps.sync-to-checkpoint.outcome == 'success'
         run: |
           short_sha=$(echo "${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}")
           echo "$short_sha" > latest-disk-state-sha.txt
@@ -280,7 +286,6 @@ jobs:
         continue-on-error: true
         run: |
           gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}"
-          sleep 60s
 
   test-stateful-sync:
     name: Test blocks sync