diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9fe0c628da5..0dcbf60c453 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,6 @@ on: - '**/Cargo.toml' - '**/Cargo.lock' - '**/deny.toml' - - 'docker/**' - '.github/workflows/ci.yml' env: diff --git a/.github/workflows/coverage.patch.yml b/.github/workflows/coverage.patch.yml index d3961267ed0..2896ae3a3a9 100644 --- a/.github/workflows/coverage.patch.yml +++ b/.github/workflows/coverage.patch.yml @@ -1,7 +1,6 @@ name: Coverage on: - workflow_dispatch: pull_request: paths-ignore: - '**/*.rs' diff --git a/.github/workflows/docs.patch.yml b/.github/workflows/docs.patch.yml deleted file mode 100644 index 5dea8d9af5a..00000000000 --- a/.github/workflows/docs.patch.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: Docs - -on: - workflow_dispatch: - push: - branches: - - main - paths-ignore: - - 'book/**' - - '**/firebase.json' - - 'katex-header.html' - - '.github/workflows/docs.yml' - -jobs: - build: - name: Build and Deploy Docs (+beta) - timeout-minutes: 30 - runs-on: ubuntu-latest - - steps: - - run: 'echo "No build required"' diff --git a/.github/workflows/regenerate-stateful-test-disks.yml b/.github/workflows/regenerate-stateful-test-disks.yml deleted file mode 100644 index 2db517c2636..00000000000 --- a/.github/workflows/regenerate-stateful-test-disks.yml +++ /dev/null @@ -1,70 +0,0 @@ -name: Regenerate test state - -on: - workflow_dispatch: - inputs: - network: - default: 'mainnet' - -env: - PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} - ZONE: europe-west1-b - MACHINE_TYPE: n2-standard-4 - DEPLOY_SA: cos-vm@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com - -jobs: - - regenerate: - name: Regenerate test state - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2.4.0 - with: - persist-credentials: false - - - name: Inject slug/short variables - uses: rlespinasse/github-slug-action@v4 - - - name: Set up gcloud - uses: google-github-actions/setup-gcloud@v0.5.0 - with: - project_id: ${{ env.PROJECT_ID }} - service_account_key: ${{ secrets.GCLOUD_AUTH }} - - # Creates Compute Engine virtual machine instance w/ disks - - name: Create instance - run: | - gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \ - --boot-disk-size 100GB \ - --boot-disk-type pd-ssd \ - --container-image rust:buster \ - --container-mount-disk mount-path='/${{ github.event.inputs.network }}',name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy" \ - --container-restart-policy never \ - --create-disk name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy",size=100GB,type=pd-balanced \ - --machine-type ${{ env.MACHINE_TYPE }} \ - --service-account ${{ env.DEPLOY_SA }} \ - --scopes cloud-platform \ - --tags zebrad \ - --zone "${{ env.ZONE }}" - # Build and run test container to sync up to activation and no further - - name: Regenerate state for tests - id: regenerate-state - run: | - gcloud compute ssh "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --zone "${{ env.ZONE }}" --command \ - "git clone -b ${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }} https://github.com/ZcashFoundation/zebra.git && - cd zebra/ && - docker build --build-arg SHORT_SHA=${{ env.GITHUB_SHA_SHORT }} -f docker/Dockerfile.test -t zebrad-test . && - docker run -i -e "ZEBRA_SKIP_IPV6_TESTS=1" --mount type=bind,source=/mnt/disks/gce-containers-mounts/gce-persistent-disks/zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy,target=/zebrad-cache zebrad-test:latest cargo test --verbose --features test_sync_to_mandatory_checkpoint_${{ github.event.inputs.network }} --manifest-path zebrad/Cargo.toml sync_to_mandatory_checkpoint_${{ github.event.inputs.network }}; - " - # Create image from disk that will be used in test.yml workflow - - name: Create image from state disk - # Only run if the earlier step succeeds - if: steps.regenerate-state.outcome == 'success' - run: | - gcloud compute images create "zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy" --source-disk="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ github.event.inputs.network }}-canopy" --source-disk-zone="${{ env.ZONE }}" - # Clean up - - name: Delete test instance - # Always run even if the earlier step fails - if: ${{ always() }} - run: | - gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}" diff --git a/.github/workflows/test.patch.yml b/.github/workflows/test.patch.yml deleted file mode 100644 index 9918e6e3cfd..00000000000 --- a/.github/workflows/test.patch.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Test - -on: - workflow_dispatch: - push: - branches: - - main - paths-ignore: - - '**/*.rs' - - '**/*.txt' - - '**/Cargo.toml' - - '**/Cargo.lock' - - 'docker/**' - - '.github/workflows/test.yml' - -jobs: - test: - name: Run all tests - runs-on: ubuntu-latest - - steps: - - run: 'echo "No build required"' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f99dabdc996..fca0dc727ce 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,7 +2,14 @@ name: Test on: workflow_dispatch: - push: + inputs: + network: + default: 'Mainnet' + regenerate-disks: + type: boolean + default: false + description: Just update stateful disks + pull_request: branches: - main paths: @@ -14,59 +21,375 @@ on: - '.github/workflows/test.yml' env: + CARGO_INCREMENTAL: '1' + ZEBRA_SKIP_IPV6_TESTS: "1" + NETWORK: Mainnet PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} - ZONE: europe-west1-b - MACHINE_TYPE: n2-standard-8 - DEPLOY_SA: cos-vm@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com + GAR_BASE: us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/zebra + GCR_BASE: gcr.io/${{ secrets.GCP_PROJECT_ID }} + REGION: us-central1 + ZONE: us-central1-a + MACHINE_TYPE: c2-standard-8 + IMAGE_NAME: zebrad-test jobs: + build: + name: Build images + timeout-minutes: 210 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2.4.0 + with: + persist-credentials: false + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + + # Setup Docker Buildx to allow use of docker cache layers from GH + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + with: + driver-opts: network=host + + - name: Login to Google Artifact Registry + uses: docker/login-action@v1.12.0 + with: + registry: us-docker.pkg.dev + username: _json_key + password: ${{ secrets.GOOGLE_CREDENTIALS }} + + - name: Login to Google Container Registry + uses: docker/login-action@v1.12.0 + with: + registry: gcr.io + username: _json_key + password: ${{ secrets.GOOGLE_CREDENTIALS }} + + # Build and push image to Google Artifact Registry + - name: Build & push + id: docker_build + uses: docker/build-push-action@v2.8.0 + with: + target: tester + context: . + file: ./docker/Dockerfile.build + tags: | + ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:latest + ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} + ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.IMAGE_NAME }}:latest + ${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} + build-args: | + NETWORK=${{ github.event.inputs.network || env.NETWORK }} + SHORT_SHA=${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} + RUST_BACKTRACE=full + ZEBRA_SKIP_NETWORK_TESTS="1" + CHECKPOINT_SYNC=${{ github.event.inputs.checkpoint_sync || true }} + RUST_LOG=debug + SENTRY_DSN=${{ secrets.SENTRY_ENDPOINT }} + push: true + cache-from: type=gha + cache-to: type=gha,mode=max + + test-all: + name: Test all + runs-on: ubuntu-latest + needs: build + if: ${{ github.event.inputs.regenerate-disks != 'true' }} + steps: + - uses: actions/checkout@v2.4.0 + with: + persist-credentials: false + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + + - name: Run all zebrad tests + run: | + docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} + docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --workspace + + test-fake-activation-heights: + name: Test with fake activation heights + runs-on: ubuntu-latest + needs: build + if: ${{ github.event.inputs.regenerate-disks != 'true' }} + steps: + - uses: actions/checkout@v2.4.0 + with: + persist-credentials: false + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + + - name: Run tests with fake activation heights + run: | + docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} + docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} cargo test --locked --release --package zebra-state --lib -- with_fake_activation_heights + + test-large-sync: + name: Test large sync + runs-on: ubuntu-latest + needs: build + if: ${{ github.event.inputs.regenerate-disks != 'true' }} + steps: + - uses: actions/checkout@v2.4.0 + with: + persist-credentials: false + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + + - name: Run zebrad large sync tests + run: | + docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} + docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --test acceptance sync_large_checkpoints_ -- --ignored + + test-zunstable: + name: Test with Zunstable options + runs-on: ubuntu-latest + needs: build + if: ${{ github.event.inputs.regenerate-disks != 'true' }} + steps: + - uses: actions/checkout@v2.4.0 + with: + persist-credentials: false + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + + - name: Run Zunstable-options tests + run: | + docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} + docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --workspace -- -Zunstable-options --include-ignored - test: - name: Run all tests + regenerate-stateful-disks: + name: Renerate state disks runs-on: ubuntu-latest + needs: build steps: - uses: actions/checkout@v2.4.0 with: persist-credentials: false + fetch-depth: '2' + + - name: Get specific changed files + id: changed-files-specific + uses: tj-actions/changed-files@v14.4 + with: + files: | + /zebra-state/**/disk_format.rs + /zebra-state/**/finalized_state.rs + /zebra-state/**/constants.rs - name: Inject slug/short variables uses: rlespinasse/github-slug-action@v4 - - name: Set up gcloud - uses: google-github-actions/setup-gcloud@v0.5.0 + - name: Downcase network name for disks + run: | + echo LOWER_NET_NAME="${{ github.event.inputs.network || env.NETWORK }}" | awk '{print tolower($0)}' >> $GITHUB_ENV + + # Setup gcloud CLI + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v0.5.0 with: - project_id: ${{ env.PROJECT_ID }} - service_account_key: ${{ secrets.GCLOUD_AUTH }} + credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }} + + - name: Create GCP compute instance + id: create-instance + if: ${{ steps.changed-files-specific.outputs.any_changed == 'true' }} || ${{ github.event.inputs.regenerate-disks == 'true' }} + run: | + gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" \ + --boot-disk-size 100GB \ + --boot-disk-type pd-ssd \ + --create-disk name="zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy",size=100GB,type=pd-ssd \ + --container-mount-disk mount-path='/zebrad-cache',name="zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy" \ + --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \ + --container-restart-policy=never \ + --container-stdin \ + --container-tty \ + --container-command="cargo" \ + --container-arg="test" \ + --container-arg="--locked" \ + --container-arg="--release" \ + --container-arg="--features" \ + --container-arg="enable-sentry,test_sync_to_mandatory_checkpoint_${{ env.lower_net_name }}" \ + --container-arg="--manifest-path" \ + --container-arg="zebrad/Cargo.toml" \ + --container-arg="sync_to_mandatory_checkpoint_${{ env.lower_net_name }}" \ + --container-env=ZEBRA_SKIP_IPV6_TESTS=1 \ + --machine-type ${{ env.MACHINE_TYPE }} \ + --scopes cloud-platform \ + --metadata=google-monitoring-enabled=true,google-logging-enabled=true \ + --tags zebrad \ + --zone "${{ env.ZONE }}" + + # TODO: this approach is very mesy, but getting the just created container name is very error prone and GCP doesn't have a workaround for this without requiring a TTY + # This TODO relates to the following issues: + # https://github.com/actions/runner/issues/241 + # https://www.googlecloudcommunity.com/gc/Infrastructure-Compute-Storage/SSH-into-Compute-Container-not-easily-possible/td-p/170915 + - name: Get container name from logs + id: get-container-name + if: steps.create-instance.outcome == 'success' + run: | + INSTANCE_ID=$(gcloud compute instances describe zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --format='value(id)') + echo "Using instance: $INSTANCE_ID" + while [[ ${CONTAINER_NAME} != *"zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}"* ]]; do + CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-....' | tr -d "'.") + echo "Using container: ${CONTAINER_NAME} from instance: ${INSTANCE_ID}" + sleep 10 + done + CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-....' | tr -d "'.") + echo "::set-output name=zebra_container::$CONTAINER_NAME" + + - name: Regenerate stateful disks logs + id: sync-to-checkpoint + run: | + gcloud compute ssh \ + zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=5" \ + --command="docker logs --follow ${{ env.ZEBRA_CONTAINER }}" + env: + ZEBRA_CONTAINER: ${{ steps.get-container-name.outputs.zebra_container }} + + # Create image from disk that will be used to sync past mandatory checkpoint test + # Force the image creation as the disk is still attached even though is not being used by the container + - name: Create image from state disk + # Only run if the earlier step succeeds + if: steps.sync-to-checkpoint.outcome == 'success' + run: | + gcloud compute images create zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy \ + --force \ + --source-disk=zebrad-cache-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-${{ env.lower_net_name }}-canopy \ + --source-disk-zone=${{ env.ZONE }} \ + --storage-location=us \ + --description="Created from head branch ${{ env.GITHUB_HEAD_REF_SLUG }} targeting ${{ env.GITHUB_BASE_REF_SLUG }} from PR ${{ env.GITHUB_REF_SLUG }} with commit ${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA }}" + + - name: Write the disk SHORT_SHA to a txt + if: steps.sync-to-checkpoint.outcome == 'success' + run: | + short_sha=$(echo "${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}") + echo "$short_sha" > latest-disk-state-sha.txt + + - name: Upload the disk state txt + uses: actions/upload-artifact@v2.3.1 + with: + name: latest-disk-state-sha + path: latest-disk-state-sha.txt + retention-days: 1095 + + - name: Delete test instance + # Do not delete the instance if the sync timeouts in GitHub + if: ${{ steps.sync-to-checkpoint.outcome == 'success' }} || ${{ steps.sync-to-checkpoint.outcome == 'failure' }} + continue-on-error: true + run: | + gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}" + + test-stateful-sync: + name: Test blocks sync + runs-on: ubuntu-latest + needs: [ build, regenerate-stateful-disks] + if: ${{ github.event.inputs.regenerate-disks != 'true' }} + steps: + - uses: actions/checkout@v2.4.0 + with: + persist-credentials: false + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + + - name: Downcase network name for disks + run: | + echo LOWER_NET_NAME="${{ github.event.inputs.network || env.NETWORK }}" | awk '{print tolower($0)}' >> $GITHUB_ENV + + # Get the latest uploaded txt with the disk SHORT_SHA from this workflow + - name: Download latest disk state SHORT_SHA + uses: dawidd6/action-download-artifact@v2.17.0 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + workflow: test.yml + name: latest-disk-state-sha + check_artifacts: true + + - name: Get disk state SHA from txt + id: get-disk-sha + run: | + output=$(cat latest-disk-state-sha.txt) + echo "::set-output name=sha::$output" + + # Setup gcloud CLI + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v0.5.0 + with: + credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }} # Creates Compute Engine virtual machine instance w/ disks - - name: Create instance + - name: Create GCP compute instance + id: create-instance run: | - gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \ + gcloud compute instances create-with-container "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" \ --boot-disk-size 100GB \ --boot-disk-type pd-ssd \ - --container-image rust:buster \ - --container-mount-disk mount-path='/mainnet',name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-mainnet-canopy" \ - --container-restart-policy never \ - --create-disk name="zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-mainnet-canopy",image=zebrad-cache-1558f3378-mainnet-canopy \ + --disk=boot=no,mode=rw,name=zebrad-cache-${{ env.DISK_SHORT_SHA }}-${{ env.lower_net_name }}-canopy \ + --container-mount-disk=mount-path='/zebrad-cache',name=zebrad-cache-${{ env.DISK_SHORT_SHA }}-${{ env.lower_net_name }}-canopy \ + --container-image ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \ + --container-restart-policy=never \ + --container-stdin \ + --container-tty \ + --container-command="cargo" \ + --container-arg="test" \ + --container-arg="--locked" \ + --container-arg="--release" \ + --container-arg="--features" \ + --container-arg="enable-sentry,test_sync_past_mandatory_checkpoint_${{ env.lower_net_name }}" \ + --container-arg="--manifest-path" \ + --container-arg="zebrad/Cargo.toml" \ + --container-arg="sync_past_mandatory_checkpoint_${{ env.lower_net_name }}" \ + --container-env=ZEBRA_SKIP_IPV6_TESTS=1 \ --machine-type ${{ env.MACHINE_TYPE }} \ - --service-account ${{ env.DEPLOY_SA }} \ --scopes cloud-platform \ + --metadata=google-monitoring-enabled=true,google-logging-enabled=true \ --tags zebrad \ --zone "${{ env.ZONE }}" + env: + DISK_SHORT_SHA: ${{ steps.get-disk-sha.outputs.sha }} + + # TODO: this approach is very mesy, but getting the just created container name is very error prone and GCP doesn't have a workaround for this without requiring a TTY + # This TODO relates to the following issues: + # https://github.com/actions/runner/issues/241 + # https://www.googlecloudcommunity.com/gc/Infrastructure-Compute-Storage/SSH-into-Compute-Container-not-easily-possible/td-p/170915 + - name: Get container name from logs + id: get-container-name + if: steps.create-instance.outcome == 'success' + run: | + INSTANCE_ID=$(gcloud compute instances describe zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --format='value(id)') + echo "Using instance: $INSTANCE_ID" + while [[ ${CONTAINER_NAME} != *"zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}"* ]]; do + CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-....' | tr -d "'.") + echo "Using container: ${CONTAINER_NAME} from instance: ${INSTANCE_ID}" + sleep 10 + done + CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}-....' | tr -d "'.") + echo "::set-output name=zebra_container::$CONTAINER_NAME" + + - name: Sync past mandatory checkpoint logs + id: sync-past-checkpoint + run: | + gcloud compute ssh \ + zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=5" \ + --command="docker logs --follow ${{ env.ZEBRA_CONTAINER }}" + env: + ZEBRA_CONTAINER: ${{ steps.get-container-name.outputs.zebra_container }} - # Build and run test container - - name: Run all tests - run: | - gcloud compute ssh "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --ssh-flag="-o ServerAliveInterval=5" --zone "${{ env.ZONE }}" --command \ - "git clone -b ${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }} https://github.com/ZcashFoundation/zebra.git && - cd zebra/ && - docker build --build-arg SHORT_SHA=${{ env.GITHUB_SHA_SHORT }} -f docker/Dockerfile.test -t zebrad-test . && - docker run -t -e ZEBRA_SKIP_IPV6_TESTS=1 zebrad-test:latest cargo test --workspace --no-fail-fast -- -Zunstable-options --include-ignored && - docker run -t -e ZEBRA_SKIP_IPV6_TESTS=1 --mount type=bind,source=/mnt/disks/gce-containers-mounts/gce-persistent-disks/zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-mainnet-canopy,target=/zebrad-cache zebrad-test:latest cargo test --verbose --features test_sync_past_mandatory_checkpoint_mainnet --manifest-path zebrad/Cargo.toml sync_past_mandatory_checkpoint_mainnet - " - # Clean up - name: Delete test instance - # Always run even if the earlier step fails - if: ${{ always() }} + # Do not delete the instance if the sync timeouts in GitHub + if: ${{ steps.sync-past-checkpoint.outcome == 'success' }} || ${{ steps.sync-past-checkpoint.outcome == 'failure' }} + continue-on-error: true run: | - gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}" + gcloud compute instances delete "zebrad-tests-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}" --delete-disks all --zone "${{ env.ZONE }}" diff --git a/.github/workflows/zcashd-manual-deploy.yml b/.github/workflows/zcashd-manual-deploy.yml index 927ac5e6d67..846a43bd436 100644 --- a/.github/workflows/zcashd-manual-deploy.yml +++ b/.github/workflows/zcashd-manual-deploy.yml @@ -4,22 +4,22 @@ on: workflow_dispatch: inputs: network: - default: 'testnet' + default: 'Testnet' size: default: 10 env: PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} REGION: us-central1 - MACHINE_TYPE: n2-standard-4 - DEPLOY_SA: cos-vm@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com + ZONE: us-central1-a + MACHINE_TYPE: c2-standard-4 jobs: - deploy: name: Deploy zcashd nodes runs-on: ubuntu-latest timeout-minutes: 30 + steps: - uses: actions/checkout@v2.4.0 with: @@ -38,14 +38,17 @@ jobs: # Create instance template from container image - name: Create instance template run: | - gcloud compute instance-templates create-with-container "zcashd-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }}" \ - --boot-disk-size 100GB \ - --container-image "electriccoinco/zcashd" \ + gcloud compute instance-templates create-with-container zcashd-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}-${{ env.GITHUB_SHA_SHORT }} \ + --boot-disk-size 10GB \ + --boot-disk-type=pd-ssd \ + --container-stdin \ + --container-tty \ + --container-image electriccoinco/zcashd \ --container-env ZCASHD_NETWORK="${{ github.event.inputs.network }}" \ --machine-type ${{ env.MACHINE_TYPE }} \ --service-account ${{ env.DEPLOY_SA }} \ --scopes cloud-platform \ - --tags zcashd \ + --tags zcashd # Check if our destination instance group exists already - name: Check if instance group exists diff --git a/zebra-chain/src/transparent/serialize.rs b/zebra-chain/src/transparent/serialize.rs index 0a9d01915c4..11cd98964e1 100644 --- a/zebra-chain/src/transparent/serialize.rs +++ b/zebra-chain/src/transparent/serialize.rs @@ -1,3 +1,5 @@ +//! Serializes and deserializes transparent data. + use std::io; use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; @@ -14,13 +16,24 @@ use crate::{ use super::{CoinbaseData, Input, OutPoint, Output, Script}; /// The maximum length of the coinbase data. +/// /// Includes the encoded coinbase height, if any. /// /// > The number of bytes in the coinbase script, up to a maximum of 100 bytes. /// -/// https://developer.bitcoin.org/reference/transactions.html#coinbase-input-the-input-of-the-first-transaction-in-a-block +/// pub const MAX_COINBASE_DATA_LEN: usize = 100; +/// The minimum length of the coinbase data. +/// +/// Includes the encoded coinbase height, if any. +/// +// TODO: Update the link below once the constant is documented in the +// protocol. +/// +/// +pub const MIN_COINBASE_DATA_LEN: usize = 2; + /// The coinbase data for a genesis block. /// /// Zcash uses the same coinbase data for the Mainnet, Testnet, and Regtest @@ -68,18 +81,19 @@ impl ZcashDeserialize for OutPoint { /// /// # Consensus /// -/// > A coinbase transaction for a block at block height greater than 0 MUST have -/// > a script that, as its first item, encodes the block height height as follows. -/// > For height in the range {1..16}, the encoding is a single byte of value -/// > 0x50 height. Otherwise, let heightBytes be the signed little-endian -/// > representation of height, using the minimum nonzero number of bytes such that -/// > the most significant byte is < 0x80. -/// > The length of heightBytes MUST be in the range {1..5}. -/// > Then the encoding is the length of heightBytes encoded as one byte, -/// > followed by heightBytes itself. This matches the encoding used by Bitcoin in the +/// > A coinbase transaction for a *block* at *block height* greater than 0 MUST have +/// > a script that, as its first item, encodes the *block height* `height` as follows. +/// > For `height` in the range {1..16}, the encoding is a single byte of value +/// > `0x50` + `height`. Otherwise, let `heightBytes` be the signed little-endian +/// > representation of `height`, using the minimum nonzero number of bytes such that +/// > the most significant byte is < `0x80`. +/// > The length of `heightBytes` MUST be in the range {1..5}. +/// > Then the encoding is the length of `heightBytes` encoded as one byte, +/// > followed by `heightBytes` itself. This matches the encoding used by Bitcoin in the /// > implementation of [BIP-34] (but the description here is to be considered normative). /// /// +/// pub(crate) fn parse_coinbase_height( mut data: Vec, ) -> Result<(block::Height, CoinbaseData), SerializationError> { @@ -262,9 +276,14 @@ impl ZcashDeserialize for Input { } let data: Vec = (&mut reader).zcash_deserialize_into()?; + + // Check the coinbase data length. if data.len() > MAX_COINBASE_DATA_LEN { - return Err(SerializationError::Parse("coinbase has too much data")); + return Err(SerializationError::Parse("coinbase data is too long")); + } else if data.len() < MIN_COINBASE_DATA_LEN { + return Err(SerializationError::Parse("coinbase data is too short")); } + let (height, data) = parse_coinbase_height(data)?; let sequence = reader.read_u32::()?; diff --git a/zebra-network/src/peer/handshake.rs b/zebra-network/src/peer/handshake.rs index d5d5b4d1b32..cc784a46730 100644 --- a/zebra-network/src/peer/handshake.rs +++ b/zebra-network/src/peer/handshake.rs @@ -588,13 +588,29 @@ where debug!(?our_version, "sending initial version message"); peer_conn.send(our_version).await?; - let remote_msg = peer_conn + let mut remote_msg = peer_conn .next() .await .ok_or(HandshakeError::ConnectionClosed)??; - // Check that we got a Version and destructure its fields into the local scope. - debug!(?remote_msg, "got message from remote peer"); + // Wait for next message if the one we got is not Version + loop { + match remote_msg { + Message::Version { .. } => { + debug!(?remote_msg, "got version message from remote peer"); + break; + } + _ => { + remote_msg = peer_conn + .next() + .await + .ok_or(HandshakeError::ConnectionClosed)??; + debug!(?remote_msg, "ignoring non-version message from remote peer"); + } + } + } + + // If we got a Version message, destructure its fields into the local scope. let (remote_nonce, remote_services, remote_version, remote_canonical_addr, user_agent) = if let Message::Version { version, @@ -700,14 +716,26 @@ where peer_conn.send(Message::Verack).await?; - let remote_msg = peer_conn + let mut remote_msg = peer_conn .next() .await .ok_or(HandshakeError::ConnectionClosed)??; - if let Message::Verack = remote_msg { - debug!("got verack from remote peer"); - } else { - Err(HandshakeError::UnexpectedMessage(Box::new(remote_msg)))?; + + // Wait for next message if the one we got is not Verack + loop { + match remote_msg { + Message::Verack => { + debug!(?remote_msg, "got verack message from remote peer"); + break; + } + _ => { + remote_msg = peer_conn + .next() + .await + .ok_or(HandshakeError::ConnectionClosed)??; + debug!(?remote_msg, "ignoring non-verack message from remote peer"); + } + } } Ok((remote_version, remote_services, remote_canonical_addr))