From 89818e84503bd52756e60f9ca9200b29c9bf4850 Mon Sep 17 00:00:00 2001 From: teor Date: Sat, 27 Aug 2022 12:58:28 +1000 Subject: [PATCH 1/6] Expand cached state disks before running tests --- .github/workflows/deploy-gcp-tests.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-gcp-tests.yml b/.github/workflows/deploy-gcp-tests.yml index f21739d239e..b723cb9c226 100644 --- a/.github/workflows/deploy-gcp-tests.yml +++ b/.github/workflows/deploy-gcp-tests.yml @@ -148,6 +148,9 @@ jobs: --zone ${{ env.ZONE }} sleep 60 + # Create a docker volume with the new disk we just created. + # + # SSH into the just created VM, and create a docker volume with the newly created disk. - name: Create ${{ inputs.test_id }} Docker volume run: | gcloud compute ssh \ @@ -157,7 +160,7 @@ jobs: --ssh-flag="-o ServerAliveInterval=5" \ --command \ "\ - sudo mkfs.ext4 /dev/sdb \ + sudo mkfs.ext4 -v /dev/sdb \ && \ docker volume create --driver local --opt type=ext4 --opt device=/dev/sdb \ ${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }} \ @@ -351,7 +354,10 @@ jobs: # Create a docker volume with the selected cached state. # - # SSH into the just created VM, and create a docker volume with the recently attached disk. + # SSH into the just created VM, expand the partition and filesystem to fill the entire disk, + # then create a docker volume with the recently attached disk. + # (The cached state and disk are usually the same size, + # but the cached state can be smaller if we just increased the disk size.) - name: Create ${{ inputs.test_id }} Docker volume run: | gcloud compute ssh \ @@ -361,6 +367,10 @@ jobs: --ssh-flag="-o ServerAliveInterval=5" \ --command \ "\ + sudo growpart --verbose /dev/sdb 1 \ + && \ + sudo resize2fs -p /dev/sdb \ + && \ docker volume create --driver local --opt type=ext4 --opt device=/dev/sdb \ ${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }} \ " From 60101a4928697f5f237ae1cdde570e948b57759e Mon Sep 17 00:00:00 2001 From: teor Date: Sat, 27 Aug 2022 13:30:36 +1000 Subject: [PATCH 2/6] Install partition management tool --- .github/workflows/deploy-gcp-tests.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/deploy-gcp-tests.yml b/.github/workflows/deploy-gcp-tests.yml index b723cb9c226..84e229be788 100644 --- a/.github/workflows/deploy-gcp-tests.yml +++ b/.github/workflows/deploy-gcp-tests.yml @@ -367,6 +367,11 @@ jobs: --ssh-flag="-o ServerAliveInterval=5" \ --command \ "\ + sudo apt-get --quiet update --yes \ + && \ + sudo apt-get --quiet install --yes --no-install-recommends \ + cloud-guest-utils \ + && \ sudo growpart --verbose /dev/sdb 1 \ && \ sudo resize2fs -p /dev/sdb \ From d50900a248e119935969fd63397a5581668fa0b1 Mon Sep 17 00:00:00 2001 From: teor Date: Sat, 27 Aug 2022 14:25:20 +1000 Subject: [PATCH 3/6] There isn't actually a partition on the cached state image --- .github/workflows/deploy-gcp-tests.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/deploy-gcp-tests.yml b/.github/workflows/deploy-gcp-tests.yml index 84e229be788..8d784f37475 100644 --- a/.github/workflows/deploy-gcp-tests.yml +++ b/.github/workflows/deploy-gcp-tests.yml @@ -367,12 +367,7 @@ jobs: --ssh-flag="-o ServerAliveInterval=5" \ --command \ "\ - sudo apt-get --quiet update --yes \ - && \ - sudo apt-get --quiet install --yes --no-install-recommends \ - cloud-guest-utils \ - && \ - sudo growpart --verbose /dev/sdb 1 \ + sudo e2fsck -v -f /dev/sdb \ && \ sudo resize2fs -p /dev/sdb \ && \ From fa218fef20fb48ac7ce225a0b00730586d09ecf1 Mon Sep 17 00:00:00 2001 From: teor Date: Sat, 27 Aug 2022 14:31:07 +1000 Subject: [PATCH 4/6] Make e2fsck non-interactive --- .github/workflows/deploy-gcp-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-gcp-tests.yml b/.github/workflows/deploy-gcp-tests.yml index 8d784f37475..5380f39a27b 100644 --- a/.github/workflows/deploy-gcp-tests.yml +++ b/.github/workflows/deploy-gcp-tests.yml @@ -367,7 +367,7 @@ jobs: --ssh-flag="-o ServerAliveInterval=5" \ --command \ "\ - sudo e2fsck -v -f /dev/sdb \ + sudo e2fsck -v -f -p /dev/sdb \ && \ sudo resize2fs -p /dev/sdb \ && \ From 0359f07bfe4213b3cdd3e481d0ff4320a21539ea Mon Sep 17 00:00:00 2001 From: teor Date: Sun, 28 Aug 2022 06:03:44 +1000 Subject: [PATCH 5/6] Limit the length of image names to 63 characters --- .github/workflows/deploy-gcp-tests.yml | 34 ++++++++++++++++++-------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/.github/workflows/deploy-gcp-tests.yml b/.github/workflows/deploy-gcp-tests.yml index 5380f39a27b..479be773eb0 100644 --- a/.github/workflows/deploy-gcp-tests.yml +++ b/.github/workflows/deploy-gcp-tests.yml @@ -938,14 +938,22 @@ jobs: with: short-length: 7 + # Performs formatting on disk name components. + # # Disk images in GCP are required to be in lowercase, but the blockchain network - # uses sentence case, so we need to downcase ${{ inputs.network }} + # uses sentence case, so we need to downcase ${{ inputs.network }}. # - # Passes ${{ inputs.network }} to subsequent steps using $NETWORK env variable - - name: Downcase network name for disks + # Disk image names in GCP are limited to 63 characters, so we need to limit + # branch names to 13 characters. + # + # Passes ${{ inputs.network }} to subsequent steps using $NETWORK env variable. + # Passes ${{ env.GITHUB_REF_SLUG_URL }} to subsequent steps using $SHORT_GITHUB_REF env variable. + - name: Format network name and branch name for disks run: | NETWORK_CAPS=${{ inputs.network }} echo "NETWORK=${NETWORK_CAPS,,}" >> $GITHUB_ENV + LONG_GITHUB_REF=${{ env.GITHUB_REF_SLUG_URL }} + echo "SHORT_GITHUB_REF=${LONG_GITHUB_REF:0:13}" >> $GITHUB_ENV # Setup gcloud CLI - name: Authenticate to Google Cloud @@ -990,7 +998,7 @@ jobs: SYNC_HEIGHT=$(echo $DOCKER_LOGS | grep -oE '${{ inputs.height_grep_text }}\([0-9]+\)' | grep -oE '[0-9]+' | tail -1 || [[ $? == 1 ]]) echo "SYNC_HEIGHT=$SYNC_HEIGHT" >> $GITHUB_ENV - # Sets the $UPDATE_SUFFIX env var to "-update" if using cached state, + # Sets the $UPDATE_SUFFIX env var to "-u" if using cached state, # and the empty string otherwise. # # Also sets a unique date and time suffix $TIME_SUFFIX. @@ -999,26 +1007,32 @@ jobs: UPDATE_SUFFIX="" if [[ "${{ inputs.needs_zebra_state }}" == "true" ]]; then - UPDATE_SUFFIX="-update" + UPDATE_SUFFIX="-u" fi - TIME_SUFFIX=$(date '+%Y-%m-%d-%H-%M-%S' --utc) + # We're going to delete old images after a month, so we don't need the year here + TIME_SUFFIX=$(date '+%m%d%H%M%S' --utc) echo "UPDATE_SUFFIX=$UPDATE_SUFFIX" >> $GITHUB_ENV echo "TIME_SUFFIX=$TIME_SUFFIX" >> $GITHUB_ENV - # Create an image from disk that will be used for following/other tests + # Create an image from disk that will be used for following/other tests. + # # This image can contain: # - Zebra cached state # - Zebra + lightwalletd cached state - # Which cached state is being saved to the disk is defined by ${{ inputs.disk_prefix }} + # Which cached state is being saved to the disk is defined by ${{ inputs.disk_prefix }}. + # + # The image name must be unique, and be 63 characters or less. + # The timestamp makes images from the same commit unique, + # as long as they don't finish in the same second. # # Force the image creation (--force) as the disk is still attached even though is not being - # used by the container + # used by the container. - name: Create image from state disk run: | gcloud compute images create \ - "${{ inputs.disk_prefix }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}-v${{ env.STATE_VERSION }}-${{ env.NETWORK }}-${{ inputs.disk_suffix }}$UPDATE_SUFFIX-$TIME_SUFFIX" \ + "${{ inputs.disk_prefix }}-${SHORT_GITHUB_REF}-${{ env.GITHUB_SHA_SHORT }}-v${{ env.STATE_VERSION }}-${{ env.NETWORK }}-${{ inputs.disk_suffix }}${UPDATE_SUFFIX}-${TIME_SUFFIX}" \ --force \ --source-disk=${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }} \ --source-disk-zone=${{ env.ZONE }} \ From e098efa92459c36732473561806dc449f7de75ee Mon Sep 17 00:00:00 2001 From: teor Date: Sun, 28 Aug 2022 06:10:45 +1000 Subject: [PATCH 6/6] Ignore possibly long branch names when matching images, just match the commit --- .github/workflows/deploy-gcp-tests.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy-gcp-tests.yml b/.github/workflows/deploy-gcp-tests.yml index 479be773eb0..e70a8415b88 100644 --- a/.github/workflows/deploy-gcp-tests.yml +++ b/.github/workflows/deploy-gcp-tests.yml @@ -288,11 +288,16 @@ jobs: DISK_PREFIX=${{ inputs.zebra_state_dir || inputs.disk_prefix }} fi - # Try to find an image generated from this branch and commit - # Fields are listed in the "Create image from state disk" step - COMMIT_DISK_PREFIX="${DISK_PREFIX}-${GITHUB_REF_SLUG_URL}-${GITHUB_SHA_SHORT}-v${LOCAL_STATE_VERSION}-${NETWORK}-${{ inputs.disk_suffix }}" + # Try to find an image generated from a previous step or run of this commit. + # Fields are listed in the "Create image from state disk" step. + # + # We can't match the full branch name here, + # because it might have been shortened for the image. + # + # The probability of two matching short commit hashes within the same month is very low. + COMMIT_DISK_PREFIX="${DISK_PREFIX}-.+-${{ env.GITHUB_SHA_SHORT }}-v${LOCAL_STATE_VERSION}-${NETWORK}-${{ inputs.disk_suffix }}" COMMIT_CACHED_DISK_NAME=$(gcloud compute images list --filter="name~${COMMIT_DISK_PREFIX}" --format="value(NAME)" --sort-by=~creationTimestamp --limit=1) - echo "${GITHUB_REF_SLUG_URL}-${GITHUB_SHA_SHORT} Disk: $COMMIT_CACHED_DISK_NAME" + echo "${GITHUB_REF_SLUG_URL}-${{ env.GITHUB_SHA_SHORT }} Disk: $COMMIT_CACHED_DISK_NAME" if [[ -n "$COMMIT_CACHED_DISK_NAME" ]]; then echo "Description: $(gcloud compute images describe $COMMIT_CACHED_DISK_NAME --format='value(DESCRIPTION)')" fi