feat(ci): validate state version with cached state

.github/workflows/test-full-sync.yml

@@ -81,6 +81,28 @@ jobs:
         with:
           short-length: 7
 
+      - name: Authenticate to Google Cloud
+        id: auth
+        uses: google-github-actions/[email protected]
+        with:
+          workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc'
+          service_account: '[email protected]'
+          token_format: 'access_token'
+
+      # Before executing any further steps, validate the local state and remote version are the same,
+      # or at least that the local state version is greater than the available cached state version from main.
+      - name: Validate constants.rs version vs cached state version
+        id: validate-state-version
+        run: |
+          LOCAL_STATE_VERSION=$(grep -oE "DATABASE_FORMAT_VERSION: .* [0-9]+" "$GITHUB_WORKSPACE/zebra-state/src/constants.rs" | grep -oE "[0-9]+" | tail -n1)
+          echo "LOCAL_STATE_VERSION: $LOCAL_STATE_VERSION"
+
+          GCP_STATE_DISK=$(gcloud compute images list --filter="name~zebrad-cache-main AND name~-tip" --format="value(NAME)" --sort-by=~creationTimestamp --limit=1)
+          GCP_STATE_VERSION=$(echo "$GCP_STATE_DISK" | grep -oE "v[0-9]+" | grep -oE "[0-9]+")
+          echo "GCP_STATE_VERSION: $GCP_STATE_VERSION"
+
+          if [[ "$LOCAL_STATE_VERSION" -lt "$GCP_STATE_VERSION" ]]; then echo "Local version is lower than cached version" && exit 1; fi
+
       # Automatic tag management and OCI Image Format Specification for labels
       - name: Docker meta
         id: meta
@@ -105,14 +127,6 @@ jobs:
         id: buildx
         uses: docker/setup-buildx-action@v1
 
-      - name: Authenticate to Google Cloud
-        id: auth
-        uses: google-github-actions/[email protected]
-        with:
-          workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc'
-          service_account: '[email protected]'
-          token_format: 'access_token'
-
       - name: Login to Google Artifact Registry
         uses: docker/[email protected]
         with:

.github/workflows/test.yml

@@ -70,6 +70,8 @@ jobs:
     name: Build images
     timeout-minutes: 210
     runs-on: ubuntu-latest
+    outputs:
+      any_changed: ${{ steps.changed-files-specific.outputs.any_changed }}
     permissions:
       contents: 'read'
       id-token: 'write'
@@ -83,6 +85,40 @@ jobs:
         with:
           short-length: 7
 
+      - name: Authenticate to Google Cloud
+        id: auth
+        uses: google-github-actions/[email protected]
+        with:
+          workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc'
+          service_account: '[email protected]'
+          token_format: 'access_token'
+
+      # If we change the version we want to rebuild the cached state disks,
+      # so we want to run the regenerate-stateful-disks to use the newer state in tests.
+      #
+      # If we change the state path without changing the version,
+      # the regenerate-stateful-disks job will take a few hours because it will do a full rebuild.
+      - name: Get specific changed files
+        id: changed-files-specific
+        uses: tj-actions/[email protected]
+        with:
+          files: |
+            zebra-state/**/constants.rs
+
+      # Before executing any further steps, validate the local state and remote version are the same,
+      # or at least that the local state version is greater than the available cached state version from main.
+      - name: Validate constants.rs version vs cached state version
+        id: validate-state-version
+        run: |
+          LOCAL_STATE_VERSION=$(grep -oE "DATABASE_FORMAT_VERSION: .* [0-9]+" "$GITHUB_WORKSPACE/zebra-state/src/constants.rs" | grep -oE "[0-9]+" | tail -n1)
+          echo "LOCAL_STATE_VERSION: $LOCAL_STATE_VERSION"
+
+          GCP_STATE_DISK=$(gcloud compute images list --filter="name~zebrad-cache-main AND name~-checkpoint" --format="value(NAME)" --sort-by=~creationTimestamp --limit=1)
+          GCP_STATE_VERSION=$(echo "$GCP_STATE_DISK" | grep -oE "v[0-9]+" | grep -oE "[0-9]+")
+          echo "GCP_STATE_VERSION: $GCP_STATE_VERSION"
+
+          if [[ "$LOCAL_STATE_VERSION" -lt "$GCP_STATE_VERSION" ]]; then echo "Local version is lower than cached version" && exit 1; fi
+
         # Automatic tag management and OCI Image Format Specification for labels
       - name: Docker meta
         id: meta
@@ -107,14 +143,6 @@ jobs:
         id: buildx
         uses: docker/setup-buildx-action@v1
 
-      - name: Authenticate to Google Cloud
-        id: auth
-        uses: google-github-actions/[email protected]
-        with:
-          workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc'
-          service_account: '[email protected]'
-          token_format: 'access_token'
-
       - name: Login to Google Artifact Registry
         uses: docker/[email protected]
         with:
@@ -232,8 +260,6 @@ jobs:
     name: Regenerate stateful disks
     runs-on: ubuntu-latest
     needs: build
-    outputs:
-      any_changed: ${{ steps.changed-files-specific.outputs.any_changed }}
     permissions:
       contents: 'read'
       id-token: 'write'
@@ -243,21 +269,6 @@ jobs:
           persist-credentials: false
           fetch-depth: '2'
 
-      # TODO move the `changed-files-specific` step to the build job for a better dependency tree
-      # Only run this job if the database format version has (likely) changed.
-      #
-      # If we have accidentally changed the format, but not changed the version,
-      # we want to run with the old cached state, so this job fails.
-      #
-      # If we change the state path without changing the version,
-      # this job will take a few hours, because it will do a full rebuild.
-      - name: Get specific changed files
-        id: changed-files-specific
-        uses: tj-actions/[email protected]
-        with:
-          files: |
-            zebra-state/**/constants.rs
-
       - name: Inject slug/short variables
         uses: rlespinasse/github-slug-action@v4
         with:
@@ -280,7 +291,7 @@ jobs:
       # Check if our destination compute instance exists and delete it
       - name: Delete existing instance with same SHA
         id: delete-old-instance
-        if: ${{ steps.changed-files-specific.outputs.any_changed == 'true' || github.event.inputs.regenerate-disks == 'true' || github.event_name == 'push'}}
+        if: ${{ needs.build.outputs.any_changed == 'true' || github.event.inputs.regenerate-disks == 'true' }}
         run: |
           INSTANCE=$(gcloud compute instances list --filter=regenerate-disk-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} --format='value(NAME)')
           if [ -z "${INSTANCE}" ]; then
@@ -291,7 +302,7 @@ jobs:
 
       - name: Create GCP compute instance
         id: create-instance
-        if: ${{ steps.changed-files-specific.outputs.any_changed == 'true' || github.event.inputs.regenerate-disks == 'true' || github.event_name == 'push'}}
+        if: ${{ steps.delete-old-instance.outcome == 'skipped' }}
         run: |
           gcloud compute instances create-with-container "regenerate-disk-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \
           --boot-disk-size 100GB \
@@ -446,10 +457,36 @@ jobs:
             gcloud compute instances delete "${INSTANCE}" --zone "${{ env.ZONE }}" --delete-disks all --quiet
           fi
 
-      - name: Get disk state name from gcloud
-        id: get-disk-name
+      # If a triggering file for a rebuild was changed on a PR or we manually triggered a rebuild,
+      # use the latest available disk state as the regenerate-stateful-disks job was executed
+      #
+      # In the regenerate-stateful-disks job we don't validate the github.event_name because we
+      # want to ensure the disk are regenerated on PR syncs and push events (to main), and when
+      # manually triggered.
+      # 
+      # In this scenario we also valida that if a file like constants.rs was changed then a disk
+      # rebuild was triggered and we if this happened on a PR we want to use the cached disk from
+      # the PR and not the cached disk from main. Otherwise we move to the next step, which is
+      # get-main-disk-name as this was most likely triggered by a push in main, or constanst.rs
+      # was not changed, which allows us to use the disk from main, as a disk rebuild did not happened,
+      # and if the rebuilt happened then it was triggered on main and we want to use main's reference.
+      - name: Get latest state disk name from gcloud
+        id: get-latest-disk-name
+        if: ${{ (needs.build.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event.inputs.regenerate-disks == 'true' }}
+        run: |
+          GCP_STATE_DISK=$(gcloud compute images list --filter="name~zebrad-cache-${{ env.GITHUB_REF_SLUG_URL }} AND name~-checkpoint" --format="value(NAME)" --sort-by=~creationTimestamp --limit=1)
+          echo "Disk: $GCP_STATE_DISK"
+          echo "Description: $(gcloud compute images describe $GCP_STATE_DISK --format='value(DESCRIPTION)')"
+
+          echo "CACHED_DISK_NAME=$GCP_STATE_DISK" >> $GITHUB_ENV
+
+      # If the conditions in the get-latest-disk-name step were not applied means this was triggered by a push event,
+      # and we need to use the cached state from the main branch. 
+      - name: Get main state disk name from gcloud
+        id: get-main-disk-name
+        if: ${{ steps.get-latest-disk-name.outcome == 'skipped' }}
         run: |
-          GCP_STATE_DISK=$(gcloud compute images list --filter="name~zebrad-cache AND name~-checkpoint" --format="value(NAME)" --sort-by=~creationTimestamp --limit=1)
+          GCP_STATE_DISK=$(gcloud compute images list --filter="name~zebrad-cache-main AND name~-checkpoint" --format="value(NAME)" --sort-by=~creationTimestamp --limit=1)
           echo "Disk: $GCP_STATE_DISK"
           echo "Description: $(gcloud compute images describe $GCP_STATE_DISK --format='value(DESCRIPTION)')"