[latest-17.0](vitessio#3938): CherryPick(vitessio#14753): Add a retry…

… to remove the vttablet directory during upgrade/downgrade backup tests (vitessio#3941)

Co-authored-by: Florent Poinsard <[email protected]>

.github/workflows/upgrade_downgrade_test_backups_manual.yml

@@ -89,6 +89,7 @@ jobs:
             - 'config/**'
             - 'bootstrap.sh'
             - '.github/workflows/upgrade_downgrade_test_backups_manual.yml'
+            - 'examples/**'
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'

.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml

@@ -92,6 +92,7 @@ jobs:
             - 'config/**'
             - 'bootstrap.sh'
             - '.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml'
+            - 'examples/**'
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'

examples/backups/stop_tablets.sh

@@ -30,7 +30,25 @@ for tablet in 100 200 300; do
       CELL=zone1 TABLET_UID=$uid ../common/scripts/mysqlctl-down.sh
       echo "Removing tablet directory zone1-$uid"
       vtctlclient DeleteTablet -- --allow_primary=true zone1-$uid
-      rm -Rf $VTDATAROOT/vt_0000000$uid
+
+      for ((i=0; i<30; i++)); do
+          # Redirect stderr to a temporary file
+          temp_file=$(mktemp)
+          rm -Rf $VTDATAROOT/vt_0000000$uid 2>"$temp_file"
+
+          if grep -q 'Directory not empty' "$temp_file"; then
+              echo "Directory not empty, retrying..."
+          elif [ ! -s "$temp_file" ]; then
+              echo "Deletion succeeded."
+              rm -f "$temp_file"
+              break
+          else
+              echo "An error occurred."
+              cat "$temp_file"
+          fi
+          rm -f "$temp_file"
+          sleep 1
+      done
     done
   fi
 done