Add retry to kubectl command

hitachienergy · Feb 17, 2022 · 1cc4d74 · 1cc4d74
1 parent 6ea36a8
commit 1cc4d74
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 0 deletions.
diff --git a/ansible/playbooks/roles/upgrade/tasks/kubernetes/backup-kubeadm-config.yml b/ansible/playbooks/roles/upgrade/tasks/kubernetes/backup-kubeadm-config.yml
@@ -1,10 +1,15 @@
 ---
 # The kubeadm-config.yml file is no longer used during upgrade process,
 # but we keep it for backup and reference purposes.
+# Retries needed for HA deployment (random failures)
 - name: k8s/master | Collect kubeadm-config ConfigMap
   command: >-
     kubectl get cm kubeadm-config -o yaml -n kube-system
   register: kubeadm_config
+  until:
+    - kubeadm_config is succeeded
+  retries: 30
+  delay: 1
 
 - name: k8s/master | Save kubeadm-config ConfigMap to file
   copy:

diff --git a/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-master0.yml b/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-master0.yml
@@ -41,9 +41,15 @@
     - name: k8s/master0 | Include set-cluster-version.yml
       include_tasks: set-cluster-version.yml # sets cluster_version
 
+    # Retries needed for HA deployment (random failures)
     - name: k8s/master0 | Add k8s annotation for containerd
       command: >-
         kubectl annotate node {{ inventory_hostname }} --overwrite kubeadm.alpha.kubernetes.io/cri-socket=unix:///run/containerd/containerd.sock
+      register: result
+      until:
+        - result is succeeded
+      retries: 30
+      delay: 1
 
     # Note: Usage of the --config flag for reconfiguring the cluster during upgrade is not recommended since v1.16
     - name: k8s/master0 | Upgrade K8s cluster to v{{ version }}

diff --git a/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-masterN.yml b/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-masterN.yml
@@ -7,9 +7,15 @@
     - name: k8s/masterN | Install kubeadm
       include_tasks: "{{ ansible_os_family }}/install-kubeadm.yml"
 
+    # Retries needed for HA deployment (random failures)
     - name: k8s/masterN | Add k8s annotation for containerd
       command: >-
         kubectl annotate node {{ inventory_hostname }} --overwrite kubeadm.alpha.kubernetes.io/cri-socket=unix:///run/containerd/containerd.sock
+      register: result
+      until:
+        - result is succeeded
+      retries: 30
+      delay: 1
 
     - name: k8s/masterN | Upgrade master {{ inventory_hostname }}
       command: >-