Add state flag file in K8s upgrades (#2702)

ansible/playbooks/roles/upgrade/defaults/main.yml

@@ -19,3 +19,6 @@ opendistro_for_elasticsearch:
       filename: demo2epiphany-certs-migration-root-CAs.pem
 
   upgrade_state_file_path: /etc/elasticsearch/epicli-upgrade-started.state
+
+kubernetes:
+  upgrade_state_file_path: /var/lib/epiphany/upgrade/state/kubernetes-{{ ver }}.uncompleted

ansible/playbooks/roles/upgrade/tasks/kubernetes.yml

@@ -1,25 +1,65 @@
 ---
+# During HA control plane upgrade server address in kubeconfig is switched to local for
+#   * compatibility between client and server versions
+#   * identifying correct server version
+
+- name: k8s/master | Switch apiserver address to local
+  include_tasks: kubernetes/utils/set-local-apiserver.yml # sets kubectl_context_cluster
+  when:
+    - groups.kubernetes_master | length > 1
+    - inventory_hostname in groups.kubernetes_master
+
 - name: Wait for kube-apiserver and get cluster version
-  delegate_to: "{{ groups.kubernetes_master[0] }}"
+  delegate_to: >-
+    {{ inventory_hostname if inventory_hostname in groups.kubernetes_master else
+      groups.kubernetes_master[0] }}
   block:
     - name: k8s | Include wait-for-kube-apiserver.yml
       import_tasks: kubernetes/utils/wait-for-kube-apiserver.yml
 
     - name: k8s | Include get-cluster-version.yml
-      import_tasks: kubernetes/get-cluster-version.yml # sets cluster_version
+      import_tasks: kubernetes/get-cluster-version.yml
+
+    - name: k8s | Set cluster version facts
+      set_fact:
+        initial_cluster_version: "{{ _cluster_version }}"
+        cluster_version: "{{ _cluster_version }}"
+      vars:
+        _cluster_version: "{{ (kubectl_cluster_version.stdout | from_yaml).serverVersion.gitVersion }}"
 
 - name: k8s | Include get-kubelet-version.yml
-  import_tasks: kubernetes/get-kubelet-version.yml # sets kubelet_version
+  import_tasks: kubernetes/get-kubelet-version.yml
   delegate_to: "{{ groups.kubernetes_master[0] }}"
 
+- name: k8s | Set kubelet version as fact
+  set_fact:
+    initial_kubelet_version: "{{ kubelet_version.stdout }}"
+
+- name: Check if upgrade state file exists
+  stat:
+    path: "{{ kubernetes.upgrade_state_file_path }}"
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: k8s_upgrade_state_file_status
+
 - name: Upgrade masters then nodes
   vars:
     version: "{{ ver }}"
     cni_version: "{{ cni_ver }}"
   block:
     - name: Upgrade masters
-      when: cluster_version is version('v' + version, '<=')
+      when:
+        - inventory_hostname in groups.kubernetes_master
+        - k8s_upgrade_state_file_status.stat.exists
+          or initial_cluster_version is version('v' + version, '<')
       block:
+        - name: Create K8s upgrade state file on master node
+          copy:
+            dest: "{{ kubernetes.upgrade_state_file_path }}"
+            content: Upgrade started
+            mode: u=rw,g=r,o=
+
         - name: k8s | Upgrade first master to v{{ version }}
           include_tasks: kubernetes/upgrade-master0.yml
           when:
@@ -30,16 +70,39 @@
           when:
             - inventory_hostname in groups.kubernetes_master[1:]
 
+        - name: Remove K8s upgrade state file on master node
+          file:
+            path: "{{ kubernetes.upgrade_state_file_path }}"
+            state: absent
+
     - name: Upgrade nodes
-      when: kubelet_version is version('v' + version, '<=')
+      when:
+        - groups.kubernetes_node is defined
+        - inventory_hostname in groups.kubernetes_node
+        - k8s_upgrade_state_file_status.stat.exists
+          or initial_kubelet_version is version('v' + version, '<')
       block:
+        - name: Create K8s upgrade state file on node
+          copy:
+            dest: "{{ kubernetes.upgrade_state_file_path }}"
+            content: Upgrade started
+            mode: u=rw,g=r,o=
+
         - name: k8s | Upgrade node to v{{ version }}
           include_tasks: kubernetes/upgrade-node.yml
-          when:
-            - groups.kubernetes_node is defined
-            - inventory_hostname in groups.kubernetes_node
+
+        - name: Remove K8s upgrade state file on node
+          file:
+            path: "{{ kubernetes.upgrade_state_file_path }}"
+            state: absent
+
+- name: k8s/master | Switch apiserver address to HAProxy
+  command: |-
+    kubectl config set-cluster {{ kubectl_context_cluster.stdout }} --server=https://localhost:3446
+  when:
+    - groups.kubernetes_master | length > 1
+    - inventory_hostname in groups.kubernetes_master
+  changed_when: true
 
 - name: k8s | Upgrade internal haproxy load-balancer
   import_tasks: kubernetes/upgrade-haproxy.yml
-
-# TODO: Create a flag file that the upgrade completed to not run it again for the same version next time

ansible/playbooks/roles/upgrade/tasks/kubernetes/get-cluster-version.yml

@@ -6,12 +6,3 @@
   retries: 60
   delay: 5
   changed_when: false
-
-- name: Set cluster version as fact
-  set_fact:
-    cluster_version: >-
-      {{ (kubectl_cluster_version.stdout | from_yaml).serverVersion.gitVersion }}
-    cluster_version_major: >-
-      {{ (kubectl_cluster_version.stdout | from_yaml).serverVersion.major }}
-    cluster_version_minor: >-
-      {{ (kubectl_cluster_version.stdout | from_yaml).serverVersion.minor }}

ansible/playbooks/roles/upgrade/tasks/kubernetes/get-kubelet-version.yml

@@ -4,7 +4,3 @@
     kubectl get node {{ inventory_hostname }} -o jsonpath='{.status.nodeInfo.kubeletVersion}'
   register: kubelet_version
   changed_when: false
-
-- name: Set kubelet version as fact
-  set_fact:
-    kubelet_version: "{{ kubelet_version.stdout }}"

ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubelet-cm.yml

@@ -1,6 +1,6 @@
 ---
-- name: k8s/kubelet-cm | Include get-cluster-version.yml
-  include_tasks: get-cluster-version.yml # sets cluster_version
+- name: k8s/kubelet-cm | Include set-cluster-version.yml
+  include_tasks: set-cluster-version.yml # sets cluster_version
 
 - name: k8s/kubelet-cm | Get kubelet config from ConfigMap
   command: |-

ansible/playbooks/roles/upgrade/tasks/kubernetes/set-cluster-version.yml

@@ -0,0 +1,12 @@
+---
+- name: k8s | Include get-cluster-version.yml
+  include_tasks: kubernetes/get-cluster-version.yml
+
+- name: Set cluster version as fact
+  set_fact:
+    cluster_version: >-
+      {{ (kubectl_cluster_version.stdout | from_yaml).serverVersion.gitVersion }}
+    cluster_version_major: >-
+      {{ (kubectl_cluster_version.stdout | from_yaml).serverVersion.major }}
+    cluster_version_minor: >-
+      {{ (kubectl_cluster_version.stdout | from_yaml).serverVersion.minor }}

ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-master0.yml

@@ -1,13 +1,4 @@
 ---
-# During HA control plane upgrade server address in kubeconfig is switched to local for
-#   * compatibility between client and server versions
-#   * identifying correct server version
-
-- name: k8s/master0 | Switch apiserver address to local
-  include_tasks: utils/set-local-apiserver.yml # sets kubectl_context_cluster
-  when:
-    - groups.kubernetes_master | length > 1
-
 - name: k8s/master0 | Wait for cluster's readiness
   include_tasks: utils/wait.yml
 
@@ -107,10 +98,3 @@
 
 - name: k8s/master0 | Verify component versions and node status
   include_tasks: kubernetes/verify-upgrade.yml
-
-- name: k8s/master0 | Switch apiserver address to HAProxy
-  command: |-
-    kubectl config set-cluster {{ kubectl_context_cluster.stdout }} --server=https://localhost:3446
-  when:
-    - groups.kubernetes_master | length > 1
-  changed_when: true

ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-masterN.yml

@@ -1,7 +1,4 @@
 ---
-- name: k8s/masterN | Switch apiserver address to local
-  include_tasks: utils/set-local-apiserver.yml # sets kubectl_context_cluster
-
 - name: k8s/masterN | Drain master in preparation for maintenance
   include_tasks: utils/drain.yml
 
@@ -55,8 +52,3 @@
 
 - name: k8s/masterN | Verify component versions and node status
   include_tasks: kubernetes/verify-upgrade.yml
-
-- name: k8s/masterN | Switch apiserver address to HAProxy
-  command: |-
-    kubectl config set-cluster {{ kubectl_context_cluster.stdout }} --server=https://localhost:3446
-  changed_when: true

ansible/playbooks/roles/upgrade/tasks/kubernetes/verify-upgrade.yml

@@ -6,8 +6,8 @@
     - name: k8s/verify | Include wait-for-kube-apiserver.yml
       include_tasks: utils/wait-for-kube-apiserver.yml
 
-    - name: k8s/verify | Include get-cluster-version.yml
-      include_tasks: get-cluster-version.yml  # sets cluster_version
+    - name: k8s/verify | Include set-cluster-version.yml
+      include_tasks: set-cluster-version.yml
 
     - name: k8s/verify | Verify cluster version
       assert:

docs/changelogs/CHANGELOG-1.3.md

@@ -21,6 +21,7 @@
 - [#2814](https://github.com/epiphany-platform/epiphany/issues/2814) - Add description how to enable TLS in Kibana
 - [#1076](https://github.com/epiphany-platform/epiphany/issues/2595) - Document connection protocols and ciphers
 - [#2665](https://github.com/epiphany-platform/epiphany/issues/2665) - Add Kubernetes prereqs to epicli preflight checks
+- [#2702](https://github.com/epiphany-platform/epiphany/issues/2702) - Use state flag file in K8s upgrades
 
 ### Fixed