diff --git a/ansible/playbooks/roles/upgrade/defaults/main.yml b/ansible/playbooks/roles/upgrade/defaults/main.yml index f90edd92b3..e7e0a5f77a 100644 --- a/ansible/playbooks/roles/upgrade/defaults/main.yml +++ b/ansible/playbooks/roles/upgrade/defaults/main.yml @@ -19,3 +19,6 @@ opendistro_for_elasticsearch: filename: demo2epiphany-certs-migration-root-CAs.pem upgrade_state_file_path: /etc/elasticsearch/epicli-upgrade-started.state + +kubernetes: + upgrade_state_file_path: /var/lib/epiphany/upgrade/state/kubernetes-{{ ver }}.uncompleted diff --git a/ansible/playbooks/roles/upgrade/tasks/kubernetes.yml b/ansible/playbooks/roles/upgrade/tasks/kubernetes.yml index 36f0b82542..7ec0fdc930 100644 --- a/ansible/playbooks/roles/upgrade/tasks/kubernetes.yml +++ b/ansible/playbooks/roles/upgrade/tasks/kubernetes.yml @@ -1,25 +1,67 @@ --- -- name: Wait for kube-apiserver and get cluster version - delegate_to: "{{ groups.kubernetes_master[0] }}" - block: - - name: k8s | Include wait-for-kube-apiserver.yml - import_tasks: kubernetes/utils/wait-for-kube-apiserver.yml +- name: k8s | Wait for kube-apiserver + delegate_to: >- + {{ inventory_hostname if inventory_hostname in groups.kubernetes_master else + groups.kubernetes_master[0] }} + import_tasks: kubernetes/utils/wait-for-kube-apiserver.yml +# During HA control plane upgrade server address in kubeconfig is switched to local for +# * compatibility between client and server versions +# * identifying correct server version +- name: k8s/master | Switch apiserver address to local + include_tasks: kubernetes/utils/set-local-apiserver.yml # sets kubectl_context_cluster + when: + - groups.kubernetes_master | length > 1 + - inventory_hostname in groups.kubernetes_master + +- name: Get cluster version and set version facts + delegate_to: >- + {{ inventory_hostname if inventory_hostname in groups.kubernetes_master else + groups.kubernetes_master[0] }} + block: - name: k8s | Include get-cluster-version.yml - import_tasks: kubernetes/get-cluster-version.yml # sets cluster_version + import_tasks: kubernetes/get-cluster-version.yml + + - name: k8s | Set cluster version facts + set_fact: + initial_cluster_version: "{{ _cluster_version }}" + cluster_version: "{{ _cluster_version }}" + vars: + _cluster_version: "{{ (kubectl_cluster_version.stdout | from_yaml).serverVersion.gitVersion }}" - name: k8s | Include get-kubelet-version.yml - import_tasks: kubernetes/get-kubelet-version.yml # sets kubelet_version + import_tasks: kubernetes/get-kubelet-version.yml delegate_to: "{{ groups.kubernetes_master[0] }}" +- name: k8s | Set kubelet version as fact + set_fact: + initial_kubelet_version: "{{ kubelet_version.stdout }}" + +- name: Check if upgrade state file exists + stat: + path: "{{ kubernetes.upgrade_state_file_path }}" + get_attributes: false + get_checksum: false + get_mime: false + register: k8s_upgrade_state_file_status + - name: Upgrade masters then nodes vars: version: "{{ ver }}" cni_version: "{{ cni_ver }}" block: - name: Upgrade masters - when: cluster_version is version('v' + version, '<=') + when: + - inventory_hostname in groups.kubernetes_master + - k8s_upgrade_state_file_status.stat.exists + or initial_cluster_version is version('v' + version, '<') block: + - name: Create K8s upgrade state file on master node + copy: + dest: "{{ kubernetes.upgrade_state_file_path }}" + content: Upgrade started + mode: u=rw,g=r,o= + - name: k8s | Upgrade first master to v{{ version }} include_tasks: kubernetes/upgrade-master0.yml when: @@ -30,16 +72,39 @@ when: - inventory_hostname in groups.kubernetes_master[1:] + - name: Remove K8s upgrade state file on master node + file: + path: "{{ kubernetes.upgrade_state_file_path }}" + state: absent + - name: Upgrade nodes - when: kubelet_version is version('v' + version, '<=') + when: + - groups.kubernetes_node is defined + - inventory_hostname in groups.kubernetes_node + - k8s_upgrade_state_file_status.stat.exists + or initial_kubelet_version is version('v' + version, '<') block: + - name: Create K8s upgrade state file on node + copy: + dest: "{{ kubernetes.upgrade_state_file_path }}" + content: Upgrade started + mode: u=rw,g=r,o= + - name: k8s | Upgrade node to v{{ version }} include_tasks: kubernetes/upgrade-node.yml - when: - - groups.kubernetes_node is defined - - inventory_hostname in groups.kubernetes_node + + - name: Remove K8s upgrade state file on node + file: + path: "{{ kubernetes.upgrade_state_file_path }}" + state: absent + +- name: k8s/master | Switch apiserver address to HAProxy + command: |- + kubectl config set-cluster {{ kubectl_context_cluster.stdout }} --server=https://localhost:3446 + when: + - groups.kubernetes_master | length > 1 + - inventory_hostname in groups.kubernetes_master + changed_when: true - name: k8s | Upgrade internal haproxy load-balancer import_tasks: kubernetes/upgrade-haproxy.yml - -# TODO: Create a flag file that the upgrade completed to not run it again for the same version next time diff --git a/ansible/playbooks/roles/upgrade/tasks/kubernetes/get-cluster-version.yml b/ansible/playbooks/roles/upgrade/tasks/kubernetes/get-cluster-version.yml index 0a6de7b1de..b543177277 100644 --- a/ansible/playbooks/roles/upgrade/tasks/kubernetes/get-cluster-version.yml +++ b/ansible/playbooks/roles/upgrade/tasks/kubernetes/get-cluster-version.yml @@ -6,12 +6,3 @@ retries: 60 delay: 5 changed_when: false - -- name: Set cluster version as fact - set_fact: - cluster_version: >- - {{ (kubectl_cluster_version.stdout | from_yaml).serverVersion.gitVersion }} - cluster_version_major: >- - {{ (kubectl_cluster_version.stdout | from_yaml).serverVersion.major }} - cluster_version_minor: >- - {{ (kubectl_cluster_version.stdout | from_yaml).serverVersion.minor }} diff --git a/ansible/playbooks/roles/upgrade/tasks/kubernetes/get-kubelet-version.yml b/ansible/playbooks/roles/upgrade/tasks/kubernetes/get-kubelet-version.yml index 9a2ca95c28..7ac6170eeb 100644 --- a/ansible/playbooks/roles/upgrade/tasks/kubernetes/get-kubelet-version.yml +++ b/ansible/playbooks/roles/upgrade/tasks/kubernetes/get-kubelet-version.yml @@ -4,7 +4,3 @@ kubectl get node {{ inventory_hostname }} -o jsonpath='{.status.nodeInfo.kubeletVersion}' register: kubelet_version changed_when: false - -- name: Set kubelet version as fact - set_fact: - kubelet_version: "{{ kubelet_version.stdout }}" diff --git a/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubelet-cm.yml b/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubelet-cm.yml index 3789ba9a28..5af0aa3094 100644 --- a/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubelet-cm.yml +++ b/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubelet-cm.yml @@ -1,6 +1,6 @@ --- -- name: k8s/kubelet-cm | Include get-cluster-version.yml - include_tasks: get-cluster-version.yml # sets cluster_version +- name: k8s/kubelet-cm | Include set-cluster-version.yml + include_tasks: set-cluster-version.yml # sets cluster_version - name: k8s/kubelet-cm | Get kubelet config from ConfigMap command: |- diff --git a/ansible/playbooks/roles/upgrade/tasks/kubernetes/set-cluster-version.yml b/ansible/playbooks/roles/upgrade/tasks/kubernetes/set-cluster-version.yml new file mode 100644 index 0000000000..61e986bcf4 --- /dev/null +++ b/ansible/playbooks/roles/upgrade/tasks/kubernetes/set-cluster-version.yml @@ -0,0 +1,12 @@ +--- +- name: k8s | Include get-cluster-version.yml + include_tasks: kubernetes/get-cluster-version.yml + +- name: Set cluster version as fact + set_fact: + cluster_version: >- + {{ (kubectl_cluster_version.stdout | from_yaml).serverVersion.gitVersion }} + cluster_version_major: >- + {{ (kubectl_cluster_version.stdout | from_yaml).serverVersion.major }} + cluster_version_minor: >- + {{ (kubectl_cluster_version.stdout | from_yaml).serverVersion.minor }} diff --git a/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-master0.yml b/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-master0.yml index 9e66c0bc8d..5afc84d972 100644 --- a/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-master0.yml +++ b/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-master0.yml @@ -1,13 +1,4 @@ --- -# During HA control plane upgrade server address in kubeconfig is switched to local for -# * compatibility between client and server versions -# * identifying correct server version - -- name: k8s/master0 | Switch apiserver address to local - include_tasks: utils/set-local-apiserver.yml # sets kubectl_context_cluster - when: - - groups.kubernetes_master | length > 1 - - name: k8s/master0 | Wait for cluster's readiness include_tasks: utils/wait.yml @@ -107,10 +98,3 @@ - name: k8s/master0 | Verify component versions and node status include_tasks: kubernetes/verify-upgrade.yml - -- name: k8s/master0 | Switch apiserver address to HAProxy - command: |- - kubectl config set-cluster {{ kubectl_context_cluster.stdout }} --server=https://localhost:3446 - when: - - groups.kubernetes_master | length > 1 - changed_when: true diff --git a/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-masterN.yml b/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-masterN.yml index 0ff766d3bc..850ba8427d 100644 --- a/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-masterN.yml +++ b/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-masterN.yml @@ -1,7 +1,4 @@ --- -- name: k8s/masterN | Switch apiserver address to local - include_tasks: utils/set-local-apiserver.yml # sets kubectl_context_cluster - - name: k8s/masterN | Drain master in preparation for maintenance include_tasks: utils/drain.yml @@ -55,8 +52,3 @@ - name: k8s/masterN | Verify component versions and node status include_tasks: kubernetes/verify-upgrade.yml - -- name: k8s/masterN | Switch apiserver address to HAProxy - command: |- - kubectl config set-cluster {{ kubectl_context_cluster.stdout }} --server=https://localhost:3446 - changed_when: true diff --git a/ansible/playbooks/roles/upgrade/tasks/kubernetes/verify-upgrade.yml b/ansible/playbooks/roles/upgrade/tasks/kubernetes/verify-upgrade.yml index bf886df408..02dfad56b0 100644 --- a/ansible/playbooks/roles/upgrade/tasks/kubernetes/verify-upgrade.yml +++ b/ansible/playbooks/roles/upgrade/tasks/kubernetes/verify-upgrade.yml @@ -6,8 +6,8 @@ - name: k8s/verify | Include wait-for-kube-apiserver.yml include_tasks: utils/wait-for-kube-apiserver.yml - - name: k8s/verify | Include get-cluster-version.yml - include_tasks: get-cluster-version.yml # sets cluster_version + - name: k8s/verify | Include set-cluster-version.yml + include_tasks: set-cluster-version.yml - name: k8s/verify | Verify cluster version assert: diff --git a/docs/changelogs/CHANGELOG-1.3.md b/docs/changelogs/CHANGELOG-1.3.md index 1c5ebf2eed..f65f93ae97 100644 --- a/docs/changelogs/CHANGELOG-1.3.md +++ b/docs/changelogs/CHANGELOG-1.3.md @@ -21,6 +21,7 @@ - [#2814](https://github.com/epiphany-platform/epiphany/issues/2814) - Add description how to enable TLS in Kibana - [#1076](https://github.com/epiphany-platform/epiphany/issues/2595) - Document connection protocols and ciphers - [#2665](https://github.com/epiphany-platform/epiphany/issues/2665) - Add Kubernetes prereqs to epicli preflight checks +- [#2702](https://github.com/epiphany-platform/epiphany/issues/2702) - Use state flag file in K8s upgrades ### Fixed