diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/main.yml index d1635a4f54..690a6b3ace 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/main.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/main.yml @@ -1,36 +1,35 @@ --- -- name: Check cluster version +- name: Get cluster version environment: KUBECONFIG: "/home/{{ admin_user.name }}/.kube/config" - shell: kubectl version --short | grep -i server + shell: >- + set -o pipefail && + kubectl version --short -o json | jq --raw-output '.serverVersion.gitVersion' register: cluster_version changed_when: false -- name: Check kubelet version +- name: Get kubelet version environment: KUBECONFIG: "/home/{{ admin_user.name }}/.kube/config" - shell: kubectl get node {{ inventory_hostname }} -o custom-columns=VERSION:.status.nodeInfo.kubeletVersion + shell: >- + kubectl get node {{ inventory_hostname }} -o jsonpath='{.status.nodeInfo.kubeletVersion}' register: kubelet_version changed_when: false -- name: Upgrade master to {{ version }} - include_tasks: "upgrade_master.yml" +- name: Upgrade master to v{{ version }} + include_tasks: upgrade_master.yml vars: version: "{{ ver }}" cni_version: "{{ cni_ver }}" when: - groups['kubernetes_master'][0] == inventory_hostname - - (version == "1.12.10" and "1.11" in cluster_version.stdout) or - (version == "1.13.8" and "1.12" in cluster_version.stdout) or - (version == "1.14.4" and "1.13" in cluster_version.stdout) + - cluster_version.stdout is version('v' + version, '<=') -- name: Upgrade nodes to {{ version }} - include_tasks: "upgrade_nodes.yml" +- name: Upgrade node to v{{ version }} + include_tasks: upgrade_node.yml vars: version: "{{ ver }}" cni_version: "{{ cni_ver }}" when: - inventory_hostname in groups['kubernetes_node'] - - (version == "1.12.10" and "1.11" in kubelet_version.stdout and kubelet_version.stdout != cluster_version.stdout) or - (version == "1.13.8" and "1.12" in kubelet_version.stdout and kubelet_version.stdout != cluster_version.stdout) or - (version == "1.14.4" and "1.13" in kubelet_version.stdout and kubelet_version.stdout != cluster_version.stdout ) + - kubelet_version.stdout is version('v' + version, '<=') diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/upgrade_master.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/upgrade_master.yml index 12d52768e7..5cd15b50ef 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/upgrade_master.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/upgrade_master.yml @@ -1,16 +1,18 @@ --- -- name: Wait for the cluster's readiness +- name: upgrade_master | Wait for cluster's readiness include_tasks: wait.yml -- name: Drain master in preparation for maintenance +- name: upgrade_master | Drain master in preparation for maintenance environment: KUBECONFIG: /home/{{ admin_user.name }}/.kube/config shell: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-local-data -- name: Wait for the cluster's readiness +- name: upgrade_master | Wait for cluster's readiness include_tasks: wait.yml -- name: Install kubernetes-cni {{ cni_version }}, kubelet {{ version }}, kubectl {{ version }} and kubeadm {{ version }} packages for RedHat family +- name: >- + upgrade_master | Install kubernetes-cni-{{ cni_version }}, kubelet-{{ version }}, kubectl-{{ version }} + and kubeadm-{{ version }} packages for RedHat family yum: name: - kubernetes-cni-{{ cni_version }} @@ -23,14 +25,14 @@ when: - ansible_os_family == "RedHat" -- name: Get information about installed packages as facts +- name: upgrade_master | Get information about installed packages as facts package_facts: manager: auto changed_when: false when: - ansible_os_family == "Debian" -- name: Remove newer Debian packages installed as dependencies if they exist # as there is no allow_downgrade parameter in ansible apt module +- name: upgrade_master | Remove newer Debian packages installed as dependencies if they exist # as there is no allow_downgrade parameter in ansible apt module apt: name: - kubernetes-cni @@ -45,7 +47,9 @@ (ansible_facts.packages['kubectl'][0].version is version (version + '-00', '>')) or (ansible_facts.packages['kubeadm'][0].version is version (version + '-00', '>')) -- name: Install kubernetes-cni {{ cni_version }}, kubelet {{ version }}, kubectl {{ version }} and kubeadm {{ version }} packages for Debian family +- name: >- + upgrade_master | Install kubernetes-cni {{ cni_version }}, kubelet {{ version }}, kubectl {{ version }} + and kubeadm {{ version }} packages for Debian family apt: name: - kubernetes-cni={{ cni_version }}-00 @@ -57,92 +61,58 @@ when: - ansible_os_family == "Debian" -- name: Wait for the cluster's readiness +- name: upgrade_master | Wait for cluster's readiness include_tasks: wait.yml -- name: Check if kubeadm configuration file exists +- name: upgrade_master | Check if kubeadm configuration file exists stat: path: /etc/kubeadm/kubeadm-config.yml changed_when: false register: kubeadm_config -# https://github.com/kubernetes/kubeadm/issues/1471 Upgrading a 1.12 cluster thru 1.13 to 1.14 fails -- name: Validate whether current cluster is upgradeable (from ver. 1.13) - block: - - name: Show upgrade plan (using kubeadm configuration file) - shell: kubeadm upgrade plan v{{ version }} --config /etc/kubeadm/kubeadm-config.yml - changed_when: false - when: - - kubeadm_config.stat.exists - - - name: Show upgrade plan - shell: kubeadm upgrade plan v{{ version }} - changed_when: false - when: - - not kubeadm_config.stat.exists - when: - - '"1.13" in cluster_version.stdout' - - rescue: - - name: Find the existing etcd server certificates - find: - paths: /etc/kubernetes/pki/etcd - patterns: "*server.*" - changed_when: false - register: files_to_delete - - - name: Remove the existing etcd server certificates - file: - path: "{{ item.path }}" - state: absent - with_items: "{{ files_to_delete.files }}" - - - name: Regenerate the etcd server certificates - shell: kubeadm init phase certs etcd-server - -- name: Validate whether current cluster is upgradeable (using kubeadm configuration file) +- name: upgrade_master | Validate whether cluster is upgradeable (using kubeadm configuration file) shell: kubeadm upgrade plan v{{ version }} --config /etc/kubeadm/kubeadm-config.yml changed_when: false when: - kubeadm_config.stat.exists -- name: Validate whether current cluster is upgradeable +- name: upgrade_master | Validate whether cluster is upgradeable shell: kubeadm upgrade plan v{{ version }} changed_when: false when: - not kubeadm_config.stat.exists -- name: Upgrade Kubernetes cluster to the specified version v{{ version }} (using kubeadm configuration file) +- name: upgrade_master | Upgrade K8s cluster to v{{ version }} (using kubeadm configuration file) shell: kubeadm upgrade apply -y v{{ version }} --config /etc/kubeadm/kubeadm-config.yml - retries: 5 - delay: 5 register: output until: output is succeeded + retries: 5 + delay: 5 when: - kubeadm_config.stat.exists -- name: Upgrade Kubernetes cluster to the specified version v{{ version }} +- name: upgrade_master | Upgrade K8s cluster to v{{ version }} shell: kubeadm upgrade apply -y v{{ version }} - retries: 5 - delay: 5 register: output until: output is succeeded + retries: 5 + delay: 5 when: - not kubeadm_config.stat.exists -- name: Wait for the cluster's readiness +- name: upgrade_master | Wait for cluster's readiness include_tasks: wait.yml -- name: Restart kubelet +- name: upgrade_master | Restart kubelet systemd: state: restarted daemon_reload: yes name: kubelet -- name: Wait for the cluster's readiness +- name: upgrade_master | Wait for cluster's readiness include_tasks: wait.yml -- name: Uncordon master - mark master as schedulable +- name: upgrade_master | Uncordon master - mark master as schedulable environment: KUBECONFIG: /home/{{ admin_user.name }}/.kube/config shell: kubectl uncordon {{ inventory_hostname }} @@ -151,5 +121,5 @@ register: output until: output is succeeded -- name: Verify cluster version +- name: upgrade_master | Verify cluster version include_tasks: verify.yml diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/upgrade_nodes.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/upgrade_node.yml similarity index 65% rename from core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/upgrade_nodes.yml rename to core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/upgrade_node.yml index 25a1c5bac6..18c8aa7d14 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/upgrade_nodes.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/upgrade_node.yml @@ -1,17 +1,19 @@ --- -- name: Wait for the cluster's readiness +- name: upgrade_node | Wait for cluster's readiness include_tasks: wait.yml -- name: Drain node in preparation for maintenance +- name: upgrade_node | Drain node in preparation for maintenance environment: KUBECONFIG: /home/{{ admin_user.name }}/.kube/config shell: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-local-data delegate_to: "{{ groups['kubernetes_master'][0] }}" -- name: Wait for the cluster's readiness +- name: upgrade_node | Wait for cluster's readiness include_tasks: wait.yml -- name: Install kubernetes-cni {{ cni_version }}, kubelet {{ version }}, kubectl {{ version }} and kubeadm {{ version }} packages for RedHat family +- name: >- + upgrade_node | Install kubernetes-cni-{{ cni_version }}, kubelet-{{ version }}, kubectl-{{ version }} + and kubeadm-{{ version }} packages for RedHat family yum: name: - kubernetes-cni-{{ cni_version }} @@ -24,14 +26,14 @@ when: - ansible_os_family == "RedHat" -- name: Get information about installed packages as facts +- name: upgrade_node | Get information about installed packages as facts package_facts: manager: auto changed_when: false when: - ansible_os_family == "Debian" -- name: Remove newer Debian packages installed as dependencies if they exist # as there is no allow_downgrade parameter in ansible apt module +- name: upgrade_node | Remove newer Debian packages installed as dependencies if they exist # as there is no allow_downgrade parameter in ansible apt module apt: name: - kubernetes-cni @@ -46,7 +48,9 @@ (ansible_facts.packages['kubectl'][0].version is version (version + '-00', '>')) or (ansible_facts.packages['kubeadm'][0].version is version (version + '-00', '>')) -- name: Install kubernetes-cni {{ cni_version }}, kubelet {{ version }}, kubectl {{ version }} and kubeadm {{ version }} packages for Debian family +- name: >- + upgrade_node | Install kubernetes-cni {{ cni_version }}, kubelet {{ version }}, kubectl {{ version }} + and kubeadm {{ version }} packages for Debian family apt: name: - kubernetes-cni={{ cni_version }}-00 @@ -58,27 +62,27 @@ when: - ansible_os_family == "Debian" -- name: Upgrade node config +- name: upgrade_node | Upgrade node config shell: kubeadm upgrade node config --kubelet-version v{{ version }} -- name: Restart kubelet +- name: upgrade_node | Restart kubelet systemd: state: restarted daemon_reload: yes name: kubelet -- name: Wait for the cluster's readiness +- name: upgrade_node | Wait for cluster's readiness include_tasks: wait.yml -- name: Uncordon node - mark node as schedulable +- name: upgrade_node | Uncordon node - mark node as schedulable environment: KUBECONFIG: /home/{{ admin_user.name }}/.kube/config shell: kubectl uncordon {{ inventory_hostname }} - retries: 5 - delay: 5 register: output until: output is succeeded + retries: 5 + delay: 5 delegate_to: "{{ groups['kubernetes_master'][0] }}" -- name: Verify cluster version +- name: upgrade_node | Verify cluster version include_tasks: verify.yml diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/verify.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/verify.yml index 7742548474..64d6578ff4 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/verify.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/verify.yml @@ -1,46 +1,47 @@ --- -- name: Get cluster version +- name: verify | Get cluster version environment: KUBECONFIG: /home/{{ admin_user.name }}/.kube/config shell: kubectl version --short | grep -i server changed_when: false register: kubectl_cluster_version -- name: Get kubectl version +- name: verify | Verify cluster version + assert: + that: "'{{ version }}' in kubectl_cluster_version.stdout" + +- name: verify | Get kubectl version environment: KUBECONFIG: /home/{{ admin_user.name }}/.kube/config shell: kubectl version --client --short | awk '{print $3}' changed_when: false register: kubectl_client_version -- name: Get kubeadm version +- name: verify | Verify kubectl version + assert: + that: "'{{ version }}' in kubectl_client_version.stdout" + +- name: verify | Get kubeadm version environment: KUBECONFIG: /home/{{ admin_user.name }}/.kube/config shell: kubeadm version -o short changed_when: false register: kubeadm_version -- name: Get node version +- name: verify | Verify kubeadm version + assert: + that: "'{{ version }}' in kubeadm_version.stdout" + +- name: verify | Verify node version environment: KUBECONFIG: /home/{{ admin_user.name }}/.kube/config shell: kubectl get nodes {{ inventory_hostname }} -o wide | awk '{print $2" "$5}' changed_when: false register: get_node_status + until: version in get_node_status.stdout + retries: 30 # 1min + delay: 2 -- name: Verify cluster version - assert: - that: "'{{ version }}' in kubectl_cluster_version.stdout" - -- name: Verify kubectl version - assert: - that: "'{{ version }}' in kubectl_client_version.stdout" - -- name: Verify kubeadm version - assert: - that: "'{{ version }}' in kubeadm_version.stdout" - -- name: Verify node version and status +- name: verify | Verify node status assert: - that: - - "'{{ version }}' in get_node_status.stdout" - - "'Ready' in get_node_status.stdout" + that: "'Ready' in get_node_status.stdout" diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/wait.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/wait.yml index 9ffb84eb2e..e3196b6cd9 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/wait.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/wait.yml @@ -1,9 +1,9 @@ --- -- name: Wait for kubectl to find and access a Kubernetes cluster +- name: Wait for kubectl to find and access K8s cluster environment: KUBECONFIG: /home/{{ admin_user.name }}/.kube/config shell: kubectl cluster-info - retries: 50 + retries: 30 # 1min delay: 2 register: output until: output is succeeded and "running" in output.stdout @@ -15,7 +15,7 @@ shell: kubectl get nodes -o json register: output until: output.stdout|from_json|json_query("items[*].status.conditions[?(@.type=='Ready')].status[]")|unique == ["True"] - retries: 600 + retries: 600 # 20min delay: 2 changed_when: false @@ -25,7 +25,7 @@ shell: kubectl get pods --all-namespaces -o json register: output until: output.stdout|from_json|json_query('items[*].status.phase')|unique == ["Running"] - retries: 150 + retries: 600 # 20min delay: 2 changed_when: false @@ -35,6 +35,6 @@ shell: kubectl get pods --all-namespaces -o json register: output until: output.stdout|from_json|json_query('items[*].status.conditions[].status')|unique == ["True"] - retries: 600 + retries: 600 # 20min delay: 2 changed_when: false \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/upgrade.yml b/core/src/epicli/data/common/ansible/playbooks/upgrade.yml index 2ce17ba3c0..3e6dd0999f 100644 --- a/core/src/epicli/data/common/ansible/playbooks/upgrade.yml +++ b/core/src/epicli/data/common/ansible/playbooks/upgrade.yml @@ -1,5 +1,5 @@ --- -# Ansible playbook for upgrading Kubernetes cluster +# Ansible playbook for upgrading Kubernetes cluster - hosts: kubernetes_master:kubernetes_node serial: 1 @@ -13,19 +13,19 @@ become: true become_method: sudo roles: - - { role: upgrade, ver: "1.13.8", cni_ver: "0.7.5" } + - { role: upgrade, ver: "1.13.12", cni_ver: "0.7.5" } - hosts: kubernetes_master:kubernetes_node serial: 1 become: true become_method: sudo roles: - - { role: upgrade, ver: "1.14.4", cni_ver: "0.7.5" } + - { role: upgrade, ver: "1.14.6", cni_ver: "0.7.5" } # latest patch versions: # 1.11.10 # 1.12.10 -# 1.13.8 -# 1.14.4 +# 1.13.12 +# 1.14.8