Skip to content

Commit

Permalink
Improvements and fix for upgrade role
Browse files Browse the repository at this point in the history
  • Loading branch information
to-bar committed Oct 29, 2019
1 parent 51f4ac4 commit b039b40
Show file tree
Hide file tree
Showing 6 changed files with 88 additions and 114 deletions.
Original file line number Diff line number Diff line change
@@ -1,36 +1,35 @@
---
- name: Check cluster version
- name: Get cluster version
environment:
KUBECONFIG: "/home/{{ admin_user.name }}/.kube/config"
shell: kubectl version --short | grep -i server
shell: >-
set -o pipefail &&
kubectl version --short -o json | jq --raw-output '.serverVersion.gitVersion'
register: cluster_version
changed_when: false

- name: Check kubelet version
- name: Get kubelet version
environment:
KUBECONFIG: "/home/{{ admin_user.name }}/.kube/config"
shell: kubectl get node {{ inventory_hostname }} -o custom-columns=VERSION:.status.nodeInfo.kubeletVersion
shell: >-
kubectl get node {{ inventory_hostname }} -o jsonpath='{.status.nodeInfo.kubeletVersion}'
register: kubelet_version
changed_when: false

- name: Upgrade master to {{ version }}
include_tasks: "upgrade_master.yml"
- name: Upgrade master to v{{ version }}
include_tasks: upgrade_master.yml
vars:
version: "{{ ver }}"
cni_version: "{{ cni_ver }}"
when:
- groups['kubernetes_master'][0] == inventory_hostname
- (version == "1.12.10" and "1.11" in cluster_version.stdout) or
(version == "1.13.8" and "1.12" in cluster_version.stdout) or
(version == "1.14.4" and "1.13" in cluster_version.stdout)
- cluster_version.stdout is version('v' + version, '<=')

- name: Upgrade nodes to {{ version }}
include_tasks: "upgrade_nodes.yml"
- name: Upgrade node to v{{ version }}
include_tasks: upgrade_node.yml
vars:
version: "{{ ver }}"
cni_version: "{{ cni_ver }}"
when:
- inventory_hostname in groups['kubernetes_node']
- (version == "1.12.10" and "1.11" in kubelet_version.stdout and kubelet_version.stdout != cluster_version.stdout) or
(version == "1.13.8" and "1.12" in kubelet_version.stdout and kubelet_version.stdout != cluster_version.stdout) or
(version == "1.14.4" and "1.13" in kubelet_version.stdout and kubelet_version.stdout != cluster_version.stdout )
- kubelet_version.stdout is version('v' + version, '<=')
Original file line number Diff line number Diff line change
@@ -1,16 +1,18 @@
---
- name: Wait for the cluster's readiness
- name: upgrade_master | Wait for cluster's readiness
include_tasks: wait.yml

- name: Drain master in preparation for maintenance
- name: upgrade_master | Drain master in preparation for maintenance
environment:
KUBECONFIG: /home/{{ admin_user.name }}/.kube/config
shell: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-local-data

- name: Wait for the cluster's readiness
- name: upgrade_master | Wait for cluster's readiness
include_tasks: wait.yml

- name: Install kubernetes-cni {{ cni_version }}, kubelet {{ version }}, kubectl {{ version }} and kubeadm {{ version }} packages for RedHat family
- name: >-
upgrade_master | Install kubernetes-cni-{{ cni_version }}, kubelet-{{ version }}, kubectl-{{ version }}
and kubeadm-{{ version }} packages for RedHat family
yum:
name:
- kubernetes-cni-{{ cni_version }}
Expand All @@ -23,14 +25,14 @@
when:
- ansible_os_family == "RedHat"

- name: Get information about installed packages as facts
- name: upgrade_master | Get information about installed packages as facts
package_facts:
manager: auto
changed_when: false
when:
- ansible_os_family == "Debian"

- name: Remove newer Debian packages installed as dependencies if they exist # as there is no allow_downgrade parameter in ansible apt module
- name: upgrade_master | Remove newer Debian packages installed as dependencies if they exist # as there is no allow_downgrade parameter in ansible apt module
apt:
name:
- kubernetes-cni
Expand All @@ -45,7 +47,9 @@
(ansible_facts.packages['kubectl'][0].version is version (version + '-00', '>')) or
(ansible_facts.packages['kubeadm'][0].version is version (version + '-00', '>'))

- name: Install kubernetes-cni {{ cni_version }}, kubelet {{ version }}, kubectl {{ version }} and kubeadm {{ version }} packages for Debian family
- name: >-
upgrade_master | Install kubernetes-cni {{ cni_version }}, kubelet {{ version }}, kubectl {{ version }}
and kubeadm {{ version }} packages for Debian family
apt:
name:
- kubernetes-cni={{ cni_version }}-00
Expand All @@ -57,92 +61,58 @@
when:
- ansible_os_family == "Debian"

- name: Wait for the cluster's readiness
- name: upgrade_master | Wait for cluster's readiness
include_tasks: wait.yml

- name: Check if kubeadm configuration file exists
- name: upgrade_master | Check if kubeadm configuration file exists
stat:
path: /etc/kubeadm/kubeadm-config.yml
changed_when: false
register: kubeadm_config

# https://github.com/kubernetes/kubeadm/issues/1471 Upgrading a 1.12 cluster thru 1.13 to 1.14 fails
- name: Validate whether current cluster is upgradeable (from ver. 1.13)
block:
- name: Show upgrade plan (using kubeadm configuration file)
shell: kubeadm upgrade plan v{{ version }} --config /etc/kubeadm/kubeadm-config.yml
changed_when: false
when:
- kubeadm_config.stat.exists

- name: Show upgrade plan
shell: kubeadm upgrade plan v{{ version }}
changed_when: false
when:
- not kubeadm_config.stat.exists
when:
- '"1.13" in cluster_version.stdout'

rescue:
- name: Find the existing etcd server certificates
find:
paths: /etc/kubernetes/pki/etcd
patterns: "*server.*"
changed_when: false
register: files_to_delete

- name: Remove the existing etcd server certificates
file:
path: "{{ item.path }}"
state: absent
with_items: "{{ files_to_delete.files }}"

- name: Regenerate the etcd server certificates
shell: kubeadm init phase certs etcd-server

- name: Validate whether current cluster is upgradeable (using kubeadm configuration file)
- name: upgrade_master | Validate whether cluster is upgradeable (using kubeadm configuration file)
shell: kubeadm upgrade plan v{{ version }} --config /etc/kubeadm/kubeadm-config.yml
changed_when: false
when:
- kubeadm_config.stat.exists

- name: Validate whether current cluster is upgradeable
- name: upgrade_master | Validate whether cluster is upgradeable
shell: kubeadm upgrade plan v{{ version }}
changed_when: false
when:
- not kubeadm_config.stat.exists

- name: Upgrade Kubernetes cluster to the specified version v{{ version }} (using kubeadm configuration file)
- name: upgrade_master | Upgrade K8s cluster to v{{ version }} (using kubeadm configuration file)
shell: kubeadm upgrade apply -y v{{ version }} --config /etc/kubeadm/kubeadm-config.yml
retries: 5
delay: 5
register: output
until: output is succeeded
retries: 5
delay: 5
when:
- kubeadm_config.stat.exists

- name: Upgrade Kubernetes cluster to the specified version v{{ version }}
- name: upgrade_master | Upgrade K8s cluster to v{{ version }}
shell: kubeadm upgrade apply -y v{{ version }}
retries: 5
delay: 5
register: output
until: output is succeeded
retries: 5
delay: 5
when:
- not kubeadm_config.stat.exists

- name: Wait for the cluster's readiness
- name: upgrade_master | Wait for cluster's readiness
include_tasks: wait.yml

- name: Restart kubelet
- name: upgrade_master | Restart kubelet
systemd:
state: restarted
daemon_reload: yes
name: kubelet

- name: Wait for the cluster's readiness
- name: upgrade_master | Wait for cluster's readiness
include_tasks: wait.yml

- name: Uncordon master - mark master as schedulable
- name: upgrade_master | Uncordon master - mark master as schedulable
environment:
KUBECONFIG: /home/{{ admin_user.name }}/.kube/config
shell: kubectl uncordon {{ inventory_hostname }}
Expand All @@ -151,5 +121,5 @@
register: output
until: output is succeeded

- name: Verify cluster version
- name: upgrade_master | Verify cluster version
include_tasks: verify.yml
Original file line number Diff line number Diff line change
@@ -1,17 +1,19 @@
---
- name: Wait for the cluster's readiness
- name: upgrade_node | Wait for cluster's readiness
include_tasks: wait.yml

- name: Drain node in preparation for maintenance
- name: upgrade_node | Drain node in preparation for maintenance
environment:
KUBECONFIG: /home/{{ admin_user.name }}/.kube/config
shell: kubectl drain {{ inventory_hostname }} --ignore-daemonsets --delete-local-data
delegate_to: "{{ groups['kubernetes_master'][0] }}"

- name: Wait for the cluster's readiness
- name: upgrade_node | Wait for cluster's readiness
include_tasks: wait.yml

- name: Install kubernetes-cni {{ cni_version }}, kubelet {{ version }}, kubectl {{ version }} and kubeadm {{ version }} packages for RedHat family
- name: >-
upgrade_node | Install kubernetes-cni-{{ cni_version }}, kubelet-{{ version }}, kubectl-{{ version }}
and kubeadm-{{ version }} packages for RedHat family
yum:
name:
- kubernetes-cni-{{ cni_version }}
Expand All @@ -24,14 +26,14 @@
when:
- ansible_os_family == "RedHat"

- name: Get information about installed packages as facts
- name: upgrade_node | Get information about installed packages as facts
package_facts:
manager: auto
changed_when: false
when:
- ansible_os_family == "Debian"

- name: Remove newer Debian packages installed as dependencies if they exist # as there is no allow_downgrade parameter in ansible apt module
- name: upgrade_node | Remove newer Debian packages installed as dependencies if they exist # as there is no allow_downgrade parameter in ansible apt module
apt:
name:
- kubernetes-cni
Expand All @@ -46,7 +48,9 @@
(ansible_facts.packages['kubectl'][0].version is version (version + '-00', '>')) or
(ansible_facts.packages['kubeadm'][0].version is version (version + '-00', '>'))

- name: Install kubernetes-cni {{ cni_version }}, kubelet {{ version }}, kubectl {{ version }} and kubeadm {{ version }} packages for Debian family
- name: >-
upgrade_node | Install kubernetes-cni {{ cni_version }}, kubelet {{ version }}, kubectl {{ version }}
and kubeadm {{ version }} packages for Debian family
apt:
name:
- kubernetes-cni={{ cni_version }}-00
Expand All @@ -58,27 +62,27 @@
when:
- ansible_os_family == "Debian"

- name: Upgrade node config
- name: upgrade_node | Upgrade node config
shell: kubeadm upgrade node config --kubelet-version v{{ version }}

- name: Restart kubelet
- name: upgrade_node | Restart kubelet
systemd:
state: restarted
daemon_reload: yes
name: kubelet

- name: Wait for the cluster's readiness
- name: upgrade_node | Wait for cluster's readiness
include_tasks: wait.yml

- name: Uncordon node - mark node as schedulable
- name: upgrade_node | Uncordon node - mark node as schedulable
environment:
KUBECONFIG: /home/{{ admin_user.name }}/.kube/config
shell: kubectl uncordon {{ inventory_hostname }}
retries: 5
delay: 5
register: output
until: output is succeeded
retries: 5
delay: 5
delegate_to: "{{ groups['kubernetes_master'][0] }}"

- name: Verify cluster version
- name: upgrade_node | Verify cluster version
include_tasks: verify.yml
Original file line number Diff line number Diff line change
@@ -1,46 +1,47 @@
---
- name: Get cluster version
- name: verify | Get cluster version
environment:
KUBECONFIG: /home/{{ admin_user.name }}/.kube/config
shell: kubectl version --short | grep -i server
changed_when: false
register: kubectl_cluster_version

- name: Get kubectl version
- name: verify | Verify cluster version
assert:
that: "'{{ version }}' in kubectl_cluster_version.stdout"

- name: verify | Get kubectl version
environment:
KUBECONFIG: /home/{{ admin_user.name }}/.kube/config
shell: kubectl version --client --short | awk '{print $3}'
changed_when: false
register: kubectl_client_version

- name: Get kubeadm version
- name: verify | Verify kubectl version
assert:
that: "'{{ version }}' in kubectl_client_version.stdout"

- name: verify | Get kubeadm version
environment:
KUBECONFIG: /home/{{ admin_user.name }}/.kube/config
shell: kubeadm version -o short
changed_when: false
register: kubeadm_version

- name: Get node version
- name: verify | Verify kubeadm version
assert:
that: "'{{ version }}' in kubeadm_version.stdout"

- name: verify | Verify node version
environment:
KUBECONFIG: /home/{{ admin_user.name }}/.kube/config
shell: kubectl get nodes {{ inventory_hostname }} -o wide | awk '{print $2" "$5}'
changed_when: false
register: get_node_status
until: version in get_node_status.stdout
retries: 30 # 1min
delay: 2

- name: Verify cluster version
assert:
that: "'{{ version }}' in kubectl_cluster_version.stdout"

- name: Verify kubectl version
assert:
that: "'{{ version }}' in kubectl_client_version.stdout"

- name: Verify kubeadm version
assert:
that: "'{{ version }}' in kubeadm_version.stdout"

- name: Verify node version and status
- name: verify | Verify node status
assert:
that:
- "'{{ version }}' in get_node_status.stdout"
- "'Ready' in get_node_status.stdout"
that: "'Ready' in get_node_status.stdout"
Loading

0 comments on commit b039b40

Please sign in to comment.