From e6019134fde78d4c39e604c0eba2c9a009775884 Mon Sep 17 00:00:00 2001
From: to-bar <46519524+to-bar@users.noreply.github.com>
Date: Fri, 3 Jul 2020 19:40:02 +0200
Subject: [PATCH 1/7] Clean up and optimize K8s upgrades

---
 .../Debian/install-packages-pre-1.15.yml      |  31 ---
 .../kubernetes/migrate-kubeadm-config.yml     |  14 --
 .../update-kubeadm-image-repository.yml       |  34 ++++
 .../kubernetes/upgrade-kubeadm-config.yml     |  60 +-----
 .../tasks/kubernetes/upgrade-master.yml       | 183 ++++--------------
 .../upgrade/tasks/kubernetes/upgrade-node.yml |  17 --
 6 files changed, 78 insertions(+), 261 deletions(-)
 delete mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/Debian/install-packages-pre-1.15.yml
 delete mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/migrate-kubeadm-config.yml
 create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/update-kubeadm-image-repository.yml

diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/Debian/install-packages-pre-1.15.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/Debian/install-packages-pre-1.15.yml
deleted file mode 100644
index 6553ee646a..0000000000
--- a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/Debian/install-packages-pre-1.15.yml
+++ /dev/null
@@ -1,31 +0,0 @@
----
-- name: install-packages | Get information about installed packages as facts
-  package_facts:
-    manager: auto
-  changed_when: false
-
-- name: install-packages | Remove newer Debian packages installed as dependencies if they exist # as there is no allow_downgrade parameter in ansible apt module
-  apt:
-    name:
-      - kubernetes-cni
-      - kubelet
-      - kubectl
-      - kubeadm
-    state: absent
-  when:
-    - (ansible_facts.packages['kubernetes-cni'][0].version is version (cni_version + '-00', '>')) or
-      (ansible_facts.packages['kubelet'][0].version is version (version + '-00', '>')) or
-      (ansible_facts.packages['kubectl'][0].version is version (version + '-00', '>')) or
-      (ansible_facts.packages['kubeadm'][0].version is version (version + '-00', '>'))
-
-- name: >-
-    install-packages | Install kubernetes-cni {{ cni_version }}, kubelet {{ version }}, kubectl {{ version }}
-    and kubeadm {{ version }} packages for Debian family
-  apt:
-    name:
-      - kubernetes-cni={{ cni_version }}-00
-      - kubelet={{ version }}-00
-      - kubectl={{ version }}-00
-      - kubeadm={{ version }}-00
-    update_cache: yes
-    state: present
\ No newline at end of file
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/migrate-kubeadm-config.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/migrate-kubeadm-config.yml
deleted file mode 100644
index 5eb73e273b..0000000000
--- a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/migrate-kubeadm-config.yml
+++ /dev/null
@@ -1,14 +0,0 @@
-
-- name: "migrate-kubeadm-config | Backup old kubeadm-config"
-  copy:
-    src: /etc/kubeadm/kubeadm-config.yml
-    dest: /etc/kubeadm/kubeadm-config.yml.bak
-    remote_src: yes
-
-- name: "migrate-kubeadm-config | Backup old kubeadm-config"
-  become: yes
-  shell: >-
-    kubeadm config migrate --old-config /etc/kubeadm/kubeadm-config.yml --new-config /etc/kubeadm/kubeadm-config.yml
-
-- name: Update kubeadm-config with latest version
-  include_tasks: upgrade-kubeadm-config.yml
\ No newline at end of file
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/update-kubeadm-image-repository.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/update-kubeadm-image-repository.yml
new file mode 100644
index 0000000000..a2b068a411
--- /dev/null
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/update-kubeadm-image-repository.yml
@@ -0,0 +1,34 @@
+---
+# Note: Usage of the --config flag for reconfiguring the cluster during upgrade is not recommended since v1.16
+- name: upgrade-master | Get value of imageRepository from kubeadm-config ConfigMap
+  shell: kubeadm config view
+  changed_when: false
+  register: result
+
+- name: upgrade-master | Set current value of imageRepository as fact
+  set_fact:
+    kubeadm_image_repository: "{{ (result.stdout|from_yaml).imageRepository }}"
+
+- name: upgrade-master | Set new value for imageRepository as fact
+  set_fact:
+    new_kubeadm_image_repository: >-
+      {%- if kubeadm_image_repository is search(':') -%}
+        {{ kubeadm_image_repository | regex_replace('^(?P<host>.+):(?P<port>\d+)', image_registry_address) }}
+      {%- else -%}
+        {{ image_registry_address }}/{{ kubeadm_image_repository }}
+      {%- endif -%}
+
+- name: upgrade-master | Patch imageRepository in kubeadm-config ConfigMap
+  when:
+    - kubeadm_image_repository != new_kubeadm_image_repository
+  environment:
+    KUBECONFIG: /home/{{ admin_user.name }}/.kube/config
+  shell: |-
+    set -o pipefail &&
+    # do not use --export option since it has been deprecated in 1.14
+    kubectl get cm kubeadm-config -n kube-system -o yaml |
+    sed 's|imageRepository: {{ kubeadm_image_repository }}|imageRepository: {{ new_kubeadm_image_repository }}|g' |
+    xargs --null -I config_map_content \
+    kubectl patch cm kubeadm-config -n kube-system --patch config_map_content
+  args:
+    executable: /bin/bash
\ No newline at end of file
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-kubeadm-config.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-kubeadm-config.yml
index 8b30788bfb..c13c94866c 100644
--- a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-kubeadm-config.yml
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-kubeadm-config.yml
@@ -1,58 +1,4 @@
 ---
-- name: Run assertions for parameters of the task file at hand
-  block:
-    - assert:
-        that:
-          - version is defined
-          - version is string
-          - version | length > 0
-        fail_msg: "Invalid version string."
-
-- name: Check the kubeadm-config.yml file
-  stat:
-    path: &kubeadm-config-yml /etc/kubeadm/kubeadm-config.yml
-    get_attributes: false
-    get_checksum: false
-    get_mime: false
-  register: stat_kubeadm_config_file
-
-- when: stat_kubeadm_config_file.stat.exists
-  block:
-    - name: Load contents of the kubeadm-config.yml file
-      slurp:
-        path: *kubeadm-config-yml
-      register: slurp_kubeadm_config
-
-    - name: Save modified contents of the kubeadm-config.yml file
-      copy:
-        dest: *kubeadm-config-yml
-
-        # Save all documents.
-        content: |
-          {% for document in _documents_updated %}
-          ---
-          {{ document | to_nice_yaml(indent=2) }}
-          {% endfor -%}
-
-      vars:
-        # Parse yaml payload (remove empty documents).
-        _documents: >-
-          {{ slurp_kubeadm_config.content | b64decode
-                                          | from_yaml_all
-                                          | select
-                                          | list }}
-        # Prepare the patch.
-        _update:
-          kubernetesVersion: "v{{ version }}"
-
-        # Process all documents (returns a list of dictionaries).
-        _documents_updated: >-
-          {%- set output = [] -%}
-          {%- for document in _documents -%}
-            {%- if document.kind is defined and document.kind == 'ClusterConfiguration' -%}
-              {{- output.append(document | combine(_update, recursive=true)) -}}
-            {%- else -%}
-              {{- output.append(document) -}}
-            {%- endif -%}
-          {%- endfor -%}
-          {{- output -}}
+- name: upgrade-kubeadm-config | Save kubeadm-config ConfigMap to file
+  shell: >-
+    kubeadm config view > /etc/kubeadm/kubeadm-config.yml
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-master.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-master.yml
index 81852212dc..b9d4a0f95e 100644
--- a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-master.yml
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-master.yml
@@ -16,44 +16,8 @@
 - name: upgrade-master | Make sure the etcd encryption feature is properly configured (if enabled)
   import_tasks: patch-kubeadm-config.yml
 
-- name: Set imageRepository in kubeadm-config ConfigMap to use {{ image_registry_address }}
-  when:
-    - not kubeadm_config_file.stat.exists
-  block:
-    - name: upgrade-master | Set parent key of imageRepository key in kubeadm-config ConfigMap
-      set_fact:
-        kubeadm_config_parent_key: >-
-          {{ 'MasterConfiguration' if 'v1.11' in cluster_version else 'ClusterConfiguration' }}
-
-    - name: upgrade-master | Get value of imageRepository key
-      shell: |
-        kubectl get cm -n kube-system kubeadm-config -o jsonpath={{ jsonpath }}
-      vars:
-        jsonpath: >-
-          '{.data.{{ kubeadm_config_parent_key }}}'
-      environment:
-        KUBECONFIG: /home/{{ admin_user.name }}/.kube/config
-      changed_when: false
-      register: result
-
-    - name: upgrade-master | Save value of imageRepository as fact
-      set_fact:
-        kubeadm_image_repository: "{{ (result.stdout|from_yaml).imageRepository }}"
-
-    - name: upgrade-master | Patch imageRepository in kubeadm-config ConfigMap
-      environment:
-        KUBECONFIG: /home/{{ admin_user.name }}/.kube/config
-      shell: |-
-        set -o pipefail &&
-        # do not use --export option since it has been deprecated in 1.14
-        kubectl get cm kubeadm-config -n kube-system -o yaml |
-        sed 's/imageRepository: {{ kubeadm_image_repository }}/imageRepository: {{ image_registry_address }}\/{{ kubeadm_image_repository }}/g' |
-        xargs --null -I config_map_content \
-        kubectl patch cm kubeadm-config -n kube-system --patch config_map_content
-      args:
-        executable: /bin/bash
-      when:
-        - not image_registry_address in kubeadm_image_repository
+- name: upgrade-master | Update imageRepository in kubeadm-config ConfigMap
+  include_tasks: update-kubeadm-image-repository.yml
 
 - name: upgrade-master | Reconfigure coredns
   include_tasks: reconfigure-coredns.yml
@@ -80,10 +44,7 @@
 - name: upgrade-master | Wait for cluster's readiness
   include_tasks: wait.yml
 
-# For Redhat and Debian K8s > 1.14.6 we want to update packages like so
 - name: upgrade-master | Upgrade, configure packages
-  when: version is version('1.15.0', '>=')
-     or ansible_os_family == "RedHat"
   block:
     - name: upgrade-master | Hold packages for Debian family
       include_tasks: "Debian/hold-packages.yml"
@@ -105,121 +66,57 @@
     - name: upgrade-master | Wait for cluster's readiness
       include_tasks: wait.yml
 
-    - name: upgrade-master | Migrate kubeadm config
-      include_tasks: migrate-kubeadm-config.yml
-      when:
-        - kubeadm_config_file.stat.exists
-        - version is version('1.17.7', '==') # Migrate kubeadm config for K8s 1.17
-
-    - name: upgrade-master | Upgrade plan block
-      block:
-        - name: "upgrade-master | Validate whether cluster is upgradeable
-                {{ '(using kubeadm-config.yml file)' if kubeadm_config_file.stat.exists else '' }}"
-          shell: >-
-            kubeadm upgrade plan v{{ version +
-            (' --config /etc/kubeadm/kubeadm-config.yml' if kubeadm_config_file.stat.exists else '') }}
-          changed_when: false
-          register: result
-          until: result is succeeded
-          retries: 20
-          delay: 30
-
-      rescue: # ignore CoreDNSUnsupportedPlugins error since coredns migration does not support all plugins that are valid and currently used
-        - name: "upgrade-master | Validate whether cluster is upgradeable with ignore CoreDNS plugin related erroros
-                {{ '(using kubeadm-config.yml file)' if kubeadm_config_file.stat.exists else '' }}"
-          shell: >-
-            kubeadm upgrade plan v{{ version +
-            (' --config /etc/kubeadm/kubeadm-config.yml' if kubeadm_config_file.stat.exists else '') }} --ignore-preflight-errors=CoreDNSUnsupportedPlugins
-          changed_when: false
-          register: result
-          until: result is succeeded
-          retries: 20
-          delay: 30
-
-    - name: upgrade-master | Upgrade apply block
-      block:
-        - name: "upgrade-master | Upgrade K8s cluster to v{{ version }}
-                {{ '(using kubeadm-config.yml file)' if kubeadm_config_file.stat.exists else '' }}"
-          shell: >-
-            kubeadm upgrade apply -y v{{ version +
-            (' --config /etc/kubeadm/kubeadm-config.yml' if kubeadm_config_file.stat.exists else '') }}
-          register: result
-          until: result is succeeded
-          retries: 20
-          delay: 30
-          when:
-            - cluster_version is version('v' + version, '<') # without this condition fails when 'upgrading' again from 1.12.10 to 1.12.10
-
-      rescue:  # ignore CoreDNSUnsupportedPlugins error since coredns migration does not support all plugins that are valid and currently used
-        - name: "upgrade-master | Upgrade K8s cluster to v{{ version }}
-                {{ '(using kubeadm-config.yml file)' if kubeadm_config_file.stat.exists else '' }}"
-          shell: >-
-            kubeadm upgrade apply -y v{{ version +
-            (' --config /etc/kubeadm/kubeadm-config.yml' if kubeadm_config_file.stat.exists else '') }} --ignore-preflight-errors=CoreDNSUnsupportedPlugins
-          register: result
-          until: result is succeeded
-          retries: 20
-          delay: 30
-          when:
-            - cluster_version is version('v' + version, '<')
-
-    - name: Install kubelet and kubectl for {{ version }}
-      include_tasks: >-
-        {%- if cni_in_kubelet is undefined or not cni_in_kubelet -%}
-          {{ ansible_os_family }}/install-packages.yml
+    - name: upgrade-master | Validate whether cluster is upgradeable
+      # Ignore CoreDNSUnsupportedPlugins error since coredns-migration does not support 'hosts' plugin.
+      # This issue is fixed in K8s v1.18, see https://github.com/kubernetes/kubernetes/pull/88482
+      shell: >-
+        {%- if version is version('1.18.0', '>=') -%}
+           kubeadm upgrade plan v{{ version }}
         {%- else -%}
-          {{ ansible_os_family }}/install-packages-cni-in-kubelet.yml
+          kubeadm upgrade plan v{{ version }} --ignore-preflight-errors=CoreDNSUnsupportedPlugins
         {%- endif -%}
-      when: result is succeeded
-
-    - name: Update kubeadm-config.yml if exists
-      include_tasks: upgrade-kubeadm-config.yml
-      when: kubeadm_config_file.stat.exists
-
-# For Debian & K8s <= 1.14.6 we want to update packages like so
-- name: upgrade-master | Upgrade, configure packages
-  when:
-    - ansible_os_family == "Debian"
-    - version is version('1.14.6', '<=')
-  block:
-    - name: upgrade-master | Install packages
-      include_tasks: "{{ ansible_os_family }}/install-packages-pre-1.15.yml"
-
-    - name: upgrade-master | Wait for cluster's readiness
-      include_tasks: wait.yml
-
-    - name: "upgrade-master | Validate whether cluster is upgradeable
-            {{ '(using kubeadm-config.yml file)' if kubeadm_config_file.stat.exists else '' }}"
-      shell: >-
-        kubeadm upgrade plan v{{ version +
-        (' --config /etc/kubeadm/kubeadm-config.yml' if kubeadm_config_file.stat.exists else '') }}
       changed_when: false
       register: result
       until: result is succeeded
-      retries: 5
-      delay: 5
+      retries: 20
+      delay: 30
 
-    - name: "upgrade-master | Upgrade K8s cluster to v{{ version }}
-            {{ '(using kubeadm-config.yml file)' if kubeadm_config_file.stat.exists else '' }}"
+    # Note: Usage of the --config flag for reconfiguring the cluster during upgrade is not recommended since v1.16
+    - name: upgrade-master | Upgrade K8s cluster to v{{ version }}
+      # Ignore CoreDNSUnsupportedPlugins error since coredns-migration does not support 'hosts' plugin.
+      # This issue is fixed in K8s v1.18, see https://github.com/kubernetes/kubernetes/pull/88482
       shell: >-
-        kubeadm upgrade apply -y v{{ version +
-        (' --config /etc/kubeadm/kubeadm-config.yml' if kubeadm_config_file.stat.exists else '') }}
+        {%- if version is version('1.18.0', '>=') -%}
+           kubeadm upgrade apply -y v{{ version }}
+        {%- else -%}
+          kubeadm upgrade apply -y v{{ version }} --ignore-preflight-errors=CoreDNSUnsupportedPlugins
+        {%- endif -%}
+      changed_when: false
       register: result
       until: result is succeeded
-      retries: 5
-      delay: 5
+      retries: 20
+      delay: 30
       when:
-        - cluster_version is version('v' + version, '<') # without this condition fails when 'upgrading' again from 1.12.10 to 1.12.10
+        - cluster_version is version('v' + version, '<')
+
+    - name: Install kubelet and kubectl for {{ version }}
+      include_tasks: >-
+        {%- if cni_in_kubelet is undefined or not cni_in_kubelet -%}
+          {{ ansible_os_family }}/install-packages.yml
+        {%- else -%}
+          {{ ansible_os_family }}/install-packages-cni-in-kubelet.yml
+        {%- endif -%}
+      when: result is succeeded
 
 - name: upgrade-master | Wait for cluster's readiness
   include_tasks: wait.yml
 
-- name: Upgrade Docker
-  block:
-    - name: upgrade-master | Upgrade Docker # this may restart Docker daemon
-      include_tasks: docker.yml
-  when:
-    - version is version('1.14.0', '>=') # Docker 18.09 validated since K8s 1.14
+- name: upgrade-master | Upgrade kubeadm-config.yml if exists
+  include_tasks: upgrade-kubeadm-config.yml
+  when: kubeadm_config_file.stat.exists
+
+- name: upgrade-master | Upgrade Docker # this may restart Docker daemon
+  include_tasks: docker.yml
 
 - name: upgrade-master | Stop Kubelet
   systemd:
@@ -264,3 +161,5 @@
 
 - name: upgrade-master | Verify cluster version
   include_tasks: verify-upgrade.yml
+
+# TODO: Create a flag file that the upgrade completed to not run it again for the same version next time
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-node.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-node.yml
index 680f721181..50d0bcd05f 100644
--- a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-node.yml
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-node.yml
@@ -10,13 +10,8 @@
 
 - name: upgrade-node | Upgrade Docker # this may restart Docker daemon
   include_tasks: docker.yml
-  when:
-    - version is version('1.14.0', '>=') # Docker 18.09 validated since K8s 1.14
 
-# For Redhat and Debian & K8s > 1.14.6 we want to update packages like so
 - name: upgrade-node | Upgrade, configure packages
-  when: version is version('1.15.0', '>=')
-     or ansible_os_family == "RedHat"
   block:
     - name: upgrade-node | Hold packages for Debian family
       include_tasks: "Debian/hold-packages.yml"
@@ -36,18 +31,6 @@
           {{ ansible_os_family }}/install-packages-cni-in-kubelet.yml
         {%- endif -%}
 
-# For Debian & K8s <= 1.14.6 we want to update packages like so
-- name: upgrade-node | Upgrade, configure packages
-  when:
-    - version is version('1.14.6', '<=')
-    - ansible_os_family == "Debian"
-  block:
-    - name: upgrade-node | Install packages
-      include_tasks: "{{ ansible_os_family }}/install-packages-pre-1.15.yml"
-
-    - name: upgrade-node | Upgrade node config
-      shell: kubeadm upgrade node config --kubelet-version v{{ version }}
-
 - name: upgrade-node | Restart kubelet
   systemd:
     state: restarted

From 44bea766eea3dc5ab345833d3ee8b7f5ab331a05 Mon Sep 17 00:00:00 2001
From: to-bar <46519524+to-bar@users.noreply.github.com>
Date: Wed, 8 Jul 2020 15:25:50 +0200
Subject: [PATCH 2/7] Patch only kubeadm-config ConfigMap

---
 .../tasks/kubernetes/patch-kubeadm-config.yml | 115 +++++-------------
 1 file changed, 29 insertions(+), 86 deletions(-)

diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubeadm-config.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubeadm-config.yml
index 3e96190218..07bd3bf115 100644
--- a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubeadm-config.yml
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubeadm-config.yml
@@ -1,82 +1,24 @@
 ---
-- name: Check the etc-encryption.conf file
-  stat:
-    path: &etc-encryption-conf /etc/kubernetes/pki/etcd/etc-encryption.conf
-    get_attributes: false
-    get_checksum: false
-    get_mime: false
-  register: stat_etcd_encryption_config_file
+# Since usage of the --config flag for reconfiguring the cluster during upgrade is not recommended
+# (warning added in v1.17), we patch kubeadm-config ConfigMap directly.
 
-# Assuming that if the etcd encryption config file is absent, then
-# the encryption feature has never been enabled for the cluster at hand.
-- when:
-    - stat_etcd_encryption_config_file.stat.exists
-  block:
-    - name: Check the kubeadm-config.yml file
-      stat:
-        path: &kubeadm-config-yml /etc/kubeadm/kubeadm-config.yml
-        get_attributes: false
-        get_checksum: false
-        get_mime: false
-      register: stat_kubeadm_config_file
-
-- when:
-    - stat_etcd_encryption_config_file.stat.exists
-    - stat_kubeadm_config_file.stat.exists
-  block:
-    - name: Load contents of the kubeadm-config.yml file
-      slurp:
-        path: *kubeadm-config-yml
-      register: slurp_kubeadm_config
-
-    - name: Save modified contents of the kubeadm-config.yml file
-      copy:
-        dest: *kubeadm-config-yml
-
-        # Save all documents.
-        content: |
-          {% for document in _documents_updated %}
-          ---
-          {{ document | to_nice_yaml(indent=2) }}
-          {% endfor -%}
-
-      vars:
-        # Parse yaml payload (remove empty documents).
-        _documents: >-
-          {{ slurp_kubeadm_config.content | b64decode
-                                          | from_yaml_all
-                                          | select
-                                          | list }}
-        # Prepare the patch.
-        # In this patch we include location of the etcd encryption config file.
-        # If it is not included, then the etcd encryption feature becomes disabled/broken.
-        # If it is not present on a cluster that has kube-system secrets already encrypted, then
-        # it may cause any upgrade attempt to freeze for a very long time (in Epiphany it has been reported to be even up to 8 hours).
-        _update:
-          apiServer:
-            extraArgs:
-              encryption-provider-config: *etc-encryption-conf
+# kube-apiserver uses --encryption-provider-config parameter to control how data is encrypted in etcd.
+# If this parameter is absent the encryption is not enabled.
+- name: upgrade-master | Check if encryption of secret data is enabled
+  shell: >-
+    grep -- '--encryption-provider-config' /etc/kubernetes/manifests/kube-apiserver.yaml
+  args:
+    executable: /bin/bash
+  register: shell_grep_encryption_flag
+  changed_when: false
+  failed_when: shell_grep_encryption_flag.rc > 1
 
-        # Process all documents (returns a list of dictionaries).
-        _documents_updated: >-
-          {%- set output = [] -%}
-          {%- for document in _documents -%}
-            {%- if document.kind is defined and document.kind == 'ClusterConfiguration' -%}
-              {{- output.append(document | combine(_update, recursive=true)) -}}
-            {%- else -%}
-              {{- output.append(document) -}}
-            {%- endif -%}
-          {%- endfor -%}
-          {{- output -}}
-
-# The `kubeadm upgrade` command can be executed with or without a config file.
-# If the kubeadm-config.yml file does not exists, then we at least patch the kubeadm-config configmap.
-- when:
-    - stat_etcd_encryption_config_file.stat.exists
-    - not stat_kubeadm_config_file.stat.exists
+- name: upgrade-master | Patch kubeadm-config ConfigMap if needed
+  when:
+    - shell_grep_encryption_flag.rc == 0 # encryption enabled
   run_once: true  # makes no sense to execute it more than once (would be redundant)
   block:
-    - name: Load the kubeadm-config configmap
+    - name: Get kubeadm-config ConfigMap
       shell: |
         kubectl get configmap kubeadm-config \
           --namespace kube-system \
@@ -88,9 +30,10 @@
       register: shell_kubeadm_configmap
       changed_when: false
 
-    # The following procedure ensures that etcd encryption is always enabled
-    # during subsequent kubeadm executions (if the config file is not present).
-    - name: Patch and re-apply the kubeadm-config configmap
+    # The following procedure ensures that etcd encryption is always enabled during subsequent kubeadm executions
+    - name: upgrade-master | Patch and re-apply the kubeadm-config ConfigMap
+      when:
+        - _kubeadm_api_server_extra_args['encryption-provider-config'] is undefined
       shell: |
         kubectl apply \
           --namespace kube-system \
@@ -100,32 +43,32 @@
         executable: /bin/bash
       environment:
         KUBECONFIG: *KUBECONFIG
-        # Render an altered kubeadm-config configmap document.
+        # Render an altered kubeadm-config configmap document
         KUBEADM_CONFIGMAP_DOCUMENT: >-
           {{ _document | combine(_update2, recursive=true) | to_nice_yaml(indent=2) }}
 
-      # Skip the task if there is no change in the cluster config.
-      when: _cluster_config_updated != _cluster_config  # comparing two dictionaries here
-
       vars:
-        # Parse yaml payload.
+        # Parse yaml payload
         _document: >-
           {{ shell_kubeadm_configmap.stdout | from_yaml }}
 
-        # Extract cluster config.
+        # Extract cluster config
         _cluster_config: >-
           {{ _document.data.ClusterConfiguration | from_yaml }}
 
-        # Prepare the cluster config patch.
+        _kubeadm_api_server_extra_args: >-
+          {{ _cluster_config.apiServer.extraArgs }}
+
+        # Prepare the cluster config patch
         _update1:
           apiServer:
             extraArgs:
-              encryption-provider-config: *etc-encryption-conf
+              encryption-provider-config: /etc/kubernetes/pki/etcd/etc-encryption.conf
 
         _cluster_config_updated: >-
           {{ _cluster_config | combine(_update1, recursive=true) }}
 
-        # Prepare the final update for the whole document.
+        # Prepare the final update for the whole document
         _update2:
           data:
             ClusterConfiguration: >-

From bb9163c81cfcce5bfaa2a10b3ff96eaed9d0884c Mon Sep 17 00:00:00 2001
From: to-bar <46519524+to-bar@users.noreply.github.com>
Date: Wed, 8 Jul 2020 22:41:11 +0200
Subject: [PATCH 3/7] Downgrade CoreDNS to K8s built-in version before 'kubeadm
 upgrade apply'

---
 .../tasks/deployments/deploy-template.yml     |   2 +-
 .../tasks/kubernetes/downgrade-coredns.yml    |  43 ++++++
 .../tasks/kubernetes/reconfigure-coredns.yml  |  52 --------
 .../tasks/kubernetes/upgrade-master.yml       |  10 +-
 .../coredns-config-for-k8s-below-1.16.yml.j2  | 126 ++++++++++++++++++
 5 files changed, 175 insertions(+), 58 deletions(-)
 create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/downgrade-coredns.yml
 delete mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/reconfigure-coredns.yml
 create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/upgrade/templates/kubernetes/coredns-config-for-k8s-below-1.16.yml.j2

diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/deployments/deploy-template.yml b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/deployments/deploy-template.yml
index 43a5dc7d8d..79f88ed63b 100644
--- a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/deployments/deploy-template.yml
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/deployments/deploy-template.yml
@@ -6,7 +6,7 @@
     state: directory
     owner: root
     group: root
-    mode: u=rw,go=r
+    mode: u=rwx,go=r
 
 - name: "Upload {{ file_name }} file"
   become: true
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/downgrade-coredns.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/downgrade-coredns.yml
new file mode 100644
index 0000000000..3069b147e7
--- /dev/null
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/downgrade-coredns.yml
@@ -0,0 +1,43 @@
+---
+# CoreDNS version matrix
+#+-----------------------------------------------------------------+
+#| Epiphany version | K8s version | Epiphany CoreDNS | K8s CoreDNS |
+#|------------------|-------------|------------------|-------------|
+#| 0.4.4            | 1.14.6      | 1.5.0            | 1.3.1       |
+#| 0.5.0            | 1.14.6      | 1.5.0            | 1.3.1       |
+#| upgrade < 0.6.0  | 1.15.10     | 1.5.0            | 1.3.1       |
+#| upgrade < 0.6.0  | 1.16.7      | 1.5.0            | 1.6.2       |
+#| 0.6.0            | 1.17.4      | 1.6.5            | 1.6.5       |
+#| 0.7.0            | 1.17.7      | 1.6.5            | 1.6.5       |
+#+-----------------------------------------------------------------+
+# Source: look for 'CoreDNSVersion' at https://github.com/kubernetes/kubernetes/blob/$TAG/cmd/kubeadm/app/constants/constants.go
+
+- name: upgrade-master | Create directory /etc/epiphany/manifests
+  become: true
+  file:
+    path: /etc/epiphany/manifests
+    state: directory
+    owner: root
+    group: root
+    mode: u=rwx,go=r
+
+- name: Upload and apply template
+  block:
+    - name: upgrade-master | Upload {{ file_name }} file
+      template:
+        src: kubernetes/{{ file_name }}.j2
+        dest: /etc/epiphany/manifests/{{ file_name }}
+        owner: "{{ admin_user.name }}"
+        group: "{{ admin_user.name }}"
+        mode: u=rw,go=r
+
+    - name: Apply /etc/epiphany/manifests/{{ file_name }} file
+      environment:
+        KUBECONFIG: /home/{{ admin_user.name }}/.kube/config
+      shell: |
+        kubectl apply \
+          -f /etc/epiphany/manifests/{{ file_name }}
+      args:
+        executable: /bin/bash
+  vars:
+    file_name: coredns-config-for-k8s-below-1.16.yml
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/reconfigure-coredns.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/reconfigure-coredns.yml
deleted file mode 100644
index aebc9602c5..0000000000
--- a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/reconfigure-coredns.yml
+++ /dev/null
@@ -1,52 +0,0 @@
----
-- name: Change coredns to use {{ image_registry_address }}
-  block:
-    - name: upgrade-master | Get coredns image
-      environment:
-        KUBECONFIG: /home/{{ admin_user.name }}/.kube/config
-      shell: |-
-        kubectl get deployments.apps coredns -n kube-system -o=jsonpath='{$.spec.template.spec.containers[:1].image}'
-      changed_when: false
-      register: coredns_image_repository
-      args:
-        executable: /bin/bash
-    - name: upgrade-master | Patch coredns to use {{ image_registry_address }}
-      environment:
-        KUBECONFIG: /home/{{ admin_user.name }}/.kube/config
-      shell: |-
-        set -o pipefail &&
-        kubectl patch deployment coredns --patch '{"spec": {"template": { "spec": { "containers": [ { "image": "{{ image_registry_address }}/{{ coredns_image_repository.stdout }}", "name": "coredns" }]}}}}' -n kube-system
-      args:
-        executable: /bin/bash
-      when:
-        - not image_registry_address in coredns_image_repository.stdout
-
-    - name: upgrade-master | Get coredns config map (Corefile)
-      environment:
-        KUBECONFIG: /home/{{ admin_user.name }}/.kube/config
-      shell: |-
-        kubectl get configmap coredns -n kube-system -o=jsonpath='{$.data.Corefile}'
-      changed_when: false
-      register: coredns_corefile
-      args:
-        executable: /bin/bash
-
-    - name: upgrade-master | Patch Corefile to not use deprecated plugin (upstream)
-      environment:
-        KUBECONFIG: /home/{{ admin_user.name }}/.kube/config
-      shell: |-
-        kubectl -n kube-system get configmap coredns -o yaml | sed  's/upstream//g' | kubectl apply -f - 
-      args:
-        executable: /bin/bash
-      when:
-        - '"upstream" in coredns_corefile.stdout'
-
-    - name: upgrade-master | Patch Corefile to not use deprecated plugin (replace proxy with forward plugin)
-      environment:
-        KUBECONFIG: /home/{{ admin_user.name }}/.kube/config
-      shell: |-
-        kubectl -n kube-system get configmap coredns -o yaml | sed  's/proxy/forward/g' | kubectl apply -f - 
-      args:
-        executable: /bin/bash
-      when:
-        - '"proxy" in coredns_corefile.stdout'
\ No newline at end of file
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-master.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-master.yml
index b9d4a0f95e..690fce5963 100644
--- a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-master.yml
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-master.yml
@@ -19,11 +19,6 @@
 - name: upgrade-master | Update imageRepository in kubeadm-config ConfigMap
   include_tasks: update-kubeadm-image-repository.yml
 
-- name: upgrade-master | Reconfigure coredns
-  include_tasks: reconfigure-coredns.yml
-  when:
-    - version is version('1.16.7', '<=')
-
 - name: upgrade-master | Reconfigure rabbitmq application
   include_tasks: reconfigure-rabbitmq-app.yml
 
@@ -58,6 +53,11 @@
       when:
         - version is version('1.16.7', '>=')
 
+    - name: upgrade-master | Downgrade CoreDNS to K8s built-in version
+      include_tasks: downgrade-coredns.yml
+      when:
+        - version is version('1.16.0', '<')
+
     - name: upgrade-master | Upgrade coredns for latest Kubernetes (1.17.7)
       include_tasks: upgrade-coredns.yml
       when:
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/templates/kubernetes/coredns-config-for-k8s-below-1.16.yml.j2 b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/templates/kubernetes/coredns-config-for-k8s-below-1.16.yml.j2
new file mode 100644
index 0000000000..67e08e4ac4
--- /dev/null
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/templates/kubernetes/coredns-config-for-k8s-below-1.16.yml.j2
@@ -0,0 +1,126 @@
+{#
+This configuration is compatible with CoreDNS v1.3.1 since 'kubeadm upgrade apply' may downgrade CoreDNS
+and then the upgrade may hang due to an error like: "/etc/coredns/Corefile:4 - Error during parsing: Unknown directive 'ready'"
+#}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: coredns
+  namespace: kube-system
+data:
+  Corefile: |
+    .:53 {
+        errors
+        health
+        hosts {
+           fallthrough
+        }
+        kubernetes cluster.local in-addr.arpa ip6.arpa {
+          pods insecure
+          upstream
+          fallthrough in-addr.arpa ip6.arpa
+        }
+        prometheus :9153
+        forward . /etc/resolv.conf
+        cache 30
+        loop
+        reload
+        loadbalance
+    }
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: coredns
+  namespace: kube-system
+  labels:
+    k8s-app: kube-dns
+    kubernetes.io/name: "CoreDNS"
+spec:
+  replicas: 2
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+  selector:
+    matchLabels:
+      k8s-app: kube-dns
+  template:
+    metadata:
+      labels:
+        k8s-app: kube-dns
+    spec:
+      priorityClassName: system-cluster-critical
+      serviceAccountName: coredns
+      tolerations:
+        - key: "CriticalAddonsOnly"
+          operator: "Exists"
+      nodeSelector:
+        beta.kubernetes.io/os: linux
+      containers:
+      - name: coredns
+        image: {{ image_registry_address }}/k8s.gcr.io/coredns:1.3.1
+        imagePullPolicy: IfNotPresent
+        resources:
+          limits:
+            memory: 170Mi
+          requests:
+            cpu: 100m
+            ephemeral-storage: 2Mi
+            memory: 70Mi
+        args: [ "-conf", "/etc/coredns/Corefile" ]
+        volumeMounts:
+        - name: config-volume
+          mountPath: /etc/coredns
+          readOnly: true
+        - name: hosts-volume
+          mountPath: /etc/hosts
+          readOnly: true
+        - name: tmp
+          mountPath: /tmp
+        ports:
+        - containerPort: 53
+          name: dns
+          protocol: UDP
+        - containerPort: 53
+          name: dns-tcp
+          protocol: TCP
+        - containerPort: 9153
+          name: metrics
+          protocol: TCP
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            add:
+            - NET_BIND_SERVICE
+            drop:
+            - all
+          readOnlyRootFilesystem: true
+        livenessProbe:
+          httpGet:
+            path: /health
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 60
+          timeoutSeconds: 5
+          successThreshold: 1
+          failureThreshold: 5
+        readinessProbe:
+          httpGet:
+            path: /health
+            port: 8080
+            scheme: HTTP
+      dnsPolicy: Default
+      volumes:
+        - name: tmp
+          emptyDir: {}
+        - name: config-volume
+          configMap:
+            name: coredns
+            items:
+            - key: Corefile
+              path: Corefile
+        - name: hosts-volume
+          hostPath:
+            path: /etc/hosts
+            type: File

From 5e494290b917bc73582bf9cdea0eace57077ee0f Mon Sep 17 00:00:00 2001
From: to-bar <46519524+to-bar@users.noreply.github.com>
Date: Thu, 9 Jul 2020 12:15:28 +0200
Subject: [PATCH 4/7] Deploy customized CoreDNS after K8s is upgraded to the
 latest version

---
 .../tasks/kubernetes/downgrade-coredns.yml       |  2 +-
 .../upgrade/tasks/kubernetes/upgrade-master.yml  | 16 +++++++++++-----
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/downgrade-coredns.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/downgrade-coredns.yml
index 3069b147e7..49b17d941b 100644
--- a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/downgrade-coredns.yml
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/downgrade-coredns.yml
@@ -31,7 +31,7 @@
         group: "{{ admin_user.name }}"
         mode: u=rw,go=r
 
-    - name: Apply /etc/epiphany/manifests/{{ file_name }} file
+    - name: upgrade-master | Apply /etc/epiphany/manifests/{{ file_name }} file
       environment:
         KUBECONFIG: /home/{{ admin_user.name }}/.kube/config
       shell: |
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-master.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-master.yml
index 690fce5963..c376a76019 100644
--- a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-master.yml
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-master.yml
@@ -53,16 +53,14 @@
       when:
         - version is version('1.16.7', '>=')
 
+    # CoreDNS is overwritten (upgraded or downgraded) by 'kubeadm upgrade apply', see CoreDNS version matrix in downgrade-coredns.yml.
+    # kubeadm upgrade is not able to downgrade coredns ConfigMap properly (at least when upgrade from 1.14.6 to 1.15.10)
+    # which may cause the update to hang.
     - name: upgrade-master | Downgrade CoreDNS to K8s built-in version
       include_tasks: downgrade-coredns.yml
       when:
         - version is version('1.16.0', '<')
 
-    - name: upgrade-master | Upgrade coredns for latest Kubernetes (1.17.7)
-      include_tasks: upgrade-coredns.yml
-      when:
-        - version is version('1.17.7', '==')
-
     - name: upgrade-master | Wait for cluster's readiness
       include_tasks: wait.yml
 
@@ -111,6 +109,14 @@
 - name: upgrade-master | Wait for cluster's readiness
   include_tasks: wait.yml
 
+# 'kubeadm upgrade apply' overwrites Epiphany's customized CoreDNS deployment so we restore it.
+# This task should be run each time K8s is upgraded to the latest version.
+# Keep Epiphany's CoreDNS version in synch with K8s CoreDNS version.
+- name: upgrade-master | Deploy customized CoreDNS for latest Kubernetes (1.17.7)
+  include_tasks: upgrade-coredns.yml
+  when:
+    - version is version('1.17.7', '==')
+
 - name: upgrade-master | Upgrade kubeadm-config.yml if exists
   include_tasks: upgrade-kubeadm-config.yml
   when: kubeadm_config_file.stat.exists

From 7693542e4fa06711978784084e158bf42c8329fa Mon Sep 17 00:00:00 2001
From: to-bar <46519524+to-bar@users.noreply.github.com>
Date: Thu, 9 Jul 2020 13:54:57 +0200
Subject: [PATCH 5/7] Update changelog

---
 CHANGELOG-0.7.md | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG-0.7.md b/CHANGELOG-0.7.md
index eb017ec681..f5a70ea173 100644
--- a/CHANGELOG-0.7.md
+++ b/CHANGELOG-0.7.md
@@ -1,5 +1,11 @@
 # Changelog 0.7
 
+## [0.7.1] 2020-07-XX
+
+### Fixed
+
+- [#1399](https://github.com/epiphany-platform/epiphany/issues/1399) - Epicli upgrade: Kubernetes upgrade may hang
+
 ## [0.7.0] 2020-06-30
 
 ### Added
@@ -48,7 +54,6 @@
 - [#1372](https://github.com/epiphany-platform/epiphany/issues/1372) - [BUG] Epicli does not create Postgresql SET\_BY\_AUTOMATION values correctly
 - [#1373](https://github.com/epiphany-platform/epiphany/issues/1373) - [BUG] permission denied for shared directory in the container when no volume was mounted
 - [#1385](https://github.com/epiphany-platform/epiphany/issues/1385) - [BUG] Regression issue with disabling etcd encryption
-- [#1399](https://github.com/epiphany-platform/epiphany/issues/1399) - [BUG] Epicli upgrade issue - the process hangs for several hours on the task kubeadm upgrade apply
 
 ### Known Issues
 

From bcdd6abeedfec034d71c4779e1aaff8710228448 Mon Sep 17 00:00:00 2001
From: to-bar <46519524+to-bar@users.noreply.github.com>
Date: Fri, 10 Jul 2020 14:28:32 +0200
Subject: [PATCH 6/7] Changes after review

---
 .../tasks/kubernetes/patch-kubeadm-config.yml |  4 +-
 .../update-kubeadm-image-repository.yml       | 87 ++++++++++++-------
 .../tasks/kubernetes/upgrade-master.yml       |  5 +-
 3 files changed, 59 insertions(+), 37 deletions(-)

diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubeadm-config.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubeadm-config.yml
index 07bd3bf115..a0301ceb9c 100644
--- a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubeadm-config.yml
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubeadm-config.yml
@@ -5,10 +5,8 @@
 # kube-apiserver uses --encryption-provider-config parameter to control how data is encrypted in etcd.
 # If this parameter is absent the encryption is not enabled.
 - name: upgrade-master | Check if encryption of secret data is enabled
-  shell: >-
+  command: >-
     grep -- '--encryption-provider-config' /etc/kubernetes/manifests/kube-apiserver.yaml
-  args:
-    executable: /bin/bash
   register: shell_grep_encryption_flag
   changed_when: false
   failed_when: shell_grep_encryption_flag.rc > 1
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/update-kubeadm-image-repository.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/update-kubeadm-image-repository.yml
index a2b068a411..83e2e2bc65 100644
--- a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/update-kubeadm-image-repository.yml
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/update-kubeadm-image-repository.yml
@@ -1,34 +1,57 @@
 ---
-# Note: Usage of the --config flag for reconfiguring the cluster during upgrade is not recommended since v1.16
-- name: upgrade-master | Get value of imageRepository from kubeadm-config ConfigMap
-  shell: kubeadm config view
-  changed_when: false
-  register: result
-
-- name: upgrade-master | Set current value of imageRepository as fact
-  set_fact:
-    kubeadm_image_repository: "{{ (result.stdout|from_yaml).imageRepository }}"
-
-- name: upgrade-master | Set new value for imageRepository as fact
-  set_fact:
-    new_kubeadm_image_repository: >-
-      {%- if kubeadm_image_repository is search(':') -%}
-        {{ kubeadm_image_repository | regex_replace('^(?P<host>.+):(?P<port>\d+)', image_registry_address) }}
-      {%- else -%}
-        {{ image_registry_address }}/{{ kubeadm_image_repository }}
-      {%- endif -%}
-
 - name: upgrade-master | Patch imageRepository in kubeadm-config ConfigMap
-  when:
-    - kubeadm_image_repository != new_kubeadm_image_repository
-  environment:
-    KUBECONFIG: /home/{{ admin_user.name }}/.kube/config
-  shell: |-
-    set -o pipefail &&
-    # do not use --export option since it has been deprecated in 1.14
-    kubectl get cm kubeadm-config -n kube-system -o yaml |
-    sed 's|imageRepository: {{ kubeadm_image_repository }}|imageRepository: {{ new_kubeadm_image_repository }}|g' |
-    xargs --null -I config_map_content \
-    kubectl patch cm kubeadm-config -n kube-system --patch config_map_content
-  args:
-    executable: /bin/bash
\ No newline at end of file
+  block:
+    - name: upgrade-master | Get kubeadm-config configmap
+      shell: |
+        kubectl get configmap kubeadm-config \
+          --namespace kube-system \
+          --output yaml
+      environment:
+        KUBECONFIG: &KUBECONFIG /etc/kubernetes/admin.conf
+      register: shell_kubeadm_configmap
+      changed_when: false
+
+    - name: upgrade-master | Patch kubeadm-config configmap (update-kubeadm-image-repository.yml)
+      when:
+        - _image_repository_updated != _image_repository  # skip the task if nothing changed
+      shell: |
+        kubectl patch configmap kubeadm-config \
+          --namespace kube-system \
+          --patch "$KUBEADM_CONFIGMAP_DOCUMENT"
+      environment:
+        KUBECONFIG: *KUBECONFIG
+        # Render an altered kubeadm-config configmap document
+        KUBEADM_CONFIGMAP_DOCUMENT: >-
+          {{ _document | combine(_update2, recursive=true) | to_nice_yaml(indent=2) }}
+
+      vars:
+        # Parse yaml payload
+        _document: >-
+          {{ shell_kubeadm_configmap.stdout | from_yaml }}
+
+        # Extract cluster config
+        _cluster_config: >-
+          {{ _document.data.ClusterConfiguration | from_yaml }}
+
+        _image_repository: >-
+          {{ _cluster_config.imageRepository }}
+
+        _image_repository_updated: >-
+          {%- if _image_repository is search(':') -%}
+            {{ _image_repository | regex_replace('^(?P<host>.+):(?P<port>\d+)', image_registry_address) }}
+          {%- else -%}
+            {{ image_registry_address }}/{{ _image_repository }}
+          {%- endif -%}
+
+        # Prepare the cluster config patch
+        _update1:
+          imageRepository: "{{ _image_repository_updated }}"
+
+        _cluster_config_updated: >-
+          {{ _cluster_config | combine(_update1, recursive=true) }}
+
+        # Prepare the final update for the whole document
+        _update2:
+          data:
+            ClusterConfiguration: >-
+              {{ _cluster_config_updated | to_nice_yaml(indent=2) }}
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-master.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-master.yml
index c376a76019..b705e1a3d1 100644
--- a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-master.yml
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-master.yml
@@ -64,12 +64,13 @@
     - name: upgrade-master | Wait for cluster's readiness
       include_tasks: wait.yml
 
+    # Note: Usage of the --config flag for reconfiguring the cluster during upgrade is not recommended since v1.16
     - name: upgrade-master | Validate whether cluster is upgradeable
       # Ignore CoreDNSUnsupportedPlugins error since coredns-migration does not support 'hosts' plugin.
       # This issue is fixed in K8s v1.18, see https://github.com/kubernetes/kubernetes/pull/88482
       shell: >-
         {%- if version is version('1.18.0', '>=') -%}
-           kubeadm upgrade plan v{{ version }}
+          kubeadm upgrade plan v{{ version }}
         {%- else -%}
           kubeadm upgrade plan v{{ version }} --ignore-preflight-errors=CoreDNSUnsupportedPlugins
         {%- endif -%}
@@ -85,7 +86,7 @@
       # This issue is fixed in K8s v1.18, see https://github.com/kubernetes/kubernetes/pull/88482
       shell: >-
         {%- if version is version('1.18.0', '>=') -%}
-           kubeadm upgrade apply -y v{{ version }}
+          kubeadm upgrade apply -y v{{ version }}
         {%- else -%}
           kubeadm upgrade apply -y v{{ version }} --ignore-preflight-errors=CoreDNSUnsupportedPlugins
         {%- endif -%}

From b77e4c7d3b7ed50503a41dee7e5184bfc2fe3de9 Mon Sep 17 00:00:00 2001
From: to-bar <46519524+to-bar@users.noreply.github.com>
Date: Fri, 10 Jul 2020 16:28:38 +0200
Subject: [PATCH 7/7] Wait for API resources to propagate

---
 .../tasks/kubernetes/upgrade-network-components.yml   | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-network-components.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-network-components.yml
index c9dfaece68..7b5b38c4da 100644
--- a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-network-components.yml
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-network-components.yml
@@ -4,6 +4,17 @@
   set_fact:
     plugin: "flannel"
 
+- name: upgrade-master | Wait for API resources to propagate
+  shell: |
+    kubectl api-resources --cached=false && kubectl -n kube-system get daemonsets
+  environment:
+    KUBECONFIG: /home/{{ admin_user.name }}/.kube/config
+  changed_when: false
+  register: daemonsets_query_result
+  until: daemonsets_query_result is success
+  retries: 20
+  delay: 30
+
 - name: upgrade-master | If canal is installed on the cluster
   environment:
     KUBECONFIG: /home/{{ admin_user.name }}/.kube/config