Fix K8s upgrade: 'kubeadm upgrade apply' hangs

CHANGELOG-0.7.md

@@ -1,5 +1,11 @@
 # Changelog 0.7
 
+## [0.7.1] 2020-07-XX
+
+### Fixed
+
+- [#1399](https://github.com/epiphany-platform/epiphany/issues/1399) - Epicli upgrade: Kubernetes upgrade may hang
+
 ## [0.7.0] 2020-06-30
 
 ### Added
@@ -48,7 +54,6 @@
 - [#1372](https://github.com/epiphany-platform/epiphany/issues/1372) - [BUG] Epicli does not create Postgresql SET\_BY\_AUTOMATION values correctly
 - [#1373](https://github.com/epiphany-platform/epiphany/issues/1373) - [BUG] permission denied for shared directory in the container when no volume was mounted
 - [#1385](https://github.com/epiphany-platform/epiphany/issues/1385) - [BUG] Regression issue with disabling etcd encryption
-- [#1399](https://github.com/epiphany-platform/epiphany/issues/1399) - [BUG] Epicli upgrade issue - the process hangs for several hours on the task kubeadm upgrade apply
 
 ### Known Issues
 

core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/deployments/deploy-template.yml

@@ -6,7 +6,7 @@
     state: directory
     owner: root
     group: root
-    mode: u=rw,go=r
+    mode: u=rwx,go=r
 
 - name: "Upload {{ file_name }} file"
   become: true

core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/downgrade-coredns.yml

@@ -0,0 +1,43 @@
+---
+# CoreDNS version matrix
+#+-----------------------------------------------------------------+
+#| Epiphany version | K8s version | Epiphany CoreDNS | K8s CoreDNS |
+#|------------------|-------------|------------------|-------------|
+#| 0.4.4            | 1.14.6      | 1.5.0            | 1.3.1       |
+#| 0.5.0            | 1.14.6      | 1.5.0            | 1.3.1       |
+#| upgrade < 0.6.0  | 1.15.10     | 1.5.0            | 1.3.1       |
+#| upgrade < 0.6.0  | 1.16.7      | 1.5.0            | 1.6.2       |
+#| 0.6.0            | 1.17.4      | 1.6.5            | 1.6.5       |
+#| 0.7.0            | 1.17.7      | 1.6.5            | 1.6.5       |
+#+-----------------------------------------------------------------+
+# Source: look for 'CoreDNSVersion' at https://github.com/kubernetes/kubernetes/blob/$TAG/cmd/kubeadm/app/constants/constants.go
+
+- name: upgrade-master | Create directory /etc/epiphany/manifests
+  become: true
+  file:
+    path: /etc/epiphany/manifests
+    state: directory
+    owner: root
+    group: root
+    mode: u=rwx,go=r
+
+- name: Upload and apply template
+  block:
+    - name: upgrade-master | Upload {{ file_name }} file
+      template:
+        src: kubernetes/{{ file_name }}.j2
+        dest: /etc/epiphany/manifests/{{ file_name }}
+        owner: "{{ admin_user.name }}"
+        group: "{{ admin_user.name }}"
+        mode: u=rw,go=r
+
+    - name: upgrade-master | Apply /etc/epiphany/manifests/{{ file_name }} file
+      environment:
+        KUBECONFIG: /home/{{ admin_user.name }}/.kube/config
+      shell: |
+        kubectl apply \
+          -f /etc/epiphany/manifests/{{ file_name }}
+      args:
+        executable: /bin/bash
+  vars:
+    file_name: coredns-config-for-k8s-below-1.16.yml

core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubeadm-config.yml

@@ -1,82 +1,24 @@
 ---
-- name: Check the etc-encryption.conf file
-  stat:
-    path: &etc-encryption-conf /etc/kubernetes/pki/etcd/etc-encryption.conf
-    get_attributes: false
-    get_checksum: false
-    get_mime: false
-  register: stat_etcd_encryption_config_file
+# Since usage of the --config flag for reconfiguring the cluster during upgrade is not recommended
+# (warning added in v1.17), we patch kubeadm-config ConfigMap directly.
 
-# Assuming that if the etcd encryption config file is absent, then
-# the encryption feature has never been enabled for the cluster at hand.
-- when:
-    - stat_etcd_encryption_config_file.stat.exists
-  block:
-    - name: Check the kubeadm-config.yml file
-      stat:
-        path: &kubeadm-config-yml /etc/kubeadm/kubeadm-config.yml
-        get_attributes: false
-        get_checksum: false
-        get_mime: false
-      register: stat_kubeadm_config_file
-
-- when:
-    - stat_etcd_encryption_config_file.stat.exists
-    - stat_kubeadm_config_file.stat.exists
-  block:
-    - name: Load contents of the kubeadm-config.yml file
-      slurp:
-        path: *kubeadm-config-yml
-      register: slurp_kubeadm_config
-
-    - name: Save modified contents of the kubeadm-config.yml file
-      copy:
-        dest: *kubeadm-config-yml
-
-        # Save all documents.
-        content: |
-          {% for document in _documents_updated %}
-          ---
-          {{ document | to_nice_yaml(indent=2) }}
-          {% endfor -%}
-
-      vars:
-        # Parse yaml payload (remove empty documents).
-        _documents: >-
-          {{ slurp_kubeadm_config.content | b64decode
-                                          | from_yaml_all
-                                          | select
-                                          | list }}
-        # Prepare the patch.
-        # In this patch we include location of the etcd encryption config file.
-        # If it is not included, then the etcd encryption feature becomes disabled/broken.
-        # If it is not present on a cluster that has kube-system secrets already encrypted, then
-        # it may cause any upgrade attempt to freeze for a very long time (in Epiphany it has been reported to be even up to 8 hours).
-        _update:
-          apiServer:
-            extraArgs:
-              encryption-provider-config: *etc-encryption-conf
+# kube-apiserver uses --encryption-provider-config parameter to control how data is encrypted in etcd.
+# If this parameter is absent the encryption is not enabled.
+- name: upgrade-master | Check if encryption of secret data is enabled
+  shell: >-
+    grep -- '--encryption-provider-config' /etc/kubernetes/manifests/kube-apiserver.yaml
+  args:
+    executable: /bin/bash
+  register: shell_grep_encryption_flag
+  changed_when: false
+  failed_when: shell_grep_encryption_flag.rc > 1
 
-        # Process all documents (returns a list of dictionaries).
-        _documents_updated: >-
-          {%- set output = [] -%}
-          {%- for document in _documents -%}
-            {%- if document.kind is defined and document.kind == 'ClusterConfiguration' -%}
-              {{- output.append(document | combine(_update, recursive=true)) -}}
-            {%- else -%}
-              {{- output.append(document) -}}
-            {%- endif -%}
-          {%- endfor -%}
-          {{- output -}}
-
-# The `kubeadm upgrade` command can be executed with or without a config file.
-# If the kubeadm-config.yml file does not exists, then we at least patch the kubeadm-config configmap.
-- when:
-    - stat_etcd_encryption_config_file.stat.exists
-    - not stat_kubeadm_config_file.stat.exists
+- name: upgrade-master | Patch kubeadm-config ConfigMap if needed
+  when:
+    - shell_grep_encryption_flag.rc == 0 # encryption enabled
   run_once: true  # makes no sense to execute it more than once (would be redundant)
   block:
-    - name: Load the kubeadm-config configmap
+    - name: Get kubeadm-config ConfigMap
       shell: |
         kubectl get configmap kubeadm-config \
           --namespace kube-system \
@@ -88,9 +30,10 @@
       register: shell_kubeadm_configmap
       changed_when: false
 
-    # The following procedure ensures that etcd encryption is always enabled
-    # during subsequent kubeadm executions (if the config file is not present).
-    - name: Patch and re-apply the kubeadm-config configmap
+    # The following procedure ensures that etcd encryption is always enabled during subsequent kubeadm executions
+    - name: upgrade-master | Patch and re-apply the kubeadm-config ConfigMap
+      when:
+        - _kubeadm_api_server_extra_args['encryption-provider-config'] is undefined
       shell: |
         kubectl apply \
           --namespace kube-system \
@@ -100,32 +43,32 @@
         executable: /bin/bash
       environment:
         KUBECONFIG: *KUBECONFIG
-        # Render an altered kubeadm-config configmap document.
+        # Render an altered kubeadm-config configmap document
         KUBEADM_CONFIGMAP_DOCUMENT: >-
           {{ _document | combine(_update2, recursive=true) | to_nice_yaml(indent=2) }}
 
-      # Skip the task if there is no change in the cluster config.
-      when: _cluster_config_updated != _cluster_config  # comparing two dictionaries here
-
       vars:
-        # Parse yaml payload.
+        # Parse yaml payload
         _document: >-
           {{ shell_kubeadm_configmap.stdout | from_yaml }}
 
-        # Extract cluster config.
+        # Extract cluster config
         _cluster_config: >-
           {{ _document.data.ClusterConfiguration | from_yaml }}
 
-        # Prepare the cluster config patch.
+        _kubeadm_api_server_extra_args: >-
+          {{ _cluster_config.apiServer.extraArgs }}
+
+        # Prepare the cluster config patch
         _update1:
           apiServer:
             extraArgs:
-              encryption-provider-config: *etc-encryption-conf
+              encryption-provider-config: /etc/kubernetes/pki/etcd/etc-encryption.conf
 
         _cluster_config_updated: >-
           {{ _cluster_config | combine(_update1, recursive=true) }}
 
-        # Prepare the final update for the whole document.
+        # Prepare the final update for the whole document
         _update2:
           data:
             ClusterConfiguration: >-

core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/update-kubeadm-image-repository.yml

@@ -0,0 +1,34 @@
+---
+# Note: Usage of the --config flag for reconfiguring the cluster during upgrade is not recommended since v1.16
+- name: upgrade-master | Get value of imageRepository from kubeadm-config ConfigMap
+  shell: kubeadm config view
+  changed_when: false
+  register: result
+
+- name: upgrade-master | Set current value of imageRepository as fact
+  set_fact:
+    kubeadm_image_repository: "{{ (result.stdout|from_yaml).imageRepository }}"
+
+- name: upgrade-master | Set new value for imageRepository as fact
+  set_fact:
+    new_kubeadm_image_repository: >-
+      {%- if kubeadm_image_repository is search(':') -%}
+        {{ kubeadm_image_repository | regex_replace('^(?P<host>.+):(?P<port>\d+)', image_registry_address) }}
+      {%- else -%}
+        {{ image_registry_address }}/{{ kubeadm_image_repository }}
+      {%- endif -%}
+
+- name: upgrade-master | Patch imageRepository in kubeadm-config ConfigMap
+  when:
+    - kubeadm_image_repository != new_kubeadm_image_repository
+  environment:
+    KUBECONFIG: /home/{{ admin_user.name }}/.kube/config
+  shell: |-
+    set -o pipefail &&
+    # do not use --export option since it has been deprecated in 1.14
+    kubectl get cm kubeadm-config -n kube-system -o yaml |
+    sed 's|imageRepository: {{ kubeadm_image_repository }}|imageRepository: {{ new_kubeadm_image_repository }}|g' |
+    xargs --null -I config_map_content \
+    kubectl patch cm kubeadm-config -n kube-system --patch config_map_content
+  args:
+    executable: /bin/bash