From 4399db86dceea51a3bb324bc291adf6bf656a84f Mon Sep 17 00:00:00 2001
From: atsikham <atsikham@users.noreply.github.com>
Date: Mon, 24 Jan 2022 09:22:56 +0100
Subject: [PATCH 01/10] Remove force join attempt for k8s node

---
 .../playbooks/roles/kubernetes_node/tasks/node-join.yml   | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/ansible/playbooks/roles/kubernetes_node/tasks/node-join.yml b/ansible/playbooks/roles/kubernetes_node/tasks/node-join.yml
index d280300cb0..8d0b4e19c8 100644
--- a/ansible/playbooks/roles/kubernetes_node/tasks/node-join.yml
+++ b/ansible/playbooks/roles/kubernetes_node/tasks/node-join.yml
@@ -46,14 +46,6 @@
 
                 - <<: *soft-join
 
-              rescue:
-                - name: Join to cluster with ignores
-                  command: |-
-                    kubeadm join \
-                      --config /etc/kubeadm/kubeadm-join-node.yml \
-                      --ignore-preflight-errors all
-                  register: kubeadm_join_result
-
           always:
             - name: Display kubeadm join stderr if any
               debug:

From 52f2677eb8fc84de04ed185275dc377d5b0cc0c0 Mon Sep 17 00:00:00 2001
From: atsikham <atsikham@users.noreply.github.com>
Date: Mon, 24 Jan 2022 15:18:02 +0100
Subject: [PATCH 02/10] Re-generate certificates only when required

---
 .../tasks/apiserver-certificates.yml          | 38 +++++---
 .../tasks/generate-certificates.yml           | 61 +++++++------
 .../roles/kubernetes_master/tasks/main.yml    | 88 ++++++++++---------
 .../kubernetes_master/tasks/master-init.yml   |  4 -
 .../templates/kubeadm-config.yml.j2           |  8 +-
 .../kubernetes/patch-kubeadm-apiserver.yml    | 21 +++--
 6 files changed, 124 insertions(+), 96 deletions(-)

diff --git a/ansible/playbooks/roles/kubernetes_master/tasks/apiserver-certificates.yml b/ansible/playbooks/roles/kubernetes_master/tasks/apiserver-certificates.yml
index d118b8aa93..6fe85e06b4 100644
--- a/ansible/playbooks/roles/kubernetes_master/tasks/apiserver-certificates.yml
+++ b/ansible/playbooks/roles/kubernetes_master/tasks/apiserver-certificates.yml
@@ -8,18 +8,28 @@
     - "{{ pki.location }}/apiserver.crt"
     - "{{ pki.location }}/apiserver.key"
 
-- name: Delete apiserver.{crt,key}
-  file:
-    path: "{{ item }}"
-    state: absent
-  loop:
-    - "{{ pki.location }}/apiserver.crt"
-    - "{{ pki.location }}/apiserver.key"
+- name: Generate new apiserver certificates with kubeadm
+  block:
+    - name: Delete apiserver.{crt,key}
+      file:
+        path: "{{ item }}"
+        state: absent
+      loop:
+        - "{{ pki.location }}/apiserver.crt"
+        - "{{ pki.location }}/apiserver.key"
 
-- name: Render new certificates apiserver.{crt,key}
-  shell: |
-    kubeadm init phase certs apiserver \
-      --config /etc/kubeadm/kubeadm-config.yml
-  args:
-    executable: /bin/bash
-    creates: "{{ pki.location }}/apiserver.key"
+    - name: Render new certificates apiserver.{crt,key}
+      command: |-
+        kubeadm init phase certs apiserver \
+          --config /etc/kubeadm/kubeadm-config.yml
+      args:
+        creates: "{{ pki.location }}/apiserver.key"
+  rescue:
+    - name: Restore apiserver.{crt,key}
+      copy:
+        dest: "{{ item }}"
+        src: "{{ item }}.OLD"
+        remote_src: true
+      loop:
+        - "{{ pki.location }}/apiserver.crt"
+        - "{{ pki.location }}/apiserver.key"
diff --git a/ansible/playbooks/roles/kubernetes_master/tasks/generate-certificates.yml b/ansible/playbooks/roles/kubernetes_master/tasks/generate-certificates.yml
index c055831864..edf83a1d0a 100644
--- a/ansible/playbooks/roles/kubernetes_master/tasks/generate-certificates.yml
+++ b/ansible/playbooks/roles/kubernetes_master/tasks/generate-certificates.yml
@@ -1,5 +1,15 @@
 ---
-- name: Generate certificates block
+- name: Set pki_backup_dir fact
+  set_fact:
+    pki_backup_dir: "{{ pki.location | regex_replace('\\/$', '') }}-backup-{{ ansible_date_time.iso8601_basic_short }}"
+
+- name: Save current certificates
+  synchronize:
+    src: "{{ pki.location }}/"
+    dest: "{{ pki_backup_dir }}"
+  delegate_to: "{{ inventory_hostname }}"
+
+- name: Generate certificates
   vars:
     # https://kubernetes.io/docs/setup/best-practices/certificates/#all-certificates
     _certificates_opt_mapping:
@@ -62,13 +72,6 @@
             certificates_renewal_list: "{{ _certificates_opt_mapping | map(attribute='name') }}"
           when: certificates_renewal_list is not defined
 
-    - name: Save old certificates
-      synchronize:
-        src: "{{ pki.location }}/"
-        dest: >-
-          {{ pki.location | regex_replace('\\/$', '') }}-backup-{{ ansible_date_time.iso8601_basic_short }}
-      delegate_to: "{{ inventory_hostname }}"
-
     - name: Ensure necessary directories exist
       file:
         path: "{{ item }}"
@@ -195,20 +198,28 @@
         - 'csr'
         - 'ext'
 
-    - name: Restart systemd services
-      when:
-        - services_to_restart is defined
-        - services_to_restart | difference(['docker', 'kubelet']) | length == 0
-      block:
-        - name: Restart services
-          systemd:
-            name: "{{ item }}"
-            state: restarted
-          loop: "{{ services_to_restart }}"
-
-        - name: Wait until cluster is available
-          command: kubectl cluster-info
-          retries: 50
-          delay: 1
-          register: result
-          until: result is succeeded and "running" in result.stdout
+  rescue:
+    - name: Restore certificates
+      synchronize:
+        src: "{{ pki_backup_dir }}/"
+        dest: "{{ pki.location | regex_replace('\\/$', '') }}"
+      delegate_to: "{{ inventory_hostname }}"
+
+
+- name: Restart systemd services
+  when:
+    - services_to_restart is defined
+    - services_to_restart | difference(['docker', 'kubelet']) | length == 0
+  block:
+    - name: Restart services
+      systemd:
+        name: "{{ item }}"
+        state: restarted
+      loop: "{{ services_to_restart }}"
+
+    - name: Wait until cluster is available
+      command: kubectl cluster-info
+      retries: 60
+      delay: 1
+      register: result
+      until: result is succeeded and "running" in result.stdout
diff --git a/ansible/playbooks/roles/kubernetes_master/tasks/main.yml b/ansible/playbooks/roles/kubernetes_master/tasks/main.yml
index 746c035673..ce8a59b184 100644
--- a/ansible/playbooks/roles/kubernetes_master/tasks/main.yml
+++ b/ansible/playbooks/roles/kubernetes_master/tasks/main.yml
@@ -1,8 +1,4 @@
 ---
-- name: Set is_first_deployment fact
-  set_fact:
-    is_first_deployment: false
-
 - when: use_ha_control_plane
   block:
     - name: Configure internal load-balancer (HAProxy)
@@ -28,21 +24,33 @@
         - import_tasks: copy-kubernetes-pki.yml
         - import_tasks: master-join.yml
 
-- name: Collect current apiserver certificate 'not_after' date by openssl
-  command: openssl x509 -enddate -noout -in apiserver.crt
+- name: Collect subject alternative names of apiserver certificate
+  shell: |-
+    set -o pipefail && \
+    openssl x509 -in /etc/kubernetes/pki/apiserver.crt -text -noout \
+      | grep DNS:
   args:
-    chdir: "{{ pki.location }}"
-  register: apiserver_certificate_info
+    executable: /bin/bash
   changed_when: false
+  register: apiserver_cert
+
+- name: Check presence of each control plane address in SANs list
+  set_fact:
+    san_search_results: "{{ san_search_results | default([]) + [apiserver_cert.stdout is search(item)] }}"
+  loop: >-
+    {{ (groups['kubernetes_master'] | map('extract', hostvars, ['ansible_default_ipv4', 'address']) | list)
+      + (groups['kubernetes_master'] | map('extract', hostvars, ['ansible_host']) | list) }}
 
 - name: Regenerate apiserver certificates
-  when: kubernetes_common.automation_designated_master != inventory_hostname or not is_first_deployment
-# It's almost always necessary to regenerate apiserver certificates for designated and non-designated masters
-# because of a few points:
-# a. Update certificates for old clusters have to be supported
-# b. Execution order is not defined, so when cluster is promoted to HA,
-#    non-designated masters may join cluster before designated master's certificate update
+  when: "not (san_search_results is all)"
   block:
+    - name: Collect current apiserver certificate 'not_after' date by openssl
+      command: openssl x509 -enddate -noout -in apiserver.crt
+      args:
+        chdir: "{{ pki.location }}"
+      register: apiserver_certificate_info
+      changed_when: false
+
     - name: Extend kubeadm config
       vars:
         update:
@@ -50,41 +58,39 @@
             certSANs: >-
               {{ (groups['kubernetes_master'] | map('extract', hostvars, ['ansible_default_ipv4', 'address']) | list)
               + (groups['kubernetes_master'] | map('extract', hostvars, ['ansible_host']) | list)
-              + [ 'localhost', '127.0.0.1' ] | unique }}
+              + [ '127.0.0.1' ] | unique }}
       include_role:
         name: kubernetes_common
         tasks_from: extend-kubeadm-config
 
-    - name: Backup and generate apiserver certificates with latest kubeadm config
+    - name: Generate apiserver certificates with latest kubeadm config
       include_tasks: apiserver-certificates.yml
 
-# kubeadm certs renewal uses the existing certificates as the authoritative source for attributes (Common Name, Organization, SAN, etc.)
-# instead of the kubeadm-config ConfigMap, so it's not possible to combine this step with previous ones
-# See https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/kubeadm-certs/#manual-certificate-renewal
-- name: Update apiserver certificate expiration date
-  when: not (specification.advanced.certificates.renew | bool)
-  block:
-    - name: Regenerate apiserver certificate with previous expiration value
-      vars:
-        certificates_renewal_list:
-          - apiserver
-        valid_days: "{{ apiserver_certificate_info.stdout | openssl_date2days }}"
-      include_tasks: generate-certificates.yml
+    - name: Update apiserver certificate expiration date
+      when: not (specification.advanced.certificates.renew | bool)
+      block:
+        - name: Regenerate apiserver certificate with previous expiration value
+          vars:
+            certificates_renewal_list:
+              - apiserver
+            valid_days: "{{ apiserver_certificate_info.stdout | openssl_date2days }}"
+          include_tasks: generate-certificates.yml
 
-    - name: Restart apiserver
-      shell: |
-        docker ps \
-          --filter 'name=kube-apiserver_kube-apiserver' \
-          --format '{{ "{{.ID}}" }}' \
-        | xargs --no-run-if-empty docker kill
-      args:
-        executable: /bin/bash
+        - name: Restart apiserver
+          shell: |-
+            set -o pipefail && \
+            docker ps \
+              --filter 'name=kube-apiserver_kube-apiserver' \
+              --format '{{ "{{.ID}}" }}' \
+            | xargs --no-run-if-empty docker kill
+          args:
+            executable: /bin/bash
 
-- name: Update in-cluster configuration
-  when: kubernetes_common.automation_designated_master == inventory_hostname
-  include_role:
-    name: kubernetes_common
-    tasks_from: update-in-cluster-config
+    - name: Update in-cluster configuration
+      when: kubernetes_common.automation_designated_master == inventory_hostname
+      include_role:
+        name: kubernetes_common
+        tasks_from: update-in-cluster-config
 
 - name: Regenerate all certificates
   when: specification.advanced.certificates.renew | bool
diff --git a/ansible/playbooks/roles/kubernetes_master/tasks/master-init.yml b/ansible/playbooks/roles/kubernetes_master/tasks/master-init.yml
index 108456d258..fa77f08a73 100644
--- a/ansible/playbooks/roles/kubernetes_master/tasks/master-init.yml
+++ b/ansible/playbooks/roles/kubernetes_master/tasks/master-init.yml
@@ -11,10 +11,6 @@
 
 - when: not stat_kube_apiserver_yaml.stat.exists
   block:
-    - name: Set is_first_deployment fact
-      set_fact:
-        is_first_deployment: true
-
     - name: Ensure /etc/kubeadm/ directory exists
       file:
         path: /etc/kubeadm/
diff --git a/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2 b/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2
index dd3ecec4d2..fbe2c6df55 100644
--- a/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2
+++ b/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2
@@ -10,12 +10,12 @@ controlPlaneEndpoint: "localhost:3446"
 apiServer:
   timeoutForControlPlane: 4m0s
   certSANs:
-{% set ip_address_list = ['127.0.0.1', 'localhost'] %}
+{% set address_list = ['127.0.0.1'] %}
 {% for host in groups['kubernetes_master'] %}
-{%   set _ = ip_address_list.extend([ hostvars[host]['ansible_default_ipv4']['address'], hostvars[host]['ansible_host'] ]) %}
+{%   set _ = address_list.extend([ hostvars[host]['ansible_default_ipv4']['address'], hostvars[host]['ansible_host'] ]) %}
 {% endfor %}
-{% for ip in ip_address_list|unique %}
-    - {{ ip }}
+{% for address in address_list|unique %}
+    - {{ address }}
 {% endfor %}
   extraArgs: # https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/
 {% if specification.advanced.etcd_args.encrypted | bool %}
diff --git a/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubeadm-apiserver.yml b/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubeadm-apiserver.yml
index 804474e2ea..2c460d6593 100644
--- a/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubeadm-apiserver.yml
+++ b/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubeadm-apiserver.yml
@@ -31,15 +31,12 @@
       register: command_kubeadm_configmap
       changed_when: false
 
-    # The following procedure ensures that etcd encryption is always enabled during subsequent kubeadm executions
-    # Starting from K8s v1.21 support for DenyEscalatingExec admission plugin is removed,
-    # but it was specified in Epiphany's default values
+    # The following procedure ensures that
+    #   - etcd encryption is always enabled during subsequent kubeadm executions
+    #   - DenyEscalatingExec admission plugin is not used as unsupported (it was specified in Epiphany's default values before v1.3)
+    #   - apiserver audit logging is enabled
+    #   - apiserver certSANs are configured the same way as for apply (GitHub issue #1520)
     - name: k8s/master | Patch kubeadm-config configmap (patch-kubeadm-apiserver.yml)
-      when: _kubeadm_api_server_extra_args['encryption-provider-config'] is undefined
-         or _cluster_config.apiServer['extraVolumes'] is undefined
-         or (version is version('1.21', '>=')
-             and version is version('1.22', '<')
-             and _kubeadm_api_server_extra_args['enable-admission-plugins'] is defined)
       command: |
         kubectl patch configmap kubeadm-config \
           --namespace kube-system \
@@ -68,6 +65,14 @@
         # Prepare the cluster config patch
         _update1: |-
           apiServer:
+            certSANs:
+          {% set address_list = ['127.0.0.1'] %}
+          {% for host in groups['kubernetes_master'] %}
+          {%   set _ = address_list.extend([ hostvars[host]['ansible_default_ipv4']['address'], hostvars[host]['ansible_host'] ]) %}
+          {% endfor %}
+          {% for address in address_list|unique %}
+              - {{ address }}
+          {% endfor %}
             extraArgs:
               encryption-provider-config: {{ kubernetes_master_defaults.pki.location }}/etcd/etc-encryption.conf
           {% if version is version('1.21', '>=') and version is version('1.22', '<') %}

From 2577e1779d91b9a8f59ebba1562b14dc0c889e5d Mon Sep 17 00:00:00 2001
From: atsikham <atsikham@users.noreply.github.com>
Date: Tue, 25 Jan 2022 15:16:50 +0100
Subject: [PATCH 03/10] Update changelog

---
 docs/changelogs/CHANGELOG-2.0.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/changelogs/CHANGELOG-2.0.md b/docs/changelogs/CHANGELOG-2.0.md
index 5b559ca2da..35deb2023a 100644
--- a/docs/changelogs/CHANGELOG-2.0.md
+++ b/docs/changelogs/CHANGELOG-2.0.md
@@ -6,8 +6,14 @@
 
 ### Fixed
 
+- [#2669](https://github.com/epiphany-platform/epiphany/issues/2669) - Restarting the installation process can cause certificate problems if K8s was not fully configured
+
 ### Updated
 
+- [#2828](https://github.com/epiphany-platform/epiphany/issues/2828) - K8s improvements
+  - Re-generate apiserver certificates only by purpose
+  - Do not ignore preflight errors in `kubeadm join`
+
 ### Removed
 
 - [#2834](https://github.com/epiphany-platform/epiphany/issues/2834) - Removal of Hashicorp Vault component

From c0864a57718249a2357a77b73ab105581bd65447 Mon Sep 17 00:00:00 2001
From: Anatoli Tsikhamirau <anatoliytihomirov@yahoo.com>
Date: Thu, 27 Jan 2022 12:32:08 +0100
Subject: [PATCH 04/10] Apply suggestions from code review

Co-authored-by: to-bar <46519524+to-bar@users.noreply.github.com>
---
 .../roles/kubernetes_master/tasks/generate-certificates.yml     | 2 +-
 .../roles/upgrade/tasks/kubernetes/patch-kubeadm-apiserver.yml  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/ansible/playbooks/roles/kubernetes_master/tasks/generate-certificates.yml b/ansible/playbooks/roles/kubernetes_master/tasks/generate-certificates.yml
index edf83a1d0a..efbcc097fd 100644
--- a/ansible/playbooks/roles/kubernetes_master/tasks/generate-certificates.yml
+++ b/ansible/playbooks/roles/kubernetes_master/tasks/generate-certificates.yml
@@ -3,7 +3,7 @@
   set_fact:
     pki_backup_dir: "{{ pki.location | regex_replace('\\/$', '') }}-backup-{{ ansible_date_time.iso8601_basic_short }}"
 
-- name: Save current certificates
+- name: Back up current certificates
   synchronize:
     src: "{{ pki.location }}/"
     dest: "{{ pki_backup_dir }}"
diff --git a/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubeadm-apiserver.yml b/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubeadm-apiserver.yml
index 2c460d6593..a609e7fd96 100644
--- a/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubeadm-apiserver.yml
+++ b/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubeadm-apiserver.yml
@@ -31,7 +31,7 @@
       register: command_kubeadm_configmap
       changed_when: false
 
-    # The following procedure ensures that
+    # The following procedure ensures that:
     #   - etcd encryption is always enabled during subsequent kubeadm executions
     #   - DenyEscalatingExec admission plugin is not used as unsupported (it was specified in Epiphany's default values before v1.3)
     #   - apiserver audit logging is enabled

From f899d5c0af3947ca9b3a839f4ac0e0cd99f198f7 Mon Sep 17 00:00:00 2001
From: atsikham <atsikham@users.noreply.github.com>
Date: Fri, 28 Jan 2022 01:35:30 +0100
Subject: [PATCH 05/10] Apply fixes after review

---
 .../tasks/apiserver-certificates.yml          |  5 +++
 .../tasks/generate-certificates.yml           |  4 ++
 .../roles/kubernetes_master/tasks/main.yml    | 10 ++---
 .../roles/kubernetes_node/tasks/node-join.yml | 37 ++++++++-----------
 4 files changed, 30 insertions(+), 26 deletions(-)

diff --git a/ansible/playbooks/roles/kubernetes_master/tasks/apiserver-certificates.yml b/ansible/playbooks/roles/kubernetes_master/tasks/apiserver-certificates.yml
index 6fe85e06b4..0e984e15b4 100644
--- a/ansible/playbooks/roles/kubernetes_master/tasks/apiserver-certificates.yml
+++ b/ansible/playbooks/roles/kubernetes_master/tasks/apiserver-certificates.yml
@@ -24,6 +24,7 @@
           --config /etc/kubeadm/kubeadm-config.yml
       args:
         creates: "{{ pki.location }}/apiserver.key"
+
   rescue:
     - name: Restore apiserver.{crt,key}
       copy:
@@ -33,3 +34,7 @@
       loop:
         - "{{ pki.location }}/apiserver.crt"
         - "{{ pki.location }}/apiserver.key"
+
+    - name: Fail certificates generation
+      fail:
+        msg: Apiserver certificates generation failed, restored an initial state
diff --git a/ansible/playbooks/roles/kubernetes_master/tasks/generate-certificates.yml b/ansible/playbooks/roles/kubernetes_master/tasks/generate-certificates.yml
index efbcc097fd..241e34cd26 100644
--- a/ansible/playbooks/roles/kubernetes_master/tasks/generate-certificates.yml
+++ b/ansible/playbooks/roles/kubernetes_master/tasks/generate-certificates.yml
@@ -205,6 +205,10 @@
         dest: "{{ pki.location | regex_replace('\\/$', '') }}"
       delegate_to: "{{ inventory_hostname }}"
 
+    - name: Fail certificates generation
+      fail:
+        msg: Certificates generation failed, restored an initial state
+
 
 - name: Restart systemd services
   when:
diff --git a/ansible/playbooks/roles/kubernetes_master/tasks/main.yml b/ansible/playbooks/roles/kubernetes_master/tasks/main.yml
index ce8a59b184..1c2981268a 100644
--- a/ansible/playbooks/roles/kubernetes_master/tasks/main.yml
+++ b/ansible/playbooks/roles/kubernetes_master/tasks/main.yml
@@ -32,11 +32,11 @@
   args:
     executable: /bin/bash
   changed_when: false
-  register: apiserver_cert
+  register: apiserver_certificate_san
 
 - name: Check presence of each control plane address in SANs list
   set_fact:
-    san_search_results: "{{ san_search_results | default([]) + [apiserver_cert.stdout is search(item)] }}"
+    san_search_results: "{{ san_search_results | default([]) + [apiserver_certificate_san.stdout is search(item)] }}"
   loop: >-
     {{ (groups['kubernetes_master'] | map('extract', hostvars, ['ansible_default_ipv4', 'address']) | list)
       + (groups['kubernetes_master'] | map('extract', hostvars, ['ansible_host']) | list) }}
@@ -48,7 +48,7 @@
       command: openssl x509 -enddate -noout -in apiserver.crt
       args:
         chdir: "{{ pki.location }}"
-      register: apiserver_certificate_info
+      register: apiserver_certificate_enddate
       changed_when: false
 
     - name: Extend kubeadm config
@@ -58,7 +58,7 @@
             certSANs: >-
               {{ (groups['kubernetes_master'] | map('extract', hostvars, ['ansible_default_ipv4', 'address']) | list)
               + (groups['kubernetes_master'] | map('extract', hostvars, ['ansible_host']) | list)
-              + [ '127.0.0.1' ] | unique }}
+              + [ '127.0.0.1' ] }}
       include_role:
         name: kubernetes_common
         tasks_from: extend-kubeadm-config
@@ -73,7 +73,7 @@
           vars:
             certificates_renewal_list:
               - apiserver
-            valid_days: "{{ apiserver_certificate_info.stdout | openssl_date2days }}"
+            valid_days: "{{ apiserver_certificate_enddate.stdout | openssl_date2days }}"
           include_tasks: generate-certificates.yml
 
         - name: Restart apiserver
diff --git a/ansible/playbooks/roles/kubernetes_node/tasks/node-join.yml b/ansible/playbooks/roles/kubernetes_node/tasks/node-join.yml
index 8d0b4e19c8..c663efc72a 100644
--- a/ansible/playbooks/roles/kubernetes_node/tasks/node-join.yml
+++ b/ansible/playbooks/roles/kubernetes_node/tasks/node-join.yml
@@ -13,7 +13,7 @@
             {{ hostvars[groups.kubernetes_master.0].ansible_default_ipv4.address }}:6443
           {%- endif -%}
       block:
-        - name: Creates directory
+        - name: Create kubeadm directory
           file:
             path: /etc/kubeadm/
             state: directory
@@ -38,27 +38,22 @@
                 register: kubeadm_join_result
 
           rescue:
-            - name: Attempt to join with kubeadm reset
-              when: kubeadm_join_result.stderr is search('/etc/kubernetes/kubelet.conf already exists')
-              block:
-                - name: Reset node
-                  command: kubeadm reset --force
-
-                - <<: *soft-join
-
-          always:
-            - name: Display kubeadm join stderr if any
+            - name: Display kubeadm join stderr
               debug:
                 msg: |
-                  Joined with warnings
+                  Node join attempt failed:
                   {{ kubeadm_join_result.stderr_lines }}
-              when: kubeadm_join_result is failed
 
-            - name: Mark node regardless of join result
-              set_fact:
-                kubernetes_common: >-
-                  {{ kubernetes_common | default({}) | combine(set_fact, recursive=true) }}
-              vars:
-                set_fact:
-                  node_already_joined: >-
-                    {{ kubeadm_join_result is succeeded }}
+            - name: Reset node
+              command: kubeadm reset --force
+
+            - <<: *soft-join
+
+        - name: Mark node as joined
+          set_fact:
+            kubernetes_common: >-
+              {{ kubernetes_common | default({}) | combine(set_fact, recursive=true) }}
+          vars:
+            set_fact:
+              node_already_joined: >-
+                {{ kubeadm_join_result is succeeded }}

From f6bbd2ae3659bc110ed80c9aef31cf2e9bf5fa0c Mon Sep 17 00:00:00 2001
From: atsikham <atsikham@users.noreply.github.com>
Date: Mon, 31 Jan 2022 13:23:43 +0100
Subject: [PATCH 06/10] Revert localhost removal from SANs

---
 ansible/playbooks/roles/kubernetes_master/tasks/main.yml        | 2 +-
 .../roles/kubernetes_master/templates/kubeadm-config.yml.j2     | 2 +-
 .../roles/upgrade/tasks/kubernetes/patch-kubeadm-apiserver.yml  | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/ansible/playbooks/roles/kubernetes_master/tasks/main.yml b/ansible/playbooks/roles/kubernetes_master/tasks/main.yml
index 1c2981268a..5c1f8d19ad 100644
--- a/ansible/playbooks/roles/kubernetes_master/tasks/main.yml
+++ b/ansible/playbooks/roles/kubernetes_master/tasks/main.yml
@@ -58,7 +58,7 @@
             certSANs: >-
               {{ (groups['kubernetes_master'] | map('extract', hostvars, ['ansible_default_ipv4', 'address']) | list)
               + (groups['kubernetes_master'] | map('extract', hostvars, ['ansible_host']) | list)
-              + [ '127.0.0.1' ] }}
+              + [ '127.0.0.1', 'localhost' ] }}
       include_role:
         name: kubernetes_common
         tasks_from: extend-kubeadm-config
diff --git a/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2 b/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2
index fbe2c6df55..a1f4bd1c1a 100644
--- a/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2
+++ b/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2
@@ -10,7 +10,7 @@ controlPlaneEndpoint: "localhost:3446"
 apiServer:
   timeoutForControlPlane: 4m0s
   certSANs:
-{% set address_list = ['127.0.0.1'] %}
+{% set address_list = ['127.0.0.1', 'localhost'] %}
 {% for host in groups['kubernetes_master'] %}
 {%   set _ = address_list.extend([ hostvars[host]['ansible_default_ipv4']['address'], hostvars[host]['ansible_host'] ]) %}
 {% endfor %}
diff --git a/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubeadm-apiserver.yml b/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubeadm-apiserver.yml
index a609e7fd96..5fe60fe541 100644
--- a/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubeadm-apiserver.yml
+++ b/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubeadm-apiserver.yml
@@ -66,7 +66,7 @@
         _update1: |-
           apiServer:
             certSANs:
-          {% set address_list = ['127.0.0.1'] %}
+          {% set address_list = ['127.0.0.1', 'localhost'] %}
           {% for host in groups['kubernetes_master'] %}
           {%   set _ = address_list.extend([ hostvars[host]['ansible_default_ipv4']['address'], hostvars[host]['ansible_host'] ]) %}
           {% endfor %}

From 05efa4d199415a3e890bdcc962c86809efa08853 Mon Sep 17 00:00:00 2001
From: atsikham <atsikham@users.noreply.github.com>
Date: Tue, 1 Feb 2022 23:16:18 +0100
Subject: [PATCH 07/10] Remove unnecessary apiserver cert generation tasks

---
 .../tasks/save-in-cluster-config.yml          | 24 +++++++
 .../tasks/apiserver-certificates.yml          | 40 -----------
 .../tasks/generate-certificates.yml           |  1 -
 .../roles/kubernetes_master/tasks/main.yml    | 67 ++++++++++++-------
 4 files changed, 66 insertions(+), 66 deletions(-)
 create mode 100644 ansible/playbooks/roles/kubernetes_common/tasks/save-in-cluster-config.yml
 delete mode 100644 ansible/playbooks/roles/kubernetes_master/tasks/apiserver-certificates.yml

diff --git a/ansible/playbooks/roles/kubernetes_common/tasks/save-in-cluster-config.yml b/ansible/playbooks/roles/kubernetes_common/tasks/save-in-cluster-config.yml
new file mode 100644
index 0000000000..88d1a91570
--- /dev/null
+++ b/ansible/playbooks/roles/kubernetes_common/tasks/save-in-cluster-config.yml
@@ -0,0 +1,24 @@
+---
+- name: Assert that dest_file variable is defined
+  assert:
+    that:
+      - dest_file is defined
+    fail_msg: Variable 'dest_file' must be defined
+
+- name: Collect kubeadm-config
+  command: |-
+    kubectl get configmap kubeadm-config \
+      --namespace kube-system \
+      --output jsonpath={{ jsonpath }}
+  vars:
+    jsonpath: >-
+      '{.data.ClusterConfiguration}'
+  register: kubeadm_config
+  changed_when: false
+
+- name: Create {{ dest_file }}
+  copy:
+    dest: "{{ dest_file }}"
+    mode: u=rw,go=
+    content: >-
+      {{ kubeadm_config.stdout }}
diff --git a/ansible/playbooks/roles/kubernetes_master/tasks/apiserver-certificates.yml b/ansible/playbooks/roles/kubernetes_master/tasks/apiserver-certificates.yml
deleted file mode 100644
index 0e984e15b4..0000000000
--- a/ansible/playbooks/roles/kubernetes_master/tasks/apiserver-certificates.yml
+++ /dev/null
@@ -1,40 +0,0 @@
----
-- name: Backup apiserver.{crt,key}
-  copy:
-    dest: "{{ item }}.OLD"
-    src: "{{ item }}"
-    remote_src: true
-  loop:
-    - "{{ pki.location }}/apiserver.crt"
-    - "{{ pki.location }}/apiserver.key"
-
-- name: Generate new apiserver certificates with kubeadm
-  block:
-    - name: Delete apiserver.{crt,key}
-      file:
-        path: "{{ item }}"
-        state: absent
-      loop:
-        - "{{ pki.location }}/apiserver.crt"
-        - "{{ pki.location }}/apiserver.key"
-
-    - name: Render new certificates apiserver.{crt,key}
-      command: |-
-        kubeadm init phase certs apiserver \
-          --config /etc/kubeadm/kubeadm-config.yml
-      args:
-        creates: "{{ pki.location }}/apiserver.key"
-
-  rescue:
-    - name: Restore apiserver.{crt,key}
-      copy:
-        dest: "{{ item }}"
-        src: "{{ item }}.OLD"
-        remote_src: true
-      loop:
-        - "{{ pki.location }}/apiserver.crt"
-        - "{{ pki.location }}/apiserver.key"
-
-    - name: Fail certificates generation
-      fail:
-        msg: Apiserver certificates generation failed, restored an initial state
diff --git a/ansible/playbooks/roles/kubernetes_master/tasks/generate-certificates.yml b/ansible/playbooks/roles/kubernetes_master/tasks/generate-certificates.yml
index 241e34cd26..9e01e11d8a 100644
--- a/ansible/playbooks/roles/kubernetes_master/tasks/generate-certificates.yml
+++ b/ansible/playbooks/roles/kubernetes_master/tasks/generate-certificates.yml
@@ -209,7 +209,6 @@
       fail:
         msg: Certificates generation failed, restored an initial state
 
-
 - name: Restart systemd services
   when:
     - services_to_restart is defined
diff --git a/ansible/playbooks/roles/kubernetes_master/tasks/main.yml b/ansible/playbooks/roles/kubernetes_master/tasks/main.yml
index 5c1f8d19ad..e08a42bc5a 100644
--- a/ansible/playbooks/roles/kubernetes_master/tasks/main.yml
+++ b/ansible/playbooks/roles/kubernetes_master/tasks/main.yml
@@ -25,12 +25,9 @@
         - import_tasks: master-join.yml
 
 - name: Collect subject alternative names of apiserver certificate
-  shell: |-
-    set -o pipefail && \
-    openssl x509 -in /etc/kubernetes/pki/apiserver.crt -text -noout \
-      | grep DNS:
+  command: openssl x509 -in apiserver.crt -ext subjectAltName -noout
   args:
-    executable: /bin/bash
+    chdir: "{{ pki.location }}"
   changed_when: false
   register: apiserver_certificate_san
 
@@ -44,13 +41,7 @@
 - name: Regenerate apiserver certificates
   when: "not (san_search_results is all)"
   block:
-    - name: Collect current apiserver certificate 'not_after' date by openssl
-      command: openssl x509 -enddate -noout -in apiserver.crt
-      args:
-        chdir: "{{ pki.location }}"
-      register: apiserver_certificate_enddate
-      changed_when: false
-
+    # Executed on all hosts as /etc/kubeadm/kubeadm-config.yml is required for apiserver certificate generation
     - name: Extend kubeadm config
       vars:
         update:
@@ -63,12 +54,23 @@
         name: kubernetes_common
         tasks_from: extend-kubeadm-config
 
-    - name: Generate apiserver certificates with latest kubeadm config
-      include_tasks: apiserver-certificates.yml
+    - name: Update in-cluster configuration
+      run_once: true
+      include_role:
+        name: kubernetes_common
+        tasks_from: update-in-cluster-config
 
+    # When specification.advanced.certificates.renew is set to true, certificate will be re-generated later
     - name: Update apiserver certificate expiration date
-      when: not (specification.advanced.certificates.renew | bool)
+      when: not specification.advanced.certificates.renew
       block:
+        - name: Collect current apiserver certificate 'not_after' date by openssl
+          command: openssl x509 -enddate -noout -in apiserver.crt
+          args:
+            chdir: "{{ pki.location }}"
+          register: apiserver_certificate_enddate
+          changed_when: false
+
         - name: Regenerate apiserver certificate with previous expiration value
           vars:
             certificates_renewal_list:
@@ -86,19 +88,34 @@
           args:
             executable: /bin/bash
 
-    - name: Update in-cluster configuration
-      when: kubernetes_common.automation_designated_master == inventory_hostname
+- name: Regenerate all certificates
+  when: specification.advanced.certificates.renew
+  block:
+    - name: Save kubeadm config to file
+      when:
+        - kubernetes_common.automation_designated_master != inventory_hostname
+        - san_search_results is all # kubeadm config was not extended/saved for apiserver cert generation
+      vars:
+        dest_file: /etc/kubeadm/kubeadm-config.yml
       include_role:
         name: kubernetes_common
-        tasks_from: update-in-cluster-config
+        tasks_from: save-in-cluster-config
 
-- name: Regenerate all certificates
-  when: specification.advanced.certificates.renew | bool
-  vars:
-    valid_days: "{{ specification.advanced.certificates.expiration_days }}"
-    services_to_restart:
-      - docker
-  include_tasks: generate-certificates.yml
+    - name: Regenerate certificates
+      vars:
+        valid_days: "{{ specification.advanced.certificates.expiration_days }}"
+        services_to_restart:
+          - docker
+      include_tasks: generate-certificates.yml
+
+# kubeadm-config.yml can appear not only on 'automation_designated_master' in 2 cases:
+# - the number of control-plane nodes was changed -> apiserver certificate had to be updated
+# - the file was created to renew all certificates
+- name: Ensure kubeadm-config.yml exists only on 'automation_designated_master'
+  when: kubernetes_common.automation_designated_master != inventory_hostname
+  file:
+    path: /etc/kubeadm/kubeadm-config.yml
+    state: absent
 
 - import_tasks: master-untaint.yml
 

From 909bfbec0131b21a1626dd0e00a5da62b83828ab Mon Sep 17 00:00:00 2001
From: atsikham <atsikham@users.noreply.github.com>
Date: Wed, 2 Feb 2022 10:31:57 +0100
Subject: [PATCH 08/10] Avoid usage of openssl ext option

---
 ansible/playbooks/roles/kubernetes_master/tasks/main.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/ansible/playbooks/roles/kubernetes_master/tasks/main.yml b/ansible/playbooks/roles/kubernetes_master/tasks/main.yml
index e08a42bc5a..75a4e43708 100644
--- a/ansible/playbooks/roles/kubernetes_master/tasks/main.yml
+++ b/ansible/playbooks/roles/kubernetes_master/tasks/main.yml
@@ -25,8 +25,13 @@
         - import_tasks: master-join.yml
 
 - name: Collect subject alternative names of apiserver certificate
-  command: openssl x509 -in apiserver.crt -ext subjectAltName -noout
+  # -ext option is not used as it requires newer openssl version
+  shell: |-
+    set -o pipefail && \
+    openssl x509 -in apiserver.crt -text -noout \
+      | grep DNS:
   args:
+    executable: /bin/bash
     chdir: "{{ pki.location }}"
   changed_when: false
   register: apiserver_certificate_san

From 3e3fbbf4297e20af1d8cfdb3433bffc4d67735ca Mon Sep 17 00:00:00 2001
From: atsikham <atsikham@users.noreply.github.com>
Date: Thu, 3 Feb 2022 15:53:07 +0100
Subject: [PATCH 09/10] Handle change of automation_designated_master

---
 ansible/playbooks/roles/kubernetes_master/tasks/main.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ansible/playbooks/roles/kubernetes_master/tasks/main.yml b/ansible/playbooks/roles/kubernetes_master/tasks/main.yml
index 75a4e43708..e3a32f6d37 100644
--- a/ansible/playbooks/roles/kubernetes_master/tasks/main.yml
+++ b/ansible/playbooks/roles/kubernetes_master/tasks/main.yml
@@ -98,7 +98,6 @@
   block:
     - name: Save kubeadm config to file
       when:
-        - kubernetes_common.automation_designated_master != inventory_hostname
         - san_search_results is all # kubeadm config was not extended/saved for apiserver cert generation
       vars:
         dest_file: /etc/kubeadm/kubeadm-config.yml

From 24871e6095c046324d6891ec1e52ec28cedfe4b4 Mon Sep 17 00:00:00 2001
From: atsikham <atsikham@users.noreply.github.com>
Date: Thu, 3 Feb 2022 15:54:15 +0100
Subject: [PATCH 10/10] Extend doc about k8s control plane certificates renewal

---
 docs/changelogs/CHANGELOG-2.0.md           |  1 +
 docs/home/howto/kubernetes/CERTIFICATES.md | 13 +++++++++++++
 2 files changed, 14 insertions(+)

diff --git a/docs/changelogs/CHANGELOG-2.0.md b/docs/changelogs/CHANGELOG-2.0.md
index 0c4aaaea40..42d71bbc3e 100644
--- a/docs/changelogs/CHANGELOG-2.0.md
+++ b/docs/changelogs/CHANGELOG-2.0.md
@@ -17,6 +17,7 @@
 - [#2828](https://github.com/epiphany-platform/epiphany/issues/2828) - K8s improvements
   - Re-generate apiserver certificates only by purpose
   - Do not ignore preflight errors in `kubeadm join`
+  - Update documentation about control plane certificates renewal
 - [#2825](https://github.com/epiphany-platform/epiphany/issues/2825) - Upgrade Terraform and providers
   - Terraform 0.12.6 to 1.1.3 ([#2706](https://github.com/epiphany-platform/epiphany/issues/2706))
   - Azurerm provider 1.38.0 to 2.91.0
diff --git a/docs/home/howto/kubernetes/CERTIFICATES.md b/docs/home/howto/kubernetes/CERTIFICATES.md
index dd05a2c26f..c2e247dd53 100644
--- a/docs/home/howto/kubernetes/CERTIFICATES.md
+++ b/docs/home/howto/kubernetes/CERTIFICATES.md
@@ -2,6 +2,19 @@
 
 ### TLS certificates in a cluster
 
+---
+**NOTE**
+
+1. There are issues encountered for K8s HA clusters when certificates renewal is enabled and applied
+   after `kubeadm reset`. If you restored control plane VMs from snapshots or used this command and plan to
+   run `epicli apply`, make sure that `renew` option is set to `false`.
+
+
+2. By default, kubeadm sets certificates expiration period to 1 year. If the cluster is upgraded, and different
+   expiration period is required, run `epicli apply` with appropriate configuration.
+
+---
+
 It's possible to regenerate Kubernetes control plane certificates with Epiphany. To do so, additional configuration
 should be specified.