From 57a2282563f249bbcf6f82d5a432cf84c8679f45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Opala?= Date: Wed, 8 Jul 2020 10:51:37 +0200 Subject: [PATCH] Etcd encryption feature refactor for deployment and upgrades (#1427) * kubernetes_master: etcd encryption simplification and refactor * upgrade: refactor of upgrade-kubeadm-config.yml (proper yaml parsing) * upgrade: adding etcd encryption patching procedure * upgrade-master.yml: small coding style improvement (highlight fix) * upgrade: enabling patching of the kubeadm config * fact naming improvements Co-authored-by: to-bar <46519524+to-bar@users.noreply.github.com> * patch-kubeadm-config.yml: skipping unnecessary kubectl apply Co-authored-by: to-bar <46519524+to-bar@users.noreply.github.com> --- .../tasks/copy-kubernetes-pki.yml | 2 +- .../tasks/etcd-encryption-init.yml | 18 +++ .../tasks/etcd-encryption.yml | 54 ------- .../roles/kubernetes_master/tasks/main.yml | 6 +- .../templates/etc-encryption.conf.j2 | 4 +- .../templates/kubeadm-config.yml.j2 | 3 + .../tasks/kubernetes/etcd-encryption.yml | 5 - .../tasks/kubernetes/patch-kubeadm-config.yml | 132 ++++++++++++++++++ .../kubernetes/upgrade-kubeadm-config.yml | 63 ++++++++- .../tasks/kubernetes/upgrade-master.yml | 46 ++---- 10 files changed, 227 insertions(+), 106 deletions(-) create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/etcd-encryption-init.yml delete mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/etcd-encryption.yml delete mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/etcd-encryption.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubeadm-config.yml diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/copy-kubernetes-pki.yml b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/copy-kubernetes-pki.yml index 7880777b96..6e3646a0b7 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/copy-kubernetes-pki.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/copy-kubernetes-pki.yml @@ -23,7 +23,7 @@ pki_front_proxy_ca_key: pki/front-proxy-ca.key _optional_pki_files_map: pki_etcd_etc_encryption_conf: >- - {{ 'pki/etcd/etc-encryption.conf' if specification.advanced.etcd_args.encrypted else '' }} + {{ 'pki/etcd/etc-encryption.conf' if (specification.advanced.etcd_args.encrypted | bool) else '' }} - name: Check if the PKI file exists delegate_to: localhost diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/etcd-encryption-init.yml b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/etcd-encryption-init.yml new file mode 100644 index 0000000000..be81a9bbf6 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/etcd-encryption-init.yml @@ -0,0 +1,18 @@ +--- +- name: Ensure the /etc/kubernetes/pki/etcd/ directory exists + file: + path: /etc/kubernetes/pki/etcd/ + state: directory + owner: root + group: root + mode: u=rwx,g=rx,o= + +# Please note, the config file is automatically redistributed to other masters (via copy-kubernetes-pki.yml). +- name: Render the etcd encryption config file (just one time) + template: + dest: /etc/kubernetes/pki/etcd/etc-encryption.conf + src: etc-encryption.conf.j2 + force: false # please keep it "false" here, otherwise the secret will change and your cluster will start failing eventually + vars: + etcd_encryption_secret: >- + {{ lookup('password', '/dev/null length=32') | b64encode }} diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/etcd-encryption.yml b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/etcd-encryption.yml deleted file mode 100644 index 6974a5cab6..0000000000 --- a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/etcd-encryption.yml +++ /dev/null @@ -1,54 +0,0 @@ ---- -- name: Check if etcd encryption configuration exists - stat: - path: /etc/kubernetes/pki/etcd/etc-encryption.conf - get_attributes: false - get_checksum: false - get_mime: false - register: stat_etcd_conf_file - -- when: kubernetes_common.automation_designated_master == inventory_hostname - block: - - when: not stat_etcd_conf_file.stat.exists - block: - - name: Generate encryption secret - shell: | - head -c 32 /dev/urandom \ - | base64 -i - - args: - executable: /bin/bash - register: random_secret - changed_when: false - - - name: Create etcd encryption config - template: - src: etc-encryption.conf.j2 - dest: /etc/kubernetes/pki/etcd/etc-encryption.conf - owner: root - group: root - mode: u=rw,go=r - -# Common to all masters, there is no other way (yet) to make this work (2020). -- name: Change kube apiserver configuration for etcd - lineinfile: - path: /etc/kubernetes/manifests/kube-apiserver.yaml - insertafter: "^ - kube-apiserver$" - line: " - --encryption-provider-config=/etc/kubernetes/pki/etcd/etc-encryption.conf" - -- when: kubernetes_common.automation_designated_master == inventory_hostname - block: - - name: Run secrets encryption - environment: - KUBECONFIG: "/home/{{ admin_user.name }}/.kube/config" - shell: | - kubectl get secrets \ - --all-namespaces \ - -o json \ - | kubectl replace -f- - args: - executable: /bin/bash - register: result - until: - - result.rc == 0 - retries: 5 - delay: 10 diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/main.yml index 8560fc51cb..6777864189 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/main.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/main.yml @@ -9,9 +9,9 @@ - name: Init Kubernetes master when: kubernetes_common.automation_designated_master == inventory_hostname block: + - import_tasks: etcd-encryption-init.yml + when: specification.advanced.etcd_args.encrypted | bool - import_tasks: master-init.yml - - import_tasks: etcd-encryption.yml - when: specification.advanced.etcd_args.encrypted - import_tasks: registry-secrets.yml - import_tasks: copy-kubernetes-pki.yml @@ -22,8 +22,6 @@ block: - import_tasks: copy-kubernetes-pki.yml - import_tasks: master-join.yml - - import_tasks: etcd-encryption.yml - when: specification.advanced.etcd_args.encrypted - import_tasks: master-untaint.yml diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/etc-encryption.conf.j2 b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/etc-encryption.conf.j2 index 37ce31addf..1f2832456e 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/etc-encryption.conf.j2 +++ b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/etc-encryption.conf.j2 @@ -7,5 +7,5 @@ resources: - aescbc: keys: - name: key1 - secret: "{{ random_secret.stdout }}" - - identity: {} \ No newline at end of file + secret: "{{ etcd_encryption_secret }}" + - identity: {} diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2 b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2 index 24ca88cfc5..c1f6f65cc2 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2 +++ b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2 @@ -10,6 +10,9 @@ controlPlaneEndpoint: "localhost:3446" apiServer: timeoutForControlPlane: 4m0s extraArgs: # https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/ +{% if specification.advanced.etcd_args.encrypted | bool %} + encryption-provider-config: /etc/kubernetes/pki/etcd/etc-encryption.conf +{% endif %} {% for key, value in specification.advanced.api_server_args.items() %} {{ key }}: "{{ value }}" {% endfor %} diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/etcd-encryption.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/etcd-encryption.yml deleted file mode 100644 index aa43774840..0000000000 --- a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/etcd-encryption.yml +++ /dev/null @@ -1,5 +0,0 @@ - -- name: Apply etcd encryption to API server manifest if etcd encryption enabled - import_role: - name: kubernetes_master - tasks_from: etcd-encryption \ No newline at end of file diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubeadm-config.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubeadm-config.yml new file mode 100644 index 0000000000..3e96190218 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubeadm-config.yml @@ -0,0 +1,132 @@ +--- +- name: Check the etc-encryption.conf file + stat: + path: &etc-encryption-conf /etc/kubernetes/pki/etcd/etc-encryption.conf + get_attributes: false + get_checksum: false + get_mime: false + register: stat_etcd_encryption_config_file + +# Assuming that if the etcd encryption config file is absent, then +# the encryption feature has never been enabled for the cluster at hand. +- when: + - stat_etcd_encryption_config_file.stat.exists + block: + - name: Check the kubeadm-config.yml file + stat: + path: &kubeadm-config-yml /etc/kubeadm/kubeadm-config.yml + get_attributes: false + get_checksum: false + get_mime: false + register: stat_kubeadm_config_file + +- when: + - stat_etcd_encryption_config_file.stat.exists + - stat_kubeadm_config_file.stat.exists + block: + - name: Load contents of the kubeadm-config.yml file + slurp: + path: *kubeadm-config-yml + register: slurp_kubeadm_config + + - name: Save modified contents of the kubeadm-config.yml file + copy: + dest: *kubeadm-config-yml + + # Save all documents. + content: | + {% for document in _documents_updated %} + --- + {{ document | to_nice_yaml(indent=2) }} + {% endfor -%} + + vars: + # Parse yaml payload (remove empty documents). + _documents: >- + {{ slurp_kubeadm_config.content | b64decode + | from_yaml_all + | select + | list }} + # Prepare the patch. + # In this patch we include location of the etcd encryption config file. + # If it is not included, then the etcd encryption feature becomes disabled/broken. + # If it is not present on a cluster that has kube-system secrets already encrypted, then + # it may cause any upgrade attempt to freeze for a very long time (in Epiphany it has been reported to be even up to 8 hours). + _update: + apiServer: + extraArgs: + encryption-provider-config: *etc-encryption-conf + + # Process all documents (returns a list of dictionaries). + _documents_updated: >- + {%- set output = [] -%} + {%- for document in _documents -%} + {%- if document.kind is defined and document.kind == 'ClusterConfiguration' -%} + {{- output.append(document | combine(_update, recursive=true)) -}} + {%- else -%} + {{- output.append(document) -}} + {%- endif -%} + {%- endfor -%} + {{- output -}} + +# The `kubeadm upgrade` command can be executed with or without a config file. +# If the kubeadm-config.yml file does not exists, then we at least patch the kubeadm-config configmap. +- when: + - stat_etcd_encryption_config_file.stat.exists + - not stat_kubeadm_config_file.stat.exists + run_once: true # makes no sense to execute it more than once (would be redundant) + block: + - name: Load the kubeadm-config configmap + shell: | + kubectl get configmap kubeadm-config \ + --namespace kube-system \ + --output yaml + args: + executable: /bin/bash + environment: + KUBECONFIG: &KUBECONFIG /etc/kubernetes/admin.conf + register: shell_kubeadm_configmap + changed_when: false + + # The following procedure ensures that etcd encryption is always enabled + # during subsequent kubeadm executions (if the config file is not present). + - name: Patch and re-apply the kubeadm-config configmap + shell: | + kubectl apply \ + --namespace kube-system \ + --filename - \ + <<< "$KUBEADM_CONFIGMAP_DOCUMENT" + args: + executable: /bin/bash + environment: + KUBECONFIG: *KUBECONFIG + # Render an altered kubeadm-config configmap document. + KUBEADM_CONFIGMAP_DOCUMENT: >- + {{ _document | combine(_update2, recursive=true) | to_nice_yaml(indent=2) }} + + # Skip the task if there is no change in the cluster config. + when: _cluster_config_updated != _cluster_config # comparing two dictionaries here + + vars: + # Parse yaml payload. + _document: >- + {{ shell_kubeadm_configmap.stdout | from_yaml }} + + # Extract cluster config. + _cluster_config: >- + {{ _document.data.ClusterConfiguration | from_yaml }} + + # Prepare the cluster config patch. + _update1: + apiServer: + extraArgs: + encryption-provider-config: *etc-encryption-conf + + _cluster_config_updated: >- + {{ _cluster_config | combine(_update1, recursive=true) }} + + # Prepare the final update for the whole document. + _update2: + data: + ClusterConfiguration: >- + {{ _cluster_config_updated | to_nice_yaml(indent=2) }} diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-kubeadm-config.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-kubeadm-config.yml index 73630b0c55..8b30788bfb 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-kubeadm-config.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-kubeadm-config.yml @@ -1,7 +1,58 @@ +--- +- name: Run assertions for parameters of the task file at hand + block: + - assert: + that: + - version is defined + - version is string + - version | length > 0 + fail_msg: "Invalid version string." -- name: Update kubeadm-config.yml with current version (v{{ version }}) - lineinfile: - dest: /etc/kubeadm/kubeadm-config.yml - regexp: "^kubernetesVersion:" - line: "kubernetesVersion: v{{ version }}" - state: present \ No newline at end of file +- name: Check the kubeadm-config.yml file + stat: + path: &kubeadm-config-yml /etc/kubeadm/kubeadm-config.yml + get_attributes: false + get_checksum: false + get_mime: false + register: stat_kubeadm_config_file + +- when: stat_kubeadm_config_file.stat.exists + block: + - name: Load contents of the kubeadm-config.yml file + slurp: + path: *kubeadm-config-yml + register: slurp_kubeadm_config + + - name: Save modified contents of the kubeadm-config.yml file + copy: + dest: *kubeadm-config-yml + + # Save all documents. + content: | + {% for document in _documents_updated %} + --- + {{ document | to_nice_yaml(indent=2) }} + {% endfor -%} + + vars: + # Parse yaml payload (remove empty documents). + _documents: >- + {{ slurp_kubeadm_config.content | b64decode + | from_yaml_all + | select + | list }} + # Prepare the patch. + _update: + kubernetesVersion: "v{{ version }}" + + # Process all documents (returns a list of dictionaries). + _documents_updated: >- + {%- set output = [] -%} + {%- for document in _documents -%} + {%- if document.kind is defined and document.kind == 'ClusterConfiguration' -%} + {{- output.append(document | combine(_update, recursive=true)) -}} + {%- else -%} + {{- output.append(document) -}} + {%- endif -%} + {%- endfor -%} + {{- output -}} diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-master.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-master.yml index 8e1f6c00e8..81852212dc 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-master.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-master.yml @@ -8,10 +8,13 @@ changed_when: false register: kubeadm_config_file -- name: Check if etcd encryption configuration exists - stat: - path: /etc/kubernetes/pki/etcd/etc-encryption.conf - register: etcd_encryption_file +# This resolves issues (related to the etcd encryption) causing upgrades to hang. +# Legacy clusters may have incomplete configs, thus it is corrected here, before any `kubeadm upgrade` command is executed. +# If config is incomplete, kubeadm rewrites the kube-apiserver.yaml manifest file without the etcd feature enabled. +# In turn, this causes Kuberentes components such as the controller-manager to lose ability to read internal (kube-system) secrets, then +# any upgrade attempt freezes and the cluster at hand becomes unusable. +- name: upgrade-master | Make sure the etcd encryption feature is properly configured (if enabled) + import_tasks: patch-kubeadm-config.yml - name: Set imageRepository in kubeadm-config ConfigMap to use {{ image_registry_address }} when: @@ -23,9 +26,13 @@ {{ 'MasterConfiguration' if 'v1.11' in cluster_version else 'ClusterConfiguration' }} - name: upgrade-master | Get value of imageRepository key + shell: | + kubectl get cm -n kube-system kubeadm-config -o jsonpath={{ jsonpath }} + vars: + jsonpath: >- + '{.data.{{ kubeadm_config_parent_key }}}' environment: KUBECONFIG: /home/{{ admin_user.name }}/.kube/config - shell: kubectl get cm -n kube-system kubeadm-config -o jsonpath='{.data.{{ kubeadm_config_parent_key }}}' changed_when: false register: result @@ -131,26 +138,6 @@ - name: upgrade-master | Upgrade apply block block: - - when: etcd_encryption_file.stat.exists - block: &kube-apiserver-etcd-encryption-fix - - name: Spawn shell task in background to correct the kube-apiserver.yaml config file (if needed) - shell: | - for (( RETRY = 0; RETRY < {{ _async // _sleep }}; RETRY++ )); do - if grep -m1 '{{ _insertafter }}' '{{ _path }}' && ! grep -m1 '{{ _line }}' '{{ _path }}'; then - sed -i '/{{ _insertafter }}/a\{{ _line }}' '{{ _path }}' - fi - sleep {{ _sleep }} - done - args: { executable: /bin/bash } - vars: - _path: /etc/kubernetes/manifests/kube-apiserver.yaml - _insertafter: '^ - kube-apiserver$' - _line: " - --encryption-provider-config=/etc/kubernetes/pki/etcd/etc-encryption.conf" - _async: 600 # this task will terminate after 10 minutes - _sleep: 30 - async: "{{ _async | int }}" - poll: 0 - - name: "upgrade-master | Upgrade K8s cluster to v{{ version }} {{ '(using kubeadm-config.yml file)' if kubeadm_config_file.stat.exists else '' }}" shell: >- @@ -164,9 +151,6 @@ - cluster_version is version('v' + version, '<') # without this condition fails when 'upgrading' again from 1.12.10 to 1.12.10 rescue: # ignore CoreDNSUnsupportedPlugins error since coredns migration does not support all plugins that are valid and currently used - - when: etcd_encryption_file.stat.exists - block: *kube-apiserver-etcd-encryption-fix - - name: "upgrade-master | Upgrade K8s cluster to v{{ version }} {{ '(using kubeadm-config.yml file)' if kubeadm_config_file.stat.exists else '' }}" shell: >- @@ -192,12 +176,6 @@ include_tasks: upgrade-kubeadm-config.yml when: kubeadm_config_file.stat.exists - - name: upgrade-master | Set etcd encryption if configured - include_tasks: etcd-encryption.yml - when: - - groups['kubernetes_master'][0] == inventory_hostname - - etcd_encryption_file.stat.exists - # For Debian & K8s <= 1.14.6 we want to update packages like so - name: upgrade-master | Upgrade, configure packages when: