From 7ceffe54bce5acc43a44ea0d21eef6828b1cec43 Mon Sep 17 00:00:00 2001 From: Anatoli Tsikhamirau Date: Mon, 7 Sep 2020 15:28:14 +0200 Subject: [PATCH] Ability to change control plane certificates expiration date (#1595) * Refactored certificates settings * Moved apicerver-certificates tasks to correct role * Added ansible tasks to regenerate certificates * Added custom handler to process openssl output date * Applied created tasks to kubernetes_master role * Added documentation for certificates management * Updated changelog * Renamed config option not to match builtin name * Do not restart kubelet during certificates renewal process * Formatting and syntax fixes * Splitted tasks by empty lines * Renamed config setting * Collect k8s apiserver cert information before renewal * Certificates renewal documentation enhancements * Regenerate certificates tasks improvement Co-authored-by: atsikham --- CHANGELOG-0.8.md | 1 + .../filter_plugins/date_processing.py | 21 ++ .../tasks/apiserver-certificates.yml | 9 - .../tasks/generate-certificates.yml | 188 ++++++++++++++++++ .../roles/kubernetes_master/tasks/main.yml | 45 ++++- .../templates/certificate-v3.ext.j2 | 5 + .../templates/kubeadm-config.yml.j2 | 2 +- .../configuration/kubernetes-master.yml | 5 +- docs/home/CERTIFICATES.md | 55 +++++ 9 files changed, 315 insertions(+), 16 deletions(-) create mode 100644 core/src/epicli/data/common/ansible/playbooks/filter_plugins/date_processing.py rename core/src/epicli/data/common/ansible/playbooks/roles/{kubernetes_common => kubernetes_master}/tasks/apiserver-certificates.yml (75%) create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/generate-certificates.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/certificate-v3.ext.j2 create mode 100644 docs/home/CERTIFICATES.md diff --git a/CHANGELOG-0.8.md b/CHANGELOG-0.8.md index e2c891b924..aa9e7db1c7 100644 --- a/CHANGELOG-0.8.md +++ b/CHANGELOG-0.8.md @@ -4,6 +4,7 @@ ### Added +- [#1302](https://github.com/epiphany-platform/epiphany/issues/1302) - Ability to update control plane certificates expiration date - [#1324](https://github.com/epiphany-platform/epiphany/issues/1324) - Added Logstash to export data from Elasticsearch to csv format - [#1300](https://github.com/epiphany-platform/epiphany/issues/1300) - Configure OpenSSH according to Mozilla Infosec guidance - [#1543](https://github.com/epiphany-platform/epiphany/issues/1543) - Add support for Azure availability sets diff --git a/core/src/epicli/data/common/ansible/playbooks/filter_plugins/date_processing.py b/core/src/epicli/data/common/ansible/playbooks/filter_plugins/date_processing.py new file mode 100644 index 0000000000..450c139c37 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/filter_plugins/date_processing.py @@ -0,0 +1,21 @@ +#!/usr/bin/python + +from datetime import datetime + + +class FilterModule(object): + def filters(self): + return { + 'openssl_date2days': self.openssl_date2days + } + + def openssl_date2days(self, openssl_date): + """ + This function is used to find difference between openssl's + '-enddate' or '-startdate' output and today + :param openssl_date: '-enddate' or '-startdate' output of openssl, example: notAfter=Apr 20 07:06:21 2022 GMT + :return: result in days + """ + date1 = datetime.strptime(openssl_date.split('=')[1], '%b %d %H:%M:%S %Y %Z').date() + date2 = datetime.now().date() + return (date1 - date2).days diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_common/tasks/apiserver-certificates.yml b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/apiserver-certificates.yml similarity index 75% rename from core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_common/tasks/apiserver-certificates.yml rename to core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/apiserver-certificates.yml index 16adcf26ca..ad809c8234 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_common/tasks/apiserver-certificates.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/apiserver-certificates.yml @@ -23,12 +23,3 @@ args: executable: /bin/bash creates: /etc/kubernetes/pki/apiserver.key - -- name: Restart apiserver - shell: | - docker ps \ - --filter 'name=kube-apiserver_kube-apiserver' \ - --format '{{ "{{.ID}}" }}' \ - | xargs --no-run-if-empty docker kill - args: - executable: /bin/bash diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/generate-certificates.yml b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/generate-certificates.yml new file mode 100644 index 0000000000..650ad3288c --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/generate-certificates.yml @@ -0,0 +1,188 @@ +--- +- name: Generate certificates block + vars: + # https://kubernetes.io/docs/setup/best-practices/certificates/#all-certificates + _certificates_opt_mapping: + - name: admin.conf + kind: ['clientAuth'] + target: "{{ specification.advanced.certificates.location }}/admin.conf" + parent_ca: ca + - name: apiserver-etcd-client + kind: ['clientAuth'] + target: "{{ specification.advanced.certificates.location }}/apiserver-etcd-client" + parent_ca: etcd/ca + - name: apiserver-kubelet-client + kind: ['clientAuth'] + target: "{{ specification.advanced.certificates.location }}/apiserver-kubelet-client" + parent_ca: ca + - name: apiserver + kind: ['serverAuth'] + target: "{{ specification.advanced.certificates.location }}/apiserver" + parent_ca: ca + - name: controller-manager.conf + kind: ['clientAuth'] + target: "{{ specification.advanced.certificates.location }}/controller-manager.conf" + parent_ca: ca + - name: etcd-healthcheck-client + kind: ['clientAuth'] + target: "{{ specification.advanced.certificates.location }}/etcd/healthcheck-client" + parent_ca: etcd/ca + - name: etcd-peer + kind: ['serverAuth', 'clientAuth'] + target: "{{ specification.advanced.certificates.location }}/etcd/peer" + parent_ca: etcd/ca + - name: etcd-server + kind: ['serverAuth', 'clientAuth'] + target: "{{ specification.advanced.certificates.location }}/etcd/server" + parent_ca: etcd/ca + - name: front-proxy-client + kind: ['clientAuth'] + target: "{{ specification.advanced.certificates.location }}/front-proxy-client" + parent_ca: front-proxy-ca + - name: scheduler.conf + kind: ['clientAuth'] + target: "{{ specification.advanced.certificates.location }}/scheduler.conf" + parent_ca: ca + block: + - name: Create certificates_opt_mapping fact + block: + - set_fact: + certificates_opt_mapping: "{{ certificates_opt_mapping | default([]) + [item] }}" + when: certificates_renewal_list is defined and item.name in certificates_renewal_list + with_items: "{{ _certificates_opt_mapping }}" + - set_fact: + certificates_opt_mapping: "{{ _certificates_opt_mapping }}" + when: certificates_renewal_list is not defined + + - name: Save old certificates + synchronize: + src: "{{ specification.advanced.certificates.location }}/" + dest: >- + "{{ specification.advanced.certificates.location | regex_replace('\\/$', '') }}-backup-{{ ansible_date_time.iso8601_basic_short }}" + delegate_to: "{{ inventory_hostname }}" + + - name: Ensure necessary directories exist + file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: u=rw + with_items: + - "{{ specification.advanced.certificates.location }}/csr" + - "{{ specification.advanced.certificates.location }}/ext" + + - name: Generate new CSR + shell: kubeadm alpha certs renew all --csr-only --csr-dir=csr + args: + executable: /bin/bash + chdir: "{{ specification.advanced.certificates.location }}" + + # ansible openssl modules and openssl tool behave different, extensions file is necessary for openssl + # https://github.com/openssl/openssl/issues/10458 + - name: Register SAN extension for all csr files + shell: |- + openssl req -text -noout \ + -reqopt no_subject,no_header,no_version,no_serial,no_signame,no_validity,no_issuer,no_pubkey,no_sigdump,no_aux \ + -in csr/{{ item.name }}.csr \ + | sed '1,3d;s/ Address//g;s/^[[:blank:]]*//;s/[[:blank:]]*$//' + args: + executable: /bin/bash + chdir: "{{ specification.advanced.certificates.location }}" + register: csr_info + with_items: "{{ certificates_opt_mapping }}" + + - name: Generate extension files + template: + src: certificate-v3.ext.j2 + dest: "{{ specification.advanced.certificates.location }}/ext/{{ item.0.name }}.ext" + with_together: + - "{{ certificates_opt_mapping }}" + - "{{ csr_info.results }}" + + - name: Create signed certificates + shell: |- + openssl x509 -req -days {{ valid_days }} -in csr/{{ item.name }}.csr -extfile ext/{{ item.name }}.ext \ + -CA {{ item.parent_ca }}.crt -CAkey {{ item.parent_ca }}.key -CAcreateserial -out {{ item.target }}.crt + args: + executable: /bin/bash + chdir: "{{ specification.advanced.certificates.location }}" + with_items: "{{ certificates_opt_mapping }}" + + - name: Copy keys to pki location and ensure that permissions are strict + copy: + src: "{{ specification.advanced.certificates.location }}/csr/{{ item.name }}.key" + remote_src: yes + dest: "{{ item.target }}.key" + owner: root + group: root + mode: '0600' + with_items: "{{ certificates_opt_mapping }}" + + - name: Remove csr and ext directories + file: + path: "{{ specification.advanced.certificates.location }}/{{ item }}" + state: absent + with_items: + - csr + - ext + + - name: Search for .conf certificates + find: + paths: [ "{{ specification.advanced.certificates.location }}" ] + pattern: "*.conf.crt" + register: _conf_certificates + + - name: Set conf_certificates fact + set_fact: + conf_certificates: >- + {{ _conf_certificates.files + | map(attribute='path') + | map('basename') + | map('regex_replace', '\.crt$', '') + | list }} + + - name: Update conf files with embedded certs + environment: + KUBECONFIG: "/etc/kubernetes/{{ item }}" + vars: + conf_account_mapping: + admin.conf: "kubernetes-admin" + scheduler.conf: "system:kube-scheduler" + controller-manager.conf: "system:kube-controller-manager" + shell: | + kubectl config set-credentials {{ conf_account_mapping[item] }} \ + --client-key {{ specification.advanced.certificates.location }}/{{ item }}.key \ + --client-certificate {{ specification.advanced.certificates.location }}/{{ item }}.crt --embed-certs + args: + executable: /bin/bash + with_items: "{{ conf_certificates }}" + + - name: Remove conf certificates + file: + path: "{{ specification.advanced.certificates.location }}/{{ item.0 }}.{{ item.1 }}" + state: absent + with_nested: + - - 'admin.conf' + - 'scheduler.conf' + - 'controller-manager.conf' + - - 'crt' + - 'key' + + - name: Restart systemd services + when: restart_services is defined and (restart_services | difference(['docker', 'kubelet']) | length == 0) + block: + - name: Restart + systemd: + name: "{{ item }}" + state: restarted + with_items: "{{ restart_services }}" + + - name: Wait until cluster is available + environment: + KUBECONFIG: /etc/kubernetes/admin.conf + shell: kubectl cluster-info + retries: 50 + delay: 1 + register: output + until: output is succeeded diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/main.yml index a9d216db68..70fd2b9209 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/main.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/main.yml @@ -28,9 +28,16 @@ - import_tasks: copy-kubernetes-pki.yml - import_tasks: master-join.yml +- name: Collect current apiserver certificate 'not_after' date by openssl + shell: openssl x509 -enddate -noout -in apiserver.crt + args: + executable: /bin/bash + chdir: "{{ specification.advanced.certificates.location }}" + register: apiserver_certificate_info + - name: Regenerate apiserver certificates when: kubernetes_common.automation_designated_master != inventory_hostname or not is_first_deployment -# It's almost always necessary to regenerate certificates for designated and non-designated masters +# It's almost always necessary to regenerate apiserver certificates for designated and non-designated masters # because of a few points: # a. Update certificates for old clusters have to be supported # b. Execution order is not defined, so when cluster is promoted to HA, @@ -47,10 +54,30 @@ name: kubernetes_common tasks_from: extend-kubeadm-config - - name: Backup and generate apiserver certificates - include_role: - name: kubernetes_common - tasks_from: apiserver-certificates + - name: Backup and generate apiserver certificates with latest kubeadm config + include_tasks: apiserver-certificates.yml + +# kubeadm certs renewal uses the existing certificates as the authoritative source for attributes (Common Name, Organization, SAN, etc.) +# instead of the kubeadm-config ConfigMap, so it's not possible to combine this step with previous ones +# See https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/kubeadm-certs/#manual-certificate-renewal +- name: Update apiserver certificate expiration date + when: not (specification.advanced.certificates.renew | bool) + block: + - name: Regenerate apiserver certificate with previous expiration value + vars: + certificates_renewal_list: + - apiserver + valid_days: "{{ apiserver_certificate_info.stdout | openssl_date2days }}" + include_tasks: generate-certificates.yml + + - name: Restart apiserver + shell: | + docker ps \ + --filter 'name=kube-apiserver_kube-apiserver' \ + --format '{{ "{{.ID}}" }}' \ + | xargs --no-run-if-empty docker kill + args: + executable: /bin/bash - name: Update in-cluster configuration when: kubernetes_common.automation_designated_master == inventory_hostname @@ -58,6 +85,14 @@ name: kubernetes_common tasks_from: update-in-cluster-config +- name: Regenerate all certificates + when: specification.advanced.certificates.renew | bool + vars: + valid_days: "{{ specification.advanced.certificates.expiration_days }}" + restart_services: + - docker + include_tasks: generate-certificates.yml + - import_tasks: master-untaint.yml - include_tasks: "{{ specification.provider }}/kubernetes-storage.yml" diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/certificate-v3.ext.j2 b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/certificate-v3.ext.j2 new file mode 100644 index 0000000000..ab8d8955fe --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/certificate-v3.ext.j2 @@ -0,0 +1,5 @@ +keyUsage = critical, digitalSignature, keyEncipherment +extendedKeyUsage = {{ item.0.kind | join(',') }} +{% if item.1.stdout %} +subjectAltName = {{ item.1.stdout }} +{% endif %} diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2 b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2 index 86c00722da..2c77c85005 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2 +++ b/core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2 @@ -46,5 +46,5 @@ imageRepository: {{ image_registry_address }}/{{ specification.advanced.imageRep imageRepository: {{ custom_image_registry_address }}/{{ specification.advanced.imageRepository }} {% endif %} -certificatesDir: {{ specification.advanced.certificatesDir }} +certificatesDir: {{ specification.advanced.certificates.location }} diff --git a/core/src/epicli/data/common/defaults/configuration/kubernetes-master.yml b/core/src/epicli/data/common/defaults/configuration/kubernetes-master.yml index a409a99212..87b0b384a9 100644 --- a/core/src/epicli/data/common/defaults/configuration/kubernetes-master.yml +++ b/core/src/epicli/data/common/defaults/configuration/kubernetes-master.yml @@ -28,7 +28,10 @@ specification: serviceSubnet: 10.96.0.0/12 plugin: flannel # valid options: calico, flannel, canal (due to lack of support for calico on Azure - use canal) imageRepository: k8s.gcr.io - certificatesDir: /etc/kubernetes/pki + certificates: + location: /etc/kubernetes/pki + expiration_days: 365 + renew: false etcd_args: encrypted: yes diff --git a/docs/home/CERTIFICATES.md b/docs/home/CERTIFICATES.md new file mode 100644 index 0000000000..ce490fe1bc --- /dev/null +++ b/docs/home/CERTIFICATES.md @@ -0,0 +1,55 @@ +# PKI certificates management + +## TLS certificates in a cluster + +It's possible to regenerate kubernetes control plane certificates with epiphany. +To do so, additional configuration should be specified. + +```yaml +kind: configuration/kubernetes-master +title: "Kubernetes Master Config" +name: default +provider: +specification: + advanced: + certificates: + expiration_days: + renew: true +``` + +Parameters (optional): + +1. expiration_days - days to expire in, default value is `365` +2. renew - whether to renew certificates or not, default value is `false` + +When `epicly apply` executes, if `renew` option is set to `true`, following certificates will be renewed with expiration period defined by `expiration_days`: + +1. admin.conf +2. apiserver +3. apiserver-etcd-client +4. apiserver-kubelet-client +5. controller-manager.conf +6. etcd-healthcheck-client +7. etcd-peer +8. etcd-server +9. front-proxy-client +10. scheduler.conf + +--- +**NOTE** + +kubelet.conf is not renewed because kubelet is configured for automatic certificate renewal. +To verify that, navigate to `/var/lib/kubelet/` and check `config.yaml` file, where `rotateCertificates` setting is `true` by default. + +--- + +## CA certificates rotation + +This part cannot be done by epiphany. Refer to official kubernetes [documentation](https://kubernetes.io/docs/tasks/tls/manual-rotation-of-ca-certificates/) to perform this task. + +## References + +1. [Best practices](https://kubernetes.io/docs/setup/best-practices/certificates/) +2. [Certificates management by kubeadm](https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/kubeadm-certs/) +3. [Kubernetes the hard way](https://github.com/kelseyhightower/kubernetes-the-hard-way/blob/master/docs/04-certificate-authority.md) +4. [Certificates generation with cfssl](https://gist.github.com/detiber/81b515df272f5911959e81e39137a8bb)