Backport ability to change control plane certificates expiration date

CHANGELOG-0.5.md

@@ -2,6 +2,10 @@
 
 ## [0.5.5] 2020-10-07
 
+### Added
+
+- [#1302](https://github.com/epiphany-platform/epiphany/issues/1302) - Ability to update control plane certificates expiration date
+
 ### Fixed
 
 - [#1705](https://github.com/epiphany-platform/epiphany/issues/1705) - [RHEL/CentOS] epicli fails on downloading requirements - Docker CE repo not available

core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/generate-certificates.yml

@@ -0,0 +1,205 @@
+---
+- name: Generate certificates block
+  vars:
+    # https://kubernetes.io/docs/setup/best-practices/certificates/#all-certificates
+    _certificates_opt_mapping:
+      - name: admin.conf
+        kind: ['clientAuth']
+        target: admin.conf
+        parent_ca: ca
+      - name: apiserver-etcd-client
+        kind: ['clientAuth']
+        target: apiserver-etcd-client
+        parent_ca: etcd/ca
+      - name: apiserver-kubelet-client
+        kind: ['clientAuth']
+        target: apiserver-kubelet-client
+        parent_ca: ca
+      - name: apiserver
+        kind: ['serverAuth']
+        target: apiserver
+        parent_ca: ca
+      - name: controller-manager.conf
+        kind: ['clientAuth']
+        target: controller-manager.conf
+        parent_ca: ca
+      - name: etcd-healthcheck-client
+        kind: ['clientAuth']
+        target: etcd/healthcheck-client
+        parent_ca: etcd/ca
+      - name: etcd-peer
+        kind: ['serverAuth', 'clientAuth']
+        target: etcd/peer
+        parent_ca: etcd/ca
+      - name: etcd-server
+        kind: ['serverAuth', 'clientAuth']
+        target: etcd/server
+        parent_ca: etcd/ca
+      - name: front-proxy-client
+        kind: ['clientAuth']
+        target: front-proxy-client
+        parent_ca: front-proxy-ca
+      - name: scheduler.conf
+        kind: ['clientAuth']
+        target: scheduler.conf
+        parent_ca: ca
+  block:
+    - name: Set certificates_opt_mapping fact
+      block:
+        - set_fact:
+            certificates_opt_mapping: "{{ certificates_opt_mapping | default([]) + [item] }}"
+          when: certificates_renewal_list is defined and item.name in certificates_renewal_list
+          with_items: "{{ _certificates_opt_mapping }}"
+        - set_fact:
+            certificates_opt_mapping: "{{ _certificates_opt_mapping }}"
+          when: certificates_renewal_list is not defined
+
+    - name: Set backup_dir fact
+      set_fact:
+        backup_dir: >-
+          {{ specification.advanced.certificates.location | regex_replace('\\/$', '') }}-backup-{{ ansible_date_time.iso8601_basic_short }}
+
+    - name: Copy old certificates
+      block:
+        - name: Copy {{ specification.advanced.certificates.location }}
+          synchronize:
+            src: "{{ specification.advanced.certificates.location }}/"
+            dest: "{{ backup_dir }}"
+          delegate_to: "{{ inventory_hostname }}"
+        - name: Copy .conf files with embedded certificates
+          copy:
+            src: "/etc/kubernetes/{{ item }}"
+            dest: "{{ backup_dir }}"
+            remote_src: true
+          with_items:
+            - admin.conf
+            - controller-manager.conf
+            - scheduler.conf
+
+    - name: Ensure necessary directories exist
+      file:
+        path: "{{ item }}"
+        state: directory
+        owner: root
+        group: root
+        mode: u=rw
+      with_items:
+        - "{{ specification.advanced.certificates.location }}/csr"
+        - "{{ specification.advanced.certificates.location }}/ext"
+
+    - name: Generate CSR for kubeadm-managed certificates
+      command: kubeadm alpha certs renew all --csr-only --csr-dir=csr
+      args:
+        chdir: "{{ specification.advanced.certificates.location }}"
+
+    - name: Generate CSR for embedded .conf certificates
+      command: |-
+        openssl req -nodes -newkey rsa:2048 \
+          -keyout csr/{{ item.key }}.key \
+          -out csr/{{ item.key }}.csr \
+          -subj "{{ item.value }}"
+      args:
+        chdir: "{{ specification.advanced.certificates.location }}"
+      with_dict:
+        # https://kubernetes.io/docs/setup/best-practices/certificates/#configure-certificates-for-user-accounts
+        admin.conf: /O=system:masters/CN=kubernetes-admin
+        scheduler.conf: /CN=system:kube-scheduler
+        controller-manager.conf: /CN=system:kube-controller-manager
+
+
+    # ansible openssl modules and openssl tool behave different, extensions file is necessary for openssl
+    # https://github.com/openssl/openssl/issues/10458
+    - name: Register SAN extension for all csr files
+      shell: |-
+        openssl req -text -noout \
+          -reqopt no_subject,no_header,no_version,no_serial,no_signame,no_validity,no_issuer,no_pubkey,no_sigdump,no_aux \
+          -in csr/{{ item.target }}.csr \
+          | sed '1,3d;s/ Address//g;s/^[[:blank:]]*//;s/[[:blank:]]*$//'
+      args:
+        chdir: "{{ specification.advanced.certificates.location }}"
+      register: csr_info
+      with_items: "{{ certificates_opt_mapping }}"
+
+    - name: Generate extension files
+      template:
+        src: certificate-v3.ext.j2
+        dest: "{{ specification.advanced.certificates.location }}/ext/{{ item.0.name }}.ext"
+      with_together:
+        - "{{ certificates_opt_mapping }}"
+        - "{{ csr_info.results }}"
+
+    - name: Create signed certificates
+      command: |-
+        openssl x509 -req \
+          -days {{ valid_days }} \
+          -in csr/{{ item.target }}.csr \
+          -extfile ext/{{ item.name }}.ext \
+          -CA {{ item.parent_ca }}.crt \
+          -CAkey {{ item.parent_ca }}.key \
+          -CAcreateserial \
+          -out {{ item.target }}.crt
+      args:
+        chdir: "{{ specification.advanced.certificates.location }}"
+      with_items: "{{ certificates_opt_mapping }}"
+
+    - name: Copy keys to pki location and ensure that permissions are strict
+      copy:
+        src: "{{ specification.advanced.certificates.location }}/csr/{{ item.target }}.key"
+        remote_src: true
+        dest: "{{ specification.advanced.certificates.location }}/{{ item.target }}.key"
+        owner: root
+        group: root
+        mode: u=rw
+      with_items: "{{ certificates_opt_mapping }}"
+
+    - name: Remove csr and ext directories
+      file:
+        path: "{{ specification.advanced.certificates.location }}/{{ item }}"
+        state: absent
+      with_items:
+        - csr
+        - ext
+
+    - name: Update .conf files with embedded certs
+      environment:
+        KUBECONFIG: "/etc/kubernetes/{{ item.key }}"
+      command: |-
+        kubectl config set-credentials {{ item.value }} \
+          --client-key {{ item.key }}.key \
+          --client-certificate {{ item.key }}.crt \
+          --embed-certs
+      args:
+        chdir: "{{ specification.advanced.certificates.location }}"
+      with_dict:
+        admin.conf: "kubernetes-admin"
+        scheduler.conf: "system:kube-scheduler"
+        controller-manager.conf: "system:kube-controller-manager"
+
+    - name: Remove .conf certificates
+      file:
+        path: "{{ specification.advanced.certificates.location }}/{{ item.0 }}.{{ item.1 }}"
+        state: absent
+      with_nested:
+        - - 'admin.conf'
+          - 'scheduler.conf'
+          - 'controller-manager.conf'
+        - - 'crt'
+          - 'key'
+
+    - name: Restart systemd services
+      when: restart_services is defined and (restart_services | difference(['docker', 'kubelet']) | length == 0)
+      block:
+        - name: Restart
+          systemd:
+            name: "{{ item }}"
+            state: restarted
+          with_items: "{{ restart_services }}"
+
+        - name: Wait until cluster is available
+          environment:
+            KUBECONFIG: /etc/kubernetes/admin.conf
+          command: kubectl cluster-info
+          retries: 50
+          delay: 1
+          register: result
+          until: result is succeeded and "running" in result.stdout

core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/main.yml

@@ -3,6 +3,14 @@
 - include_tasks: single-master.yml
   when: groups['kubernetes_master'][0] == inventory_hostname
 
+- name: Regenerate all certificates
+  when: specification.advanced.certificates.renew | bool
+  vars:
+    valid_days: "{{ specification.advanced.certificates.expiration_days }}"
+    restart_services:
+      - docker
+  include_tasks: generate-certificates.yml
+
 - include_tasks: "{{ specification.provider }}/kubernetes-storage.yml"
   when: specification.storage.enable == True
 

core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/certificate-v3.ext.j2

@@ -0,0 +1,5 @@
+keyUsage = critical, digitalSignature, keyEncipherment
+extendedKeyUsage = {{ item.0.kind | join(',') }}
+{% if item.1.stdout %}
+subjectAltName = {{ item.1.stdout }}
+{% endif %}

core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2

@@ -28,4 +28,4 @@ imageRepository: {{ custom_image_registry_address }}/{{ specification.advanced.i
 {% endif %}
 
 
-certificatesDir: {{ specification.advanced.certificatesDir }}
+certificatesDir: {{ specification.advanced.certificates.location }}

core/src/epicli/data/common/defaults/configuration/kubernetes-master.yml

@@ -28,7 +28,10 @@ specification:
       serviceSubnet: 10.96.0.0/12
       plugin: flannel # valid options: calico, flannel, canal (due to lack of support for calico on Azure - use canal)
     imageRepository: k8s.gcr.io
-    certificatesDir: /etc/kubernetes/pki
+    certificates:
+      location: /etc/kubernetes/pki
+      expiration_days: 365
+      renew: false
     etcd_args:
       encrypted: yes
 

docs/home/CERTIFICATES.md

@@ -0,0 +1,55 @@
+# PKI certificates management
+
+## TLS certificates in a cluster
+
+It's possible to regenerate kubernetes control plane certificates with epiphany.
+To do so, additional configuration should be specified.
+
+```yaml
+kind: configuration/kubernetes-master
+title: "Kubernetes Master Config"
+name: default
+provider: <provider>
+specification:
+  advanced:
+    certificates:
+      expiration_days: <int>
+      renew: true
+```
+
+Parameters (optional):
+
+1. expiration_days - days to expire in, default value is `365`
+2. renew - whether to renew certificates or not, default value is `false`
+
+When `epicly apply` executes, if `renew` option is set to `true`, following certificates will be renewed with expiration period defined by `expiration_days`:
+
+1. admin.conf
+2. apiserver
+3. apiserver-etcd-client
+4. apiserver-kubelet-client
+5. controller-manager.conf
+6. etcd-healthcheck-client
+7. etcd-peer
+8. etcd-server
+9. front-proxy-client
+10. scheduler.conf
+
+---
+**NOTE**
+
+kubelet.conf is not renewed because kubelet is configured for automatic certificate renewal.
+To verify that, navigate to `/var/lib/kubelet/` and check `config.yaml` file, where `rotateCertificates` setting is `true` by default.
+
+---
+
+## CA certificates rotation
+
+This part cannot be done by epiphany. Refer to official kubernetes [documentation](https://kubernetes.io/docs/tasks/tls/manual-rotation-of-ca-certificates/) to perform this task.
+
+## References
+
+1. [Best practices](https://kubernetes.io/docs/setup/best-practices/certificates/)
+2. [Certificates management by kubeadm](https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/kubeadm-certs/)
+3. [Kubernetes the hard way](https://github.com/kelseyhightower/kubernetes-the-hard-way/blob/master/docs/04-certificate-authority.md)
+4. [Certificates generation with cfssl](https://gist.github.com/detiber/81b515df272f5911959e81e39137a8bb)