Enabled TLS encrypted communication for etcd

automation/deploy_pgcluster.yml

@@ -83,18 +83,6 @@
         - pgbackrest_auto_conf | default(true) | bool  # to be able to disable auto backup settings
       tags: always
 
-  roles:
-    # (optional) if 'ssh_public_keys' is defined
-    - role: authorized-keys
-      tags: ssh_public_keys
-
-    - role: pre-checks
-      vars:
-        minimal_ansible_version: 2.16.0
-        timescale_minimal_pg_version: 12 # if enable_timescale is defined
-      tags: always
-
-  tasks:
     - name: Clean dnf cache
       ansible.builtin.command: dnf clean all
       when:
@@ -202,6 +190,22 @@
       when: pre_deploy_command | default('') | length > 0
       tags: pre_deploy, pre_deploy_command
 
+  roles:
+    # (optional) if 'ssh_public_keys' is defined
+    - role: authorized-keys
+      tags: ssh_public_keys
+
+    - role: pre-checks
+      vars:
+        minimal_ansible_version: 2.16.0
+        timescale_minimal_pg_version: 12 # if enable_timescale is defined
+      tags: always
+
+    - role: hostname
+
+    - role: tls_certificate/generate
+      when: tls_cert_generate|bool
+
 - name: deploy_pgcluster.yml | Deploy etcd cluster
   ansible.builtin.import_playbook: etcd_cluster.yml
   when: not dcs_exists|bool and dcs_type == "etcd"
@@ -261,7 +265,6 @@
       when: firewall_enabled_at_boot|bool
       tags: firewall
 
-    - role: hostname
     - role: resolv_conf
     - role: etc_hosts
     - role: add-repository
@@ -356,7 +359,7 @@
 
     - role: cron
 
-    - role: tls_certificate
+    - role: tls_certificate/copy
       when: tls_cert_generate|bool
 
     - role: pgbouncer

automation/molecule/tests/etcd/etcd.yml

@@ -1,8 +1,13 @@
 ---
 - name: Check etcd health
   ansible.builtin.uri:
-    url: "http://{{ inventory_hostname }}:2379/health"
+    url: "{{ patroni_etcd_protocol | default('http', true) }}://{{ inventory_hostname }}:2379/health"
+    method: GET
     return_content: true
+    validate_certs: "{{ tls_cert_generate | default(false) | bool }}"
+    ca_path: "{{ tls_etcd_ca_cert_path if tls_cert_generate | default(false) | bool else omit }}"
+    client_cert: "{{ tls_etcd_cert_path if tls_cert_generate | default(false) | bool else omit }}"
+    client_key: "{{ tls_etcd_privatekey_path if tls_cert_generate | default(false) | bool else omit }}"
   register: etcd_health_status
   failed_when: "(etcd_health_status.content | from_json).health != 'true'"
   when: dcs_type == "etcd"

automation/roles/confd/templates/confd.toml.j2

@@ -1,22 +1,26 @@
 backend = "etcdv3"
-interval = 10
-watch = true
 nodes = [
-{% if not dcs_exists|bool and dcs_type == 'etcd' %}
+{% if not dcs_exists|bool %}
   {% for host in groups['etcd_cluster'] %}
-  "http://{{ hostvars[host]['inventory_hostname'] }}:2379",
+  "{{ patroni_etcd_protocol | default('http', true) }}://{{ hostvars[host]['inventory_hostname'] }}:2379",
   {% endfor %}
 {% endif %}
-{% if dcs_exists|bool and dcs_type == 'etcd' %}
+{% if dcs_exists|bool %}
   {% for etcd_hosts in patroni_etcd_hosts %}
   "{{ patroni_etcd_protocol | default('http', true) }}://{{etcd_hosts.host}}:{{etcd_hosts.port}}",
   {% endfor %}
 {% endif %}
 ]
-{% if dcs_exists|bool and dcs_type == 'etcd' %}
+{% if tls_cert_generate | default(false) | bool %}
+scheme = "https"
+client_cakeys = "{{ tls_ca_cert_path | default('/etc/tls/ca.crt') }}"
+client_cert = "{{ tls_cert_path | default('/etc/tls/server.crt') }}"
+client_key = "{{ tls_privatekey_path | default('/etc/tls/server.key') }}"
+{% endif %}
 {% if patroni_etcd_username | default('') | length > 0 %}
 basic_auth = true
 username = "{{ patroni_etcd_username | default('') }}"
 password = "{{ patroni_etcd_password | default('') }}"
 {% endif %}
-{% endif %}
+watch = true
+interval = 10

automation/roles/etcd/tasks/main.yml

@@ -86,6 +86,17 @@
     state: directory
   tags: etcd, etcd_conf
 
+- name: Fetch etcd TLS certificate, key and CA from the master node
+  ansible.builtin.include_role:
+    name: ../roles/tls_certificate/copy
+  vars:
+    copy_tls_cert_path: "{{ tls_etcd_cert_path | default('/etc/etcd/tls/server.crt') }}"
+    copy_tls_ca_cert_path: "{{ tls_etcd_ca_cert_path | default('/etc/etcd/tls/ca.crt') }}"
+    copy_tls_privatekey_path: "{{ tls_etcd_privatekey_path | default('/etc/etcd/tls/server.key') }}"
+    copy_tls_owner: "etcd"
+  when: tls_cert_generate|bool
+  tags: etcd, etcd_conf
+
 - name: Create etcd data directory
   ansible.builtin.file:
     path: "{{ etcd_data_dir }}"
@@ -128,7 +139,12 @@
     - name: Wait until the etcd cluster is healthy
       ansible.builtin.command: >
         /usr/local/bin/etcdctl endpoint health
-        --endpoints=http://{{ inventory_hostname }}:2379
+        --endpoints={{ patroni_etcd_protocol | default('http', true) }}://{{ inventory_hostname }}:2379
+        {% if tls_cert_generate | default(false) | bool %}
+        --cacert={{ tls_etcd_ca_cert_path | default('/etc/etcd/tls/ca.crt') }}
+        --cert={{ tls_etcd_cert_path | default('/etc/etcd/tls/server.crt') }}
+        --key={{ tls_etcd_privatekey_path | default('/etc/etcd/tls/server.key') }}
+        {% endif %}
       environment:
         ETCDCTL_API: "3"
       register: etcd_health_result

automation/roles/etcd/templates/etcd.conf.j2

@@ -1,13 +1,24 @@
 ETCD_NAME="{{ ansible_hostname }}"
-ETCD_LISTEN_CLIENT_URLS="http://{{ inventory_hostname }}:2379,http://127.0.0.1:2379"
-ETCD_ADVERTISE_CLIENT_URLS="http://{{ inventory_hostname }}:2379"
-ETCD_LISTEN_PEER_URLS="http://{{ inventory_hostname }}:2380"
-ETCD_INITIAL_ADVERTISE_PEER_URLS="http://{{ inventory_hostname }}:2380"
+ETCD_LISTEN_CLIENT_URLS="{{ patroni_etcd_protocol | default('http', true) }}://{{ inventory_hostname }}:2379,{{ patroni_etcd_protocol | default('http', true) }}://127.0.0.1:2379"
+ETCD_ADVERTISE_CLIENT_URLS="{{ patroni_etcd_protocol | default('http', true) }}://{{ inventory_hostname }}:2379"
+ETCD_LISTEN_PEER_URLS="{{ patroni_etcd_protocol | default('http', true) }}://{{ inventory_hostname }}:2380"
+ETCD_INITIAL_ADVERTISE_PEER_URLS="{{ patroni_etcd_protocol | default('http', true) }}://{{ inventory_hostname }}:2380"
 ETCD_INITIAL_CLUSTER_TOKEN="{{ etcd_cluster_name }}"
-ETCD_INITIAL_CLUSTER="{% for host in groups['etcd_cluster'] %}{{ hostvars[host]['ansible_hostname'] }}=http://{{ hostvars[host]['inventory_hostname'] }}:2380{% if not loop.last %},{% endif %}{% endfor %}"
+ETCD_INITIAL_CLUSTER="{% for host in groups['etcd_cluster'] %}{{ hostvars[host]['ansible_hostname'] }}={{ patroni_etcd_protocol | default('http', true) }}://{{ hostvars[host]['inventory_hostname'] }}:2380{% if not loop.last %},{% endif %}{% endfor %}"
 ETCD_INITIAL_CLUSTER_STATE="new"
 ETCD_DATA_DIR="{{ etcd_data_dir }}"
 ETCD_ELECTION_TIMEOUT="5000"
 ETCD_HEARTBEAT_INTERVAL="1000"
 ETCD_INITIAL_ELECTION_TICK_ADVANCE="false"
 ETCD_AUTO_COMPACTION_RETENTION="1"
+{% if tls_cert_generate | default(false) | bool %}
+ETCD_CERT_FILE="{{ tls_etcd_cert_path | default('/etc/etcd/tls/server.crt') }}"
+ETCD_KEY_FILE="{{ tls_etcd_privatekey_path | default('/etc/etcd/tls/server.key') }}"
+ETCD_TRUSTED_CA_FILE="{{ tls_etcd_ca_cert_path | default('/etc/etcd/tls/ca.crt') }}"
+ETCD_PEER_CERT_FILE="{{ tls_etcd_cert_path | default('/etc/etcd/tls/server.crt') }}"
+ETCD_PEER_KEY_FILE="{{ tls_etcd_privatekey_path | default('/etc/etcd/tls/server.key') }}"
+ETCD_PEER_TRUSTED_CA_FILE="{{ tls_etcd_ca_cert_path | default('/etc/etcd/tls/ca.crt') }}"
+ETCD_PEER_CLIENT_CERT_AUTH="true"
+ETCD_CLIENT_CERT_AUTH="true"
+ETCD_TLS_MIN_VERSION="TLS1.2"
+{% endif %}

automation/roles/patroni/templates/patroni.yml.j2

@@ -35,23 +35,27 @@ restapi:
 #    password: password
 {% endif %}
 
-{% if not dcs_exists|bool and dcs_type == 'etcd' %}
+{% if dcs_type == 'etcd' %}
 etcd3:
+  {% if not dcs_exists|bool %}
   hosts: {% for host in groups['etcd_cluster'] %}{{ hostvars[host]['inventory_hostname'] }}:2379{% if not loop.last %},{% endif %}{% endfor %}
-{% endif %}
-{% if dcs_exists|bool and dcs_type == 'etcd' %}
-etcd3:
+  {% endif %}
+  {% if dcs_exists|bool %}
   hosts: {% for etcd_hosts in patroni_etcd_hosts %}{{etcd_hosts.host}}:{{etcd_hosts.port}}{% if not loop.last %},{% endif %}{% endfor %}
+  {% endif %}
 
+  {% if tls_cert_generate | default(false) | bool %}
+  protocol: https
+  cacert: {{ tls_ca_cert_path | default('/etc/tls/ca.crt') }}
+  cert: {{ tls_cert_path | default('/etc/tls/server.crt') }}
+  key: {{ tls_privatekey_path | default('/etc/tls/server.key') }}
+  {% endif %}
   {% if patroni_etcd_username | default('') | length > 0 %}
   username: {{ patroni_etcd_username | default('') }}
   {% endif %}
   {% if patroni_etcd_password | default('') | length > 0 %}
   password: {{ patroni_etcd_password }}
   {% endif %}
-  {% if patroni_etcd_protocol | default('') | length > 0 %}
-  protocol: {{ patroni_etcd_protocol }}
-  {% endif %}
 {% endif %}
 
 {% if dcs_type == 'consul' %}

automation/roles/patroni/templates/pg_hba.conf.j2

@@ -85,14 +85,14 @@
 
 # TYPE      DATABASE                 USER                     ADDRESS                  METHOD
 {% for client in postgresql_pg_hba %}
-  {{ client.type.ljust(10) |default('host') }}{{ client.database.ljust(25) |default('all') }}{{ client.user.ljust(25) |default('all') }}{{ client.address.ljust(25) |default('') }}{{ client.method |default('md5') }} {{ client.options |default(None) }}
+  {{ client.type.ljust(10) |default('{% if tls_cert_generate | default(false) | bool %}hostssl{% else %}host{% endif %}') }}{{ client.database.ljust(25) |default('all') }}{{ client.user.ljust(25) |default('all') }}{{ client.address.ljust(25) |default('') }}{{ client.method |default('md5') }} {{ client.options |default(None) }}
 {% endfor %}
 {% for patroni in groups['postgres_cluster'] %}
-  host      all                      all                      {{ hostvars[patroni]['inventory_hostname'] }}/32         {{ postgresql_password_encryption_algorithm }}
+  {% if tls_cert_generate | default(false) | bool %}hostssl{% else %}host{% endif %}      all                      all                      {{ hostvars[patroni]['inventory_hostname'] }}/32         {{ postgresql_password_encryption_algorithm }}
 {% endfor %}
 # Allow replication connections from localhost, by a user with the
 # replication privilege.
-  host      replication              {{ patroni_replication_username }}               localhost                trust
+  {% if tls_cert_generate | default(false) | bool %}hostssl{% else %}host{% endif %}      replication              {{ patroni_replication_username }}               localhost                trust
 {% for host in groups['postgres_cluster'] %}
-  host      replication              {{ patroni_replication_username }}               {{ hostvars[host]['inventory_hostname'] }}/32         {{ postgresql_password_encryption_algorithm }}
+  {% if tls_cert_generate | default(false) | bool %}hostssl{% else %}host{% endif %}      replication              {{ patroni_replication_username }}               {{ hostvars[host]['inventory_hostname'] }}/32         {{ postgresql_password_encryption_algorithm }}
 {% endfor %}

automation/roles/tls_certificate/copy/tasks/main.yml

@@ -1,42 +1,34 @@
 ---
-# for add_pgnode.yml
-
-- name: Ensure TLS directories exist
-  ansible.builtin.file:
-    path: "{{ item | dirname }}"
-    state: directory
-    owner: "{{ tls_owner | default('postgres') }}"
-    group: "{{ tls_owner | default('postgres') }}"
-    mode: "0750"
-  loop:
-    - "{{ tls_privatekey_path | default('/etc/tls/server.key') }}"
-    - "{{ tls_cert_path | default('/etc/tls/server.crt') }}"
-
-- name: Fetch TLS certificate and key from master
+- name: Fetch TLS certificate, key and CA from the master node into memory
   run_once: true
-  ansible.builtin.fetch:
+  ansible.builtin.slurp:
     src: "{{ item }}"
-    dest: "files/tls/"
-    validate_checksum: true
-    flat: true
   delegate_to: "{{ groups.master[0] }}"
+  register: tls_files
   loop:
     - "{{ tls_privatekey_path | default('/etc/tls/server.key') }}"
     - "{{ tls_cert_path | default('/etc/tls/server.crt') }}"
+    - "{{ tls_ca_cert_path | default('/etc/tls/ca.crt') }}"
+  tags: tls, tls_cert_copy
+
+- name: Create directory {{ copy_tls_privatekey_path | default(tls_privatekey_path | default('/etc/tls/server.key')) | dirname }}
+  ansible.builtin.file:
+    path: "{{ copy_tls_privatekey_path | default(tls_privatekey_path | default('/etc/tls/server.key')) | dirname }}"
+    state: directory
+    owner: "{{ copy_tls_owner | default(tls_owner | default('postgres')) }}"
+    group: "{{ copy_tls_owner | default(tls_owner | default('postgres')) }}"
+    mode: "0755"
+  tags: tls, tls_cert_copy
 
-- name: Copy TLS certificate and key to replica
+- name: Copy PostgreSQL TLS certificate, key and CA to all nodes
   ansible.builtin.copy:
-    src: "files/tls/{{ item.path | basename }}"
+    content: "{{ tls_files.results[item.index].content | b64decode }}"
     dest: "{{ item.path }}"
-    owner: "{{ tls_owner | default('postgres') }}"
-    group: "{{ tls_owner | default('postgres') }}"
+    owner: "{{ copy_tls_owner | default(tls_owner | default('postgres')) }}"
+    group: "{{ copy_tls_owner | default(tls_owner | default('postgres')) }}"
     mode: "{{ item.mode }}"
   loop:
-    - { path: "{{ tls_privatekey_path | default('/etc/tls/server.key') }}", mode: "{{ tls_privatekey_mode | default('0400') }}" }
-    - { path: "{{ tls_cert_path | default('/etc/tls/server.crt') }}", mode: "{{ tls_cert_mode | default('0644') }}" }
-
-- name: Delete TLS certificate and key from the ansible controller
-  ansible.builtin.file:
-    path: "files/tls/"
-    state: absent
-  delegate_to: localhost
+    - { index: 1, path: "{{ copy_tls_cert_path | default(tls_cert_path | default('/etc/tls/server.crt')) }}", mode: "0644" }
+    - { index: 2, path: "{{ copy_tls_ca_cert_path | default(tls_ca_cert_path | default('/etc/tls/ca.crt')) }}", mode: "0644" }
+    - { index: 0, path: "{{ copy_tls_privatekey_path | default(tls_privatekey_path | default('/etc/tls/server.key')) }}", mode: "0400" }
+  tags: tls, tls_cert_copy