From 6ed248ffac70930d5dcef223b59c22f02d279bb4 Mon Sep 17 00:00:00 2001 From: klention Date: Sun, 15 Dec 2024 19:05:39 +0100 Subject: [PATCH 01/23] 1. Enabled TLS encryption for PostgreSQL and PGBouncer. 2. Automatically set the number of max_worker_processes and max_parallel_workers based on the system resources. --- automation/roles/patroni/tasks/main.yml | 18 +++++++++++++++- automation/roles/pgbouncer/tasks/main.yml | 16 ++++++++++++++ .../pgbouncer/templates/pgbouncer.ini.j2 | 5 +++++ automation/vars/main.yml | 21 +++++++++++++------ 4 files changed, 53 insertions(+), 7 deletions(-) diff --git a/automation/roles/patroni/tasks/main.yml b/automation/roles/patroni/tasks/main.yml index 2d5d6f49c..cf643f521 100644 --- a/automation/roles/patroni/tasks/main.yml +++ b/automation/roles/patroni/tasks/main.yml @@ -418,7 +418,7 @@ when: postgresql_wal_dir is defined and postgresql_wal_dir | length > 0 tags: patroni, custom_wal_dir -- block: # wheh postgresql NOT exists or PITR +- block: # when postgresql NOT exists or PITR - name: Prepare PostgreSQL | make sure PostgreSQL data directory "{{ postgresql_data_dir }}" exists ansible.builtin.file: path: "{{ postgresql_data_dir }}" @@ -427,6 +427,22 @@ state: directory mode: "0700" + # Generating TLS Certificates + - name: Generating TLS Certificates to {{ postgresql_home_dir }}/{{ postgresql_version }} + become: true + become_user: postgres + shell: + cmd: openssl req -nodes -new -x509 -days 3650 -keyout server.key -out server.crt -subj '/C=AL/L=City/O=Org/CN=PostgreSQL' + chdir: "{{ postgresql_home_dir }}/{{ postgresql_version }}/" + + - name: Changing permissions for {{ postgresql_home_dir }}/{{ postgresql_version }}/server.key + file: + path: "{{ postgresql_home_dir }}/{{ postgresql_version }}/server.key" + state: file + owner: postgres + group: postgres + mode: "0400" + # for Debian based distros only # patroni bootstrap failure is possible if the PostgreSQL config files are missing - name: Prepare PostgreSQL | make sure PostgreSQL config directory exists diff --git a/automation/roles/pgbouncer/tasks/main.yml b/automation/roles/pgbouncer/tasks/main.yml index cd2fb08fe..63fe0b51e 100644 --- a/automation/roles/pgbouncer/tasks/main.yml +++ b/automation/roles/pgbouncer/tasks/main.yml @@ -124,6 +124,22 @@ label: "{{ 'pgbouncer' if idx == 0 else 'pgbouncer-%d' % (idx + 1) }}" tags: pgbouncer_logrotate, pgbouncer +# Generating TLS Certificates +- name: Generating TLS Certificates to {{ pgbouncer_conf_dir }} + become: true + become_user: postgres + shell: + cmd: openssl req -nodes -new -x509 -days 3650 -keyout server.key -out server.crt -subj '/C=AL/L=City/O=Org/CN=PostgreSQL' + chdir: "{{ pgbouncer_conf_dir }}/" + +- name: Changing permissions for {{ pgbouncer_conf_dir }}/server.key + file: + path: "{{ pgbouncer_conf_dir }}/server.key" + state: file + owner: postgres + group: postgres + mode: 0400 + - name: Configure pgbouncer.ini ansible.builtin.template: src: templates/pgbouncer.ini.j2 diff --git a/automation/roles/pgbouncer/templates/pgbouncer.ini.j2 b/automation/roles/pgbouncer/templates/pgbouncer.ini.j2 index 8926d97bb..f2b6bd56d 100644 --- a/automation/roles/pgbouncer/templates/pgbouncer.ini.j2 +++ b/automation/roles/pgbouncer/templates/pgbouncer.ini.j2 @@ -42,6 +42,11 @@ client_tls_cert_file = {{ pgbouncer_client_tls_cert_file }} client_tls_ca_file = {{ pgbouncer_client_tls_ca_file }} client_tls_protocols = {{ pgbouncer_client_tls_protocols }} client_tls_ciphers = {{ pgbouncer_client_tls_ciphers }} +server_tls_sslmode = {{ pgbouncer_server_tls_sslmode }} +server_tls_protocols = {{ pgbouncer_server_tls_protocols }} +server_tls_ciphers = {{ pgbouncer_server_tls_ciphers }} +server_tls_cert_file = {{ pgbouncer_server_tls_cert_file }} +server_tls_key_file = {{ pgbouncer_server_tls_key_file }} {% endif %} log_connections = 0 log_disconnections = 0 diff --git a/automation/vars/main.yml b/automation/vars/main.yml index bf7e55cc2..dbaff85e2 100644 --- a/automation/vars/main.yml +++ b/automation/vars/main.yml @@ -235,6 +235,10 @@ postgresql_parameters: - { option: "max_connections", value: "1000" } - { option: "superuser_reserved_connections", value: "5" } - { option: "password_encryption", value: "{{ postgresql_password_encryption_algorithm }}" } + - { option: "ssl", value: "on"} + - { option: "ssl_cert_file", value: "{{ postgresql_home_dir }}/{{ postgresql_version }}/server.crt"} + - { option: "ssl_key_file", value: "{{ postgresql_home_dir }}/{{ postgresql_version }}/server.key"} + - { option: "ssl_min_protocol_version", value: "TLSv1.2"} - { option: "max_locks_per_transaction", value: "512" } - { option: "max_prepared_transactions", value: "0" } - { option: "huge_pages", value: "try" } # "vm.nr_hugepages" is auto-configured for shared_buffers >= 8GB (if huge_pages_auto_conf is true) @@ -305,8 +309,8 @@ postgresql_parameters: - { option: "wal_receiver_status_interval", value: "10s" } - { option: "idle_in_transaction_session_timeout", value: "10min" } # reduce this timeout if possible - { option: "jit", value: "off" } - - { option: "max_worker_processes", value: "24" } - - { option: "max_parallel_workers", value: "8" } + - { option: "max_worker_processes", value: "{{ ansible_processor_nproc | int }}" } + - { option: "max_parallel_workers", value: "{{ ansible_processor_nproc | int }}" } - { option: "max_parallel_workers_per_gather", value: "2" } - { option: "max_parallel_maintenance_workers", value: "2" } - { option: "tcp_keepalives_count", value: "10" } @@ -366,12 +370,17 @@ pgbouncer_auth_user: true # or 'false' if you want to manage the list of users f pgbouncer_auth_username: pgbouncer # user who can query the database via the user_search function pgbouncer_auth_password: "" # If not defined, a password will be generated automatically during deployment pgbouncer_auth_dbname: "postgres" -pgbouncer_client_tls_sslmode: "disable" -pgbouncer_client_tls_key_file: "" -pgbouncer_client_tls_cert_file: "" +pgbouncer_client_tls_sslmode: "require" +pgbouncer_client_tls_key_file: "{{ pgbouncer_conf_dir }}/server.key" +pgbouncer_client_tls_cert_file: "{{ pgbouncer_conf_dir }}/server.crt" pgbouncer_client_tls_ca_file: "" pgbouncer_client_tls_protocols: "secure" # allowed values: tlsv1.0, tlsv1.1, tlsv1.2, tlsv1.3, all, secure (tlsv1.2,tlsv1.3) -pgbouncer_client_tls_ciphers: "default" # allowed values: default, secure, fast, normal, all (not recommended) +pgbouncer_client_tls_ciphers: "secure" # allowed values: default, secure, fast, normal, all (not recommended) +pgbouncer_server_tls_sslmode: "require" +pgbouncer_server_tls_protocols: "secure" +pgbouncer_server_tls_ciphers: "secure" +pgbouncer_server_tls_cert_file: "{{ pgbouncer_conf_dir }}/server.crt" +pgbouncer_server_tls_key_file: "{{ pgbouncer_conf_dir }}/server.key" pgbouncer_pools: - { name: "postgres", dbname: "postgres", pool_parameters: "" } From 8ac1c91e04478fc3992b22006803829247bd2b00 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik <37010174+vitabaks@users.noreply.github.com> Date: Mon, 16 Dec 2024 12:54:14 +0500 Subject: [PATCH 02/23] Apply suggestions from code review --- automation/roles/patroni/tasks/main.yml | 5 ++--- automation/roles/pgbouncer/tasks/main.yml | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/automation/roles/patroni/tasks/main.yml b/automation/roles/patroni/tasks/main.yml index cf643f521..914c0314a 100644 --- a/automation/roles/patroni/tasks/main.yml +++ b/automation/roles/patroni/tasks/main.yml @@ -431,12 +431,11 @@ - name: Generating TLS Certificates to {{ postgresql_home_dir }}/{{ postgresql_version }} become: true become_user: postgres - shell: + ansible.builtin.command: cmd: openssl req -nodes -new -x509 -days 3650 -keyout server.key -out server.crt -subj '/C=AL/L=City/O=Org/CN=PostgreSQL' chdir: "{{ postgresql_home_dir }}/{{ postgresql_version }}/" - - name: Changing permissions for {{ postgresql_home_dir }}/{{ postgresql_version }}/server.key - file: + ansible.builtin.file: path: "{{ postgresql_home_dir }}/{{ postgresql_version }}/server.key" state: file owner: postgres diff --git a/automation/roles/pgbouncer/tasks/main.yml b/automation/roles/pgbouncer/tasks/main.yml index 63fe0b51e..f33573b21 100644 --- a/automation/roles/pgbouncer/tasks/main.yml +++ b/automation/roles/pgbouncer/tasks/main.yml @@ -128,17 +128,17 @@ - name: Generating TLS Certificates to {{ pgbouncer_conf_dir }} become: true become_user: postgres - shell: + ansible.builtin.command: cmd: openssl req -nodes -new -x509 -days 3650 -keyout server.key -out server.crt -subj '/C=AL/L=City/O=Org/CN=PostgreSQL' chdir: "{{ pgbouncer_conf_dir }}/" - name: Changing permissions for {{ pgbouncer_conf_dir }}/server.key - file: + ansible.builtin.file: path: "{{ pgbouncer_conf_dir }}/server.key" state: file owner: postgres group: postgres - mode: 0400 + mode: "0400" - name: Configure pgbouncer.ini ansible.builtin.template: From f4b5e81e33aec7cb92f28b39e1663825f653ddd7 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik <37010174+vitabaks@users.noreply.github.com> Date: Mon, 16 Dec 2024 12:56:55 +0500 Subject: [PATCH 03/23] formatting: add a new line between tasks --- automation/roles/patroni/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/automation/roles/patroni/tasks/main.yml b/automation/roles/patroni/tasks/main.yml index 914c0314a..601123d25 100644 --- a/automation/roles/patroni/tasks/main.yml +++ b/automation/roles/patroni/tasks/main.yml @@ -434,6 +434,7 @@ ansible.builtin.command: cmd: openssl req -nodes -new -x509 -days 3650 -keyout server.key -out server.crt -subj '/C=AL/L=City/O=Org/CN=PostgreSQL' chdir: "{{ postgresql_home_dir }}/{{ postgresql_version }}/" + - name: Changing permissions for {{ postgresql_home_dir }}/{{ postgresql_version }}/server.key ansible.builtin.file: path: "{{ postgresql_home_dir }}/{{ postgresql_version }}/server.key" From 12d51960cea888d6b581091c3d4bdd71a0aab759 Mon Sep 17 00:00:00 2001 From: Klention Mali <45871249+klention@users.noreply.github.com> Date: Mon, 16 Dec 2024 14:49:15 +0100 Subject: [PATCH 04/23] Update automation/vars/main.yml Co-authored-by: Vitaliy Kukharik <37010174+vitabaks@users.noreply.github.com> --- automation/vars/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automation/vars/main.yml b/automation/vars/main.yml index dbaff85e2..f9957f092 100644 --- a/automation/vars/main.yml +++ b/automation/vars/main.yml @@ -309,8 +309,8 @@ postgresql_parameters: - { option: "wal_receiver_status_interval", value: "10s" } - { option: "idle_in_transaction_session_timeout", value: "10min" } # reduce this timeout if possible - { option: "jit", value: "off" } - - { option: "max_worker_processes", value: "{{ ansible_processor_nproc | int }}" } - - { option: "max_parallel_workers", value: "{{ ansible_processor_nproc | int }}" } + - { option: "max_worker_processes", value: "{{ [ansible_processor_vcpus | int, 16] | max }}" } + - { option: "max_parallel_workers", value: "{{ [(ansible_processor_vcpus | int // 2), 8] | max }}" } - { option: "max_parallel_workers_per_gather", value: "2" } - { option: "max_parallel_maintenance_workers", value: "2" } - { option: "tcp_keepalives_count", value: "10" } From 38159d598d5cc0c1c29b07ee94fee66b0e4733b6 Mon Sep 17 00:00:00 2001 From: klention Date: Mon, 16 Dec 2024 18:02:57 +0100 Subject: [PATCH 05/23] Use ansible module to generate TLS certificates instead of OpenSSL --- automation/roles/patroni/tasks/main.yml | 21 ++++++++++++--------- automation/roles/pgbouncer/tasks/main.yml | 21 ++++++++++++--------- automation/vars/Debian.yml | 1 + automation/vars/RedHat.yml | 1 + 4 files changed, 26 insertions(+), 18 deletions(-) diff --git a/automation/roles/patroni/tasks/main.yml b/automation/roles/patroni/tasks/main.yml index 601123d25..a833d8bc7 100644 --- a/automation/roles/patroni/tasks/main.yml +++ b/automation/roles/patroni/tasks/main.yml @@ -428,20 +428,23 @@ mode: "0700" # Generating TLS Certificates - - name: Generating TLS Certificates to {{ postgresql_home_dir }}/{{ postgresql_version }} + - name: Generate a private key for TLS become: true become_user: postgres - ansible.builtin.command: - cmd: openssl req -nodes -new -x509 -days 3650 -keyout server.key -out server.crt -subj '/C=AL/L=City/O=Org/CN=PostgreSQL' - chdir: "{{ postgresql_home_dir }}/{{ postgresql_version }}/" - - - name: Changing permissions for {{ postgresql_home_dir }}/{{ postgresql_version }}/server.key - ansible.builtin.file: + community.crypto.openssl_privatekey: path: "{{ postgresql_home_dir }}/{{ postgresql_version }}/server.key" - state: file + size: '4096' + mode: '0400' + + - name: Generate a self-signed certificate for Postgres + community.crypto.x509_certificate: + path: "{{ postgresql_home_dir }}/{{ postgresql_version }}/server.crt" + privatekey_path: "{{ postgresql_home_dir }}/{{ postgresql_version }}/server.key" owner: postgres group: postgres - mode: "0400" + mode: '0644' + provider: selfsigned + entrust_not_after: "+1000d" # for Debian based distros only # patroni bootstrap failure is possible if the PostgreSQL config files are missing diff --git a/automation/roles/pgbouncer/tasks/main.yml b/automation/roles/pgbouncer/tasks/main.yml index f33573b21..7e76c6b8f 100644 --- a/automation/roles/pgbouncer/tasks/main.yml +++ b/automation/roles/pgbouncer/tasks/main.yml @@ -125,20 +125,23 @@ tags: pgbouncer_logrotate, pgbouncer # Generating TLS Certificates -- name: Generating TLS Certificates to {{ pgbouncer_conf_dir }} +- name: Generate a private key for TLS become: true become_user: postgres - ansible.builtin.command: - cmd: openssl req -nodes -new -x509 -days 3650 -keyout server.key -out server.crt -subj '/C=AL/L=City/O=Org/CN=PostgreSQL' - chdir: "{{ pgbouncer_conf_dir }}/" - -- name: Changing permissions for {{ pgbouncer_conf_dir }}/server.key - ansible.builtin.file: + community.crypto.openssl_privatekey: path: "{{ pgbouncer_conf_dir }}/server.key" - state: file + size: '4096' + mode: '0400' + +- name: Generate a self-signed certificate for PGBouncer + community.crypto.x509_certificate: + path: "{{ pgbouncer_conf_dir }}/server.crt" + privatekey_path: "{{ pgbouncer_conf_dir }}/server.key" owner: postgres group: postgres - mode: "0400" + mode: '0644' + provider: selfsigned + entrust_not_after: "+1000d" - name: Configure pgbouncer.ini ansible.builtin.template: diff --git a/automation/vars/Debian.yml b/automation/vars/Debian.yml index 5e2d99900..4f1ea629b 100644 --- a/automation/vars/Debian.yml +++ b/automation/vars/Debian.yml @@ -40,6 +40,7 @@ system_packages: - python3-psycopg2 - python3-setuptools - python3-pip + - python3-cryptography - curl - less - sudo diff --git a/automation/vars/RedHat.yml b/automation/vars/RedHat.yml index 46e158bcc..86fb6be98 100644 --- a/automation/vars/RedHat.yml +++ b/automation/vars/RedHat.yml @@ -60,6 +60,7 @@ system_packages: - python{{ python_version }}-setuptools - python{{ python_version }}-pip - python{{ python_version }}-urllib3 + - python3-cryptography - less - sudo - vim From 790ddbfaf07d3bef58f324adcf32bc9bf6b67908 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik <37010174+vitabaks@users.noreply.github.com> Date: Tue, 17 Dec 2024 11:44:17 +0500 Subject: [PATCH 06/23] Update postgres cert path --- automation/roles/patroni/tasks/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/automation/roles/patroni/tasks/main.yml b/automation/roles/patroni/tasks/main.yml index a833d8bc7..f87437ca3 100644 --- a/automation/roles/patroni/tasks/main.yml +++ b/automation/roles/patroni/tasks/main.yml @@ -432,14 +432,14 @@ become: true become_user: postgres community.crypto.openssl_privatekey: - path: "{{ postgresql_home_dir }}/{{ postgresql_version }}/server.key" + path: "{{ postgresql_conf_dir }}/server.key" size: '4096' mode: '0400' - name: Generate a self-signed certificate for Postgres community.crypto.x509_certificate: - path: "{{ postgresql_home_dir }}/{{ postgresql_version }}/server.crt" - privatekey_path: "{{ postgresql_home_dir }}/{{ postgresql_version }}/server.key" + path: "{{ postgresql_conf_dir }}/server.crt" + privatekey_path: "{{ postgresql_conf_dir }}/server.key" owner: postgres group: postgres mode: '0644' From 0debea8aa468426981ce0cce2449dee471162e1f Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik <37010174+vitabaks@users.noreply.github.com> Date: Tue, 17 Dec 2024 11:44:37 +0500 Subject: [PATCH 07/23] Update postgres cert path in vars --- automation/vars/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automation/vars/main.yml b/automation/vars/main.yml index f9957f092..5802ee0d4 100644 --- a/automation/vars/main.yml +++ b/automation/vars/main.yml @@ -236,8 +236,8 @@ postgresql_parameters: - { option: "superuser_reserved_connections", value: "5" } - { option: "password_encryption", value: "{{ postgresql_password_encryption_algorithm }}" } - { option: "ssl", value: "on"} - - { option: "ssl_cert_file", value: "{{ postgresql_home_dir }}/{{ postgresql_version }}/server.crt"} - - { option: "ssl_key_file", value: "{{ postgresql_home_dir }}/{{ postgresql_version }}/server.key"} + - { option: "ssl_cert_file", value: "{{ postgresql_conf_dir }}/server.crt"} + - { option: "ssl_key_file", value: "{{ postgresql_conf_dir }}/server.key"} - { option: "ssl_min_protocol_version", value: "TLSv1.2"} - { option: "max_locks_per_transaction", value: "512" } - { option: "max_prepared_transactions", value: "0" } From cbae342bd1c7fecdab8f3be3354804b2f79a84b2 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Tue, 17 Dec 2024 12:02:59 +0500 Subject: [PATCH 08/23] add a condition for TLS --- automation/roles/patroni/tasks/main.yml | 36 ++++++++++++----------- automation/roles/pgbouncer/tasks/main.yml | 36 ++++++++++++----------- 2 files changed, 38 insertions(+), 34 deletions(-) diff --git a/automation/roles/patroni/tasks/main.yml b/automation/roles/patroni/tasks/main.yml index f87437ca3..f26a01ac8 100644 --- a/automation/roles/patroni/tasks/main.yml +++ b/automation/roles/patroni/tasks/main.yml @@ -428,23 +428,25 @@ mode: "0700" # Generating TLS Certificates - - name: Generate a private key for TLS - become: true - become_user: postgres - community.crypto.openssl_privatekey: - path: "{{ postgresql_conf_dir }}/server.key" - size: '4096' - mode: '0400' - - - name: Generate a self-signed certificate for Postgres - community.crypto.x509_certificate: - path: "{{ postgresql_conf_dir }}/server.crt" - privatekey_path: "{{ postgresql_conf_dir }}/server.key" - owner: postgres - group: postgres - mode: '0644' - provider: selfsigned - entrust_not_after: "+1000d" + - block: + - name: Generate a private key for TLS + become: true + become_user: postgres + community.crypto.openssl_privatekey: + path: "{{ postgresql_conf_dir }}/server.key" + size: '4096' + mode: '0400' + + - name: Generate a self-signed certificate for Postgres + community.crypto.x509_certificate: + path: "{{ postgresql_conf_dir }}/server.crt" + privatekey_path: "{{ postgresql_conf_dir }}/server.key" + owner: postgres + group: postgres + mode: '0644' + provider: selfsigned + entrust_not_after: "+1000d" + when: (postgresql_parameters | selectattr('option', 'equalto', 'ssl') | map(attribute='value') | first | default('off')) == 'on' # for Debian based distros only # patroni bootstrap failure is possible if the PostgreSQL config files are missing diff --git a/automation/roles/pgbouncer/tasks/main.yml b/automation/roles/pgbouncer/tasks/main.yml index 7e76c6b8f..7dd70e281 100644 --- a/automation/roles/pgbouncer/tasks/main.yml +++ b/automation/roles/pgbouncer/tasks/main.yml @@ -125,23 +125,25 @@ tags: pgbouncer_logrotate, pgbouncer # Generating TLS Certificates -- name: Generate a private key for TLS - become: true - become_user: postgres - community.crypto.openssl_privatekey: - path: "{{ pgbouncer_conf_dir }}/server.key" - size: '4096' - mode: '0400' - -- name: Generate a self-signed certificate for PGBouncer - community.crypto.x509_certificate: - path: "{{ pgbouncer_conf_dir }}/server.crt" - privatekey_path: "{{ pgbouncer_conf_dir }}/server.key" - owner: postgres - group: postgres - mode: '0644' - provider: selfsigned - entrust_not_after: "+1000d" +- block: + - name: Generate a private key for TLS + become: true + become_user: postgres + community.crypto.openssl_privatekey: + path: "{{ pgbouncer_conf_dir }}/server.key" + size: '4096' + mode: '0400' + + - name: Generate a self-signed certificate for PGBouncer + community.crypto.x509_certificate: + path: "{{ pgbouncer_conf_dir }}/server.crt" + privatekey_path: "{{ pgbouncer_conf_dir }}/server.key" + owner: postgres + group: postgres + mode: '0644' + provider: selfsigned + entrust_not_after: "+1000d" + when: pgbouncer_server_tls_sslmode | default('disable') != 'disable' - name: Configure pgbouncer.ini ansible.builtin.template: From 6878b913349dcc8b149cb2e58666e6f416fd54e7 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Tue, 17 Dec 2024 12:16:17 +0500 Subject: [PATCH 09/23] Move "Generating TLS Certificates" block --- automation/roles/patroni/tasks/main.yml | 42 ++++++++++++------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/automation/roles/patroni/tasks/main.yml b/automation/roles/patroni/tasks/main.yml index f26a01ac8..f97af3c1d 100644 --- a/automation/roles/patroni/tasks/main.yml +++ b/automation/roles/patroni/tasks/main.yml @@ -427,27 +427,6 @@ state: directory mode: "0700" - # Generating TLS Certificates - - block: - - name: Generate a private key for TLS - become: true - become_user: postgres - community.crypto.openssl_privatekey: - path: "{{ postgresql_conf_dir }}/server.key" - size: '4096' - mode: '0400' - - - name: Generate a self-signed certificate for Postgres - community.crypto.x509_certificate: - path: "{{ postgresql_conf_dir }}/server.crt" - privatekey_path: "{{ postgresql_conf_dir }}/server.key" - owner: postgres - group: postgres - mode: '0644' - provider: selfsigned - entrust_not_after: "+1000d" - when: (postgresql_parameters | selectattr('option', 'equalto', 'ssl') | map(attribute='value') | first | default('off')) == 'on' - # for Debian based distros only # patroni bootstrap failure is possible if the PostgreSQL config files are missing - name: Prepare PostgreSQL | make sure PostgreSQL config directory exists @@ -618,6 +597,27 @@ when: postgresql_exists | bool and patroni_cluster_bootstrap_method == "initdb" tags: patroni, patroni_start_master +# Generating TLS Certificates +- block: + - name: Prepare PostgreSQL | Generate a private key for TLS + become: true + become_user: postgres + community.crypto.openssl_privatekey: + path: "{{ postgresql_conf_dir }}/server.key" + size: '4096' + mode: '0400' + + - name: Prepare PostgreSQL | Generate a self-signed certificate for Postgres + community.crypto.x509_certificate: + path: "{{ postgresql_conf_dir }}/server.crt" + privatekey_path: "{{ postgresql_conf_dir }}/server.key" + owner: postgres + group: postgres + mode: '0644' + provider: selfsigned + entrust_not_after: "+1000d" + when: (postgresql_parameters | selectattr('option', 'equalto', 'ssl') | map(attribute='value') | first | default('off')) == 'on' + - block: # PITR (custom bootstrap) # Prepare (install pexpect, ruamel.yaml) - name: Prepare | Make sure the ansible required python library is exist From 16e7ea994a2a6ea79c15747c3b009bf23ab1bce9 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Tue, 17 Dec 2024 12:16:53 +0500 Subject: [PATCH 10/23] Update main.yml --- automation/roles/patroni/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automation/roles/patroni/tasks/main.yml b/automation/roles/patroni/tasks/main.yml index f97af3c1d..28353983f 100644 --- a/automation/roles/patroni/tasks/main.yml +++ b/automation/roles/patroni/tasks/main.yml @@ -599,7 +599,7 @@ # Generating TLS Certificates - block: - - name: Prepare PostgreSQL | Generate a private key for TLS + - name: Generate a private key for TLS become: true become_user: postgres community.crypto.openssl_privatekey: @@ -607,7 +607,7 @@ size: '4096' mode: '0400' - - name: Prepare PostgreSQL | Generate a self-signed certificate for Postgres + - name: Generate a self-signed certificate for Postgres community.crypto.x509_certificate: path: "{{ postgresql_conf_dir }}/server.crt" privatekey_path: "{{ postgresql_conf_dir }}/server.key" From 828c548b287597cad7c978ab5750a6a7d6020670 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Tue, 17 Dec 2024 12:20:10 +0500 Subject: [PATCH 11/23] Update main.yml --- automation/roles/patroni/tasks/main.yml | 42 ++++++++++++------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/automation/roles/patroni/tasks/main.yml b/automation/roles/patroni/tasks/main.yml index 28353983f..f21066324 100644 --- a/automation/roles/patroni/tasks/main.yml +++ b/automation/roles/patroni/tasks/main.yml @@ -597,27 +597,6 @@ when: postgresql_exists | bool and patroni_cluster_bootstrap_method == "initdb" tags: patroni, patroni_start_master -# Generating TLS Certificates -- block: - - name: Generate a private key for TLS - become: true - become_user: postgres - community.crypto.openssl_privatekey: - path: "{{ postgresql_conf_dir }}/server.key" - size: '4096' - mode: '0400' - - - name: Generate a self-signed certificate for Postgres - community.crypto.x509_certificate: - path: "{{ postgresql_conf_dir }}/server.crt" - privatekey_path: "{{ postgresql_conf_dir }}/server.key" - owner: postgres - group: postgres - mode: '0644' - provider: selfsigned - entrust_not_after: "+1000d" - when: (postgresql_parameters | selectattr('option', 'equalto', 'ssl') | map(attribute='value') | first | default('off')) == 'on' - - block: # PITR (custom bootstrap) # Prepare (install pexpect, ruamel.yaml) - name: Prepare | Make sure the ansible required python library is exist @@ -837,6 +816,27 @@ (existing_pgcluster is not defined or not existing_pgcluster|bool) tags: patroni +# Generating TLS Certificates +- block: + - name: Generate a private key for TLS (if not exist) + become: true + become_user: postgres + community.crypto.openssl_privatekey: + path: "{{ postgresql_conf_dir }}/server.key" + size: '4096' + mode: '0400' + + - name: Generate a self-signed certificate for Postgres (if not exist) + community.crypto.x509_certificate: + path: "{{ postgresql_conf_dir }}/server.crt" + privatekey_path: "{{ postgresql_conf_dir }}/server.key" + owner: postgres + group: postgres + mode: '0644' + provider: selfsigned + entrust_not_after: "+1000d" + when: (postgresql_parameters | selectattr('option', 'equalto', 'ssl') | map(attribute='value') | first | default('off')) == 'on' + - block: # start patroni on master - name: Start patroni service on the Master server ansible.builtin.systemd: From a9b8ed62d3064b2e98452d91fd8f36a192c9e72f Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Tue, 17 Dec 2024 12:22:48 +0500 Subject: [PATCH 12/23] Add pgbouncer_server_tls_ca_file variable --- automation/roles/pgbouncer/templates/pgbouncer.ini.j2 | 1 + automation/vars/main.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/automation/roles/pgbouncer/templates/pgbouncer.ini.j2 b/automation/roles/pgbouncer/templates/pgbouncer.ini.j2 index f2b6bd56d..434037d5c 100644 --- a/automation/roles/pgbouncer/templates/pgbouncer.ini.j2 +++ b/automation/roles/pgbouncer/templates/pgbouncer.ini.j2 @@ -47,6 +47,7 @@ server_tls_protocols = {{ pgbouncer_server_tls_protocols }} server_tls_ciphers = {{ pgbouncer_server_tls_ciphers }} server_tls_cert_file = {{ pgbouncer_server_tls_cert_file }} server_tls_key_file = {{ pgbouncer_server_tls_key_file }} +server_tls_ca_file = {{ pgbouncer_server_tls_ca_file }} {% endif %} log_connections = 0 log_disconnections = 0 diff --git a/automation/vars/main.yml b/automation/vars/main.yml index 5802ee0d4..e66193563 100644 --- a/automation/vars/main.yml +++ b/automation/vars/main.yml @@ -381,6 +381,7 @@ pgbouncer_server_tls_protocols: "secure" pgbouncer_server_tls_ciphers: "secure" pgbouncer_server_tls_cert_file: "{{ pgbouncer_conf_dir }}/server.crt" pgbouncer_server_tls_key_file: "{{ pgbouncer_conf_dir }}/server.key" +pgbouncer_server_tls_ca_file: "" pgbouncer_pools: - { name: "postgres", dbname: "postgres", pool_parameters: "" } From a641891d08be6a46c4f9046d0210da6c55c99016 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Tue, 17 Dec 2024 12:25:59 +0500 Subject: [PATCH 13/23] Update pgbouncer.ini.j2 --- automation/roles/pgbouncer/templates/pgbouncer.ini.j2 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/automation/roles/pgbouncer/templates/pgbouncer.ini.j2 b/automation/roles/pgbouncer/templates/pgbouncer.ini.j2 index 434037d5c..8bac8c6e3 100644 --- a/automation/roles/pgbouncer/templates/pgbouncer.ini.j2 +++ b/automation/roles/pgbouncer/templates/pgbouncer.ini.j2 @@ -39,16 +39,22 @@ so_reuseport = 1 client_tls_sslmode = {{ pgbouncer_client_tls_sslmode }} client_tls_key_file = {{ pgbouncer_client_tls_key_file }} client_tls_cert_file = {{ pgbouncer_client_tls_cert_file }} +{% if pgbouncer_client_tls_ca_file | default('') | length > 0 %} client_tls_ca_file = {{ pgbouncer_client_tls_ca_file }} +{% endif %} client_tls_protocols = {{ pgbouncer_client_tls_protocols }} client_tls_ciphers = {{ pgbouncer_client_tls_ciphers }} +{% endif %} +{% if pgbouncer_server_tls_sslmode != 'disable' %} server_tls_sslmode = {{ pgbouncer_server_tls_sslmode }} server_tls_protocols = {{ pgbouncer_server_tls_protocols }} server_tls_ciphers = {{ pgbouncer_server_tls_ciphers }} server_tls_cert_file = {{ pgbouncer_server_tls_cert_file }} server_tls_key_file = {{ pgbouncer_server_tls_key_file }} +{% if pgbouncer_server_tls_ca_file | default('') | length > 0 %} server_tls_ca_file = {{ pgbouncer_server_tls_ca_file }} {% endif %} +{% endif %} log_connections = 0 log_disconnections = 0 From 2c84108ec54f49177c00890179b99342d1dd89b1 Mon Sep 17 00:00:00 2001 From: klention Date: Tue, 17 Dec 2024 13:30:35 +0100 Subject: [PATCH 14/23] Generate TLS cert only on PGBouncer role --- automation/roles/patroni/tasks/main.yml | 21 --------------------- automation/roles/pgbouncer/tasks/main.yml | 12 ++++++------ automation/vars/main.yml | 4 ++-- 3 files changed, 8 insertions(+), 29 deletions(-) diff --git a/automation/roles/patroni/tasks/main.yml b/automation/roles/patroni/tasks/main.yml index f21066324..d6ca5cf62 100644 --- a/automation/roles/patroni/tasks/main.yml +++ b/automation/roles/patroni/tasks/main.yml @@ -816,27 +816,6 @@ (existing_pgcluster is not defined or not existing_pgcluster|bool) tags: patroni -# Generating TLS Certificates -- block: - - name: Generate a private key for TLS (if not exist) - become: true - become_user: postgres - community.crypto.openssl_privatekey: - path: "{{ postgresql_conf_dir }}/server.key" - size: '4096' - mode: '0400' - - - name: Generate a self-signed certificate for Postgres (if not exist) - community.crypto.x509_certificate: - path: "{{ postgresql_conf_dir }}/server.crt" - privatekey_path: "{{ postgresql_conf_dir }}/server.key" - owner: postgres - group: postgres - mode: '0644' - provider: selfsigned - entrust_not_after: "+1000d" - when: (postgresql_parameters | selectattr('option', 'equalto', 'ssl') | map(attribute='value') | first | default('off')) == 'on' - - block: # start patroni on master - name: Start patroni service on the Master server ansible.builtin.systemd: diff --git a/automation/roles/pgbouncer/tasks/main.yml b/automation/roles/pgbouncer/tasks/main.yml index 7dd70e281..95aa7b75f 100644 --- a/automation/roles/pgbouncer/tasks/main.yml +++ b/automation/roles/pgbouncer/tasks/main.yml @@ -126,24 +126,24 @@ # Generating TLS Certificates - block: - - name: Generate a private key for TLS + - name: Generate a private TLS key for PostgreSQL & PGBouncer become: true become_user: postgres community.crypto.openssl_privatekey: path: "{{ pgbouncer_conf_dir }}/server.key" - size: '4096' - mode: '0400' + size: "4096" + mode: "0400" - - name: Generate a self-signed certificate for PGBouncer + - name: Generate a self-signed TLS certificate for PostgreSQL & PGBouncer community.crypto.x509_certificate: path: "{{ pgbouncer_conf_dir }}/server.crt" privatekey_path: "{{ pgbouncer_conf_dir }}/server.key" owner: postgres group: postgres - mode: '0644' + mode: "0644" provider: selfsigned entrust_not_after: "+1000d" - when: pgbouncer_server_tls_sslmode | default('disable') != 'disable' + when: (pgbouncer_server_tls_sslmode | default('disable') != 'disable') or ((postgresql_parameters | selectattr('option', 'equalto', 'ssl') | map(attribute='value') | first | default('off')) == 'on') - name: Configure pgbouncer.ini ansible.builtin.template: diff --git a/automation/vars/main.yml b/automation/vars/main.yml index e66193563..a2a5ca04b 100644 --- a/automation/vars/main.yml +++ b/automation/vars/main.yml @@ -236,8 +236,8 @@ postgresql_parameters: - { option: "superuser_reserved_connections", value: "5" } - { option: "password_encryption", value: "{{ postgresql_password_encryption_algorithm }}" } - { option: "ssl", value: "on"} - - { option: "ssl_cert_file", value: "{{ postgresql_conf_dir }}/server.crt"} - - { option: "ssl_key_file", value: "{{ postgresql_conf_dir }}/server.key"} + - { option: "ssl_cert_file", value: "{{ pgbouncer_conf_dir }}/server.crt"} + - { option: "ssl_key_file", value: "{{ pgbouncer_conf_dir }}/server.key"} - { option: "ssl_min_protocol_version", value: "TLSv1.2"} - { option: "max_locks_per_transaction", value: "512" } - { option: "max_prepared_transactions", value: "0" } From eff28c19db4bcd84e7afbea9aca458e0f03cb09e Mon Sep 17 00:00:00 2001 From: klention Date: Wed, 18 Dec 2024 00:14:15 +0100 Subject: [PATCH 15/23] Moved TLS certificate generation tasks to its own role - tls_certificate --- automation/roles/pgbouncer/tasks/main.yml | 28 ++++++------------- .../roles/tls_certificate/tasks/main.yml | 22 +++++++++++++++ 2 files changed, 31 insertions(+), 19 deletions(-) create mode 100644 automation/roles/tls_certificate/tasks/main.yml diff --git a/automation/roles/pgbouncer/tasks/main.yml b/automation/roles/pgbouncer/tasks/main.yml index 95aa7b75f..1d962c494 100644 --- a/automation/roles/pgbouncer/tasks/main.yml +++ b/automation/roles/pgbouncer/tasks/main.yml @@ -124,25 +124,15 @@ label: "{{ 'pgbouncer' if idx == 0 else 'pgbouncer-%d' % (idx + 1) }}" tags: pgbouncer_logrotate, pgbouncer -# Generating TLS Certificates -- block: - - name: Generate a private TLS key for PostgreSQL & PGBouncer - become: true - become_user: postgres - community.crypto.openssl_privatekey: - path: "{{ pgbouncer_conf_dir }}/server.key" - size: "4096" - mode: "0400" - - - name: Generate a self-signed TLS certificate for PostgreSQL & PGBouncer - community.crypto.x509_certificate: - path: "{{ pgbouncer_conf_dir }}/server.crt" - privatekey_path: "{{ pgbouncer_conf_dir }}/server.key" - owner: postgres - group: postgres - mode: "0644" - provider: selfsigned - entrust_not_after: "+1000d" +- name: Generate TLS certificate... + ansible.builtin.import_role: + name: tls_certificate + tasks_from: main + vars: + tls_title: "PostgreSQL & PGBouncer" + tls_privatekey_path: "{{ pgbouncer_conf_dir }}/server.key" + tls_cert_path: "{{ pgbouncer_conf_dir }}/server.crt" + tls_owner: "postgres" when: (pgbouncer_server_tls_sslmode | default('disable') != 'disable') or ((postgresql_parameters | selectattr('option', 'equalto', 'ssl') | map(attribute='value') | first | default('off')) == 'on') - name: Configure pgbouncer.ini diff --git a/automation/roles/tls_certificate/tasks/main.yml b/automation/roles/tls_certificate/tasks/main.yml new file mode 100644 index 000000000..02e634d73 --- /dev/null +++ b/automation/roles/tls_certificate/tasks/main.yml @@ -0,0 +1,22 @@ +--- +- name: Generate private TLS key for {{ tls_title }} + community.crypto.openssl_privatekey: + path: "{{ tls_privatekey_path }}" + size: "4096" + +- name: Set the owner of the private TLS key {{ tls_privatekey_path }} to {{ tls_owner }} + ansible.builtin.file: + path: "{{ tls_privatekey_path }}" + owner: "{{ tls_owner }}" + group: "{{ tls_owner }}" + mode: "0400" + +- name: Generate self-signed TLS certificate for {{ tls_title }} + community.crypto.x509_certificate: + path: "{{ tls_cert_path }}" + privatekey_path: "{{ tls_privatekey_path }}" + owner: "{{ tls_owner }}" + group: "{{ tls_owner }}" + mode: "0644" + provider: selfsigned + entrust_not_after: "+1000d" From 0aed35fbcef6919a78c8dac7aef538ef21bb9466 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Wed, 18 Dec 2024 13:21:04 +0500 Subject: [PATCH 16/23] tls_certificate role improvements --- automation/deploy_pgcluster.yml | 3 ++ automation/roles/pgbouncer/tasks/main.yml | 11 ----- .../roles/tls_certificate/tasks/main.yml | 44 +++++++++++-------- automation/vars/main.yml | 17 ++++--- 4 files changed, 40 insertions(+), 35 deletions(-) diff --git a/automation/deploy_pgcluster.yml b/automation/deploy_pgcluster.yml index 0968f1fa6..cf2aed8b8 100644 --- a/automation/deploy_pgcluster.yml +++ b/automation/deploy_pgcluster.yml @@ -356,6 +356,9 @@ - role: cron + - role: tls_certificate + when: tls_cert_generate|bool + - role: pgbouncer when: pgbouncer_install|bool diff --git a/automation/roles/pgbouncer/tasks/main.yml b/automation/roles/pgbouncer/tasks/main.yml index 1d962c494..cd2fb08fe 100644 --- a/automation/roles/pgbouncer/tasks/main.yml +++ b/automation/roles/pgbouncer/tasks/main.yml @@ -124,17 +124,6 @@ label: "{{ 'pgbouncer' if idx == 0 else 'pgbouncer-%d' % (idx + 1) }}" tags: pgbouncer_logrotate, pgbouncer -- name: Generate TLS certificate... - ansible.builtin.import_role: - name: tls_certificate - tasks_from: main - vars: - tls_title: "PostgreSQL & PGBouncer" - tls_privatekey_path: "{{ pgbouncer_conf_dir }}/server.key" - tls_cert_path: "{{ pgbouncer_conf_dir }}/server.crt" - tls_owner: "postgres" - when: (pgbouncer_server_tls_sslmode | default('disable') != 'disable') or ((postgresql_parameters | selectattr('option', 'equalto', 'ssl') | map(attribute='value') | first | default('off')) == 'on') - - name: Configure pgbouncer.ini ansible.builtin.template: src: templates/pgbouncer.ini.j2 diff --git a/automation/roles/tls_certificate/tasks/main.yml b/automation/roles/tls_certificate/tasks/main.yml index 02e634d73..135af83d2 100644 --- a/automation/roles/tls_certificate/tasks/main.yml +++ b/automation/roles/tls_certificate/tasks/main.yml @@ -1,22 +1,30 @@ --- -- name: Generate private TLS key for {{ tls_title }} - community.crypto.openssl_privatekey: - path: "{{ tls_privatekey_path }}" - size: "4096" - -- name: Set the owner of the private TLS key {{ tls_privatekey_path }} to {{ tls_owner }} +- name: "Ensure TLS directories exist" ansible.builtin.file: - path: "{{ tls_privatekey_path }}" - owner: "{{ tls_owner }}" - group: "{{ tls_owner }}" - mode: "0400" + path: "{{ item }}" + state: directory + owner: "{{ tls_owner | default('postgres') }}" + group: "{{ tls_owner | default('postgres') }}" + mode: "0750" + loop: + - "{{ tls_privatekey_path | default('/etc/tls/server.key') | dirname }}" + - "{{ tls_cert_path | default('/etc/tls/server.crt') | dirname }}" + +- name: "Generate private TLS key {{ tls_privatekey_path | default('/etc/tls/server.key') }}" + community.crypto.openssl_privatekey: + path: "{{ tls_privatekey_path | default('/etc/tls/server.key') }}" + owner: "{{ tls_owner | default('postgres') }}" + group: "{{ tls_owner | default('postgres') }}" + mode: "{{ tls_privatekey_mode | default('0400') }}" + size: "{{ tls_privatekey_size | default(4096) }}" + type: "{{ tls_privatekey_type | default('RSA') }}" -- name: Generate self-signed TLS certificate for {{ tls_title }} +- name: "Generate self-signed TLS certificate {{ tls_cert_path | default('/etc/tls/server.crt') }}" community.crypto.x509_certificate: - path: "{{ tls_cert_path }}" - privatekey_path: "{{ tls_privatekey_path }}" - owner: "{{ tls_owner }}" - group: "{{ tls_owner }}" - mode: "0644" - provider: selfsigned - entrust_not_after: "+1000d" + path: "{{ tls_cert_path | default('/etc/tls/server.crt') }}" + privatekey_path: "{{ tls_privatekey_path | default('/etc/tls/server.key') }}" + owner: "{{ tls_owner | default('postgres') }}" + group: "{{ tls_owner | default('postgres') }}" + mode: "{{ tls_cert_mode | default('0644') }}" + provider: "{{ tls_cert_provider | default('selfsigned') }}" + entrust_not_after: "{{ tls_cert_entrust_not_after | default('+3650d') }}" diff --git a/automation/vars/main.yml b/automation/vars/main.yml index a2a5ca04b..cddb53db6 100644 --- a/automation/vars/main.yml +++ b/automation/vars/main.yml @@ -174,6 +174,11 @@ consul_services: # - { http: "http://{{ inventory_hostname }}:{{ patroni_restapi_port }}/async?lag={{ patroni_maximum_lag_on_replica }}", interval: "2s" } # - { args: ["systemctl", "status", "pgbouncer"], interval: "5s" } +# TLS certificate (for PostgreSQL & PGBouncer) +tls_cert_generate: true +tls_cert_path: "{{ postgresql_home_dir }}/tls/server.crt" +tls_privatekey_path: "{{ postgresql_home_dir }}/tls/server.key" +tls_owner: "postgres" # PostgreSQL variables postgresql_version: 17 @@ -236,8 +241,8 @@ postgresql_parameters: - { option: "superuser_reserved_connections", value: "5" } - { option: "password_encryption", value: "{{ postgresql_password_encryption_algorithm }}" } - { option: "ssl", value: "on"} - - { option: "ssl_cert_file", value: "{{ pgbouncer_conf_dir }}/server.crt"} - - { option: "ssl_key_file", value: "{{ pgbouncer_conf_dir }}/server.key"} + - { option: "ssl_cert_file", value: "{{ tls_cert_path }}"} + - { option: "ssl_key_file", value: "{{ tls_privatekey_path }}"} - { option: "ssl_min_protocol_version", value: "TLSv1.2"} - { option: "max_locks_per_transaction", value: "512" } - { option: "max_prepared_transactions", value: "0" } @@ -371,16 +376,16 @@ pgbouncer_auth_username: pgbouncer # user who can query the database via the use pgbouncer_auth_password: "" # If not defined, a password will be generated automatically during deployment pgbouncer_auth_dbname: "postgres" pgbouncer_client_tls_sslmode: "require" -pgbouncer_client_tls_key_file: "{{ pgbouncer_conf_dir }}/server.key" -pgbouncer_client_tls_cert_file: "{{ pgbouncer_conf_dir }}/server.crt" +pgbouncer_client_tls_key_file: "{{ tls_privatekey_path }}" +pgbouncer_client_tls_cert_file: "{{ tls_cert_path }}" pgbouncer_client_tls_ca_file: "" pgbouncer_client_tls_protocols: "secure" # allowed values: tlsv1.0, tlsv1.1, tlsv1.2, tlsv1.3, all, secure (tlsv1.2,tlsv1.3) pgbouncer_client_tls_ciphers: "secure" # allowed values: default, secure, fast, normal, all (not recommended) pgbouncer_server_tls_sslmode: "require" pgbouncer_server_tls_protocols: "secure" pgbouncer_server_tls_ciphers: "secure" -pgbouncer_server_tls_cert_file: "{{ pgbouncer_conf_dir }}/server.crt" -pgbouncer_server_tls_key_file: "{{ pgbouncer_conf_dir }}/server.key" +pgbouncer_server_tls_cert_file: "{{ tls_cert_path }}" +pgbouncer_server_tls_key_file: "{{ tls_privatekey_path }}" pgbouncer_server_tls_ca_file: "" pgbouncer_pools: From dc7196abb128e45f90f872416a885c47d8064f28 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Wed, 18 Dec 2024 13:36:50 +0500 Subject: [PATCH 17/23] tls_cert_valid_days --- automation/roles/tls_certificate/tasks/main.yml | 2 +- automation/vars/main.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/automation/roles/tls_certificate/tasks/main.yml b/automation/roles/tls_certificate/tasks/main.yml index 135af83d2..a032e6cbd 100644 --- a/automation/roles/tls_certificate/tasks/main.yml +++ b/automation/roles/tls_certificate/tasks/main.yml @@ -27,4 +27,4 @@ group: "{{ tls_owner | default('postgres') }}" mode: "{{ tls_cert_mode | default('0644') }}" provider: "{{ tls_cert_provider | default('selfsigned') }}" - entrust_not_after: "{{ tls_cert_entrust_not_after | default('+3650d') }}" + entrust_not_after: "+{{ tls_cert_valid_days | default(3650) }}d" diff --git a/automation/vars/main.yml b/automation/vars/main.yml index cddb53db6..4eb600936 100644 --- a/automation/vars/main.yml +++ b/automation/vars/main.yml @@ -176,6 +176,7 @@ consul_services: # TLS certificate (for PostgreSQL & PGBouncer) tls_cert_generate: true +tls_cert_valid_days: 3650 tls_cert_path: "{{ postgresql_home_dir }}/tls/server.crt" tls_privatekey_path: "{{ postgresql_home_dir }}/tls/server.key" tls_owner: "postgres" From 4412bd1e285b433339947eafc5bdcb867723609b Mon Sep 17 00:00:00 2001 From: klention Date: Wed, 18 Dec 2024 13:48:42 +0100 Subject: [PATCH 18/23] Reverted max_worker_processes and max_parallel_workers to their original values --- automation/vars/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automation/vars/main.yml b/automation/vars/main.yml index 4eb600936..22e6f5b41 100644 --- a/automation/vars/main.yml +++ b/automation/vars/main.yml @@ -315,8 +315,8 @@ postgresql_parameters: - { option: "wal_receiver_status_interval", value: "10s" } - { option: "idle_in_transaction_session_timeout", value: "10min" } # reduce this timeout if possible - { option: "jit", value: "off" } - - { option: "max_worker_processes", value: "{{ [ansible_processor_vcpus | int, 16] | max }}" } - - { option: "max_parallel_workers", value: "{{ [(ansible_processor_vcpus | int // 2), 8] | max }}" } + - { option: "max_worker_processes", value: "24" } + - { option: "max_parallel_workers", value: "8" } - { option: "max_parallel_workers_per_gather", value: "2" } - { option: "max_parallel_maintenance_workers", value: "2" } - { option: "tcp_keepalives_count", value: "10" } From 3097f7ae1bd4bc4cffd043d06839dd7cba59a4ea Mon Sep 17 00:00:00 2001 From: klention Date: Wed, 18 Dec 2024 19:55:14 +0100 Subject: [PATCH 19/23] Copy TLS certificate to the newly added nodes --- automation/add_pgnode.yml | 16 +++++++ .../tasks/copy_certificate.yml | 47 +++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 automation/roles/tls_certificate/tasks/copy_certificate.yml diff --git a/automation/add_pgnode.yml b/automation/add_pgnode.yml index 08127325c..b3095a22a 100644 --- a/automation/add_pgnode.yml +++ b/automation/add_pgnode.yml @@ -105,6 +105,14 @@ - firewall_enabled_at_boot | bool tags: firewall + - name: Fetch TLS certificate + ansible.builtin.include_role: + name: tls_certificate + tasks_from: copy_certificate + vars: + tls_job: fetch + when: tls_cert_generate|bool + roles: - role: ansible-role-firewall environment: "{{ proxy_env | default({}) }}" @@ -228,6 +236,14 @@ ansible.builtin.include_vars: "vars/{{ ansible_os_family }}.yml" tags: always + - name: Copy TLS certificate + ansible.builtin.include_role: + name: tls_certificate + tasks_from: copy_certificate + vars: + tls_job: copy + when: tls_cert_generate|bool + roles: - role: wal-g when: wal_g_install|bool diff --git a/automation/roles/tls_certificate/tasks/copy_certificate.yml b/automation/roles/tls_certificate/tasks/copy_certificate.yml new file mode 100644 index 000000000..e42bd60d1 --- /dev/null +++ b/automation/roles/tls_certificate/tasks/copy_certificate.yml @@ -0,0 +1,47 @@ +--- +- name: Fetch TLS certificate and key + run_once: true + ansible.builtin.fetch: + src: "{{ item }}" + dest: "/tmp/tls/" + flat: yes + loop: + - "{{ tls_privatekey_path | default('/etc/tls/server.key') }}" + - "{{ tls_cert_path | default('/etc/tls/server.crt') }}" + when: + - tls_job is defined + - tls_job == 'fetch' + +- block: + - name: Ensure TLS directories exist + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: "{{ tls_owner | default('postgres') }}" + group: "{{ tls_owner | default('postgres') }}" + mode: "0750" + loop: + - "{{ tls_privatekey_path | default('/etc/tls/server.key') | dirname }}" + - "{{ tls_cert_path | default('/etc/tls/server.crt') | dirname }}" + + - name: Copy TLS certificate and key to the new node + ansible.builtin.copy: + src: "/tmp/tls/{{ item | basename }}" + dest: "{{ item }}" + loop: + - "{{ tls_privatekey_path | default('/etc/tls/server.key') }}" + - "{{ tls_cert_path | default('/etc/tls/server.crt') }}" + + - name: Set proper permissions for TLS files + ansible.builtin.file: + path: "{{ item.path }}" + owner: "{{ tls_owner | default('postgres') }}" + group: "{{ tls_owner | default('postgres') }}" + mode: "{{ item.mode }}" + state: file + loop: + - { path: "{{ tls_privatekey_path | default('/etc/tls/server.key') }}", mode: "0400" } + - { path: "{{ tls_cert_path | default('/etc/tls/server.crt') }}", mode: "0644" } + when: + - tls_job is defined + - tls_job == 'copy' From 5069421f391df33ef80f3d591fede4cab0923f38 Mon Sep 17 00:00:00 2001 From: klention Date: Thu, 19 Dec 2024 00:05:28 +0100 Subject: [PATCH 20/23] Fixed linting --- automation/add_pgnode.yml | 4 +- .../tasks/copy_certificate.yml | 62 ++++++++++--------- 2 files changed, 36 insertions(+), 30 deletions(-) diff --git a/automation/add_pgnode.yml b/automation/add_pgnode.yml index b3095a22a..61ec658dd 100644 --- a/automation/add_pgnode.yml +++ b/automation/add_pgnode.yml @@ -111,7 +111,7 @@ tasks_from: copy_certificate vars: tls_job: fetch - when: tls_cert_generate|bool + when: tls_cert_generate | bool roles: - role: ansible-role-firewall @@ -242,7 +242,7 @@ tasks_from: copy_certificate vars: tls_job: copy - when: tls_cert_generate|bool + when: tls_cert_generate | bool roles: - role: wal-g diff --git a/automation/roles/tls_certificate/tasks/copy_certificate.yml b/automation/roles/tls_certificate/tasks/copy_certificate.yml index e42bd60d1..f86ae32c8 100644 --- a/automation/roles/tls_certificate/tasks/copy_certificate.yml +++ b/automation/roles/tls_certificate/tasks/copy_certificate.yml @@ -4,7 +4,7 @@ ansible.builtin.fetch: src: "{{ item }}" dest: "/tmp/tls/" - flat: yes + flat: true loop: - "{{ tls_privatekey_path | default('/etc/tls/server.key') }}" - "{{ tls_cert_path | default('/etc/tls/server.crt') }}" @@ -13,35 +13,41 @@ - tls_job == 'fetch' - block: - - name: Ensure TLS directories exist - ansible.builtin.file: - path: "{{ item }}" - state: directory - owner: "{{ tls_owner | default('postgres') }}" - group: "{{ tls_owner | default('postgres') }}" - mode: "0750" - loop: - - "{{ tls_privatekey_path | default('/etc/tls/server.key') | dirname }}" - - "{{ tls_cert_path | default('/etc/tls/server.crt') | dirname }}" + - name: Ensure TLS directories exist + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: "{{ tls_owner | default('postgres') }}" + group: "{{ tls_owner | default('postgres') }}" + mode: "0750" + loop: + - "{{ tls_privatekey_path | default('/etc/tls/server.key') | dirname }}" + - "{{ tls_cert_path | default('/etc/tls/server.crt') | dirname }}" - - name: Copy TLS certificate and key to the new node - ansible.builtin.copy: - src: "/tmp/tls/{{ item | basename }}" - dest: "{{ item }}" - loop: - - "{{ tls_privatekey_path | default('/etc/tls/server.key') }}" - - "{{ tls_cert_path | default('/etc/tls/server.crt') }}" + - name: Copy TLS certificate and key to the new node + ansible.builtin.copy: + src: "/tmp/tls/{{ item | basename }}" + dest: "{{ item }}" + loop: + - "{{ tls_privatekey_path | default('/etc/tls/server.key') }}" + - "{{ tls_cert_path | default('/etc/tls/server.crt') }}" - - name: Set proper permissions for TLS files - ansible.builtin.file: - path: "{{ item.path }}" - owner: "{{ tls_owner | default('postgres') }}" - group: "{{ tls_owner | default('postgres') }}" - mode: "{{ item.mode }}" - state: file - loop: - - { path: "{{ tls_privatekey_path | default('/etc/tls/server.key') }}", mode: "0400" } - - { path: "{{ tls_cert_path | default('/etc/tls/server.crt') }}", mode: "0644" } + - name: Set proper permissions for TLS files + ansible.builtin.file: + path: "{{ item.path }}" + owner: "{{ tls_owner | default('postgres') }}" + group: "{{ tls_owner | default('postgres') }}" + mode: "{{ item.mode }}" + state: file + loop: + - { path: "{{ tls_privatekey_path | default('/etc/tls/server.key') }}", mode: "0400" } + - { path: "{{ tls_cert_path | default('/etc/tls/server.crt') }}", mode: "0644" } + + - name: Delete TLS certificate and key from the ansible controller + ansible.builtin.file: + path: "/tmp/tls/" + state: absent + delegate_to: localhost when: - tls_job is defined - tls_job == 'copy' From fac17d416163a41eac1cbb5225fbeb8c551013ee Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Thu, 19 Dec 2024 13:29:10 +0500 Subject: [PATCH 21/23] update loop --- automation/roles/tls_certificate/tasks/copy_certificate.yml | 6 +++--- automation/roles/tls_certificate/tasks/main.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/automation/roles/tls_certificate/tasks/copy_certificate.yml b/automation/roles/tls_certificate/tasks/copy_certificate.yml index f86ae32c8..538248cb5 100644 --- a/automation/roles/tls_certificate/tasks/copy_certificate.yml +++ b/automation/roles/tls_certificate/tasks/copy_certificate.yml @@ -15,14 +15,14 @@ - block: - name: Ensure TLS directories exist ansible.builtin.file: - path: "{{ item }}" + path: "{{ item | dirname }}" state: directory owner: "{{ tls_owner | default('postgres') }}" group: "{{ tls_owner | default('postgres') }}" mode: "0750" loop: - - "{{ tls_privatekey_path | default('/etc/tls/server.key') | dirname }}" - - "{{ tls_cert_path | default('/etc/tls/server.crt') | dirname }}" + - "{{ tls_privatekey_path | default('/etc/tls/server.key') }}" + - "{{ tls_cert_path | default('/etc/tls/server.crt') }}" - name: Copy TLS certificate and key to the new node ansible.builtin.copy: diff --git a/automation/roles/tls_certificate/tasks/main.yml b/automation/roles/tls_certificate/tasks/main.yml index a032e6cbd..2f3021022 100644 --- a/automation/roles/tls_certificate/tasks/main.yml +++ b/automation/roles/tls_certificate/tasks/main.yml @@ -1,14 +1,14 @@ --- - name: "Ensure TLS directories exist" ansible.builtin.file: - path: "{{ item }}" + path: "{{ item | dirname }}" state: directory owner: "{{ tls_owner | default('postgres') }}" group: "{{ tls_owner | default('postgres') }}" mode: "0750" loop: - - "{{ tls_privatekey_path | default('/etc/tls/server.key') | dirname }}" - - "{{ tls_cert_path | default('/etc/tls/server.crt') | dirname }}" + - "{{ tls_privatekey_path | default('/etc/tls/server.key') }}" + - "{{ tls_cert_path | default('/etc/tls/server.crt') }}" - name: "Generate private TLS key {{ tls_privatekey_path | default('/etc/tls/server.key') }}" community.crypto.openssl_privatekey: From 94fff76655083c05fa034574abdc946525450ad7 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Thu, 19 Dec 2024 13:29:13 +0500 Subject: [PATCH 22/23] Update main.yml --- automation/vars/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/vars/main.yml b/automation/vars/main.yml index 22e6f5b41..7ef57c851 100644 --- a/automation/vars/main.yml +++ b/automation/vars/main.yml @@ -174,7 +174,7 @@ consul_services: # - { http: "http://{{ inventory_hostname }}:{{ patroni_restapi_port }}/async?lag={{ patroni_maximum_lag_on_replica }}", interval: "2s" } # - { args: ["systemctl", "status", "pgbouncer"], interval: "5s" } -# TLS certificate (for PostgreSQL & PGBouncer) +# TLS certificate (for PostgreSQL & PgBouncer) tls_cert_generate: true tls_cert_valid_days: 3650 tls_cert_path: "{{ postgresql_home_dir }}/tls/server.crt" From 66b35227b33263b9051c37561fbea6f85e4e8a15 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Thu, 19 Dec 2024 15:00:37 +0500 Subject: [PATCH 23/23] tls_certificate/copy --- automation/add_pgnode.yml | 19 ++----- .../roles/tls_certificate/copy/tasks/main.yml | 42 +++++++++++++++ .../tasks/copy_certificate.yml | 53 ------------------- .../roles/tls_certificate/tasks/main.yml | 2 +- 4 files changed, 46 insertions(+), 70 deletions(-) create mode 100644 automation/roles/tls_certificate/copy/tasks/main.yml delete mode 100644 automation/roles/tls_certificate/tasks/copy_certificate.yml diff --git a/automation/add_pgnode.yml b/automation/add_pgnode.yml index 61ec658dd..9dd7c15f1 100644 --- a/automation/add_pgnode.yml +++ b/automation/add_pgnode.yml @@ -105,14 +105,6 @@ - firewall_enabled_at_boot | bool tags: firewall - - name: Fetch TLS certificate - ansible.builtin.include_role: - name: tls_certificate - tasks_from: copy_certificate - vars: - tls_job: fetch - when: tls_cert_generate | bool - roles: - role: ansible-role-firewall environment: "{{ proxy_env | default({}) }}" @@ -236,14 +228,6 @@ ansible.builtin.include_vars: "vars/{{ ansible_os_family }}.yml" tags: always - - name: Copy TLS certificate - ansible.builtin.include_role: - name: tls_certificate - tasks_from: copy_certificate - vars: - tls_job: copy - when: tls_cert_generate | bool - roles: - role: wal-g when: wal_g_install|bool @@ -251,6 +235,9 @@ - role: pg_probackup when: pg_probackup_install|bool + - role: tls_certificate/copy + when: tls_cert_generate|bool + - role: pgbouncer when: pgbouncer_install|bool diff --git a/automation/roles/tls_certificate/copy/tasks/main.yml b/automation/roles/tls_certificate/copy/tasks/main.yml new file mode 100644 index 000000000..a0038ecc4 --- /dev/null +++ b/automation/roles/tls_certificate/copy/tasks/main.yml @@ -0,0 +1,42 @@ +--- +# for add_pgnode.yml + +- name: Ensure TLS directories exist + ansible.builtin.file: + path: "{{ item | dirname }}" + state: directory + owner: "{{ tls_owner | default('postgres') }}" + group: "{{ tls_owner | default('postgres') }}" + mode: "0750" + loop: + - "{{ tls_privatekey_path | default('/etc/tls/server.key') }}" + - "{{ tls_cert_path | default('/etc/tls/server.crt') }}" + +- name: Fetch TLS certificate and key from master + run_once: true + ansible.builtin.fetch: + src: "{{ item }}" + dest: "files/tls/" + validate_checksum: true + flat: true + delegate_to: "{{ groups.master[0] }}" + loop: + - "{{ tls_privatekey_path | default('/etc/tls/server.key') }}" + - "{{ tls_cert_path | default('/etc/tls/server.crt') }}" + +- name: Copy TLS certificate and key to replica + ansible.builtin.copy: + src: "files/tls/{{ item.path | basename }}" + dest: "{{ item.path }}" + owner: "{{ tls_owner | default('postgres') }}" + group: "{{ tls_owner | default('postgres') }}" + mode: "{{ item.mode }}" + loop: + - { path: "{{ tls_privatekey_path | default('/etc/tls/server.key') }}", mode: "{{ tls_privatekey_mode | default('0400') }}" } + - { path: "{{ tls_cert_path | default('/etc/tls/server.crt') }}", mode: "{{ tls_cert_mode | default('0644') }}" } + +- name: Delete TLS certificate and key from the ansible controller + ansible.builtin.file: + path: "files/tls/" + state: absent + delegate_to: localhost diff --git a/automation/roles/tls_certificate/tasks/copy_certificate.yml b/automation/roles/tls_certificate/tasks/copy_certificate.yml deleted file mode 100644 index 538248cb5..000000000 --- a/automation/roles/tls_certificate/tasks/copy_certificate.yml +++ /dev/null @@ -1,53 +0,0 @@ ---- -- name: Fetch TLS certificate and key - run_once: true - ansible.builtin.fetch: - src: "{{ item }}" - dest: "/tmp/tls/" - flat: true - loop: - - "{{ tls_privatekey_path | default('/etc/tls/server.key') }}" - - "{{ tls_cert_path | default('/etc/tls/server.crt') }}" - when: - - tls_job is defined - - tls_job == 'fetch' - -- block: - - name: Ensure TLS directories exist - ansible.builtin.file: - path: "{{ item | dirname }}" - state: directory - owner: "{{ tls_owner | default('postgres') }}" - group: "{{ tls_owner | default('postgres') }}" - mode: "0750" - loop: - - "{{ tls_privatekey_path | default('/etc/tls/server.key') }}" - - "{{ tls_cert_path | default('/etc/tls/server.crt') }}" - - - name: Copy TLS certificate and key to the new node - ansible.builtin.copy: - src: "/tmp/tls/{{ item | basename }}" - dest: "{{ item }}" - loop: - - "{{ tls_privatekey_path | default('/etc/tls/server.key') }}" - - "{{ tls_cert_path | default('/etc/tls/server.crt') }}" - - - name: Set proper permissions for TLS files - ansible.builtin.file: - path: "{{ item.path }}" - owner: "{{ tls_owner | default('postgres') }}" - group: "{{ tls_owner | default('postgres') }}" - mode: "{{ item.mode }}" - state: file - loop: - - { path: "{{ tls_privatekey_path | default('/etc/tls/server.key') }}", mode: "0400" } - - { path: "{{ tls_cert_path | default('/etc/tls/server.crt') }}", mode: "0644" } - - - name: Delete TLS certificate and key from the ansible controller - ansible.builtin.file: - path: "/tmp/tls/" - state: absent - delegate_to: localhost - when: - - tls_job is defined - - tls_job == 'copy' diff --git a/automation/roles/tls_certificate/tasks/main.yml b/automation/roles/tls_certificate/tasks/main.yml index 2f3021022..07ee54dd1 100644 --- a/automation/roles/tls_certificate/tasks/main.yml +++ b/automation/roles/tls_certificate/tasks/main.yml @@ -1,5 +1,5 @@ --- -- name: "Ensure TLS directories exist" +- name: Ensure TLS directories exist ansible.builtin.file: path: "{{ item | dirname }}" state: directory