From 9091ee6669861879c8626e800b16f0ff7f5f8fdc Mon Sep 17 00:00:00 2001 From: Irek Glownia Date: Thu, 5 Aug 2021 15:40:29 +0200 Subject: [PATCH] documentation, small fixes, merge conflicts --- .../postgresql/tasks/preflight/apply.yml | 7 - .../postgresql/tasks/preflight/upgrade.yml | 2 + .../extensions/replication/repmgr-main.yml | 37 ---- .../extensions/replication/repmgr-upgrade.yml | 159 ------------------ .../roles/postgresql/templates/pg_hba.conf.j2 | 4 +- .../ubuntu-18.04/requirements.x86_64.txt | 8 +- docs/home/howto/DATABASES.md | 78 +++++---- 7 files changed, 57 insertions(+), 238 deletions(-) delete mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/upgrade/extensions/replication/repmgr-main.yml delete mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/upgrade/extensions/replication/repmgr-upgrade.yml diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/preflight/apply.yml b/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/preflight/apply.yml index cda23a67f6..e93cf7e6c0 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/preflight/apply.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/preflight/apply.yml @@ -33,13 +33,6 @@ _unsupported_enabled_extensions: >- {{ _enabled_extensions | selectattr('name', 'in', unsupported_postgres_extensions[_pg_arch]) | list }} - - name: Check the number of PostgreSQL hosts for replication - when: postgresql.specification.extensions.replication.enabled | default(false) - assert: - that: groups.postgresql | length == 2 - fail_msg: The supported number of PostgreSQL hosts with enabled replication is 2 # TODO: issue 2346 - success_msg: The number of PostgreSQL hosts is correct - - name: Check PostgreSQL password encryption support for applications block: - name: Include applications vars from applications role diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/preflight/upgrade.yml b/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/preflight/upgrade.yml index 2b9b91ebe6..0a88ab26ad 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/preflight/upgrade.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/preflight/upgrade.yml @@ -22,6 +22,8 @@ shell: |- set -o pipefail && \ df --output=source {{ pg_new.pg.base_dir[ansible_os_family] }} | tail -1 + args: + executable: /bin/bash changed_when: false register: pg_data_disk diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/upgrade/extensions/replication/repmgr-main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/upgrade/extensions/replication/repmgr-main.yml deleted file mode 100644 index 5dd873338f..0000000000 --- a/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/upgrade/extensions/replication/repmgr-main.yml +++ /dev/null @@ -1,37 +0,0 @@ ---- -- name: repmgr for PG {{ pg_version }} | Load default variables for upgrade - include_vars: - file: defaults/upgrade.yml - name: upgrade_defaults - -- name: repmgr for PG {{ pg_version }} | Get information about installed packages as facts - package_facts: - manager: auto - when: ansible_facts.packages is undefined - -- name: Run upgrade if needed - when: ansible_facts.packages[repmgr_package_name] is defined - vars: - repmgr_package_name: "{{ upgrade_defaults.repmgr.package_name[ansible_os_family] }}" - installed_version: "{{ ansible_facts.packages[repmgr_package_name][0].version }}" - target_version: "{{ repmgr.version[ansible_os_family] }}" - block: - - name: repmgr for PG {{ pg_version }} | Print repmgr versions - debug: - msg: - - "Installed version: {{ installed_version }}" - - "Target version: {{ target_version }}" - - # If state file exists it means the previous run failed - - name: repmgr for PG {{ pg_version }} | Check if upgrade state file exists - stat: - path: "{{ upgrade_defaults.repmgr.upgrade.state_file_path }}" - get_attributes: false - get_checksum: false - get_mime: false - register: stat_upgrade_state_file - - - name: repmgr for PG {{ pg_version }} | Upgrade repmgr - include_tasks: roles/postgresql/tasks/upgrade/extensions/replication/repmgr-upgrade.yml - when: target_version is version(installed_version, '>') - or stat_upgrade_state_file.stat.exists diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/upgrade/extensions/replication/repmgr-upgrade.yml b/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/upgrade/extensions/replication/repmgr-upgrade.yml deleted file mode 100644 index 7ad8a40463..0000000000 --- a/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/upgrade/extensions/replication/repmgr-upgrade.yml +++ /dev/null @@ -1,159 +0,0 @@ ---- -# Documentation: https://repmgr.org/docs/5.2/upgrading-repmgr-extension.html#UPGRADING-MAJOR-VERSION# -# Compatibility: https://repmgr.org/docs/5.2/install-requirements.html#INSTALL-COMPATIBILITY-MATRIX - -- name: repmgr for PG {{ pg_version }} | Create upgrade state file - file: - path: "{{ upgrade_defaults.repmgr.upgrade.state_file_path }}" - state: touch - mode: u=rw,g=r,o= - -- name: repmgr for PG {{ pg_version }} | Load postgresql role manifest - include_vars: - file: manifest.yml - name: postgresql_manifest - -- name: repmgr for PG {{ pg_version }} | Search for primary node - become_user: postgres - # command prints primary node name (hostname) - shell: |- - set -o pipefail && \ - {{ upgrade_defaults.repmgr.bin_dir[ansible_os_family] }}/repmgr cluster show \ - | awk 'BEGIN{FS="|"} {gsub(/ /,""); if ($3 == "primary") print $2}' - changed_when: false - register: find_pg_primary_node - failed_when: find_pg_primary_node.rc != 0 or find_pg_primary_node.stdout == "" - run_once: true - args: - executable: /bin/bash - -# Step: Stop repmgr service -- name: repmgr for PG {{ pg_version }} | Stop repmgr service - systemd: - name: "{{ upgrade_defaults.repmgr.service_name[ansible_os_family] }}" - state: stopped - -# Step: Disable repmgr service to prevent packages from prematurely restarting repmgr -- name: repmgr for PG {{ pg_version }} | Disable repmgr service - systemd: - name: "{{ upgrade_defaults.repmgr.service_name[ansible_os_family] }}" - enabled: false - -# Step: Install repmgr packages - -# On Ubuntu there is dependent 'repmgr-common' package. -# apt module doesn't support --allow-downgrades (see https://github.com/ansible/ansible/issues/29451) -# so we keep installed version if it's newer. -- name: repmgr for PG {{ pg_version }} | Set target version for repmgr-common package - set_fact: - repmgr_common_target_version: >- - {{ _installed_version is version(target_version, '>') | ternary(_installed_version, target_version + '-*') }} - when: - - ansible_os_family == 'Debian' - - ansible_facts.packages['repmgr-common'] is defined - vars: - _installed_version: "{{ ansible_facts.packages['repmgr-common'][0].version }}" - -- name: repmgr for PG {{ pg_version }} | Install repmgr package(s) - package: - name: "{{ _packages[ansible_os_family] }}" - state: present - vars: - _packages: - Debian: - - "{{ repmgr_package_name }}={{ target_version + '-*' }}" - - repmgr-common={{ repmgr_common_target_version | default(target_version + '-*') }} - RedHat: - - "{{ repmgr_package_name }}-{{ target_version }}" - module_defaults: - yum: { lock_timeout: "{{ yum_lock_timeout }}" } - -- name: repmgr for PG {{ pg_version }} | Update postgres user in sudoers file - lineinfile: - path: /etc/sudoers - regexp: "^postgres ALL=\\(ALL:ALL\\) NOPASSWD:" - line: >- - postgres ALL=(ALL:ALL) NOPASSWD: - /bin/systemctl start {{ upgrade_defaults.pg.service_name[ansible_os_family] }}, - /bin/systemctl stop {{ upgrade_defaults.pg.service_name[ansible_os_family] }}, - /bin/systemctl restart {{ upgrade_defaults.pg.service_name[ansible_os_family] }}, - /bin/systemctl reload {{ upgrade_defaults.pg.service_name[ansible_os_family] }}, - /bin/systemctl start {{ upgrade_defaults.repmgr.service_name[ansible_os_family] }}, - /bin/systemctl stop {{ upgrade_defaults.repmgr.service_name[ansible_os_family] }} - validate: 'visudo -cf %s' - -# Step: Update /etc/default/repmgrd -- name: repmgr for PG {{ pg_version }} | Set 'daemonize=false' option in /etc/default/repmgrd - lineinfile: - path: /etc/default/repmgrd - regexp: ^[#]?REPMGRD_OPTS= - line: REPMGRD_OPTS="--daemonize=false" - when: ansible_os_family == 'Debian' - -# Step: systemctl daemon-reload -# Step: Restart PostgreSQL -- name: repmgr for PG {{ pg_version }} | Restart PostgreSQL service - systemd: - name: "{{ upgrade_defaults.pg.service_name[ansible_os_family] }}" - state: restarted - daemon_reload: true - -# On Ubuntu the previous task indirectly restarts instantiated service but without waiting for the child service -- name: repmgr for PG {{ pg_version }} | Ensure PostgreSQL instantiated service is running - systemd: - name: "{{ upgrade_defaults.pg.instantiated_service_name[ansible_os_family] }}" - state: started - when: upgrade_defaults.pg.instantiated_service_name[ansible_os_family] != None - -# Step: Update config file -- name: Update repmgr config file - block: - - name: repmgr for PG {{ pg_version }} | Get node id - command: >- - grep -Po '(?<=^node_id=)\d+' "{{ upgrade_defaults.repmgr.config_dir[ansible_os_family] }}/repmgr.conf" - register: pg_node_id - changed_when: false - - - name: repmgr for PG {{ pg_version }} | Replace repmgr config file - template: - src: repmgr.conf.j2 - dest: "{{ upgrade_defaults.repmgr.config_dir[ansible_os_family] }}/repmgr.conf" - owner: postgres - group: postgres - mode: u=rw,g=,o= - vars: - node_id: "{{ pg_node_id.stdout }}" - pg_bin_dir: "{{ upgrade_defaults.pg.bin_dir[ansible_os_family] }}" - pg_data_dir: "{{ upgrade_defaults.pg.data_dir[ansible_os_family] }}" - pg_service_name: "{{ upgrade_defaults.pg.service_name[ansible_os_family] }}" - repmgr_service_name: "{{ upgrade_defaults.repmgr.service_name[ansible_os_family] }}" - specification: - extensions: - replication: - replication_user_name: "{{ postgresql_manifest.specification.extensions.replication.replication_user_name }}" - repmgr_database: "{{ postgresql_manifest.specification.extensions.replication.repmgr_database }}" - -# Step: Execute 'ALTER EXTENSION repmgr UPDATE' (on primary only) -- name: repmgr for PG {{ pg_version }} | Update extension - become_user: postgres - postgresql_query: - db: "{{ postgresql_manifest.specification.extensions.replication.repmgr_database }}" - query: ALTER EXTENSION repmgr UPDATE - when: inventory_hostname == find_pg_primary_node.stdout - -# Step: Re-enable repmgr service -- name: repmgr for PG {{ pg_version }} | Re-enable repmgr service - systemd: - name: "{{ upgrade_defaults.repmgr.service_name[ansible_os_family] }}" - enabled: true - -# Step: Start repmgr service -- name: repmgr for PG {{ pg_version }} | Start repmgr service - systemd: - name: "{{ upgrade_defaults.repmgr.service_name[ansible_os_family] }}" - state: started - -- name: repmgr for PG {{ pg_version }} | Remove upgrade state file - file: - path: "{{ upgrade_defaults.repmgr.upgrade.state_file_path }}" - state: absent diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/templates/pg_hba.conf.j2 b/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/templates/pg_hba.conf.j2 index 3f37ecdcf7..b27924eb65 100755 --- a/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/templates/pg_hba.conf.j2 +++ b/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/templates/pg_hba.conf.j2 @@ -94,6 +94,6 @@ local replication all peer host replication all 127.0.0.1/32 {{ specification.configuration.password_encryption }} host replication all ::1/128 {{ specification.configuration.password_encryption }} -{% if specification.extensions.replication.enabled | default(false) %} -host replication {{ specification.extensions.replication.replication_user_name }} 0.0.0.0/0 {{ specification.configuration.password_encryption }} +{% if _specification.extensions.replication.enabled | default(false) %} +host replication {{ _specification.extensions.replication.replication_user_name }} 0.0.0.0/0 {{ specification.configuration.password_encryption }} {% endif %} diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.x86_64.txt b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.x86_64.txt index e62390cbfd..1d3c37dbe7 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.x86_64.txt +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.x86_64.txt @@ -15,7 +15,8 @@ curl docker-ce 5:19.03.14 docker-ce-cli 5:19.03.14 ebtables -elasticsearch-oss 7.10.2 # for opendistroforelasticsearch & logging roles +# for opendistroforelasticsearch & logging roles +elasticsearch-oss 7.10.2 # Erlang packages must be compatible with RabbitMQ version. # Metapackages such as erlang and erlang-nox must only be used @@ -139,10 +140,11 @@ smbclient samba-libs libsmbclient -# for postgres +# postgres related packages +# if version is not specified, it's not related to postgres version and the latest is used pgbouncer 1.15.* pgdg-keyring -postgresql-13-pgaudit +postgresql-13-pgaudit 1.5.0 postgresql-10-repmgr 5.2.1 postgresql-13-repmgr 5.2.1 postgresql-client-13 diff --git a/docs/home/howto/DATABASES.md b/docs/home/howto/DATABASES.md index 7bad1d9150..105666f4db 100644 --- a/docs/home/howto/DATABASES.md +++ b/docs/home/howto/DATABASES.md @@ -9,6 +9,22 @@ sudo -u postgres -i Then configure database server using psql according to your needs and [PostgreSQL documentation](https://www.postgresql.org/docs/). +## PostgreSQL passwords encryption + +Epiphany sets up MD5 password encryption by default. Although PostgreSQL since version 10 is able to use SCRAM SHA-256 password encryption, Epiphany does support this encryption only for one-node Postgresql configuration. HA components like pgbouncer and pgpool are not able to refresh users/passwords list while this encryption is enabled. So those components can be used only with MD5 passwords. + +However this parameter may be set up as showed below: + +```yaml +kind: configuration/postgresql +title: PostgreSQL +name: default +specification: + configuration: + password_encryption: md5 # md5 or scram-sha-256 +``` +We do not recommend to change this option especially on running database, since this require to encrypt again all passwords. + ## How to set up PostgreSQL connection pooling --- @@ -63,35 +79,36 @@ specification: parameter_groups: ... # This block is optional, you can use it to override default values - - name: REPLICATION - subgroups: - - name: Sending Server(s) - parameters: - - name: max_wal_senders - value: 10 # default value - comment: maximum number of simultaneously running WAL sender processes - when: replication - - name: wal_keep_segments - value: 34 # default value - comment: number of WAL files held for standby servers - when: replication - - name: Standby Servers - parameters: - - name: hot_standby - value: 'on' # default value - comment: must be 'on' for repmgr needs, ignored on primary but recommended in case primary becomes standby - when: replication + - name: REPLICATION + subgroups: + - name: Sending Server(s) + parameters: + - name: max_wal_senders + value: 10 + comment: maximum number of simultaneously running WAL sender processes + when: replication + - name: wal_keep_size + value: 500 + comment: the size of WAL files held for standby servers (MB) + when: replication + - name: Standby Servers + parameters: + - name: hot_standby + value: on + comment: must be 'on' for repmgr needs, ignored on primary but recommended + in case primary becomes standby + when: replication extensions: ... replication: enabled: true - replication_user_name: your_privileged_user_name + replication_user_name: epi_repmgr replication_user_password: PASSWORD_TO_CHANGE - privileged_user_name: your_privileged_user_name + privileged_user_name: epi_repmgr_admin privileged_user_password: PASSWORD_TO_CHANGE - repmgr_database: repmgr + repmgr_database: epi_repmgr shared_preload_libraries: - - repmgr + - repmgr ... ``` @@ -192,7 +209,7 @@ specification: ## --- pgpool --- - name: pgpool - enabled: yes + enabled: true ... namespace: postgres-pool service: @@ -200,7 +217,7 @@ specification: port: 5432 replicas: 3 ... - resources: # Adjust to your configuration, see https://www.pgpool.net/docs/41/en/html/resource-requiremente.html + resources: # Adjust to your configuration, see https://www.pgpool.net/docs/42/en/html/resource-requiremente.html limits: # cpu: 900m # Set according to your env memory: 176Mi @@ -216,8 +233,9 @@ specification: PGPOOL_SR_CHECK_USER: epi_pgpool_sr_check # with pg_monitor role, for streaming replication checks and health checks # --- PGPOOL_ADMIN_USERNAME: epi_pgpool_admin # Pgpool administrator (local pcp user) - PGPOOL_ENABLE_LOAD_BALANCING: yes # set to 'no' if there is no replication + PGPOOL_ENABLE_LOAD_BALANCING: false # set to 'false' if there is no replication PGPOOL_MAX_POOL: 4 + PGPOOL_CHILD_LIFE_TIME: 0 PGPOOL_POSTGRES_PASSWORD_FILE: /opt/bitnami/pgpool/secrets/pgpool_postgres_password PGPOOL_SR_CHECK_PASSWORD_FILE: /opt/bitnami/pgpool/secrets/pgpool_sr_check_password PGPOOL_ADMIN_PASSWORD_FILE: /opt/bitnami/pgpool/secrets/pgpool_admin_password @@ -225,7 +243,7 @@ specification: pgpool_postgres_password: PASSWORD_TO_CHANGE pgpool_sr_check_password: PASSWORD_TO_CHANGE pgpool_admin_password: PASSWORD_TO_CHANGE - # https://www.pgpool.net/docs/41/en/html/runtime-config.html + # https://www.pgpool.net/docs/42/en/html/runtime-config.html pgpool_conf_content_to_append: | #------------------------------------------------------------------------------ # CUSTOM SETTINGS (appended by Epiphany to override defaults) @@ -239,7 +257,7 @@ specification: ## --- pgbouncer --- - name: pgbouncer - enabled: yes + enabled: true ... namespace: postgres-pool service: @@ -257,10 +275,10 @@ specification: env: DB_HOST: pgpool.postgres-pool.svc.cluster.local # pgpool service name DB_LISTEN_PORT: 5432 - LISTEN_ADDR: "*" + LISTEN_ADDR: 0.0.0.0 LISTEN_PORT: 5432 - AUTH_FILE: "/etc/pgbouncer/auth/users.txt" - AUTH_TYPE: md5 + CONFIG_FILE: /opt/bitnami/pgbouncer/conf/pgbouncer.ini + AUTH_FILE: /opt/bitnami/pgbouncer/conf/userlist.txt MAX_CLIENT_CONN: 150 DEFAULT_POOL_SIZE: 25 RESERVE_POOL_SIZE: 25