From 2f2a80be3c2cc62de919f3a2898b3a809b1a58a0 Mon Sep 17 00:00:00 2001 From: to-bar <46519524+to-bar@users.noreply.github.com> Date: Wed, 28 Jul 2021 12:38:08 +0200 Subject: [PATCH] Fix repmgr upgrade (#2459) * Do not use 'specification' var as task var * Fix backup & restore tasks * Wait for repmgr version propagation to standby * Ensure symlink to config file exists * Use postgresql_ext module --- .../roles/backup/tasks/postgresql.yml | 2 +- .../extensions/replication/extension.yml | 6 ++- .../extensions/replication/repmgr-upgrade.yml | 44 ++++++++++++++----- .../roles/postgresql/templates/repmgr.conf.j2 | 2 +- .../roles/recovery/tasks/postgresql.yml | 11 ++--- 5 files changed, 46 insertions(+), 19 deletions(-) diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/postgresql.yml b/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/postgresql.yml index d4bd207245..6a060d61d5 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/postgresql.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/postgresql.yml @@ -10,7 +10,7 @@ - name: Check if database is running on node0 database server become: true become_user: postgres - command: "{{ repmgr.pg_bindir[ansible_os_family] }}/pg_isready" + command: "{{ pg.bin_dir[ansible_os_family] }}/pg_isready" register: node0 ignore_errors: True when: groups['postgresql'][0] == inventory_hostname diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/extensions/replication/extension.yml b/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/extensions/replication/extension.yml index b0aed2a5a1..1b4caa1075 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/extensions/replication/extension.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/extensions/replication/extension.yml @@ -76,11 +76,13 @@ pg_bin_dir: "{{ pg.bin_dir[ansible_os_family] }}" pg_data_dir: "{{ pg.data_dir[ansible_os_family] }}" pg_service_name: "{{ pg.service_name[ansible_os_family] }}" + replication_user_name: "{{ specification.extensions.replication.replication_user_name }}" + repmgr_database: "{{ specification.extensions.replication.repmgr_database }}" repmgr_service_name: "{{ repmgr.service_name[ansible_os_family] }}" -# On Ubuntu config file location is not set by package (see https://repmgr.org/docs/4.0/packages-debian-ubuntu.html). +# On Ubuntu config file location is not set by package (see https://repmgr.org/docs/5.2/packages-debian-ubuntu.html). # Create symlink to allow using repmgr commands without specifying config file location (which is custom). -# See https://repmgr.org/docs/4.0/configuration-file.html +# See https://repmgr.org/docs/5.2/configuration-file.html#CONFIGURATION-FILE-LOCATION - name: Extensions | repmgr | Debian specific tasks when: ansible_os_family == 'Debian' block: diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/upgrade/extensions/replication/repmgr-upgrade.yml b/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/upgrade/extensions/replication/repmgr-upgrade.yml index 7ad8a40463..e641b83d92 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/upgrade/extensions/replication/repmgr-upgrade.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/upgrade/extensions/replication/repmgr-upgrade.yml @@ -13,6 +13,16 @@ file: manifest.yml name: postgresql_manifest +# repmgr was introduced with epicli v0.6 but the following symlink was added in v0.7 so this task is to ensure it exists +- name: repmgr for PG {{ pg_version }} | Debian | Ensure symlink to config file exists + file: + src: "{{ upgrade_defaults.repmgr.config_dir[ansible_os_family] }}/repmgr.conf" + dest: /etc/repmgr.conf + state: link + owner: postgres + group: postgres + when: ansible_os_family == 'Debian' + - name: repmgr for PG {{ pg_version }} | Search for primary node become_user: postgres # command prints primary node name (hostname) @@ -126,21 +136,35 @@ pg_bin_dir: "{{ upgrade_defaults.pg.bin_dir[ansible_os_family] }}" pg_data_dir: "{{ upgrade_defaults.pg.data_dir[ansible_os_family] }}" pg_service_name: "{{ upgrade_defaults.pg.service_name[ansible_os_family] }}" + replication_user_name: "{{ postgresql_manifest.specification.extensions.replication.replication_user_name }}" + repmgr_database: "{{ postgresql_manifest.specification.extensions.replication.repmgr_database }}" repmgr_service_name: "{{ upgrade_defaults.repmgr.service_name[ansible_os_family] }}" - specification: - extensions: - replication: - replication_user_name: "{{ postgresql_manifest.specification.extensions.replication.replication_user_name }}" - repmgr_database: "{{ postgresql_manifest.specification.extensions.replication.repmgr_database }}" - -# Step: Execute 'ALTER EXTENSION repmgr UPDATE' (on primary only) -- name: repmgr for PG {{ pg_version }} | Update extension + +# Step: Execute 'ALTER EXTENSION repmgr UPDATE' (on primary only). +# On standby it fails with error "cannot execute ALTER EXTENSION in a read-only transaction". +- name: repmgr for PG {{ pg_version }} | primary | Update extension become_user: postgres - postgresql_query: + postgresql_ext: + name: repmgr db: "{{ postgresql_manifest.specification.extensions.replication.repmgr_database }}" - query: ALTER EXTENSION repmgr UPDATE + version: "{{ _split_version[0] }}.{{ _split_version[1] }}" + vars: + _split_version: "{{ repmgr.version[ansible_os_family].split('.') }}" when: inventory_hostname == find_pg_primary_node.stdout +# Wait for repmgr version propagation to standby +- name: repmgr for PG {{ pg_version }} | standby | Wait for repmgr version sync with primary + become_user: postgres + postgresql_query: + db: "{{ postgresql_manifest.specification.extensions.replication.repmgr_database }}" + query: SELECT installed_version FROM pg_available_extensions WHERE name = 'repmgr' + register: query_repmgr_version + # 'installed_version' keeps only major version (e.g. '5.2') + until: repmgr.version[ansible_os_family] is match(query_repmgr_version.query_result | json_query('[0].installed_version')) + retries: 10 + delay: 1 + when: inventory_hostname != find_pg_primary_node.stdout + # Step: Re-enable repmgr service - name: repmgr for PG {{ pg_version }} | Re-enable repmgr service systemd: diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/templates/repmgr.conf.j2 b/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/templates/repmgr.conf.j2 index bedcea6a49..a23ac2cd81 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/templates/repmgr.conf.j2 +++ b/core/src/epicli/data/common/ansible/playbooks/roles/postgresql/templates/repmgr.conf.j2 @@ -2,7 +2,7 @@ node_id={{ node_id }} node_name={{ inventory_hostname }} -conninfo='host={{ ansible_default_ipv4.address }} user={{ specification.extensions.replication.replication_user_name }} dbname={{ specification.extensions.replication.repmgr_database }} connect_timeout=2' +conninfo='host={{ ansible_default_ipv4.address }} user={{ replication_user_name }} dbname={{ repmgr_database }} connect_timeout=2' data_directory='{{ pg_data_dir }}' failover=automatic diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/postgresql.yml b/core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/postgresql.yml index 3c1bcc39ce..27c8a55033 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/postgresql.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/postgresql.yml @@ -9,7 +9,8 @@ /bin/systemctl stop {{ pg.service_name[ansible_os_family] }}, /bin/systemctl restart {{ pg.service_name[ansible_os_family] }}, /bin/systemctl reload {{ pg.service_name[ansible_os_family] }}, - {{ repmgr.bindir[ansible_os_family] }}/repmgr standby follow + /bin/systemctl start {{ repmgr.service_name[ansible_os_family] }}, + /bin/systemctl stop {{ repmgr.service_name[ansible_os_family] }} validate: 'visudo -cf %s' - name: Stop repmgr service @@ -134,7 +135,7 @@ - name: Register primary node in repmgr become: yes become_user: postgres - shell: "{{ repmgr.bindir[ansible_os_family] }}/repmgr -f {{ repmgr.config_dir[ansible_os_family] }}/repmgr.conf + shell: "{{ repmgr.bin_dir[ansible_os_family] }}/repmgr -f {{ repmgr.config_dir[ansible_os_family] }}/repmgr.conf --force --superuser={{ component_vars.specification.extensions.replication.privileged_user_name }} primary register -F" - name: Start repmgr on primary node @@ -200,7 +201,7 @@ - name: Clone content from primary node using repmgr become_user: postgres - shell: "{{ repmgr.bindir[ansible_os_family] }}/repmgr -f {{ repmgr.config_dir[ansible_os_family] }}/repmgr.conf -h {{ hostvars[groups['postgresql'][0]]['ansible_default_ipv4']['address'] }} -U {{ component_vars.specification.extensions.replication.privileged_user_name }} -d {{ component_vars.specification.extensions.replication.repmgr_database }} -p 5432 -F standby clone" + shell: "{{ repmgr.bin_dir[ansible_os_family] }}/repmgr -f {{ repmgr.config_dir[ansible_os_family] }}/repmgr.conf -h {{ hostvars[groups['postgresql'][0]]['ansible_default_ipv4']['address'] }} -U {{ component_vars.specification.extensions.replication.privileged_user_name }} -d {{ component_vars.specification.extensions.replication.repmgr_database }} -p 5432 -F standby clone" - name: Copy cached config files back to database configuration location copy: @@ -217,7 +218,7 @@ - name: Register secondary node to repmgr cluster become_user: postgres - shell: "{{ repmgr.bindir[ansible_os_family] }}/repmgr -f {{ repmgr.config_dir[ansible_os_family] }}/repmgr.conf standby register -F" + shell: "{{ repmgr.bin_dir[ansible_os_family] }}/repmgr -f {{ repmgr.config_dir[ansible_os_family] }}/repmgr.conf standby register -F" - name: Start repmgr service service: @@ -228,7 +229,7 @@ become: true become_user: postgres become_method: sudo - command: "{{ repmgr.bindir[ansible_os_family] }}/repmgr standby follow" + command: "{{ repmgr.bin_dir[ansible_os_family] }}/repmgr standby follow" when: - component_vars.specification.extensions.replication.enabled | default(false) - groups['postgresql'][1] == inventory_hostname