Fix repmgr upgrade (hitachienergy#2459)

* Do not use 'specification' var as task var

* Fix backup & restore tasks

* Wait for repmgr version propagation to standby

* Ensure symlink to config file exists

* Use postgresql_ext module

core/src/epicli/data/common/ansible/playbooks/roles/backup/tasks/postgresql.yml

@@ -10,7 +10,7 @@
 - name: Check if database is running on node0 database server
   become: true
   become_user: postgres
-  command: "{{ repmgr.pg_bindir[ansible_os_family] }}/pg_isready"
+  command: "{{ pg.bin_dir[ansible_os_family] }}/pg_isready"
   register: node0
   ignore_errors: True
   when: groups['postgresql'][0] == inventory_hostname

core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/extensions/replication/extension.yml

@@ -76,11 +76,13 @@
     pg_bin_dir: "{{ pg.bin_dir[ansible_os_family] }}"
     pg_data_dir: "{{ pg.data_dir[ansible_os_family] }}"
     pg_service_name: "{{ pg.service_name[ansible_os_family] }}"
+    replication_user_name: "{{ specification.extensions.replication.replication_user_name }}"
+    repmgr_database: "{{ specification.extensions.replication.repmgr_database }}"
     repmgr_service_name: "{{ repmgr.service_name[ansible_os_family] }}"
 
-# On Ubuntu config file location is not set by package (see https://repmgr.org/docs/4.0/packages-debian-ubuntu.html).
+# On Ubuntu config file location is not set by package (see https://repmgr.org/docs/5.2/packages-debian-ubuntu.html).
 # Create symlink to allow using repmgr commands without specifying config file location (which is custom).
-# See https://repmgr.org/docs/4.0/configuration-file.html
+# See https://repmgr.org/docs/5.2/configuration-file.html#CONFIGURATION-FILE-LOCATION
 - name: Extensions | repmgr | Debian specific tasks
   when: ansible_os_family == 'Debian'
   block:

core/src/epicli/data/common/ansible/playbooks/roles/postgresql/tasks/upgrade/extensions/replication/repmgr-upgrade.yml

@@ -13,6 +13,16 @@
     file: manifest.yml
     name: postgresql_manifest
 
+# repmgr was introduced with epicli v0.6 but the following symlink was added in v0.7 so this task is to ensure it exists
+- name: repmgr for PG {{ pg_version }} | Debian | Ensure symlink to config file exists
+  file:
+    src: "{{ upgrade_defaults.repmgr.config_dir[ansible_os_family] }}/repmgr.conf"
+    dest: /etc/repmgr.conf
+    state: link
+    owner: postgres
+    group: postgres
+  when: ansible_os_family == 'Debian'
+
 - name: repmgr for PG {{ pg_version }} | Search for primary node
   become_user: postgres
   # command prints primary node name (hostname)
@@ -126,21 +136,35 @@
         pg_bin_dir: "{{ upgrade_defaults.pg.bin_dir[ansible_os_family] }}"
         pg_data_dir: "{{ upgrade_defaults.pg.data_dir[ansible_os_family] }}"
         pg_service_name: "{{ upgrade_defaults.pg.service_name[ansible_os_family] }}"
+        replication_user_name: "{{ postgresql_manifest.specification.extensions.replication.replication_user_name }}"
+        repmgr_database: "{{ postgresql_manifest.specification.extensions.replication.repmgr_database }}"
         repmgr_service_name: "{{ upgrade_defaults.repmgr.service_name[ansible_os_family] }}"
-        specification:
-          extensions:
-            replication:
-              replication_user_name: "{{ postgresql_manifest.specification.extensions.replication.replication_user_name }}"
-              repmgr_database:       "{{ postgresql_manifest.specification.extensions.replication.repmgr_database }}"
-
-# Step: Execute 'ALTER EXTENSION repmgr UPDATE' (on primary only)
-- name: repmgr for PG {{ pg_version }} | Update extension
+
+# Step: Execute 'ALTER EXTENSION repmgr UPDATE' (on primary only).
+# On standby it fails with error "cannot execute ALTER EXTENSION in a read-only transaction".
+- name: repmgr for PG {{ pg_version }} | primary | Update extension
   become_user: postgres
-  postgresql_query:
+  postgresql_ext:
+    name: repmgr
     db: "{{ postgresql_manifest.specification.extensions.replication.repmgr_database }}"
-    query: ALTER EXTENSION repmgr UPDATE
+    version: "{{ _split_version[0] }}.{{ _split_version[1] }}"
+  vars:
+    _split_version: "{{ repmgr.version[ansible_os_family].split('.') }}"
   when: inventory_hostname == find_pg_primary_node.stdout
 
+# Wait for repmgr version propagation to standby
+- name: repmgr for PG {{ pg_version }} | standby | Wait for repmgr version sync with primary
+  become_user: postgres
+  postgresql_query:
+    db: "{{ postgresql_manifest.specification.extensions.replication.repmgr_database }}"
+    query: SELECT installed_version FROM pg_available_extensions WHERE name = 'repmgr'
+  register: query_repmgr_version
+  # 'installed_version' keeps only major version (e.g. '5.2')
+  until: repmgr.version[ansible_os_family] is match(query_repmgr_version.query_result | json_query('[0].installed_version'))
+  retries: 10
+  delay: 1
+  when: inventory_hostname != find_pg_primary_node.stdout
+
 # Step: Re-enable repmgr service
 - name: repmgr for PG {{ pg_version }} | Re-enable repmgr service
   systemd:

core/src/epicli/data/common/ansible/playbooks/roles/postgresql/templates/repmgr.conf.j2

@@ -2,7 +2,7 @@
 
 node_id={{ node_id }}
 node_name={{ inventory_hostname }}
-conninfo='host={{ ansible_default_ipv4.address }} user={{ specification.extensions.replication.replication_user_name }} dbname={{ specification.extensions.replication.repmgr_database }} connect_timeout=2'
+conninfo='host={{ ansible_default_ipv4.address }} user={{ replication_user_name }} dbname={{ repmgr_database }} connect_timeout=2'
 data_directory='{{ pg_data_dir }}'
 
 failover=automatic

core/src/epicli/data/common/ansible/playbooks/roles/recovery/tasks/postgresql.yml

@@ -9,7 +9,8 @@
       /bin/systemctl stop {{ pg.service_name[ansible_os_family] }},
       /bin/systemctl restart {{ pg.service_name[ansible_os_family] }},
       /bin/systemctl reload {{ pg.service_name[ansible_os_family] }},
-      {{ repmgr.bindir[ansible_os_family] }}/repmgr standby follow
+      /bin/systemctl start {{ repmgr.service_name[ansible_os_family] }},
+      /bin/systemctl stop {{ repmgr.service_name[ansible_os_family] }}
     validate: 'visudo -cf %s'
 
 - name: Stop repmgr service
@@ -134,7 +135,7 @@
       - name: Register primary node in repmgr
         become: yes
         become_user: postgres
-        shell: "{{ repmgr.bindir[ansible_os_family] }}/repmgr -f {{ repmgr.config_dir[ansible_os_family] }}/repmgr.conf
+        shell: "{{ repmgr.bin_dir[ansible_os_family] }}/repmgr -f {{ repmgr.config_dir[ansible_os_family] }}/repmgr.conf
         --force --superuser={{ component_vars.specification.extensions.replication.privileged_user_name }} primary register -F"
 
       - name: Start repmgr on primary node
@@ -200,7 +201,7 @@
 
     - name: Clone content from primary node using repmgr
       become_user: postgres
-      shell: "{{ repmgr.bindir[ansible_os_family] }}/repmgr -f {{ repmgr.config_dir[ansible_os_family] }}/repmgr.conf -h {{ hostvars[groups['postgresql'][0]]['ansible_default_ipv4']['address'] }} -U {{ component_vars.specification.extensions.replication.privileged_user_name }} -d {{ component_vars.specification.extensions.replication.repmgr_database }} -p 5432 -F standby clone"
+      shell: "{{ repmgr.bin_dir[ansible_os_family] }}/repmgr -f {{ repmgr.config_dir[ansible_os_family] }}/repmgr.conf -h {{ hostvars[groups['postgresql'][0]]['ansible_default_ipv4']['address'] }} -U {{ component_vars.specification.extensions.replication.privileged_user_name }} -d {{ component_vars.specification.extensions.replication.repmgr_database }} -p 5432 -F standby clone"
 
     - name: Copy cached config files back to database configuration location
       copy:
@@ -217,7 +218,7 @@
 
     - name: Register secondary node to repmgr cluster
       become_user: postgres
-      shell: "{{ repmgr.bindir[ansible_os_family] }}/repmgr -f {{ repmgr.config_dir[ansible_os_family] }}/repmgr.conf standby register -F"
+      shell: "{{ repmgr.bin_dir[ansible_os_family] }}/repmgr -f {{ repmgr.config_dir[ansible_os_family] }}/repmgr.conf standby register -F"
 
     - name: Start repmgr service
       service:
@@ -228,7 +229,7 @@
       become: true
       become_user: postgres
       become_method: sudo
-      command: "{{ repmgr.bindir[ansible_os_family] }}/repmgr standby follow"
+      command: "{{ repmgr.bin_dir[ansible_os_family] }}/repmgr standby follow"
   when:
     - component_vars.specification.extensions.replication.enabled | default(false)
     - groups['postgresql'][1] == inventory_hostname