update_pgcluster.yml: Improve error handling (#578)

roles/update/tasks/patroni.yml

@@ -11,10 +11,12 @@
       environment:
         PATH: "{{ ansible_env.PATH }}:/usr/local/bin:/usr/bin"
         PIP_BREAK_SYSTEM_PACKAGES: "1"
-  when: installation_method == "repo" and patroni_installation_method == "pip"
+      register: update_patroni_package_pip
+  ignore_errors: true
   environment: "{{ proxy_env | default({}) }}"
   vars:
     ansible_python_interpreter: /usr/bin/python3
+  when: installation_method == "repo" and patroni_installation_method == "pip"
 
 # patroni_installation_method: "rpm/deb"
 - block:
@@ -23,8 +25,8 @@
       ansible.builtin.package:
         name: "{{ patroni_packages | default('patroni') }}"
         state: latest
-      register: package_status
-      until: package_status is success
+      register: update_patroni_package_debian
+      until: update_patroni_package_debian is success
       delay: 5
       retries: 3
       when: ansible_os_family == "Debian" and patroni_deb_package_repo | length < 1
@@ -34,8 +36,8 @@
       ansible.builtin.package:
         name: "{{ patroni_packages | default('patroni') }}"
         state: latest
-      register: package_status
-      until: package_status is success
+      register: update_patroni_package_rhel
+      until: update_patroni_package_rhel is success
       delay: 5
       retries: 3
       when: ansible_os_family == "RedHat" and patroni_rpm_package_repo | length < 1
@@ -57,8 +59,8 @@
         deb: "/tmp/{{ item }}"
         state: present
       loop: "{{ patroni_deb_package_repo | map('basename') | list }}"
-      register: apt_status
-      until: apt_status is success
+      register: update_patroni_package_deb
+      until: update_patroni_package_deb is success
       delay: 5
       retries: 3
       when: ansible_os_family == "Debian" and patroni_deb_package_repo | length > 0
@@ -78,13 +80,26 @@
         name: "/tmp/{{ item }}"
         state: present
       loop: "{{ patroni_rpm_package_repo | map('basename') | list }}"
-      register: package_status
-      until: package_status is success
+      register: update_patroni_package_rpm
+      until: update_patroni_package_rpm is success
       delay: 5
       retries: 3
       when: ansible_os_family == "RedHat" and patroni_rpm_package_repo | length > 0
+  ignore_errors: true
   environment: "{{ proxy_env | default({}) }}"
   when:
     - installation_method == "repo"
     - (patroni_installation_method == "rpm" or patroni_installation_method == "deb")
+
+# Set flag if any update failed
+- name: "Set variable: update_patroni_failed"
+  ansible.builtin.set_fact:
+    update_patroni_failed: true
+  when: >
+    (update_patroni_package_pip is defined and update_patroni_package_pip is failed) or
+    (update_patroni_package_debian is defined and update_patroni_package_debian is failed) or
+    (update_patroni_package_rhel is defined and update_patroni_package_rhel is failed) or
+    (update_patroni_package_deb is defined and update_patroni_package_deb is failed) or
+    (update_patroni_package_rpm is defined and update_patroni_package_rpm is failed)
+
 ...

roles/update/tasks/postgres.yml

@@ -3,6 +3,7 @@
   ansible.builtin.shell: yum clean all && yum -y makecache
   args:
     executable: /bin/bash
+  ignore_errors: true
   when:
     - ansible_os_family == "RedHat"
     - ansible_distribution_major_version == '7'
@@ -11,6 +12,7 @@
   ansible.builtin.shell: dnf clean all && dnf -y makecache
   args:
     executable: /bin/bash
+  ignore_errors: true
   when:
     - ansible_os_family == "RedHat"
     - ansible_distribution_major_version >= '8'
@@ -23,15 +25,24 @@
   until: apt_status is success
   delay: 5
   retries: 3
+  ignore_errors: true
   when: ansible_os_family == "Debian"
 
 - name: Install the latest version of PostgreSQL packages
   ansible.builtin.package:
     name: "{{ item }}"
     state: latest
   loop: "{{ postgresql_packages }}"
-  register: package_status
-  until: package_status is success
+  register: update_postgres_package
+  until: update_postgres_package is success
   delay: 5
   retries: 3
+  ignore_errors: true
+
+# Set flag if any update failed
+- name: "Set variable: update_postgres_failed"
+  ansible.builtin.set_fact:
+    update_postgres_failed: true
+  when: update_postgres_package is failed
+
 ...

roles/update/tasks/system.yml

@@ -3,6 +3,7 @@
   ansible.builtin.shell: yum clean all && yum -y makecache
   args:
     executable: /bin/bash
+  ignore_errors: true
   when:
     - ansible_os_family == "RedHat"
     - ansible_distribution_major_version == '7'
@@ -11,6 +12,7 @@
   ansible.builtin.shell: dnf clean all && dnf -y makecache
   args:
     executable: /bin/bash
+  ignore_errors: true
   when:
     - ansible_os_family == "RedHat"
     - ansible_distribution_major_version >= '8'
@@ -23,14 +25,15 @@
   until: apt_status is success
   delay: 5
   retries: 3
+  ignore_errors: true
   when: ansible_os_family == "Debian"
 
 - name: Update all system packages
   ansible.builtin.package:
     name: '*'
     state: latest
-  register: package_status
-  until: package_status is success
+  register: update_system_packages_debian
+  until: update_system_packages_debian is success
   delay: 5
   retries: 3
   ignore_errors: true
@@ -41,10 +44,11 @@
     name: '*'
     state: latest
     disablerepo: 'pgdg*'
-  register: package_status
-  until: package_status is success
+  register: update_system_packages_rhel7
+  until: update_system_packages_rhel7 is success
   delay: 5
   retries: 3
+  ignore_errors: true
   when:
     - ansible_os_family == "RedHat"
     - ansible_distribution_major_version == '7'
@@ -54,10 +58,11 @@
     name: '*'
     state: latest
     disablerepo: 'pgdg*'
-  register: package_status
-  until: package_status is success
+  register: update_system_packages_rhel
+  until: update_system_packages_rhel is success
   delay: 5
   retries: 3
+  ignore_errors: true
   when:
     - ansible_os_family == "RedHat"
     - ansible_distribution_major_version >= '8'
@@ -66,6 +71,7 @@
   ansible.builtin.stat:
     path: /var/run/reboot-required
   register: reboot_required_debian
+  failed_when: false
   changed_when: false
   when:
     - ansible_os_family == "Debian"
@@ -87,4 +93,14 @@
     test_command: uptime
   when: (reboot_required_debian.stat.exists is defined and reboot_required_debian.stat.exists) or
         (reboot_required_rhel.rc is defined and reboot_required_rhel.rc != 0)
+
+# Set flag if any update failed
+- name: "Set variable: update_system_failed"
+  ansible.builtin.set_fact:
+    update_system_failed: true
+  when: >
+    (update_system_packages_debian is defined and update_system_packages_debian is failed) or
+    (update_system_packages_rhel is defined and update_system_packages_rhel is failed) or
+    (update_system_packages_rhel7 is defined and update_system_packages_rhel7 is failed)
+
 ...

update_pgcluster.yml

@@ -206,6 +206,15 @@
       when: update_extensions | bool
 
     # finish (info)
+    - name: Check for any update failure
+      run_once: true # noqa run-once
+      ansible.builtin.set_fact:
+        any_update_failed: "{{ any_update_failed | default(false) or
+          hostvars[item].update_postgres_failed | default(false) or
+          hostvars[item].update_patroni_failed | default(false) or
+          hostvars[item].update_system_failed | default(false) }}"
+      loop: "{{ groups['postgres_cluster'] }}"
+
     - name: Check the Patroni cluster state
       run_once: true # noqa run-once
       become: true
@@ -230,13 +239,23 @@
         msg: "{{ patronictl_result.stdout_lines }}"
       when: patronictl_result.stdout_lines is defined
 
+    # if there are no update errors
     - name: Update completed
       run_once: true # noqa run-once
       ansible.builtin.debug:
         msg:
           - "PostgreSQL HA cluster update completed."
           - "Current version: {{ postgres_version.stdout }}"
-      when: postgres_version.stdout is defined
+      when: not any_update_failed
+
+    # if there were errors during the update
+    - name: Update completed with error
+      run_once: true # noqa run-once
+      ansible.builtin.debug:
+        msg:
+          - "Update of PostgreSQL HA cluster completed with errors. Check the Ansible log."
+          - "Current version: {{ postgres_version.stdout }}"
+      when: any_update_failed
   tags:
     - update
     - update-extensions