Add logic for upgrading from Postgres 13 to 15

config/crd/bases/eda.ansible.com_edas.yaml

@@ -2151,6 +2151,9 @@ spec:
               dbFieldsEncryptionSecret:
                 description: Database Fields Encryption secret name of the deployed instance
                 type: string
+              upgradedPostgresVersion:
+                description: Status to indicate that the database has been upgraded to the version in the status
+                type: string
               image:
                 description: URL of the image used for the deployed instance
                 type: string

roles/backup/vars/main.yml

@@ -1,7 +1,7 @@
 ---
 deployment_type: "eda"
-_postgres_image: quay.io/sclorg/postgresql-13-c8s
+_postgres_image: quay.io/sclorg/postgresql-15-c8s
 _postgres_image_version: latest
 backup_complete: false
 database_type: "unmanaged"
-supported_pg_version: 13
+supported_pg_version: 15

roles/eda/tasks/update_status.yml

@@ -53,3 +53,14 @@
         status:
           URL: "https://{{ route_url['resources'][0]['status']['ingress'][0]['host'] }}"
   when: ingress_type | lower == 'route'
+
+
+- name: Update upgradedPostgresVersion status
+  operator_sdk.util.k8s_status:
+    api_version: '{{ api_version }}'
+    kind: "{{ kind }}"
+    name: "{{ ansible_operator_meta.name }}"
+    namespace: "{{ ansible_operator_meta.namespace }}"
+    status:
+      upgradedPostgresVersion: "{{ upgraded_postgres_version | string }}"
+  when: upgraded_postgres_version is defined

roles/postgres/defaults/main.yml

@@ -4,7 +4,7 @@ deployment_type: eda
 database_name: "{{ deployment_type }}"
 database_username: "{{ deployment_type }}"
 
-_postgres_image: quay.io/sclorg/postgresql-13-c8s
+_postgres_image: quay.io/sclorg/postgresql-15-c8s
 _postgres_image_version: latest
 
 # Add a nodeSelector for the Postgres pods.

roles/postgres/tasks/check_postgres_version.yml

@@ -0,0 +1,76 @@
+---
+
+# It is possible that N-2 postgres pods may still be present in the namespace from previous upgrades.
+# So we have to take that into account and preferentially set the most recent one.
+- name: Get the old postgres pod (N-1)
+  k8s_info:
+    kind: Pod
+    namespace: "{{ ansible_operator_meta.namespace }}"
+    field_selectors:
+      - status.phase=Running
+  register: _running_pods
+
+- block:
+  - name: Filter pods by name
+    set_fact:
+      filtered_old_postgres_pods: "{{ _running_pods.resources |
+        selectattr('metadata.name', 'match', ansible_operator_meta.name + '-postgres.*-0') |
+        rejectattr('metadata.name', 'search', '-' + supported_pg_version | string + '-0') |
+        list }}"
+
+  # Sort pods by name in reverse order (most recent PG version first) and set
+  - name: Set info for previous postgres pod
+    set_fact:
+      sorted_old_postgres_pods: "{{ filtered_old_postgres_pods |
+        sort(attribute='metadata.name') |
+        reverse }}"
+    when: filtered_old_postgres_pods | length
+
+  - name: Set info for previous postgres pod
+    set_fact:
+      old_postgres_pod: "{{ sorted_old_postgres_pods | first }}"
+    when: filtered_old_postgres_pods | length
+  when: _running_pods.resources | length
+
+- name: Look up details for this deployment
+  k8s_info:
+    api_version: "{{ api_version }}"
+    kind: "{{ kind }}"
+    name: "{{ ansible_operator_meta.name }}"
+    namespace: "{{ ansible_operator_meta.namespace }}"
+  register: this_eda
+
+# If this deployment has been upgraded before or if upgrade has already been started, set this var
+- name: Set previous PG version var
+  set_fact:
+    _previous_upgraded_pg_version: "{{ this_eda['resources'][0]['status']['upgradedPostgresVersion'] | default(false) }}"
+  when:
+    - "'upgradedPostgresVersion' in this_eda['resources'][0]['status']"
+
+- name: Check if postgres pod is running an older version
+  block:
+    - name: Set path to PG_VERSION file for given container image
+      set_fact:
+        path_to_pg_version: '{{ combined_database.postgres_data_path }}/PG_VERSION'
+
+    - name: Get old PostgreSQL version
+      k8s_exec:
+        namespace: "{{ ansible_operator_meta.namespace }}"
+        pod: "{{ old_postgres_pod['metadata']['name'] }}"
+        command: |
+          bash -c """
+          cat {{ path_to_pg_version }}
+          """
+      register: _old_pg_version
+
+    - debug:
+        msg: "--- Upgrading from {{ old_postgres_pod['metadata']['name'] | default('NONE')}} Pod ---"
+
+    - name: Upgrade data dir from old Postgres to {{ supported_pg_version }} if applicable
+      include_tasks: upgrade_postgres.yml
+      when:
+        - (_old_pg_version.stdout | default(0) | int ) < supported_pg_version
+  when:
+    - managed_database
+    - (_previous_upgraded_pg_version | default(false)) | ternary(_previous_upgraded_pg_version < supported_pg_version, true)
+    - old_postgres_pod | length  # If empty, then old pg pod has been removed and we can assume the upgrade is complete

roles/postgres/tasks/main.yml

@@ -13,6 +13,11 @@
 - name: Set variables to be used in Postgres templates
   import_tasks: set_variables.yml
 
+# Managed Database block
+- name: Check PostgreSQL version to determine if an upgrade is needed
+  import_tasks: check_postgres_version.yml
+  when: managed_database
+
 - name: Create managed Postgres StatefulSet if no external db is defined
   import_tasks: create_managed_postgres.yml
   when: managed_database

roles/postgres/tasks/upgrade_postgres.yml

@@ -0,0 +1,165 @@
+---
+
+# Upgrade Posgres (Managed Databases only)
+#  * If postgres version is not supported_pg_version, and not an external postgres instance (when managed_database is yes),
+#    then run this playbook with include_tasks from database_configuration.yml
+#  * Data will be streamed via a pg_dump from the postgres 12/13 pod to the postgres supported_pg_version
+#    pod via a pg_restore.
+
+
+- name: Scale down Deployment for migration
+  include_tasks: scale_down_deployment.yml
+
+- name: Delete existing postgres configuration secret
+  k8s:
+    api_version: v1
+    kind: Secret
+    name: "{{ ansible_operator_meta.name }}-postgres-configuration"
+    namespace: "{{ ansible_operator_meta.namespace }}"
+    state: absent
+    wait: yes
+
+- name: Create Database configuration with new -postgres-{{ supported_pg_version }} hostname
+  k8s:
+    apply: true
+    definition: "{{ lookup('template', 'postgres_upgrade_secret.yaml.j2') }}"
+  no_log: "{{ no_log }}"
+
+- name: Set new database var to be used when configuring app credentials (resources_configuration.yml)
+  set_fact:
+    eda_postgres_host: "{{ ansible_operator_meta.name }}-postgres-{{ supported_pg_version }}"
+  no_log: "{{ no_log }}"
+
+- name: Create Database and StatefulSet if no database is specified
+  k8s:
+    apply: yes
+    definition: "{{ lookup('template', item + '.yaml.j2') }}"
+    wait: yes
+  loop:
+    - 'postgres.service'
+    - 'postgres.statefulset'
+  register: create_statefulset_result
+
+- name: Apply database resources
+  k8s:
+    apply: yes
+    definition: "{{ lookup('template', item + '.yaml.j2') }}"
+    wait: no
+  register: postgres_resources_result
+  loop:
+    - 'postgres.service'
+    - 'postgres.statefulset'
+  no_log: "{{ no_log }}"
+
+- name: Set postgres label if not defined by user
+  set_fact:
+    postgres_label_selector: "app.kubernetes.io/instance=postgres-{{ supported_pg_version }}-{{ ansible_operator_meta.name }}"
+  when: postgres_label_selector is not defined
+
+- name: Get new postgres pod information
+  k8s_info:
+    kind: Pod
+    namespace: "{{ ansible_operator_meta.namespace }}"
+    label_selectors:
+      - "{{ postgres_label_selector }}"
+    field_selectors:
+      - status.phase=Running
+  register: postgres_pod
+  until:
+    - "postgres_pod['resources'] | length"
+    - "postgres_pod['resources'][0]['status']['phase'] == 'Running'"
+    - "postgres_pod['resources'][0]['status']['containerStatuses'][0]['ready'] == true"
+  delay: 5
+  retries: 60
+
+- name: Set the resource pod name as a variable.
+  set_fact:
+    postgres_pod_name: "{{ postgres_pod['resources'][0]['metadata']['name'] }}"
+
+- name: Get the name of the service for the old postgres pod
+  k8s_info:
+    kind: Service
+    namespace: "{{ ansible_operator_meta.namespace }}"
+    label_selectors:
+      - "app.kubernetes.io/component=database"
+      - "app.kubernetes.io/instance={{ old_postgres_pod.metadata.labels['app.kubernetes.io/instance'] }}"
+      - "app.kubernetes.io/managed-by=eda-operator"
+  register: old_postgres_svc
+
+- name: Set full resolvable host name for postgres pod
+  set_fact:
+    resolvable_db_host: "{{ old_postgres_svc['resources'][0]['metadata']['name'] }}.{{ ansible_operator_meta.namespace }}.svc"  # yamllint disable-line rule:line-length
+  no_log: "{{ no_log }}"
+
+- name: Set pg_dump command
+  set_fact:
+    pgdump: >-
+      pg_dump
+      -h {{ resolvable_db_host }}
+      -U {{ eda_postgres_user }}
+      -d {{ eda_postgres_database }}
+      -p {{ eda_postgres_port }}
+      -F custom
+  no_log: "{{ no_log }}"
+
+- name: Set pg_restore command
+  set_fact:
+    pg_restore: >-
+      pg_restore
+      -U {{ eda_postgres_user }}
+      -d {{ eda_postgres_database }}
+  no_log: "{{ no_log }}"
+
+- name: Stream backup from pg_dump to the new postgresql container
+  k8s_exec:
+    namespace: "{{ ansible_operator_meta.namespace }}"
+    pod: "{{ postgres_pod_name }}"
+    command: |
+      bash -c """
+      set -e -o pipefail
+      PGPASSWORD='{{ eda_postgres_pass }}' {{ pgdump }} | PGPASSWORD='{{ eda_postgres_pass }}' {{ pg_restore }}
+      echo 'Successful'
+      """
+  no_log: "{{ no_log }}"
+  register: data_migration
+  failed_when: "'Successful' not in data_migration.stdout"
+
+- name: Set flag signifying that this instance has been migrated
+  set_fact:
+    upgraded_postgres_version: '{{ supported_pg_version }}'
+
+# Cleanup old Postgres resources
+- name: Remove old Postgres StatefulSet
+  k8s:
+    kind: StatefulSet
+    api_version: v1
+    namespace: "{{ ansible_operator_meta.namespace }}"
+    name: "{{ item }}"
+    state: absent
+    wait: true
+  loop:
+    - "{{ ansible_operator_meta.name }}-postgres"
+    - "{{ ansible_operator_meta.name }}-postgres-13"
+
+- name: Remove old Postgres Service
+  k8s:
+    kind: Service
+    api_version: v1
+    namespace: "{{ ansible_operator_meta.namespace }}"
+    name: "{{ item }}"
+    state: absent
+  loop:
+    - "{{ ansible_operator_meta.name }}-postgres"
+    - "{{ ansible_operator_meta.name }}-postgres-13"
+
+- name: Remove old persistent volume claim
+  k8s:
+    kind: PersistentVolumeClaim
+    api_version: v1
+    namespace: "{{ ansible_operator_meta.namespace }}"
+    name: "{{ item }}"
+    state: absent
+  loop:
+    - "postgres-{{ ansible_operator_meta.name }}-postgres-0"
+    - "postgres-{{ ansible_operator_meta.name }}-postgres-13-0"
+  when: postgres_keep_pvc_after_upgrade

roles/postgres/vars/main.yml

@@ -1,4 +1,6 @@
 ---
 # vars file for PostgreSQL database
 
-supported_pg_version: 13
+supported_pg_version: 15
+_previous_upgraded_pg_version: 0
+old_postgres_pod: []

roles/restore/README.md

@@ -19,7 +19,7 @@ This role assumes you are authenticated with an Openshift or Kubernetes cluster:
 
 *Before Restoring from a backup*, be sure to:
   - delete the old existing EDA CR
-  - delete the persistent volume claim (PVC) for the database from the old deployment, which has a name like `postgres-13-<deployment-name>-postgres-13-0`
+  - delete the persistent volume claim (PVC) for the database from the old deployment, which has a name like `postgres-15-<deployment-name>-postgres-15-0`
 
 **Note**: Do not delete the namespace/project, as that will delete the backup and the backup's PVC as well.
 

roles/restore/vars/main.yml

@@ -1,8 +1,8 @@
 ---
 
 deployment_type: "eda"
-supported_pg_version: 13
-_postgres_image: quay.io/sclorg/postgresql-13-c8s
+supported_pg_version: 15
+_postgres_image: quay.io/sclorg/postgresql-15-c8s
 _postgres_image_version: latest
 
 backup_api_version: '{{ deployment_type }}.ansible.com/v1alpha1'