diff --git a/config/crd/bases/eda.ansible.com_edas.yaml b/config/crd/bases/eda.ansible.com_edas.yaml index 4256b9fa..b36c8be0 100644 --- a/config/crd/bases/eda.ansible.com_edas.yaml +++ b/config/crd/bases/eda.ansible.com_edas.yaml @@ -2151,6 +2151,9 @@ spec: dbFieldsEncryptionSecret: description: Database Fields Encryption secret name of the deployed instance type: string + upgradedPostgresVersion: + description: Status to indicate that the database has been upgraded to the version in the status + type: string image: description: URL of the image used for the deployed instance type: string diff --git a/roles/backup/vars/main.yml b/roles/backup/vars/main.yml index e7ce5a75..fe35903d 100644 --- a/roles/backup/vars/main.yml +++ b/roles/backup/vars/main.yml @@ -1,7 +1,7 @@ --- deployment_type: "eda" -_postgres_image: quay.io/sclorg/postgresql-13-c8s +_postgres_image: quay.io/sclorg/postgresql-15-c8s _postgres_image_version: latest backup_complete: false database_type: "unmanaged" -supported_pg_version: 13 +supported_pg_version: 15 diff --git a/roles/eda/tasks/update_status.yml b/roles/eda/tasks/update_status.yml index 21d03a67..6189375c 100644 --- a/roles/eda/tasks/update_status.yml +++ b/roles/eda/tasks/update_status.yml @@ -53,3 +53,14 @@ status: URL: "https://{{ route_url['resources'][0]['status']['ingress'][0]['host'] }}" when: ingress_type | lower == 'route' + + +- name: Update upgradedPostgresVersion status + operator_sdk.util.k8s_status: + api_version: '{{ api_version }}' + kind: "{{ kind }}" + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ ansible_operator_meta.namespace }}" + status: + upgradedPostgresVersion: "{{ upgraded_postgres_version | string }}" + when: upgraded_postgres_version is defined diff --git a/roles/postgres/defaults/main.yml b/roles/postgres/defaults/main.yml index 78c450ab..cfe0d594 100644 --- a/roles/postgres/defaults/main.yml +++ b/roles/postgres/defaults/main.yml @@ -4,7 +4,7 @@ deployment_type: eda database_name: "{{ deployment_type }}" database_username: "{{ deployment_type }}" -_postgres_image: quay.io/sclorg/postgresql-13-c8s +_postgres_image: quay.io/sclorg/postgresql-15-c8s _postgres_image_version: latest # Add a nodeSelector for the Postgres pods. diff --git a/roles/postgres/tasks/check_postgres_version.yml b/roles/postgres/tasks/check_postgres_version.yml new file mode 100644 index 00000000..c5a29389 --- /dev/null +++ b/roles/postgres/tasks/check_postgres_version.yml @@ -0,0 +1,76 @@ +--- + +# It is possible that N-2 postgres pods may still be present in the namespace from previous upgrades. +# So we have to take that into account and preferentially set the most recent one. +- name: Get the old postgres pod (N-1) + k8s_info: + kind: Pod + namespace: "{{ ansible_operator_meta.namespace }}" + field_selectors: + - status.phase=Running + register: _running_pods + +- block: + - name: Filter pods by name + set_fact: + filtered_old_postgres_pods: "{{ _running_pods.resources | + selectattr('metadata.name', 'match', ansible_operator_meta.name + '-postgres.*-0') | + rejectattr('metadata.name', 'search', '-' + supported_pg_version | string + '-0') | + list }}" + + # Sort pods by name in reverse order (most recent PG version first) and set + - name: Set info for previous postgres pod + set_fact: + sorted_old_postgres_pods: "{{ filtered_old_postgres_pods | + sort(attribute='metadata.name') | + reverse }}" + when: filtered_old_postgres_pods | length + + - name: Set info for previous postgres pod + set_fact: + old_postgres_pod: "{{ sorted_old_postgres_pods | first }}" + when: filtered_old_postgres_pods | length + when: _running_pods.resources | length + +- name: Look up details for this deployment + k8s_info: + api_version: "{{ api_version }}" + kind: "{{ kind }}" + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ ansible_operator_meta.namespace }}" + register: this_eda + +# If this deployment has been upgraded before or if upgrade has already been started, set this var +- name: Set previous PG version var + set_fact: + _previous_upgraded_pg_version: "{{ this_eda['resources'][0]['status']['upgradedPostgresVersion'] | default(false) }}" + when: + - "'upgradedPostgresVersion' in this_eda['resources'][0]['status']" + +- name: Check if postgres pod is running an older version + block: + - name: Set path to PG_VERSION file for given container image + set_fact: + path_to_pg_version: '{{ combined_database.postgres_data_path }}/PG_VERSION' + + - name: Get old PostgreSQL version + k8s_exec: + namespace: "{{ ansible_operator_meta.namespace }}" + pod: "{{ old_postgres_pod['metadata']['name'] }}" + command: | + bash -c """ + cat {{ path_to_pg_version }} + """ + register: _old_pg_version + + - debug: + msg: "--- Upgrading from {{ old_postgres_pod['metadata']['name'] | default('NONE')}} Pod ---" + + - name: Upgrade data dir from old Postgres to {{ supported_pg_version }} if applicable + include_tasks: upgrade_postgres.yml + when: + - (_old_pg_version.stdout | default(0) | int ) < supported_pg_version + when: + - managed_database + - (_previous_upgraded_pg_version | default(false)) | ternary(_previous_upgraded_pg_version < supported_pg_version, true) + - old_postgres_pod | length # If empty, then old pg pod has been removed and we can assume the upgrade is complete diff --git a/roles/postgres/tasks/main.yml b/roles/postgres/tasks/main.yml index 130f2fc0..4297ace3 100644 --- a/roles/postgres/tasks/main.yml +++ b/roles/postgres/tasks/main.yml @@ -13,6 +13,11 @@ - name: Set variables to be used in Postgres templates import_tasks: set_variables.yml +# Managed Database block +- name: Check PostgreSQL version to determine if an upgrade is needed + import_tasks: check_postgres_version.yml + when: managed_database + - name: Create managed Postgres StatefulSet if no external db is defined import_tasks: create_managed_postgres.yml when: managed_database diff --git a/roles/postgres/tasks/upgrade_postgres.yml b/roles/postgres/tasks/upgrade_postgres.yml new file mode 100644 index 00000000..429dfb69 --- /dev/null +++ b/roles/postgres/tasks/upgrade_postgres.yml @@ -0,0 +1,165 @@ +--- + +# Upgrade Posgres (Managed Databases only) +# * If postgres version is not supported_pg_version, and not an external postgres instance (when managed_database is yes), +# then run this playbook with include_tasks from database_configuration.yml +# * Data will be streamed via a pg_dump from the postgres 12/13 pod to the postgres supported_pg_version +# pod via a pg_restore. + + +- name: Scale down Deployment for migration + include_tasks: scale_down_deployment.yml + +- name: Delete existing postgres configuration secret + k8s: + api_version: v1 + kind: Secret + name: "{{ ansible_operator_meta.name }}-postgres-configuration" + namespace: "{{ ansible_operator_meta.namespace }}" + state: absent + wait: yes + +- name: Create Database configuration with new -postgres-{{ supported_pg_version }} hostname + k8s: + apply: true + definition: "{{ lookup('template', 'postgres_upgrade_secret.yaml.j2') }}" + no_log: "{{ no_log }}" + +- name: Set new database var to be used when configuring app credentials (resources_configuration.yml) + set_fact: + eda_postgres_host: "{{ ansible_operator_meta.name }}-postgres-{{ supported_pg_version }}" + no_log: "{{ no_log }}" + +- name: Create Database and StatefulSet if no database is specified + k8s: + apply: yes + definition: "{{ lookup('template', item + '.yaml.j2') }}" + wait: yes + loop: + - 'postgres.service' + - 'postgres.statefulset' + register: create_statefulset_result + +- name: Apply database resources + k8s: + apply: yes + definition: "{{ lookup('template', item + '.yaml.j2') }}" + wait: no + register: postgres_resources_result + loop: + - 'postgres.service' + - 'postgres.statefulset' + no_log: "{{ no_log }}" + +- name: Set postgres label if not defined by user + set_fact: + postgres_label_selector: "app.kubernetes.io/instance=postgres-{{ supported_pg_version }}-{{ ansible_operator_meta.name }}" + when: postgres_label_selector is not defined + +- name: Get new postgres pod information + k8s_info: + kind: Pod + namespace: "{{ ansible_operator_meta.namespace }}" + label_selectors: + - "{{ postgres_label_selector }}" + field_selectors: + - status.phase=Running + register: postgres_pod + until: + - "postgres_pod['resources'] | length" + - "postgres_pod['resources'][0]['status']['phase'] == 'Running'" + - "postgres_pod['resources'][0]['status']['containerStatuses'][0]['ready'] == true" + delay: 5 + retries: 60 + +- name: Set the resource pod name as a variable. + set_fact: + postgres_pod_name: "{{ postgres_pod['resources'][0]['metadata']['name'] }}" + +- name: Get the name of the service for the old postgres pod + k8s_info: + kind: Service + namespace: "{{ ansible_operator_meta.namespace }}" + label_selectors: + - "app.kubernetes.io/component=database" + - "app.kubernetes.io/instance={{ old_postgres_pod.metadata.labels['app.kubernetes.io/instance'] }}" + - "app.kubernetes.io/managed-by=eda-operator" + register: old_postgres_svc + +- name: Set full resolvable host name for postgres pod + set_fact: + resolvable_db_host: "{{ old_postgres_svc['resources'][0]['metadata']['name'] }}.{{ ansible_operator_meta.namespace }}.svc" # yamllint disable-line rule:line-length + no_log: "{{ no_log }}" + +- name: Set pg_dump command + set_fact: + pgdump: >- + pg_dump + -h {{ resolvable_db_host }} + -U {{ eda_postgres_user }} + -d {{ eda_postgres_database }} + -p {{ eda_postgres_port }} + -F custom + no_log: "{{ no_log }}" + +- name: Set pg_restore command + set_fact: + pg_restore: >- + pg_restore + -U {{ eda_postgres_user }} + -d {{ eda_postgres_database }} + no_log: "{{ no_log }}" + +- name: Stream backup from pg_dump to the new postgresql container + k8s_exec: + namespace: "{{ ansible_operator_meta.namespace }}" + pod: "{{ postgres_pod_name }}" + command: | + bash -c """ + set -e -o pipefail + PGPASSWORD='{{ eda_postgres_pass }}' {{ pgdump }} | PGPASSWORD='{{ eda_postgres_pass }}' {{ pg_restore }} + echo 'Successful' + """ + no_log: "{{ no_log }}" + register: data_migration + failed_when: "'Successful' not in data_migration.stdout" + +- name: Set flag signifying that this instance has been migrated + set_fact: + upgraded_postgres_version: '{{ supported_pg_version }}' + +# Cleanup old Postgres resources +- name: Remove old Postgres StatefulSet + k8s: + kind: StatefulSet + api_version: v1 + namespace: "{{ ansible_operator_meta.namespace }}" + name: "{{ item }}" + state: absent + wait: true + loop: + - "{{ ansible_operator_meta.name }}-postgres" + - "{{ ansible_operator_meta.name }}-postgres-13" + +- name: Remove old Postgres Service + k8s: + kind: Service + api_version: v1 + namespace: "{{ ansible_operator_meta.namespace }}" + name: "{{ item }}" + state: absent + loop: + - "{{ ansible_operator_meta.name }}-postgres" + - "{{ ansible_operator_meta.name }}-postgres-13" + +- name: Remove old persistent volume claim + k8s: + kind: PersistentVolumeClaim + api_version: v1 + namespace: "{{ ansible_operator_meta.namespace }}" + name: "{{ item }}" + state: absent + loop: + - "postgres-{{ ansible_operator_meta.name }}-postgres-0" + - "postgres-{{ ansible_operator_meta.name }}-postgres-13-0" + when: postgres_keep_pvc_after_upgrade diff --git a/roles/postgres/vars/main.yml b/roles/postgres/vars/main.yml index 08c74dee..0dbc544e 100644 --- a/roles/postgres/vars/main.yml +++ b/roles/postgres/vars/main.yml @@ -1,4 +1,6 @@ --- # vars file for PostgreSQL database -supported_pg_version: 13 +supported_pg_version: 15 +_previous_upgraded_pg_version: 0 +old_postgres_pod: [] diff --git a/roles/restore/README.md b/roles/restore/README.md index 7122d652..6a0741e9 100644 --- a/roles/restore/README.md +++ b/roles/restore/README.md @@ -19,7 +19,7 @@ This role assumes you are authenticated with an Openshift or Kubernetes cluster: *Before Restoring from a backup*, be sure to: - delete the old existing EDA CR - - delete the persistent volume claim (PVC) for the database from the old deployment, which has a name like `postgres-13--postgres-13-0` + - delete the persistent volume claim (PVC) for the database from the old deployment, which has a name like `postgres-15--postgres-15-0` **Note**: Do not delete the namespace/project, as that will delete the backup and the backup's PVC as well. diff --git a/roles/restore/vars/main.yml b/roles/restore/vars/main.yml index 9feb3de4..79a9b3e6 100644 --- a/roles/restore/vars/main.yml +++ b/roles/restore/vars/main.yml @@ -1,8 +1,8 @@ --- deployment_type: "eda" -supported_pg_version: 13 -_postgres_image: quay.io/sclorg/postgresql-13-c8s +supported_pg_version: 15 +_postgres_image: quay.io/sclorg/postgresql-15-c8s _postgres_image_version: latest backup_api_version: '{{ deployment_type }}.ansible.com/v1alpha1'