Skip to content

Commit

Permalink
Rework the upgrade logic to be more resiliant for multiple upgrades
Browse files Browse the repository at this point in the history
  • Loading branch information
rooftopcellist committed Aug 15, 2023
1 parent 94cb1e1 commit 1c87383
Show file tree
Hide file tree
Showing 7 changed files with 77 additions and 25 deletions.
20 changes: 11 additions & 9 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,15 @@ Have questions about this document or anything not covered here? Please file a n

## Table of contents

* [Things to know prior to submitting code](#things-to-know-prior-to-submitting-code)
* [Submmiting your Work](#submitting-your-work)
* [Testing](#testing)
* [Testing in Docker](#testing-in-docker)
* [Testing in Minikube](#testing-in-minikube)
* [Generating a bundle](#generating-a-bundle)
* [Reporting Issues](#reporting-issues)
- [AWX-Operator Contributing Guidelines](#awx-operator-contributing-guidelines)
- [Table of contents](#table-of-contents)
- [Things to know prior to submitting code](#things-to-know-prior-to-submitting-code)
- [Submmiting your work](#submmiting-your-work)
- [Testing](#testing)
- [Testing in Kind](#testing-in-kind)
- [Testing in Minikube](#testing-in-minikube)
- [Generating a bundle](#generating-a-bundle)
- [Reporting Issues](#reporting-issues)


## Things to know prior to submitting code
Expand Down Expand Up @@ -44,12 +46,12 @@ Have questions about this document or anything not covered here? Please file a n

## Testing

This Operator includes a [Molecule](https://molecule.readthedocs.io/en/stable/)-based test environment, which can be executed standalone in Docker (e.g. in CI or in a single Docker container anywhere), or inside any kind of Kubernetes cluster (e.g. Minikube).
This Operator includes a [Molecule](https://ansible.readthedocs.io/projects/molecule/)-based test environment, which can be executed standalone in Docker (e.g. in CI or in a single Docker container anywhere), or inside any kind of Kubernetes cluster (e.g. Minikube).

You need to make sure you have Molecule installed before running the following commands. You can install Molecule with:

```sh
#> pip install 'molecule[docker]'
#> python -m pip install molecule-plugins[docker]
```

Running `molecule test` sets up a clean environment, builds the operator, runs all configured tests on an example operator instance, then tears down the environment (at least in the case of Docker).
Expand Down
4 changes: 2 additions & 2 deletions config/manager/kustomization.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@ generatorOptions:
disableNameSuffixHash: true

configMapGenerator:
- name: awx-manager-config
files:
- files:
- controller_manager_config.yaml
name: awx-manager-config

apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
Expand Down
2 changes: 1 addition & 1 deletion molecule/default/tasks/awx_replicas_test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -61,4 +61,4 @@
expected_web_replicas: 3
expected_task_replicas: 3
tags:
- replicas
- replicas
46 changes: 38 additions & 8 deletions roles/installer/tasks/database_configuration.yml
Original file line number Diff line number Diff line change
Expand Up @@ -106,14 +106,35 @@
set_fact:
managed_database: "{{ pg_config['resources'][0]['data']['type'] | default('') | b64decode == 'managed' }}"

- name: Get the old postgres pod information
# It is possible that N-2 postgres pods may still be present in the namespace from previous upgrades.
# So we have to take that into account and preferentially set the most recent one.
- name: Get the old postgres pod (N-1)
k8s_info:
kind: Pod
namespace: "{{ ansible_operator_meta.namespace }}"
name: "{{ ansible_operator_meta.name }}-postgres-0"
field_selectors:
- status.phase=Running
register: old_postgres_pod
register: _old_postgres_pods

- name: Filter pods by name
set_fact:
filtered_old_postgres_pods: "{{ _old_postgres_pods.resources |
selectattr('metadata.name', 'match', ansible_operator_meta.name + '-postgres.*-0') |
rejectattr('metadata.name', 'search', '-' + supported_pg_version | string + '-0') |
list }}"

# Sort pods by name in reverse order (most recent PG version first) and set
- name: Set info for previous postgres pod
set_fact:
sorted_old_postgres_pods: "{{ filtered_old_postgres_pods |
sort(attribute='metadata.name') |
reverse }}"
when: _old_postgres_pods.resources | length

- name: Set info for previous postgres pod
set_fact:
old_postgres_pod: "{{ sorted_old_postgres_pods | first }}"
when: _old_postgres_pods | length

- name: Look up details for this deployment
k8s_info:
Expand All @@ -123,6 +144,13 @@
namespace: "{{ ansible_operator_meta.namespace }}"
register: this_awx

# If this deployment has been upgraded before or if upgrade has already been started, set this var
- name: Set previous PG version var
set_fact:
_previous_upgraded_pg_version: "{{ this_awx['resources'][0]['status']['upgradedPostgresVersion'] | default(false) }}"
when:
- "'upgradedPostgresVersion' in this_awx['resources'][0]['status']"

- name: Check if postgres pod is running an older version
block:
- name: Set path to PG_VERSION file for given container image
Expand All @@ -132,22 +160,24 @@
- name: Get old PostgreSQL version
k8s_exec:
namespace: "{{ ansible_operator_meta.namespace }}"
pod: "{{ ansible_operator_meta.name }}-postgres-0"
pod: "{{ old_postgres_pod['metadata']['name'] }}"
command: |
bash -c """
cat {{ path_to_pg_version }}
"""
register: _old_pg_version

- debug:
msg: "--- Upgrading from {{ old_postgres_pod['metadata']['name'] | default('NONE')}} Pod ---"

- name: Upgrade data dir from old Postgres to {{ supported_pg_version }} if applicable
include_tasks: upgrade_postgres.yml
when:
- _old_pg_version.stdout | default('0') | trim < supported_pg_version
- (_old_pg_version.stdout | default(0) | int ) < supported_pg_version
when:
- managed_database
- "'upgradedPostgresVersion' is in this_awx['resources'][0]['status']"
- (this_awx['resources'][0]['status']['upgradedPostgresVersion'] | default(false)) | ternary (this_awx['resources'][0]['status']['upgradedPostgresVersion'] < supported_pg_version, false)
- old_postgres_pod['resources'] | length # upgrade is complete and old pg pod has been removed
- (_previous_upgraded_pg_version | default(false)) | ternary(_previous_upgraded_pg_version < supported_pg_version, true)
- old_postgres_pod | length # If empty, then old pg pod has been removed and we can assume the upgrade is complete

- block:
- name: Create Database if no database is specified
Expand Down
2 changes: 1 addition & 1 deletion roles/installer/tasks/update_status.yml
Original file line number Diff line number Diff line change
Expand Up @@ -111,5 +111,5 @@
name: "{{ ansible_operator_meta.name }}"
namespace: "{{ ansible_operator_meta.namespace }}"
status:
upgradedPostgresVersion: "{{ upgraded_postgres_version }}"
upgradedPostgresVersion: "{{ upgraded_postgres_version | string }}"
when: upgraded_postgres_version is defined
27 changes: 23 additions & 4 deletions roles/installer/tasks/upgrade_postgres.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,9 +62,19 @@
set_fact:
postgres_pod_name: "{{ postgres_pod['resources'][0]['metadata']['name'] }}"

- name: Get the name of the service for the old postgres pod
k8s_info:
kind: Service
namespace: "{{ ansible_operator_meta.namespace }}"
label_selectors:
- "app.kubernetes.io/component=database"
- "app.kubernetes.io/instance={{ old_postgres_pod.metadata.labels['app.kubernetes.io/instance'] }}"
- "app.kubernetes.io/managed-by=awx-operator"
register: old_postgres_svc

- name: Set full resolvable host name for postgres pod
set_fact:
resolvable_db_host: "{{ ansible_operator_meta.name }}-postgres.{{ ansible_operator_meta.namespace }}.svc" # yamllint disable-line rule:line-length
resolvable_db_host: "{{ old_postgres_svc['resources'][0]['metadata']['name'] }}.{{ ansible_operator_meta.namespace }}.svc" # yamllint disable-line rule:line-length
no_log: "{{ no_log }}"

- name: Set pg_dump command
Expand Down Expand Up @@ -110,23 +120,32 @@
kind: StatefulSet
api_version: v1
namespace: "{{ ansible_operator_meta.namespace }}"
name: "{{ ansible_operator_meta.name }}-postgres"
name: "{{ item }}"
state: absent
wait: true
loop:
- "{{ ansible_operator_meta.name }}-postgres"
- "{{ ansible_operator_meta.name }}-postgres-13"

- name: Remove old Postgres Service
k8s:
kind: Service
api_version: v1
namespace: "{{ ansible_operator_meta.namespace }}"
name: "{{ ansible_operator_meta.name }}-postgres"
name: "{{ item }}"
state: absent
loop:
- "{{ ansible_operator_meta.name }}-postgres"
- "{{ ansible_operator_meta.name }}-postgres-13"

- name: Remove old persistent volume claim
k8s:
kind: PersistentVolumeClaim
api_version: v1
namespace: "{{ ansible_operator_meta.namespace }}"
name: "postgres-{{ ansible_operator_meta.name }}-postgres-0"
name: "{{ item }}"
state: absent
loop:
- "postgres-{{ ansible_operator_meta.name }}-postgres-0"
- "postgres-{{ ansible_operator_meta.name }}-postgres-13-0"
when: postgres_keep_pvc_after_upgrade
1 change: 1 addition & 0 deletions roles/installer/vars/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,4 @@ ldap_cacert_ca_crt: ''
bundle_ca_crt: ''
projects_existing_claim: ''
supported_pg_version: 15
_previous_upgraded_pg_version: 0

0 comments on commit 1c87383

Please sign in to comment.