Merge pull request #1264 from jackyalbo/ja-pr1221-pg-upgrade

[Clone] Add upgrade process for postgres DB version 15

Makefile

@@ -156,7 +156,7 @@ gen-olm: gen
 
 gen-odf-package: cli
 	rm -rf $(MANIFESTS)
-	MANIFESTS="$(MANIFESTS)" CSV_NAME="$(csv-name)" SKIP_RANGE="$(skip-range)" REPLACES="$(replaces)" CORE_IMAGE="$(core-image)" DB_IMAGE="$(db-image)" OPERATOR_IMAGE="$(operator-image)" COSI_SIDECAR_IMAGE="$(cosi-sidecar-image)" OBC_CRD="$(obc-crd)" build/gen-odf-package.sh
+	MANIFESTS="$(MANIFESTS)" CSV_NAME="$(csv-name)" SKIP_RANGE="$(skip-range)" REPLACES="$(replaces)" CORE_IMAGE="$(core-image)" DB_IMAGE="$(db-image)" OPERATOR_IMAGE="$(operator-image)" COSI_SIDECAR_IMAGE="$(cosi-sidecar-image)" OBC_CRD="$(obc-crd)" PSQL_12_IMAGE="$(psql-12-image)" build/gen-odf-package.sh
 	@echo "✅ gen-odf-package"
 .PHONY: gen-odf-package
 

build/gen-odf-package.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-if [ "${MANIFESTS}" == "" ] || [ "${CSV_NAME}" == "" ] || [ "${CORE_IMAGE}" == "" ] || [ "${DB_IMAGE}" == "" ] || [ "${OPERATOR_IMAGE}" == "" ] || [ "${COSI_SIDECAR_IMAGE}" == "" ]
+if [ "${MANIFESTS}" == "" ] || [ "${CSV_NAME}" == "" ] || [ "${CORE_IMAGE}" == "" ] || [ "${PSQL_12_IMAGE}" == "" ] || [ "${DB_IMAGE}" == "" ] || [ "${OPERATOR_IMAGE}" == "" ] || [ "${COSI_SIDECAR_IMAGE}" == "" ]
 then
   echo "gen-odf-package.sh: not all required envs were supplied"
   exit 1
@@ -16,6 +16,7 @@ echo "--obc-crd=${OBC_CRD}"
 --replaces "${REPLACES}" \
 --noobaa-image ${CORE_IMAGE} \
 --db-image ${DB_IMAGE} \
+--psql-12-image ${PSQL_12_IMAGE} \
 --operator-image ${OPERATOR_IMAGE} \
 --cosi-sidecar-image ${COSI_SIDECAR_IMAGE} \
 --obc-crd=${OBC_CRD} 
@@ -34,6 +35,8 @@ cat >> ${temp_csv} << EOF
     name: noobaa-core
   - image: ${DB_IMAGE}
     name: noobaa-db
+  - image: ${PSQL_12_IMAGE}
+    name: noobaa-psql-12
   - image: ${OPERATOR_IMAGE}
     name: noobaa-operator
 EOF

deploy/crds/noobaa.io_noobaas.yaml

@@ -1630,6 +1630,10 @@ spec:
                 description: ActualImage is set to report which image the operator
                   is using
                 type: string
+              beforeUpgradeDbImage:
+                description: BeforeUpgradeDbImage is the db image used before last
+                  db upgrade
+                type: string
               conditions:
                 description: Conditions is a list of conditions related to operator
                   reconciliation
@@ -1689,6 +1693,10 @@ spec:
                 description: Phase is a simple, high-level summary of where the System
                   is in its lifecycle
                 type: string
+              postgresUpdatePhase:
+                description: Upgrade reports the status of the ongoing postgres upgrade
+                  process
+                type: string
               readme:
                 description: Readme is a user readable string with explanations on
                   the system

deploy/internal/configmap-postgres-initdb.yaml

@@ -37,3 +37,51 @@ data:
                  -e 's/^pg_ctl\sstart.*/pg_ctl start || true/'                                   \
                     /usr/bin/run-postgresql
           su postgres -c "bash -x /usr/bin/run-postgresql"
+
+            
+  dumpdb.sh: |
+          set -e
+          sed -i -e 's/^\(postgres:[^:]\):[0-9]*:[0-9]*:/\1:10001:0:/' /etc/passwd
+          su postgres -c "bash -x /usr/bin/run-postgresql" &
+          THRESHOLD=33
+          USE=$(df -h --output=pcent "/$HOME/data" | tail -n 1 | tr -d '[:space:]%')
+          # Check if the used space is more than the threshold
+          if [ "$USE" -gt "$THRESHOLD" ]; then
+            echo "Warning: Free space $USE% is above $THRESHOLD% threshold. Can't start upgrade!"
+            exit 1
+          fi
+          echo "Info: Free space $USE% is below $THRESHOLD% threshold. Starting upgrade!"
+          until pg_isready; do sleep 1; done;
+            pg_dumpall -U postgres > /$HOME/data/dump.sql
+          exit 0
+
+  upgradedb.sh: |
+          set -e
+          PGDATA=$HOME/data/userdata
+          PGDATA_12=$HOME/data/userdata-12
+          THRESHOLD=33
+          USE=$(df -h --output=pcent "/$HOME/data" | tail -n 1 | tr -d '[:space:]%')
+          # Check if the used space is more than the threshold
+          if [ "$USE" -gt "$THRESHOLD" ]; then
+            echo "Warning: Free space $USE% is above $THRESHOLD% threshold. Can't start upgrade!"
+            exit 1
+          fi
+          echo "Info: Free space $USE% is below $THRESHOLD% threshold. Starting upgrade!"
+          if [ ! -d $PGDATA_12 ]; then
+            mv $PGDATA $PGDATA_12
+          fi
+          sed -i -e 's/^\(postgres:[^:]\):[0-9]*:[0-9]*:/\1:10001:0:/' /etc/passwd
+          su postgres -c "bash -x /usr/bin/run-postgresql" &
+          until pg_isready; do sleep 1; done;
+          psql -U postgres < /$HOME/data/dump.sql
+          rm /$HOME/data/dump.sql
+          exit 0
+
+  revertdb.sh: |
+          PGDATA=$HOME/data/userdata
+          PGDATA_12=$HOME/data/userdata-12
+          if [ -d $PGDATA_12 ]; then
+            rm -rf $PGDATA
+            mv $PGDATA_12 $PGDATA
+          fi
+          exit 0

pkg/apis/noobaa/v1alpha1/noobaa_types.go

@@ -323,13 +323,21 @@ type NooBaaStatus struct {
 	// +optional
 	UpgradePhase UpgradePhase `json:"upgradePhase,omitempty"`
 
+	// Upgrade reports the status of the ongoing postgres upgrade process
+	// +optional
+	PostgresUpdatePhase UpgradePhase `json:"postgresUpdatePhase,omitempty"`
+
 	// Readme is a user readable string with explanations on the system
 	// +optional
 	Readme string `json:"readme,omitempty"`
 
 	// LastKeyRotateTime is the time system ran an encryption key rotate
 	// +optional
 	LastKeyRotateTime metav1.Time `json:"lastKeyRotateTime,omitempty"`
+
+	// BeforeUpgradeDbImage is the db image used before last db upgrade
+	// +optional
+	BeforeUpgradeDbImage *string `json:"beforeUpgradeDbImage,omitempty"`
 }
 
 // SystemPhase is a string enum type for system phases
@@ -472,6 +480,12 @@ const (
 	UpgradePhaseClean UpgradePhase = "Cleanning"
 
 	UpgradePhaseFinished UpgradePhase = "DoneUpgrade"
+
+	UpgradePhaseReverting UpgradePhase = "Reverting"
+
+	UpgradePhaseFailed UpgradePhase = "Failed"
+
+	UpgradePhaseUpgrade UpgradePhase = "Upgrading"
 )
 
 // CleanupPolicySpec specifies the cleanup policy

pkg/bundle/deploy.go

@@ -1471,7 +1471,7 @@ spec:
       status: {}
 `
 
-const Sha256_deploy_crds_noobaa_io_noobaas_yaml = "8f111f5299fc274593af4efb9565b34f36d36bd49f3666b19279504411ed0df0"
+const Sha256_deploy_crds_noobaa_io_noobaas_yaml = "87997842fc69b23f31b716777b6c3df7b12db2ae12758f890f63ce8d910a89f2"
 
 const File_deploy_crds_noobaa_io_noobaas_yaml = `---
 apiVersion: apiextensions.k8s.io/v1
@@ -3105,6 +3105,10 @@ spec:
                 description: ActualImage is set to report which image the operator
                   is using
                 type: string
+              beforeUpgradeDbImage:
+                description: BeforeUpgradeDbImage is the db image used before last
+                  db upgrade
+                type: string
               conditions:
                 description: Conditions is a list of conditions related to operator
                   reconciliation
@@ -3164,6 +3168,10 @@ spec:
                 description: Phase is a simple, high-level summary of where the System
                   is in its lifecycle
                 type: string
+              postgresUpdatePhase:
+                description: Upgrade reports the status of the ongoing postgres upgrade
+                  process
+                type: string
               readme:
                 description: Readme is a user readable string with explanations on
                   the system
@@ -3642,7 +3650,7 @@ data:
     shared_preload_libraries = 'pg_stat_statements'
 `
 
-const Sha256_deploy_internal_configmap_postgres_initdb_yaml = "016881f9a5e0561dbf10e7034dead0ee636556c162439d4d54c974a65253357c"
+const Sha256_deploy_internal_configmap_postgres_initdb_yaml = "9ce7163b6de6bf58c2804ca6be2efc69fef7c90951fa549d17fa08a3a2684fc8"
 
 const File_deploy_internal_configmap_postgres_initdb_yaml = `apiVersion: v1
 kind: ConfigMap
@@ -3683,6 +3691,54 @@ data:
                  -e 's/^pg_ctl\sstart.*/pg_ctl start || true/'                                   \
                     /usr/bin/run-postgresql
           su postgres -c "bash -x /usr/bin/run-postgresql"
+
+            
+  dumpdb.sh: |
+          set -e
+          sed -i -e 's/^\(postgres:[^:]\):[0-9]*:[0-9]*:/\1:10001:0:/' /etc/passwd
+          su postgres -c "bash -x /usr/bin/run-postgresql" &
+          THRESHOLD=33
+          USE=$(df -h --output=pcent "/$HOME/data" | tail -n 1 | tr -d '[:space:]%')
+          # Check if the used space is more than the threshold
+          if [ "$USE" -gt "$THRESHOLD" ]; then
+            echo "Warning: Free space $USE% is above $THRESHOLD% threshold. Can't start upgrade!"
+            exit 1
+          fi
+          echo "Info: Free space $USE% is below $THRESHOLD% threshold. Starting upgrade!"
+          until pg_isready; do sleep 1; done;
+            pg_dumpall -U postgres > /$HOME/data/dump.sql
+          exit 0
+
+  upgradedb.sh: |
+          set -e
+          PGDATA=$HOME/data/userdata
+          PGDATA_12=$HOME/data/userdata-12
+          THRESHOLD=33
+          USE=$(df -h --output=pcent "/$HOME/data" | tail -n 1 | tr -d '[:space:]%')
+          # Check if the used space is more than the threshold
+          if [ "$USE" -gt "$THRESHOLD" ]; then
+            echo "Warning: Free space $USE% is above $THRESHOLD% threshold. Can't start upgrade!"
+            exit 1
+          fi
+          echo "Info: Free space $USE% is below $THRESHOLD% threshold. Starting upgrade!"
+          if [ ! -d $PGDATA_12 ]; then
+            mv $PGDATA $PGDATA_12
+          fi
+          sed -i -e 's/^\(postgres:[^:]\):[0-9]*:[0-9]*:/\1:10001:0:/' /etc/passwd
+          su postgres -c "bash -x /usr/bin/run-postgresql" &
+          until pg_isready; do sleep 1; done;
+          psql -U postgres < /$HOME/data/dump.sql
+          rm /$HOME/data/dump.sql
+          exit 0
+
+  revertdb.sh: |
+          PGDATA=$HOME/data/userdata
+          PGDATA_12=$HOME/data/userdata-12
+          if [ -d $PGDATA_12 ]; then
+            rm -rf $PGDATA
+            mv $PGDATA_12 $PGDATA
+          fi
+          exit 0
 `
 
 const Sha256_deploy_internal_deployment_endpoint_yaml = "b3dab0839de6aa382833772ab278feb245067b27591dee988b29184de90961b6"

pkg/olm/olm.go

@@ -293,6 +293,10 @@ func GenerateCSV(opConf *operator.Conf, csvParams *generateCSVParams) *operv1.Cl
 					Name:  "NOOBAA_DB_IMAGE",
 					Value: options.DBImage,
 				},
+				corev1.EnvVar{
+					Name:  "NOOBAA_PSQL_12_IMAGE",
+					Value: options.Psql12Image,
+				},
 				corev1.EnvVar{
 					Name:  "ENABLE_NOOBAA_ADMISSION",
 					Value: "true",

pkg/options/options.go

@@ -76,7 +76,11 @@ var DBImage = "centos/mongodb-36-centos7"
 
 // DBPostgresImage is the default postgres db image url
 // currently it can not be overridden.
-var DBPostgresImage = "centos/postgresql-12-centos7"
+var DBPostgresImage = "quay.io/sclorg/postgresql-15-c9s"
+
+// Psql12Image is the default postgres12 db image url
+// currently it can not be overridden.
+var Psql12Image = "centos/postgresql-12-centos7"
 
 // DBMongoImage is the default mongo db image url
 // this is used during migration to solve issues where mongo STS referencing to postgres image
@@ -226,6 +230,10 @@ func init() {
 		&DBImage, "db-image",
 		DBImage, "The database container image",
 	)
+	FlagSet.StringVar(
+		&Psql12Image, "psql-12-image",
+		Psql12Image, "The database old container image",
+	)
 	FlagSet.StringVar(
 		&CosiSideCarImage, "cosi-sidecar-image",
 		CosiSideCarImage, "The cosi side car container image",