db upgrage: upgrade postgres 15 to postgres 16

The Postgres upgrade from 15 to 16 or any later versions can be done by adding the env variable named POSTGRES_UPGRADE = copy in the pg db pod.

More about upgrade process is here: https://www.postgresql.org/docs/current/pgupgrade.html

In our case, we achieve the upgrade by setting this env var in noobaa-db-pg STS.

Once after upgrade is done, we have to remove the POSTGRES_UPGRADE env from the STS. Otherwise pg db pod fails to come up because the db pod check PG directory version against upgrade image version and exits if there is no upgrade required and POSTGRES_UPGRADE is still provided.

To remove this env, we store the upgrade status in noobaa-core CR after the PG upgrade and refer the same status during db reconcile. If the upgrade is already done, we remove this env from desired sts status.

This will help to upgrade the db smoothly.

Signed-off-by: Vinayak Hariharmath <[email protected]>

pkg/options/options.go

@@ -72,7 +72,7 @@ var NooBaaImage = ContainerImage
 
 // DBImage is the default db image url
 // it can be overridden for testing or different registry locations.
-var DBImage = "quay.io/sclorg/postgresql-15-c9s"
+var DBImage = "quay.io/sclorg/postgresql-16-c9s"
 
 // Psql12Image is the default postgres12 db image url
 // currently it can not be overridden.

pkg/system/phase2_creating.go

@@ -33,6 +33,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 )
 
 const (
@@ -130,6 +131,12 @@ func (r *Reconciler) ReconcilePhaseCreatingForMainClusters() error {
 			return err
 		}
 	}
+
+	// Once the db is reconciled, upgrade it if it is required
+	if err := r.UpgradeDb(); err != nil {
+		return err
+	}
+
 	// create bucket logging pvc if not provided by user for 'Guaranteed' logging in ODF env
 	if r.NooBaa.Spec.BucketLogging.LoggingType == nbv1.BucketLoggingTypeGuaranteed {
 		if err := r.ReconcileODFBucketLoggingPVC(); err != nil {
@@ -152,6 +159,47 @@ func (r *Reconciler) ReconcilePhaseCreatingForMainClusters() error {
 	return nil
 }
 
+func (r *Reconciler) UpgradeDb() error {
+	var result controllerutil.OperationResult
+	var err error
+
+	// Check the need for upgrade
+	if r.checkDBUpgradeReqd() {
+		r.NooBaa.Status.PostgresUpdatePhase = nbv1.UpgradePhaseUpgrade
+	} else {
+		if r.NooBaa.Status.PostgresUpdatePhase == nbv1.UpgradePhaseFinished {
+			_, err = controllerutil.CreateOrUpdate(
+				r.Ctx, r.Client, r.NooBaaPostgresDB, func() error {
+					r.unSetPGUpgradeEnvInSTS()
+					return nil
+				},
+			)
+			return err
+		} else {
+			// No need of upgrade and also env for upgrade is removed
+			return nil
+		}
+	}
+
+	// Upgrade db image in db sts
+	if r.NooBaa.Status.PostgresUpdatePhase != nbv1.UpgradePhaseFinished {
+		result, err = controllerutil.CreateOrUpdate(
+			r.Ctx, r.Client, r.NooBaaPostgresDB, func() error {
+				r.setPGUpgradeEnvInSTS()
+				r.updateDBImageForUpgrade()
+				return nil
+			},
+		)
+	}
+
+	// Set the upgrade finished flag here and refer the same to remove the
+	// POSTGRES_UPGRADE flag in next reconcile.
+	if result == controllerutil.OperationResultUpdated {
+		r.NooBaa.Status.PostgresUpdatePhase = nbv1.UpgradePhaseFinished
+	}
+	return err
+}
+
 // SetDesiredServiceAccount updates the ServiceAccount as desired for reconciling
 func (r *Reconciler) SetDesiredServiceAccount() error {
 	if r.ServiceAccount.Annotations == nil {
@@ -223,6 +271,82 @@ func (r *Reconciler) SetDesiredServiceDBForPostgres() error {
 	return nil
 }
 
+func (r *Reconciler) updateDBImageForUpgrade() {
+	var podSpec = &r.NooBaaPostgresDB.Spec.Template.Spec
+
+	for i := range podSpec.Containers {
+		c := &podSpec.Containers[i]
+		if c.Name == "db" {
+			// Fetch the DB image from options.go
+			c.Image = options.DBImage
+			if r.NooBaa.Spec.DBResources != nil {
+				c.Resources = *r.NooBaa.Spec.DBResources
+			}
+
+			c.Lifecycle = &corev1.Lifecycle{
+				PreStop: &corev1.LifecycleHandler{
+					Exec: &corev1.ExecAction{
+						Command: []string{"/bin/sh", "-c", "pg_ctl -D /var/lib/pgsql/data/userdata/ -w -t 60 -m fast stop"},
+					},
+				},
+			}
+
+		}
+	}
+}
+
+// Check wheter db upgrade required
+func (r *Reconciler) checkDBUpgradeReqd() bool {
+	for _, container := range r.NooBaaPostgresDB.Spec.Template.Spec.Containers {
+		if container.Name == "db" {
+			if container.Image != options.DBImage {
+				return true
+			}
+			break
+		}
+	}
+	return false
+}
+
+// Set env "POSTGRESQL_UPGRADE=copy" in PG sts to iniate the upgrade
+func (r *Reconciler) setPGUpgradeEnvInSTS() {
+	for i, container := range r.NooBaaPostgresDB.Spec.Template.Spec.Containers {
+		if container.Name == "db" {
+			for _, env := range container.Env {
+				if env.Name == "POSTGRESQL_UPGRADE" {
+					return
+				}
+			}
+			envVars := container.Env
+			newEnvVar := corev1.EnvVar{
+				Name:  "POSTGRESQL_UPGRADE",
+				Value: "copy",
+			}
+
+			envVars = append(envVars, newEnvVar)
+			r.NooBaaPostgresDB.Spec.Template.Spec.Containers[i].Env = envVars
+			return
+		}
+	}
+}
+
+// unSet env "POSTGRESQL_UPGRADE=copy" in PG sts after upgrade
+func (r *Reconciler) unSetPGUpgradeEnvInSTS() {
+	for i, container := range r.NooBaaPostgresDB.Spec.Template.Spec.Containers {
+		if container.Name == "db" {
+			newEnvVars := []corev1.EnvVar{}
+			for _, env := range container.Env {
+				if env.Name != "POSTGRESQL_UPGRADE" {
+					newEnvVars = append(newEnvVars, env)
+				}
+			}
+			r.NooBaaPostgresDB.Spec.Template.Spec.Containers[i].Env = newEnvVars
+			return
+		}
+	}
+	r.NooBaa.Status.PostgresUpdatePhase = nbv1.UpgradePhaseNone
+}
+
 // SetDesiredNooBaaDB updates the NooBaaDB as desired for reconciling
 func (r *Reconciler) SetDesiredNooBaaDB() error {
 	var NooBaaDBTemplate *appsv1.StatefulSet = nil