From 0fa629c83c210e538c634597cda69a278afd40ea Mon Sep 17 00:00:00 2001 From: Vinayak Hariharmath Date: Mon, 19 Aug 2024 16:44:27 +0530 Subject: [PATCH] db upgrage: upgrade postgres 15 to postgres 16 The Postgres upgrade from 15 to 16 or any later versions can be done by adding the env variable named POSTGRES_UPGRADE = copy in the pg db pod. More about upgrade process is here: https://www.postgresql.org/docs/current/pgupgrade.html In our case, we achieve the upgrade by setting this env var in noobaa-db-pg STS. Once after upgrade is done, we have to remove the POSTGRES_UPGRADE env from the STS. Otherwise pg db pod fails to come up because the db pod check PG directory version against upgrade image version and exits if there is no upgrade required and POSTGRES_UPGRADE is still provided. To remove this env, we store the upgrade status in noobaa-core CR after the PG upgrade and refer the same status during db reconcile. If the upgrade is already done, we remove this env from desired sts status. This will help to upgrade the db smoothly. Signed-off-by: Vinayak Hariharmath --- deploy/crds/noobaa.io_noobaas.yaml | 4 + pkg/apis/noobaa/v1alpha1/noobaa_types.go | 12 +- pkg/bundle/deploy.go | 6 +- pkg/options/options.go | 7 +- pkg/system/phase2_creating.go | 273 +++++++++++++++++++++-- 5 files changed, 280 insertions(+), 22 deletions(-) diff --git a/deploy/crds/noobaa.io_noobaas.yaml b/deploy/crds/noobaa.io_noobaas.yaml index 639a844541..babdca7e56 100644 --- a/deploy/crds/noobaa.io_noobaas.yaml +++ b/deploy/crds/noobaa.io_noobaas.yaml @@ -1911,6 +1911,10 @@ spec: description: Phase is a simple, high-level summary of where the System is in its lifecycle type: string + postgresMajorVersion: + description: PostgresMajorVersion is the major version of postgress + db image + type: string postgresUpdatePhase: description: Upgrade reports the status of the ongoing postgres upgrade process diff --git a/pkg/apis/noobaa/v1alpha1/noobaa_types.go b/pkg/apis/noobaa/v1alpha1/noobaa_types.go index 829621dda7..648b335360 100644 --- a/pkg/apis/noobaa/v1alpha1/noobaa_types.go +++ b/pkg/apis/noobaa/v1alpha1/noobaa_types.go @@ -385,6 +385,10 @@ type NooBaaStatus struct { // BeforeUpgradeDbImage is the db image used before last db upgrade // +optional BeforeUpgradeDbImage *string `json:"beforeUpgradeDbImage,omitempty"` + + // PostgresMajorVersion is the major version of postgress db image + // +optional + PostgresMajorVersion PostgresMajorVersion `json:"postgresMajorVersion,omitempty"` } // SystemPhase is a string enum type for system phases @@ -514,6 +518,12 @@ type EndpointsStatus struct { VirtualHosts []string `json:"virtualHosts"` } +type PostgresMajorVersion string + +const ( + PostgresMajorVersionV16 PostgresMajorVersion = "16" +) + // UpgradePhase is a string enum type for upgrade phases type UpgradePhase string @@ -525,7 +535,7 @@ const ( UpgradePhaseMigrate UpgradePhase = "Migrating" - UpgradePhaseClean UpgradePhase = "Cleanning" + UpgradePhaseClean UpgradePhase = "Cleaning" UpgradePhaseFinished UpgradePhase = "DoneUpgrade" diff --git a/pkg/bundle/deploy.go b/pkg/bundle/deploy.go index 1b95f6a199..0abf708b3c 100644 --- a/pkg/bundle/deploy.go +++ b/pkg/bundle/deploy.go @@ -1423,7 +1423,7 @@ spec: status: {} ` -const Sha256_deploy_crds_noobaa_io_noobaas_yaml = "e862d263d097ed43f774784eaaf9a616967746b67608fadbe4ca71d93b220ab6" +const Sha256_deploy_crds_noobaa_io_noobaas_yaml = "5bfeed22b7201dfc6d56cda47e573dc0f5cac7350806b142c76509ad73b070b0" const File_deploy_crds_noobaa_io_noobaas_yaml = `--- apiVersion: apiextensions.k8s.io/v1 @@ -3338,6 +3338,10 @@ spec: description: Phase is a simple, high-level summary of where the System is in its lifecycle type: string + postgresMajorVersion: + description: PostgresMajorVersion is the major version of postgress + db image + type: string postgresUpdatePhase: description: Upgrade reports the status of the ongoing postgres upgrade process diff --git a/pkg/options/options.go b/pkg/options/options.go index ebb9e46b12..d524e88a04 100644 --- a/pkg/options/options.go +++ b/pkg/options/options.go @@ -4,6 +4,7 @@ import ( "github.com/noobaa/noobaa-operator/v5/pkg/util" "github.com/noobaa/noobaa-operator/v5/version" + nbv1 "github.com/noobaa/noobaa-operator/v5/pkg/apis/noobaa/v1alpha1" "github.com/spf13/cobra" "github.com/spf13/pflag" ) @@ -72,7 +73,11 @@ var NooBaaImage = ContainerImage // DBImage is the default db image url // it can be overridden for testing or different registry locations. -var DBImage = "quay.io/sclorg/postgresql-15-c9s" +var DBImage = "quay.io/sclorg/postgresql-16-c9s" + +// Postgress Major version is the major version of Postgress image +// This is referred for postgres upgrade. +var PostgresMajorVersion = nbv1.PostgresMajorVersionV16 // Psql12Image is the default postgres12 db image url // currently it can not be overridden. diff --git a/pkg/system/phase2_creating.go b/pkg/system/phase2_creating.go index d01ce0e0a6..6ab90395c2 100644 --- a/pkg/system/phase2_creating.go +++ b/pkg/system/phase2_creating.go @@ -123,6 +123,10 @@ func (r *Reconciler) ReconcilePhaseCreatingForMainClusters() error { // create the db only if postgres secret is not given if r.NooBaa.Spec.ExternalPgSecret == nil { + if err := r.UpgradeDbPostgres16(); err != nil { + return err + } + if err := r.ReconcileDB(); err != nil { return err } @@ -131,6 +135,7 @@ func (r *Reconciler) ReconcilePhaseCreatingForMainClusters() error { return err } } + // create bucket logging pvc if not provided by user for 'Guaranteed' logging in ODF env if r.NooBaa.Spec.BucketLogging.LoggingType == nbv1.BucketLoggingTypeGuaranteed { if err := r.ReconcileODFPersistentLoggingPVC( @@ -145,13 +150,13 @@ func (r *Reconciler) ReconcilePhaseCreatingForMainClusters() error { // create notification log pvc if bucket notifications is enabled and pvc was not set explicitly if r.NooBaa.Spec.BucketNotifications.Enabled { - if err := r.ReconcileODFPersistentLoggingPVC( - "bucketNotifications.pvc", - "InvalidBucketNotificationConfiguration", - "Bucket notifications requires a Persistent Volume Claim (PVC) with ReadWriteMany (RWX) access mode. Please specify the 'bucketNotifications.pvc'.", - r.NooBaa.Spec.BucketNotifications.PVC, - r.BucketNotificationsPVC); err != nil { - return err + if err := r.ReconcileODFPersistentLoggingPVC( + "bucketNotifications.pvc", + "InvalidBucketNotificationConfiguration", + "Bucket notifications requires a Persistent Volume Claim (PVC) with ReadWriteMany (RWX) access mode. Please specify the 'bucketNotifications.pvc'.", + r.NooBaa.Spec.BucketNotifications.PVC, + r.BucketNotificationsPVC); err != nil { + return err } } @@ -171,6 +176,153 @@ func (r *Reconciler) ReconcilePhaseCreatingForMainClusters() error { return nil } +func (r *Reconciler) UpgradeDbPostgres16() error { + if r.NooBaa.Status.PostgresUpdatePhase == nbv1.UpgradePhaseFinished && + r.NooBaa.Status.PostgresMajorVersion == nbv1.PostgresMajorVersionV16 { + r.Logger.Infof("UpgradePostgresDB: DB is already upgraded to postgresql-16") + return nil + } + + if r.NooBaa.Status.PostgresUpdatePhase != nbv1.UpgradePhasePrepare && + r.NooBaa.Status.PostgresUpdatePhase != nbv1.UpgradePhaseUpgrade && + r.NooBaa.Status.PostgresUpdatePhase != nbv1.UpgradePhaseClean { + sts := util.KubeObject(bundle.File_deploy_internal_statefulset_postgres_db_yaml).(*appsv1.StatefulSet) + sts.Name = "noobaa-db-pg" + sts.Namespace = r.NooBaa.Namespace + + // Postgres database doesn't exists, no need to upgrade. + if !util.KubeCheckQuiet(sts) { + r.Logger.Infof("UpgradePostgresDB: old STS doesn't exist - no need for upgrade") + r.NooBaa.Status.PostgresUpdatePhase = nbv1.UpgradePhaseNone + } else { + desiredImage := GetDesiredDBImage(r.NooBaa, "") + // Check whether upgrade is reqd, set phase depending the requirement + if r.isPostgresUpgradeReqd(desiredImage, sts) { + r.Logger.Infof("UpgradePostgresDB: upgrading postgres16") + r.NooBaa.Status.PostgresUpdatePhase = nbv1.UpgradePhasePrepare + } else { + if r.NooBaa.Status.PostgresMajorVersion == nbv1.PostgresMajorVersionV16 { + r.NooBaa.Status.PostgresUpdatePhase = nbv1.UpgradePhaseFinished + } else if strings.Contains(desiredImage, "postgresql-16") { + r.NooBaa.Status.PostgresMajorVersion = nbv1.PostgresMajorVersionV16 + r.NooBaa.Status.PostgresUpdatePhase = nbv1.UpgradePhaseFinished + } else { + r.NooBaa.Status.PostgresUpdatePhase = nbv1.UpgradePhaseNone + } + + } + } + + } + + switch r.NooBaa.Status.PostgresUpdatePhase { + case nbv1.UpgradePhasePrepare: + if err := r.setEndpointsDeploymentReplicas(0); err != nil { + r.Logger.Errorf("UpgradePostgresDB::got error on endpoints deployment reconcile %v", err) + return err + } + desiredImage := GetDesiredDBImage(r.NooBaa, "") + if err := r.ReconcileObject( + r.NooBaaPostgresDB, func() error { + r.updateDBImageForUpgrade(desiredImage) + r.setPGUpgradeEnvInSTS() + return nil + }, + ); err != nil { + r.Logger.Errorf("got error on postgres STS reconcile %v", err) + break + } + + // Update the DB pod + restartError := r.RestartDbPods() + if restartError != nil { + r.Logger.Warn("UpgradePostgresDB: Unable to restart db pods") + } + + r.NooBaa.Status.PostgresUpdatePhase = nbv1.UpgradePhaseUpgrade + case nbv1.UpgradePhaseUpgrade: + dbPod := &corev1.Pod{} + dbPod.Name = "noobaa-db-pg-0" + dbPod.Namespace = r.NooBaaPostgresDB.Namespace + if !util.KubeCheckQuiet(dbPod) { + return nil + } + // make sure previous step has finished + if dbPod.ObjectMeta.DeletionTimestamp != nil { + r.Logger.Infof("UpgradePostgresDB: upgrade-db is not yet running, phase is: %s and deletion time stamp is %v", + dbPod.Status.Phase, dbPod.ObjectMeta.DeletionTimestamp) + return nil + } + + status := r.checkDbContainerStatus(dbPod) + if status == "PodInitializing" { + return nil + } + + if status == "Running" { + r.NooBaa.Status.PostgresUpdatePhase = nbv1.UpgradePhaseClean + break + } + // TODO: status = CLBO and status = ERROR cases need to be handled + // Till then check for the status again + + case nbv1.UpgradePhaseClean: + // Check the DB status again, Set phaseFinished only it is in Running state + dbPod := &corev1.Pod{} + dbPod.Name = "noobaa-db-pg-0" + dbPod.Namespace = r.NooBaaPostgresDB.Namespace + if !util.KubeCheckQuiet(dbPod) { + return nil + } + status := r.checkDbContainerStatus(dbPod) + if status == "Running" { + r.NooBaa.Status.PostgresMajorVersion = nbv1.PostgresMajorVersionV16 + r.NooBaa.Status.PostgresUpdatePhase = nbv1.UpgradePhaseFinished + } else { + // TODO: status = CLBO and status = ERROR cases need to be handled + // Till then check for the status again + break + } + + if err := r.ReconcileObject(r.NooBaaPostgresDB, func() error { + r.unSetPGUpgradeEnvInSTS() + return nil + }); err != nil { + r.Logger.Infof("UpgradePostgresDB: Cleaning of POSTGRES_UPGRADE env failed: err %v, retry", err) + return nil + } + + if err := r.setEndpointsDeploymentReplicas(1); err != nil { + r.Logger.Errorf("UpgradePostgresDB::got error on endpoints deployment reconcile %v", err) + return err + } + } + + if err := r.UpdateStatus(); err != nil { + return err + } + return nil +} + +// checkDbContainerStatus checks if postgres upgrade container failed +func (r *Reconciler) checkDbContainerStatus(dbPod *corev1.Pod) string { + for i := range dbPod.Status.ContainerStatuses { + if dbPod.Status.ContainerStatuses[i].Name == "db" { + r.Logger.Infof("HasDbContainerFailed: Checking state of upgrade-db container: %v", dbPod.Status.ContainerStatuses[i].State) + if dbPod.Status.ContainerStatuses[i].State.Terminated != nil { + return dbPod.Status.ContainerStatuses[i].State.Terminated.Reason + } + if dbPod.Status.ContainerStatuses[i].State.Waiting != nil { + return dbPod.Status.ContainerStatuses[i].State.Waiting.Reason + } + if dbPod.Status.ContainerStatuses[i].State.Running != nil { + return "Running" + } + } + } + return "NotFound" +} + // SetDesiredServiceAccount updates the ServiceAccount as desired for reconciling func (r *Reconciler) SetDesiredServiceAccount() error { if r.ServiceAccount.Annotations == nil { @@ -242,6 +394,82 @@ func (r *Reconciler) SetDesiredServiceDBForPostgres() error { return nil } +func (r *Reconciler) updateDBImageForUpgrade(dbImage string) { + var podSpec = &r.NooBaaPostgresDB.Spec.Template.Spec + + for i := range podSpec.Containers { + c := &podSpec.Containers[i] + if c.Name == "db" { + // Fetch the DB image from options.go + c.Image = dbImage + } + } +} + +// SetEndpointsDeploymentReplicas updates the number of replicas on the endpoints deployment +func (r *Reconciler) setEndpointsDeploymentReplicas(replicas int32) error { + r.Logger.Infof("UpgradeMigrateDB:: setting endpoints replica count to %d", replicas) + return r.ReconcileObject(r.DeploymentEndpoint, func() error { + r.DeploymentEndpoint.Spec.Replicas = &replicas + return nil + }) +} + +// Returns true only if desired image and PG major version does not match +func (r *Reconciler) isPostgresUpgradeReqd(desiredImage string, sts *appsv1.StatefulSet) bool { + var currentImage string + + for _, container := range sts.Spec.Template.Spec.Containers { + if container.Name == "db" { + currentImage = container.Image + } + } + if currentImage != desiredImage && + r.NooBaa.Status.PostgresMajorVersion != nbv1.PostgresMajorVersionV16 { + return true + } + return false +} + +// Set env "POSTGRESQL_UPGRADE=copy" in PG sts to iniate the upgrade +func (r *Reconciler) setPGUpgradeEnvInSTS() { + for i, container := range r.NooBaaPostgresDB.Spec.Template.Spec.Containers { + if container.Name == "db" { + for _, env := range container.Env { + if env.Name == "POSTGRESQL_UPGRADE" { + return + } + } + envVars := container.Env + newEnvVar := corev1.EnvVar{ + Name: "POSTGRESQL_UPGRADE", + Value: "copy", + } + + envVars = append(envVars, newEnvVar) + r.NooBaaPostgresDB.Spec.Template.Spec.Containers[i].Env = envVars + return + } + } +} + +// unSet env "POSTGRESQL_UPGRADE=copy" in PG sts after upgrade +func (r *Reconciler) unSetPGUpgradeEnvInSTS() { + for i, container := range r.NooBaaPostgresDB.Spec.Template.Spec.Containers { + if container.Name == "db" { + newEnvVars := []corev1.EnvVar{} + for _, env := range container.Env { + if env.Name != "POSTGRESQL_UPGRADE" { + newEnvVars = append(newEnvVars, env) + } + } + r.NooBaaPostgresDB.Spec.Template.Spec.Containers[i].Env = newEnvVars + return + } + } + r.NooBaa.Status.PostgresUpdatePhase = nbv1.UpgradePhaseNone +} + // SetDesiredNooBaaDB updates the NooBaaDB as desired for reconciling func (r *Reconciler) SetDesiredNooBaaDB() error { var NooBaaDBTemplate *appsv1.StatefulSet = nil @@ -472,10 +700,10 @@ func (r *Reconciler) setDesiredCoreEnv(c *corev1.Container) { if r.NooBaa.Spec.BucketNotifications.Enabled { envVar := corev1.EnvVar{ - Name: "NOTIFICATION_LOG_DIR", + Name: "NOTIFICATION_LOG_DIR", Value: "/var/logs/notifications", } - util.MergeEnvArrays(&c.Env, &[]corev1.EnvVar{envVar}); + util.MergeEnvArrays(&c.Env, &[]corev1.EnvVar{envVar}) } } @@ -555,10 +783,10 @@ func (r *Reconciler) SetDesiredCoreApp() error { }} util.MergeVolumeMountList(&c.VolumeMounts, ¬ificationVolumeMounts) - notificationVolumes := []corev1.Volume {{ + notificationVolumes := []corev1.Volume{{ Name: notificationsVolume, - VolumeSource: corev1.VolumeSource { - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource { + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ ClaimName: r.BucketNotificationsPVC.Name, }, }, @@ -1187,6 +1415,12 @@ func (r *Reconciler) reconcileDBRBAC() error { // ReconcileDB choose between different types of DB func (r *Reconciler) ReconcileDB() error { + if r.NooBaa.Status.PostgresUpdatePhase == nbv1.UpgradePhasePrepare || + r.NooBaa.Status.PostgresUpdatePhase == nbv1.UpgradePhaseUpgrade || + r.NooBaa.Status.PostgresUpdatePhase == nbv1.UpgradePhaseClean { + r.Logger.Infof("UpgradePostgresDB: Postgres Update in progress") + return nil + } if r.NooBaa.Spec.ExternalPgSecret != nil { return nil @@ -1217,6 +1451,7 @@ func (r *Reconciler) ReconcileDB() error { } } + return nil } @@ -1230,8 +1465,8 @@ func (r *Reconciler) ReconcileDBConfigMap(cm *corev1.ConfigMap, desiredFunc func return r.isObjectUpdated(result), nil } -//ReconcileODFPersistentLoggingPVC ensures a persistent logging pvc (either for bucket logging or bucket notificatoins) -//is properly configured. If needed and possible, allocate one from CephFS +// ReconcileODFPersistentLoggingPVC ensures a persistent logging pvc (either for bucket logging or bucket notificatoins) +// is properly configured. If needed and possible, allocate one from CephFS func (r *Reconciler) ReconcileODFPersistentLoggingPVC( fieldName string, errorName string, @@ -1243,7 +1478,7 @@ func (r *Reconciler) ReconcileODFPersistentLoggingPVC( // Return if persistent logging PVC already exists if pvcName != nil { - pvc.Name = *pvcName; + pvc.Name = *pvcName log.Infof("PersistentLoggingPVC %s already exists and supports RWX access mode. Skipping ReconcileODFPersistentLoggingPVC.", *pvcName) return nil } @@ -1257,7 +1492,7 @@ func (r *Reconciler) ReconcileODFPersistentLoggingPVC( if !r.preparePersistentLoggingPVC(pvc, fieldName) { return util.NewPersistentError(errorName, errorText) } - r.Own(pvc); + r.Own(pvc) log.Infof("Persistent logging PVC %s does not exist. Creating...", pvc.Name) err := r.Client.Create(r.Ctx, pvc) @@ -1269,7 +1504,7 @@ func (r *Reconciler) ReconcileODFPersistentLoggingPVC( } -//prepare persistent logging pvc +// prepare persistent logging pvc func (r *Reconciler) preparePersistentLoggingPVC(pvc *corev1.PersistentVolumeClaim, fieldName string) bool { pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} @@ -1282,9 +1517,9 @@ func (r *Reconciler) preparePersistentLoggingPVC(pvc *corev1.PersistentVolumeCla if util.KubeCheck(sc) { r.Logger.Infof("%s not provided, defaulting to 'cephfs' storage class %s to create persistent logging pvc", fieldName, sc.Name) pvc.Spec.StorageClassName = &sc.Name - return true; + return true } else { - return false; + return false } }