Switch cinder-api to a statefulset

Change cinder-api from a deployment to a statefulset. Although the service doesn't require preserving stateful data, it helps with the pod naming convention so they're similar to glance-api pods, and eliminates the squirrely suffixes like "cinder-api-xzkpl"
ASBishop · Dec 13, 2023 · 486345c · 486345c
1 parent a3f1200
commit 486345c
Show file tree

Hide file tree

Showing 15 changed files with 31 additions and 43 deletions.
diff --git a/api/bases/cinder.openstack.org_cinderapis.yaml b/api/bases/cinder.openstack.org_cinderapis.yaml
@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.11.1
+    controller-gen.kubebuilder.io/version: v0.10.0
   creationTimestamp: null
   name: cinderapis.cinder.openstack.org
 spec:

diff --git a/api/bases/cinder.openstack.org_cinderbackups.yaml b/api/bases/cinder.openstack.org_cinderbackups.yaml
@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.11.1
+    controller-gen.kubebuilder.io/version: v0.10.0
   creationTimestamp: null
   name: cinderbackups.cinder.openstack.org
 spec:

diff --git a/api/bases/cinder.openstack.org_cinders.yaml b/api/bases/cinder.openstack.org_cinders.yaml
@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.11.1
+    controller-gen.kubebuilder.io/version: v0.10.0
   creationTimestamp: null
   name: cinders.cinder.openstack.org
 spec:

diff --git a/api/bases/cinder.openstack.org_cinderschedulers.yaml b/api/bases/cinder.openstack.org_cinderschedulers.yaml
@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.11.1
+    controller-gen.kubebuilder.io/version: v0.10.0
   creationTimestamp: null
   name: cinderschedulers.cinder.openstack.org
 spec:

diff --git a/api/bases/cinder.openstack.org_cindervolumes.yaml b/api/bases/cinder.openstack.org_cindervolumes.yaml
@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.11.1
+    controller-gen.kubebuilder.io/version: v0.10.0
   creationTimestamp: null
   name: cindervolumes.cinder.openstack.org
 spec:

diff --git a/config/crd/bases/cinder.openstack.org_cinderapis.yaml b/config/crd/bases/cinder.openstack.org_cinderapis.yaml
@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.11.1
+    controller-gen.kubebuilder.io/version: v0.10.0
   creationTimestamp: null
   name: cinderapis.cinder.openstack.org
 spec:

diff --git a/config/crd/bases/cinder.openstack.org_cinderbackups.yaml b/config/crd/bases/cinder.openstack.org_cinderbackups.yaml
@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.11.1
+    controller-gen.kubebuilder.io/version: v0.10.0
   creationTimestamp: null
   name: cinderbackups.cinder.openstack.org
 spec:

diff --git a/config/crd/bases/cinder.openstack.org_cinders.yaml b/config/crd/bases/cinder.openstack.org_cinders.yaml
@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.11.1
+    controller-gen.kubebuilder.io/version: v0.10.0
   creationTimestamp: null
   name: cinders.cinder.openstack.org
 spec:

diff --git a/config/crd/bases/cinder.openstack.org_cinderschedulers.yaml b/config/crd/bases/cinder.openstack.org_cinderschedulers.yaml
@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.11.1
+    controller-gen.kubebuilder.io/version: v0.10.0
   creationTimestamp: null
   name: cinderschedulers.cinder.openstack.org
 spec:

diff --git a/config/crd/bases/cinder.openstack.org_cindervolumes.yaml b/config/crd/bases/cinder.openstack.org_cindervolumes.yaml
@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.11.1
+    controller-gen.kubebuilder.io/version: v0.10.0
   creationTimestamp: null
   name: cindervolumes.cinder.openstack.org
 spec:

diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml
@@ -27,18 +27,6 @@ rules:
   - list
   - update
   - watch
-- apiGroups:
-  - apps
-  resources:
-  - deployments
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
 - apiGroups:
   - apps
   resources:

diff --git a/controllers/cinderapi_controller.go b/controllers/cinderapi_controller.go
@@ -41,14 +41,14 @@ import (
 	keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1"
 	"github.com/openstack-k8s-operators/lib-common/modules/common"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/condition"
-	"github.com/openstack-k8s-operators/lib-common/modules/common/deployment"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/endpoint"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/env"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/helper"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/labels"
 	nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/secret"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/service"
+	"github.com/openstack-k8s-operators/lib-common/modules/common/statefulset"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/util"
 )
 
@@ -97,7 +97,7 @@ var (
 // +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;create;update;patch;delete;watch
 // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;
 // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;create;update;patch;delete;watch
-// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;create;update;patch;delete;watch
+// +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;create;update;patch;delete;watch
 // +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneservices,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneendpoints,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch
@@ -255,7 +255,7 @@ func (r *CinderAPIReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		For(&cinderv1beta1.CinderAPI{}).
 		Owns(&keystonev1.KeystoneService{}).
 		Owns(&keystonev1.KeystoneEndpoint{}).
-		Owns(&appsv1.Deployment{}).
+		Owns(&appsv1.StatefulSet{}).
 		Owns(&corev1.Service{}).
 		// watch the secrets we don't own
 		Watches(&source.Kind{Type: &corev1.Secret{}},
@@ -658,14 +658,14 @@ func (r *CinderAPIReconciler) reconcileNormal(ctx context.Context, instance *cin
 	// normal reconcile tasks
 	//
 
-	// Define a new Deployment object
-	deplDef := cinderapi.Deployment(instance, inputHash, serviceLabels, serviceAnnotations)
-	depl := deployment.NewDeployment(
-		deplDef,
+	// Deploy a statefulset
+	ssDef := cinderapi.StatefulSet(instance, inputHash, serviceLabels, serviceAnnotations)
+	ss := statefulset.NewStatefulSet(
+		ssDef,
 		time.Duration(5)*time.Second,
 	)
 
-	ctrlResult, err = depl.CreateOrPatch(ctx, helper)
+	ctrlResult, err = ss.CreateOrPatch(ctx, helper)
 	if err != nil {
 		instance.Status.Conditions.Set(condition.FalseCondition(
 			condition.DeploymentReadyCondition,
@@ -682,7 +682,7 @@ func (r *CinderAPIReconciler) reconcileNormal(ctx context.Context, instance *cin
 			condition.DeploymentReadyRunningMessage))
 		return ctrlResult, nil
 	}
-	instance.Status.ReadyCount = depl.GetDeployment().Status.ReadyReplicas
+	instance.Status.ReadyCount = ss.GetStatefulSet().Status.ReadyReplicas
 
 	// verify if network attachment matches expectations
 	networkReady := false
@@ -720,7 +720,7 @@ func (r *CinderAPIReconciler) reconcileNormal(ctx context.Context, instance *cin
 	if instance.Status.ReadyCount > 0 {
 		instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage)
 	}
-	// create Deployment - end
+	// create StatefulSet - end
 
 	r.Log.Info(fmt.Sprintf("Reconciled Service '%s' successfully", instance.Name))
 	return ctrl.Result{}, nil

diff --git a/pkg/cinderapi/deployment.go → pkg/cinderapi/statefuleset.go b/pkg/cinderapi/deployment.go → pkg/cinderapi/statefuleset.go
@@ -33,13 +33,13 @@ const (
 	ServiceCommand = "/usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start"
 )
 
-// Deployment func
-func Deployment(
+// StatefulSet func
+func StatefulSet(
 	instance *cinderv1beta1.CinderAPI,
 	configHash string,
 	labels map[string]string,
 	annotations map[string]string,
-) *appsv1.Deployment {
+) *appsv1.StatefulSet {
 	runAsUser := int64(0)
 
 	livenessProbe := &corev1.Probe{
@@ -80,13 +80,13 @@ func Deployment(
 	envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS")
 	envVars["CONFIG_HASH"] = env.SetValue(configHash)
 
-	deployment := &appsv1.Deployment{
+	statefulset := &appsv1.StatefulSet{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      instance.Name,
 			Namespace: instance.Namespace,
 			Labels:    labels,
 		},
-		Spec: appsv1.DeploymentSpec{
+		Spec: appsv1.StatefulSetSpec{
 			Selector: &metav1.LabelSelector{
 				MatchLabels: labels,
 			},
@@ -137,24 +137,24 @@ func Deployment(
 			},
 		},
 	}
-	deployment.Spec.Template.Spec.Volumes = GetVolumes(
+	statefulset.Spec.Template.Spec.Volumes = GetVolumes(
 		cinder.GetOwningCinderName(instance),
 		instance.Name,
 		instance.Spec.ExtraMounts)
 
 	// If possible two pods of the same service should not
 	// run on the same worker node. If this is not possible
 	// the get still created on the same worker node.
-	deployment.Spec.Template.Spec.Affinity = affinity.DistributePods(
+	statefulset.Spec.Template.Spec.Affinity = affinity.DistributePods(
 		common.AppSelector,
 		[]string{
 			cinder.ServiceName,
 		},
 		corev1.LabelHostname,
 	)
 	if instance.Spec.NodeSelector != nil && len(instance.Spec.NodeSelector) > 0 {
-		deployment.Spec.Template.Spec.NodeSelector = instance.Spec.NodeSelector
+		statefulset.Spec.Template.Spec.NodeSelector = instance.Spec.NodeSelector
 	}
 
-	return deployment
+	return statefulset
 }
diff --git a/test/functional/cinder_controller_test.go b/test/functional/cinder_controller_test.go
@@ -385,7 +385,7 @@ var _ = Describe("Cinder controller", func() {
 			keystone.SimulateKeystoneServiceReady(cinderTest.CinderKeystoneService)
 		})
 		It("Check the resulting endpoints of the generated sub-CRs", func() {
-			th.SimulateDeploymentReadyWithPods(
+			th.SimulateStatefulSetReplicaReadyWithPods(
 				cinderTest.CinderAPI,
 				map[string][]string{cinderName.Namespace + "/internalapi": {"10.0.0.1"}},
 			)

diff --git a/test/kuttl/common/assert_sample_deployment.yaml b/test/kuttl/common/assert_sample_deployment.yaml
@@ -21,7 +21,7 @@ spec:
       replicas: 1
 ---
 apiVersion: apps/v1
-kind: Deployment
+kind: StatefulSet
 metadata:
   name: cinder-api
   ownerReferences: