From 530a2bc472cfcbf3bef554ad6791c6f3f2db2d9b Mon Sep 17 00:00:00 2001 From: Francesco Pantano Date: Tue, 26 Nov 2024 14:24:29 +0100 Subject: [PATCH] Consume Topology CR Signed-off-by: Francesco Pantano --- .../glance.openstack.org_glanceapis.yaml | 2 + api/bases/glance.openstack.org_glances.yaml | 4 + api/go.mod | 2 + api/v1beta1/common_types.go | 6 + api/v1beta1/conditions.go | 9 ++ api/v1beta1/glance_types.go | 6 + api/v1beta1/zz_generated.deepcopy.go | 10 ++ .../glance.openstack.org_glanceapis.yaml | 2 + .../bases/glance.openstack.org_glances.yaml | 4 + config/rbac/role.yaml | 10 ++ controllers/glance_controller.go | 7 + controllers/glanceapi_controller.go | 152 +++++++++++++++++- go.mod | 2 + go.sum | 4 +- main.go | 2 + pkg/glanceapi/statefulset.go | 24 ++- test/functional/suite_test.go | 3 + 17 files changed, 239 insertions(+), 10 deletions(-) diff --git a/api/bases/glance.openstack.org_glanceapis.yaml b/api/bases/glance.openstack.org_glanceapis.yaml index 4720440c..154c7f99 100644 --- a/api/bases/glance.openstack.org_glanceapis.yaml +++ b/api/bases/glance.openstack.org_glanceapis.yaml @@ -705,6 +705,8 @@ spec: caBundleSecretName: type: string type: object + topology: + type: string type: default: split enum: diff --git a/api/bases/glance.openstack.org_glances.yaml b/api/bases/glance.openstack.org_glances.yaml index bf0e7271..eaf533df 100644 --- a/api/bases/glance.openstack.org_glances.yaml +++ b/api/bases/glance.openstack.org_glances.yaml @@ -697,6 +697,8 @@ spec: caBundleSecretName: type: string type: object + topology: + type: string type: default: split enum: @@ -777,6 +779,8 @@ spec: storageRequest: type: string type: object + topologySpreadConstraint: + type: string required: - containerImage - databaseInstance diff --git a/api/go.mod b/api/go.mod index 46f04076..d6efabc0 100644 --- a/api/go.mod +++ b/api/go.mod @@ -70,3 +70,5 @@ require ( // mschuppert: map to latest commit from release-4.16 tag // must consistent within modules and service operators replace github.com/openshift/api => github.com/openshift/api v0.0.0-20240830023148-b7d0481c9094 //allow-merging + +replace github.com/openstack-k8s-operators/infra-operator/apis => github.com/fmount/infra-operator/apis v0.0.0-20241205172025-610bcf59b0c7 diff --git a/api/v1beta1/common_types.go b/api/v1beta1/common_types.go index d9b98663..d91fbbe2 100644 --- a/api/v1beta1/common_types.go +++ b/api/v1beta1/common_types.go @@ -60,6 +60,12 @@ type GlanceAPITemplate struct { // NodeSelector to target subset of worker nodes running this service NodeSelector *map[string]string `json:"nodeSelector,omitempty"` + // +kubebuilder:validation:Optional + // Topology to apply the Policy defined by the associated CR referenced by + // name + // NOTE: we do not use a targetRef as it should live in the same namespace + Topology *string `json:"topology,omitempty"` + // +kubebuilder:validation:Optional // CustomServiceConfig - customize the service config using this parameter to change service defaults, // or overwrite rendered information using raw OpenStack config format. The content gets added to diff --git a/api/v1beta1/conditions.go b/api/v1beta1/conditions.go index 6977fcd6..a216fcaa 100644 --- a/api/v1beta1/conditions.go +++ b/api/v1beta1/conditions.go @@ -43,4 +43,13 @@ const ( InvalidBackendErrorMessageSplit = "The GlanceAPI layout type: split cannot be used in combination with File and NFS backend" // InvalidBackendErrorMessageSingle InvalidBackendErrorMessageSingle = "The GlanceAPI layout type: single can only be used in combination with File and NFS backend" + + // TopologyConfigReadyInitMessage + TopologyConfigReadyInitMessage = "Topology config create not started" + // TopologyConfigReadyMessage + TopologyConfigReadyMessage = "Topology config create completed" + // TopologyConfigReadyErrorMessage + TopologyConfigReadyErrorMessage = "Topology config create error occurred %s" + // TopologyConfigReadyCondition Status=True condition which indicates a CR exists and is referenced by the Glance + TopologyConfigReadyCondition condition.Type = "TopologyConfigReady" ) diff --git a/api/v1beta1/glance_types.go b/api/v1beta1/glance_types.go index 508042c7..c90c0671 100644 --- a/api/v1beta1/glance_types.go +++ b/api/v1beta1/glance_types.go @@ -72,6 +72,12 @@ type GlanceSpecCore struct { // NodeSelector to target subset of worker nodes running this service NodeSelector *map[string]string `json:"nodeSelector,omitempty"` + // +kubebuilder:validation:Optional + // Topology to apply the Policy defined by the associated CR referenced by + // name + // NOTE: we do not use a targetRef as it should live in the same namespace + Topology *string `json:"topologySpreadConstraint,omitempty"` + // +kubebuilder:validation:Optional // +kubebuilder:default=false // PreserveJobs - do not delete jobs after they finished e.g. to check logs diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index e0f6c7d5..a45bddc9 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -260,6 +260,11 @@ func (in *GlanceAPITemplate) DeepCopyInto(out *GlanceAPITemplate) { } } } + if in.Topology != nil { + in, out := &in.Topology, &out.Topology + *out = new(string) + **out = **in + } if in.CustomServiceConfigSecrets != nil { in, out := &in.CustomServiceConfigSecrets, &out.CustomServiceConfigSecrets *out = make([]string, len(*in)) @@ -387,6 +392,11 @@ func (in *GlanceSpecCore) DeepCopyInto(out *GlanceSpecCore) { } } } + if in.Topology != nil { + in, out := &in.Topology, &out.Topology + *out = new(string) + **out = **in + } if in.CustomServiceConfigSecrets != nil { in, out := &in.CustomServiceConfigSecrets, &out.CustomServiceConfigSecrets *out = make([]string, len(*in)) diff --git a/config/crd/bases/glance.openstack.org_glanceapis.yaml b/config/crd/bases/glance.openstack.org_glanceapis.yaml index 4720440c..154c7f99 100644 --- a/config/crd/bases/glance.openstack.org_glanceapis.yaml +++ b/config/crd/bases/glance.openstack.org_glanceapis.yaml @@ -705,6 +705,8 @@ spec: caBundleSecretName: type: string type: object + topology: + type: string type: default: split enum: diff --git a/config/crd/bases/glance.openstack.org_glances.yaml b/config/crd/bases/glance.openstack.org_glances.yaml index bf0e7271..eaf533df 100644 --- a/config/crd/bases/glance.openstack.org_glances.yaml +++ b/config/crd/bases/glance.openstack.org_glances.yaml @@ -697,6 +697,8 @@ spec: caBundleSecretName: type: string type: object + topology: + type: string type: default: split enum: @@ -777,6 +779,8 @@ spec: storageRequest: type: string type: object + topologySpreadConstraint: + type: string required: - containerImage - databaseInstance diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 9952166c..f0518b9b 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -287,3 +287,13 @@ rules: - securitycontextconstraints verbs: - use +- apiGroups: + - topology.openstack.org + resources: + - topologies + verbs: + - get + - list + - patch + - update + - watch diff --git a/controllers/glance_controller.go b/controllers/glance_controller.go index 4bbbb46c..1d3553c6 100644 --- a/controllers/glance_controller.go +++ b/controllers/glance_controller.go @@ -866,6 +866,13 @@ func (r *GlanceReconciler) apiDeploymentCreateOrUpdate( if instance.Spec.KeystoneEndpoint == apiName { apiAnnotations[glance.KeystoneEndpoint] = "true" } + + // If topology is not present in the underlying GlanceAPI, + // inherit from the top-level CR + if apiSpec.GlanceAPITemplate.Topology == nil { + apiSpec.GlanceAPITemplate.Topology = instance.Spec.Topology + } + // Add the API name to the GlanceAPI instance as a label serviceLabels[glancev1.APINameLabel] = apiName glanceStatefulset := &glancev1.GlanceAPI{ diff --git a/controllers/glanceapi_controller.go b/controllers/glanceapi_controller.go index 4fe153ab..9540f6cd 100644 --- a/controllers/glanceapi_controller.go +++ b/controllers/glanceapi_controller.go @@ -22,6 +22,7 @@ import ( "strings" batchv1 "k8s.io/api/batch/v1" + "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" @@ -33,6 +34,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" @@ -45,6 +47,7 @@ import ( "github.com/openstack-k8s-operators/glance-operator/pkg/glance" "github.com/openstack-k8s-operators/glance-operator/pkg/glanceapi" memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" @@ -73,9 +76,9 @@ type GlanceAPIReconciler struct { Scheme *runtime.Scheme } -//+kubebuilder:rbac:groups=glance.openstack.org,resources=glanceapis,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=glance.openstack.org,resources=glanceapis/status,verbs=get;update;patch -//+kubebuilder:rbac:groups=glance.openstack.org,resources=glanceapis/finalizers,verbs=update;patch +// +kubebuilder:rbac:groups=glance.openstack.org,resources=glanceapis,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=glance.openstack.org,resources=glanceapis/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=glance.openstack.org,resources=glanceapis/finalizers,verbs=update;patch // +kubebuilder:rbac:groups=cinder.openstack.org,resources=cinders,verbs=get;list;watch // +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete; // +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete @@ -86,6 +89,7 @@ type GlanceAPIReconciler struct { // +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneendpoints,verbs=get;list;watch;create;update;patch;delete; // +kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch // +kubebuilder:rbac:groups=memcached.openstack.org,resources=memcacheds,verbs=get;list;watch; +// +kubebuilder:rbac:groups=topology.openstack.org,resources=topologies,verbs=get;list;watch;update;patch // Reconcile reconcile Glance API requests func (r *GlanceAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, _err error) { @@ -158,6 +162,7 @@ func (r *GlanceAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( condition.UnknownCondition(condition.CreateServiceReadyCondition, condition.InitReason, condition.CreateServiceReadyInitMessage), condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyInitMessage), + condition.UnknownCondition(glancev1.TopologyConfigReadyCondition, condition.InitReason, glancev1.TopologyConfigReadyInitMessage), condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage), condition.UnknownCondition(condition.TLSInputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), // right now we have no dedicated KeystoneEndpointReadyInitMessage @@ -331,6 +336,43 @@ func (r *GlanceAPIReconciler) SetupWithManager(mgr ctrl.Manager) error { } return nil } + tpFn := predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + oldObj := e.ObjectOld.(*topologyv1.Topology) + newObj := e.ObjectNew.(*topologyv1.Topology) + // Compare spec + return !equality.Semantic.DeepEqual(oldObj.Spec, newObj.Spec) + }, + } + + topologyFn := func(_ context.Context, o client.Object) []reconcile.Request { + result := []reconcile.Request{} + + // get all GlanceAPIs CRs + glanceAPIs := &glancev1.GlanceAPIList{} + listOpts := []client.ListOption{ + client.InNamespace(o.GetNamespace()), + } + if err := r.Client.List(context.Background(), glanceAPIs, listOpts...); err != nil { + r.Log.Error(err, "Unable to retrieve GlanceAPI CRs %w") + return nil + } + + for _, cr := range glanceAPIs.Items { + if o.GetName() == *cr.Spec.Topology { + name := client.ObjectKey{ + Namespace: o.GetNamespace(), + Name: cr.Name, + } + r.Log.Info(fmt.Sprintf("Topology %s is used by GlanceAPI CR %s", o.GetName(), cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + if len(result) > 0 { + return result + } + return nil + } return ctrl.NewControllerManagedBy(mgr). For(&glancev1.GlanceAPI{}). @@ -349,6 +391,9 @@ func (r *GlanceAPIReconciler) SetupWithManager(mgr ctrl.Manager) error { ). Watches(&memcachedv1.Memcached{}, handler.EnqueueRequestsFromMapFunc(memcachedFn)). + Watches(&topologyv1.Topology{}, + handler.EnqueueRequestsFromMapFunc(topologyFn), + builder.WithPredicates(tpFn)). Complete(r) } @@ -388,9 +433,14 @@ func (r *GlanceAPIReconciler) findObjectsForSrc(ctx context.Context, src client. func (r *GlanceAPIReconciler) reconcileDelete(ctx context.Context, instance *glancev1.GlanceAPI, helper *helper.Helper) (ctrl.Result, error) { r.Log.Info(fmt.Sprintf("Reconciling Service '%s' delete", instance.Name)) + // Remove finalizer on the KeystoneEndpoints CR if ctrlResult, err := r.ensureDeletedEndpoints(ctx, instance, helper); err != nil { return ctrlResult, err } + // Remove finalizer on the Topology CR + if ctrlResult, err := r.ensureDeletedTopology(ctx, instance, helper); err != nil { + return ctrlResult, err + } // Endpoints are deleted so remove the finalizer. controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) @@ -797,10 +847,31 @@ func (r *GlanceAPIReconciler) reconcileNormal( err.Error())) return ctrl.Result{}, err } + // At this point the config is generated and the inputHash is computed // we can mark the ServiceConfigReady as True and rollout the new pods instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) + var topology *topologyv1.Topology + if instance.Spec.Topology != nil { + topology, err = r.ensureGlanceAPITopology(ctx, helper, instance) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + glancev1.TopologyConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + glancev1.TopologyConfigReadyErrorMessage, + err.Error())) + r.Log.Info("Glance config is waiting for Topology requirements, requeueing...") + return ctrl.Result{}, err + } + } + + // At this point Glance has a Topology CR (or not in case it's not referenced in the + // top-level CR), and we can mark the TopologyReady condition as True (and rollout the + // new pods) + instance.Status.Conditions.MarkTrue(glancev1.TopologyConfigReadyCondition, glancev1.TopologyConfigReadyMessage) + // This is currently required because cleaner and pruner cronJobs // mount the same pvc to clean data present in /var/lib/glance/image-cache // TODO (fpantano) reference a Glance spec/proposal to move to a different @@ -815,6 +886,7 @@ func (r *GlanceAPIReconciler) reconcileNormal( GetServiceLabels(instance), serviceAnnotations, privileged, + topology, ) if err != nil { return ctrlResult, err @@ -1251,6 +1323,7 @@ func (r *GlanceAPIReconciler) ensureDeletedEndpoints( return ctrl.Result{}, err } +// ensureImageCacheJob - func (r *GlanceAPIReconciler) ensureImageCacheJob( ctx context.Context, h *helper.Helper, @@ -1427,3 +1500,76 @@ func (r *GlanceAPIReconciler) glanceAPIRefresh( } return nil } + +// ensureGlanceAPITopology - retrieve the associated Topology/Affinity CR +func (r *GlanceAPIReconciler) ensureGlanceAPITopology( + ctx context.Context, + h *helper.Helper, + instance *glancev1.GlanceAPI, +) (*topologyv1.Topology, error) { + + var err error + var hash string + + topology, hash, err := topologyv1.GetTopologyByName( + ctx, + h, + *instance.Spec.Topology, + instance.Namespace, + ) + if err != nil { + if k8s_errors.IsNotFound(err) { + return topology, err + } + return topology, err + } + + // Add finalizer to the resource because it is going to be consumed by GlanceAPI + if !controllerutil.ContainsFinalizer(topology, h.GetFinalizer()) { + controllerutil.AddFinalizer(topology, h.GetFinalizer()) + } + // Update the resource + if err := h.GetClient().Update(ctx, topology); err != nil { + return topology, err + } + if hashMap, changed := util.SetHash(instance.Status.Hash, "topology", hash); changed { + instance.Status.Hash = hashMap + r.Log.Info(fmt.Sprintf("Set Topology hash %s - %s", "topology", hash)) + } + return topology, nil +} + +// ensureDeletedTopology - +func (r *GlanceAPIReconciler) ensureDeletedTopology( + ctx context.Context, + instance *glancev1.GlanceAPI, + h *helper.Helper, +) (ctrl.Result, error) { + if instance.Spec.Topology == nil { + return ctrl.Result{}, nil + } + // Remove the finalizer from the Topology CR + topology, _, err := topologyv1.GetTopologyByName( + ctx, + h, + *instance.Spec.Topology, + instance.Namespace, + ) + + if k8s_errors.IsNotFound(err) { + return ctrl.Result{}, nil + } + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + if err == nil { + if controllerutil.RemoveFinalizer(topology, h.GetFinalizer()) { + err = r.Update(ctx, topology) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + util.LogForObject(h, "Removed finalizer from Topology", instance) + } + } + return ctrl.Result{}, err +} diff --git a/go.mod b/go.mod index 8e9754bc..0e8af8ac 100644 --- a/go.mod +++ b/go.mod @@ -89,3 +89,5 @@ replace github.com/openstack-k8s-operators/glance-operator/api => ./api // mschuppert: map to latest commit from release-4.16 tag // must consistent within modules and service operators replace github.com/openshift/api => github.com/openshift/api v0.0.0-20240830023148-b7d0481c9094 //allow-merging + +replace github.com/openstack-k8s-operators/infra-operator/apis => github.com/fmount/infra-operator/apis v0.0.0-20241205172025-610bcf59b0c7 diff --git a/go.sum b/go.sum index d270c19b..fa7b35af 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,8 @@ github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/fmount/infra-operator/apis v0.0.0-20241205172025-610bcf59b0c7 h1:Rkul0l8hVVwBXrMg5qnLBAsnqhzkr+8ysoeo6UcV/+A= +github.com/fmount/infra-operator/apis v0.0.0-20241205172025-610bcf59b0c7/go.mod h1:45GCQTvY41txq3sS5hWjZxYFv3AWYSwfa2ghk0qNcd0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= @@ -80,8 +82,6 @@ github.com/openshift/api v0.0.0-20240830023148-b7d0481c9094 h1:J1wuGhVxpsHykZBa6 github.com/openshift/api v0.0.0-20240830023148-b7d0481c9094/go.mod h1:CxgbWAlvu2iQB0UmKTtRu1YfepRg1/vJ64n2DlIEVz4= github.com/openstack-k8s-operators/cinder-operator/api v0.5.1-0.20241129103654-95d11aacd353 h1:HJuuASmkgXEG+c5IwPruNKj+7y53P9SgD6CPgHwIis8= github.com/openstack-k8s-operators/cinder-operator/api v0.5.1-0.20241129103654-95d11aacd353/go.mod h1:3NvCHOoCyzcZm046XctbsU/Dx+en3l8jG3nHhZyWOAA= -github.com/openstack-k8s-operators/infra-operator/apis v0.5.1-0.20241128131522-70981ed1ed1a h1:A0ryzkskW8hYi0OSBIrpKnSsI80+s934tvw8HhjSlZQ= -github.com/openstack-k8s-operators/infra-operator/apis v0.5.1-0.20241128131522-70981ed1ed1a/go.mod h1:6x5zgJJBjrOhsTtNALYrM2ecUH92kIoZbZ6w1fKHPfs= github.com/openstack-k8s-operators/keystone-operator/api v0.5.1-0.20241128130137-80d688a88c19 h1:H5apEipo6Tq8ewOKy3KRUxPKfs/7VwhfctoJoZefjQ8= github.com/openstack-k8s-operators/keystone-operator/api v0.5.1-0.20241128130137-80d688a88c19/go.mod h1:AZhHY6dZzGyG9iVOf1poD7pTS9c7ZG/f99Fg+GdFVEk= github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20241114091812-6dc9fd0961dc h1:Ufa/q/nC9wmKblvsc0kJppsXHOJoY4fbUamb3ItWCOk= diff --git a/main.go b/main.go index 1e8414bf..c851c544 100644 --- a/main.go +++ b/main.go @@ -41,6 +41,7 @@ import ( networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" cinderv1 "github.com/openstack-k8s-operators/cinder-operator/api/v1beta1" memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" @@ -62,6 +63,7 @@ func init() { utilruntime.Must(mariadbv1.AddToScheme(scheme)) utilruntime.Must(keystonev1.AddToScheme(scheme)) utilruntime.Must(cinderv1.AddToScheme(scheme)) + utilruntime.Must(topologyv1.AddToScheme(scheme)) utilruntime.Must(networkv1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } diff --git a/pkg/glanceapi/statefulset.go b/pkg/glanceapi/statefulset.go index 7bfddd5e..650aeb4a 100644 --- a/pkg/glanceapi/statefulset.go +++ b/pkg/glanceapi/statefulset.go @@ -20,6 +20,7 @@ import ( glancev1 "github.com/openstack-k8s-operators/glance-operator/api/v1beta1" glance "github.com/openstack-k8s-operators/glance-operator/pkg/glance" + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" common "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/affinity" "github.com/openstack-k8s-operators/lib-common/modules/common/env" @@ -49,6 +50,7 @@ func StatefulSet( labels map[string]string, annotations map[string]string, privileged bool, + topologyOverride *topologyv1.Topology, ) (*appsv1.StatefulSet, error) { userID := glance.GlanceUID startupProbe := &corev1.Probe{ @@ -290,19 +292,31 @@ func StatefulSet( extraVolPropagation), apiVolumes...) + // initialize an Affinity Object + aff := &affinity.Overrides{ + Affinity: nil, + AntiAffinity: nil, + } + + if instance.Spec.NodeSelector != nil { + statefulset.Spec.Template.Spec.NodeSelector = *instance.Spec.NodeSelector + } + + if topologyOverride != nil { + statefulset.Spec.Template.Spec.TopologySpreadConstraints = *topologyOverride.Spec.TopologySpreadConstraint + aff = topologyOverride.Spec.APIAffinity + } + // If possible two pods of the same service should not // run on the same worker node. If this is not possible // the get still created on the same worker node. - statefulset.Spec.Template.Spec.Affinity = affinity.DistributePods( + statefulset.Spec.Template.Spec.Affinity, err = affinity.DistributePods( common.AppSelector, []string{ glance.ServiceName, }, corev1.LabelHostname, + aff, ) - if instance.Spec.NodeSelector != nil { - statefulset.Spec.Template.Spec.NodeSelector = *instance.Spec.NodeSelector - } - return statefulset, err } diff --git a/test/functional/suite_test.go b/test/functional/suite_test.go index 026d9621..59e951ce 100644 --- a/test/functional/suite_test.go +++ b/test/functional/suite_test.go @@ -47,6 +47,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook" memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" @@ -152,6 +153,8 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) err = memcachedv1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) + err = topologyv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) //+kubebuilder:scaffold:scheme