From 1918e5fcde05766065021ab116f15a5bf96a0f8d Mon Sep 17 00:00:00 2001 From: Gregory Thiemonge Date: Mon, 11 Dec 2023 04:05:43 -0500 Subject: [PATCH] Convert AmphoraControllers to DaemonSet --- ...enstack.org_octaviaamphoracontrollers.yaml | 12 +++-- api/bases/octavia.openstack.org_octavias.yaml | 21 --------- api/v1beta1/amphoracontroller_types.go | 10 ++--- api/v1beta1/zz_generated.deepcopy.go | 5 --- ...enstack.org_octaviaamphoracontrollers.yaml | 12 +++-- .../bases/octavia.openstack.org_octavias.yaml | 21 --------- config/rbac/role.yaml | 12 +++++ controllers/amphoracontroller_controller.go | 44 +++++++++---------- controllers/octavia_controller.go | 43 +++++++++--------- .../{deployment.go => daemonset.go} | 19 ++++---- 10 files changed, 76 insertions(+), 123 deletions(-) rename pkg/amphoracontrollers/{deployment.go => daemonset.go} (90%) diff --git a/api/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml b/api/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml index 6623c123..77d27756 100644 --- a/api/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml +++ b/api/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml @@ -139,13 +139,6 @@ spec: from the Secret type: string type: object - replicas: - default: 1 - description: Replicas - Octavia Worker Replicas - format: int32 - maximum: 32 - minimum: 0 - type: integer resources: description: Resources - Compute Resources required by this service (Limits/Requests). https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ @@ -276,6 +269,11 @@ spec: databaseHostname: description: Octavia Database Hostname type: string + desiredNumberScheduled: + description: DesiredNumberScheduled - total number of the nodes which + should be running Daemon + format: int32 + type: integer hash: additionalProperties: type: string diff --git a/api/bases/octavia.openstack.org_octavias.yaml b/api/bases/octavia.openstack.org_octavias.yaml index 9df71518..6bd2f4b6 100644 --- a/api/bases/octavia.openstack.org_octavias.yaml +++ b/api/bases/octavia.openstack.org_octavias.yaml @@ -546,13 +546,6 @@ spec: from the Secret type: string type: object - replicas: - default: 1 - description: Replicas - Octavia Worker Replicas - format: int32 - maximum: 32 - minimum: 0 - type: integer resources: description: Resources - Compute Resources required by this service (Limits/Requests). https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ @@ -728,13 +721,6 @@ spec: from the Secret type: string type: object - replicas: - default: 1 - description: Replicas - Octavia Worker Replicas - format: int32 - maximum: 32 - minimum: 0 - type: integer resources: description: Resources - Compute Resources required by this service (Limits/Requests). https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ @@ -910,13 +896,6 @@ spec: from the Secret type: string type: object - replicas: - default: 1 - description: Replicas - Octavia Worker Replicas - format: int32 - maximum: 32 - minimum: 0 - type: integer resources: description: Resources - Compute Resources required by this service (Limits/Requests). https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ diff --git a/api/v1beta1/amphoracontroller_types.go b/api/v1beta1/amphoracontroller_types.go index dad57810..34cf2304 100644 --- a/api/v1beta1/amphoracontroller_types.go +++ b/api/v1beta1/amphoracontroller_types.go @@ -66,13 +66,6 @@ type OctaviaAmphoraControllerSpec struct { // Role - the role for the controller (one of worker, housekeeping, healthmanager) Role string `json:"role"` - // +kubebuilder:validation:Optional - // +kubebuilder:default=1 - // +kubebuilder:validation:Maximum=32 - // +kubebuilder:validation:Minimum=0 - // Replicas - Octavia Worker Replicas - Replicas *int32 `json:"replicas"` - // +kubebuilder:validation:Required // Secret containing OpenStack password information for octavia OctaviaDatabasePassword, AdminPassword Secret string `json:"secret"` @@ -138,6 +131,9 @@ type OctaviaAmphoraControllerStatus struct { // ReadyCount of Octavia Amphora Controllers ReadyCount int32 `json:"readyCount,omitempty"` + // DesiredNumberScheduled - total number of the nodes which should be running Daemon + DesiredNumberScheduled int32 `json:"desiredNumberScheduled,omitempty"` + // Map of hashes to track e.g. job status Hash map[string]string `json:"hash,omitempty"` diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index d474924c..eb30def4 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -299,11 +299,6 @@ func (in *OctaviaAmphoraControllerList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OctaviaAmphoraControllerSpec) DeepCopyInto(out *OctaviaAmphoraControllerSpec) { *out = *in - if in.Replicas != nil { - in, out := &in.Replicas, &out.Replicas - *out = new(int32) - **out = **in - } out.PasswordSelectors = in.PasswordSelectors if in.NodeSelector != nil { in, out := &in.NodeSelector, &out.NodeSelector diff --git a/config/crd/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml b/config/crd/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml index 6623c123..77d27756 100644 --- a/config/crd/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml +++ b/config/crd/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml @@ -139,13 +139,6 @@ spec: from the Secret type: string type: object - replicas: - default: 1 - description: Replicas - Octavia Worker Replicas - format: int32 - maximum: 32 - minimum: 0 - type: integer resources: description: Resources - Compute Resources required by this service (Limits/Requests). https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ @@ -276,6 +269,11 @@ spec: databaseHostname: description: Octavia Database Hostname type: string + desiredNumberScheduled: + description: DesiredNumberScheduled - total number of the nodes which + should be running Daemon + format: int32 + type: integer hash: additionalProperties: type: string diff --git a/config/crd/bases/octavia.openstack.org_octavias.yaml b/config/crd/bases/octavia.openstack.org_octavias.yaml index 9df71518..6bd2f4b6 100644 --- a/config/crd/bases/octavia.openstack.org_octavias.yaml +++ b/config/crd/bases/octavia.openstack.org_octavias.yaml @@ -546,13 +546,6 @@ spec: from the Secret type: string type: object - replicas: - default: 1 - description: Replicas - Octavia Worker Replicas - format: int32 - maximum: 32 - minimum: 0 - type: integer resources: description: Resources - Compute Resources required by this service (Limits/Requests). https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ @@ -728,13 +721,6 @@ spec: from the Secret type: string type: object - replicas: - default: 1 - description: Replicas - Octavia Worker Replicas - format: int32 - maximum: 32 - minimum: 0 - type: integer resources: description: Resources - Compute Resources required by this service (Limits/Requests). https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ @@ -910,13 +896,6 @@ spec: from the Secret type: string type: object - replicas: - default: 1 - description: Replicas - Octavia Worker Replicas - format: int32 - maximum: 32 - minimum: 0 - type: integer resources: description: Resources - Compute Resources required by this service (Limits/Requests). https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 2cec8e05..7a8e4ebb 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -27,6 +27,18 @@ rules: - list - update - watch +- apiGroups: + - apps + resources: + - daemonsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - apps resources: diff --git a/controllers/amphoracontroller_controller.go b/controllers/amphoracontroller_controller.go index 974a100a..bf74ca50 100644 --- a/controllers/amphoracontroller_controller.go +++ b/controllers/amphoracontroller_controller.go @@ -25,7 +25,7 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/configmap" - "github.com/openstack-k8s-operators/lib-common/modules/common/deployment" + "github.com/openstack-k8s-operators/lib-common/modules/common/daemonset" "github.com/openstack-k8s-operators/lib-common/modules/common/endpoint" "github.com/openstack-k8s-operators/lib-common/modules/common/env" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" @@ -329,9 +329,9 @@ func (r *OctaviaAmphoraControllerReconciler) reconcileNormal(ctx context.Context common.AppSelector: instance.ObjectMeta.Name, } - // Define a new Deployment object - depl := deployment.NewDeployment( - amphoracontrollers.Deployment( + // Define a new DaemonSet object + dset := daemonset.NewDaemonSet( + amphoracontrollers.DaemonSet( instance, inputHash, serviceLabels, @@ -339,7 +339,7 @@ func (r *OctaviaAmphoraControllerReconciler) reconcileNormal(ctx context.Context 5, ) - ctrlResult, err = depl.CreateOrPatch(ctx, helper) + ctrlResult, err = dset.CreateOrPatch(ctx, helper) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.DeploymentReadyCondition, @@ -358,21 +358,15 @@ func (r *OctaviaAmphoraControllerReconciler) reconcileNormal(ctx context.Context } // verify if network attachment matches expectations - networkReady := false - networkAttachmentStatus := map[string][]string{} - if *instance.Spec.Replicas > 0 { - networkReady, networkAttachmentStatus, err = nad.VerifyNetworkStatusFromAnnotation( - ctx, - helper, - instance.Spec.NetworkAttachments, - serviceLabels, - instance.Status.ReadyCount, - ) - if err != nil { - return ctrl.Result{}, err - } - } else { - networkReady = true + networkReady, networkAttachmentStatus, err := nad.VerifyNetworkStatusFromAnnotation( + ctx, + helper, + instance.Spec.NetworkAttachments, + serviceLabels, + instance.Status.ReadyCount, + ) + if err != nil { + return ctrl.Result{}, err } instance.Status.NetworkAttachments = networkAttachmentStatus @@ -390,12 +384,14 @@ func (r *OctaviaAmphoraControllerReconciler) reconcileNormal(ctx context.Context return ctrl.Result{}, err } - instance.Status.ReadyCount = depl.GetDeployment().Status.ReadyReplicas - if instance.Status.ReadyCount > 0 { + instance.Status.DesiredNumberScheduled = dset.GetDaemonSet().Status.DesiredNumberScheduled + // TODO(gthiemonge) change for NumberReady? + instance.Status.ReadyCount = dset.GetDaemonSet().Status.NumberReady + if instance.Status.ReadyCount == instance.Status.DesiredNumberScheduled { instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) } - // create Deployment - end + // create DaemonSet - end Log.Info("Reconciled Service successfully") return ctrl.Result{}, nil @@ -509,6 +505,6 @@ func (r *OctaviaAmphoraControllerReconciler) SetupWithManager(mgr ctrl.Manager) Owns(&corev1.Service{}). Owns(&corev1.Secret{}). Owns(&corev1.ConfigMap{}). - Owns(&appsv1.Deployment{}). + Owns(&appsv1.DaemonSet{}). Complete(r) } diff --git a/controllers/octavia_controller.go b/controllers/octavia_controller.go index 1c92523a..3a7f537d 100644 --- a/controllers/octavia_controller.go +++ b/controllers/octavia_controller.go @@ -75,6 +75,7 @@ func (r *OctaviaReconciler) GetLogger(ctx context.Context) logr.Logger { // +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete; // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;patch;delete; // +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete; +// +kubebuilder:rbac:groups=apps,resources=daemonsets,verbs=get;list;watch;create;update;patch;delete; // +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch;create;update;patch;delete; // +kubebuilder:rbac:groups=mariadb.openstack.org,resources=mariadbdatabases,verbs=get;list;watch;create;update;patch;delete; // +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneapis,verbs=get;list;watch; @@ -588,7 +589,7 @@ func (r *OctaviaReconciler) reconcileNormal(ctx context.Context, instance *octav instance.Status.Conditions.MarkTrue(octaviav1.OctaviaAPIReadyCondition, condition.DeploymentReadyMessage) } - octaviaHousekeeping, op, err := r.amphoraControllerDeploymentCreateOrUpdate(instance, instance.Spec.OctaviaHousekeeping, octaviav1.Housekeeping) + octaviaHousekeeping, op, err := r.amphoraControllerDaemonSetCreateOrUpdate(instance, instance.Spec.OctaviaHousekeeping, octaviav1.Housekeeping) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( amphoraControllerReadyCondition(octaviav1.Housekeeping), @@ -611,7 +612,7 @@ func (r *OctaviaReconciler) reconcileNormal(ctx context.Context, instance *octav instance.Status.Conditions.MarkTrue(amphoraControllerReadyCondition(octaviav1.Housekeeping), condition.DeploymentReadyMessage) } - octaviaHealthManager, op, err := r.amphoraControllerDeploymentCreateOrUpdate(instance, instance.Spec.OctaviaHealthManager, octaviav1.HealthManager) + octaviaHealthManager, op, err := r.amphoraControllerDaemonSetCreateOrUpdate(instance, instance.Spec.OctaviaHealthManager, octaviav1.HealthManager) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( amphoraControllerReadyCondition(octaviav1.HealthManager), @@ -634,7 +635,7 @@ func (r *OctaviaReconciler) reconcileNormal(ctx context.Context, instance *octav instance.Status.Conditions.MarkTrue(amphoraControllerReadyCondition(octaviav1.HealthManager), condition.DeploymentReadyMessage) } - octaviaWorker, op, err := r.amphoraControllerDeploymentCreateOrUpdate(instance, instance.Spec.OctaviaWorker, octaviav1.Worker) + octaviaWorker, op, err := r.amphoraControllerDaemonSetCreateOrUpdate(instance, instance.Spec.OctaviaWorker, octaviav1.Worker) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( amphoraControllerReadyCondition(octaviav1.Worker), @@ -794,43 +795,43 @@ func (r *OctaviaReconciler) transportURLCreateOrUpdate( return transportURL, op, err } -func (r *OctaviaReconciler) amphoraControllerDeploymentCreateOrUpdate( +func (r *OctaviaReconciler) amphoraControllerDaemonSetCreateOrUpdate( instance *octaviav1.Octavia, controllerSpec octaviav1.OctaviaAmphoraControllerSpec, role string, ) (*octaviav1.OctaviaAmphoraController, controllerutil.OperationResult, error) { - deployment := &octaviav1.OctaviaAmphoraController{ + daemonset := &octaviav1.OctaviaAmphoraController{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-%s", instance.Name, role), Namespace: instance.Namespace, }, } - op, err := controllerutil.CreateOrUpdate(context.TODO(), r.Client, deployment, func() error { - deployment.Spec = controllerSpec - deployment.Spec.Role = role - deployment.Spec.DatabaseInstance = instance.Spec.DatabaseInstance - deployment.Spec.DatabaseHostname = instance.Status.DatabaseHostname - deployment.Spec.DatabaseUser = instance.Spec.DatabaseUser - deployment.Spec.ServiceUser = instance.Spec.ServiceUser - deployment.Spec.Secret = instance.Spec.Secret - deployment.Spec.TransportURLSecret = instance.Status.TransportURLSecret - deployment.Spec.ServiceAccount = instance.RbacResourceName() - deployment.Spec.LbMgmtNetworks.ManageLbMgmtNetworks = instance.Spec.LbMgmtNetworks.ManageLbMgmtNetworks - deployment.Spec.LbMgmtNetworks.SubnetIPVersion = instance.Spec.LbMgmtNetworks.SubnetIPVersion - if len(deployment.Spec.NodeSelector) == 0 { - deployment.Spec.NodeSelector = instance.Spec.NodeSelector + op, err := controllerutil.CreateOrUpdate(context.TODO(), r.Client, daemonset, func() error { + daemonset.Spec = controllerSpec + daemonset.Spec.Role = role + daemonset.Spec.DatabaseInstance = instance.Spec.DatabaseInstance + daemonset.Spec.DatabaseHostname = instance.Status.DatabaseHostname + daemonset.Spec.DatabaseUser = instance.Spec.DatabaseUser + daemonset.Spec.ServiceUser = instance.Spec.ServiceUser + daemonset.Spec.Secret = instance.Spec.Secret + daemonset.Spec.TransportURLSecret = instance.Status.TransportURLSecret + daemonset.Spec.ServiceAccount = instance.RbacResourceName() + daemonset.Spec.LbMgmtNetworks.ManageLbMgmtNetworks = instance.Spec.LbMgmtNetworks.ManageLbMgmtNetworks + daemonset.Spec.LbMgmtNetworks.SubnetIPVersion = instance.Spec.LbMgmtNetworks.SubnetIPVersion + if len(daemonset.Spec.NodeSelector) == 0 { + daemonset.Spec.NodeSelector = instance.Spec.NodeSelector } - err := controllerutil.SetControllerReference(instance, deployment, r.Scheme) + err := controllerutil.SetControllerReference(instance, daemonset, r.Scheme) if err != nil { return err } return nil }) - return deployment, op, err + return daemonset, op, err } func amphoraControllerReadyCondition(role string) condition.Type { diff --git a/pkg/amphoracontrollers/deployment.go b/pkg/amphoracontrollers/daemonset.go similarity index 90% rename from pkg/amphoracontrollers/deployment.go rename to pkg/amphoracontrollers/daemonset.go index 06d7d544..3cce1315 100644 --- a/pkg/amphoracontrollers/deployment.go +++ b/pkg/amphoracontrollers/daemonset.go @@ -29,13 +29,13 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// Deployment func -func Deployment( +// DaemonSet func +func DaemonSet( instance *octaviav1.OctaviaAmphoraController, configHash string, labels map[string]string, annotations map[string]string, -) *appsv1.Deployment { +) *appsv1.DaemonSet { serviceName := fmt.Sprintf("octavia-%s", instance.Spec.Role) // The API pod has an extra volume so the API and the provider agent can @@ -82,16 +82,15 @@ func Deployment( envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") envVars["CONFIG_HASH"] = env.SetValue(configHash) - deployment := &appsv1.Deployment{ + daemonset := &appsv1.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Name: serviceName, Namespace: instance.Namespace, }, - Spec: appsv1.DeploymentSpec{ + Spec: appsv1.DaemonSetSpec{ Selector: &metav1.LabelSelector{ MatchLabels: labels, }, - Replicas: instance.Spec.Replicas, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: annotations, @@ -118,7 +117,7 @@ func Deployment( // If possible two pods of the same service should not // run on the same worker node. If this is not possible // the get still created on the same worker node. - deployment.Spec.Template.Spec.Affinity = affinity.DistributePods( + daemonset.Spec.Template.Spec.Affinity = affinity.DistributePods( common.AppSelector, []string{ serviceName, @@ -126,7 +125,7 @@ func Deployment( corev1.LabelHostname, ) if instance.Spec.NodeSelector != nil && len(instance.Spec.NodeSelector) > 0 { - deployment.Spec.Template.Spec.NodeSelector = instance.Spec.NodeSelector + daemonset.Spec.Template.Spec.NodeSelector = instance.Spec.NodeSelector } initContainerDetails := octavia.APIDetails{ @@ -140,7 +139,7 @@ func Deployment( UserPasswordSelector: instance.Spec.PasswordSelectors.Service, VolumeMounts: octavia.GetInitVolumeMounts(), } - deployment.Spec.Template.Spec.InitContainers = octavia.InitContainer(initContainerDetails) + daemonset.Spec.Template.Spec.InitContainers = octavia.InitContainer(initContainerDetails) - return deployment + return daemonset }