diff --git a/README.md b/README.md index e52c654c..fd3e2fb3 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,7 @@ make manifests More information can be found via the [Kubebuilder Documentation](https://book.kubebuilder.io/introduction.html) + ## License Copyright 2022. diff --git a/api/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml b/api/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml index eb8062a7..15496196 100644 --- a/api/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml +++ b/api/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml @@ -16,6 +16,10 @@ spec: scope: Namespaced versions: - additionalPrinterColumns: + - description: NetworkAttachments + jsonPath: .status.spec.networkAttachments + name: NetworkAttachments + type: string - description: Status jsonPath: .status.conditions[0].status name: Status @@ -83,6 +87,12 @@ spec: to add additional files. Those get added to the service config dir in /etc/ . TODO: -> implement' type: object + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment resource + names to expose the services to the given network + items: + type: string + type: array nodeSelector: additionalProperties: type: string @@ -245,6 +255,13 @@ spec: type: string description: Map of hashes to track e.g. job status type: object + networkAttachments: + additionalProperties: + items: + type: string + type: array + description: NetworkAttachment status of the deployment pods + type: object readyCount: description: ReadyCount of Octavia Amphora Controllers format: int32 diff --git a/api/bases/octavia.openstack.org_octaviaapis.yaml b/api/bases/octavia.openstack.org_octaviaapis.yaml index 8b02bd89..a639e445 100644 --- a/api/bases/octavia.openstack.org_octaviaapis.yaml +++ b/api/bases/octavia.openstack.org_octaviaapis.yaml @@ -16,6 +16,10 @@ spec: scope: Namespaced versions: - additionalPrinterColumns: + - description: NetworkAttachments + jsonPath: .status.spec.networkAttachments + name: NetworkAttachments + type: string - description: Status jsonPath: .status.conditions[0].status name: Status @@ -91,6 +95,12 @@ spec: to add additional files. Those get added to the service config dir in /etc/ . TODO: -> implement' type: object + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment resource + names to expose the services to the given network + items: + type: string + type: array nodeSelector: additionalProperties: type: string @@ -422,6 +432,13 @@ spec: type: string description: Map of hashes to track e.g. job status type: object + networkAttachments: + additionalProperties: + items: + type: string + type: array + description: NetworkAttachment status of the deployment pods + type: object readyCount: description: ReadyCount of octavia API instances format: int32 diff --git a/api/bases/octavia.openstack.org_octavias.yaml b/api/bases/octavia.openstack.org_octavias.yaml index eda30002..a377e416 100644 --- a/api/bases/octavia.openstack.org_octavias.yaml +++ b/api/bases/octavia.openstack.org_octavias.yaml @@ -142,6 +142,12 @@ spec: also be used to add additional files. Those get added to the service config dir in /etc/ . TODO: -> implement' type: object + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment + resource names to expose the services to the given network + items: + type: string + type: array nodeSelector: additionalProperties: type: string @@ -470,6 +476,12 @@ spec: also be used to add additional files. Those get added to the service config dir in /etc/ . TODO: -> implement' type: object + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment + resource names to expose the services to the given network + items: + type: string + type: array nodeSelector: additionalProperties: type: string @@ -620,6 +632,12 @@ spec: also be used to add additional files. Those get added to the service config dir in /etc/ . TODO: -> implement' type: object + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment + resource names to expose the services to the given network + items: + type: string + type: array nodeSelector: additionalProperties: type: string @@ -770,6 +788,12 @@ spec: also be used to add additional files. Those get added to the service config dir in /etc/ . TODO: -> implement' type: object + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment + resource names to expose the services to the given network + items: + type: string + type: array nodeSelector: additionalProperties: type: string diff --git a/api/v1beta1/amphoracontroller_types.go b/api/v1beta1/amphoracontroller_types.go index cedf1daa..4e2e836c 100644 --- a/api/v1beta1/amphoracontroller_types.go +++ b/api/v1beta1/amphoracontroller_types.go @@ -72,7 +72,7 @@ type OctaviaAmphoraControllerSpec struct { // +kubebuilder:validation:Maximum=32 // +kubebuilder:validation:Minimum=0 // Replicas - Octavia Worker Replicas - Replicas int32 `json:"replicas"` + Replicas *int32 `json:"replicas"` // +kubebuilder:validation:Required // Secret containing OpenStack password information for octavia OctaviaDatabasePassword, AdminPassword @@ -112,6 +112,10 @@ type OctaviaAmphoraControllerSpec struct { // Resources - Compute Resources required by this service (Limits/Requests). // https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ Resources corev1.ResourceRequirements `json:"resources,omitempty"` + + // +kubebuilder:validation:Optional + // NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network + NetworkAttachments []string `json:"networkAttachments,omitempty"` } // OctaviaAmphoraControllerStatus defines the observed state of the Octavia Amphora Controller @@ -127,10 +131,14 @@ type OctaviaAmphoraControllerStatus struct { // Octavia Database Hostname DatabaseHostname string `json:"databaseHostname,omitempty"` + + // NetworkAttachment status of the deployment pods + NetworkAttachments map[string][]string `json:"networkAttachments,omitempty"` } //+kubebuilder:object:root=true //+kubebuilder:subresource:status +//+kubebuilder:printcolumn:name="NetworkAttachments",type="string",JSONPath=".status.spec.networkAttachments",description="NetworkAttachments" //+kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[0].status",description="Status" //+kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.conditions[0].message",description="Message" diff --git a/api/v1beta1/octaviaapi_types.go b/api/v1beta1/octaviaapi_types.go index 9beb9092..a3dbdc86 100644 --- a/api/v1beta1/octaviaapi_types.go +++ b/api/v1beta1/octaviaapi_types.go @@ -118,6 +118,10 @@ type OctaviaAPISpec struct { // +kubebuilder:validation:Optional // Override, provides the ability to override the generated manifest of several child resources. Override APIOverrideSpec `json:"override,omitempty"` + + // +kubebuilder:validation:Optional + // NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network + NetworkAttachments []string `json:"networkAttachments,omitempty"` } // APIOverrideSpec to override the generated manifest of several child resources. @@ -153,10 +157,14 @@ type OctaviaAPIStatus struct { // Octavia Database Hostname DatabaseHostname string `json:"databaseHostname,omitempty"` + + // NetworkAttachment status of the deployment pods + NetworkAttachments map[string][]string `json:"networkAttachments,omitempty"` } //+kubebuilder:object:root=true //+kubebuilder:subresource:status +//+kubebuilder:printcolumn:name="NetworkAttachments",type="string",JSONPath=".status.spec.networkAttachments",description="NetworkAttachments" //+kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[0].status",description="Status" //+kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.conditions[0].message",description="Message" diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 6f3f193b..47956523 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -176,6 +176,11 @@ func (in *OctaviaAPISpec) DeepCopyInto(out *OctaviaAPISpec) { } in.Resources.DeepCopyInto(&out.Resources) in.Override.DeepCopyInto(&out.Override) + if in.NetworkAttachments != nil { + in, out := &in.NetworkAttachments, &out.NetworkAttachments + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OctaviaAPISpec. @@ -205,6 +210,21 @@ func (in *OctaviaAPIStatus) DeepCopyInto(out *OctaviaAPIStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.NetworkAttachments != nil { + in, out := &in.NetworkAttachments, &out.NetworkAttachments + *out = make(map[string][]string, len(*in)) + for key, val := range *in { + var outVal []string + if val == nil { + (*out)[key] = nil + } else { + in, out := &val, &outVal + *out = make([]string, len(*in)) + copy(*out, *in) + } + (*out)[key] = outVal + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OctaviaAPIStatus. @@ -279,6 +299,11 @@ func (in *OctaviaAmphoraControllerList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OctaviaAmphoraControllerSpec) DeepCopyInto(out *OctaviaAmphoraControllerSpec) { *out = *in + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } out.PasswordSelectors = in.PasswordSelectors if in.NodeSelector != nil { in, out := &in.NodeSelector, &out.NodeSelector @@ -295,6 +320,11 @@ func (in *OctaviaAmphoraControllerSpec) DeepCopyInto(out *OctaviaAmphoraControll } } in.Resources.DeepCopyInto(&out.Resources) + if in.NetworkAttachments != nil { + in, out := &in.NetworkAttachments, &out.NetworkAttachments + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OctaviaAmphoraControllerSpec. @@ -324,6 +354,21 @@ func (in *OctaviaAmphoraControllerStatus) DeepCopyInto(out *OctaviaAmphoraContro (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.NetworkAttachments != nil { + in, out := &in.NetworkAttachments, &out.NetworkAttachments + *out = make(map[string][]string, len(*in)) + for key, val := range *in { + var outVal []string + if val == nil { + (*out)[key] = nil + } else { + in, out := &val, &outVal + *out = make([]string, len(*in)) + copy(*out, *in) + } + (*out)[key] = outVal + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OctaviaAmphoraControllerStatus. diff --git a/config/crd/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml b/config/crd/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml index eb8062a7..15496196 100644 --- a/config/crd/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml +++ b/config/crd/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml @@ -16,6 +16,10 @@ spec: scope: Namespaced versions: - additionalPrinterColumns: + - description: NetworkAttachments + jsonPath: .status.spec.networkAttachments + name: NetworkAttachments + type: string - description: Status jsonPath: .status.conditions[0].status name: Status @@ -83,6 +87,12 @@ spec: to add additional files. Those get added to the service config dir in /etc/ . TODO: -> implement' type: object + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment resource + names to expose the services to the given network + items: + type: string + type: array nodeSelector: additionalProperties: type: string @@ -245,6 +255,13 @@ spec: type: string description: Map of hashes to track e.g. job status type: object + networkAttachments: + additionalProperties: + items: + type: string + type: array + description: NetworkAttachment status of the deployment pods + type: object readyCount: description: ReadyCount of Octavia Amphora Controllers format: int32 diff --git a/config/crd/bases/octavia.openstack.org_octaviaapis.yaml b/config/crd/bases/octavia.openstack.org_octaviaapis.yaml index 8b02bd89..a639e445 100644 --- a/config/crd/bases/octavia.openstack.org_octaviaapis.yaml +++ b/config/crd/bases/octavia.openstack.org_octaviaapis.yaml @@ -16,6 +16,10 @@ spec: scope: Namespaced versions: - additionalPrinterColumns: + - description: NetworkAttachments + jsonPath: .status.spec.networkAttachments + name: NetworkAttachments + type: string - description: Status jsonPath: .status.conditions[0].status name: Status @@ -91,6 +95,12 @@ spec: to add additional files. Those get added to the service config dir in /etc/ . TODO: -> implement' type: object + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment resource + names to expose the services to the given network + items: + type: string + type: array nodeSelector: additionalProperties: type: string @@ -422,6 +432,13 @@ spec: type: string description: Map of hashes to track e.g. job status type: object + networkAttachments: + additionalProperties: + items: + type: string + type: array + description: NetworkAttachment status of the deployment pods + type: object readyCount: description: ReadyCount of octavia API instances format: int32 diff --git a/config/crd/bases/octavia.openstack.org_octavias.yaml b/config/crd/bases/octavia.openstack.org_octavias.yaml index eda30002..a377e416 100644 --- a/config/crd/bases/octavia.openstack.org_octavias.yaml +++ b/config/crd/bases/octavia.openstack.org_octavias.yaml @@ -142,6 +142,12 @@ spec: also be used to add additional files. Those get added to the service config dir in /etc/ . TODO: -> implement' type: object + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment + resource names to expose the services to the given network + items: + type: string + type: array nodeSelector: additionalProperties: type: string @@ -470,6 +476,12 @@ spec: also be used to add additional files. Those get added to the service config dir in /etc/ . TODO: -> implement' type: object + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment + resource names to expose the services to the given network + items: + type: string + type: array nodeSelector: additionalProperties: type: string @@ -620,6 +632,12 @@ spec: also be used to add additional files. Those get added to the service config dir in /etc/ . TODO: -> implement' type: object + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment + resource names to expose the services to the given network + items: + type: string + type: array nodeSelector: additionalProperties: type: string @@ -770,6 +788,12 @@ spec: also be used to add additional files. Those get added to the service config dir in /etc/ . TODO: -> implement' type: object + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment + resource names to expose the services to the given network + items: + type: string + type: array nodeSelector: additionalProperties: type: string diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 41faccdf..e3b67ece 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -87,6 +87,14 @@ rules: - patch - update - watch +- apiGroups: + - k8s.cni.cncf.io + resources: + - network-attachment-definitions + verbs: + - get + - list + - watch - apiGroups: - keystone.openstack.org resources: diff --git a/config/samples/octavia_v1beta1_octavia.yaml b/config/samples/octavia_v1beta1_octavia.yaml index 59796bbd..93125cd3 100644 --- a/config/samples/octavia_v1beta1_octavia.yaml +++ b/config/samples/octavia_v1beta1_octavia.yaml @@ -7,7 +7,6 @@ spec: databaseUser: octavia serviceUser: octavia rabbitMqClusterName: rabbitmq - replicas: 1 secret: osp-secret debug: dbSync: false @@ -23,7 +22,6 @@ spec: serviceAccount: octavia role: housekeeping certssecret: todo - replicas: 1 secret: osp-secret preserveJobs: false customServiceConfig: | @@ -36,7 +34,6 @@ spec: serviceAccount: octavia role: healthmanager certssecret: todo - replicas: 1 secret: osp-secret preserveJobs: false customServiceConfig: | @@ -47,7 +44,6 @@ spec: databaseUser: octavia serviceUser: octavia serviceAccount: octavia - replicas: 1 role: worker certssecret: todo secret: osp-secret diff --git a/controllers/amphoracontroller_controller.go b/controllers/amphoracontroller_controller.go index 13a0ddc6..bf2d32b2 100644 --- a/controllers/amphoracontroller_controller.go +++ b/controllers/amphoracontroller_controller.go @@ -19,6 +19,7 @@ package controllers import ( "context" "fmt" + "time" "github.com/go-logr/logr" "github.com/openstack-k8s-operators/lib-common/modules/common" @@ -29,6 +30,7 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/env" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" "github.com/openstack-k8s-operators/lib-common/modules/common/labels" + nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment" "github.com/openstack-k8s-operators/lib-common/modules/common/util" keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" @@ -56,6 +58,7 @@ type OctaviaAmphoraControllerReconciler struct { //+kubebuilder:rbac:groups=octavia.openstack.org,resources=octaviaamphoracontrollers,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=octavia.openstack.org,resources=octaviaamphoracontrollers/status,verbs=get;update;patch //+kubebuilder:rbac:groups=octavia.openstack.org,resources=octaviaamphoracontrollers/finalizers,verbs=update +// +kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch // Reconcile implementation of the reconcile loop for amphora // controllers like the octavia housekeeper, worker and health manager @@ -84,6 +87,7 @@ func (r *OctaviaAmphoraControllerReconciler) Reconcile(ctx context.Context, req condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyInitMessage), condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage), + condition.UnknownCondition(condition.NetworkAttachmentsReadyCondition, condition.InitReason, condition.NetworkAttachmentsReadyInitMessage), ) // TODO(beagles): what other conditions? instance.Status.Conditions.Init(&cl) @@ -96,6 +100,10 @@ func (r *OctaviaAmphoraControllerReconciler) Reconcile(ctx context.Context, req instance.Status.Hash = map[string]string{} } + if instance.Status.NetworkAttachments == nil { + instance.Status.NetworkAttachments = map[string][]string{} + } + helper, err := helper.NewHelper( instance, r.Client, @@ -202,6 +210,34 @@ func (r *OctaviaAmphoraControllerReconciler) reconcileNormal(ctx context.Context instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) + for _, networkAttachment := range instance.Spec.NetworkAttachments { + _, err := nad.GetNADWithName(ctx, helper, networkAttachment, instance.Namespace) + if err != nil { + if k8s_errors.IsNotFound(err) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.NetworkAttachmentsReadyWaitingMessage, + networkAttachment)) + return ctrl.Result{RequeueAfter: time.Second * 10}, fmt.Errorf("network-attachment-definition %s not found", networkAttachment) + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + } + + serviceAnnotations, err := nad.CreateNetworksAnnotation(instance.Namespace, instance.Spec.NetworkAttachments) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed create network annotation from %s: %w", + instance.Spec.NetworkAttachments, err) + } + // Handle service update ctrlResult, err := r.reconcileUpdate(ctx, instance, helper) if err != nil { @@ -231,7 +267,8 @@ func (r *OctaviaAmphoraControllerReconciler) reconcileNormal(ctx context.Context amphoracontrollers.Deployment( instance, inputHash, - serviceLabels), + serviceLabels, + serviceAnnotations), 5, ) @@ -252,10 +289,45 @@ func (r *OctaviaAmphoraControllerReconciler) reconcileNormal(ctx context.Context condition.DeploymentReadyRunningMessage)) return ctrlResult, nil } + + // verify if network attachment matches expectations + networkReady := false + networkAttachmentStatus := map[string][]string{} + if *instance.Spec.Replicas > 0 { + networkReady, networkAttachmentStatus, err = nad.VerifyNetworkStatusFromAnnotation( + ctx, + helper, + instance.Spec.NetworkAttachments, + serviceLabels, + instance.Status.ReadyCount, + ) + if err != nil { + return ctrl.Result{}, err + } + } else { + networkReady = true + } + + instance.Status.NetworkAttachments = networkAttachmentStatus + if networkReady { + instance.Status.Conditions.MarkTrue(condition.NetworkAttachmentsReadyCondition, condition.NetworkAttachmentsReadyMessage) + } else { + err := fmt.Errorf("not all pods have interfaces with ips as configured in NetworkAttachments: %s", instance.Spec.NetworkAttachments) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error())) + + return ctrl.Result{}, err + } + instance.Status.ReadyCount = depl.GetDeployment().Status.ReadyReplicas if instance.Status.ReadyCount > 0 { instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) } + // create Deployment - end util.LogForObject(helper, "Reconciled Service successfully", instance) diff --git a/controllers/octavia_controller.go b/controllers/octavia_controller.go index a9403d7e..82337cac 100644 --- a/controllers/octavia_controller.go +++ b/controllers/octavia_controller.go @@ -30,6 +30,7 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/helper" "github.com/openstack-k8s-operators/lib-common/modules/common/job" "github.com/openstack-k8s-operators/lib-common/modules/common/labels" + nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment" common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac" oko_secret "github.com/openstack-k8s-operators/lib-common/modules/common/secret" "github.com/openstack-k8s-operators/lib-common/modules/common/util" @@ -76,6 +77,7 @@ type OctaviaReconciler struct { // +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneservices,verbs=get;list;watch;create;update;patch;delete; // +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneendpoints,verbs=get;list;watch;create;update;patch;delete; // +kubebuilder:rbac:groups=rabbitmq.openstack.org,resources=transporturls,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch // service account, role, rolebinding // +kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;list;watch;create;update @@ -157,6 +159,7 @@ func (r *OctaviaReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re condition.UnknownCondition(condition.RoleReadyCondition, condition.InitReason, condition.RoleReadyInitMessage), condition.UnknownCondition(condition.RoleBindingReadyCondition, condition.InitReason, condition.RoleBindingReadyInitMessage), condition.UnknownCondition(octaviav1.OctaviaAPIReadyCondition, condition.InitReason, octaviav1.OctaviaAPIReadyInitMessage), + condition.UnknownCondition(condition.NetworkAttachmentsReadyCondition, condition.InitReason, condition.NetworkAttachmentsReadyInitMessage), amphoraControllerInitCondition(octaviav1.HealthManager), amphoraControllerInitCondition(octaviav1.Housekeeping), amphoraControllerInitCondition(octaviav1.Worker), @@ -226,6 +229,7 @@ func (r *OctaviaReconciler) reconcileInit( instance *octaviav1.Octavia, helper *helper.Helper, serviceLabels map[string]string, + serviceAnnotations map[string]string, ) (ctrl.Result, error) { r.Log.Info("Reconciling Service init") @@ -292,7 +296,7 @@ func (r *OctaviaReconciler) reconcileInit( // run octavia db sync // dbSyncHash := instance.Status.Hash[octaviav1.DbSyncHash] - jobDef := octavia.DbSyncJob(instance, serviceLabels) + jobDef := octavia.DbSyncJob(instance, serviceLabels, serviceAnnotations) r.Log.Info("Initializing db sync job") dbSyncjob := job.NewJob( jobDef, @@ -494,13 +498,42 @@ func (r *OctaviaReconciler) reconcileNormal(ctx context.Context, instance *octav common.AppSelector: octavia.ServiceName, } + for _, networkAttachment := range instance.Spec.OctaviaAPI.NetworkAttachments { + _, err := nad.GetNADWithName(ctx, helper, networkAttachment, instance.Namespace) + if err != nil { + if k8s_errors.IsNotFound(err) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.NetworkAttachmentsReadyWaitingMessage, + networkAttachment)) + return ctrl.Result{RequeueAfter: time.Second * 10}, fmt.Errorf("network-attachment-definition %s not found", networkAttachment) + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + } + + serviceAnnotations, err := nad.CreateNetworksAnnotation(instance.Namespace, instance.Spec.OctaviaAPI.NetworkAttachments) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed create network annotation from %s: %w", + instance.Spec.OctaviaAPI.NetworkAttachments, err) + } + // Handle service init - ctrlResult, err := r.reconcileInit(ctx, instance, helper, serviceLabels) + ctrlResult, err := r.reconcileInit(ctx, instance, helper, serviceLabels, serviceAnnotations) if err != nil { return ctrlResult, err } else if (ctrlResult != ctrl.Result{}) { return ctrlResult, nil } + instance.Status.Conditions.MarkTrue(condition.NetworkAttachmentsReadyCondition, condition.NetworkAttachmentsReadyMessage) // Handle service update ctrlResult, err = r.reconcileUpdate(ctx, instance, helper) diff --git a/controllers/octaviaapi_controller.go b/controllers/octaviaapi_controller.go index 68f18575..dbae38e4 100644 --- a/controllers/octaviaapi_controller.go +++ b/controllers/octaviaapi_controller.go @@ -32,6 +32,7 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/env" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" "github.com/openstack-k8s-operators/lib-common/modules/common/labels" + nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment" oko_secret "github.com/openstack-k8s-operators/lib-common/modules/common/secret" "github.com/openstack-k8s-operators/lib-common/modules/common/service" "github.com/openstack-k8s-operators/lib-common/modules/common/util" @@ -75,6 +76,7 @@ type OctaviaAPIReconciler struct { // +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneservices,verbs=get;list;watch;create;update;patch;delete; // +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneendpoints,verbs=get;list;watch;create;update;patch;delete; // +kubebuilder:rbac:groups=ovn.openstack.org,resources=ovndbclusters,verbs=get;list;watch; +// +kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -146,6 +148,7 @@ func (r *OctaviaAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) // right now we have no dedicated KeystoneServiceReadyInitMessage condition.UnknownCondition(condition.KeystoneServiceReadyCondition, condition.InitReason, ""), condition.UnknownCondition(condition.KeystoneEndpointReadyCondition, condition.InitReason, ""), + condition.UnknownCondition(condition.NetworkAttachmentsReadyCondition, condition.InitReason, condition.NetworkAttachmentsReadyInitMessage), ) instance.Status.Conditions.Init(&cl) @@ -487,6 +490,34 @@ func (r *OctaviaAPIReconciler) reconcileNormal(ctx context.Context, instance *oc instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) + for _, networkAttachment := range instance.Spec.NetworkAttachments { + _, err := nad.GetNADWithName(ctx, helper, networkAttachment, instance.Namespace) + if err != nil { + if k8s_errors.IsNotFound(err) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.NetworkAttachmentsReadyWaitingMessage, + networkAttachment)) + return ctrl.Result{RequeueAfter: time.Second * 10}, fmt.Errorf("network-attachment-definition %s not found", networkAttachment) + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + } + + serviceAnnotations, err := nad.CreateNetworksAnnotation(instance.Namespace, instance.Spec.NetworkAttachments) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed create network annotation from %s: %w", + instance.Spec.NetworkAttachments, err) + } + // Create ConfigMaps and Secrets - end // @@ -527,7 +558,7 @@ func (r *OctaviaAPIReconciler) reconcileNormal(ctx context.Context, instance *oc // Define a new Deployment object depl := deployment.NewDeployment( - octaviaapi.Deployment(instance, inputHash, serviceLabels), + octaviaapi.Deployment(instance, inputHash, serviceLabels, serviceAnnotations), time.Duration(5)*time.Second, ) @@ -548,6 +579,39 @@ func (r *OctaviaAPIReconciler) reconcileNormal(ctx context.Context, instance *oc condition.DeploymentReadyRunningMessage)) return ctrlResult, nil } + + // verify if network attachment matches expectations + networkReady := false + networkAttachmentStatus := map[string][]string{} + if *instance.Spec.Replicas > 0 { + networkReady, networkAttachmentStatus, err = nad.VerifyNetworkStatusFromAnnotation( + ctx, + helper, + instance.Spec.NetworkAttachments, + serviceLabels, + instance.Status.ReadyCount, + ) + if err != nil { + return ctrl.Result{}, err + } + } else { + networkReady = true + } + + instance.Status.NetworkAttachments = networkAttachmentStatus + if networkReady { + instance.Status.Conditions.MarkTrue(condition.NetworkAttachmentsReadyCondition, condition.NetworkAttachmentsReadyMessage) + } else { + err := fmt.Errorf("not all pods have interfaces with ips as configured in NetworkAttachments: %s", instance.Spec.NetworkAttachments) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error())) + + return ctrl.Result{}, err + } instance.Status.ReadyCount = depl.GetDeployment().Status.ReadyReplicas if instance.Status.ReadyCount > 0 { instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) diff --git a/go.mod b/go.mod index 9d66ebdc..6106bdca 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.19 require ( github.com/go-logr/logr v1.2.4 + github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0 github.com/onsi/ginkgo/v2 v2.12.1 github.com/onsi/gomega v1.27.10 github.com/openstack-k8s-operators/infra-operator/apis v0.1.1-0.20230926144332-61ec188379c1 diff --git a/go.sum b/go.sum index 91ccecac..cad0c26d 100644 --- a/go.sum +++ b/go.sum @@ -105,6 +105,8 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0 h1:VzM3TYHDgqPkettiP6I6q2jOeQFL4nrJM+UcAc4f6Fs= +github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0/go.mod h1:nqCI7aelBJU61wiBeeZWJ6oi4bJy5nrjkM6lWIMA4j0= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= diff --git a/main.go b/main.go index e5cdb79c..d9196c83 100644 --- a/main.go +++ b/main.go @@ -35,6 +35,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1" keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" @@ -57,6 +58,7 @@ func init() { utilruntime.Must(rabbitmqv1.AddToScheme(scheme)) utilruntime.Must(octaviav1.AddToScheme(scheme)) utilruntime.Must(ovn1beta1.AddToScheme(scheme)) + utilruntime.Must(networkv1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } diff --git a/pkg/amphoracontrollers/deployment.go b/pkg/amphoracontrollers/deployment.go index 33b65ad1..60115860 100644 --- a/pkg/amphoracontrollers/deployment.go +++ b/pkg/amphoracontrollers/deployment.go @@ -34,6 +34,7 @@ func Deployment( instance *octaviav1.OctaviaAmphoraController, configHash string, labels map[string]string, + annotations map[string]string, ) *appsv1.Deployment { serviceName := fmt.Sprintf("octavia-%s", instance.Spec.Role) @@ -86,10 +87,11 @@ func Deployment( Selector: &metav1.LabelSelector{ MatchLabels: labels, }, - Replicas: &instance.Spec.Replicas, + Replicas: instance.Spec.Replicas, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: labels, + Annotations: annotations, + Labels: labels, }, Spec: corev1.PodSpec{ ServiceAccountName: instance.Spec.ServiceAccount, diff --git a/pkg/octavia/dbsync.go b/pkg/octavia/dbsync.go index b3846783..8901820d 100644 --- a/pkg/octavia/dbsync.go +++ b/pkg/octavia/dbsync.go @@ -34,6 +34,7 @@ const ( func DbSyncJob( instance *octaviav1.Octavia, labels map[string]string, + annotations map[string]string, ) *batchv1.Job { runAsUser := int64(0) initVolumeMounts := GetInitVolumeMounts() @@ -58,6 +59,9 @@ func DbSyncJob( }, Spec: batchv1.JobSpec{ Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annotations, + }, Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyOnFailure, ServiceAccountName: instance.RbacResourceName(), diff --git a/pkg/octaviaapi/deployment.go b/pkg/octaviaapi/deployment.go index 30d48f45..65a669b5 100644 --- a/pkg/octaviaapi/deployment.go +++ b/pkg/octaviaapi/deployment.go @@ -40,6 +40,7 @@ func Deployment( instance *octaviav1.OctaviaAPI, configHash string, labels map[string]string, + annotations map[string]string, ) *appsv1.Deployment { runAsUser := int64(0) initVolumeMounts := octavia.GetInitVolumeMounts() @@ -110,7 +111,8 @@ func Deployment( Replicas: instance.Spec.Replicas, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: labels, + Annotations: annotations, + Labels: labels, }, Spec: corev1.PodSpec{ ServiceAccountName: instance.Spec.ServiceAccount,