From 231281a8ce1ff7e9bdc4e5534fb23d76879a0878 Mon Sep 17 00:00:00 2001 From: Martin Schuppert Date: Fri, 4 Aug 2023 17:15:14 +0200 Subject: [PATCH] Create cinderapi route and svc endpoint overrides Creates the route for the cinderapi, also allows to customize the route via override. Generats the service override for the env with what is configured in the externalEndpoints, or specified in the service template override. Depends-On: https://github.com/openstack-k8s-operators/lib-common/pull/313 Depends-On: https://github.com/openstack-k8s-operators/keystone-operator/pull/289 Depends-On: https://github.com/openstack-k8s-operators/cinder-operator/pull/248 Jira: OSP-26690 --- ....openstack.org_openstackcontrolplanes.yaml | 101 ++++++++++++++++++ apis/core/v1beta1/conditions.go | 3 + .../v1beta1/openstackcontrolplane_types.go | 5 + apis/core/v1beta1/zz_generated.deepcopy.go | 1 + ....openstack.org_openstackcontrolplanes.yaml | 101 ++++++++++++++++++ ...nstack-operator.clusterserviceversion.yaml | 18 ++++ ...controlplane_galera_network_isolation.yaml | 17 ++- ...ne_galera_network_isolation_3replicas.yaml | 17 ++- ...enstackcontrolplane_network_isolation.yaml | 17 ++- ...ckcontrolplane_network_isolation_ceph.yaml | 17 ++- pkg/openstack/cinder.go | 54 ++++++++++ 11 files changed, 331 insertions(+), 20 deletions(-) diff --git a/apis/bases/core.openstack.org_openstackcontrolplanes.yaml b/apis/bases/core.openstack.org_openstackcontrolplanes.yaml index 836db0a91..e5cf1f0b9 100644 --- a/apis/bases/core.openstack.org_openstackcontrolplanes.yaml +++ b/apis/bases/core.openstack.org_openstackcontrolplanes.yaml @@ -99,6 +99,107 @@ spec: type: object cinder: properties: + apiOverride: + properties: + route: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + alternateBackends: + items: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + maxItems: 3 + type: array + host: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + path: + pattern: ^/ + type: string + port: + properties: + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - targetPort + type: object + subdomain: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + tls: + properties: + caCertificate: + type: string + certificate: + type: string + destinationCACertificate: + type: string + insecureEdgeTerminationPolicy: + type: string + key: + type: string + termination: + enum: + - edge + - reencrypt + - passthrough + type: string + required: + - termination + type: object + to: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + wildcardPolicy: + enum: + - None + - Subdomain + - "" + type: string + type: object + type: object + type: object enabled: default: true type: boolean diff --git a/apis/core/v1beta1/conditions.go b/apis/core/v1beta1/conditions.go index c74cfbbb5..b2a5de511 100644 --- a/apis/core/v1beta1/conditions.go +++ b/apis/core/v1beta1/conditions.go @@ -51,6 +51,9 @@ const ( // OpenStackControlPlaneCinderReadyCondition Status=True condition which indicates if Cinder is configured and operational OpenStackControlPlaneCinderReadyCondition condition.Type = "OpenStackControlPlaneCinderReady" + // OpenStackControlPlaneExposeCinderReadyCondition Status=True condition which indicates if Cinder is exposed via a route + OpenStackControlPlaneExposeCinderReadyCondition condition.Type = "OpenStackControlPlaneExposeCinderReady" + // OpenStackControlPlaneOVNReadyCondition Status=True condition which indicates if OVN is configured and operational OpenStackControlPlaneOVNReadyCondition condition.Type = "OpenStackControlPlaneOVNReady" diff --git a/apis/core/v1beta1/openstackcontrolplane_types.go b/apis/core/v1beta1/openstackcontrolplane_types.go index d34bb15df..e51675943 100644 --- a/apis/core/v1beta1/openstackcontrolplane_types.go +++ b/apis/core/v1beta1/openstackcontrolplane_types.go @@ -259,6 +259,11 @@ type CinderSection struct { //+operator-sdk:csv:customresourcedefinitions:type=spec // Template - Overrides to use when creating Cinder Resources Template cinderv1.CinderSpec `json:"template,omitempty"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // APIOverride, provides the ability to override the generated manifest of several child resources. + APIOverride Override `json:"apiOverride,omitempty"` } // MariadbSection defines the desired state of MariaDB service diff --git a/apis/core/v1beta1/zz_generated.deepcopy.go b/apis/core/v1beta1/zz_generated.deepcopy.go index d1d34b202..be945048f 100644 --- a/apis/core/v1beta1/zz_generated.deepcopy.go +++ b/apis/core/v1beta1/zz_generated.deepcopy.go @@ -52,6 +52,7 @@ func (in *CeilometerSection) DeepCopy() *CeilometerSection { func (in *CinderSection) DeepCopyInto(out *CinderSection) { *out = *in in.Template.DeepCopyInto(&out.Template) + in.APIOverride.DeepCopyInto(&out.APIOverride) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CinderSection. diff --git a/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml b/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml index 836db0a91..e5cf1f0b9 100644 --- a/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml +++ b/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml @@ -99,6 +99,107 @@ spec: type: object cinder: properties: + apiOverride: + properties: + route: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + alternateBackends: + items: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + maxItems: 3 + type: array + host: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + path: + pattern: ^/ + type: string + port: + properties: + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - targetPort + type: object + subdomain: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + tls: + properties: + caCertificate: + type: string + certificate: + type: string + destinationCACertificate: + type: string + insecureEdgeTerminationPolicy: + type: string + key: + type: string + termination: + enum: + - edge + - reencrypt + - passthrough + type: string + required: + - termination + type: object + to: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + wildcardPolicy: + enum: + - None + - Subdomain + - "" + type: string + type: object + type: object + type: object enabled: default: true type: boolean diff --git a/config/manifests/bases/openstack-operator.clusterserviceversion.yaml b/config/manifests/bases/openstack-operator.clusterserviceversion.yaml index 9867387e0..b1766e6c0 100644 --- a/config/manifests/bases/openstack-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/openstack-operator.clusterserviceversion.yaml @@ -43,6 +43,24 @@ spec: path: cinder.enabled x-descriptors: - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: IPAddressPool expose VIP via MetalLB on the IPAddressPool + displayName: IPAddress Pool + path: cinder.externalEndpoints[0].ipAddressPool + - description: LoadBalancerIPs, request given IPs from the pool if available. + Using a list to allow dual stack (IPv4/IPv6) support + displayName: Load Balancer IPs + path: cinder.externalEndpoints[0].loadBalancerIPs + - description: SharedIP if true, VIP/VIPs get shared with multiple services + displayName: Shared IP + path: cinder.externalEndpoints[0].sharedIP + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: SharedIPKey specifies the sharing key which gets set as the annotation + on the LoadBalancer service. Services which share the same VIP must have + the same SharedIPKey. Defaults to the IPAddressPool if SharedIP is true, + but no SharedIPKey specified. + displayName: Shared IPKey + path: cinder.externalEndpoints[0].sharedIPKey - description: Template - Overrides to use when creating Cinder Resources displayName: Template path: cinder.template diff --git a/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml b/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml index d70197594..89851d551 100644 --- a/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml +++ b/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml @@ -17,15 +17,22 @@ spec: - 192.168.122.1 replicas: 1 cinder: + apiOverride: + route: {} template: databaseInstance: openstack secret: osp-secret cinderAPI: - externalEndpoints: - - endpoint: internal - ipAddressPool: internalapi - loadBalancerIPs: - - 172.17.0.80 + override: + service: + internal: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.80 + spec: + type: LoadBalancer cinderScheduler: replicas: 1 cinderBackup: diff --git a/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation_3replicas.yaml b/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation_3replicas.yaml index 89fb36ec2..9aee87914 100644 --- a/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation_3replicas.yaml +++ b/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation_3replicas.yaml @@ -17,15 +17,22 @@ spec: - 192.168.122.1 replicas: 1 cinder: + apiOverride: + route: {} template: databaseInstance: openstack secret: osp-secret cinderAPI: - externalEndpoints: - - endpoint: internal - ipAddressPool: internalapi - loadBalancerIPs: - - 172.17.0.80 + override: + service: + internal: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.80 + spec: + type: LoadBalancer cinderScheduler: replicas: 1 cinderBackup: diff --git a/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml b/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml index ad608becc..1a6dd873e 100644 --- a/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml +++ b/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml @@ -17,15 +17,22 @@ spec: - 192.168.122.1 replicas: 1 cinder: + apiOverride: + route: {} template: databaseInstance: openstack secret: osp-secret cinderAPI: - externalEndpoints: - - endpoint: internal - ipAddressPool: internalapi - loadBalancerIPs: - - 172.17.0.80 + override: + service: + internal: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.80 + spec: + type: LoadBalancer cinderScheduler: replicas: 1 cinderBackup: diff --git a/config/samples/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml b/config/samples/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml index 5b497c61c..82e07adae 100644 --- a/config/samples/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml +++ b/config/samples/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml @@ -37,15 +37,22 @@ spec: - 192.168.122.1 replicas: 1 cinder: + apiOverride: + route: {} template: databaseInstance: openstack secret: osp-secret cinderAPI: - externalEndpoints: - - endpoint: internal - ipAddressPool: internalapi - loadBalancerIPs: - - 172.17.0.80 + override: + service: + internal: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.80 + spec: + type: LoadBalancer cinderBackup: customServiceConfig: | [DEFAULT] diff --git a/pkg/openstack/cinder.go b/pkg/openstack/cinder.go index 01f5daccc..a0be02e74 100644 --- a/pkg/openstack/cinder.go +++ b/pkg/openstack/cinder.go @@ -4,14 +4,19 @@ import ( "context" "fmt" + "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/reconcile" cinderv1 "github.com/openstack-k8s-operators/cinder-operator/api/v1beta1" corev1beta1 "github.com/openstack-k8s-operators/openstack-operator/apis/core/v1beta1" + k8s_errors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" ) @@ -29,12 +34,61 @@ func ReconcileCinder(ctx context.Context, instance *corev1beta1.OpenStackControl return res, err } instance.Status.Conditions.Remove(corev1beta1.OpenStackControlPlaneCinderReadyCondition) + instance.Status.Conditions.Remove(corev1beta1.OpenStackControlPlaneExposeCinderReadyCondition) return ctrl.Result{}, nil } + // add selector to service overrides + for _, endpointType := range []service.Endpoint{service.EndpointPublic, service.EndpointInternal} { + if instance.Spec.Cinder.Template.CinderAPI.Override.Service == nil { + instance.Spec.Cinder.Template.CinderAPI.Override.Service = map[service.Endpoint]service.RoutedOverrideSpec{} + } + instance.Spec.Cinder.Template.CinderAPI.Override.Service[endpointType] = + AddServiceComponentLabel( + instance.Spec.Cinder.Template.CinderAPI.Override.Service[endpointType], + cinder.Name) + } + + // When component services got created check if there is the need to create a route + if err := helper.GetClient().Get(ctx, types.NamespacedName{Name: "cinder", Namespace: instance.Namespace}, cinder); err != nil { + if !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + } + + if cinder.Status.Conditions.IsTrue(cinderv1.CinderAPIReadyCondition) { + svcs, err := service.GetServicesListWithLabel( + ctx, + helper, + instance.Namespace, + map[string]string{common.AppSelector: cinder.Name}, + ) + if err != nil { + return ctrl.Result{}, err + } + + var ctrlResult reconcile.Result + instance.Spec.Cinder.Template.CinderAPI.Override.Service, ctrlResult, err = EnsureRoute( + ctx, + instance, + helper, + cinder, + svcs, + instance.Spec.Cinder.Template.CinderAPI.Override.Service, + instance.Spec.Cinder.APIOverride.Route, + corev1beta1.OpenStackControlPlaneExposeCinderReadyCondition, + ) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + } + helper.GetLogger().Info("Reconciling Cinder", "Cinder.Namespace", instance.Namespace, "Cinder.Name", "cinder") op, err := controllerutil.CreateOrPatch(ctx, helper.GetClient(), cinder, func() error { instance.Spec.Cinder.Template.DeepCopyInto(&cinder.Spec) + if cinder.Spec.Secret == "" { cinder.Spec.Secret = instance.Spec.Secret }