From fdd6c1821c34847bd9cb608a7742b2c840cef3d5 Mon Sep 17 00:00:00 2001 From: Martin Schuppert Date: Fri, 4 Aug 2023 17:15:14 +0200 Subject: [PATCH] Create cinderapi route and svc endpoint overrides Creates the route for the cinderapi, also allows to customize the route via override. Generats the service override for the env with what is configured in the externalEndpoints, or specified in the service template override. Depends-On: https://github.com/openstack-k8s-operators/lib-common/pull/313 Depends-On: https://github.com/openstack-k8s-operators/keystone-operator/pull/289 Depends-On: https://github.com/openstack-k8s-operators/cinder-operator/pull/248 Jira: OSP-26690 --- ....openstack.org_openstackcontrolplanes.yaml | 127 ++++++++++++++++++ .../v1beta1/openstackcontrolplane_types.go | 5 + apis/core/v1beta1/zz_generated.deepcopy.go | 1 + ....openstack.org_openstackcontrolplanes.yaml | 127 ++++++++++++++++++ ...nstack-operator.clusterserviceversion.yaml | 18 +++ ...controlplane_galera_network_isolation.yaml | 11 +- ...ne_galera_network_isolation_3replicas.yaml | 11 +- ...enstackcontrolplane_network_isolation.yaml | 11 +- ...ckcontrolplane_network_isolation_ceph.yaml | 11 +- pkg/openstack/cinder.go | 67 +++++++++ 10 files changed, 369 insertions(+), 20 deletions(-) diff --git a/apis/bases/core.openstack.org_openstackcontrolplanes.yaml b/apis/bases/core.openstack.org_openstackcontrolplanes.yaml index b3b57cb09..90356ef0c 100644 --- a/apis/bases/core.openstack.org_openstackcontrolplanes.yaml +++ b/apis/bases/core.openstack.org_openstackcontrolplanes.yaml @@ -101,6 +101,133 @@ spec: type: object cinder: properties: + apiOverride: + properties: + externalEndpoints: + items: + properties: + endpoint: + default: internal + enum: + - internal + - public + type: string + ipAddressPool: + minLength: 1 + type: string + loadBalancerIPs: + items: + type: string + type: array + sharedIP: + default: true + type: boolean + sharedIPKey: + default: "" + type: string + required: + - ipAddressPool + type: object + type: array + route: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + alternateBackends: + items: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + maxItems: 3 + type: array + host: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + path: + pattern: ^/ + type: string + port: + properties: + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - targetPort + type: object + subdomain: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + tls: + properties: + caCertificate: + type: string + certificate: + type: string + destinationCACertificate: + type: string + insecureEdgeTerminationPolicy: + type: string + key: + type: string + termination: + enum: + - edge + - reencrypt + - passthrough + type: string + required: + - termination + type: object + to: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + wildcardPolicy: + enum: + - None + - Subdomain + - "" + type: string + type: object + type: object + type: object enabled: default: true type: boolean diff --git a/apis/core/v1beta1/openstackcontrolplane_types.go b/apis/core/v1beta1/openstackcontrolplane_types.go index 3b08af2bf..f7413ade1 100644 --- a/apis/core/v1beta1/openstackcontrolplane_types.go +++ b/apis/core/v1beta1/openstackcontrolplane_types.go @@ -260,6 +260,11 @@ type CinderSection struct { //+operator-sdk:csv:customresourcedefinitions:type=spec // Template - Overrides to use when creating Cinder Resources Template cinderv1.CinderSpec `json:"template,omitempty"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // APIOverride, provides the ability to override the generated manifest of several child resources. + APIOverride Override `json:"apiOverride,omitempty"` } // MariadbSection defines the desired state of MariaDB service diff --git a/apis/core/v1beta1/zz_generated.deepcopy.go b/apis/core/v1beta1/zz_generated.deepcopy.go index cac7f1fc5..828919e56 100644 --- a/apis/core/v1beta1/zz_generated.deepcopy.go +++ b/apis/core/v1beta1/zz_generated.deepcopy.go @@ -51,6 +51,7 @@ func (in *CeilometerSection) DeepCopy() *CeilometerSection { func (in *CinderSection) DeepCopyInto(out *CinderSection) { *out = *in in.Template.DeepCopyInto(&out.Template) + in.APIOverride.DeepCopyInto(&out.APIOverride) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CinderSection. diff --git a/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml b/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml index b3b57cb09..90356ef0c 100644 --- a/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml +++ b/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml @@ -101,6 +101,133 @@ spec: type: object cinder: properties: + apiOverride: + properties: + externalEndpoints: + items: + properties: + endpoint: + default: internal + enum: + - internal + - public + type: string + ipAddressPool: + minLength: 1 + type: string + loadBalancerIPs: + items: + type: string + type: array + sharedIP: + default: true + type: boolean + sharedIPKey: + default: "" + type: string + required: + - ipAddressPool + type: object + type: array + route: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + alternateBackends: + items: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + maxItems: 3 + type: array + host: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + path: + pattern: ^/ + type: string + port: + properties: + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - targetPort + type: object + subdomain: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + tls: + properties: + caCertificate: + type: string + certificate: + type: string + destinationCACertificate: + type: string + insecureEdgeTerminationPolicy: + type: string + key: + type: string + termination: + enum: + - edge + - reencrypt + - passthrough + type: string + required: + - termination + type: object + to: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + wildcardPolicy: + enum: + - None + - Subdomain + - "" + type: string + type: object + type: object + type: object enabled: default: true type: boolean diff --git a/config/manifests/bases/openstack-operator.clusterserviceversion.yaml b/config/manifests/bases/openstack-operator.clusterserviceversion.yaml index d69ba1dee..da7a64790 100644 --- a/config/manifests/bases/openstack-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/openstack-operator.clusterserviceversion.yaml @@ -42,6 +42,24 @@ spec: path: cinder.enabled x-descriptors: - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: IPAddressPool expose VIP via MetalLB on the IPAddressPool + displayName: IPAddress Pool + path: cinder.externalEndpoints[0].ipAddressPool + - description: LoadBalancerIPs, request given IPs from the pool if available. + Using a list to allow dual stack (IPv4/IPv6) support + displayName: Load Balancer IPs + path: cinder.externalEndpoints[0].loadBalancerIPs + - description: SharedIP if true, VIP/VIPs get shared with multiple services + displayName: Shared IP + path: cinder.externalEndpoints[0].sharedIP + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: SharedIPKey specifies the sharing key which gets set as the annotation + on the LoadBalancer service. Services which share the same VIP must have + the same SharedIPKey. Defaults to the IPAddressPool if SharedIP is true, + but no SharedIPKey specified. + displayName: Shared IPKey + path: cinder.externalEndpoints[0].sharedIPKey - description: Template - Overrides to use when creating Cinder Resources displayName: Template path: cinder.template diff --git a/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml b/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml index 9546b9bcc..f010bd3cf 100644 --- a/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml +++ b/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml @@ -17,15 +17,16 @@ spec: - 192.168.122.1 replicas: 1 cinder: + apiOverride: + externalEndpoints: + - endpoint: internal + ipAddressPool: internalapi + loadBalancerIPs: + - 172.17.0.80 template: databaseInstance: openstack secret: osp-secret cinderAPI: - externalEndpoints: - - endpoint: internal - ipAddressPool: internalapi - loadBalancerIPs: - - 172.17.0.80 cinderScheduler: replicas: 1 cinderBackup: diff --git a/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation_3replicas.yaml b/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation_3replicas.yaml index 7ad4464b7..adcb9bc03 100644 --- a/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation_3replicas.yaml +++ b/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation_3replicas.yaml @@ -17,15 +17,16 @@ spec: - 192.168.122.1 replicas: 1 cinder: + apiOverride: + externalEndpoints: + - endpoint: internal + ipAddressPool: internalapi + loadBalancerIPs: + - 172.17.0.80 template: databaseInstance: openstack secret: osp-secret cinderAPI: - externalEndpoints: - - endpoint: internal - ipAddressPool: internalapi - loadBalancerIPs: - - 172.17.0.80 cinderScheduler: replicas: 1 cinderBackup: diff --git a/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml b/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml index c0f5da4a6..48332c9b9 100644 --- a/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml +++ b/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml @@ -17,15 +17,16 @@ spec: - 192.168.122.1 replicas: 1 cinder: + apiOverride: + externalEndpoints: + - endpoint: internal + ipAddressPool: internalapi + loadBalancerIPs: + - 172.17.0.80 template: databaseInstance: openstack secret: osp-secret cinderAPI: - externalEndpoints: - - endpoint: internal - ipAddressPool: internalapi - loadBalancerIPs: - - 172.17.0.80 cinderScheduler: replicas: 1 cinderBackup: diff --git a/config/samples/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml b/config/samples/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml index 63f0c06fb..d9983b343 100644 --- a/config/samples/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml +++ b/config/samples/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml @@ -36,15 +36,16 @@ spec: - 192.168.122.1 replicas: 1 cinder: + apiOverride: + externalEndpoints: + - endpoint: internal + ipAddressPool: internalapi + loadBalancerIPs: + - 172.17.0.80 template: databaseInstance: openstack secret: osp-secret cinderAPI: - externalEndpoints: - - endpoint: internal - ipAddressPool: internalapi - loadBalancerIPs: - - 172.17.0.80 cinderBackup: customServiceConfig: | [DEFAULT] diff --git a/pkg/openstack/cinder.go b/pkg/openstack/cinder.go index 01f5daccc..175b06ccf 100644 --- a/pkg/openstack/cinder.go +++ b/pkg/openstack/cinder.go @@ -5,13 +5,17 @@ import ( "fmt" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/endpoint" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" cinderv1 "github.com/openstack-k8s-operators/cinder-operator/api/v1beta1" corev1beta1 "github.com/openstack-k8s-operators/openstack-operator/apis/core/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" ctrl "sigs.k8s.io/controller-runtime" ) @@ -32,9 +36,50 @@ func ReconcileCinder(ctx context.Context, instance *corev1beta1.OpenStackControl return ctrl.Result{}, nil } + // Create service overrides to pass into the service CR + // and expose the public endpoint using a route per default. + // Any trailing path will be added on the service-operator level. + var endpoints = map[service.Endpoint]endpoint.Data{ + service.EndpointPublic: {}, + service.EndpointInternal: {}, + } + + serviceOverrides := []service.OverrideSpec{} + serviceDetails := []ServiceDetails{} + + for endpointType := range endpoints { + + sd := ServiceDetails{ + ServiceName: cinder.Name, + Namespace: instance.Namespace, + Endpoint: endpointType, + ExternalEndpoints: instance.Spec.Cinder.APIOverride.ExternalEndpoints, + ServiceOverrideSpec: instance.Spec.Cinder.Template.CinderAPI.Override.Service, + RouteOverrideSpec: instance.Spec.Cinder.APIOverride.Route, + } + + svcOverride, ctrlResult, err := sd.CreateRouteAndServiceOverride(ctx, instance, helper, endpointType) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + serviceDetails = append( + serviceDetails, + sd, + ) + if svcOverride != nil { + serviceOverrides = append(serviceOverrides, *svcOverride) + } + } + instance.Status.Conditions.MarkTrue(corev1beta1.OpenStackControlPlaneServiceOverrideReadyCondition, corev1beta1.OpenStackControlPlaneServiceOverrideReadyMessage) + helper.GetLogger().Info("Reconciling Cinder", "Cinder.Namespace", instance.Namespace, "Cinder.Name", "cinder") op, err := controllerutil.CreateOrPatch(ctx, helper.GetClient(), cinder, func() error { instance.Spec.Cinder.Template.DeepCopyInto(&cinder.Spec) + cinder.Spec.CinderAPI.Override.Service = serviceOverrides + if cinder.Spec.Secret == "" { cinder.Spec.Secret = instance.Spec.Secret } @@ -90,6 +135,28 @@ func ReconcileCinder(ctx context.Context, instance *corev1beta1.OpenStackControl corev1beta1.OpenStackControlPlaneCinderReadyRunningMessage)) } + for _, sd := range serviceDetails { + // Add the service CR to the ownerRef list of the route to prevent the route being deleted + // before the service is deleted. Otherwise this can result cleanup issues which require + // the endpoint to be reachable. + // If ALL objects in the list have been deleted, this object will be garbage collected. + // https://github.com/kubernetes/apimachinery/blob/15d95c0b2af3f4fcf46dce24105e5fbb9379af5a/pkg/apis/meta/v1/types.go#L240-L247 + scheme := runtime.NewScheme() + gvk := schema.GroupVersionKind{ + Group: cinderv1.GroupVersion.Group, + Version: cinderv1.GroupVersion.Version, + Kind: cinder.Kind, + } + + // Add the GVK to the scheme + scheme.AddKnownTypeWithName(gvk, &cinderv1.Cinder{}) + + err = sd.AddOwnerRef(ctx, helper, cinder, scheme) + if err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{}, nil }