From fa1ade686249547163cf637fe871ac0ef12986ce Mon Sep 17 00:00:00 2001 From: Martin Schuppert Date: Tue, 1 Aug 2023 11:34:41 +0200 Subject: [PATCH] Create glanceapi route and svc endpoint overrides Creates the route for the glanceapi, also allows to customize the route via override. Generats the service override for the env with what is configured in the externalEndpoints, or specified in the service template override. Depends-On: https://github.com/openstack-k8s-operators/lib-common/pull/313 Depends-On: https://github.com/openstack-k8s-operators/keystone-operator/pull/289 Depends-On: https://github.com/openstack-k8s-operators/glance-operator/pull/285 Jira: OSP-26690 --- ....openstack.org_openstackcontrolplanes.yaml | 101 ++++++++++++++++++ apis/core/v1beta1/conditions.go | 3 + .../v1beta1/openstackcontrolplane_types.go | 5 + apis/core/v1beta1/zz_generated.deepcopy.go | 1 + ....openstack.org_openstackcontrolplanes.yaml | 101 ++++++++++++++++++ ...nstack-operator.clusterserviceversion.yaml | 18 ++++ ...controlplane_galera_network_isolation.yaml | 16 ++- ...ne_galera_network_isolation_3replicas.yaml | 16 ++- ...enstackcontrolplane_network_isolation.yaml | 16 ++- ...ckcontrolplane_network_isolation_ceph.yaml | 16 ++- pkg/openstack/glance.go | 63 ++++++++++- 11 files changed, 335 insertions(+), 21 deletions(-) diff --git a/apis/bases/core.openstack.org_openstackcontrolplanes.yaml b/apis/bases/core.openstack.org_openstackcontrolplanes.yaml index 04b477e16..3a348655d 100644 --- a/apis/bases/core.openstack.org_openstackcontrolplanes.yaml +++ b/apis/bases/core.openstack.org_openstackcontrolplanes.yaml @@ -2108,6 +2108,107 @@ spec: type: object glance: properties: + apiOverride: + properties: + route: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + alternateBackends: + items: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + maxItems: 3 + type: array + host: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + path: + pattern: ^/ + type: string + port: + properties: + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - targetPort + type: object + subdomain: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + tls: + properties: + caCertificate: + type: string + certificate: + type: string + destinationCACertificate: + type: string + insecureEdgeTerminationPolicy: + type: string + key: + type: string + termination: + enum: + - edge + - reencrypt + - passthrough + type: string + required: + - termination + type: object + to: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + wildcardPolicy: + enum: + - None + - Subdomain + - "" + type: string + type: object + type: object + type: object enabled: default: true type: boolean diff --git a/apis/core/v1beta1/conditions.go b/apis/core/v1beta1/conditions.go index cd5aa2c0b..da1e13252 100644 --- a/apis/core/v1beta1/conditions.go +++ b/apis/core/v1beta1/conditions.go @@ -42,6 +42,9 @@ const ( // OpenStackControlPlaneGlanceReadyCondition Status=True condition which indicates if Glance is configured and operational OpenStackControlPlaneGlanceReadyCondition condition.Type = "OpenStackControlPlaneGlanceReady" + // OpenStackControlPlaneExposeGlanceReadyCondition Status=True condition which indicates if Glance is exposed via a route + OpenStackControlPlaneExposeGlanceReadyCondition condition.Type = "OpenStackControlPlaneExposeGlanceReady" + // OpenStackControlPlaneCinderReadyCondition Status=True condition which indicates if Cinder is configured and operational OpenStackControlPlaneCinderReadyCondition condition.Type = "OpenStackControlPlaneCinderReady" diff --git a/apis/core/v1beta1/openstackcontrolplane_types.go b/apis/core/v1beta1/openstackcontrolplane_types.go index ed170b64f..c4da6f416 100644 --- a/apis/core/v1beta1/openstackcontrolplane_types.go +++ b/apis/core/v1beta1/openstackcontrolplane_types.go @@ -231,6 +231,11 @@ type GlanceSection struct { //+operator-sdk:csv:customresourcedefinitions:type=spec // Template - Overrides to use when creating the Glance Service Template glancev1.GlanceSpec `json:"template,omitempty"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // APIOverride, provides the ability to override the generated manifest of several child resources. + APIOverride Override `json:"apiOverride,omitempty"` } // CinderSection defines the desired state of Cinder service diff --git a/apis/core/v1beta1/zz_generated.deepcopy.go b/apis/core/v1beta1/zz_generated.deepcopy.go index 96ee0d99d..70a69e7e6 100644 --- a/apis/core/v1beta1/zz_generated.deepcopy.go +++ b/apis/core/v1beta1/zz_generated.deepcopy.go @@ -105,6 +105,7 @@ func (in *GaleraSection) DeepCopy() *GaleraSection { func (in *GlanceSection) DeepCopyInto(out *GlanceSection) { *out = *in in.Template.DeepCopyInto(&out.Template) + in.APIOverride.DeepCopyInto(&out.APIOverride) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GlanceSection. diff --git a/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml b/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml index 04b477e16..3a348655d 100644 --- a/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml +++ b/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml @@ -2108,6 +2108,107 @@ spec: type: object glance: properties: + apiOverride: + properties: + route: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + alternateBackends: + items: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + maxItems: 3 + type: array + host: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + path: + pattern: ^/ + type: string + port: + properties: + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - targetPort + type: object + subdomain: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + tls: + properties: + caCertificate: + type: string + certificate: + type: string + destinationCACertificate: + type: string + insecureEdgeTerminationPolicy: + type: string + key: + type: string + termination: + enum: + - edge + - reencrypt + - passthrough + type: string + required: + - termination + type: object + to: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + wildcardPolicy: + enum: + - None + - Subdomain + - "" + type: string + type: object + type: object + type: object enabled: default: true type: boolean diff --git a/config/manifests/bases/openstack-operator.clusterserviceversion.yaml b/config/manifests/bases/openstack-operator.clusterserviceversion.yaml index 06e7017b4..e1e8dfd42 100644 --- a/config/manifests/bases/openstack-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/openstack-operator.clusterserviceversion.yaml @@ -81,6 +81,24 @@ spec: path: glance.enabled x-descriptors: - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: IPAddressPool expose VIP via MetalLB on the IPAddressPool + displayName: IPAddress Pool + path: glance.externalEndpoints[0].ipAddressPool + - description: LoadBalancerIPs, request given IPs from the pool if available. + Using a list to allow dual stack (IPv4/IPv6) support + displayName: Load Balancer IPs + path: glance.externalEndpoints[0].loadBalancerIPs + - description: SharedIP if true, VIP/VIPs get shared with multiple services + displayName: Shared IP + path: glance.externalEndpoints[0].sharedIP + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: SharedIPKey specifies the sharing key which gets set as the annotation + on the LoadBalancer service. Services which share the same VIP must have + the same SharedIPKey. Defaults to the IPAddressPool if SharedIP is true, + but no SharedIPKey specified. + displayName: Shared IPKey + path: glance.externalEndpoints[0].sharedIPKey - description: Template - Overrides to use when creating the Glance Service displayName: Template path: glance.template diff --git a/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml b/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml index 056f56154..1a8fe37d6 100644 --- a/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml +++ b/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml @@ -38,16 +38,22 @@ spec: - storage replicas: 0 # backend needs to be configured glance: + apiOverride: + route: {} template: databaseInstance: openstack storageClass: "" storageRequest: 10G glanceAPIInternal: - externalEndpoints: - - endpoint: internal - ipAddressPool: internalapi - loadBalancerIPs: - - 172.17.0.80 + override: + service: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.80 + spec: + type: LoadBalancer networkAttachments: - storage glanceAPIExternal: diff --git a/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation_3replicas.yaml b/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation_3replicas.yaml index 93c60b689..9a6db0997 100644 --- a/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation_3replicas.yaml +++ b/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation_3replicas.yaml @@ -38,16 +38,22 @@ spec: - storage replicas: 0 # backend needs to be configured glance: + apiOverride: + route: {} template: databaseInstance: openstack storageClass: "" storageRequest: 10G glanceAPIInternal: - externalEndpoints: - - endpoint: internal - ipAddressPool: internalapi - loadBalancerIPs: - - 172.17.0.80 + override: + service: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.80 + spec: + type: LoadBalancer networkAttachments: - storage glanceAPIExternal: diff --git a/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml b/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml index e0b94177a..a137f5d0a 100644 --- a/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml +++ b/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml @@ -38,16 +38,22 @@ spec: - storage replicas: 0 # backend needs to be configured glance: + apiOverride: + route: {} template: databaseInstance: openstack storageClass: "" storageRequest: 10G glanceAPIInternal: - externalEndpoints: - - endpoint: internal - ipAddressPool: internalapi - loadBalancerIPs: - - 172.17.0.80 + override: + service: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.80 + spec: + type: LoadBalancer networkAttachments: - storage glanceAPIExternal: diff --git a/config/samples/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml b/config/samples/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml index ca5fff9c0..b01e91f8d 100644 --- a/config/samples/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml +++ b/config/samples/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml @@ -73,6 +73,8 @@ spec: - storage replicas: 0 # backend needs to be configured glance: + apiOverride: + route: {} template: databaseInstance: openstack customServiceConfig: | @@ -88,11 +90,15 @@ spec: storageClass: "" storageRequest: 10G glanceAPIInternal: - externalEndpoints: - - endpoint: internal - ipAddressPool: internalapi - loadBalancerIPs: - - 172.17.0.80 + override: + service: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.80 + spec: + type: LoadBalancer networkAttachments: - storage glanceAPIExternal: diff --git a/pkg/openstack/glance.go b/pkg/openstack/glance.go index 5decc025b..66db48a32 100644 --- a/pkg/openstack/glance.go +++ b/pkg/openstack/glance.go @@ -4,14 +4,20 @@ import ( "context" "fmt" + "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/reconcile" glancev1 "github.com/openstack-k8s-operators/glance-operator/api/v1beta1" corev1beta1 "github.com/openstack-k8s-operators/openstack-operator/apis/core/v1beta1" + k8s_errors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" ) @@ -29,12 +35,68 @@ func ReconcileGlance(ctx context.Context, instance *corev1beta1.OpenStackControl return res, err } instance.Status.Conditions.Remove(corev1beta1.OpenStackControlPlaneGlanceReadyCondition) + instance.Status.Conditions.Remove(corev1beta1.OpenStackControlPlaneExposeGlanceReadyCondition) return ctrl.Result{}, nil } + serviceOverrides := map[string]service.OverrideSpec{} + if instance.Spec.Glance.Template.GlanceAPIExternal.Override.Service != nil { + serviceOverrides[string(service.EndpointPublic)] = *instance.Spec.Glance.Template.GlanceAPIExternal.Override.Service + } + if instance.Spec.Glance.Template.GlanceAPIInternal.Override.Service != nil { + serviceOverrides[string(service.EndpointInternal)] = *instance.Spec.Glance.Template.GlanceAPIInternal.Override.Service + } + + // add selector to service overrides + for _, endpointType := range []service.Endpoint{service.EndpointPublic, service.EndpointInternal} { + svcOverride := ptr.To(serviceOverrides[string(endpointType)]) + serviceOverrides[string(endpointType)] = AddServiceComponentLabel(svcOverride, glance.Name) + } + + // When component services got created check if there is the need to create a route + if err := helper.GetClient().Get(ctx, types.NamespacedName{Name: "glance", Namespace: instance.Namespace}, glance); err != nil { + if !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + } + + if glance.Status.Conditions.IsTrue(glancev1.GlanceAPIReadyCondition) { + svcs, err := service.GetServicesListWithLabel( + ctx, + helper, + instance.Namespace, + map[string]string{common.AppSelector: glance.Name}, + ) + if err != nil { + return ctrl.Result{}, err + } + + var ctrlResult reconcile.Result + serviceOverrides, ctrlResult, err = EnsureRoute( + ctx, + instance, + helper, + glance, + svcs, + serviceOverrides, + instance.Spec.Glance.APIOverride.Route, + corev1beta1.OpenStackControlPlaneExposeGlanceReadyCondition, + ) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + } + + //instance.Status.Conditions.MarkTrue(corev1beta1.OpenStackControlPlaneServiceOverrideReadyCondition, corev1beta1.OpenStackControlPlaneServiceOverrideReadyMessage) + helper.GetLogger().Info("Reconciling Glance", "Glance.Namespace", instance.Namespace, "Glance.Name", "glance") op, err := controllerutil.CreateOrPatch(ctx, helper.GetClient(), glance, func() error { instance.Spec.Glance.Template.DeepCopyInto(&glance.Spec) + glance.Spec.GlanceAPIExternal.Override.Service = ptr.To(serviceOverrides[string(service.EndpointPublic)]) + glance.Spec.GlanceAPIInternal.Override.Service = ptr.To(serviceOverrides[string(service.EndpointInternal)]) + if glance.Spec.Secret == "" { glance.Spec.Secret = instance.Spec.Secret } @@ -90,5 +152,4 @@ func ReconcileGlance(ctx context.Context, instance *corev1beta1.OpenStackControl } return ctrl.Result{}, nil - }