From cd473966886386bfa9e41a1a389dceb014103b3d Mon Sep 17 00:00:00 2001 From: Martin Schuppert Date: Fri, 11 Aug 2023 12:00:49 +0200 Subject: [PATCH] Create horizon route and svc overrides Creates the route for the horizon, also allows to customize the route via override. Generats the service override for the env with what is configured in the externalEndpoints, or specified in the service template override. Depends-On: https://github.com/openstack-k8s-operators/lib-common/pull/313 Depends-On: https://github.com/openstack-k8s-operators/keystone-operator/pull/289 Depends-On: https://github.com/openstack-k8s-operators/horizon-operator/pull/202 Jira: OSP-26690 --- ....openstack.org_openstackcontrolplanes.yaml | 173 ++++++++++++++++-- .../v1beta1/openstackcontrolplane_types.go | 12 ++ apis/core/v1beta1/zz_generated.deepcopy.go | 21 +++ ....openstack.org_openstackcontrolplanes.yaml | 173 ++++++++++++++++-- pkg/openstack/horizon.go | 66 +++++++ 5 files changed, 417 insertions(+), 28 deletions(-) diff --git a/apis/bases/core.openstack.org_openstackcontrolplanes.yaml b/apis/bases/core.openstack.org_openstackcontrolplanes.yaml index 4954fdfbe..69a1ab325 100644 --- a/apis/bases/core.openstack.org_openstackcontrolplanes.yaml +++ b/apis/bases/core.openstack.org_openstackcontrolplanes.yaml @@ -4076,6 +4076,107 @@ spec: type: object horizon: properties: + apiOverride: + properties: + route: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + alternateBackends: + items: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + maxItems: 3 + type: array + host: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + path: + pattern: ^/ + type: string + port: + properties: + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - targetPort + type: object + subdomain: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + tls: + properties: + caCertificate: + type: string + certificate: + type: string + destinationCACertificate: + type: string + insecureEdgeTerminationPolicy: + type: string + key: + type: string + termination: + enum: + - edge + - reencrypt + - passthrough + type: string + required: + - termination + type: object + to: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + wildcardPolicy: + enum: + - None + - Subdomain + - "" + type: string + type: object + type: object + type: object enabled: default: false type: boolean @@ -4103,6 +4204,64 @@ spec: additionalProperties: type: string type: object + override: + properties: + service: + items: + properties: + endpoint: + enum: + - internal + - public + type: string + endpointURL: + type: string + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + externalName: + type: string + externalTrafficPolicy: + type: string + internalTrafficPolicy: + type: string + ipFamilyPolicy: + type: string + loadBalancerClass: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + sessionAffinity: + type: string + sessionAffinityConfig: + properties: + clientIP: + properties: + timeoutSeconds: + format: int32 + type: integer + type: object + type: object + type: + type: string + type: object + required: + - endpoint + type: object + type: array + type: object preserveJobs: default: false type: boolean @@ -4143,20 +4302,6 @@ spec: x-kubernetes-int-or-string: true type: object type: object - route: - properties: - routeLocation: - type: string - routeName: - default: horizon - type: string - routeTLSCA: - type: string - routeTLSEnabled: - type: string - routeTLSKey: - type: string - type: object secret: type: string required: diff --git a/apis/core/v1beta1/openstackcontrolplane_types.go b/apis/core/v1beta1/openstackcontrolplane_types.go index e6b8c5734..1c8b360bb 100644 --- a/apis/core/v1beta1/openstackcontrolplane_types.go +++ b/apis/core/v1beta1/openstackcontrolplane_types.go @@ -522,6 +522,18 @@ type HorizonSection struct { // +kubebuilder:validation:Optional // Template - Overrides to use when creating the Horizon services Template horizonv1.HorizonSpec `json:"template,omitempty"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // APIOverride, provides the ability to override the generated manifest of several child resources. + APIOverride HorizonOverride `json:"apiOverride,omitempty"` +} + +// HorizonOverride to override the generated manifest of several child resources. +type HorizonOverride struct { + // +kubebuilder:validation:Optional + // Route overrides to use when creating the public service endpoint + Route *route.OverrideSpec `json:"route,omitempty"` } // CeilometerSection defines the desired state of OpenStack Telemetry services diff --git a/apis/core/v1beta1/zz_generated.deepcopy.go b/apis/core/v1beta1/zz_generated.deepcopy.go index 8ab11dfb5..c39ec2d62 100644 --- a/apis/core/v1beta1/zz_generated.deepcopy.go +++ b/apis/core/v1beta1/zz_generated.deepcopy.go @@ -137,10 +137,31 @@ func (in *HeatSection) DeepCopy() *HeatSection { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HorizonOverride) DeepCopyInto(out *HorizonOverride) { + *out = *in + if in.Route != nil { + in, out := &in.Route, &out.Route + *out = new(route.OverrideSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HorizonOverride. +func (in *HorizonOverride) DeepCopy() *HorizonOverride { + if in == nil { + return nil + } + out := new(HorizonOverride) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HorizonSection) DeepCopyInto(out *HorizonSection) { *out = *in in.Template.DeepCopyInto(&out.Template) + in.APIOverride.DeepCopyInto(&out.APIOverride) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HorizonSection. diff --git a/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml b/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml index 4954fdfbe..69a1ab325 100644 --- a/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml +++ b/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml @@ -4076,6 +4076,107 @@ spec: type: object horizon: properties: + apiOverride: + properties: + route: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + alternateBackends: + items: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + maxItems: 3 + type: array + host: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + path: + pattern: ^/ + type: string + port: + properties: + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - targetPort + type: object + subdomain: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + tls: + properties: + caCertificate: + type: string + certificate: + type: string + destinationCACertificate: + type: string + insecureEdgeTerminationPolicy: + type: string + key: + type: string + termination: + enum: + - edge + - reencrypt + - passthrough + type: string + required: + - termination + type: object + to: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + wildcardPolicy: + enum: + - None + - Subdomain + - "" + type: string + type: object + type: object + type: object enabled: default: false type: boolean @@ -4103,6 +4204,64 @@ spec: additionalProperties: type: string type: object + override: + properties: + service: + items: + properties: + endpoint: + enum: + - internal + - public + type: string + endpointURL: + type: string + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + externalName: + type: string + externalTrafficPolicy: + type: string + internalTrafficPolicy: + type: string + ipFamilyPolicy: + type: string + loadBalancerClass: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + sessionAffinity: + type: string + sessionAffinityConfig: + properties: + clientIP: + properties: + timeoutSeconds: + format: int32 + type: integer + type: object + type: object + type: + type: string + type: object + required: + - endpoint + type: object + type: array + type: object preserveJobs: default: false type: boolean @@ -4143,20 +4302,6 @@ spec: x-kubernetes-int-or-string: true type: object type: object - route: - properties: - routeLocation: - type: string - routeName: - default: horizon - type: string - routeTLSCA: - type: string - routeTLSEnabled: - type: string - routeTLSKey: - type: string - type: object secret: type: string required: diff --git a/pkg/openstack/horizon.go b/pkg/openstack/horizon.go index 84f7a6ed8..06a54dd6d 100644 --- a/pkg/openstack/horizon.go +++ b/pkg/openstack/horizon.go @@ -5,13 +5,17 @@ import ( "fmt" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/endpoint" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" horizonv1 "github.com/openstack-k8s-operators/horizon-operator/api/v1beta1" corev1beta1 "github.com/openstack-k8s-operators/openstack-operator/apis/core/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" ctrl "sigs.k8s.io/controller-runtime" ) @@ -31,9 +35,49 @@ func ReconcileHorizon(ctx context.Context, instance *corev1beta1.OpenStackContro instance.Status.Conditions.Remove(corev1beta1.OpenStackControlPlaneHorizonReadyCondition) return ctrl.Result{}, nil } + + // Create service overrides to pass into the service CR + // and expose the public endpoint using a route per default. + // Any trailing path will be added on the service-operator level. + var endpoints = map[service.Endpoint]endpoint.Data{ + service.EndpointPublic: {}, + service.EndpointInternal: {}, + } + serviceDetails := []ServiceDetails{} + + serviceOverrides := []service.OverrideSpec{} + for endpointType := range endpoints { + + sd := ServiceDetails{ + ServiceName: horizon.Name, + Namespace: instance.Namespace, + Endpoint: endpointType, + ExternalEndpoints: nil, + ServiceOverrideSpec: instance.Spec.Horizon.Template.Override.Service, + RouteOverrideSpec: instance.Spec.Horizon.APIOverride.Route, + } + + svcOverride, ctrlResult, err := sd.CreateRouteAndServiceOverride(ctx, instance, helper) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + serviceDetails = append( + serviceDetails, + sd, + ) + if svcOverride != nil { + serviceOverrides = append(serviceOverrides, *svcOverride) + } + } + helper.GetLogger().Info("Reconcile Horizon", "horizon.Namespace", instance.Namespace, "horizon.Name", "horizon") op, err := controllerutil.CreateOrPatch(ctx, helper.GetClient(), horizon, func() error { instance.Spec.Horizon.Template.DeepCopyInto(&horizon.Spec) + horizon.Spec.Override.Service = serviceOverrides + err := controllerutil.SetControllerReference(helper.GetBeforeObject(), horizon, helper.GetScheme()) if err != nil { return err @@ -64,5 +108,27 @@ func ReconcileHorizon(ctx context.Context, instance *corev1beta1.OpenStackContro corev1beta1.OpenStackControlPlaneHorizonReadyRunningMessage)) } + for _, sd := range serviceDetails { + // Add the service CR to the ownerRef list of the route to prevent the route being deleted + // before the service is deleted. Otherwise this can result cleanup issues which require + // the endpoint to be reachable. + // If ALL objects in the list have been deleted, this object will be garbage collected. + // https://github.com/kubernetes/apimachinery/blob/15d95c0b2af3f4fcf46dce24105e5fbb9379af5a/pkg/apis/meta/v1/types.go#L240-L247 + scheme := runtime.NewScheme() + gvk := schema.GroupVersionKind{ + Group: horizonv1.GroupVersion.Group, + Version: horizonv1.GroupVersion.Version, + Kind: horizon.Kind, + } + + // Add the GVK to the scheme + scheme.AddKnownTypeWithName(gvk, &horizonv1.Horizon{}) + + err = sd.AddOwnerRef(ctx, helper, horizon, scheme) + if err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{}, nil }