From 37cf6a1fe11459f23a22a7d636c37939ea1f2227 Mon Sep 17 00:00:00 2001 From: Martin Schuppert Date: Fri, 4 Aug 2023 11:32:23 +0200 Subject: [PATCH] Create placementapi route and svc endpoint overrides Creates the route for the placementapi, also allows to customize the route via override. Generats the service override for the env with what is configured in the externalEndpoints, or specified in the service template override. Depends-On: https://github.com/openstack-k8s-operators/lib-common/pull/313 Depends-On: https://github.com/openstack-k8s-operators/keystone-operator/pull/289 Depends-On: https://github.com/openstack-k8s-operators/placement-operator/pull/48 Jira: OSP-26690 --- ....openstack.org_openstackcontrolplanes.yaml | 101 ++++++++++++++++++ apis/core/v1beta1/conditions.go | 3 + .../v1beta1/openstackcontrolplane_types.go | 5 + apis/core/v1beta1/zz_generated.deepcopy.go | 1 + ....openstack.org_openstackcontrolplanes.yaml | 101 ++++++++++++++++++ ...nstack-operator.clusterserviceversion.yaml | 18 ++++ ...controlplane_galera_network_isolation.yaml | 17 ++- ...ne_galera_network_isolation_3replicas.yaml | 17 ++- ...enstackcontrolplane_network_isolation.yaml | 17 ++- ...ckcontrolplane_network_isolation_ceph.yaml | 17 ++- pkg/openstack/placement.go | 54 ++++++++++ 11 files changed, 331 insertions(+), 20 deletions(-) diff --git a/apis/bases/core.openstack.org_openstackcontrolplanes.yaml b/apis/bases/core.openstack.org_openstackcontrolplanes.yaml index 22fd81e4f..175d828e4 100644 --- a/apis/bases/core.openstack.org_openstackcontrolplanes.yaml +++ b/apis/bases/core.openstack.org_openstackcontrolplanes.yaml @@ -7602,6 +7602,107 @@ spec: type: object placement: properties: + apiOverride: + properties: + route: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + alternateBackends: + items: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + maxItems: 3 + type: array + host: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + path: + pattern: ^/ + type: string + port: + properties: + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - targetPort + type: object + subdomain: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + tls: + properties: + caCertificate: + type: string + certificate: + type: string + destinationCACertificate: + type: string + insecureEdgeTerminationPolicy: + type: string + key: + type: string + termination: + enum: + - edge + - reencrypt + - passthrough + type: string + required: + - termination + type: object + to: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + wildcardPolicy: + enum: + - None + - Subdomain + - "" + type: string + type: object + type: object + type: object enabled: default: true type: boolean diff --git a/apis/core/v1beta1/conditions.go b/apis/core/v1beta1/conditions.go index da1e13252..6999943f9 100644 --- a/apis/core/v1beta1/conditions.go +++ b/apis/core/v1beta1/conditions.go @@ -39,6 +39,9 @@ const ( // OpenStackControlPlanePlacementAPIReadyCondition Status=True condition which indicates if PlacementAPI is configured and operational OpenStackControlPlanePlacementAPIReadyCondition condition.Type = "OpenStackControlPlanePlacementAPIReady" + // OpenStackControlPlaneExposePlacementAPIReadyCondition Status=True condition which indicates if PlacementAPI is exposed via a route + OpenStackControlPlaneExposePlacementAPIReadyCondition condition.Type = "OpenStackControlPlaneExposePlacementAPIReady" + // OpenStackControlPlaneGlanceReadyCondition Status=True condition which indicates if Glance is configured and operational OpenStackControlPlaneGlanceReadyCondition condition.Type = "OpenStackControlPlaneGlanceReady" diff --git a/apis/core/v1beta1/openstackcontrolplane_types.go b/apis/core/v1beta1/openstackcontrolplane_types.go index c4da6f416..e272ecd57 100644 --- a/apis/core/v1beta1/openstackcontrolplane_types.go +++ b/apis/core/v1beta1/openstackcontrolplane_types.go @@ -217,6 +217,11 @@ type PlacementSection struct { //+operator-sdk:csv:customresourcedefinitions:type=spec // Template - Overrides to use when creating the Placement API Template placementv1.PlacementAPISpec `json:"template,omitempty"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // APIOverride, provides the ability to override the generated manifest of several child resources. + APIOverride Override `json:"apiOverride,omitempty"` } // GlanceSection defines the desired state of Glance service diff --git a/apis/core/v1beta1/zz_generated.deepcopy.go b/apis/core/v1beta1/zz_generated.deepcopy.go index 70a69e7e6..d25bada84 100644 --- a/apis/core/v1beta1/zz_generated.deepcopy.go +++ b/apis/core/v1beta1/zz_generated.deepcopy.go @@ -541,6 +541,7 @@ func (in *OvnSection) DeepCopy() *OvnSection { func (in *PlacementSection) DeepCopyInto(out *PlacementSection) { *out = *in in.Template.DeepCopyInto(&out.Template) + in.APIOverride.DeepCopyInto(&out.APIOverride) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlacementSection. diff --git a/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml b/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml index 22fd81e4f..175d828e4 100644 --- a/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml +++ b/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml @@ -7602,6 +7602,107 @@ spec: type: object placement: properties: + apiOverride: + properties: + route: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + alternateBackends: + items: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + maxItems: 3 + type: array + host: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + path: + pattern: ^/ + type: string + port: + properties: + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - targetPort + type: object + subdomain: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + tls: + properties: + caCertificate: + type: string + certificate: + type: string + destinationCACertificate: + type: string + insecureEdgeTerminationPolicy: + type: string + key: + type: string + termination: + enum: + - edge + - reencrypt + - passthrough + type: string + required: + - termination + type: object + to: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + wildcardPolicy: + enum: + - None + - Subdomain + - "" + type: string + type: object + type: object + type: object enabled: default: true type: boolean diff --git a/config/manifests/bases/openstack-operator.clusterserviceversion.yaml b/config/manifests/bases/openstack-operator.clusterserviceversion.yaml index e1e8dfd42..d69ba1dee 100644 --- a/config/manifests/bases/openstack-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/openstack-operator.clusterserviceversion.yaml @@ -248,6 +248,24 @@ spec: path: placement.enabled x-descriptors: - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: IPAddressPool expose VIP via MetalLB on the IPAddressPool + displayName: IPAddress Pool + path: placement.externalEndpoints[0].ipAddressPool + - description: LoadBalancerIPs, request given IPs from the pool if available. + Using a list to allow dual stack (IPv4/IPv6) support + displayName: Load Balancer IPs + path: placement.externalEndpoints[0].loadBalancerIPs + - description: SharedIP if true, VIP/VIPs get shared with multiple services + displayName: Shared IP + path: placement.externalEndpoints[0].sharedIP + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: SharedIPKey specifies the sharing key which gets set as the annotation + on the LoadBalancer service. Services which share the same VIP must have + the same SharedIPKey. Defaults to the IPAddressPool if SharedIP is true, + but no SharedIPKey specified. + displayName: Shared IPKey + path: placement.externalEndpoints[0].sharedIPKey - description: Template - Overrides to use when creating the Placement API displayName: Template path: placement.template diff --git a/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml b/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml index 1a8fe37d6..daa8d078e 100644 --- a/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml +++ b/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml @@ -165,14 +165,21 @@ spec: ovn-encap-type: "geneve" networkAttachment: tenant placement: + apiOverride: + route: {} template: + override: + service: + internal: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.80 + spec: + type: LoadBalancer databaseInstance: openstack secret: osp-secret - externalEndpoints: - - endpoint: internal - ipAddressPool: internalapi - loadBalancerIPs: - - 172.17.0.80 rabbitmq: templates: rabbitmq: diff --git a/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation_3replicas.yaml b/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation_3replicas.yaml index 9a6db0997..f450e54b4 100644 --- a/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation_3replicas.yaml +++ b/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation_3replicas.yaml @@ -165,14 +165,21 @@ spec: ovn-encap-type: "geneve" networkAttachment: tenant placement: + apiOverride: + route: {} template: + override: + service: + internal: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.80 + spec: + type: LoadBalancer databaseInstance: openstack secret: osp-secret - externalEndpoints: - - endpoint: internal - ipAddressPool: internalapi - loadBalancerIPs: - - 172.17.0.80 rabbitmq: templates: rabbitmq: diff --git a/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml b/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml index a137f5d0a..878dd5790 100644 --- a/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml +++ b/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml @@ -153,14 +153,21 @@ spec: ovn-encap-type: "geneve" networkAttachment: tenant placement: + apiOverride: + route: {} template: + override: + service: + internal: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.80 + spec: + type: LoadBalancer databaseInstance: openstack secret: osp-secret - externalEndpoints: - - endpoint: internal - ipAddressPool: internalapi - loadBalancerIPs: - - 172.17.0.80 rabbitmq: templates: rabbitmq: diff --git a/config/samples/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml b/config/samples/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml index b01e91f8d..0c0ae2f82 100644 --- a/config/samples/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml +++ b/config/samples/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml @@ -211,14 +211,21 @@ spec: ovn-encap-type: "geneve" networkAttachment: tenant placement: + apiOverride: + route: {} template: + override: + service: + internal: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.80 + spec: + type: LoadBalancer databaseInstance: openstack secret: osp-secret - externalEndpoints: - - endpoint: internal - ipAddressPool: internalapi - loadBalancerIPs: - - 172.17.0.80 rabbitmq: templates: rabbitmq: diff --git a/pkg/openstack/placement.go b/pkg/openstack/placement.go index 398a052c1..d9f9402d7 100644 --- a/pkg/openstack/placement.go +++ b/pkg/openstack/placement.go @@ -4,14 +4,20 @@ import ( "context" "fmt" + "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/reconcile" corev1beta1 "github.com/openstack-k8s-operators/openstack-operator/apis/core/v1beta1" placementv1 "github.com/openstack-k8s-operators/placement-operator/api/v1beta1" + k8s_errors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" ) @@ -29,12 +35,60 @@ func ReconcilePlacementAPI(ctx context.Context, instance *corev1beta1.OpenStackC return res, err } instance.Status.Conditions.Remove(corev1beta1.OpenStackControlPlanePlacementAPIReadyCondition) + instance.Status.Conditions.Remove(corev1beta1.OpenStackControlPlaneExposePlacementAPIReadyCondition) return ctrl.Result{}, nil } + // add selector to service overrides + for _, endpointType := range []service.Endpoint{service.EndpointPublic, service.EndpointInternal} { + if instance.Spec.Placement.Template.Override.Service == nil { + instance.Spec.Placement.Template.Override.Service = map[string]service.RoutedOverrideSpec{} + } + instance.Spec.Placement.Template.Override.Service[string(endpointType)] = AddServiceComponentLabel( + ptr.To(instance.Spec.Placement.Template.Override.Service[string(endpointType)]), + placementAPI.Name) + } + + // When component services got created check if there is the need to create a route + if err := helper.GetClient().Get(ctx, types.NamespacedName{Name: "placement", Namespace: instance.Namespace}, placementAPI); err != nil { + if !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + } + + if placementAPI.Status.Conditions.IsTrue(condition.ExposeServiceReadyCondition) { + svcs, err := service.GetServicesListWithLabel( + ctx, + helper, + instance.Namespace, + map[string]string{common.AppSelector: placementAPI.Name}, + ) + if err != nil { + return ctrl.Result{}, err + } + + var ctrlResult reconcile.Result + instance.Spec.Placement.Template.Override.Service, ctrlResult, err = EnsureRoute( + ctx, + instance, + helper, + placementAPI, + svcs, + instance.Spec.Placement.Template.Override.Service, + instance.Spec.Placement.APIOverride.Route, + corev1beta1.OpenStackControlPlaneExposePlacementAPIReadyCondition, + ) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + } + helper.GetLogger().Info("Reconciling PlacementAPI", "PlacementAPI.Namespace", instance.Namespace, "PlacementAPI.Name", "placement") op, err := controllerutil.CreateOrPatch(ctx, helper.GetClient(), placementAPI, func() error { instance.Spec.Placement.Template.DeepCopyInto(&placementAPI.Spec) + if placementAPI.Spec.Secret == "" { placementAPI.Spec.Secret = instance.Spec.Secret }