From 8b3105dee9c412dc23c513e77e3838651a785998 Mon Sep 17 00:00:00 2001 From: Martin Schuppert Date: Thu, 27 Jul 2023 12:01:36 +0200 Subject: [PATCH] Create keystoneapi route and svc endpoint overrides Creates the route for the keystoneapi, also allows to customize the route via override. Generats the service override for the env with what is configured in the externalEndpoints, or specified in the service template override. Depends-On: https://github.com/openstack-k8s-operators/lib-common/pull/313 Depends-On: https://github.com/openstack-k8s-operators/keystone-operator/pull/289 Jira: OSP-26690 --- ....openstack.org_openstackcontrolplanes.yaml | 276 ++++++++++++++++-- .../v1beta1/openstackcontrolplane_types.go | 22 ++ apis/core/v1beta1/zz_generated.deepcopy.go | 29 ++ apis/go.mod | 8 + apis/go.sum | 8 +- ....openstack.org_openstackcontrolplanes.yaml | 276 ++++++++++++++++-- ...nstack-operator.clusterserviceversion.yaml | 18 ++ config/rbac/role.yaml | 12 + ...controlplane_galera_network_isolation.yaml | 10 +- ...enstackcontrolplane_network_isolation.yaml | 10 +- .../core/openstackcontrolplane_controller.go | 3 + go.mod | 12 +- go.sum | 8 +- main.go | 2 + pkg/openstack/common.go | 203 +++++++++++++ pkg/openstack/keystone.go | 82 ++++++ 16 files changed, 907 insertions(+), 72 deletions(-) diff --git a/apis/bases/core.openstack.org_openstackcontrolplanes.yaml b/apis/bases/core.openstack.org_openstackcontrolplanes.yaml index 8bd155512..f27024429 100644 --- a/apis/bases/core.openstack.org_openstackcontrolplanes.yaml +++ b/apis/bases/core.openstack.org_openstackcontrolplanes.yaml @@ -3929,6 +3929,133 @@ spec: enabled: default: true type: boolean + externalEndpoints: + items: + properties: + endpoint: + default: internal + enum: + - internal + - public + type: string + ipAddressPool: + minLength: 1 + type: string + loadBalancerIPs: + items: + type: string + type: array + sharedIP: + default: true + type: boolean + sharedIPKey: + default: "" + type: string + required: + - ipAddressPool + type: object + type: array + override: + properties: + route: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + alternateBackends: + items: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + maxItems: 3 + type: array + host: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + path: + pattern: ^/ + type: string + port: + properties: + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - targetPort + type: object + subdomain: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + tls: + properties: + caCertificate: + type: string + certificate: + type: string + destinationCACertificate: + type: string + insecureEdgeTerminationPolicy: + type: string + key: + type: string + termination: + enum: + - edge + - reencrypt + - passthrough + type: string + required: + - termination + type: object + to: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + wildcardPolicy: + enum: + - None + - Subdomain + - "" + type: string + type: object + type: object + type: object template: properties: adminProject: @@ -3962,32 +4089,6 @@ spec: additionalProperties: type: string type: object - externalEndpoints: - items: - properties: - endpoint: - enum: - - internal - - public - type: string - ipAddressPool: - minLength: 1 - type: string - loadBalancerIPs: - items: - type: string - type: array - sharedIP: - default: true - type: boolean - sharedIPKey: - default: "" - type: string - required: - - endpoint - - ipAddressPool - type: object - type: array memcachedInstance: default: memcached type: string @@ -3999,6 +4100,123 @@ spec: additionalProperties: type: string type: object + override: + properties: + service: + items: + properties: + endpoint: + enum: + - internal + - public + type: string + endpointURL: + type: string + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + allocateLoadBalancerNodePorts: + type: boolean + clusterIP: + type: string + clusterIPs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalIPs: + items: + type: string + type: array + externalName: + type: string + externalTrafficPolicy: + type: string + healthCheckNodePort: + format: int32 + type: integer + internalTrafficPolicy: + type: string + ipFamilies: + items: + type: string + type: array + x-kubernetes-list-type: atomic + ipFamilyPolicy: + type: string + loadBalancerClass: + type: string + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + ports: + items: + properties: + appProtocol: + type: string + name: + type: string + nodePort: + format: int32 + type: integer + port: + format: int32 + type: integer + protocol: + default: TCP + type: string + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + - protocol + x-kubernetes-list-type: map + publishNotReadyAddresses: + type: boolean + selector: + additionalProperties: + type: string + type: object + x-kubernetes-map-type: atomic + sessionAffinity: + type: string + sessionAffinityConfig: + properties: + clientIP: + properties: + timeoutSeconds: + format: int32 + type: integer + type: object + type: object + type: + type: string + type: object + required: + - endpoint + type: object + type: array + type: object passwordSelectors: default: admin: AdminPassword @@ -7341,6 +7559,12 @@ spec: type: object externalEndpoint: properties: + endpoint: + default: internal + enum: + - internal + - public + type: string ipAddressPool: minLength: 1 type: string diff --git a/apis/core/v1beta1/openstackcontrolplane_types.go b/apis/core/v1beta1/openstackcontrolplane_types.go index f86e57c74..3fadb84a6 100644 --- a/apis/core/v1beta1/openstackcontrolplane_types.go +++ b/apis/core/v1beta1/openstackcontrolplane_types.go @@ -28,6 +28,8 @@ import ( ironicv1 "github.com/openstack-k8s-operators/ironic-operator/api/v1beta1" keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/route" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" "github.com/openstack-k8s-operators/lib-common/modules/common/util" "github.com/openstack-k8s-operators/lib-common/modules/storage" manilav1 "github.com/openstack-k8s-operators/manila-operator/api/v1beta1" @@ -186,6 +188,19 @@ type KeystoneSection struct { //+operator-sdk:csv:customresourcedefinitions:type=spec // Template - Overrides to use when creating the Keystone service Template keystonev1.KeystoneAPISpec `json:"template,omitempty"` + + // ExternalEndpoints, expose a VIP using a pre-created IPAddressPool + ExternalEndpoints []MetalLBConfig `json:"externalEndpoints,omitempty"` + + // +kubebuilder:validation:Optional + // Override, provides the ability to override the generated manifest of several child resources. + Override KeystoneOverrideSpec `json:"override,omitempty"` +} + +// KeystoneOverrideSpec to override the generated manifest of several child resources. +type KeystoneOverrideSpec struct { + // +kubebuilder:validation:Optional + Route *route.OverrideSpec `json:"route,omitempty"` } // PlacementSection defines the desired state of Placement service @@ -301,6 +316,12 @@ type RabbitmqTemplate struct { // MetalLBConfig to configure the MetalLB loadbalancer service type MetalLBConfig struct { + // +kubebuilder:validation:Optional + // +kubebuilder:validation:Enum=internal;public + // +kubebuilder:default=internal + // Endpoint, OpenStack endpoint this service maps to + Endpoint service.Endpoint `json:"endpoint"` + // +kubebuilder:validation:Required // +kubebuilder:validation:MinLength=1 //+operator-sdk:csv:customresourcedefinitions:type=spec @@ -537,6 +558,7 @@ func (instance *OpenStackControlPlane) InitConditions() { instance.Status.Conditions = condition.Conditions{} } cl := condition.CreateList( + condition.UnknownCondition(condition.ExposeServiceReadyCondition, condition.InitReason, condition.ExposeServiceReadyInitMessage), condition.UnknownCondition(OpenStackControlPlaneRabbitMQReadyCondition, condition.InitReason, OpenStackControlPlaneRabbitMQReadyInitMessage), condition.UnknownCondition(OpenStackControlPlaneOVNReadyCondition, condition.InitReason, OpenStackControlPlaneOVNReadyInitMessage), condition.UnknownCondition(OpenStackControlPlaneNeutronReadyCondition, condition.InitReason, OpenStackControlPlaneNeutronReadyInitMessage), diff --git a/apis/core/v1beta1/zz_generated.deepcopy.go b/apis/core/v1beta1/zz_generated.deepcopy.go index 1514626cd..dd345f2ba 100644 --- a/apis/core/v1beta1/zz_generated.deepcopy.go +++ b/apis/core/v1beta1/zz_generated.deepcopy.go @@ -24,6 +24,7 @@ package v1beta1 import ( memcachedv1beta1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/route" "github.com/openstack-k8s-operators/lib-common/modules/storage" apiv1beta1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" ovn_operatorapiv1beta1 "github.com/openstack-k8s-operators/ovn-operator/api/v1beta1" @@ -164,10 +165,38 @@ func (in *IronicSection) DeepCopy() *IronicSection { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KeystoneOverrideSpec) DeepCopyInto(out *KeystoneOverrideSpec) { + *out = *in + if in.Route != nil { + in, out := &in.Route, &out.Route + *out = new(route.OverrideSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KeystoneOverrideSpec. +func (in *KeystoneOverrideSpec) DeepCopy() *KeystoneOverrideSpec { + if in == nil { + return nil + } + out := new(KeystoneOverrideSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KeystoneSection) DeepCopyInto(out *KeystoneSection) { *out = *in in.Template.DeepCopyInto(&out.Template) + if in.ExternalEndpoints != nil { + in, out := &in.ExternalEndpoints, &out.ExternalEndpoints + *out = make([]MetalLBConfig, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + in.Override.DeepCopyInto(&out.Override) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KeystoneSection. diff --git a/apis/go.mod b/apis/go.mod index 9cb653dcc..518008430 100644 --- a/apis/go.mod +++ b/apis/go.mod @@ -100,3 +100,11 @@ require ( // mschuppert: map to latest commit from release-4.13 tag // must consistent within modules and service operators replace github.com/openshift/api => github.com/openshift/api v0.0.0-20230414143018-3367bc7e6ac7 //allow-merging + +replace github.com/openstack-k8s-operators/lib-common/modules/common => github.com/stuggi/lib-common/modules/common v0.0.0-20230801084758-6169f85dea1d + +//replace github.com/openstack-k8s-operators/lib-common/modules/common => /home/mschuppe/src/github.com/openstack-k8s-operators/lib-common/modules/common + +replace github.com/openstack-k8s-operators/keystone-operator/api => github.com/stuggi/keystone-operator/api v0.0.0-20230801072240-78012cee7ee6 + +//replace github.com/openstack-k8s-operators/keystone-operator/api => /home/mschuppe/src/github.com/openstack-k8s-operators/keystone-operator/api diff --git a/apis/go.sum b/apis/go.sum index 107c07470..a34ef9961 100644 --- a/apis/go.sum +++ b/apis/go.sum @@ -138,10 +138,6 @@ github.com/openstack-k8s-operators/infra-operator/apis v0.1.0 h1:cavVlTrKeW2xcyt github.com/openstack-k8s-operators/infra-operator/apis v0.1.0/go.mod h1:t1xmsiZDqM3wXcLMqgHp7/iilK8ozuOkydV4Vi2Qibk= github.com/openstack-k8s-operators/ironic-operator/api v0.1.0 h1:Bm0mo2iRT16CG/WNj3stuheDGQDDZY3CGVv52AFDH1w= github.com/openstack-k8s-operators/ironic-operator/api v0.1.0/go.mod h1:slFWxIEbjO2tkyPR6GoKI0pIDgctKsPVB+Wg6D+CbQ0= -github.com/openstack-k8s-operators/keystone-operator/api v0.1.0 h1:p98vKnS4KzdgU/+vrVKFY3y9n9v1Z6cpo4JvbTNRxlM= -github.com/openstack-k8s-operators/keystone-operator/api v0.1.0/go.mod h1:LNJJdteQG4E2fhWDerE+f8S2/ephEJg8yBkH1eqYYOo= -github.com/openstack-k8s-operators/lib-common/modules/common v0.1.0 h1:F1iYRBwa0cZ2VHw8Zs4frqSWQ1B/tiCuSwH/DuHb8VM= -github.com/openstack-k8s-operators/lib-common/modules/common v0.1.0/go.mod h1:3hAC5Ce0AOSt85BqD6DgTKNkJHmpXwqbwL8mVWRJQqo= github.com/openstack-k8s-operators/lib-common/modules/openstack v0.1.0 h1:mMeJvCQfZmakssvMyHjzp/ngxKysETDj9GJYhRwydzg= github.com/openstack-k8s-operators/lib-common/modules/openstack v0.1.0/go.mod h1:+paEFOL5IlJzhg9fy7/1+HSErVkWUgUj1ORLFwgvxnI= github.com/openstack-k8s-operators/lib-common/modules/storage v0.1.0 h1:rHn0wlwBBggRl65gWDniF97XW+1XB2+4PsGZS2RIJ5E= @@ -195,6 +191,10 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stuggi/keystone-operator/api v0.0.0-20230801072240-78012cee7ee6 h1:AGAL1xVaiufVo765y5uRKb/zOWw/M78hAOsdSy7nZuo= +github.com/stuggi/keystone-operator/api v0.0.0-20230801072240-78012cee7ee6/go.mod h1:fMstyYIgZTTLLO9q2jya/aF9ay4CedCLc9Mb6eXsxaA= +github.com/stuggi/lib-common/modules/common v0.0.0-20230801084758-6169f85dea1d h1:7yXrIPpHM0hYy6YQxbv6UHO7GUm36QE/DW2Hg4abSv4= +github.com/stuggi/lib-common/modules/common v0.0.0-20230801084758-6169f85dea1d/go.mod h1:2fZFojsJsTA4MfGy50JEwbXaIP7Hr7h7x8hbhlMBedY= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= diff --git a/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml b/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml index 8bd155512..f27024429 100644 --- a/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml +++ b/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml @@ -3929,6 +3929,133 @@ spec: enabled: default: true type: boolean + externalEndpoints: + items: + properties: + endpoint: + default: internal + enum: + - internal + - public + type: string + ipAddressPool: + minLength: 1 + type: string + loadBalancerIPs: + items: + type: string + type: array + sharedIP: + default: true + type: boolean + sharedIPKey: + default: "" + type: string + required: + - ipAddressPool + type: object + type: array + override: + properties: + route: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + alternateBackends: + items: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + maxItems: 3 + type: array + host: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + path: + pattern: ^/ + type: string + port: + properties: + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - targetPort + type: object + subdomain: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + tls: + properties: + caCertificate: + type: string + certificate: + type: string + destinationCACertificate: + type: string + insecureEdgeTerminationPolicy: + type: string + key: + type: string + termination: + enum: + - edge + - reencrypt + - passthrough + type: string + required: + - termination + type: object + to: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + wildcardPolicy: + enum: + - None + - Subdomain + - "" + type: string + type: object + type: object + type: object template: properties: adminProject: @@ -3962,32 +4089,6 @@ spec: additionalProperties: type: string type: object - externalEndpoints: - items: - properties: - endpoint: - enum: - - internal - - public - type: string - ipAddressPool: - minLength: 1 - type: string - loadBalancerIPs: - items: - type: string - type: array - sharedIP: - default: true - type: boolean - sharedIPKey: - default: "" - type: string - required: - - endpoint - - ipAddressPool - type: object - type: array memcachedInstance: default: memcached type: string @@ -3999,6 +4100,123 @@ spec: additionalProperties: type: string type: object + override: + properties: + service: + items: + properties: + endpoint: + enum: + - internal + - public + type: string + endpointURL: + type: string + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + allocateLoadBalancerNodePorts: + type: boolean + clusterIP: + type: string + clusterIPs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalIPs: + items: + type: string + type: array + externalName: + type: string + externalTrafficPolicy: + type: string + healthCheckNodePort: + format: int32 + type: integer + internalTrafficPolicy: + type: string + ipFamilies: + items: + type: string + type: array + x-kubernetes-list-type: atomic + ipFamilyPolicy: + type: string + loadBalancerClass: + type: string + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + ports: + items: + properties: + appProtocol: + type: string + name: + type: string + nodePort: + format: int32 + type: integer + port: + format: int32 + type: integer + protocol: + default: TCP + type: string + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + - protocol + x-kubernetes-list-type: map + publishNotReadyAddresses: + type: boolean + selector: + additionalProperties: + type: string + type: object + x-kubernetes-map-type: atomic + sessionAffinity: + type: string + sessionAffinityConfig: + properties: + clientIP: + properties: + timeoutSeconds: + format: int32 + type: integer + type: object + type: object + type: + type: string + type: object + required: + - endpoint + type: object + type: array + type: object passwordSelectors: default: admin: AdminPassword @@ -7341,6 +7559,12 @@ spec: type: object externalEndpoint: properties: + endpoint: + default: internal + enum: + - internal + - public + type: string ipAddressPool: minLength: 1 type: string diff --git a/config/manifests/bases/openstack-operator.clusterserviceversion.yaml b/config/manifests/bases/openstack-operator.clusterserviceversion.yaml index 3734864f9..f7f922ef0 100644 --- a/config/manifests/bases/openstack-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/openstack-operator.clusterserviceversion.yaml @@ -117,6 +117,24 @@ spec: path: keystone.enabled x-descriptors: - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: IPAddressPool expose VIP via MetalLB on the IPAddressPool + displayName: IPAddress Pool + path: keystone.externalEndpoints[0].ipAddressPool + - description: LoadBalancerIPs, request given IPs from the pool if available. + Using a list to allow dual stack (IPv4/IPv6) support + displayName: Load Balancer IPs + path: keystone.externalEndpoints[0].loadBalancerIPs + - description: SharedIP if true, VIP/VIPs get shared with multiple services + displayName: Shared IP + path: keystone.externalEndpoints[0].sharedIP + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: SharedIPKey specifies the sharing key which gets set as the annotation + on the LoadBalancer service. Services which share the same VIP must have + the same SharedIPKey. Defaults to the IPAddressPool if SharedIP is true, + but no SharedIPKey specified. + displayName: Shared IPKey + path: keystone.externalEndpoints[0].sharedIPKey - description: Template - Overrides to use when creating the Keystone service displayName: Template path: keystone.template diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 32ad844b0..e43290c9e 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -331,6 +331,18 @@ rules: - list - update - watch +- apiGroups: + - route.openshift.io + resources: + - routes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - security.openshift.io resourceNames: diff --git a/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml b/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml index e42a54a66..e082b1c94 100644 --- a/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml +++ b/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml @@ -54,14 +54,14 @@ spec: networkAttachments: - storage keystone: + externalEndpoints: + - endpoint: internal + ipAddressPool: internalapi + loadBalancerIPs: + - 172.17.0.80 template: databaseInstance: openstack secret: osp-secret - externalEndpoints: - - endpoint: internal - ipAddressPool: internalapi - loadBalancerIPs: - - 172.17.0.80 mariadb: enabled: false templates: diff --git a/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml b/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml index ed3a7f5f3..8473f0ccd 100644 --- a/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml +++ b/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml @@ -54,14 +54,14 @@ spec: networkAttachments: - storage keystone: + externalEndpoints: + - endpoint: internal + ipAddressPool: internalapi + loadBalancerIPs: + - 172.17.0.80 template: databaseInstance: openstack secret: osp-secret - externalEndpoints: - - endpoint: internal - ipAddressPool: internalapi - loadBalancerIPs: - - 172.17.0.80 mariadb: templates: openstack: diff --git a/controllers/core/openstackcontrolplane_controller.go b/controllers/core/openstackcontrolplane_controller.go index 582809af4..7a61e55d2 100644 --- a/controllers/core/openstackcontrolplane_controller.go +++ b/controllers/core/openstackcontrolplane_controller.go @@ -20,6 +20,7 @@ import ( "context" "fmt" + routev1 "github.com/openshift/api/route/v1" cinderv1 "github.com/openstack-k8s-operators/cinder-operator/api/v1beta1" glancev1 "github.com/openstack-k8s-operators/glance-operator/api/v1beta1" heatv1 "github.com/openstack-k8s-operators/heat-operator/api/v1beta1" @@ -83,6 +84,7 @@ type OpenStackControlPlaneReconciler struct { //+kubebuilder:rbac:groups=network.openstack.org,resources=dnsmasqs,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=telemetry.openstack.org,resources=ceilometercentrals,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=swift.openstack.org,resources=swifts,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch;create;update;patch;delete; // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -313,5 +315,6 @@ func (r *OpenStackControlPlaneReconciler) SetupWithManager(mgr ctrl.Manager) err Owns(&ironicv1.Ironic{}). Owns(&horizonv1.Horizon{}). Owns(&telemetryv1.CeilometerCentral{}). + Owns(&routev1.Route{}). Complete(r) } diff --git a/go.mod b/go.mod index 1d02e9d20..ab9671fc0 100644 --- a/go.mod +++ b/go.mod @@ -73,7 +73,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/openshift/api v3.9.0+incompatible // indirect + github.com/openshift/api v3.9.0+incompatible github.com/openstack-k8s-operators/lib-common/modules/openstack v0.1.0 //indirect github.com/openstack-k8s-operators/lib-common/modules/storage v0.1.0 //indirect github.com/pkg/errors v0.9.1 // indirect @@ -101,7 +101,7 @@ require ( k8s.io/component-base v0.26.7 //indirect k8s.io/klog/v2 v2.100.1 // indirect k8s.io/kube-openapi v0.0.0-20230515203736-54b630e78af5 //indirect - k8s.io/utils v0.0.0-20230711102312-30195339c3c7 //indirect + k8s.io/utils v0.0.0-20230711102312-30195339c3c7 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd //indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect @@ -112,3 +112,11 @@ replace github.com/openstack-k8s-operators/openstack-operator/apis => ./apis // mschuppert: map to latest commit from release-4.13 tag // must consistent within modules and service operators replace github.com/openshift/api => github.com/openshift/api v0.0.0-20230414143018-3367bc7e6ac7 //allow-merging + +replace github.com/openstack-k8s-operators/lib-common/modules/common => github.com/stuggi/lib-common/modules/common v0.0.0-20230801084758-6169f85dea1d + +//replace github.com/openstack-k8s-operators/lib-common/modules/common => /home/mschuppe/src/github.com/openstack-k8s-operators/lib-common/modules/common + +replace github.com/openstack-k8s-operators/keystone-operator/api => github.com/stuggi/keystone-operator/api v0.0.0-20230801072240-78012cee7ee6 + +//replace github.com/openstack-k8s-operators/keystone-operator/api => /home/mschuppe/src/github.com/openstack-k8s-operators/keystone-operator/api diff --git a/go.sum b/go.sum index 5dce52250..30386b19f 100644 --- a/go.sum +++ b/go.sum @@ -147,10 +147,6 @@ github.com/openstack-k8s-operators/infra-operator/apis v0.1.0 h1:cavVlTrKeW2xcyt github.com/openstack-k8s-operators/infra-operator/apis v0.1.0/go.mod h1:t1xmsiZDqM3wXcLMqgHp7/iilK8ozuOkydV4Vi2Qibk= github.com/openstack-k8s-operators/ironic-operator/api v0.1.0 h1:Bm0mo2iRT16CG/WNj3stuheDGQDDZY3CGVv52AFDH1w= github.com/openstack-k8s-operators/ironic-operator/api v0.1.0/go.mod h1:slFWxIEbjO2tkyPR6GoKI0pIDgctKsPVB+Wg6D+CbQ0= -github.com/openstack-k8s-operators/keystone-operator/api v0.1.0 h1:p98vKnS4KzdgU/+vrVKFY3y9n9v1Z6cpo4JvbTNRxlM= -github.com/openstack-k8s-operators/keystone-operator/api v0.1.0/go.mod h1:LNJJdteQG4E2fhWDerE+f8S2/ephEJg8yBkH1eqYYOo= -github.com/openstack-k8s-operators/lib-common/modules/common v0.1.0 h1:F1iYRBwa0cZ2VHw8Zs4frqSWQ1B/tiCuSwH/DuHb8VM= -github.com/openstack-k8s-operators/lib-common/modules/common v0.1.0/go.mod h1:3hAC5Ce0AOSt85BqD6DgTKNkJHmpXwqbwL8mVWRJQqo= github.com/openstack-k8s-operators/lib-common/modules/openstack v0.1.0 h1:mMeJvCQfZmakssvMyHjzp/ngxKysETDj9GJYhRwydzg= github.com/openstack-k8s-operators/lib-common/modules/openstack v0.1.0/go.mod h1:+paEFOL5IlJzhg9fy7/1+HSErVkWUgUj1ORLFwgvxnI= github.com/openstack-k8s-operators/lib-common/modules/storage v0.1.0 h1:rHn0wlwBBggRl65gWDniF97XW+1XB2+4PsGZS2RIJ5E= @@ -212,6 +208,10 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stuggi/keystone-operator/api v0.0.0-20230801072240-78012cee7ee6 h1:AGAL1xVaiufVo765y5uRKb/zOWw/M78hAOsdSy7nZuo= +github.com/stuggi/keystone-operator/api v0.0.0-20230801072240-78012cee7ee6/go.mod h1:fMstyYIgZTTLLO9q2jya/aF9ay4CedCLc9Mb6eXsxaA= +github.com/stuggi/lib-common/modules/common v0.0.0-20230801084758-6169f85dea1d h1:7yXrIPpHM0hYy6YQxbv6UHO7GUm36QE/DW2Hg4abSv4= +github.com/stuggi/lib-common/modules/common v0.0.0-20230801084758-6169f85dea1d/go.mod h1:2fZFojsJsTA4MfGy50JEwbXaIP7Hr7h7x8hbhlMBedY= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= diff --git a/main.go b/main.go index 31cf5a972..490f14136 100644 --- a/main.go +++ b/main.go @@ -60,6 +60,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + routev1 "github.com/openshift/api/route/v1" clientv1 "github.com/openstack-k8s-operators/openstack-operator/apis/client/v1beta1" corev1 "github.com/openstack-k8s-operators/openstack-operator/apis/core/v1beta1" clientcontrollers "github.com/openstack-k8s-operators/openstack-operator/controllers/client" @@ -97,6 +98,7 @@ func init() { utilruntime.Must(telemetryv1.AddToScheme(scheme)) utilruntime.Must(swiftv1.AddToScheme(scheme)) utilruntime.Must(clientv1.AddToScheme(scheme)) + utilruntime.Must(routev1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } diff --git a/pkg/openstack/common.go b/pkg/openstack/common.go index 760a87c0a..7c6a93633 100644 --- a/pkg/openstack/common.go +++ b/pkg/openstack/common.go @@ -2,9 +2,19 @@ package openstack import ( "context" + "strings" + "time" + "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + "github.com/openstack-k8s-operators/lib-common/modules/common/route" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" + "github.com/openstack-k8s-operators/lib-common/modules/common/util" + corev1 "github.com/openstack-k8s-operators/openstack-operator/apis/core/v1beta1" + k8s_corev1 "k8s.io/api/core/v1" k8s_errors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -27,3 +37,196 @@ func EnsureDeleted(ctx context.Context, helper *helper.Helper, obj client.Object return ctrl.Result{}, nil } + +// GetExternalEnpointDetailsForEndpoint - returns the MetalLBData for the endpoint, if not specified nil. +func GetExternalEnpointDetailsForEndpoint( + externalEndpoints []corev1.MetalLBConfig, + endpt service.Endpoint, +) *corev1.MetalLBConfig { + for _, metallbcfg := range externalEndpoints { + if metallbcfg.Endpoint == endpt { + return metallbcfg.DeepCopy() + } + } + + return nil +} + +/* +// GetServiceName - +// TODO need this? +func GetServiceName(obj client.Object, crClient client.Client) (string, error) { + gvk, err := apiutil.GVKForObject(obj, crClient.Scheme()) + if err != nil { + return "", err + } + + return strings.ToLower(gvk.Kind), nil +} +*/ + +// ServiceDetails - service details to create service.Service with overrides +type ServiceDetails struct { + ServiceName string + Namespace string + Endpoint service.Endpoint + ExternalEndpoints []corev1.MetalLBConfig + ServiceOverrideSpec []service.OverrideSpec + RouteOverrideSpec *route.OverrideSpec + endpointName string + endpointURL string + baseServiceLabels map[string]string + serviceLabels map[string]string + hostname *string +} + +// GetServiceLabels - returns the labels for the service, used as selector on the route +func (sd *ServiceDetails) GetServiceLabels() map[string]string { + return sd.serviceLabels +} + +// GetEndpointName - returns the name of the endpoint +func (sd *ServiceDetails) GetEndpointName() string { + return sd.endpointName +} + +// GetEndpointURL - returns the URL of the endpoint (proto://hostname/path) +func (sd *ServiceDetails) GetEndpointURL() string { + return sd.endpointURL +} + +// public endpoint +// 1) Default (service override nil, no definition for public endpt in externalEndpoints) +// - create route + pass in service selector +// 2) externalEndpoints != nil for public endpt +// - pass in override for metallb LoadBalancer service +// 3) service override provided == LoadBalancer service type +// - do nothing let the service operator handle it +// 4) service override provided == ClusterIP service type +// - create route , merge override with service selector +// +// internal endpoint +// 1) Default (service override nil, no definition for public endpt in externalEndpoints) +// - service operator will create default service +// 2) externalEndpoints != nil for internal endpt, no service override provided +// - pass in override for metallb LoadBalancer service +// 3) externalEndpoints != nil for internal endpt, service override provided +// - merge override with metallb service and pass in override +// 4) externalEndpoints == nil for internal endpt, service override provided +// - do nothing let the service operator handle it + +// CreateEndpointServiceOverride - +func (sd *ServiceDetails) CreateEndpointServiceOverride() (*service.Service, error) { + + // set the endpoint name to service name - endpoint type + sd.endpointName = sd.ServiceName + "-" + string(sd.Endpoint) + + // create the service label + sd.baseServiceLabels = map[string]string{ + common.AppSelector: sd.ServiceName, + } + + // create the label for the service , used for the route -> service connection + sd.serviceLabels = util.MergeStringMaps( + sd.baseServiceLabels, + map[string]string{ + string(sd.Endpoint): "true", + }, + ) + + // initialize the service override if not specified via the CR + svcOverride := service.GetOverrideSpecForEndpoint(sd.ServiceOverrideSpec, sd.Endpoint).DeepCopy() + if svcOverride == nil { + svcOverride = &service.OverrideSpec{} + svcOverride.Endpoint = sd.Endpoint + } + + // get MetalLB config data for the endpoint + metalLBData := GetExternalEnpointDetailsForEndpoint(sd.ExternalEndpoints, sd.Endpoint) + + // Create generic service definitions for either MetalLB, or generic service + svcDef := &k8s_corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: sd.endpointName, + Namespace: sd.Namespace, + Labels: sd.serviceLabels, + }, + Spec: k8s_corev1.ServiceSpec{ + Type: k8s_corev1.ServiceTypeClusterIP, + }, + } + + // if the externalEndpoints is configured for the service and for this endpoint provided, + // set MetalLB annotations and service type to LoadBalancer + if metalLBData != nil { + // create MetalLB service definition + annotations := map[string]string{ + service.MetalLBAddressPoolAnnotation: metalLBData.IPAddressPool, + } + if len(metalLBData.LoadBalancerIPs) > 0 { + annotations[service.MetalLBLoadBalancerIPs] = strings.Join(metalLBData.LoadBalancerIPs, ",") + } + if metalLBData.SharedIP { + if metalLBData.SharedIPKey == "" { + annotations[service.MetalLBAllowSharedIPAnnotation] = metalLBData.IPAddressPool + } else { + annotations[service.MetalLBAllowSharedIPAnnotation] = metalLBData.SharedIPKey + } + } + svcDef.Annotations = annotations + svcDef.Spec.Type = k8s_corev1.ServiceTypeLoadBalancer + } + + // Create the service + svc, err := service.NewService( + svcDef, + time.Duration(5)*time.Second, //not really required as we don't create the service here + svcOverride, + ) + if err != nil { + return nil, err + } + + // add annotation to register service name in dnsmasq if it is a LoadBalancer service + if svc.GetServiceType() == k8s_corev1.ServiceTypeLoadBalancer { + svc.AddAnnotation(map[string]string{ + service.AnnotationHostnameKey: svc.GetServiceHostname(), + }) + } + + return svc, nil +} + +// CreateRoute - +func (sd *ServiceDetails) CreateRoute( + ctx context.Context, + helper *helper.Helper, + svc service.Service, +) (ctrl.Result, error) { + // TODO TLS + route, err := route.NewRoute( + route.GenericRoute(&route.GenericRouteDetails{ + Name: sd.ServiceName, + Namespace: sd.Namespace, + Labels: sd.baseServiceLabels, + ServiceName: sd.endpointName, + TargetPortName: sd.endpointName, + }), + time.Duration(5)*time.Second, + sd.RouteOverrideSpec, + ) + if err != nil { + return ctrl.Result{}, err + } + + ctrlResult, err := route.CreateOrPatch(ctx, helper) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + sd.hostname = pointer.String(route.GetHostname()) + + return ctrl.Result{}, nil +} diff --git a/pkg/openstack/keystone.go b/pkg/openstack/keystone.go index b84846c9b..6eb3f11d2 100644 --- a/pkg/openstack/keystone.go +++ b/pkg/openstack/keystone.go @@ -11,12 +11,18 @@ import ( keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" corev1beta1 "github.com/openstack-k8s-operators/openstack-operator/apis/core/v1beta1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" ctrl "sigs.k8s.io/controller-runtime" + + "github.com/openstack-k8s-operators/lib-common/modules/common/endpoint" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" ) // ReconcileKeystoneAPI - func ReconcileKeystoneAPI(ctx context.Context, instance *corev1beta1.OpenStackControlPlane, helper *helper.Helper) (ctrl.Result, error) { + keystoneAPI := &keystonev1.KeystoneAPI{ ObjectMeta: metav1.ObjectMeta{ Name: "keystone", //FIXME (keystone doesn't seem to work unless named "keystone") @@ -32,9 +38,85 @@ func ReconcileKeystoneAPI(ctx context.Context, instance *corev1beta1.OpenStackCo return ctrl.Result{}, nil } + spec := instance.Spec.Keystone.DeepCopy() + + // Create service overrides to pass into the service CR + // and expose the public endpoint using a route per default + var endpoints = map[service.Endpoint]endpoint.Data{ + service.EndpointPublic: {}, + service.EndpointInternal: {}, + } + serviceOverrides := []service.OverrideSpec{} + + for endpointType := range endpoints { + + sd := ServiceDetails{ + ServiceName: keystoneAPI.Name, + Namespace: instance.Namespace, + Endpoint: endpointType, + ExternalEndpoints: spec.ExternalEndpoints, + ServiceOverrideSpec: spec.Template.Override.Service, + RouteOverrideSpec: spec.Override.Route, + } + + svc, err := sd.CreateEndpointServiceOverride() + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + // TODO new ServiceOverrideCondition + condition.ExposeServiceReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ExposeServiceReadyErrorMessage, + err.Error())) + + return ctrl.Result{}, err + } + + // Create the route if it is public endpoint and the service type is ClusterIP + if svc != nil && sd.Endpoint == service.EndpointPublic && + svc.GetServiceType() == corev1.ServiceTypeClusterIP { + ctrlResult, err := sd.CreateRoute(ctx, helper, *svc) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ExposeServiceReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ExposeServiceReadyErrorMessage, + err.Error())) + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + // set the URL for the endpoint. Any trailing path needs to be added by the service operator + // TODO: TLS, pass in https as protocol, create TLS cert + endpointProto := endpoint.EndptProtocol(endpoint.PtrProtocol(endpoint.ProtocolHTTP)) + sd.endpointURL, err = svc.GetAPIEndpoint(nil, endpointProto, "") + if err != nil { + return ctrl.Result{}, err + } + } + + svcOverride := service.OverrideSpec{ + Endpoint: sd.Endpoint, + EmbeddedLabelsAnnotations: &service.EmbeddedLabelsAnnotations{ + Annotations: svc.GetAnnotations(), + Labels: svc.GetLabels(), + }, + Spec: svc.GetSpec(), + } + + if sd.GetEndpointURL() != "" { + svcOverride.EndpointURL = pointer.String(sd.GetEndpointURL()) + } + + serviceOverrides = append(serviceOverrides, svcOverride) + } + helper.GetLogger().Info("Reconciling KeystoneAPI", "KeystoneAPI.Namespace", instance.Namespace, "KeystoneAPI.Name", "keystone") op, err := controllerutil.CreateOrPatch(ctx, helper.GetClient(), keystoneAPI, func() error { instance.Spec.Keystone.Template.DeepCopyInto(&keystoneAPI.Spec) + keystoneAPI.Spec.Override.Service = serviceOverrides if keystoneAPI.Spec.Secret == "" { keystoneAPI.Spec.Secret = instance.Spec.Secret }