From 16d83ab042a3bee69db58ee2b9baa29d73758d36 Mon Sep 17 00:00:00 2001 From: Martin Schuppert Date: Fri, 4 Aug 2023 16:59:52 +0200 Subject: [PATCH] Update cinderapi to use service override Removes creation of routes.Those get done in the openstack-operator. Via service overrides the service can be customized. The service operator adds annotation to the service which needs to be exposed as a route. Jira: OSP-26690 Depends-On: openstack-k8s-operators/lib-common#332 --- README.md | 17 ++- .../cinder.openstack.org_cinderapis.yaml | 77 ++++++---- api/bases/cinder.openstack.org_cinders.yaml | 77 ++++++---- api/go.mod | 15 +- api/go.sum | 30 ++-- api/v1beta1/cinderapi_types.go | 12 +- api/v1beta1/common_types.go | 30 ---- api/v1beta1/zz_generated.deepcopy.go | 51 +++---- .../cinder.openstack.org_cinderapis.yaml | 77 ++++++---- .../bases/cinder.openstack.org_cinders.yaml | 77 ++++++---- config/rbac/role.yaml | 12 -- controllers/cinderapi_controller.go | 137 ++++++++++++------ go.mod | 22 +-- go.sum | 40 ++--- main.go | 2 - test/kuttl/tests/cinder_scale/05-errors.yaml | 1 - 16 files changed, 393 insertions(+), 284 deletions(-) diff --git a/README.md b/README.md index 516acacb..a61c3034 100644 --- a/README.md +++ b/README.md @@ -286,13 +286,16 @@ spec: ... cinderAPI: ... - externalEndpoints: - - endpoint: internal - ipAddressPool: osp-internalapi - loadBalancerIPs: - - 172.17.0.202 - sharedIP: true - sharedIPKey: "" + override: + service: + internal: + metadata: + annotations: + metallb.universe.tf/address-pool: osp-internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.202 + spec: + type: LoadBalancer ... ... ``` diff --git a/api/bases/cinder.openstack.org_cinderapis.yaml b/api/bases/cinder.openstack.org_cinderapis.yaml index 469b0f09..bc667953 100644 --- a/api/bases/cinder.openstack.org_cinderapis.yaml +++ b/api/bases/cinder.openstack.org_cinderapis.yaml @@ -63,32 +63,6 @@ spec: additionalProperties: type: string type: object - externalEndpoints: - items: - properties: - endpoint: - enum: - - internal - - public - type: string - ipAddressPool: - minLength: 1 - type: string - loadBalancerIPs: - items: - type: string - type: array - sharedIP: - default: true - type: boolean - sharedIPKey: - default: "" - type: string - required: - - endpoint - - ipAddressPool - type: object - type: array extraMounts: items: properties: @@ -853,6 +827,57 @@ spec: additionalProperties: type: string type: object + override: + properties: + service: + additionalProperties: + properties: + endpointURL: + type: string + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + externalName: + type: string + externalTrafficPolicy: + type: string + internalTrafficPolicy: + type: string + ipFamilyPolicy: + type: string + loadBalancerClass: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + sessionAffinity: + type: string + sessionAffinityConfig: + properties: + clientIP: + properties: + timeoutSeconds: + format: int32 + type: integer + type: object + type: object + type: + type: string + type: object + type: object + type: object + type: object passwordSelectors: default: database: CinderDatabasePassword diff --git a/api/bases/cinder.openstack.org_cinders.yaml b/api/bases/cinder.openstack.org_cinders.yaml index 3647b3e7..113920a1 100644 --- a/api/bases/cinder.openstack.org_cinders.yaml +++ b/api/bases/cinder.openstack.org_cinders.yaml @@ -56,32 +56,6 @@ spec: additionalProperties: type: string type: object - externalEndpoints: - items: - properties: - endpoint: - enum: - - internal - - public - type: string - ipAddressPool: - minLength: 1 - type: string - loadBalancerIPs: - items: - type: string - type: array - sharedIP: - default: true - type: boolean - sharedIPKey: - default: "" - type: string - required: - - endpoint - - ipAddressPool - type: object - type: array networkAttachments: items: type: string @@ -90,6 +64,57 @@ spec: additionalProperties: type: string type: object + override: + properties: + service: + additionalProperties: + properties: + endpointURL: + type: string + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + externalName: + type: string + externalTrafficPolicy: + type: string + internalTrafficPolicy: + type: string + ipFamilyPolicy: + type: string + loadBalancerClass: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + sessionAffinity: + type: string + sessionAffinityConfig: + properties: + clientIP: + properties: + timeoutSeconds: + format: int32 + type: integer + type: object + type: object + type: + type: string + type: object + type: object + type: object + type: object replicas: default: 1 format: int32 diff --git a/api/go.mod b/api/go.mod index 9e5c0aeb..94e65936 100644 --- a/api/go.mod +++ b/api/go.mod @@ -5,11 +5,11 @@ go 1.19 require ( github.com/onsi/ginkgo/v2 v2.12.0 github.com/onsi/gomega v1.27.10 - github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20230913075424-2680ce4b6ad2 - github.com/openstack-k8s-operators/lib-common/modules/storage v0.1.1-0.20230913075424-2680ce4b6ad2 - k8s.io/api v0.26.8 - k8s.io/apimachinery v0.26.8 - k8s.io/client-go v0.26.8 + github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20230919113507-d74c2f31d216 + github.com/openstack-k8s-operators/lib-common/modules/storage v0.1.1-0.20230919113507-d74c2f31d216 + k8s.io/api v0.26.9 + k8s.io/apimachinery v0.26.9 + k8s.io/client-go v0.26.9 sigs.k8s.io/controller-runtime v0.14.6 ) @@ -46,7 +46,6 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/openshift/api v3.9.0+incompatible // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.14.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect @@ -67,8 +66,8 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.26.8 //indirect - k8s.io/component-base v0.26.8 //indirect + k8s.io/apiextensions-apiserver v0.26.9 //indirect + k8s.io/component-base v0.26.9 //indirect k8s.io/klog/v2 v2.100.1 // indirect k8s.io/kube-openapi v0.0.0-20230327201221-f5883ff37f0c //indirect k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect diff --git a/api/go.sum b/api/go.sum index 37da30d7..e13033b6 100644 --- a/api/go.sum +++ b/api/go.sum @@ -228,12 +228,10 @@ github.com/onsi/ginkgo/v2 v2.12.0 h1:UIVDowFPwpg6yMUpPjGkYvf06K3RAiJXUhCxEwQVHRI github.com/onsi/ginkgo/v2 v2.12.0/go.mod h1:ZNEzXISYlqpb8S36iN71ifqLi3vVD1rVJGvWRCJOUpQ= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= -github.com/openshift/api v0.0.0-20230414143018-3367bc7e6ac7 h1:rncLxJBpFGqBztyxCMwNRnMjhhIDOWHJowi6q8G6koI= -github.com/openshift/api v0.0.0-20230414143018-3367bc7e6ac7/go.mod h1:ctXNyWanKEjGj8sss1KjjHQ3ENKFm33FFnS5BKaIPh4= -github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20230913075424-2680ce4b6ad2 h1:/ez+9PSwtucQ9v1I5X72xlP5UJztTMPH4M5gDAJAatc= -github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20230913075424-2680ce4b6ad2/go.mod h1:bG2JdbaO4bR4u8rtXZ7MgmMELuEseTkL2BPgk9JBYmY= -github.com/openstack-k8s-operators/lib-common/modules/storage v0.1.1-0.20230913075424-2680ce4b6ad2 h1:WqquubnK/hLqeZpTnNjjBCloo3HgvKhCrhEDNyQg21I= -github.com/openstack-k8s-operators/lib-common/modules/storage v0.1.1-0.20230913075424-2680ce4b6ad2/go.mod h1:UKzRiNlmYVIE7Ggc6+qd7MMrNBL20sgW46rhP+U0Ilc= +github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20230919113507-d74c2f31d216 h1:arYbQA6bLyXJkHm+6M6gPc4YpWMFjs5qkG16Yii4UDo= +github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20230919113507-d74c2f31d216/go.mod h1:Ge7Yf6AUmjEvJK9AIW2bT5udLzBIcK59b1QxqymncaQ= +github.com/openstack-k8s-operators/lib-common/modules/storage v0.1.1-0.20230919113507-d74c2f31d216 h1:1dMwzjuZJSRgHcwblOo6jHrWAe2anwWu4w35Rukz1kw= +github.com/openstack-k8s-operators/lib-common/modules/storage v0.1.1-0.20230919113507-d74c2f31d216/go.mod h1:DnsRBWOFEwecwrVAntM4lbgiHRvqfoz/gKYpDjkvCYE= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -633,16 +631,16 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.26.8 h1:k2OtFmQPWfDUyAuYAwQPftVygF/vz4BMGSKnd15iddM= -k8s.io/api v0.26.8/go.mod h1:QaflR7cmG3V9lIz0VLBM+ylndNN897OAUAoJDcgwiQw= -k8s.io/apiextensions-apiserver v0.26.8 h1:ESVQ22MH6YfcpflpZMIvkgnHs/EwOgKKSCkS9AfxJOY= -k8s.io/apiextensions-apiserver v0.26.8/go.mod h1:ySo6rPc9ulNtKoZczw7ljCAdZN3DbyxLNat8wuYk4r8= -k8s.io/apimachinery v0.26.8 h1:SzpGtRX3/j/Ylg8Eg65Iobpxi9Jz4vOvI0qcBZyPVrM= -k8s.io/apimachinery v0.26.8/go.mod h1:qYzLkrQ9lhrZRh0jNKo2cfvf/R1/kQONnSiyB7NUJU0= -k8s.io/client-go v0.26.8 h1:pPuTYaVtLlg/7n6rqs3MsKLi4XgNaJ3rTMyS37Y5CKU= -k8s.io/client-go v0.26.8/go.mod h1:1sBQqKmdy9rWZYQnoedpc0gnRXG7kU3HrKZvBe2QbGM= -k8s.io/component-base v0.26.8 h1:j+W9y9id4CLW85+5GhRMgcYLaezw6bK+ZQ2eN3uZtJc= -k8s.io/component-base v0.26.8/go.mod h1:tOQmHjTJBLjzWLWqbxz7sVgX9XMMphEcy0tWhk+u2BI= +k8s.io/api v0.26.9 h1:s8Y+G1u2JM55b90+Yo2RVb3PGT/hkWNVPN4idPERxJg= +k8s.io/api v0.26.9/go.mod h1:W/W4fEWRVzPD36820LlVUQfNBiSbiq0VPWRFJKwzmUg= +k8s.io/apiextensions-apiserver v0.26.9 h1:aJqWRuBj9i9J6tIDniqUDYM5QCRajTKXK/GO+zEccGQ= +k8s.io/apiextensions-apiserver v0.26.9/go.mod h1:L1uysxOP2kC1vkZTlHGUlUl5WSpa7e4GHJmGEZY7yLg= +k8s.io/apimachinery v0.26.9 h1:5yAV9cFR7Z4gIorKcAjWnx4uxtxiFsERwq4Pvmx0CCg= +k8s.io/apimachinery v0.26.9/go.mod h1:qYzLkrQ9lhrZRh0jNKo2cfvf/R1/kQONnSiyB7NUJU0= +k8s.io/client-go v0.26.9 h1:TGWi/6guEjIgT0Hg871Gsmx0qFuoGyGFjlFedrk7It0= +k8s.io/client-go v0.26.9/go.mod h1:tU1FZS0bwAmAFyPYpZycUQrQnUMzQ5MHloop7EbX6ow= +k8s.io/component-base v0.26.9 h1:qQVdQgyEIUe8EUkB3EEuQ9l5sgVlG2KgOB519yWEBGw= +k8s.io/component-base v0.26.9/go.mod h1:3WmW9lH9tbjpuvpAc22cPF/6C3VxCjMxkOU1j2mpzr8= k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20230327201221-f5883ff37f0c h1:EFfsozyzZ/pggw5qNx7ftTVZdp7WZl+3ih89GEjYEK8= diff --git a/api/v1beta1/cinderapi_types.go b/api/v1beta1/cinderapi_types.go index 7d78d631..e8199675 100644 --- a/api/v1beta1/cinderapi_types.go +++ b/api/v1beta1/cinderapi_types.go @@ -18,6 +18,7 @@ package v1beta1 import ( condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -33,8 +34,15 @@ type CinderAPITemplate struct { Replicas *int32 `json:"replicas"` // +kubebuilder:validation:Optional - // ExternalEndpoints, expose a VIP via MetalLB on the pre-created address pool - ExternalEndpoints []MetalLBConfig `json:"externalEndpoints,omitempty"` + // Override, provides the ability to override the generated manifest of several child resources. + Override APIOverrideSpec `json:"override,omitempty"` +} + +// APIOverrideSpec to override the generated manifest of several child resources. +type APIOverrideSpec struct { + // Override configuration for the Service created to serve traffic to the cluster. + // The key must be the endpoint type (public, internal) + Service map[service.Endpoint]service.RoutedOverrideSpec `json:"service,omitempty"` } // CinderAPISpec defines the desired state of CinderAPI diff --git a/api/v1beta1/common_types.go b/api/v1beta1/common_types.go index 05111706..14d76fe0 100644 --- a/api/v1beta1/common_types.go +++ b/api/v1beta1/common_types.go @@ -17,7 +17,6 @@ limitations under the License. package v1beta1 import ( - "github.com/openstack-k8s-operators/lib-common/modules/common/endpoint" corev1 "k8s.io/api/core/v1" ) @@ -119,32 +118,3 @@ type CinderServiceDebug struct { // service enable debug Service bool `json:"service"` } - -// MetalLBConfig to configure the MetalLB loadbalancer service -type MetalLBConfig struct { - // +kubebuilder:validation:Required - // +kubebuilder:validation:Enum=internal;public - // Endpoint, OpenStack endpoint this service maps to - Endpoint endpoint.Endpoint `json:"endpoint"` - - // +kubebuilder:validation:Required - // +kubebuilder:validation:MinLength=1 - // IPAddressPool expose VIP via MetalLB on the IPAddressPool - IPAddressPool string `json:"ipAddressPool"` - - // +kubebuilder:validation:Optional - // +kubebuilder:default=true - // SharedIP if true, VIP/VIPs get shared with multiple services - SharedIP bool `json:"sharedIP"` - - // +kubebuilder:validation:Optional - // +kubebuilder:default="" - // SharedIPKey specifies the sharing key which gets set as the annotation on the LoadBalancer service. - // Services which share the same VIP must have the same SharedIPKey. Defaults to the IPAddressPool if - // SharedIP is true, but no SharedIPKey specified. - SharedIPKey string `json:"sharedIPKey"` - - // +kubebuilder:validation:Optional - // LoadBalancerIPs, request given IPs from the pool if available. Using a list to allow dual stack (IPv4/IPv6) support - LoadBalancerIPs []string `json:"loadBalancerIPs,omitempty"` -} diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index b98eb476..aeeb981b 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -23,10 +23,33 @@ package v1beta1 import ( "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" "github.com/openstack-k8s-operators/lib-common/modules/storage" "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *APIOverrideSpec) DeepCopyInto(out *APIOverrideSpec) { + *out = *in + if in.Service != nil { + in, out := &in.Service, &out.Service + *out = make(map[service.Endpoint]service.RoutedOverrideSpec, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new APIOverrideSpec. +func (in *APIOverrideSpec) DeepCopy() *APIOverrideSpec { + if in == nil { + return nil + } + out := new(APIOverrideSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Cinder) DeepCopyInto(out *Cinder) { *out = *in @@ -214,13 +237,7 @@ func (in *CinderAPITemplate) DeepCopyInto(out *CinderAPITemplate) { *out = new(int32) **out = **in } - if in.ExternalEndpoints != nil { - in, out := &in.ExternalEndpoints, &out.ExternalEndpoints - *out = make([]MetalLBConfig, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } + in.Override.DeepCopyInto(&out.Override) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CinderAPITemplate. @@ -941,26 +958,6 @@ func (in *CinderVolumeTemplate) DeepCopy() *CinderVolumeTemplate { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MetalLBConfig) DeepCopyInto(out *MetalLBConfig) { - *out = *in - if in.LoadBalancerIPs != nil { - in, out := &in.LoadBalancerIPs, &out.LoadBalancerIPs - *out = make([]string, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetalLBConfig. -func (in *MetalLBConfig) DeepCopy() *MetalLBConfig { - if in == nil { - return nil - } - out := new(MetalLBConfig) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PasswordSelector) DeepCopyInto(out *PasswordSelector) { *out = *in diff --git a/config/crd/bases/cinder.openstack.org_cinderapis.yaml b/config/crd/bases/cinder.openstack.org_cinderapis.yaml index 469b0f09..bc667953 100644 --- a/config/crd/bases/cinder.openstack.org_cinderapis.yaml +++ b/config/crd/bases/cinder.openstack.org_cinderapis.yaml @@ -63,32 +63,6 @@ spec: additionalProperties: type: string type: object - externalEndpoints: - items: - properties: - endpoint: - enum: - - internal - - public - type: string - ipAddressPool: - minLength: 1 - type: string - loadBalancerIPs: - items: - type: string - type: array - sharedIP: - default: true - type: boolean - sharedIPKey: - default: "" - type: string - required: - - endpoint - - ipAddressPool - type: object - type: array extraMounts: items: properties: @@ -853,6 +827,57 @@ spec: additionalProperties: type: string type: object + override: + properties: + service: + additionalProperties: + properties: + endpointURL: + type: string + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + externalName: + type: string + externalTrafficPolicy: + type: string + internalTrafficPolicy: + type: string + ipFamilyPolicy: + type: string + loadBalancerClass: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + sessionAffinity: + type: string + sessionAffinityConfig: + properties: + clientIP: + properties: + timeoutSeconds: + format: int32 + type: integer + type: object + type: object + type: + type: string + type: object + type: object + type: object + type: object passwordSelectors: default: database: CinderDatabasePassword diff --git a/config/crd/bases/cinder.openstack.org_cinders.yaml b/config/crd/bases/cinder.openstack.org_cinders.yaml index 3647b3e7..113920a1 100644 --- a/config/crd/bases/cinder.openstack.org_cinders.yaml +++ b/config/crd/bases/cinder.openstack.org_cinders.yaml @@ -56,32 +56,6 @@ spec: additionalProperties: type: string type: object - externalEndpoints: - items: - properties: - endpoint: - enum: - - internal - - public - type: string - ipAddressPool: - minLength: 1 - type: string - loadBalancerIPs: - items: - type: string - type: array - sharedIP: - default: true - type: boolean - sharedIPKey: - default: "" - type: string - required: - - endpoint - - ipAddressPool - type: object - type: array networkAttachments: items: type: string @@ -90,6 +64,57 @@ spec: additionalProperties: type: string type: object + override: + properties: + service: + additionalProperties: + properties: + endpointURL: + type: string + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + externalName: + type: string + externalTrafficPolicy: + type: string + internalTrafficPolicy: + type: string + ipFamilyPolicy: + type: string + loadBalancerClass: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + sessionAffinity: + type: string + sessionAffinityConfig: + properties: + clientIP: + properties: + timeoutSeconds: + format: int32 + type: integer + type: object + type: object + type: + type: string + type: object + type: object + type: object + type: object replicas: default: 1 format: int32 diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 79bd14d7..84b24ee8 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -308,18 +308,6 @@ rules: - list - update - watch -- apiGroups: - - route.openshift.io - resources: - - routes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - apiGroups: - security.openshift.io resourceNames: diff --git a/controllers/cinderapi_controller.go b/controllers/cinderapi_controller.go index 915519a2..619200b2 100644 --- a/controllers/cinderapi_controller.go +++ b/controllers/cinderapi_controller.go @@ -21,7 +21,6 @@ import ( "fmt" "time" - routev1 "github.com/openshift/api/route/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" k8s_errors "k8s.io/apimachinery/pkg/api/errors" @@ -49,6 +48,7 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/labels" nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment" "github.com/openstack-k8s-operators/lib-common/modules/common/secret" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" "github.com/openstack-k8s-operators/lib-common/modules/common/util" ) @@ -96,7 +96,6 @@ var ( // +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;create;update;patch;delete;watch // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list; -// +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;create;update;patch;delete;watch // +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;create;update;patch;delete;watch // +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneservices,verbs=get;list;watch;create;update;patch;delete @@ -257,7 +256,6 @@ func (r *CinderAPIReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&keystonev1.KeystoneService{}). Owns(&keystonev1.KeystoneEndpoint{}). Owns(&appsv1.Deployment{}). - Owns(&routev1.Route{}). Owns(&corev1.Service{}). // watch the secrets we don't own Watches(&source.Kind{Type: &corev1.Secret{}}, @@ -318,7 +316,7 @@ func (r *CinderAPIReconciler) reconcileInit( r.Log.Info(fmt.Sprintf("Reconciling Service '%s' init", instance.Name)) // - // expose the service (create service, route and return the created endpoint URLs) + // expose the service (create service and return the created endpoint URLs) // // V3 @@ -330,62 +328,113 @@ func (r *CinderAPIReconciler) reconcileInit( Port: cinder.CinderInternalPort, Path: "/v3", } - data := map[endpoint.Endpoint]endpoint.Data{ - endpoint.EndpointPublic: publicEndpointData, - endpoint.EndpointInternal: internalEndpointData, + cinderEndpoints := map[service.Endpoint]endpoint.Data{ + service.EndpointPublic: publicEndpointData, + service.EndpointInternal: internalEndpointData, } - for _, metallbcfg := range instance.Spec.ExternalEndpoints { - portCfg := data[metallbcfg.Endpoint] + apiEndpointsV3 := make(map[string]string) - portCfg.MetalLB = &endpoint.MetalLBData{ - IPAddressPool: metallbcfg.IPAddressPool, - SharedIP: metallbcfg.SharedIP, - SharedIPKey: metallbcfg.SharedIPKey, - LoadBalancerIPs: metallbcfg.LoadBalancerIPs, + for endpointType, data := range cinderEndpoints { + endpointTypeStr := string(endpointType) + endpointName := cinder.ServiceName + "-" + endpointTypeStr + svcOverride := instance.Spec.Override.Service[endpointType] + if svcOverride.EmbeddedLabelsAnnotations == nil { + svcOverride.EmbeddedLabelsAnnotations = &service.EmbeddedLabelsAnnotations{} } - data[metallbcfg.Endpoint] = portCfg - } + exportLabels := util.MergeStringMaps( + serviceLabels, + map[string]string{ + service.AnnotationEndpointKey: endpointTypeStr, + }, + ) - apiEndpointsV3, ctrlResult, err := endpoint.ExposeEndpoints( - ctx, - helper, - cinder.ServiceName, - serviceLabels, - data, - time.Duration(5)*time.Second, - ) - if err != nil { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.ExposeServiceReadyCondition, - condition.ErrorReason, - condition.SeverityWarning, - condition.ExposeServiceReadyErrorMessage, - err.Error())) - return ctrlResult, err - } else if (ctrlResult != ctrl.Result{}) { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.ExposeServiceReadyCondition, - condition.RequestedReason, - condition.SeverityInfo, - condition.ExposeServiceReadyRunningMessage)) - return ctrlResult, nil + // Create the service + svc, err := service.NewService( + service.GenericService(&service.GenericServiceDetails{ + Name: endpointName, + Namespace: instance.Namespace, + Labels: exportLabels, + Selector: serviceLabels, + Port: service.GenericServicePort{ + Name: endpointName, + Port: data.Port, + Protocol: corev1.ProtocolTCP, + }, + }), + 5, + &svcOverride.OverrideSpec, + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ExposeServiceReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ExposeServiceReadyErrorMessage, + err.Error())) + + return ctrl.Result{}, err + } + + svc.AddAnnotation(map[string]string{ + service.AnnotationEndpointKey: endpointTypeStr, + }) + + // add Annotation to whether creating an ingress is required or not + if endpointType == service.EndpointPublic && svc.GetServiceType() == corev1.ServiceTypeClusterIP { + svc.AddAnnotation(map[string]string{ + service.AnnotationIngressCreateKey: "true", + }) + } else { + svc.AddAnnotation(map[string]string{ + service.AnnotationIngressCreateKey: "false", + }) + if svc.GetServiceType() == corev1.ServiceTypeLoadBalancer { + svc.AddAnnotation(map[string]string{ + service.AnnotationHostnameKey: svc.GetServiceHostname(), // add annotation to register service name in dnsmasq + }) + } + } + + ctrlResult, err := svc.CreateOrPatch(ctx, helper) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ExposeServiceReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ExposeServiceReadyErrorMessage, + err.Error())) + + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ExposeServiceReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.ExposeServiceReadyRunningMessage)) + return ctrlResult, nil + } + // create service - end + + // TODO: TLS, pass in https as protocol, create TLS cert + apiEndpointsV3[string(endpointType)], err = svc.GetAPIEndpoint( + svcOverride.EndpointURL, data.Protocol, data.Path) + if err != nil { + return ctrl.Result{}, err + } } + instance.Status.Conditions.MarkTrue(condition.ExposeServiceReadyCondition, condition.ExposeServiceReadyMessage) // // Update instance status with service endpoint url from route host information // - // TODO: need to support https default here if instance.Status.APIEndpoints == nil { instance.Status.APIEndpoints = map[string]map[string]string{} } - instance.Status.APIEndpoints[cinder.ServiceNameV3] = apiEndpointsV3 // V3 - end - instance.Status.Conditions.MarkTrue(condition.ExposeServiceReadyCondition, condition.ExposeServiceReadyMessage) - // expose service - end // @@ -408,7 +457,7 @@ func (r *CinderAPIReconciler) reconcileInit( } ksSvcObj := keystonev1.NewKeystoneService(ksSvcSpec, instance.Namespace, serviceLabels, time.Duration(10)*time.Second) - ctrlResult, err = ksSvcObj.CreateOrPatch(ctx, helper) + ctrlResult, err := ksSvcObj.CreateOrPatch(ctx, helper) if err != nil { return ctrlResult, err } diff --git a/go.mod b/go.mod index 8fdf45ed..07d8091d 100644 --- a/go.mod +++ b/go.mod @@ -7,17 +7,16 @@ require ( github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0 github.com/onsi/ginkgo/v2 v2.12.0 github.com/onsi/gomega v1.27.10 - github.com/openshift/api v3.9.0+incompatible github.com/openstack-k8s-operators/cinder-operator/api v0.1.2-0.20230904185016-c3f3b79f0e7f github.com/openstack-k8s-operators/infra-operator/apis v0.1.1-0.20230914145253-116f307c7875 - github.com/openstack-k8s-operators/keystone-operator/api v0.1.1-0.20230914163026-da9aa9de960a - github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20230913075424-2680ce4b6ad2 - github.com/openstack-k8s-operators/lib-common/modules/database v0.1.1-0.20230913075424-2680ce4b6ad2 - github.com/openstack-k8s-operators/lib-common/modules/storage v0.1.1-0.20230913075424-2680ce4b6ad2 + github.com/openstack-k8s-operators/keystone-operator/api v0.1.1-0.20230920085319-92ae0260bbf3 + github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20230919113507-d74c2f31d216 + github.com/openstack-k8s-operators/lib-common/modules/database v0.1.1-0.20230919113507-d74c2f31d216 + github.com/openstack-k8s-operators/lib-common/modules/storage v0.1.1-0.20230919113507-d74c2f31d216 github.com/openstack-k8s-operators/mariadb-operator/api v0.1.1-0.20230913081601-9e4fc8aadad5 - k8s.io/api v0.26.8 - k8s.io/apimachinery v0.26.8 - k8s.io/client-go v0.26.8 + k8s.io/api v0.26.9 + k8s.io/apimachinery v0.26.9 + k8s.io/client-go v0.26.9 sigs.k8s.io/controller-runtime v0.14.6 ) @@ -50,7 +49,8 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/openstack-k8s-operators/lib-common/modules/openstack v0.1.1-0.20230913075424-2680ce4b6ad2 //indirect + github.com/openshift/api v3.9.0+incompatible // indirect + github.com/openstack-k8s-operators/lib-common/modules/openstack v0.1.1-0.20230919113507-d74c2f31d216 //indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.14.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect @@ -72,8 +72,8 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.26.8 //indirect - k8s.io/component-base v0.26.8 //indirect + k8s.io/apiextensions-apiserver v0.26.9 //indirect + k8s.io/component-base v0.26.9 //indirect k8s.io/klog/v2 v2.100.1 // indirect k8s.io/kube-openapi v0.0.0-20230327201221-f5883ff37f0c //indirect k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect diff --git a/go.sum b/go.sum index 3d8c8327..e86e30f7 100644 --- a/go.sum +++ b/go.sum @@ -236,16 +236,16 @@ github.com/openshift/api v0.0.0-20230414143018-3367bc7e6ac7 h1:rncLxJBpFGqBztyxC github.com/openshift/api v0.0.0-20230414143018-3367bc7e6ac7/go.mod h1:ctXNyWanKEjGj8sss1KjjHQ3ENKFm33FFnS5BKaIPh4= github.com/openstack-k8s-operators/infra-operator/apis v0.1.1-0.20230914145253-116f307c7875 h1:DUlCjbi3XxH66oL97MFZF5wgL28HdU+r8TkBZVw7WIc= github.com/openstack-k8s-operators/infra-operator/apis v0.1.1-0.20230914145253-116f307c7875/go.mod h1:NgrvT3CKMu6fE8Nt1H79qHx11L3I7Bb2eItniM7c9ow= -github.com/openstack-k8s-operators/keystone-operator/api v0.1.1-0.20230914163026-da9aa9de960a h1:MFYwi2Xk9r3OMPToCSbvqYVNrm7P+aFzGDN0eVNpgu8= -github.com/openstack-k8s-operators/keystone-operator/api v0.1.1-0.20230914163026-da9aa9de960a/go.mod h1:nxrbUOIGMJ1h2pNlawhURdt/XJ95dW2wZGedmOVo2aw= -github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20230913075424-2680ce4b6ad2 h1:/ez+9PSwtucQ9v1I5X72xlP5UJztTMPH4M5gDAJAatc= -github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20230913075424-2680ce4b6ad2/go.mod h1:bG2JdbaO4bR4u8rtXZ7MgmMELuEseTkL2BPgk9JBYmY= -github.com/openstack-k8s-operators/lib-common/modules/database v0.1.1-0.20230913075424-2680ce4b6ad2 h1:6TgG90h8i5kxcZxPSNqi5lAx/CyggHhqOcCW0US8YSA= -github.com/openstack-k8s-operators/lib-common/modules/database v0.1.1-0.20230913075424-2680ce4b6ad2/go.mod h1:MwA2qrztcNGM7X82/IvG8Th3A0q/3GE5Evxji7clFLI= -github.com/openstack-k8s-operators/lib-common/modules/openstack v0.1.1-0.20230913075424-2680ce4b6ad2 h1:4L5DRfSnomBfyRwCfAzqQwk0+osnIbyJ1VvKBy+tzyY= -github.com/openstack-k8s-operators/lib-common/modules/openstack v0.1.1-0.20230913075424-2680ce4b6ad2/go.mod h1:NZ6weu5xOAkNPTqg1luC22DO7ZbyqiilRkvrFfhjFm0= -github.com/openstack-k8s-operators/lib-common/modules/storage v0.1.1-0.20230913075424-2680ce4b6ad2 h1:WqquubnK/hLqeZpTnNjjBCloo3HgvKhCrhEDNyQg21I= -github.com/openstack-k8s-operators/lib-common/modules/storage v0.1.1-0.20230913075424-2680ce4b6ad2/go.mod h1:UKzRiNlmYVIE7Ggc6+qd7MMrNBL20sgW46rhP+U0Ilc= +github.com/openstack-k8s-operators/keystone-operator/api v0.1.1-0.20230920085319-92ae0260bbf3 h1:6VCz/ZBTJEQJTx4+z8UxLv3WITa4Bgx5CSP237wJ5xM= +github.com/openstack-k8s-operators/keystone-operator/api v0.1.1-0.20230920085319-92ae0260bbf3/go.mod h1:ta6w/29i4WuWkQp6I4cOLwMGQ5/vJI0y8Em7u+M34jo= +github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20230919113507-d74c2f31d216 h1:arYbQA6bLyXJkHm+6M6gPc4YpWMFjs5qkG16Yii4UDo= +github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20230919113507-d74c2f31d216/go.mod h1:Ge7Yf6AUmjEvJK9AIW2bT5udLzBIcK59b1QxqymncaQ= +github.com/openstack-k8s-operators/lib-common/modules/database v0.1.1-0.20230919113507-d74c2f31d216 h1:PykqfrahtBUbhIuM0IDRyrtVCOzy7jdEZz3yVEnwpp0= +github.com/openstack-k8s-operators/lib-common/modules/database v0.1.1-0.20230919113507-d74c2f31d216/go.mod h1:gdmDHoWrDtq8gfJHl20rQwe9vfzJJkY+KnvM6aD9+08= +github.com/openstack-k8s-operators/lib-common/modules/openstack v0.1.1-0.20230919113507-d74c2f31d216 h1:h76faqi4WAXBs3D2B0GLUdlCjS0dh78wRL0d5RZUwhk= +github.com/openstack-k8s-operators/lib-common/modules/openstack v0.1.1-0.20230919113507-d74c2f31d216/go.mod h1:GHi64tgyC75/vuT8Crda0yN5iCIYiSyS4bpzYJjX7MA= +github.com/openstack-k8s-operators/lib-common/modules/storage v0.1.1-0.20230919113507-d74c2f31d216 h1:1dMwzjuZJSRgHcwblOo6jHrWAe2anwWu4w35Rukz1kw= +github.com/openstack-k8s-operators/lib-common/modules/storage v0.1.1-0.20230919113507-d74c2f31d216/go.mod h1:DnsRBWOFEwecwrVAntM4lbgiHRvqfoz/gKYpDjkvCYE= github.com/openstack-k8s-operators/mariadb-operator/api v0.1.1-0.20230913081601-9e4fc8aadad5 h1:dQcSQuXfgzgOhc4v+zD0jE6WWhn6FHr5nALOjJBPxyI= github.com/openstack-k8s-operators/mariadb-operator/api v0.1.1-0.20230913081601-9e4fc8aadad5/go.mod h1:mJyhm/YiQZaYhLvOuLng/ITpwx8HvsYVht+VotS1Ed8= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -649,16 +649,16 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.26.8 h1:k2OtFmQPWfDUyAuYAwQPftVygF/vz4BMGSKnd15iddM= -k8s.io/api v0.26.8/go.mod h1:QaflR7cmG3V9lIz0VLBM+ylndNN897OAUAoJDcgwiQw= -k8s.io/apiextensions-apiserver v0.26.8 h1:ESVQ22MH6YfcpflpZMIvkgnHs/EwOgKKSCkS9AfxJOY= -k8s.io/apiextensions-apiserver v0.26.8/go.mod h1:ySo6rPc9ulNtKoZczw7ljCAdZN3DbyxLNat8wuYk4r8= -k8s.io/apimachinery v0.26.8 h1:SzpGtRX3/j/Ylg8Eg65Iobpxi9Jz4vOvI0qcBZyPVrM= -k8s.io/apimachinery v0.26.8/go.mod h1:qYzLkrQ9lhrZRh0jNKo2cfvf/R1/kQONnSiyB7NUJU0= -k8s.io/client-go v0.26.8 h1:pPuTYaVtLlg/7n6rqs3MsKLi4XgNaJ3rTMyS37Y5CKU= -k8s.io/client-go v0.26.8/go.mod h1:1sBQqKmdy9rWZYQnoedpc0gnRXG7kU3HrKZvBe2QbGM= -k8s.io/component-base v0.26.8 h1:j+W9y9id4CLW85+5GhRMgcYLaezw6bK+ZQ2eN3uZtJc= -k8s.io/component-base v0.26.8/go.mod h1:tOQmHjTJBLjzWLWqbxz7sVgX9XMMphEcy0tWhk+u2BI= +k8s.io/api v0.26.9 h1:s8Y+G1u2JM55b90+Yo2RVb3PGT/hkWNVPN4idPERxJg= +k8s.io/api v0.26.9/go.mod h1:W/W4fEWRVzPD36820LlVUQfNBiSbiq0VPWRFJKwzmUg= +k8s.io/apiextensions-apiserver v0.26.9 h1:aJqWRuBj9i9J6tIDniqUDYM5QCRajTKXK/GO+zEccGQ= +k8s.io/apiextensions-apiserver v0.26.9/go.mod h1:L1uysxOP2kC1vkZTlHGUlUl5WSpa7e4GHJmGEZY7yLg= +k8s.io/apimachinery v0.26.9 h1:5yAV9cFR7Z4gIorKcAjWnx4uxtxiFsERwq4Pvmx0CCg= +k8s.io/apimachinery v0.26.9/go.mod h1:qYzLkrQ9lhrZRh0jNKo2cfvf/R1/kQONnSiyB7NUJU0= +k8s.io/client-go v0.26.9 h1:TGWi/6guEjIgT0Hg871Gsmx0qFuoGyGFjlFedrk7It0= +k8s.io/client-go v0.26.9/go.mod h1:tU1FZS0bwAmAFyPYpZycUQrQnUMzQ5MHloop7EbX6ow= +k8s.io/component-base v0.26.9 h1:qQVdQgyEIUe8EUkB3EEuQ9l5sgVlG2KgOB519yWEBGw= +k8s.io/component-base v0.26.9/go.mod h1:3WmW9lH9tbjpuvpAc22cPF/6C3VxCjMxkOU1j2mpzr8= k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20230327201221-f5883ff37f0c h1:EFfsozyzZ/pggw5qNx7ftTVZdp7WZl+3ih89GEjYEK8= diff --git a/main.go b/main.go index 43651483..94a6a42d 100644 --- a/main.go +++ b/main.go @@ -26,7 +26,6 @@ import ( "k8s.io/client-go/kubernetes" _ "k8s.io/client-go/plugin/pkg/client/auth" - routev1 "github.com/openshift/api/route/v1" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -57,7 +56,6 @@ func init() { utilruntime.Must(mariadbv1beta1.AddToScheme(scheme)) utilruntime.Must(keystonev1beta1.AddToScheme(scheme)) utilruntime.Must(rabbitmqv1.AddToScheme(scheme)) - utilruntime.Must(routev1.AddToScheme(scheme)) utilruntime.Must(networkv1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } diff --git a/test/kuttl/tests/cinder_scale/05-errors.yaml b/test/kuttl/tests/cinder_scale/05-errors.yaml index 745317f7..39a63483 100644 --- a/test/kuttl/tests/cinder_scale/05-errors.yaml +++ b/test/kuttl/tests/cinder_scale/05-errors.yaml @@ -5,7 +5,6 @@ # No Deployment for CinderAPI CR # No Pods in cinder Deployment # No Cinder Services -# No Cinder Routes # apiVersion: cinder.openstack.org/v1beta1 kind: Cinder