From 0c554ea3fd4a1b9e69ba778578518406703a2567 Mon Sep 17 00:00:00 2001 From: Martin Schuppert Date: Fri, 11 Aug 2023 09:48:58 +0200 Subject: [PATCH] Create heatapi and heatcfnapi route and svc endpoint overrides Creates the route for the heatapi and heatcfnapi, also allows to customize the route via override. Generats the service override for the env with what is configured in the externalEndpoints, or specified in the service template override. Depends-On: https://github.com/openstack-k8s-operators/lib-common/pull/313 Depends-On: https://github.com/openstack-k8s-operators/keystone-operator/pull/289 Depends-On: https://github.com/openstack-k8s-operators/heat-operator/pull/227 Jira: OSP-26690 --- ....openstack.org_openstackcontrolplanes.yaml | 202 ++++++++++++++++++ apis/core/v1beta1/conditions.go | 3 + .../v1beta1/openstackcontrolplane_types.go | 10 + apis/core/v1beta1/zz_generated.deepcopy.go | 2 + ....openstack.org_openstackcontrolplanes.yaml | 202 ++++++++++++++++++ ...controlplane_galera_network_isolation.yaml | 24 +++ ...ne_galera_network_isolation_3replicas.yaml | 24 +++ ...enstackcontrolplane_network_isolation.yaml | 24 +++ ...ckcontrolplane_network_isolation_ceph.yaml | 24 +++ pkg/openstack/heat.go | 94 ++++++++ 10 files changed, 609 insertions(+) diff --git a/apis/bases/core.openstack.org_openstackcontrolplanes.yaml b/apis/bases/core.openstack.org_openstackcontrolplanes.yaml index 3c652089a..5700592a7 100644 --- a/apis/bases/core.openstack.org_openstackcontrolplanes.yaml +++ b/apis/bases/core.openstack.org_openstackcontrolplanes.yaml @@ -3423,6 +3423,208 @@ spec: type: object heat: properties: + apiOverride: + properties: + route: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + alternateBackends: + items: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + maxItems: 3 + type: array + host: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + path: + pattern: ^/ + type: string + port: + properties: + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - targetPort + type: object + subdomain: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + tls: + properties: + caCertificate: + type: string + certificate: + type: string + destinationCACertificate: + type: string + insecureEdgeTerminationPolicy: + type: string + key: + type: string + termination: + enum: + - edge + - reencrypt + - passthrough + type: string + required: + - termination + type: object + to: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + wildcardPolicy: + enum: + - None + - Subdomain + - "" + type: string + type: object + type: object + type: object + cnfAPIOverride: + properties: + route: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + alternateBackends: + items: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + maxItems: 3 + type: array + host: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + path: + pattern: ^/ + type: string + port: + properties: + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - targetPort + type: object + subdomain: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + tls: + properties: + caCertificate: + type: string + certificate: + type: string + destinationCACertificate: + type: string + insecureEdgeTerminationPolicy: + type: string + key: + type: string + termination: + enum: + - edge + - reencrypt + - passthrough + type: string + required: + - termination + type: object + to: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + wildcardPolicy: + enum: + - None + - Subdomain + - "" + type: string + type: object + type: object + type: object enabled: default: false type: boolean diff --git a/apis/core/v1beta1/conditions.go b/apis/core/v1beta1/conditions.go index 4c68e4d77..512a4f82c 100644 --- a/apis/core/v1beta1/conditions.go +++ b/apis/core/v1beta1/conditions.go @@ -72,6 +72,9 @@ const ( // OpenStackControlPlaneHeatReadyCondition Status=True condition which indicates if Heat is configured and operational OpenStackControlPlaneHeatReadyCondition condition.Type = "OpenStackControlPlaneHeatReady" + // OpenStackControlPlaneExposeHeatReadyCondition Status=True condition which indicates if Heat is exposed via a route + OpenStackControlPlaneExposeHeatReadyCondition condition.Type = "OpenStackControlPlaneExposeHeatReady" + // OpenStackControlPlaneIronicReadyCondition Status=True condition which indicates if Ironic is configured and operational OpenStackControlPlaneIronicReadyCondition condition.Type = "OpenStackControlPlaneIronicReady" diff --git a/apis/core/v1beta1/openstackcontrolplane_types.go b/apis/core/v1beta1/openstackcontrolplane_types.go index c16bd5b73..0aa4fae6c 100644 --- a/apis/core/v1beta1/openstackcontrolplane_types.go +++ b/apis/core/v1beta1/openstackcontrolplane_types.go @@ -464,6 +464,16 @@ type HeatSection struct { //+operator-sdk:csv:customresourcedefinitions:type=spec // Template - Overrides to use when creating the Heat services Template heatv1.HeatSpec `json:"template,omitempty"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // APIOverride, provides the ability to override the generated manifest of several child resources. + APIOverride Override `json:"apiOverride,omitempty"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // CnfAPIOverride, provides the ability to override the generated manifest of several child resources. + CnfAPIOverride Override `json:"cnfAPIOverride,omitempty"` } // IronicSection defines the desired state of Ironic services diff --git a/apis/core/v1beta1/zz_generated.deepcopy.go b/apis/core/v1beta1/zz_generated.deepcopy.go index 4a5f1fcb4..de9d2a2d3 100644 --- a/apis/core/v1beta1/zz_generated.deepcopy.go +++ b/apis/core/v1beta1/zz_generated.deepcopy.go @@ -124,6 +124,8 @@ func (in *GlanceSection) DeepCopy() *GlanceSection { func (in *HeatSection) DeepCopyInto(out *HeatSection) { *out = *in in.Template.DeepCopyInto(&out.Template) + in.APIOverride.DeepCopyInto(&out.APIOverride) + in.CnfAPIOverride.DeepCopyInto(&out.CnfAPIOverride) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HeatSection. diff --git a/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml b/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml index 3c652089a..5700592a7 100644 --- a/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml +++ b/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml @@ -3423,6 +3423,208 @@ spec: type: object heat: properties: + apiOverride: + properties: + route: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + alternateBackends: + items: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + maxItems: 3 + type: array + host: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + path: + pattern: ^/ + type: string + port: + properties: + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - targetPort + type: object + subdomain: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + tls: + properties: + caCertificate: + type: string + certificate: + type: string + destinationCACertificate: + type: string + insecureEdgeTerminationPolicy: + type: string + key: + type: string + termination: + enum: + - edge + - reencrypt + - passthrough + type: string + required: + - termination + type: object + to: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + wildcardPolicy: + enum: + - None + - Subdomain + - "" + type: string + type: object + type: object + type: object + cnfAPIOverride: + properties: + route: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + alternateBackends: + items: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + maxItems: 3 + type: array + host: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + path: + pattern: ^/ + type: string + port: + properties: + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - targetPort + type: object + subdomain: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + tls: + properties: + caCertificate: + type: string + certificate: + type: string + destinationCACertificate: + type: string + insecureEdgeTerminationPolicy: + type: string + key: + type: string + termination: + enum: + - edge + - reencrypt + - passthrough + type: string + required: + - termination + type: object + to: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + wildcardPolicy: + enum: + - None + - Subdomain + - "" + type: string + type: object + type: object + type: object enabled: default: false type: boolean diff --git a/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml b/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml index bf7e982b9..960af247a 100644 --- a/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml +++ b/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml @@ -239,12 +239,36 @@ spec: ipAddressPool: internalapi sharedIP: false heat: + apiOverride: + route: {} + cnfAPIOverride: + route: {} enabled: false template: databaseInstance: openstack heatAPI: + override: + service: + internal: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.80 + spec: + type: LoadBalancer replicas: 1 heatEngine: + override: + service: + internal: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.80 + spec: + type: LoadBalancer replicas: 1 secret: osp-secret ironic: diff --git a/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation_3replicas.yaml b/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation_3replicas.yaml index 1ff0a3a7b..028c2dc31 100644 --- a/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation_3replicas.yaml +++ b/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation_3replicas.yaml @@ -239,12 +239,36 @@ spec: ipAddressPool: internalapi sharedIP: false heat: + apiOverride: + route: {} + cnfAPIOverride: + route: {} enabled: false template: databaseInstance: openstack heatAPI: + override: + service: + internal: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.80 + spec: + type: LoadBalancer replicas: 1 heatEngine: + override: + service: + internal: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.80 + spec: + type: LoadBalancer replicas: 1 secret: osp-secret ironic: diff --git a/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml b/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml index f371493c6..6c7d5d5bd 100644 --- a/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml +++ b/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml @@ -227,12 +227,36 @@ spec: ipAddressPool: internalapi sharedIP: false heat: + apiOverride: + route: {} + cnfAPIOverride: + route: {} enabled: false template: databaseInstance: openstack heatAPI: + override: + service: + internal: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.80 + spec: + type: LoadBalancer replicas: 1 heatEngine: + override: + service: + internal: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.80 + spec: + type: LoadBalancer replicas: 1 secret: osp-secret ironic: diff --git a/config/samples/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml b/config/samples/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml index 05b23d144..8f8ac8258 100644 --- a/config/samples/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml +++ b/config/samples/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml @@ -285,12 +285,36 @@ spec: ipAddressPool: internalapi sharedIP: false heat: + apiOverride: + route: {} + cnfAPIOverride: + route: {} enabled: false template: databaseInstance: openstack heatAPI: + override: + service: + internal: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.80 + spec: + type: LoadBalancer replicas: 1 heatEngine: + override: + service: + internal: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.80 + spec: + type: LoadBalancer replicas: 1 secret: osp-secret ironic: diff --git a/pkg/openstack/heat.go b/pkg/openstack/heat.go index ff3ef8ade..033699b2d 100644 --- a/pkg/openstack/heat.go +++ b/pkg/openstack/heat.go @@ -4,14 +4,19 @@ import ( "context" "fmt" + "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/reconcile" heatv1 "github.com/openstack-k8s-operators/heat-operator/api/v1beta1" corev1beta1 "github.com/openstack-k8s-operators/openstack-operator/apis/core/v1beta1" + k8s_errors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" ) @@ -29,11 +34,100 @@ func ReconcileHeat(ctx context.Context, instance *corev1beta1.OpenStackControlPl return res, err } instance.Status.Conditions.Remove(corev1beta1.OpenStackControlPlaneHeatReadyCondition) + instance.Status.Conditions.Remove(corev1beta1.OpenStackControlPlaneExposeHeatReadyCondition) return ctrl.Result{}, nil } + + // add selector to service overrides + for _, endpointType := range []service.Endpoint{service.EndpointPublic, service.EndpointInternal} { + if instance.Spec.Heat.Template.HeatAPI.Override.Service == nil { + instance.Spec.Heat.Template.HeatAPI.Override.Service = map[service.Endpoint]service.RoutedOverrideSpec{} + } + instance.Spec.Heat.Template.HeatAPI.Override.Service[endpointType] = + AddServiceComponentLabel( + instance.Spec.Heat.Template.HeatAPI.Override.Service[endpointType], + heat.Name+"-api") + + if instance.Spec.Heat.Template.HeatCfnAPI.Override.Service == nil { + instance.Spec.Heat.Template.HeatCfnAPI.Override.Service = map[service.Endpoint]service.RoutedOverrideSpec{} + } + instance.Spec.Heat.Template.HeatCfnAPI.Override.Service[endpointType] = + AddServiceComponentLabel( + instance.Spec.Heat.Template.HeatCfnAPI.Override.Service[endpointType], + heat.Name+"-cfn") + } + + // When component services got created check if there is the need to create a route + if err := helper.GetClient().Get(ctx, types.NamespacedName{Name: "heat", Namespace: instance.Namespace}, heat); err != nil { + if !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + } + + // Heat API + if heat.Status.Conditions.IsTrue(heatv1.HeatAPIReadyCondition) { + svcs, err := service.GetServicesListWithLabel( + ctx, + helper, + instance.Namespace, + map[string]string{common.AppSelector: heat.Name + "-api"}, + ) + if err != nil { + return ctrl.Result{}, err + } + + var ctrlResult reconcile.Result + instance.Spec.Heat.Template.HeatAPI.Override.Service, ctrlResult, err = EnsureRoute( + ctx, + instance, + helper, + heat, + svcs, + instance.Spec.Heat.Template.HeatAPI.Override.Service, + instance.Spec.Heat.APIOverride.Route, + corev1beta1.OpenStackControlPlaneExposeHeatReadyCondition, + ) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + } + + // Heat CFNAPI + if heat.Status.Conditions.IsTrue(heatv1.HeatCfnAPIReadyCondition) { + svcs, err := service.GetServicesListWithLabel( + ctx, + helper, + instance.Namespace, + map[string]string{common.AppSelector: heat.Name + "-cfn"}, + ) + if err != nil { + return ctrl.Result{}, err + } + + var ctrlResult reconcile.Result + instance.Spec.Heat.Template.HeatCfnAPI.Override.Service, ctrlResult, err = EnsureRoute( + ctx, + instance, + helper, + heat, + svcs, + instance.Spec.Heat.Template.HeatCfnAPI.Override.Service, + instance.Spec.Heat.CnfAPIOverride.Route, + corev1beta1.OpenStackControlPlaneExposeHeatReadyCondition, + ) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + } + helper.GetLogger().Info("Reconcile heat", "heat.Namespace", instance.Namespace, "heat.Name", "heat") op, err := controllerutil.CreateOrPatch(ctx, helper.GetClient(), heat, func() error { instance.Spec.Heat.Template.DeepCopyInto(&heat.Spec) + err := controllerutil.SetControllerReference(helper.GetBeforeObject(), heat, helper.GetScheme()) if err != nil { return err