From 1d3564cb6d167030f68f8a4552d5e697dbb52e95 Mon Sep 17 00:00:00 2001 From: Prajyot-Parab Date: Mon, 29 Aug 2022 16:47:54 +0530 Subject: [PATCH] LoadBalancer integration for VPC Signed-off-by: Prajyot-Parab --- api/v1alpha3/conversion.go | 18 +++ api/v1alpha3/zz_generated.conversion.go | 49 +++---- api/v1alpha4/ibmvpc_conversion.go | 19 +++ api/v1alpha4/zz_generated.conversion.go | 49 +++---- api/v1beta1/conditions_consts.go | 10 ++ api/v1beta1/ibmvpccluster_types.go | 35 ++++- api/v1beta1/ibmvpcmachine_types.go | 2 + api/v1beta1/types.go | 19 ++- api/v1beta1/zz_generated.deepcopy.go | 34 ++++- cloud/scope/cluster.go | 132 ++++++++++++++++++ cloud/scope/machine.go | 101 ++++++++++++++ ...cture.cluster.x-k8s.io_ibmvpcclusters.yaml | 67 ++++++++- ...cture.cluster.x-k8s.io_ibmvpcmachines.yaml | 3 +- controllers/ibmvpccluster_controller.go | 75 +++++++++- controllers/ibmvpcmachine_controller.go | 46 ++++-- pkg/cloud/services/vpc/mock/vpc_generated.go | 110 +++++++++++++++ pkg/cloud/services/vpc/service.go | 37 ++++- pkg/cloud/services/vpc/vpc.go | 7 + 18 files changed, 726 insertions(+), 87 deletions(-) diff --git a/api/v1alpha3/conversion.go b/api/v1alpha3/conversion.go index d985008b1..27d3d36c8 100644 --- a/api/v1alpha3/conversion.go +++ b/api/v1alpha3/conversion.go @@ -104,3 +104,21 @@ func (dst *IBMVPCMachineTemplateList) ConvertFrom(srcRaw conversion.Hub) error { return Convert_v1beta1_IBMVPCMachineTemplateList_To_v1alpha3_IBMVPCMachineTemplateList(src, dst, nil) } + +// Convert_v1beta1_IBMVPCClusterSpec_To_v1alpha3_IBMVPCClusterSpec is an autogenerated conversion function. +// Requires manual conversion as ControlPlaneLoadBalancer does not exist in v1alpha4 version of IBMVPCClusterSpec. +func Convert_v1beta1_IBMVPCClusterSpec_To_v1alpha3_IBMVPCClusterSpec(in *infrav1beta1.IBMVPCClusterSpec, out *IBMVPCClusterSpec, s apiconversion.Scope) error { + return autoConvert_v1beta1_IBMVPCClusterSpec_To_v1alpha3_IBMVPCClusterSpec(in, out, s) +} + +// Convert_v1beta1_IBMVPCClusterStatus_To_v1alpha3_IBMVPCClusterStatus is an autogenerated conversion function. +// Requires manual conversion as ControlPlaneLoadBalancerState and Conditions does not exist in v1alpha4 version of IBMVPCClusterStatus. +func Convert_v1beta1_IBMVPCClusterStatus_To_v1alpha3_IBMVPCClusterStatus(in *infrav1beta1.IBMVPCClusterStatus, out *IBMVPCClusterStatus, s apiconversion.Scope) error { + return autoConvert_v1beta1_IBMVPCClusterStatus_To_v1alpha3_IBMVPCClusterStatus(in, out, s) +} + +// Convert_v1beta1_VPCEndpoint_To_v1alpha3_VPCEndpoint is an autogenerated conversion function. +// Requires manual conversion as LBID does not exist in v1alpha4 version of VPCEndpoint. +func Convert_v1beta1_VPCEndpoint_To_v1alpha3_VPCEndpoint(in *infrav1beta1.VPCEndpoint, out *VPCEndpoint, s apiconversion.Scope) error { + return autoConvert_v1beta1_VPCEndpoint_To_v1alpha3_VPCEndpoint(in, out, s) +} diff --git a/api/v1alpha3/zz_generated.conversion.go b/api/v1alpha3/zz_generated.conversion.go index 3ed2620fd..7f35b5fd2 100644 --- a/api/v1alpha3/zz_generated.conversion.go +++ b/api/v1alpha3/zz_generated.conversion.go @@ -64,21 +64,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1beta1.IBMVPCClusterSpec)(nil), (*IBMVPCClusterSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_IBMVPCClusterSpec_To_v1alpha3_IBMVPCClusterSpec(a.(*v1beta1.IBMVPCClusterSpec), b.(*IBMVPCClusterSpec), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*IBMVPCClusterStatus)(nil), (*v1beta1.IBMVPCClusterStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha3_IBMVPCClusterStatus_To_v1beta1_IBMVPCClusterStatus(a.(*IBMVPCClusterStatus), b.(*v1beta1.IBMVPCClusterStatus), scope) }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1beta1.IBMVPCClusterStatus)(nil), (*IBMVPCClusterStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_IBMVPCClusterStatus_To_v1alpha3_IBMVPCClusterStatus(a.(*v1beta1.IBMVPCClusterStatus), b.(*IBMVPCClusterStatus), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*IBMVPCMachine)(nil), (*v1beta1.IBMVPCMachine)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha3_IBMVPCMachine_To_v1beta1_IBMVPCMachine(a.(*IBMVPCMachine), b.(*v1beta1.IBMVPCMachine), scope) }); err != nil { @@ -194,11 +184,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1beta1.VPCEndpoint)(nil), (*VPCEndpoint)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_VPCEndpoint_To_v1alpha3_VPCEndpoint(a.(*v1beta1.VPCEndpoint), b.(*VPCEndpoint), scope) - }); err != nil { - return err - } if err := s.AddConversionFunc((*apiv1alpha3.APIEndpoint)(nil), (*apiv1beta1.APIEndpoint)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha3_APIEndpoint_To_v1beta1_APIEndpoint(a.(*apiv1alpha3.APIEndpoint), b.(*apiv1beta1.APIEndpoint), scope) }); err != nil { @@ -209,6 +194,21 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1beta1.IBMVPCClusterSpec)(nil), (*IBMVPCClusterSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_IBMVPCClusterSpec_To_v1alpha3_IBMVPCClusterSpec(a.(*v1beta1.IBMVPCClusterSpec), b.(*IBMVPCClusterSpec), scope) + }); err != nil { + return err + } + if err := s.AddConversionFunc((*v1beta1.IBMVPCClusterStatus)(nil), (*IBMVPCClusterStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_IBMVPCClusterStatus_To_v1alpha3_IBMVPCClusterStatus(a.(*v1beta1.IBMVPCClusterStatus), b.(*IBMVPCClusterStatus), scope) + }); err != nil { + return err + } + if err := s.AddConversionFunc((*v1beta1.VPCEndpoint)(nil), (*VPCEndpoint)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_VPCEndpoint_To_v1alpha3_VPCEndpoint(a.(*v1beta1.VPCEndpoint), b.(*VPCEndpoint), scope) + }); err != nil { + return err + } return nil } @@ -310,14 +310,10 @@ func autoConvert_v1beta1_IBMVPCClusterSpec_To_v1alpha3_IBMVPCClusterSpec(in *v1b if err := Convert_v1beta1_APIEndpoint_To_v1alpha3_APIEndpoint(&in.ControlPlaneEndpoint, &out.ControlPlaneEndpoint, s); err != nil { return err } + // WARNING: in.ControlPlaneLoadBalancer requires manual conversion: does not exist in peer-type return nil } -// Convert_v1beta1_IBMVPCClusterSpec_To_v1alpha3_IBMVPCClusterSpec is an autogenerated conversion function. -func Convert_v1beta1_IBMVPCClusterSpec_To_v1alpha3_IBMVPCClusterSpec(in *v1beta1.IBMVPCClusterSpec, out *IBMVPCClusterSpec, s conversion.Scope) error { - return autoConvert_v1beta1_IBMVPCClusterSpec_To_v1alpha3_IBMVPCClusterSpec(in, out, s) -} - func autoConvert_v1alpha3_IBMVPCClusterStatus_To_v1beta1_IBMVPCClusterStatus(in *IBMVPCClusterStatus, out *v1beta1.IBMVPCClusterStatus, s conversion.Scope) error { if err := Convert_v1alpha3_VPC_To_v1beta1_VPC(&in.VPC, &out.VPC, s); err != nil { return err @@ -348,14 +344,11 @@ func autoConvert_v1beta1_IBMVPCClusterStatus_To_v1alpha3_IBMVPCClusterStatus(in if err := Convert_v1beta1_VPCEndpoint_To_v1alpha3_VPCEndpoint(&in.VPCEndpoint, &out.VPCEndpoint, s); err != nil { return err } + // WARNING: in.ControlPlaneLoadBalancerState requires manual conversion: does not exist in peer-type + // WARNING: in.Conditions requires manual conversion: does not exist in peer-type return nil } -// Convert_v1beta1_IBMVPCClusterStatus_To_v1alpha3_IBMVPCClusterStatus is an autogenerated conversion function. -func Convert_v1beta1_IBMVPCClusterStatus_To_v1alpha3_IBMVPCClusterStatus(in *v1beta1.IBMVPCClusterStatus, out *IBMVPCClusterStatus, s conversion.Scope) error { - return autoConvert_v1beta1_IBMVPCClusterStatus_To_v1alpha3_IBMVPCClusterStatus(in, out, s) -} - func autoConvert_v1alpha3_IBMVPCMachine_To_v1beta1_IBMVPCMachine(in *IBMVPCMachine, out *v1beta1.IBMVPCMachine, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta if err := Convert_v1alpha3_IBMVPCMachineSpec_To_v1beta1_IBMVPCMachineSpec(&in.Spec, &out.Spec, s); err != nil { @@ -650,10 +643,6 @@ func Convert_v1alpha3_VPCEndpoint_To_v1beta1_VPCEndpoint(in *VPCEndpoint, out *v func autoConvert_v1beta1_VPCEndpoint_To_v1alpha3_VPCEndpoint(in *v1beta1.VPCEndpoint, out *VPCEndpoint, s conversion.Scope) error { out.Address = (*string)(unsafe.Pointer(in.Address)) out.FIPID = (*string)(unsafe.Pointer(in.FIPID)) + // WARNING: in.LBID requires manual conversion: does not exist in peer-type return nil } - -// Convert_v1beta1_VPCEndpoint_To_v1alpha3_VPCEndpoint is an autogenerated conversion function. -func Convert_v1beta1_VPCEndpoint_To_v1alpha3_VPCEndpoint(in *v1beta1.VPCEndpoint, out *VPCEndpoint, s conversion.Scope) error { - return autoConvert_v1beta1_VPCEndpoint_To_v1alpha3_VPCEndpoint(in, out, s) -} diff --git a/api/v1alpha4/ibmvpc_conversion.go b/api/v1alpha4/ibmvpc_conversion.go index 80ba4327e..c3a6d3adf 100644 --- a/api/v1alpha4/ibmvpc_conversion.go +++ b/api/v1alpha4/ibmvpc_conversion.go @@ -17,6 +17,7 @@ limitations under the License. package v1alpha4 import ( + apiconversion "k8s.io/apimachinery/pkg/conversion" "sigs.k8s.io/controller-runtime/pkg/conversion" infrav1beta1 "sigs.k8s.io/cluster-api-provider-ibmcloud/api/v1beta1" @@ -93,3 +94,21 @@ func (dst *IBMVPCMachineTemplateList) ConvertFrom(srcRaw conversion.Hub) error { return Convert_v1beta1_IBMVPCMachineTemplateList_To_v1alpha4_IBMVPCMachineTemplateList(src, dst, nil) } + +// Convert_v1beta1_IBMVPCClusterSpec_To_v1alpha4_IBMVPCClusterSpec is an autogenerated conversion function. +// Requires manual conversion as ControlPlaneLoadBalancer does not exist in v1alpha4 version of IBMVPCClusterSpec. +func Convert_v1beta1_IBMVPCClusterSpec_To_v1alpha4_IBMVPCClusterSpec(in *infrav1beta1.IBMVPCClusterSpec, out *IBMVPCClusterSpec, s apiconversion.Scope) error { + return autoConvert_v1beta1_IBMVPCClusterSpec_To_v1alpha4_IBMVPCClusterSpec(in, out, s) +} + +// Convert_v1beta1_IBMVPCClusterStatus_To_v1alpha4_IBMVPCClusterStatus is an autogenerated conversion function. +// Requires manual conversion as ControlPlaneLoadBalancerState and Conditions does not exist in v1alpha4 version of IBMVPCClusterStatus. +func Convert_v1beta1_IBMVPCClusterStatus_To_v1alpha4_IBMVPCClusterStatus(in *infrav1beta1.IBMVPCClusterStatus, out *IBMVPCClusterStatus, s apiconversion.Scope) error { + return autoConvert_v1beta1_IBMVPCClusterStatus_To_v1alpha4_IBMVPCClusterStatus(in, out, s) +} + +// Convert_v1beta1_VPCEndpoint_To_v1alpha4_VPCEndpoint is an autogenerated conversion function. +// Requires manual conversion as LBID does not exist in v1alpha4 version of VPCEndpoint. +func Convert_v1beta1_VPCEndpoint_To_v1alpha4_VPCEndpoint(in *infrav1beta1.VPCEndpoint, out *VPCEndpoint, s apiconversion.Scope) error { + return autoConvert_v1beta1_VPCEndpoint_To_v1alpha4_VPCEndpoint(in, out, s) +} diff --git a/api/v1alpha4/zz_generated.conversion.go b/api/v1alpha4/zz_generated.conversion.go index 3f6543859..104307b46 100644 --- a/api/v1alpha4/zz_generated.conversion.go +++ b/api/v1alpha4/zz_generated.conversion.go @@ -189,21 +189,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1beta1.IBMVPCClusterSpec)(nil), (*IBMVPCClusterSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_IBMVPCClusterSpec_To_v1alpha4_IBMVPCClusterSpec(a.(*v1beta1.IBMVPCClusterSpec), b.(*IBMVPCClusterSpec), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*IBMVPCClusterStatus)(nil), (*v1beta1.IBMVPCClusterStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha4_IBMVPCClusterStatus_To_v1beta1_IBMVPCClusterStatus(a.(*IBMVPCClusterStatus), b.(*v1beta1.IBMVPCClusterStatus), scope) }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1beta1.IBMVPCClusterStatus)(nil), (*IBMVPCClusterStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_IBMVPCClusterStatus_To_v1alpha4_IBMVPCClusterStatus(a.(*v1beta1.IBMVPCClusterStatus), b.(*IBMVPCClusterStatus), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*IBMVPCMachine)(nil), (*v1beta1.IBMVPCMachine)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha4_IBMVPCMachine_To_v1beta1_IBMVPCMachine(a.(*IBMVPCMachine), b.(*v1beta1.IBMVPCMachine), scope) }); err != nil { @@ -319,11 +309,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1beta1.VPCEndpoint)(nil), (*VPCEndpoint)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_VPCEndpoint_To_v1alpha4_VPCEndpoint(a.(*v1beta1.VPCEndpoint), b.(*VPCEndpoint), scope) - }); err != nil { - return err - } if err := s.AddConversionFunc((*apiv1alpha4.APIEndpoint)(nil), (*apiv1beta1.APIEndpoint)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha4_APIEndpoint_To_v1beta1_APIEndpoint(a.(*apiv1alpha4.APIEndpoint), b.(*apiv1beta1.APIEndpoint), scope) }); err != nil { @@ -349,6 +334,21 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1beta1.IBMVPCClusterSpec)(nil), (*IBMVPCClusterSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_IBMVPCClusterSpec_To_v1alpha4_IBMVPCClusterSpec(a.(*v1beta1.IBMVPCClusterSpec), b.(*IBMVPCClusterSpec), scope) + }); err != nil { + return err + } + if err := s.AddConversionFunc((*v1beta1.IBMVPCClusterStatus)(nil), (*IBMVPCClusterStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_IBMVPCClusterStatus_To_v1alpha4_IBMVPCClusterStatus(a.(*v1beta1.IBMVPCClusterStatus), b.(*IBMVPCClusterStatus), scope) + }); err != nil { + return err + } + if err := s.AddConversionFunc((*v1beta1.VPCEndpoint)(nil), (*VPCEndpoint)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_VPCEndpoint_To_v1alpha4_VPCEndpoint(a.(*v1beta1.VPCEndpoint), b.(*VPCEndpoint), scope) + }); err != nil { + return err + } return nil } @@ -873,14 +873,10 @@ func autoConvert_v1beta1_IBMVPCClusterSpec_To_v1alpha4_IBMVPCClusterSpec(in *v1b if err := Convert_v1beta1_APIEndpoint_To_v1alpha4_APIEndpoint(&in.ControlPlaneEndpoint, &out.ControlPlaneEndpoint, s); err != nil { return err } + // WARNING: in.ControlPlaneLoadBalancer requires manual conversion: does not exist in peer-type return nil } -// Convert_v1beta1_IBMVPCClusterSpec_To_v1alpha4_IBMVPCClusterSpec is an autogenerated conversion function. -func Convert_v1beta1_IBMVPCClusterSpec_To_v1alpha4_IBMVPCClusterSpec(in *v1beta1.IBMVPCClusterSpec, out *IBMVPCClusterSpec, s conversion.Scope) error { - return autoConvert_v1beta1_IBMVPCClusterSpec_To_v1alpha4_IBMVPCClusterSpec(in, out, s) -} - func autoConvert_v1alpha4_IBMVPCClusterStatus_To_v1beta1_IBMVPCClusterStatus(in *IBMVPCClusterStatus, out *v1beta1.IBMVPCClusterStatus, s conversion.Scope) error { if err := Convert_v1alpha4_VPC_To_v1beta1_VPC(&in.VPC, &out.VPC, s); err != nil { return err @@ -911,14 +907,11 @@ func autoConvert_v1beta1_IBMVPCClusterStatus_To_v1alpha4_IBMVPCClusterStatus(in if err := Convert_v1beta1_VPCEndpoint_To_v1alpha4_VPCEndpoint(&in.VPCEndpoint, &out.VPCEndpoint, s); err != nil { return err } + // WARNING: in.ControlPlaneLoadBalancerState requires manual conversion: does not exist in peer-type + // WARNING: in.Conditions requires manual conversion: does not exist in peer-type return nil } -// Convert_v1beta1_IBMVPCClusterStatus_To_v1alpha4_IBMVPCClusterStatus is an autogenerated conversion function. -func Convert_v1beta1_IBMVPCClusterStatus_To_v1alpha4_IBMVPCClusterStatus(in *v1beta1.IBMVPCClusterStatus, out *IBMVPCClusterStatus, s conversion.Scope) error { - return autoConvert_v1beta1_IBMVPCClusterStatus_To_v1alpha4_IBMVPCClusterStatus(in, out, s) -} - func autoConvert_v1alpha4_IBMVPCMachine_To_v1beta1_IBMVPCMachine(in *IBMVPCMachine, out *v1beta1.IBMVPCMachine, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta if err := Convert_v1alpha4_IBMVPCMachineSpec_To_v1beta1_IBMVPCMachineSpec(&in.Spec, &out.Spec, s); err != nil { @@ -1213,10 +1206,6 @@ func Convert_v1alpha4_VPCEndpoint_To_v1beta1_VPCEndpoint(in *VPCEndpoint, out *v func autoConvert_v1beta1_VPCEndpoint_To_v1alpha4_VPCEndpoint(in *v1beta1.VPCEndpoint, out *VPCEndpoint, s conversion.Scope) error { out.Address = (*string)(unsafe.Pointer(in.Address)) out.FIPID = (*string)(unsafe.Pointer(in.FIPID)) + // WARNING: in.LBID requires manual conversion: does not exist in peer-type return nil } - -// Convert_v1beta1_VPCEndpoint_To_v1alpha4_VPCEndpoint is an autogenerated conversion function. -func Convert_v1beta1_VPCEndpoint_To_v1alpha4_VPCEndpoint(in *v1beta1.VPCEndpoint, out *VPCEndpoint, s conversion.Scope) error { - return autoConvert_v1beta1_VPCEndpoint_To_v1alpha4_VPCEndpoint(in, out, s) -} diff --git a/api/v1beta1/conditions_consts.go b/api/v1beta1/conditions_consts.go index 41ff9fcf0..cdc8fc595 100644 --- a/api/v1beta1/conditions_consts.go +++ b/api/v1beta1/conditions_consts.go @@ -65,3 +65,13 @@ const ( // ImageImportedCondition reports on current status of the image import job. Ready indicates the import job is finished. ImageImportedCondition capiv1beta1.ConditionType = "ImageImported" ) + +const ( + // LoadBalancerNotReadyReason used when cluster is waiting for load balancer to be ready before proceeding. + LoadBalancerNotReadyReason = "LoadBalancerNotReady" +) + +const ( + // LoadBalancerReadyCondition reports on current status of the load balancer. Ready indicates the load balancer is in a active state. + LoadBalancerReadyCondition capiv1beta1.ConditionType = "LoadBalancerReady" +) diff --git a/api/v1beta1/ibmvpccluster_types.go b/api/v1beta1/ibmvpccluster_types.go index 835eac96e..b3e01b5ff 100644 --- a/api/v1beta1/ibmvpccluster_types.go +++ b/api/v1beta1/ibmvpccluster_types.go @@ -49,6 +49,19 @@ type IBMVPCClusterSpec struct { // ControlPlaneEndpoint represents the endpoint used to communicate with the control plane. // +optional ControlPlaneEndpoint capiv1beta1.APIEndpoint `json:"controlPlaneEndpoint"` + + // ControlPlaneLoadBalancer is optional configuration for customizing control plane behavior. + // +optional + ControlPlaneLoadBalancer *VPCLoadBalancerSpec `json:"controlPlaneLoadBalancer,omitempty"` +} + +// VPCLoadBalancerSpec defines the desired state of an VPC load balancer. +type VPCLoadBalancerSpec struct { + // Name sets the name of the VPC load balancer. + // +kubebuilder:validation:MaxLength:=64 + // +kubebuilder:validation:Pattern=`^[A-Za-z0-9]([A-Za-z0-9]{0,31}|[-A-Za-z0-9]{0,30}[A-Za-z0-9])$` + // +optional + Name string `json:"name,omitempty"` } // IBMVPCClusterStatus defines the observed state of IBMVPCCluster. @@ -56,10 +69,20 @@ type IBMVPCClusterStatus struct { // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster // Important: Run "make" to regenerate code after modifying this file VPC VPC `json:"vpc,omitempty"` - // Bastion Instance `json:"bastion,omitempty"` + + // Ready is true when the provider resource is ready. + // +optional Ready bool `json:"ready"` Subnet Subnet `json:"subnet,omitempty"` VPCEndpoint VPCEndpoint `json:"vpcEndpoint,omitempty"` + + // ControlPlaneLoadBalancerState is the status of the load balancer. + // +optional + ControlPlaneLoadBalancerState VPCLoadBalancerState `json:"controlPlaneLoadBalancerState,omitempty"` + + // Conditions defines current service state of the load balancer. + // +optional + Conditions capiv1beta1.Conditions `json:"conditions,omitempty"` } // VPC holds the VPC information. @@ -96,3 +119,13 @@ type IBMVPCClusterList struct { func init() { SchemeBuilder.Register(&IBMVPCCluster{}, &IBMVPCClusterList{}) } + +// GetConditions returns the observations of the operational state of the IBMVPCCluster resource. +func (r *IBMVPCCluster) GetConditions() capiv1beta1.Conditions { + return r.Status.Conditions +} + +// SetConditions sets the underlying service state of the IBMVPCCluster to the predescribed clusterv1.Conditions. +func (r *IBMVPCCluster) SetConditions(conditions capiv1beta1.Conditions) { + r.Status.Conditions = conditions +} diff --git a/api/v1beta1/ibmvpcmachine_types.go b/api/v1beta1/ibmvpcmachine_types.go index 2fa6c3b80..dd0e58904 100644 --- a/api/v1beta1/ibmvpcmachine_types.go +++ b/api/v1beta1/ibmvpcmachine_types.go @@ -68,6 +68,8 @@ type IBMVPCMachineStatus struct { InstanceID string `json:"instanceID,omitempty"` + // Ready is true when the provider resource is ready. + // +optional Ready bool `json:"ready"` // Addresses contains the GCP instance associated addresses. diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index e1d311dfb..ae55fb115 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -53,6 +53,20 @@ var ( PowerVSImageStateImporting = PowerVSImageState("importing") ) +// VPCLoadBalancerState describes the state of the load balancer. +type VPCLoadBalancerState string + +var ( + // VPCLoadBalancerStateActive is the string representing the load balancer in a active state. + VPCLoadBalancerStateActive = VPCLoadBalancerState("active") + + // VPCLoadBalancerStateCreatePending is the string representing the load balancer in a queued state. + VPCLoadBalancerStateCreatePending = VPCLoadBalancerState("create_pending") + + // VPCLoadBalancerStateDeletePending is the string representing the load balancer in a failed state. + VPCLoadBalancerStateDeletePending = VPCLoadBalancerState("delete_pending") +) + // DeletePolicy defines the policy used to identify images to be preserved. type DeletePolicy string @@ -78,5 +92,8 @@ type Subnet struct { // VPCEndpoint describes a VPCEndpoint. type VPCEndpoint struct { Address *string `json:"address"` - FIPID *string `json:"floatingIPID"` + // +optional + FIPID *string `json:"floatingIPID,omitempty"` + // +optional + LBID *string `json:"loadBalancerIPID,omitempty"` } diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 9871cec71..c9d886905 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -503,7 +503,7 @@ func (in *IBMVPCCluster) DeepCopyInto(out *IBMVPCCluster) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) } @@ -561,6 +561,11 @@ func (in *IBMVPCClusterList) DeepCopyObject() runtime.Object { func (in *IBMVPCClusterSpec) DeepCopyInto(out *IBMVPCClusterSpec) { *out = *in out.ControlPlaneEndpoint = in.ControlPlaneEndpoint + if in.ControlPlaneLoadBalancer != nil { + in, out := &in.ControlPlaneLoadBalancer, &out.ControlPlaneLoadBalancer + *out = new(VPCLoadBalancerSpec) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IBMVPCClusterSpec. @@ -579,6 +584,13 @@ func (in *IBMVPCClusterStatus) DeepCopyInto(out *IBMVPCClusterStatus) { out.VPC = in.VPC in.Subnet.DeepCopyInto(&out.Subnet) in.VPCEndpoint.DeepCopyInto(&out.VPCEndpoint) + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(apiv1beta1.Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IBMVPCClusterStatus. @@ -870,6 +882,11 @@ func (in *VPCEndpoint) DeepCopyInto(out *VPCEndpoint) { *out = new(string) **out = **in } + if in.LBID != nil { + in, out := &in.LBID, &out.LBID + *out = new(string) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VPCEndpoint. @@ -881,3 +898,18 @@ func (in *VPCEndpoint) DeepCopy() *VPCEndpoint { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VPCLoadBalancerSpec) DeepCopyInto(out *VPCLoadBalancerSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VPCLoadBalancerSpec. +func (in *VPCLoadBalancerSpec) DeepCopy() *VPCLoadBalancerSpec { + if in == nil { + return nil + } + out := new(VPCLoadBalancerSpec) + in.DeepCopyInto(out) + return out +} diff --git a/cloud/scope/cluster.go b/cloud/scope/cluster.go index 5bbb6fe0a..d17cd9186 100644 --- a/cloud/scope/cluster.go +++ b/cloud/scope/cluster.go @@ -383,6 +383,138 @@ func (s *ClusterScope) detachPublicGateway(subnetID string, pgwID string) error return err } +// CreateLoadBalancer creates a new IBM VPC load balancer in specified resource group. +func (s *ClusterScope) CreateLoadBalancer() (*vpcv1.LoadBalancer, error) { + loadBalancerReply, err := s.ensureLoadBalancerUnique(s.IBMVPCCluster.Spec.ControlPlaneLoadBalancer.Name) + if err != nil { + return nil, err + } else if loadBalancerReply != nil { + // TODO need a reasonable wrapped error + return loadBalancerReply, nil + } + + options := &vpcv1.CreateLoadBalancerOptions{} + options.SetName(s.IBMVPCCluster.Spec.ControlPlaneLoadBalancer.Name) + options.SetIsPublic(true) + options.SetResourceGroup(&vpcv1.ResourceGroupIdentity{ + ID: &s.IBMVPCCluster.Spec.ResourceGroup, + }) + + if s.IBMVPCCluster.Status.Subnet.ID != nil { + subnet := &vpcv1.SubnetIdentity{ + ID: s.IBMVPCCluster.Status.Subnet.ID, + } + options.Subnets = append(options.Subnets, subnet) + } else { + return nil, errors.Wrap(err, "Error subnet required for load balancer creation") + } + + options.SetPools([]vpcv1.LoadBalancerPoolPrototype{ + { + Algorithm: core.StringPtr("round_robin"), + HealthMonitor: &vpcv1.LoadBalancerPoolHealthMonitorPrototype{Delay: core.Int64Ptr(5), MaxRetries: core.Int64Ptr(2), Timeout: core.Int64Ptr(2), Type: core.StringPtr("tcp")}, + Name: core.StringPtr(s.IBMVPCCluster.Spec.ControlPlaneLoadBalancer.Name + "-pool"), + Protocol: core.StringPtr("tcp"), + }, + }) + + options.SetListeners([]vpcv1.LoadBalancerListenerPrototypeLoadBalancerContext{ + { + Protocol: core.StringPtr("tcp"), + Port: core.Int64Ptr(6443), + DefaultPool: &vpcv1.LoadBalancerPoolIdentityByName{ + Name: core.StringPtr(s.IBMVPCCluster.Spec.ControlPlaneLoadBalancer.Name + "-pool"), + }, + }, + }) + + loadBalancer, _, err := s.IBMVPCClient.CreateLoadBalancer(options) + if err != nil { + record.Warnf(s.IBMVPCCluster, "FailedCreateLoadBalancer", "Failed loadBalancer creation - %v", err) + return nil, err + } + + record.Eventf(s.IBMVPCCluster, "SuccessfulCreateLoadBalancer", "Created loadBalancer %q", *loadBalancer.Name) + return loadBalancer, nil +} + +func (s *ClusterScope) ensureLoadBalancerUnique(loadBalancerName string) (*vpcv1.LoadBalancer, error) { + listLoadBalancersOptions := &vpcv1.ListLoadBalancersOptions{} + loadBalancers, _, err := s.IBMVPCClient.ListLoadBalancers(listLoadBalancersOptions) + if err != nil { + return nil, err + } + for _, loadBalancer := range loadBalancers.LoadBalancers { + if (*loadBalancer.Name) == loadBalancerName { + return &loadBalancer, nil + } + } + return nil, nil +} + +// DeleteLoadBalancer deletes IBM VPC load balancer associated with a VPC id. +func (s *ClusterScope) DeleteLoadBalancer() (bool, error) { + deleted := false + if lbipID := s.GetLoadBalancerID(); lbipID != "" { + listLoadBalancersOptions := &vpcv1.ListLoadBalancersOptions{} + loadBalancers, _, err := s.IBMVPCClient.ListLoadBalancers(listLoadBalancersOptions) + if err != nil { + return deleted, err + } + + for _, loadBalancer := range loadBalancers.LoadBalancers { + if (*loadBalancer.ID) == lbipID { + deleted = true + if *loadBalancer.ProvisioningStatus != string(infrav1beta1.VPCLoadBalancerStateDeletePending) { + deleteLoadBalancerOption := &vpcv1.DeleteLoadBalancerOptions{} + deleteLoadBalancerOption.SetID(lbipID) + _, err := s.IBMVPCClient.DeleteLoadBalancer(deleteLoadBalancerOption) + if err != nil { + record.Warnf(s.IBMVPCCluster, "FailedDeleteLoadBalancer", "Failed loadBalancer deletion - %v", err) + return deleted, err + } + } + } + } + } + return deleted, nil +} + +// SetReady will set the status as ready for the cluster. +func (s *ClusterScope) SetReady() { + s.IBMVPCCluster.Status.Ready = true +} + +// SetNotReady will set the status as not ready for the cluster. +func (s *ClusterScope) SetNotReady() { + s.IBMVPCCluster.Status.Ready = false +} + +// IsReady will return the status for the cluster. +func (s *ClusterScope) IsReady() bool { + return s.IBMVPCCluster.Status.Ready +} + +// SetLoadBalancerState will set the state for the load balancer. +func (s *ClusterScope) SetLoadBalancerState(status string) { + s.IBMVPCCluster.Status.ControlPlaneLoadBalancerState = infrav1beta1.VPCLoadBalancerState(status) +} + +// GetLoadBalancerState will get the state for the load balancer. +func (s *ClusterScope) GetLoadBalancerState() infrav1beta1.VPCLoadBalancerState { + return s.IBMVPCCluster.Status.ControlPlaneLoadBalancerState +} + +// SetLoadBalancerID will set the id for the load balancer. +func (s *ClusterScope) SetLoadBalancerID(id *string) { + s.IBMVPCCluster.Status.VPCEndpoint.LBID = id +} + +// GetLoadBalancerID will get the id for the load balancer. +func (s *ClusterScope) GetLoadBalancerID() string { + return *s.IBMVPCCluster.Status.VPCEndpoint.LBID +} + // PatchObject persists the cluster configuration and status. func (s *ClusterScope) PatchObject() error { return s.patchHelper.Patch(context.TODO(), s.IBMVPCCluster) diff --git a/cloud/scope/machine.go b/cloud/scope/machine.go index 6fa9434d3..ccefe65ec 100644 --- a/cloud/scope/machine.go +++ b/cloud/scope/machine.go @@ -192,6 +192,107 @@ func (m *MachineScope) ensureInstanceUnique(instanceName string) (*vpcv1.Instanc return nil, nil } +// CreateVPCLoadBalancerPoolMember creates a new pool member and adds it to the load balancer pool. +func (m *MachineScope) CreateVPCLoadBalancerPoolMember(internalIP *string, targetPort int64) (*vpcv1.LoadBalancerPoolMember, error) { + loadBalancer, _, err := m.IBMVPCClient.GetLoadBalancer(&vpcv1.GetLoadBalancerOptions{ + ID: m.IBMVPCCluster.Status.VPCEndpoint.LBID, + }) + if err != nil { + return nil, err + } + + if *loadBalancer.ProvisioningStatus != string(infrav1beta1.VPCLoadBalancerStateActive) { + return nil, errors.Wrap(err, "load balancer is not in active state") + } + + if len(loadBalancer.Pools) == 0 { + return nil, errors.Wrap(err, "no pools exist for the load balancer") + } + + options := &vpcv1.CreateLoadBalancerPoolMemberOptions{} + options.SetLoadBalancerID(*loadBalancer.ID) + options.SetPoolID(*loadBalancer.Pools[0].ID) + options.SetTarget(&vpcv1.LoadBalancerPoolMemberTargetPrototype{ + Address: internalIP, + }) + options.SetPort(targetPort) + + listOptions := &vpcv1.ListLoadBalancerPoolMembersOptions{} + listOptions.SetLoadBalancerID(*loadBalancer.ID) + listOptions.SetPoolID(*loadBalancer.Pools[0].ID) + listLoadBalancerPoolMembers, _, err := m.IBMVPCClient.ListLoadBalancerPoolMembers(listOptions) + if err != nil { + return nil, errors.Wrapf(err, "failed to bind ListLoadBalancerPoolMembers to control plane %s/%s", m.IBMVPCMachine.Namespace, m.IBMVPCMachine.Name) + } + + for _, member := range listLoadBalancerPoolMembers.Members { + if _, ok := member.Target.(*vpcv1.LoadBalancerPoolMemberTarget); ok { + mtarget := member.Target.(*vpcv1.LoadBalancerPoolMemberTarget) + if *mtarget.Address == *internalIP && *member.Port == targetPort { + m.Logger.V(3).Info("PoolMember already exist") + return nil, nil + } + } + } + + loadBalancerPoolMember, _, err := m.IBMVPCClient.CreateLoadBalancerPoolMember(options) + if err != nil { + return nil, err + } + return loadBalancerPoolMember, nil +} + +// DeleteVPCLoadBalancerPoolMember deletes a pool member from the load balancer pool. +func (m *MachineScope) DeleteVPCLoadBalancerPoolMember() error { + loadBalancer, _, err := m.IBMVPCClient.GetLoadBalancer(&vpcv1.GetLoadBalancerOptions{ + ID: m.IBMVPCCluster.Status.VPCEndpoint.LBID, + }) + if err != nil { + return err + } + + if len(loadBalancer.Pools) == 0 { + return nil + } + + instance, _, err := m.IBMVPCClient.GetInstance(&vpcv1.GetInstanceOptions{ + ID: core.StringPtr(m.IBMVPCMachine.Status.InstanceID), + }) + if err != nil { + return err + } + + listOptions := &vpcv1.ListLoadBalancerPoolMembersOptions{} + listOptions.SetLoadBalancerID(*loadBalancer.ID) + listOptions.SetPoolID(*loadBalancer.Pools[0].ID) + listLoadBalancerPoolMembers, _, err := m.IBMVPCClient.ListLoadBalancerPoolMembers(listOptions) + if err != nil { + return err + } + + for _, member := range listLoadBalancerPoolMembers.Members { + if _, ok := member.Target.(*vpcv1.LoadBalancerPoolMemberTarget); ok { + mtarget := member.Target.(*vpcv1.LoadBalancerPoolMemberTarget) + if *mtarget.Address == *instance.PrimaryNetworkInterface.PrimaryIP.Address { + if *loadBalancer.ProvisioningStatus != string(infrav1beta1.VPCLoadBalancerStateActive) { + return errors.Wrap(err, "load balancer is not in active state") + } + + deleteOptions := &vpcv1.DeleteLoadBalancerPoolMemberOptions{} + deleteOptions.SetLoadBalancerID(*loadBalancer.ID) + deleteOptions.SetPoolID(*loadBalancer.Pools[0].ID) + deleteOptions.SetID(*member.ID) + + if _, err := m.IBMVPCClient.DeleteLoadBalancerPoolMember(deleteOptions); err != nil { + return err + } + return nil + } + } + } + return nil +} + // PatchObject persists the cluster configuration and status. func (m *MachineScope) PatchObject() error { return m.patchHelper.Patch(context.TODO(), m.IBMVPCMachine) diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_ibmvpcclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_ibmvpcclusters.yaml index 63d4de81f..889be5d86 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_ibmvpcclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_ibmvpcclusters.yaml @@ -293,6 +293,16 @@ spec: - host - port type: object + controlPlaneLoadBalancer: + description: ControlPlaneLoadBalancer is optional configuration for + customizing control plane behavior. + properties: + name: + description: Name sets the name of the VPC load balancer. + maxLength: 64 + pattern: ^[A-Za-z0-9]([A-Za-z0-9]{0,31}|[-A-Za-z0-9]{0,30}[A-Za-z0-9])$ + type: string + type: object region: description: The IBM Cloud Region the cluster lives in. type: string @@ -313,8 +323,58 @@ spec: status: description: IBMVPCClusterStatus defines the observed state of IBMVPCCluster. properties: + conditions: + description: Conditions defines current service state of the load + balancer. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + controlPlaneLoadBalancerState: + description: ControlPlaneLoadBalancerState is the status of the load + balancer. + type: string ready: - description: Bastion Instance `json:"bastion,omitempty"` + description: Ready is true when the provider resource is ready. type: boolean subnet: description: Subnet describes a subnet. @@ -353,12 +413,11 @@ spec: type: string floatingIPID: type: string + loadBalancerIPID: + type: string required: - address - - floatingIPID type: object - required: - - ready type: object type: object served: true diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_ibmvpcmachines.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_ibmvpcmachines.yaml index dba94d0c1..676938569 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_ibmvpcmachines.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_ibmvpcmachines.yaml @@ -310,9 +310,8 @@ spec: this machine. type: string ready: + description: Ready is true when the provider resource is ready. type: boolean - required: - - ready type: object type: object served: true diff --git a/controllers/ibmvpccluster_controller.go b/controllers/ibmvpccluster_controller.go index fe5446adb..c837f2aa0 100644 --- a/controllers/ibmvpccluster_controller.go +++ b/controllers/ibmvpccluster_controller.go @@ -18,6 +18,7 @@ package controllers import ( "context" + "time" "github.com/IBM/vpc-go-sdk/vpcv1" "github.com/go-logr/logr" @@ -27,6 +28,7 @@ import ( "k8s.io/client-go/tools/record" capiv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/cluster-api/util/conditions" "sigs.k8s.io/cluster-api/util/predicates" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -118,7 +120,7 @@ func (r *IBMVPCClusterReconciler) reconcile(clusterScope *scope.ClusterScope) (c } } - if clusterScope.IBMVPCCluster.Spec.ControlPlaneEndpoint.Host == "" { + if clusterScope.IBMVPCCluster.Spec.ControlPlaneEndpoint.Host == "" && clusterScope.IBMVPCCluster.Spec.ControlPlaneLoadBalancer == nil { fip, err := clusterScope.ReserveFIP() if err != nil { return ctrl.Result{}, errors.Wrapf(err, "failed to reconcile Control Plane Endpoint for IBMVPCCluster %s/%s", clusterScope.IBMVPCCluster.Namespace, clusterScope.IBMVPCCluster.Name) @@ -135,6 +137,7 @@ func (r *IBMVPCClusterReconciler) reconcile(clusterScope *scope.ClusterScope) (c FIPID: fip.ID, } } + clusterScope.SetReady() } if clusterScope.IBMVPCCluster.Status.Subnet.ID == nil { @@ -152,7 +155,51 @@ func (r *IBMVPCClusterReconciler) reconcile(clusterScope *scope.ClusterScope) (c } } - clusterScope.IBMVPCCluster.Status.Ready = true + if clusterScope.IBMVPCCluster.Spec.ControlPlaneLoadBalancer != nil { + loadBalancer, err := r.getOrCreate(clusterScope) + if err != nil { + return ctrl.Result{}, errors.Wrapf(err, "failed to reconcile Control Plane LoadBalancer for IBMVPCCluster %s/%s", clusterScope.IBMVPCCluster.Namespace, clusterScope.IBMVPCCluster.Name) + } + + if loadBalancer != nil { + clusterScope.SetLoadBalancerID(loadBalancer.ID) + clusterScope.Logger.V(3).Info("LoadBalancerID - " + clusterScope.GetLoadBalancerID()) + clusterScope.SetLoadBalancerState(*loadBalancer.ProvisioningStatus) + clusterScope.Logger.V(3).Info("LoadBalancerState - " + string(clusterScope.GetLoadBalancerState())) + + clusterScope.IBMVPCCluster.Spec.ControlPlaneEndpoint = capiv1beta1.APIEndpoint{ + Host: *loadBalancer.Hostname, + // TODO: Support usage of user supplied port. + Port: 6443, + } + + clusterScope.IBMVPCCluster.Status.VPCEndpoint = infrav1beta1.VPCEndpoint{ + Address: loadBalancer.Hostname, + LBID: loadBalancer.ID, + } + + switch clusterScope.GetLoadBalancerState() { + case infrav1beta1.VPCLoadBalancerStateCreatePending: + clusterScope.Logger.V(3).Info("LoadBalancer is in create state") + clusterScope.SetNotReady() + conditions.MarkFalse(clusterScope.IBMVPCCluster, infrav1beta1.LoadBalancerReadyCondition, string(infrav1beta1.VPCLoadBalancerStateCreatePending), capiv1beta1.ConditionSeverityInfo, *loadBalancer.OperatingStatus) + case infrav1beta1.VPCLoadBalancerStateActive: + clusterScope.Logger.V(3).Info("LoadBalancer is in active state") + clusterScope.SetReady() + conditions.MarkTrue(clusterScope.IBMVPCCluster, infrav1beta1.LoadBalancerReadyCondition) + default: + clusterScope.Logger.V(3).Info("LoadBalancer state is undefined", "state", clusterScope.GetLoadBalancerState(), "loadbalancer-id", clusterScope.GetLoadBalancerID()) + clusterScope.SetNotReady() + conditions.MarkUnknown(clusterScope.IBMVPCCluster, infrav1beta1.LoadBalancerReadyCondition, *loadBalancer.ProvisioningStatus, "") + } + } + } + + // Requeue after 1 minute if cluster is not ready to update status of the cluster properly. + if !clusterScope.IsReady() { + clusterScope.Info("Cluster is not yet ready") + return ctrl.Result{RequeueAfter: 1 * time.Minute}, nil + } return ctrl.Result{}, nil } @@ -167,15 +214,28 @@ func (r *IBMVPCClusterReconciler) reconcileDelete(clusterScope *scope.ClusterSco } // skip deleting other resources if still have vsis running. if *vsis.TotalCount != int64(0) { - return ctrl.Result{}, nil + return ctrl.Result{RequeueAfter: 1 * time.Minute}, nil + } + + if clusterScope.IBMVPCCluster.Spec.ControlPlaneLoadBalancer != nil { + deleted, err := clusterScope.DeleteLoadBalancer() + if err != nil { + return ctrl.Result{}, errors.Wrap(err, "failed to delete loadBalancer") + } + // skip deleting other resources if still have loadBalancers running. + if deleted { + return ctrl.Result{RequeueAfter: 1 * time.Minute}, nil + } } if err := clusterScope.DeleteSubnet(); err != nil { return ctrl.Result{}, errors.Wrap(err, "failed to delete subnet") } - if err := clusterScope.DeleteFloatingIP(); err != nil { - return ctrl.Result{}, errors.Wrap(err, "failed to delete floatingIP") + if clusterScope.IBMVPCCluster.Spec.ControlPlaneLoadBalancer == nil { + if err := clusterScope.DeleteFloatingIP(); err != nil { + return ctrl.Result{}, errors.Wrap(err, "failed to delete floatingIP") + } } if err := clusterScope.DeleteVPC(); err != nil { @@ -185,6 +245,11 @@ func (r *IBMVPCClusterReconciler) reconcileDelete(clusterScope *scope.ClusterSco return ctrl.Result{}, nil } +func (r *IBMVPCClusterReconciler) getOrCreate(clusterScope *scope.ClusterScope) (*vpcv1.LoadBalancer, error) { + loadBalancer, err := clusterScope.CreateLoadBalancer() + return loadBalancer, err +} + // SetupWithManager creates a new IBMVPCCluster controller for a manager. func (r *IBMVPCClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). diff --git a/controllers/ibmvpcmachine_controller.go b/controllers/ibmvpcmachine_controller.go index 2ec767acd..7ea12ce50 100644 --- a/controllers/ibmvpcmachine_controller.go +++ b/controllers/ibmvpcmachine_controller.go @@ -19,6 +19,7 @@ package controllers import ( "context" "fmt" + "time" "github.com/IBM/vpc-go-sdk/vpcv1" "github.com/go-logr/logr" @@ -165,19 +166,34 @@ func (r *IBMVPCMachineReconciler) reconcileNormal(machineScope *scope.MachineSco _, ok := machineScope.IBMVPCMachine.Labels[capiv1beta1.MachineControlPlaneLabelName] machineScope.IBMVPCMachine.Spec.ProviderID = pointer.StringPtr(fmt.Sprintf("ibmvpc://%s/%s", machineScope.Machine.Spec.ClusterName, machineScope.IBMVPCMachine.Name)) if ok { - options := &vpcv1.AddInstanceNetworkInterfaceFloatingIPOptions{} - options.SetID(*machineScope.IBMVPCCluster.Status.VPCEndpoint.FIPID) - options.SetInstanceID(*instance.ID) - options.SetNetworkInterfaceID(*instance.PrimaryNetworkInterface.ID) - floatingIP, _, err := - machineScope.IBMVPCClient.AddInstanceNetworkInterfaceFloatingIP(options) - if err != nil { - return ctrl.Result{}, errors.Wrapf(err, "failed to bind floating IP to control plane %s/%s", machineScope.IBMVPCMachine.Namespace, machineScope.IBMVPCMachine.Name) + if machineScope.IBMVPCCluster.Spec.ControlPlaneLoadBalancer == nil { + options := &vpcv1.AddInstanceNetworkInterfaceFloatingIPOptions{} + options.SetID(*machineScope.IBMVPCCluster.Status.VPCEndpoint.FIPID) + options.SetInstanceID(*instance.ID) + options.SetNetworkInterfaceID(*instance.PrimaryNetworkInterface.ID) + floatingIP, _, err := + machineScope.IBMVPCClient.AddInstanceNetworkInterfaceFloatingIP(options) + if err != nil { + return ctrl.Result{}, errors.Wrapf(err, "failed to bind floating IP to control plane %s/%s", machineScope.IBMVPCMachine.Namespace, machineScope.IBMVPCMachine.Name) + } + machineScope.IBMVPCMachine.Status.Addresses = append(machineScope.IBMVPCMachine.Status.Addresses, corev1.NodeAddress{ + Type: corev1.NodeExternalIP, + Address: *floatingIP.Address, + }) + } else { + if instance.PrimaryNetworkInterface.PrimaryIP.Address == nil || *instance.PrimaryNetworkInterface.PrimaryIP.Address == "0.0.0.0" { + return ctrl.Result{}, errors.Wrapf(err, "invalid primary ip address") + } + internalIP := instance.PrimaryNetworkInterface.PrimaryIP.Address + port := int64(6443) + poolMember, err := machineScope.CreateVPCLoadBalancerPoolMember(internalIP, port) + if err != nil { + return ctrl.Result{}, errors.Wrapf(err, "failed to bind port %d to control plane %s/%s", port, machineScope.IBMVPCMachine.Namespace, machineScope.IBMVPCMachine.Name) + } + if poolMember != nil && *poolMember.ProvisioningStatus != string(infrav1beta1.VPCLoadBalancerStateActive) { + return ctrl.Result{RequeueAfter: 1 * time.Minute}, nil + } } - machineScope.IBMVPCMachine.Status.Addresses = append(machineScope.IBMVPCMachine.Status.Addresses, corev1.NodeAddress{ - Type: corev1.NodeExternalIP, - Address: *floatingIP.Address, - }) } machineScope.IBMVPCMachine.Status.Ready = true machineScope.Info(*instance.ID) @@ -194,6 +210,12 @@ func (r *IBMVPCMachineReconciler) getOrCreate(scope *scope.MachineScope) (*vpcv1 func (r *IBMVPCMachineReconciler) reconcileDelete(scope *scope.MachineScope) (_ ctrl.Result, reterr error) { scope.Info("Handling deleted IBMVPCMachine") + if _, ok := scope.IBMVPCMachine.Labels[capiv1beta1.MachineControlPlaneLabelName]; ok && scope.IBMVPCCluster.Spec.ControlPlaneLoadBalancer != nil { + if err := scope.DeleteVPCLoadBalancerPoolMember(); err != nil { + return ctrl.Result{}, errors.Wrap(err, "failed to delete loadBalancer pool member") + } + } + if err := scope.DeleteMachine(); err != nil { scope.Info("error deleting IBMVPCMachine") return ctrl.Result{}, errors.Wrapf(err, "error deleting IBMVPCMachine %s/%s", scope.IBMVPCMachine.Namespace, scope.IBMVPCMachine.Spec.Name) diff --git a/pkg/cloud/services/vpc/mock/vpc_generated.go b/pkg/cloud/services/vpc/mock/vpc_generated.go index 4939427cc..81da9b78f 100644 --- a/pkg/cloud/services/vpc/mock/vpc_generated.go +++ b/pkg/cloud/services/vpc/mock/vpc_generated.go @@ -98,6 +98,38 @@ func (mr *MockVpcMockRecorder) CreateInstance(options interface{}) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateInstance", reflect.TypeOf((*MockVpc)(nil).CreateInstance), options) } +// CreateLoadBalancer mocks base method. +func (m *MockVpc) CreateLoadBalancer(options *vpcv1.CreateLoadBalancerOptions) (*vpcv1.LoadBalancer, *core.DetailedResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateLoadBalancer", options) + ret0, _ := ret[0].(*vpcv1.LoadBalancer) + ret1, _ := ret[1].(*core.DetailedResponse) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// CreateLoadBalancer indicates an expected call of CreateLoadBalancer. +func (mr *MockVpcMockRecorder) CreateLoadBalancer(options interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateLoadBalancer", reflect.TypeOf((*MockVpc)(nil).CreateLoadBalancer), options) +} + +// CreateLoadBalancerPoolMember mocks base method. +func (m *MockVpc) CreateLoadBalancerPoolMember(options *vpcv1.CreateLoadBalancerPoolMemberOptions) (*vpcv1.LoadBalancerPoolMember, *core.DetailedResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateLoadBalancerPoolMember", options) + ret0, _ := ret[0].(*vpcv1.LoadBalancerPoolMember) + ret1, _ := ret[1].(*core.DetailedResponse) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// CreateLoadBalancerPoolMember indicates an expected call of CreateLoadBalancerPoolMember. +func (mr *MockVpcMockRecorder) CreateLoadBalancerPoolMember(options interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateLoadBalancerPoolMember", reflect.TypeOf((*MockVpc)(nil).CreateLoadBalancerPoolMember), options) +} + // CreatePublicGateway mocks base method. func (m *MockVpc) CreatePublicGateway(options *vpcv1.CreatePublicGatewayOptions) (*vpcv1.PublicGateway, *core.DetailedResponse, error) { m.ctrl.T.Helper() @@ -192,6 +224,36 @@ func (mr *MockVpcMockRecorder) DeleteInstance(options interface{}) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteInstance", reflect.TypeOf((*MockVpc)(nil).DeleteInstance), options) } +// DeleteLoadBalancer mocks base method. +func (m *MockVpc) DeleteLoadBalancer(options *vpcv1.DeleteLoadBalancerOptions) (*core.DetailedResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteLoadBalancer", options) + ret0, _ := ret[0].(*core.DetailedResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteLoadBalancer indicates an expected call of DeleteLoadBalancer. +func (mr *MockVpcMockRecorder) DeleteLoadBalancer(options interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLoadBalancer", reflect.TypeOf((*MockVpc)(nil).DeleteLoadBalancer), options) +} + +// DeleteLoadBalancerPoolMember mocks base method. +func (m *MockVpc) DeleteLoadBalancerPoolMember(options *vpcv1.DeleteLoadBalancerPoolMemberOptions) (*core.DetailedResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteLoadBalancerPoolMember", options) + ret0, _ := ret[0].(*core.DetailedResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteLoadBalancerPoolMember indicates an expected call of DeleteLoadBalancerPoolMember. +func (mr *MockVpcMockRecorder) DeleteLoadBalancerPoolMember(options interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLoadBalancerPoolMember", reflect.TypeOf((*MockVpc)(nil).DeleteLoadBalancerPoolMember), options) +} + // DeletePublicGateway mocks base method. func (m *MockVpc) DeletePublicGateway(options *vpcv1.DeletePublicGatewayOptions) (*core.DetailedResponse, error) { m.ctrl.T.Helper() @@ -253,6 +315,22 @@ func (mr *MockVpcMockRecorder) GetInstance(options interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInstance", reflect.TypeOf((*MockVpc)(nil).GetInstance), options) } +// GetLoadBalancer mocks base method. +func (m *MockVpc) GetLoadBalancer(options *vpcv1.GetLoadBalancerOptions) (*vpcv1.LoadBalancer, *core.DetailedResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetLoadBalancer", options) + ret0, _ := ret[0].(*vpcv1.LoadBalancer) + ret1, _ := ret[1].(*core.DetailedResponse) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// GetLoadBalancer indicates an expected call of GetLoadBalancer. +func (mr *MockVpcMockRecorder) GetLoadBalancer(options interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLoadBalancer", reflect.TypeOf((*MockVpc)(nil).GetLoadBalancer), options) +} + // GetSubnetPublicGateway mocks base method. func (m *MockVpc) GetSubnetPublicGateway(options *vpcv1.GetSubnetPublicGatewayOptions) (*vpcv1.PublicGateway, *core.DetailedResponse, error) { m.ctrl.T.Helper() @@ -301,6 +379,38 @@ func (mr *MockVpcMockRecorder) ListInstances(options interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListInstances", reflect.TypeOf((*MockVpc)(nil).ListInstances), options) } +// ListLoadBalancerPoolMembers mocks base method. +func (m *MockVpc) ListLoadBalancerPoolMembers(options *vpcv1.ListLoadBalancerPoolMembersOptions) (*vpcv1.LoadBalancerPoolMemberCollection, *core.DetailedResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListLoadBalancerPoolMembers", options) + ret0, _ := ret[0].(*vpcv1.LoadBalancerPoolMemberCollection) + ret1, _ := ret[1].(*core.DetailedResponse) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// ListLoadBalancerPoolMembers indicates an expected call of ListLoadBalancerPoolMembers. +func (mr *MockVpcMockRecorder) ListLoadBalancerPoolMembers(options interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListLoadBalancerPoolMembers", reflect.TypeOf((*MockVpc)(nil).ListLoadBalancerPoolMembers), options) +} + +// ListLoadBalancers mocks base method. +func (m *MockVpc) ListLoadBalancers(options *vpcv1.ListLoadBalancersOptions) (*vpcv1.LoadBalancerCollection, *core.DetailedResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListLoadBalancers", options) + ret0, _ := ret[0].(*vpcv1.LoadBalancerCollection) + ret1, _ := ret[1].(*core.DetailedResponse) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// ListLoadBalancers indicates an expected call of ListLoadBalancers. +func (mr *MockVpcMockRecorder) ListLoadBalancers(options interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListLoadBalancers", reflect.TypeOf((*MockVpc)(nil).ListLoadBalancers), options) +} + // ListSubnets mocks base method. func (m *MockVpc) ListSubnets(options *vpcv1.ListSubnetsOptions) (*vpcv1.SubnetCollection, *core.DetailedResponse, error) { m.ctrl.T.Helper() diff --git a/pkg/cloud/services/vpc/service.go b/pkg/cloud/services/vpc/service.go index d63c4ed35..6a80dc8b8 100644 --- a/pkg/cloud/services/vpc/service.go +++ b/pkg/cloud/services/vpc/service.go @@ -93,7 +93,7 @@ func (s *Service) ListSubnets(options *vpcv1.ListSubnetsOptions) (*vpcv1.SubnetC return s.vpcService.ListSubnets(options) } -// GetSubnetPublicGateway returns a public gateway attched to the subnet. +// GetSubnetPublicGateway returns a public gateway attached to the subnet. func (s *Service) GetSubnetPublicGateway(options *vpcv1.GetSubnetPublicGatewayOptions) (*vpcv1.PublicGateway, *core.DetailedResponse, error) { return s.vpcService.GetSubnetPublicGateway(options) } @@ -133,6 +133,41 @@ func (s *Service) AddInstanceNetworkInterfaceFloatingIP(options *vpcv1.AddInstan return s.vpcService.AddInstanceNetworkInterfaceFloatingIP(options) } +// CreateLoadBalancer creates a new load balancer. +func (s *Service) CreateLoadBalancer(options *vpcv1.CreateLoadBalancerOptions) (*vpcv1.LoadBalancer, *core.DetailedResponse, error) { + return s.vpcService.CreateLoadBalancer(options) +} + +// DeleteLoadBalancer deletes a load balancer. +func (s *Service) DeleteLoadBalancer(options *vpcv1.DeleteLoadBalancerOptions) (*core.DetailedResponse, error) { + return s.vpcService.DeleteLoadBalancer(options) +} + +// ListLoadBalancers returns list of load balancers in a region. +func (s *Service) ListLoadBalancers(options *vpcv1.ListLoadBalancersOptions) (*vpcv1.LoadBalancerCollection, *core.DetailedResponse, error) { + return s.vpcService.ListLoadBalancers(options) +} + +// GetLoadBalancer returns a load balancer. +func (s *Service) GetLoadBalancer(options *vpcv1.GetLoadBalancerOptions) (*vpcv1.LoadBalancer, *core.DetailedResponse, error) { + return s.vpcService.GetLoadBalancer(options) +} + +// CreateLoadBalancerPoolMember creates a new member and adds the member to the pool. +func (s *Service) CreateLoadBalancerPoolMember(options *vpcv1.CreateLoadBalancerPoolMemberOptions) (*vpcv1.LoadBalancerPoolMember, *core.DetailedResponse, error) { + return s.vpcService.CreateLoadBalancerPoolMember(options) +} + +// DeleteLoadBalancerPoolMember deletes a member from the load balancer pool. +func (s *Service) DeleteLoadBalancerPoolMember(options *vpcv1.DeleteLoadBalancerPoolMemberOptions) (*core.DetailedResponse, error) { + return s.vpcService.DeleteLoadBalancerPoolMember(options) +} + +// ListLoadBalancerPoolMembers returns members of a load balancer pool. +func (s *Service) ListLoadBalancerPoolMembers(options *vpcv1.ListLoadBalancerPoolMembersOptions) (*vpcv1.LoadBalancerPoolMemberCollection, *core.DetailedResponse, error) { + return s.vpcService.ListLoadBalancerPoolMembers(options) +} + // NewService returns a new VPC Service. func NewService(svcEndpoint string) (Vpc, error) { service := &Service{} diff --git a/pkg/cloud/services/vpc/vpc.go b/pkg/cloud/services/vpc/vpc.go index ba1534b42..41cba1e2b 100644 --- a/pkg/cloud/services/vpc/vpc.go +++ b/pkg/cloud/services/vpc/vpc.go @@ -47,4 +47,11 @@ type Vpc interface { ListVPCAddressPrefixes(options *vpcv1.ListVPCAddressPrefixesOptions) (*vpcv1.AddressPrefixCollection, *core.DetailedResponse, error) CreateSecurityGroupRule(options *vpcv1.CreateSecurityGroupRuleOptions) (vpcv1.SecurityGroupRuleIntf, *core.DetailedResponse, error) AddInstanceNetworkInterfaceFloatingIP(options *vpcv1.AddInstanceNetworkInterfaceFloatingIPOptions) (*vpcv1.FloatingIP, *core.DetailedResponse, error) + CreateLoadBalancer(options *vpcv1.CreateLoadBalancerOptions) (*vpcv1.LoadBalancer, *core.DetailedResponse, error) + DeleteLoadBalancer(options *vpcv1.DeleteLoadBalancerOptions) (*core.DetailedResponse, error) + ListLoadBalancers(options *vpcv1.ListLoadBalancersOptions) (*vpcv1.LoadBalancerCollection, *core.DetailedResponse, error) + GetLoadBalancer(options *vpcv1.GetLoadBalancerOptions) (*vpcv1.LoadBalancer, *core.DetailedResponse, error) + CreateLoadBalancerPoolMember(options *vpcv1.CreateLoadBalancerPoolMemberOptions) (*vpcv1.LoadBalancerPoolMember, *core.DetailedResponse, error) + DeleteLoadBalancerPoolMember(options *vpcv1.DeleteLoadBalancerPoolMemberOptions) (*core.DetailedResponse, error) + ListLoadBalancerPoolMembers(options *vpcv1.ListLoadBalancerPoolMembersOptions) (*vpcv1.LoadBalancerPoolMemberCollection, *core.DetailedResponse, error) }