From cefb846fb6df736edd59029005ba38590aead1e4 Mon Sep 17 00:00:00 2001 From: hello2mao Date: Mon, 25 Feb 2019 22:39:09 +0800 Subject: [PATCH 1/7] update validate for udp --- pkg/cloud-provider/load_balancer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cloud-provider/load_balancer.go b/pkg/cloud-provider/load_balancer.go index 3644ccd..8ab1e58 100644 --- a/pkg/cloud-provider/load_balancer.go +++ b/pkg/cloud-provider/load_balancer.go @@ -202,8 +202,8 @@ func (bc *Baiducloud) validateService(service *v1.Service) error { return fmt.Errorf("requested load balancer with no ports") } for _, port := range service.Spec.Ports { - if port.Protocol != v1.ProtocolTCP { - return fmt.Errorf("only TCP LoadBalancer is supported for Baidu CCE") + if port.Protocol != v1.ProtocolTCP && port.Protocol != v1.ProtocolUDP { + return fmt.Errorf("only TCP,UDP LoadBalancer is supported for Baidu CCE") } } return nil From 1ba3a96e1cc6882d217a58cc6665ee028926bff3 Mon Sep 17 00:00:00 2001 From: hello2mao Date: Mon, 25 Feb 2019 22:59:48 +0800 Subject: [PATCH 2/7] add comment for ServiceAnnotationLoadBalancerAllocateVip --- pkg/cloud-provider/options.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/cloud-provider/options.go b/pkg/cloud-provider/options.go index 07f4d05..fbd92d1 100644 --- a/pkg/cloud-provider/options.go +++ b/pkg/cloud-provider/options.go @@ -30,6 +30,7 @@ const ( ServiceAnnotationLoadBalancerId = ServiceAnnotationLoadBalancerPrefix + "id" // ServiceAnnotationLoadBalancerInternalVpc is the annotation of LoadBalancerInternalVpc ServiceAnnotationLoadBalancerInternalVpc = ServiceAnnotationLoadBalancerPrefix + "internal-vpc" + // ServiceAnnotationLoadBalancerAllocateVip is the annotation which indicates BLB with a VIP ServiceAnnotationLoadBalancerAllocateVip = ServiceAnnotationLoadBalancerPrefix + "allocate-vip" // ServiceAnnotationElasticIPPrefix is the annotation prefix of ElasticIP From 78fc272f667202f455cbee57abd93c96a125b4c1 Mon Sep 17 00:00:00 2001 From: hello2mao Date: Mon, 25 Feb 2019 23:13:40 +0800 Subject: [PATCH 3/7] add more LB health check annotation define --- pkg/cloud-provider/options.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pkg/cloud-provider/options.go b/pkg/cloud-provider/options.go index fbd92d1..0b479b2 100644 --- a/pkg/cloud-provider/options.go +++ b/pkg/cloud-provider/options.go @@ -32,6 +32,19 @@ const ( ServiceAnnotationLoadBalancerInternalVpc = ServiceAnnotationLoadBalancerPrefix + "internal-vpc" // ServiceAnnotationLoadBalancerAllocateVip is the annotation which indicates BLB with a VIP ServiceAnnotationLoadBalancerAllocateVip = ServiceAnnotationLoadBalancerPrefix + "allocate-vip" + // ServiceAnnotationLoadBalancerScheduler is the annotation of load balancer which can be "RoundRobin"/"LeastConnection"/"Hash" + ServiceAnnotationLoadBalancerScheduler = ServiceAnnotationLoadBalancerPrefix + "scheduler" + // ServiceAnnotationLoadBalancerHealthCheckTimeoutInSecond is the annotation of health check timeout + ServiceAnnotationLoadBalancerHealthCheckTimeoutInSecond = ServiceAnnotationLoadBalancerPrefix + "health-check-timeout-in-second" + // ServiceAnnotationLoadBalancerHealthCheckInterval is the annotation of health check interval + ServiceAnnotationLoadBalancerHealthCheckInterval = ServiceAnnotationLoadBalancerPrefix + "health-check-interval" + // ServiceAnnotationLoadBalancerUnhealthyThreshold is the annotation of unhealthy threshold + ServiceAnnotationLoadBalancerUnhealthyThreshold = ServiceAnnotationLoadBalancerPrefix + "unhealthy-threshold" + // ServiceAnnotationLoadBalancerHealthyThreshold is the annotation of healthy threshold + ServiceAnnotationLoadBalancerHealthyThreshold = ServiceAnnotationLoadBalancerPrefix + "healthy-threshold" + // ServiceAnnotationLoadBalancerHealthCheckString is the annotation of health check string + ServiceAnnotationLoadBalancerHealthCheckString = ServiceAnnotationLoadBalancerPrefix + "health-check-string" + // ServiceAnnotationElasticIPPrefix is the annotation prefix of ElasticIP ServiceAnnotationElasticIPPrefix = "service.beta.kubernetes.io/cce-elastic-ip-" From cb3864203fbbd0fd95f8f9b4ae946b348ad1ae2c Mon Sep 17 00:00:00 2001 From: hello2mao Date: Tue, 26 Feb 2019 08:41:05 +0800 Subject: [PATCH 4/7] parse service annotation of LB health check --- pkg/cloud-provider/cce.go | 20 -------- pkg/cloud-provider/options.go | 88 +++++++++++++++++++++++++++++++++-- 2 files changed, 83 insertions(+), 25 deletions(-) diff --git a/pkg/cloud-provider/cce.go b/pkg/cloud-provider/cce.go index 603dbbd..eb204e7 100644 --- a/pkg/cloud-provider/cce.go +++ b/pkg/cloud-provider/cce.go @@ -63,26 +63,6 @@ type CloudConfig struct { Debug bool `json:"Debug"` } -// ServiceAnnotation contains annotations from service -type ServiceAnnotation struct { - LoadBalancerId string - LoadBalancerInternalVpc string - LoadBalancerAllocateVip string - ElasticIPName string - ElasticIPPaymentTiming string - ElasticIPBillingMethod string - ElasticIPBandwidthInMbps int - ElasticIPReservationLength int -} - -// NodeAnnotation contains annotations from node -type NodeAnnotation struct { - VpcId string - VpcRouteTableId string - VpcRouteRuleId string - CCMVersion string -} - // CCMVersion is the version of CCM var CCMVersion string diff --git a/pkg/cloud-provider/options.go b/pkg/cloud-provider/options.go index 0b479b2..ba1c130 100644 --- a/pkg/cloud-provider/options.go +++ b/pkg/cloud-provider/options.go @@ -34,17 +34,16 @@ const ( ServiceAnnotationLoadBalancerAllocateVip = ServiceAnnotationLoadBalancerPrefix + "allocate-vip" // ServiceAnnotationLoadBalancerScheduler is the annotation of load balancer which can be "RoundRobin"/"LeastConnection"/"Hash" ServiceAnnotationLoadBalancerScheduler = ServiceAnnotationLoadBalancerPrefix + "scheduler" - // ServiceAnnotationLoadBalancerHealthCheckTimeoutInSecond is the annotation of health check timeout + // ServiceAnnotationLoadBalancerHealthCheckTimeoutInSecond is the annotation of health check timeout, default 3s, [1, 60] ServiceAnnotationLoadBalancerHealthCheckTimeoutInSecond = ServiceAnnotationLoadBalancerPrefix + "health-check-timeout-in-second" - // ServiceAnnotationLoadBalancerHealthCheckInterval is the annotation of health check interval + // ServiceAnnotationLoadBalancerHealthCheckInterval is the annotation of health check interval, default 3s, [1, 10] ServiceAnnotationLoadBalancerHealthCheckInterval = ServiceAnnotationLoadBalancerPrefix + "health-check-interval" - // ServiceAnnotationLoadBalancerUnhealthyThreshold is the annotation of unhealthy threshold + // ServiceAnnotationLoadBalancerUnhealthyThreshold is the annotation of unhealthy threshold, default 3, [2, 5] ServiceAnnotationLoadBalancerUnhealthyThreshold = ServiceAnnotationLoadBalancerPrefix + "unhealthy-threshold" - // ServiceAnnotationLoadBalancerHealthyThreshold is the annotation of healthy threshold + // ServiceAnnotationLoadBalancerHealthyThreshold is the annotation of healthy threshold, default 3, [2, 5] ServiceAnnotationLoadBalancerHealthyThreshold = ServiceAnnotationLoadBalancerPrefix + "healthy-threshold" // ServiceAnnotationLoadBalancerHealthCheckString is the annotation of health check string ServiceAnnotationLoadBalancerHealthCheckString = ServiceAnnotationLoadBalancerPrefix + "health-check-string" - // ServiceAnnotationElasticIPPrefix is the annotation prefix of ElasticIP ServiceAnnotationElasticIPPrefix = "service.beta.kubernetes.io/cce-elastic-ip-" @@ -74,6 +73,35 @@ const ( NodeAnnotationCCMVersion = NodeAnnotationPrefix + "ccm-version" ) +// ServiceAnnotation contains annotations from service +type ServiceAnnotation struct { + /* BLB */ + LoadBalancerId string + LoadBalancerInternalVpc string + LoadBalancerAllocateVip string + LoadBalancerScheduler string + LoadBalancerHealthCheckTimeoutInSecond int + LoadBalancerHealthCheckInterval int + LoadBalancerUnhealthyThreshold int + LoadBalancerHealthyThreshold int + LoadBalancerHealthCheckString string + + /* EIP */ + ElasticIPName string + ElasticIPPaymentTiming string + ElasticIPBillingMethod string + ElasticIPBandwidthInMbps int + ElasticIPReservationLength int +} + +// NodeAnnotation contains annotations from node +type NodeAnnotation struct { + VpcId string + VpcRouteTableId string + VpcRouteRuleId string + CCMVersion string +} + // ExtractServiceAnnotation extract annotations from service func ExtractServiceAnnotation(service *v1.Service) *ServiceAnnotation { glog.V(4).Infof("start to ExtractServiceAnnotation: %v", service.Annotations) @@ -98,6 +126,56 @@ func ExtractServiceAnnotation(service *v1.Service) *ServiceAnnotation { result.LoadBalancerAllocateVip = loadBalancerAllocateVip } + loadBalancerScheduler, ok := annotation[ServiceAnnotationLoadBalancerScheduler] + if ok { + result.LoadBalancerScheduler = loadBalancerScheduler + } + + loadBalancerHealthCheckTimeoutInSecond, exist := annotation[ServiceAnnotationLoadBalancerHealthCheckTimeoutInSecond] + if exist { + i, err := strconv.Atoi(loadBalancerHealthCheckTimeoutInSecond) + if err != nil { + glog.V(4).Infof("ServiceAnnotationLoadBalancerHealthCheckTimeoutInSecond must be int") + } else { + result.LoadBalancerHealthCheckTimeoutInSecond = i + } + } + + loadBalancerHealthCheckInterval, exist := annotation[ServiceAnnotationLoadBalancerHealthCheckInterval] + if exist { + i, err := strconv.Atoi(loadBalancerHealthCheckInterval) + if err != nil { + glog.V(4).Infof("ServiceAnnotationLoadBalancerHealthCheckInterval must be int") + } else { + result.LoadBalancerHealthCheckInterval = i + } + } + + loadBalancerUnhealthyThreshold, exist := annotation[ServiceAnnotationLoadBalancerUnhealthyThreshold] + if exist { + i, err := strconv.Atoi(loadBalancerUnhealthyThreshold) + if err != nil { + glog.V(4).Infof("ServiceAnnotationLoadBalancerUnhealthyThreshold must be int") + } else { + result.LoadBalancerUnhealthyThreshold = i + } + } + + loadBalancerHealthyThreshold, exist := annotation[ServiceAnnotationLoadBalancerHealthyThreshold] + if exist { + i, err := strconv.Atoi(loadBalancerHealthyThreshold) + if err != nil { + glog.V(4).Infof("ServiceAnnotationLoadBalancerHealthyThreshold must be int") + } else { + result.LoadBalancerHealthyThreshold = i + } + } + + loadBalancerHealthCheckString, exist := annotation[ServiceAnnotationLoadBalancerHealthCheckString] + if exist { + result.LoadBalancerHealthCheckString = loadBalancerHealthCheckString + } + elasticIPName, exist := annotation[ServiceAnnotationElasticIPName] if exist { result.ElasticIPName = elasticIPName From 1ff4901c201e0142247a23ead9241f6743e2d85e Mon Sep 17 00:00:00 2001 From: hello2mao Date: Tue, 26 Feb 2019 09:25:27 +0800 Subject: [PATCH 5/7] update BLB cloud-sdk for UDP LB Service --- pkg/cloud-provider/cce.go | 4 +- pkg/cloud-provider/instances.go | 2 +- pkg/cloud-provider/load_balancer.go | 20 +-- pkg/cloud-provider/routes.go | 4 +- pkg/{sdk => cloud-sdk}/bcc/cds.go | 2 +- pkg/{sdk => cloud-sdk}/bcc/client.go | 2 +- pkg/{sdk => cloud-sdk}/bcc/instance.go | 2 +- pkg/{sdk => cloud-sdk}/bce/core.go | 4 +- pkg/{sdk => cloud-sdk}/bce/error.go | 0 pkg/{sdk => cloud-sdk}/bce/request.go | 2 +- pkg/{sdk => cloud-sdk}/bce/response.go | 0 pkg/{sdk => cloud-sdk}/bce/types.go | 0 pkg/{sdk => cloud-sdk}/blb/backendserver.go | 56 +++--- pkg/{sdk => cloud-sdk}/blb/client.go | 15 +- pkg/{sdk => cloud-sdk}/blb/listener.go | 163 +++++++++++------- pkg/{sdk => cloud-sdk}/blb/loadbalancer.go | 58 ++++--- pkg/{sdk => cloud-sdk}/bos/bucket.go | 4 +- pkg/{sdk => cloud-sdk}/bos/client.go | 4 +- pkg/{sdk => cloud-sdk}/cce/client.go | 2 +- pkg/{sdk => cloud-sdk}/cce/cluster.go | 4 +- pkg/{sdk => cloud-sdk}/clientset/clientset.go | 12 +- pkg/{sdk => cloud-sdk}/eip/client.go | 2 +- pkg/{sdk => cloud-sdk}/eip/eip.go | 2 +- pkg/{sdk => cloud-sdk}/util/cidr.go | 0 pkg/{sdk => cloud-sdk}/util/mime_types.go | 0 pkg/{sdk => cloud-sdk}/util/util.go | 0 pkg/{sdk => cloud-sdk}/vpc/client.go | 2 +- pkg/{sdk => cloud-sdk}/vpc/route.go | 2 +- pkg/{sdk => cloud-sdk}/vpc/subnet.go | 2 +- pkg/{sdk => cloud-sdk}/vpc/vpc.go | 2 +- 30 files changed, 209 insertions(+), 163 deletions(-) rename pkg/{sdk => cloud-sdk}/bcc/cds.go (99%) rename pkg/{sdk => cloud-sdk}/bcc/client.go (96%) rename pkg/{sdk => cloud-sdk}/bcc/instance.go (98%) rename pkg/{sdk => cloud-sdk}/bce/core.go (99%) rename pkg/{sdk => cloud-sdk}/bce/error.go (100%) rename pkg/{sdk => cloud-sdk}/bce/request.go (98%) rename pkg/{sdk => cloud-sdk}/bce/response.go (100%) rename pkg/{sdk => cloud-sdk}/bce/types.go (100%) rename pkg/{sdk => cloud-sdk}/blb/backendserver.go (90%) rename pkg/{sdk => cloud-sdk}/blb/client.go (75%) rename pkg/{sdk => cloud-sdk}/blb/listener.go (85%) rename pkg/{sdk => cloud-sdk}/blb/loadbalancer.go (90%) rename pkg/{sdk => cloud-sdk}/bos/bucket.go (99%) rename pkg/{sdk => cloud-sdk}/bos/client.go (99%) rename pkg/{sdk => cloud-sdk}/cce/client.go (96%) rename pkg/{sdk => cloud-sdk}/cce/cluster.go (98%) rename pkg/{sdk => cloud-sdk}/clientset/clientset.go (87%) rename pkg/{sdk => cloud-sdk}/eip/client.go (95%) rename pkg/{sdk => cloud-sdk}/eip/eip.go (99%) rename pkg/{sdk => cloud-sdk}/util/cidr.go (100%) rename pkg/{sdk => cloud-sdk}/util/mime_types.go (100%) rename pkg/{sdk => cloud-sdk}/util/util.go (100%) rename pkg/{sdk => cloud-sdk}/vpc/client.go (96%) rename pkg/{sdk => cloud-sdk}/vpc/route.go (98%) rename pkg/{sdk => cloud-sdk}/vpc/subnet.go (98%) rename pkg/{sdk => cloud-sdk}/vpc/vpc.go (97%) diff --git a/pkg/cloud-provider/cce.go b/pkg/cloud-provider/cce.go index eb204e7..6214e1d 100644 --- a/pkg/cloud-provider/cce.go +++ b/pkg/cloud-provider/cce.go @@ -31,8 +31,8 @@ import ( "k8s.io/kubernetes/pkg/cloudprovider" "k8s.io/kubernetes/pkg/controller" - "k8s.io/cloud-provider-baiducloud/pkg/sdk/bce" - "k8s.io/cloud-provider-baiducloud/pkg/sdk/clientset" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/bce" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/clientset" ) // ProviderName is the name of this cloud provider. diff --git a/pkg/cloud-provider/instances.go b/pkg/cloud-provider/instances.go index 84224b7..f4607ba 100644 --- a/pkg/cloud-provider/instances.go +++ b/pkg/cloud-provider/instances.go @@ -27,7 +27,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/pkg/cloudprovider" - "k8s.io/cloud-provider-baiducloud/pkg/sdk/cce" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/cce" ) // Instances returns an instances interface. Also returns true if the interface is supported, false otherwise. diff --git a/pkg/cloud-provider/load_balancer.go b/pkg/cloud-provider/load_balancer.go index 8ab1e58..2304bb5 100644 --- a/pkg/cloud-provider/load_balancer.go +++ b/pkg/cloud-provider/load_balancer.go @@ -27,10 +27,10 @@ import ( "k8s.io/api/core/v1" "k8s.io/kubernetes/pkg/cloudprovider" - "k8s.io/cloud-provider-baiducloud/pkg/sdk/blb" - "k8s.io/cloud-provider-baiducloud/pkg/sdk/eip" - "k8s.io/cloud-provider-baiducloud/pkg/sdk/util" - "k8s.io/cloud-provider-baiducloud/pkg/sdk/vpc" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/blb" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/eip" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/util" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/vpc" ) // LoadBalancer returns a balancer interface. Also returns true if the interface is supported, false otherwise. @@ -324,9 +324,9 @@ func (bc *Baiducloud) ensureEIPWithNoSpecificIP(ctx context.Context, clusterName pubIP := lb.PublicIp if len(pubIP) == 0 { // blb not bind eip, mostly case ==> glog.V(2).Infof("[%v %v] EnsureLoadBalancer: createEIP!", service.Namespace, service.Name) - args, err := bc.getEipCreateArgsFromAnnotation(serviceAnnotation) + args, err := bc.getEipArgsFromAnnotation(serviceAnnotation) if err != nil { - glog.Errorf("[%v %v] getEipCreateArgsFromAnnotation failed: %v", service.Namespace, service.Name, err) + glog.Errorf("[%v %v] getEipArgsFromAnnotation failed: %v", service.Namespace, service.Name, err) return "", err } if len(args.Name) == 0 { @@ -369,7 +369,7 @@ func (bc *Baiducloud) ensureEIPWithNoSpecificIP(ctx context.Context, clusterName if serviceAnnotation.ElasticIPBandwidthInMbps != 0 && serviceAnnotation.ElasticIPBandwidthInMbps != targetEip.BandwidthInMbps { glog.V(3).Infof("[%v %v] EnsureLoadBalancer: EIP config change, need change ElasticIPBandwidthInMbps", service.Namespace, service.Name) // just validate args - _, err := bc.getEipCreateArgsFromAnnotation(serviceAnnotation) + _, err := bc.getEipArgsFromAnnotation(serviceAnnotation) if err != nil { glog.Errorf("[%v %v] Eip Args error: %v", service.Namespace, service.Name, err) return "", err @@ -976,7 +976,7 @@ func (bc *Baiducloud) getVpcInfoForBLB() (string, string, error) { } } -func (bc *Baiducloud) getEipCreateArgsFromAnnotation(serviceAnnotation *ServiceAnnotation) (*eip.CreateEipArgs, error) { +func (bc *Baiducloud) getEipArgsFromAnnotation(serviceAnnotation *ServiceAnnotation) (*eip.CreateEipArgs, error) { var args *eip.CreateEipArgs paymentTiming := serviceAnnotation.ElasticIPPaymentTiming @@ -1064,9 +1064,9 @@ func (bc *Baiducloud) deleteOldAndCreateNewEip(service *v1.Service, serviceAnnot return "", err } glog.V(2).Infof("[%v %v] EnsureLoadBalancer: createEIP!", service.Namespace, service.Name) - args, err := bc.getEipCreateArgsFromAnnotation(serviceAnnotation) + args, err := bc.getEipArgsFromAnnotation(serviceAnnotation) if err != nil { - glog.Errorf("[%v %v] getEipCreateArgsFromAnnotation failed: %v", service.Namespace, service.Name, err) + glog.Errorf("[%v %v] getEipArgsFromAnnotation failed: %v", service.Namespace, service.Name, err) return "", err } if len(args.Name) == 0 { diff --git a/pkg/cloud-provider/routes.go b/pkg/cloud-provider/routes.go index 9ff0591..a4e44ac 100644 --- a/pkg/cloud-provider/routes.go +++ b/pkg/cloud-provider/routes.go @@ -26,8 +26,8 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/pkg/cloudprovider" - "k8s.io/cloud-provider-baiducloud/pkg/sdk/cce" - "k8s.io/cloud-provider-baiducloud/pkg/sdk/vpc" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/cce" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/vpc" ) // Routes returns a routes interface along with whether the interface is supported. diff --git a/pkg/sdk/bcc/cds.go b/pkg/cloud-sdk/bcc/cds.go similarity index 99% rename from pkg/sdk/bcc/cds.go rename to pkg/cloud-sdk/bcc/cds.go index 01b38ac..d2f4727 100644 --- a/pkg/sdk/bcc/cds.go +++ b/pkg/cloud-sdk/bcc/cds.go @@ -21,7 +21,7 @@ import ( "encoding/json" "fmt" - "k8s.io/cloud-provider-baiducloud/pkg/sdk/bce" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/bce" ) type Volume struct { diff --git a/pkg/sdk/bcc/client.go b/pkg/cloud-sdk/bcc/client.go similarity index 96% rename from pkg/sdk/bcc/client.go rename to pkg/cloud-sdk/bcc/client.go index df4d3f2..531ce4a 100644 --- a/pkg/sdk/bcc/client.go +++ b/pkg/cloud-sdk/bcc/client.go @@ -17,7 +17,7 @@ limitations under the License. package bcc import ( - "k8s.io/cloud-provider-baiducloud/pkg/sdk/bce" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/bce" ) // Endpoint contains all endpoints of Baidu Cloud BCC. diff --git a/pkg/sdk/bcc/instance.go b/pkg/cloud-sdk/bcc/instance.go similarity index 98% rename from pkg/sdk/bcc/instance.go rename to pkg/cloud-sdk/bcc/instance.go index 13910d9..ae378aa 100644 --- a/pkg/sdk/bcc/instance.go +++ b/pkg/cloud-sdk/bcc/instance.go @@ -19,7 +19,7 @@ package bcc import ( "encoding/json" - "k8s.io/cloud-provider-baiducloud/pkg/sdk/bce" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/bce" ) const ( diff --git a/pkg/sdk/bce/core.go b/pkg/cloud-sdk/bce/core.go similarity index 99% rename from pkg/sdk/bce/core.go rename to pkg/cloud-sdk/bce/core.go index 32df60c..d312552 100644 --- a/pkg/sdk/bce/core.go +++ b/pkg/cloud-sdk/bce/core.go @@ -32,7 +32,7 @@ import ( "strings" "time" - "k8s.io/cloud-provider-baiducloud/pkg/sdk/util" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/util" ) const ( @@ -46,7 +46,7 @@ const ( // // We can change it by specifying the UserAgent field of bce.Config. var DefaultUserAgent = strings.Join([]string{ - "baidubce-sdk-go", + "baidubce-cloud-sdk-go", Version, runtime.GOOS, runtime.Version(), diff --git a/pkg/sdk/bce/error.go b/pkg/cloud-sdk/bce/error.go similarity index 100% rename from pkg/sdk/bce/error.go rename to pkg/cloud-sdk/bce/error.go diff --git a/pkg/sdk/bce/request.go b/pkg/cloud-sdk/bce/request.go similarity index 98% rename from pkg/sdk/bce/request.go rename to pkg/cloud-sdk/bce/request.go index a81fc59..62f9b46 100644 --- a/pkg/sdk/bce/request.go +++ b/pkg/cloud-sdk/bce/request.go @@ -22,7 +22,7 @@ import ( "os" "strings" - "k8s.io/cloud-provider-baiducloud/pkg/sdk/util" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/util" ) var canonicalHeaders = []string{ diff --git a/pkg/sdk/bce/response.go b/pkg/cloud-sdk/bce/response.go similarity index 100% rename from pkg/sdk/bce/response.go rename to pkg/cloud-sdk/bce/response.go diff --git a/pkg/sdk/bce/types.go b/pkg/cloud-sdk/bce/types.go similarity index 100% rename from pkg/sdk/bce/types.go rename to pkg/cloud-sdk/bce/types.go diff --git a/pkg/sdk/blb/backendserver.go b/pkg/cloud-sdk/blb/backendserver.go similarity index 90% rename from pkg/sdk/blb/backendserver.go rename to pkg/cloud-sdk/blb/backendserver.go index 5e13b95..960b91b 100644 --- a/pkg/sdk/blb/backendserver.go +++ b/pkg/cloud-sdk/blb/backendserver.go @@ -21,25 +21,54 @@ import ( "encoding/json" "fmt" - "k8s.io/cloud-provider-baiducloud/pkg/sdk/bce" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/bce" ) +// BackendServer define BackendServer type BackendServer struct { InstanceId string `json:"instanceId"` Weight int `json:"weight,omitempty"` } +// BackendServerStatus define BackendServer Status type BackendServerStatus struct { InstanceId string `json:"instanceId"` Weight int `json:"weight"` Status string `json:"status"` } +// AddBackendServersArgs is the args to add BackendServer type AddBackendServersArgs struct { LoadBalancerId string `json:"-"` BackendServerList []BackendServer `json:"backendServerList"` } +// DescribeBackendServersArgs is the args to describe BackendServer +type DescribeBackendServersArgs struct { + LoadBalancerId string `json:"-"` +} + +// DescribeBackendServersResponse is the response of DescribeBackendServers +type DescribeBackendServersResponse struct { + Marker string `json:"marker"` + IsTruncated bool `json:"isTruncated"` + NextMarker string `json:"nextMarker"` + MaxKeys int `json:"maxKeys"` + BackendServerList []BackendServer `json:"backendServerList"` +} + +// UpdateBackendServersArgs is the args to update the BackendServer +type UpdateBackendServersArgs struct { + LoadBalancerId string `json:"-"` + BackendServerList []BackendServer `json:"backendServerList"` +} + +// RemoveBackendServersArgs is the argds to remove BackendServer +type RemoveBackendServersArgs struct { + LoadBalancerId string `json:"-"` + BackendServerList []string `json:"backendServerList"` +} + func (args *AddBackendServersArgs) validate() error { if args == nil { return fmt.Errorf("AddBackendServersArgs need args") @@ -53,6 +82,8 @@ func (args *AddBackendServersArgs) validate() error { return nil } + +// AddBackendServers add BackendServers func (c *Client) AddBackendServers(args *AddBackendServersArgs) error { err := args.validate() if err != nil { @@ -76,18 +107,6 @@ func (c *Client) AddBackendServers(args *AddBackendServersArgs) error { return nil } -type DescribeBackendServersArgs struct { - LoadBalancerId string `json:"-"` -} - -type DescribeBackendServersResponse struct { - Marker string `json:"marker"` - IsTruncated bool `json:"isTruncated"` - NextMarker string `json:"nextMarker"` - MaxKeys int `json:"maxKeys"` - BackendServerList []BackendServer `json:"backendServerList"` -} - func (args *DescribeBackendServersArgs) validate() error { if args == nil { return fmt.Errorf("DescribeBackendServersArgs need args") @@ -98,6 +117,7 @@ func (args *DescribeBackendServersArgs) validate() error { return nil } +// DescribeBackendServers describe BackendServers func (c *Client) DescribeBackendServers(args *DescribeBackendServersArgs) ([]BackendServer, error) { err := args.validate() if err != nil { @@ -127,11 +147,6 @@ func (c *Client) DescribeBackendServers(args *DescribeBackendServersArgs) ([]Bac } -type UpdateBackendServersArgs struct { - LoadBalancerId string `json:"-"` - BackendServerList []BackendServer `json:"backendServerList"` -} - func (args *UpdateBackendServersArgs) validate() error { if args == nil { return fmt.Errorf("UpdateBackendServersArgs need args") @@ -170,11 +185,6 @@ func (c *Client) UpdateBackendServers(args *UpdateBackendServersArgs) error { return nil } -type RemoveBackendServersArgs struct { - LoadBalancerId string `json:"-"` - BackendServerList []string `json:"backendServerList"` -} - func (args *RemoveBackendServersArgs) validate() error { if args == nil { return fmt.Errorf("UpdateBackendServersArgs need args") diff --git a/pkg/sdk/blb/client.go b/pkg/cloud-sdk/blb/client.go similarity index 75% rename from pkg/sdk/blb/client.go rename to pkg/cloud-sdk/blb/client.go index 2d395bd..448644c 100644 --- a/pkg/sdk/blb/client.go +++ b/pkg/cloud-sdk/blb/client.go @@ -17,35 +17,40 @@ limitations under the License. package blb import ( - "k8s.io/cloud-provider-baiducloud/pkg/sdk/bce" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/bce" ) -// Endpoint contains all endpoints of Baidu Cloud BCC. +// Endpoint contains all endpoints of BLB. var Endpoint = map[string]string{ "bj": "blb.bj.baidubce.com", "gz": "blb.gz.baidubce.com", + "su": "blb.su.baidubce.com", + "hk": "blb.hkg.baidubce.com", + "bd": "blb.bd.baidubce.com", } -// Config contains all options for bos.Client. +// Config contains all options for BLB. type Config struct { *bce.Config } +// NewConfig returns BLB config func NewConfig(config *bce.Config) *Config { return &Config{config} } -// Client is the bos client implemention for Baidu Cloud BOS API. +// Client is the BLB client type Client struct { *bce.Client } +// NewBLBClient returns BLB client func NewBLBClient(config *Config) *Client { bceClient := bce.NewClient(config.Config) return &Client{bceClient} } -// GetURL generates the full URL of http request for Baidu Cloud BOS API. +// GetURL generates the full URL of http request for BLB func (c *Client) GetURL(version string, params map[string]string) string { host := c.Endpoint if host == "" { diff --git a/pkg/sdk/blb/listener.go b/pkg/cloud-sdk/blb/listener.go similarity index 85% rename from pkg/sdk/blb/listener.go rename to pkg/cloud-sdk/blb/listener.go index c29a455..8bd42bc 100644 --- a/pkg/sdk/blb/listener.go +++ b/pkg/cloud-sdk/blb/listener.go @@ -20,12 +20,12 @@ import ( "bytes" "encoding/json" "fmt" - "strconv" - "k8s.io/cloud-provider-baiducloud/pkg/sdk/bce" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/bce" ) +// TCPListener define TCPListener type TCPListener struct { ListenerPort int `json:"listenerPort"` BackendPort int `json:"backendPort"` @@ -36,6 +36,7 @@ type TCPListener struct { HealthyThreshold int `json:"healthyThreshold"` } +// UDPListener define UDPListener type UDPListener struct { ListenerPort int `json:"listenerPort"` BackendPort int `json:"backendPort"` @@ -47,6 +48,7 @@ type UDPListener struct { HealthCheckString string `json:"healthCheckString"` } +// HTTPListener define HTTPListener type HTTPListener struct { ListenerPort int `json:"listenerPort"` BackendPort int `json:"backendPort"` @@ -68,6 +70,7 @@ type HTTPListener struct { RedirectPort int `json:"redirectPort"` } +// HTTPSListener define HTTPSListener type HTTPSListener struct { ListenerPort int `json:"listenerPort"` BackendPort int `json:"backendPort"` @@ -90,6 +93,7 @@ type HTTPSListener struct { Ie6Compatible bool `json:"ie6Compatible"` } +// CreateTCPListenerArgs is the args to create TCPListener type CreateTCPListenerArgs struct { LoadBalancerId string `json:"-"` ListenerPort int `json:"listenerPort"` @@ -101,6 +105,7 @@ type CreateTCPListenerArgs struct { HealthyThreshold int `json:"healthyThreshold,omitempty"` } +// CreateUDPListenerArgs is the args to create UDPListener type CreateUDPListenerArgs struct { LoadBalancerId string `json:"-"` ListenerPort int `json:"listenerPort"` @@ -113,6 +118,7 @@ type CreateUDPListenerArgs struct { HealthCheckString string `json:"healthCheckString"` } +// CreateHTTPListenerArgs is the args to create HTTPListener type CreateHTTPListenerArgs struct { LoadBalancerId string `json:"-"` ListenerPort int `json:"listenerPort"` @@ -135,8 +141,92 @@ type CreateHTTPListenerArgs struct { RedirectPort int `json:"redirectPort,omitempty"` } -// CreateLoadBalancerHTTPListener create HTTP listener on loadbalancer -// You can read doc at https://cloud.baidu.com/doc/BLB/API.html#.C0.F3.F3.ED.5C.D8.4D.66.19.FF.DA.7A.0F.75.05.7C +// CreateHTTPSListenerArgs is the args to create HTTPSListener +type CreateHTTPSListenerArgs struct { + LoadBalancerId string `json:"-"` + ListenerPort int `json:"listenerPort"` + BackendPort int `json:"backendPort"` + Scheduler string `json:"scheduler"` + KeepSession bool `json:"keepSession,omitempty"` + KeepSessionType string `json:"keepSessionType,omitempty"` + KeepSessionDuration int `json:"keepSessionDuration,omitempty"` + KeepSessionCookieName int `json:"keepSessionCookieName,omitempty"` + XForwardFor bool `json:"xForwardFor,omitempty"` + HealthCheckType string `json:"healthCheckType,omitempty"` + HealthCheckPort int `json:"healthCheckPort,omitempty"` + HealthCheckURI string `json:"healthCheckURI,omitempty"` + HealthCheckTimeoutInSecond int `json:"healthCheckTimeoutInSecond,omitempty"` + HealthCheckInterval int `json:"healthCheckInterval,omitempty"` + UnhealthyThreshold int `json:"unhealthyThreshold,omitempty"` + HealthyThreshold int `json:"healthyThreshold,omitempty"` + HealthCheckNormalStatus string `json:"healthCheckNormalStatus,omitempty"` + ServerTimeout int `json:"serverTimeout,omitempty"` + CertIds []int `json:"certIds,omitempty"` + Ie6Compatible bool `json:"ie6Compatible"` +} + +// DescribeTCPListenerArgs is the args to describe TCPListener +type DescribeTCPListenerArgs struct { + LoadBalancerId string + ListenerPort int +} + +// DescribeTCPListenerResponse is the response of DescribeTCPListener +type DescribeTCPListenerResponse struct { + Marker string `json:"marker"` + IsTruncated bool `json:"isTruncated"` + NextMarker string `json:"nextMarker"` + MaxKeys int `json:"maxKeys"` + TCPListenerList []TCPListener `json:"listenerList"` +} + +// DescribeUDPListenerArgs is the args to describe UDPListener +type DescribeUDPListenerArgs struct { + LoadBalancerId string + ListenerPort int +} + +// DescribeUDPListenerResponse is the response of DescribeUDPListener +type DescribeUDPListenerResponse struct { + Marker string `json:"marker"` + IsTruncated bool `json:"isTruncated"` + NextMarker string `json:"nextMarker"` + MaxKeys int `json:"maxKeys"` + UDPListenerList []UDPListener `json:"listenerList"` +} + +// UpdateTCPListenerArgs is the args to UpdateTCPListener +type UpdateTCPListenerArgs struct { + LoadBalancerId string `json:"-"` + ListenerPort int `json:"-"` + BackendPort int `json:"backendPort,omitempty"` + Scheduler string `json:"scheduler,omitempty"` + HealthCheckTimeoutInSecond int `json:"healthCheckTimeoutInSecond,omitempty"` + HealthCheckInterval int `json:"healthCheckInterval,omitempty"` + UnhealthyThreshold int `json:"unhealthyThreshold,omitempty"` + HealthyThreshold int `json:"healthyThreshold,omitempty"` +} + +// UpdateUDPListenerArgs is the args to update UDPListener +type UpdateUDPListenerArgs struct { + LoadBalancerId string `json:"-"` + ListenerPort int `json:"listenerPort"` + BackendPort int `json:"backendPort"` + Scheduler string `json:"scheduler"` + HealthCheckTimeoutInSecond int `json:"healthCheckTimeoutInSecond,omitempty"` + HealthCheckInterval int `json:"healthCheckInterval,omitempty"` + UnhealthyThreshold int `json:"unhealthyThreshold,omitempty"` + HealthyThreshold int `json:"healthyThreshold,omitempty"` + HealthCheckString string `json:"healthCheckString"` +} + +// DeleteListenersArgs is the args to DeleteListeners +type DeleteListenersArgs struct { + LoadBalancerId string `json:"-"` + PortList []int `json:"portList"` +} + +// CreateTCPListener create TCP listener on loadbalancer func (c *Client) CreateTCPListener(args *CreateTCPListenerArgs) (err error) { params := map[string]string{ "clientToken": c.GenerateClientToken(), @@ -159,9 +249,7 @@ func (c *Client) CreateTCPListener(args *CreateTCPListenerArgs) (err error) { return nil } -// CreateLoadBalancerUDPListener create UDP listener on loadbalancer -// -// You can read doc at https://cloud.baidu.com/doc/BLB/API.html#.D7.A3.9B.E1.45.BD.9E.FA.B0.2F.60.12.B3.39.E8.9D +// CreateUDPListener create UDP listener on loadbalancer func (c *Client) CreateUDPListener(args *CreateUDPListenerArgs) (err error) { params := map[string]string{ "clientToken": c.GenerateClientToken(), @@ -185,8 +273,6 @@ func (c *Client) CreateUDPListener(args *CreateUDPListenerArgs) (err error) { } // CreateHTTPListener create HTTP listener on loadbalancer -// -// You can read doc at https://cloud.baidu.com/doc/BLB/API.html#.D7.A3.9B.E1.45.BD.9E.FA.B0.2F.60.12.B3.39.E8.9D func (c *Client) CreateHTTPListener(args *CreateHTTPListenerArgs) (err error) { params := map[string]string{ "clientToken": c.GenerateClientToken(), @@ -209,20 +295,7 @@ func (c *Client) CreateHTTPListener(args *CreateHTTPListenerArgs) (err error) { return nil } -type DescribeTCPListenerArgs struct { - LoadBalancerId string - ListenerPort int -} -type DescribeTCPListenerResponse struct { - Marker string `json:"marker"` - IsTruncated bool `json:"isTruncated"` - NextMarker string `json:"nextMarker"` - MaxKeys int `json:"maxKeys"` - TCPListenerList []TCPListener `json:"listenerList"` -} - // DescribeTCPListener Describe TCPListener -// TODO: args need to validate func (c *Client) DescribeTCPListener(args *DescribeTCPListenerArgs) ([]TCPListener, error) { if args == nil { return nil, fmt.Errorf("DescribeTCPListener need args") @@ -257,20 +330,7 @@ func (c *Client) DescribeTCPListener(args *DescribeTCPListenerArgs) ([]TCPListen return blbsResp.TCPListenerList, nil } -type DescribeUDPListenerArgs struct { - LoadBalancerId string - ListenerPort int -} -type DescribeUDPListenerResponse struct { - Marker string `json:"marker"` - IsTruncated bool `json:"isTruncated"` - NextMarker string `json:"nextMarker"` - MaxKeys int `json:"maxKeys"` - UDPListenerList []UDPListener `json:"listenerList"` -} - // DescribeUDPListeners Describe UDPListeners -// TODO: args need to validate func (c *Client) DescribeUDPListener(args *DescribeUDPListenerArgs) ([]UDPListener, error) { if args == nil { return nil, fmt.Errorf("DescribeUDPListeners need args") @@ -305,19 +365,7 @@ func (c *Client) DescribeUDPListener(args *DescribeUDPListenerArgs) ([]UDPListen return blbsResp.UDPListenerList, nil } -type UpdateTCPListenerArgs struct { - LoadBalancerId string `json:"-"` - ListenerPort int `json:"-"` - BackendPort int `json:"backendPort,omitempty"` - Scheduler string `json:"scheduler,omitempty"` - HealthCheckTimeoutInSecond int `json:"healthCheckTimeoutInSecond,omitempty"` - HealthCheckInterval int `json:"healthCheckInterval,omitempty"` - UnhealthyThreshold int `json:"unhealthyThreshold,omitempty"` - HealthyThreshold int `json:"healthyThreshold,omitempty"` -} - // UpdateTCPListener update a TCPListener -// TODO: args need to validate func (c *Client) UpdateTCPListener(args *UpdateTCPListenerArgs) error { if args == nil || args.LoadBalancerId == "" || args.ListenerPort == 0 { @@ -344,18 +392,6 @@ func (c *Client) UpdateTCPListener(args *UpdateTCPListenerArgs) error { return nil } -type UpdateUDPListenerArgs struct { - LoadBalancerId string `json:"-"` - ListenerPort int `json:"listenerPort"` - BackendPort int `json:"backendPort"` - Scheduler string `json:"scheduler"` - HealthCheckTimeoutInSecond int `json:"healthCheckTimeoutInSecond,omitempty"` - HealthCheckInterval int `json:"healthCheckInterval,omitempty"` - UnhealthyThreshold int `json:"unhealthyThreshold,omitempty"` - HealthyThreshold int `json:"healthyThreshold,omitempty"` - HealthCheckString string `json:"healthCheckString"` -} - func (args *UpdateUDPListenerArgs) validate() error { if args.LoadBalancerId == "" { return fmt.Errorf("UpdateUDPListener need LoadBalancerId") @@ -399,12 +435,6 @@ func (c *Client) UpdateUDPListener(args *UpdateUDPListenerArgs) error { return nil } -type DeleteListenersArgs struct { - LoadBalancerId string `json:"-"` - // action string `json:"-"` - PortList []int `json:"portList"` -} - func (args *DeleteListenersArgs) validate() error { if args.LoadBalancerId == "" { return fmt.Errorf("DeleteListenersArgs need LoadBalancerId") @@ -415,7 +445,7 @@ func (args *DeleteListenersArgs) validate() error { return nil } -// UpdateUDPListener update a UDPListener +// DeleteListeners delete a Listener func (c *Client) DeleteListeners(args *DeleteListenersArgs) error { err := args.validate() if err != nil { @@ -429,7 +459,6 @@ func (c *Client) DeleteListeners(args *DeleteListenersArgs) error { if err != nil { return err } - // url := "http://" + Endpoint[c.GetRegion()] + "/v1/blb" + "/" + args.LoadBalancerId + "/listener?" + "batchdelete=&" + "clientToken=" + c.GenerateClientToken() req, err := bce.NewRequest("PUT", c.GetURL("v1/blb"+"/"+args.LoadBalancerId+"/listener", params), bytes.NewBuffer(postContent)) if err != nil { return err diff --git a/pkg/sdk/blb/loadbalancer.go b/pkg/cloud-sdk/blb/loadbalancer.go similarity index 90% rename from pkg/sdk/blb/loadbalancer.go rename to pkg/cloud-sdk/blb/loadbalancer.go index 94c0ae3..f08126b 100644 --- a/pkg/sdk/blb/loadbalancer.go +++ b/pkg/cloud-sdk/blb/loadbalancer.go @@ -19,13 +19,13 @@ package blb import ( "bytes" "encoding/json" - "fmt" - "k8s.io/cloud-provider-baiducloud/pkg/sdk/bce" - "k8s.io/cloud-provider-baiducloud/pkg/sdk/util" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/bce" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/util" ) +// LoadBalancer define LoadBalancer type LoadBalancer struct { BlbId string `json:"blbId"` Name string `json:"name"` @@ -35,6 +35,7 @@ type LoadBalancer struct { PublicIp string `json:"publicIp"` } +// DescribeLoadBalancersArgs is the args to describe LoadBalancer type DescribeLoadBalancersArgs struct { LoadBalancerId string LoadBalancerName string @@ -42,6 +43,7 @@ type DescribeLoadBalancersArgs struct { Address string } +// DescribeLoadBalancersResponse is the response of DescribeLoadBalancers type DescribeLoadBalancersResponse struct { Marker string `json:"marker"` IsTruncated bool `json:"isTruncated"` @@ -59,6 +61,7 @@ type CreateLoadBalancerArgs struct { AllocateVip bool `json:"allocateVip,omitempty"` } +// CreateLoadBalancerResponse is the response of CreateLoadBalancer type CreateLoadBalancerResponse struct { LoadBalancerId string `json:"blbId"` Address string `json:"address"` @@ -66,88 +69,87 @@ type CreateLoadBalancerResponse struct { Name string `json:"name"` } +// UpdateLoadBalancerArgs is the args to UpdateLoadBalancer type UpdateLoadBalancerArgs struct { LoadBalancerId string `json:"blbId"` Desc string `json:"desc,omitempty"` Name string `json:"name,omitempty"` } +// DeleteLoadBalancerArgs is the args to delete LoadBalancer type DeleteLoadBalancerArgs struct { LoadBalancerId string `json:"blbId"` } -// DescribeLoadBalancers Describe loadbalancers -// TODO: args need to validate -func (c *Client) DescribeLoadBalancers(args *DescribeLoadBalancersArgs) ([]LoadBalancer, error) { +// CreateLoadBalancer Create a loadbalancer +func (c *Client) CreateLoadBalancer(args *CreateLoadBalancerArgs) (*CreateLoadBalancerResponse, error) { var params map[string]string if args != nil { params = map[string]string{ - "blbId": args.LoadBalancerId, - "name": args.LoadBalancerName, - "bccId": args.BCCId, - "address": args.Address, + "clientToken": c.GenerateClientToken(), } } - req, err := bce.NewRequest("GET", c.GetURL("v1/blb", params), nil) - + postContent, err := json.Marshal(args) + if err != nil { + return nil, err + } + req, err := bce.NewRequest("POST", c.GetURL("v1/blb", params), bytes.NewBuffer(postContent)) if err != nil { return nil, err } resp, err := c.SendRequest(req, nil) - if err != nil { return nil, err } bodyContent, err := resp.GetBodyContent() - if err != nil { return nil, err } - var blbsResp *DescribeLoadBalancersResponse + var blbsResp *CreateLoadBalancerResponse err = json.Unmarshal(bodyContent, &blbsResp) if err != nil { return nil, err } - return blbsResp.BLBList, nil + return blbsResp, nil } -// CreateLoadBalancer Create a loadbalancer -// TODO: args need to validate -func (c *Client) CreateLoadBalancer(args *CreateLoadBalancerArgs) (*CreateLoadBalancerResponse, error) { +// DescribeLoadBalancers Describe loadbalancers +func (c *Client) DescribeLoadBalancers(args *DescribeLoadBalancersArgs) ([]LoadBalancer, error) { var params map[string]string if args != nil { params = map[string]string{ - "clientToken": c.GenerateClientToken(), + "blbId": args.LoadBalancerId, + "name": args.LoadBalancerName, + "bccId": args.BCCId, + "address": args.Address, } } - postContent, err := json.Marshal(args) - if err != nil { - return nil, err - } - req, err := bce.NewRequest("POST", c.GetURL("v1/blb", params), bytes.NewBuffer(postContent)) + req, err := bce.NewRequest("GET", c.GetURL("v1/blb", params), nil) + if err != nil { return nil, err } resp, err := c.SendRequest(req, nil) + if err != nil { return nil, err } bodyContent, err := resp.GetBodyContent() + if err != nil { return nil, err } - var blbsResp *CreateLoadBalancerResponse + var blbsResp *DescribeLoadBalancersResponse err = json.Unmarshal(bodyContent, &blbsResp) if err != nil { return nil, err } - return blbsResp, nil + return blbsResp.BLBList, nil } // UpdateLoadBalancer update a loadbalancer -// TODO: args need to validate func (c *Client) UpdateLoadBalancer(args *UpdateLoadBalancerArgs) error { params := map[string]string{ "clientToken": c.GenerateClientToken(), diff --git a/pkg/sdk/bos/bucket.go b/pkg/cloud-sdk/bos/bucket.go similarity index 99% rename from pkg/sdk/bos/bucket.go rename to pkg/cloud-sdk/bos/bucket.go index 01d623a..51a9b52 100644 --- a/pkg/sdk/bos/bucket.go +++ b/pkg/cloud-sdk/bos/bucket.go @@ -26,8 +26,8 @@ import ( "strings" "time" - "k8s.io/cloud-provider-baiducloud/pkg/sdk/bce" - "k8s.io/cloud-provider-baiducloud/pkg/sdk/util" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/bce" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/util" ) // MIN_PART_NUMBER is the min part number for multipart upload. diff --git a/pkg/sdk/bos/client.go b/pkg/cloud-sdk/bos/client.go similarity index 99% rename from pkg/sdk/bos/client.go rename to pkg/cloud-sdk/bos/client.go index 9e13117..4cfac93 100644 --- a/pkg/sdk/bos/client.go +++ b/pkg/cloud-sdk/bos/client.go @@ -29,8 +29,8 @@ import ( "strings" "sync" - "k8s.io/cloud-provider-baiducloud/pkg/sdk/bce" - "k8s.io/cloud-provider-baiducloud/pkg/sdk/util" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/bce" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/util" ) // Endpoint contains all endpoints of Baidu Cloud BOS. diff --git a/pkg/sdk/cce/client.go b/pkg/cloud-sdk/cce/client.go similarity index 96% rename from pkg/sdk/cce/client.go rename to pkg/cloud-sdk/cce/client.go index 05b6149..5885c1f 100644 --- a/pkg/sdk/cce/client.go +++ b/pkg/cloud-sdk/cce/client.go @@ -17,7 +17,7 @@ limitations under the License. package cce import ( - "k8s.io/cloud-provider-baiducloud/pkg/sdk/bce" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/bce" ) // Endpoint contains all endpoints of Baidu Cloud BCC. diff --git a/pkg/sdk/cce/cluster.go b/pkg/cloud-sdk/cce/cluster.go similarity index 98% rename from pkg/sdk/cce/cluster.go rename to pkg/cloud-sdk/cce/cluster.go index 21e3098..4141dae 100644 --- a/pkg/sdk/cce/cluster.go +++ b/pkg/cloud-sdk/cce/cluster.go @@ -21,8 +21,8 @@ import ( "encoding/json" "fmt" - "k8s.io/cloud-provider-baiducloud/pkg/sdk/bcc" - "k8s.io/cloud-provider-baiducloud/pkg/sdk/bce" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/bcc" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/bce" ) const ( diff --git a/pkg/sdk/clientset/clientset.go b/pkg/cloud-sdk/clientset/clientset.go similarity index 87% rename from pkg/sdk/clientset/clientset.go rename to pkg/cloud-sdk/clientset/clientset.go index 354ce2e..fda2e7a 100644 --- a/pkg/sdk/clientset/clientset.go +++ b/pkg/cloud-sdk/clientset/clientset.go @@ -19,12 +19,12 @@ package clientset import ( "fmt" - "k8s.io/cloud-provider-baiducloud/pkg/sdk/bcc" - "k8s.io/cloud-provider-baiducloud/pkg/sdk/bce" - "k8s.io/cloud-provider-baiducloud/pkg/sdk/blb" - "k8s.io/cloud-provider-baiducloud/pkg/sdk/cce" - "k8s.io/cloud-provider-baiducloud/pkg/sdk/eip" - "k8s.io/cloud-provider-baiducloud/pkg/sdk/vpc" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/bcc" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/bce" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/blb" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/cce" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/eip" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/vpc" ) // Interface contains all methods of clients diff --git a/pkg/sdk/eip/client.go b/pkg/cloud-sdk/eip/client.go similarity index 95% rename from pkg/sdk/eip/client.go rename to pkg/cloud-sdk/eip/client.go index 792085b..6f31c0f 100644 --- a/pkg/sdk/eip/client.go +++ b/pkg/cloud-sdk/eip/client.go @@ -16,7 +16,7 @@ limitations under the License. package eip -import "k8s.io/cloud-provider-baiducloud/pkg/sdk/bce" +import "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/bce" // Endpoint contains all endpoints of Baidu Cloud BCC. var Endpoint = map[string]string{ diff --git a/pkg/sdk/eip/eip.go b/pkg/cloud-sdk/eip/eip.go similarity index 99% rename from pkg/sdk/eip/eip.go rename to pkg/cloud-sdk/eip/eip.go index 20e26e4..958544c 100644 --- a/pkg/sdk/eip/eip.go +++ b/pkg/cloud-sdk/eip/eip.go @@ -21,7 +21,7 @@ import ( "encoding/json" "fmt" - "k8s.io/cloud-provider-baiducloud/pkg/sdk/bce" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/bce" ) type Eip struct { diff --git a/pkg/sdk/util/cidr.go b/pkg/cloud-sdk/util/cidr.go similarity index 100% rename from pkg/sdk/util/cidr.go rename to pkg/cloud-sdk/util/cidr.go diff --git a/pkg/sdk/util/mime_types.go b/pkg/cloud-sdk/util/mime_types.go similarity index 100% rename from pkg/sdk/util/mime_types.go rename to pkg/cloud-sdk/util/mime_types.go diff --git a/pkg/sdk/util/util.go b/pkg/cloud-sdk/util/util.go similarity index 100% rename from pkg/sdk/util/util.go rename to pkg/cloud-sdk/util/util.go diff --git a/pkg/sdk/vpc/client.go b/pkg/cloud-sdk/vpc/client.go similarity index 96% rename from pkg/sdk/vpc/client.go rename to pkg/cloud-sdk/vpc/client.go index 0ab241e..bd8ba6e 100644 --- a/pkg/sdk/vpc/client.go +++ b/pkg/cloud-sdk/vpc/client.go @@ -17,7 +17,7 @@ limitations under the License. package vpc import ( - "k8s.io/cloud-provider-baiducloud/pkg/sdk/bce" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/bce" ) // Endpoint contains all endpoints of Baidu Cloud BCC. diff --git a/pkg/sdk/vpc/route.go b/pkg/cloud-sdk/vpc/route.go similarity index 98% rename from pkg/sdk/vpc/route.go rename to pkg/cloud-sdk/vpc/route.go index 33191ce..75ca730 100644 --- a/pkg/sdk/vpc/route.go +++ b/pkg/cloud-sdk/vpc/route.go @@ -21,7 +21,7 @@ import ( "encoding/json" "fmt" - "k8s.io/cloud-provider-baiducloud/pkg/sdk/bce" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/bce" ) // RouteRule define route diff --git a/pkg/sdk/vpc/subnet.go b/pkg/cloud-sdk/vpc/subnet.go similarity index 98% rename from pkg/sdk/vpc/subnet.go rename to pkg/cloud-sdk/vpc/subnet.go index 583664e..4112768 100644 --- a/pkg/sdk/vpc/subnet.go +++ b/pkg/cloud-sdk/vpc/subnet.go @@ -21,7 +21,7 @@ import ( "encoding/json" "fmt" - "k8s.io/cloud-provider-baiducloud/pkg/sdk/bce" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/bce" ) // Subnet define subnet of vpc diff --git a/pkg/sdk/vpc/vpc.go b/pkg/cloud-sdk/vpc/vpc.go similarity index 97% rename from pkg/sdk/vpc/vpc.go rename to pkg/cloud-sdk/vpc/vpc.go index 962a5af..ae3612f 100644 --- a/pkg/sdk/vpc/vpc.go +++ b/pkg/cloud-sdk/vpc/vpc.go @@ -20,7 +20,7 @@ import ( "encoding/json" "strconv" - "k8s.io/cloud-provider-baiducloud/pkg/sdk/bce" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/bce" ) // Vpc type define From 5acb921d4391388045eccbcdedfec5628de0fc41 Mon Sep 17 00:00:00 2001 From: hello2mao Date: Tue, 26 Feb 2019 10:13:52 +0800 Subject: [PATCH 6/7] extract load_balancer.go to parts --- pkg/cloud-provider/load_balancer.go | 921 +----------------- pkg/cloud-provider/load_balancer_blb.go | 134 +++ .../load_balancer_blb_backend_server.go | 88 ++ .../load_balancer_blb_listener.go | 199 ++++ pkg/cloud-provider/load_balancer_eip.go | 423 ++++++++ pkg/cloud-provider/load_balancer_util.go | 93 ++ pkg/cloud-provider/load_balancer_vpc.go | 105 ++ pkg/cloud-sdk/blb/backendserver.go | 2 +- pkg/cloud-sdk/blb/listener.go | 6 +- 9 files changed, 1047 insertions(+), 924 deletions(-) create mode 100644 pkg/cloud-provider/load_balancer_blb.go create mode 100644 pkg/cloud-provider/load_balancer_blb_backend_server.go create mode 100644 pkg/cloud-provider/load_balancer_blb_listener.go create mode 100644 pkg/cloud-provider/load_balancer_eip.go create mode 100644 pkg/cloud-provider/load_balancer_util.go create mode 100644 pkg/cloud-provider/load_balancer_vpc.go diff --git a/pkg/cloud-provider/load_balancer.go b/pkg/cloud-provider/load_balancer.go index 2304bb5..9000b1e 100644 --- a/pkg/cloud-provider/load_balancer.go +++ b/pkg/cloud-provider/load_balancer.go @@ -18,19 +18,12 @@ package cloud_provider import ( "context" - "fmt" - "net" - "strings" - "time" "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/kubernetes/pkg/cloudprovider" "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/blb" - "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/eip" - "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/util" - "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/vpc" ) // LoadBalancer returns a balancer interface. Also returns true if the interface is supported, false otherwise. @@ -178,916 +171,4 @@ func (bc *Baiducloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName } glog.V(2).Infof("[%v %v] EnsureLoadBalancerDeleted: delete %v FINISH", serviceName, clusterName, serviceName) return nil -} - -// workaround to support old version, can be removed if not support old version -func (bc *Baiducloud) workAround(service *v1.Service) { - lb, exists, err := bc.getBCELoadBalancer(cloudprovider.GetLoadBalancerName(service)) - if err != nil { - return - } - if !exists { - return - } - if service.Annotations == nil { - service.Annotations = make(map[string]string) - } - // TODO: 不会更新最终Service的annotaion，因为ip没变 - service.Annotations[ServiceAnnotationLoadBalancerId] = lb.BlbId - glog.V(2).Infof("[%v %v] WorkAround for old version, lb: %v", service.Namespace, service.Name, lb) -} - -func (bc *Baiducloud) validateService(service *v1.Service) error { - if len(service.Spec.Ports) == 0 { - return fmt.Errorf("requested load balancer with no ports") - } - for _, port := range service.Spec.Ports { - if port.Protocol != v1.ProtocolTCP && port.Protocol != v1.ProtocolUDP { - return fmt.Errorf("only TCP,UDP LoadBalancer is supported for Baidu CCE") - } - } - return nil -} - -func (bc *Baiducloud) ensureBLB(ctx context.Context, clusterName string, service *v1.Service, nodes []*v1.Node, serviceAnnotation *ServiceAnnotation) (*blb.LoadBalancer, error) { - var lb *blb.LoadBalancer - var err error - if len(serviceAnnotation.LoadBalancerId) == 0 { // blb not exist, create one and update annotation - glog.V(3).Infof("[%v %v] EnsureLoadBalancer create blb!", service.Namespace, service.Name) - vpcId, subnetId, err := bc.getVpcInfoForBLB() - if err != nil { - return nil, fmt.Errorf("Can't get VPC info for BLB: %v\n", err) - } - allocateVip := false - if serviceAnnotation.LoadBalancerAllocateVip == "true" { - allocateVip = true - } - args := blb.CreateLoadBalancerArgs{ - Name: bc.ClusterID + "/" + getServiceName(service), - VpcID: vpcId, - SubnetID: subnetId, - Desc: "auto generated by cce:" + bc.ClusterID, - AllocateVip: allocateVip, - } - glog.V(3).Infof("[%v %v] EnsureLoadBalancer create blb args: %v", service.Namespace, service.Name, args) - resp, err := bc.clientSet.Blb().CreateLoadBalancer(&args) - if err != nil { - return nil, err - } - glog.V(3).Infof("[%v %v] EnsureLoadBalancer create blb success, BLB name: %s, BLB id: %s, BLB address: %s.", service.Namespace, service.Name, resp.Name, resp.LoadBalancerId, resp.Address) - argsDesc := blb.DescribeLoadBalancersArgs{ - LoadBalancerId: resp.LoadBalancerId, - } - lbs, err := bc.clientSet.Blb().DescribeLoadBalancers(&argsDesc) - if err != nil { - return nil, err - } - if len(lbs) != 1 { - tryCount := 0 - for { - tryCount ++ - if tryCount > 10 { - return nil, fmt.Errorf("EnsureLoadBalancer create blb success but query get none") - } - glog.V(3).Infof("[%v %v] EnsureLoadBalancer create blb success but query get none, tryCount: ", service.Namespace, service.Name, tryCount) - lbs, err = bc.clientSet.Blb().DescribeLoadBalancers(&argsDesc) - if err != nil { - return nil, err - } - if len(lbs) == 1 { - glog.V(3).Infof("[%v %v] EnsureLoadBalancer create blb success and query get one, tryCount: ", service.Namespace, service.Name, tryCount) - break - } - time.Sleep(10 * time.Second) - } - } - lb = &lbs[0] - if service.Annotations == nil { - service.Annotations = make(map[string]string) - } - service.Annotations[ServiceAnnotationLoadBalancerId] = lb.BlbId - } else { // blb already exist, get info from cloud - var exists bool - lb, exists, err = bc.getBCELoadBalancerById(serviceAnnotation.LoadBalancerId) - if err != nil { - return nil, err - } - if !exists { - return nil, fmt.Errorf("EnsureLoadBalancer getBCELoadBalancerById failed, target blb not exist, blb id: %v", serviceAnnotation.LoadBalancerId) - } - glog.V(3).Infof("[%v %v] EnsureLoadBalancer: blb already exists: %v", service.Namespace, service.Name, lb) - } - lb, err = bc.waitForLoadBalancer(lb) - if err != nil { - return nil, err - } - - // update listener - glog.V(2).Infof("[%v %v] EnsureLoadBalancer: reconcileListeners!", service.Namespace, service.Name) - err = bc.reconcileListeners(service, lb) - if err != nil { - return nil, err - } - lb, err = bc.waitForLoadBalancer(lb) - if err != nil { - return nil, err - } - - // update backend server - glog.V(2).Infof("[%v %v] EnsureLoadBalancer: reconcileBackendServers!", service.Namespace, service.Name) - err = bc.reconcileBackendServers(nodes, lb) - if err != nil { - return nil, err - } - lb, err = bc.waitForLoadBalancer(lb) - if err != nil { - return nil, err - } - - return lb, nil -} - -func (bc *Baiducloud) ensureEIP(ctx context.Context, clusterName string, service *v1.Service, nodes []*v1.Node, serviceAnnotation *ServiceAnnotation, lb *blb.LoadBalancer) (string, error) { - if lb == nil { - return "", fmt.Errorf("[%v %v] EnsureLoadBalancer: ensureEIP need not nil lb", service.Namespace, service.Name) - } - if len(service.Spec.LoadBalancerIP) == 0 { - // not set LoadBalancerIP - return bc.ensureEIPWithNoSpecificIP(ctx, clusterName, service, nodes, serviceAnnotation, lb) - } else { - // set LoadBalancerIP - return bc.ensureEIPWithSpecificIP(ctx, clusterName, service, nodes, serviceAnnotation, lb) - } -} - -func (bc *Baiducloud) ensureEIPWithNoSpecificIP(ctx context.Context, clusterName string, service *v1.Service, nodes []*v1.Node, serviceAnnotation *ServiceAnnotation, lb *blb.LoadBalancer) (string, error) { - pubIP := lb.PublicIp - if len(pubIP) == 0 { // blb not bind eip, mostly case ==> - glog.V(2).Infof("[%v %v] EnsureLoadBalancer: createEIP!", service.Namespace, service.Name) - args, err := bc.getEipArgsFromAnnotation(serviceAnnotation) - if err != nil { - glog.Errorf("[%v %v] getEipArgsFromAnnotation failed: %v", service.Namespace, service.Name, err) - return "", err - } - if len(args.Name) == 0 { - args.Name = lb.Name // default EIP name = lb name - } - pubIP, err = bc.createEIP(args, lb) - if err != nil { - if pubIP != "" { - args := eip.EipArgs{ - Ip: pubIP, - } - bc.clientSet.Eip().DeleteEip(&args) - } - return "", err - } - } else { // blb already bind eip - glog.V(3).Infof("[%v %v] EnsureLoadBalancer: blb's eip already exists, start to ensure...", service.Namespace, service.Name) - argsGet := eip.GetEipsArgs{ - Ip: pubIP, - } - eips, err := bc.clientSet.Eip().GetEips(&argsGet) - if err != nil { - return "", err - } - if len(eips) == 0 { - err = fmt.Errorf("[%v %v] EnsureLoadBalancer: EIP %s not Exist", service.Namespace, service.Name, pubIP) - return "", err - } - targetEip := eips[0] - if (len(serviceAnnotation.ElasticIPPaymentTiming) != 0 && serviceAnnotation.ElasticIPPaymentTiming != targetEip.PaymentTiming) || - (len(serviceAnnotation.ElasticIPBillingMethod) != 0 && serviceAnnotation.ElasticIPBillingMethod != targetEip.BillingMethod) { - glog.V(3).Infof("[%v %v] EnsureLoadBalancer: EIP config change, need delete old eip and create new one", service.Namespace, service.Name) - // TODO - //pubIP, err = bc.deleteOldAndCreateNewEip(service, serviceAnnotation, pubIP, lb) - //if err != nil { - // return "", err - //} - return "", fmt.Errorf("not support change ElasticIP PaymentTiming or ElasticIP BillingMethod, you can delete old and create a new one") - } - if serviceAnnotation.ElasticIPBandwidthInMbps != 0 && serviceAnnotation.ElasticIPBandwidthInMbps != targetEip.BandwidthInMbps { - glog.V(3).Infof("[%v %v] EnsureLoadBalancer: EIP config change, need change ElasticIPBandwidthInMbps", service.Namespace, service.Name) - // just validate args - _, err := bc.getEipArgsFromAnnotation(serviceAnnotation) - if err != nil { - glog.Errorf("[%v %v] Eip Args error: %v", service.Namespace, service.Name, err) - return "", err - } - err = bc.resizeEip(service, serviceAnnotation, pubIP) - if err != nil { - return "", err - } - } - } - return pubIP, nil -} - -func (bc *Baiducloud) ensureEIPWithSpecificIP(ctx context.Context, clusterName string, service *v1.Service, nodes []*v1.Node, serviceAnnotation *ServiceAnnotation, lb *blb.LoadBalancer) (string, error) { - pubIP := lb.PublicIp - loadBalancerIP := service.Spec.LoadBalancerIP - glog.V(3).Infof("[%v %v] EnsureLoadBalancer: Try to bind Custom LoadBalancerIP %s to BLB %s.", service.Namespace, service.Name, loadBalancerIP, lb.BlbId) - if len(pubIP) == 0 { // blb not bind target eip - // check eip status - argsGet := eip.GetEipsArgs{ - Ip: loadBalancerIP, - } - eips, err := bc.clientSet.Eip().GetEips(&argsGet) - if err != nil { - return "", err - } - if len(eips) == 0 { - err = fmt.Errorf("[%v %v] EnsureLoadBalancer: EIP %s not Exist", service.Namespace, service.Name, loadBalancerIP) - return "", err - } else { - eipStatus := eips[0].Status - for index := 0; (index < 10) && (eipStatus != "available"); index++ { - glog.V(3).Infof("[%v %v] Eip: %s is not available, retry: %d", service.Namespace, service.Name, loadBalancerIP, index) - time.Sleep(10 * time.Second) - eips, err := bc.clientSet.Eip().GetEips(&argsGet) - if err != nil { - return "", err - } - eipStatus = eips[0].Status - } - glog.V(3).Infof("Eip final status is: %s", eipStatus) - if eipStatus != "available" { - return "", fmt.Errorf("[%v %v] EnsureLoadBalancer: target eip not available", service.Namespace, service.Name) - } - } - - // bind - lb.Status = "unknown" // add here to do loop - for index := 0; (index < 10) && (lb.Status != "available"); index++ { - glog.V(3).Infof("[%v %v] BLB: %s is not available, retry: %d", service.Namespace, service.Name, lb.BlbId, index) - time.Sleep(10 * time.Second) - newlb, exist, err := bc.getBCELoadBalancerById(lb.BlbId) - if err != nil { - glog.V(3).Infof("getBCELoadBalancer error: %s", lb.BlbId) - return "", err - } - if !exist { - glog.V(3).Infof("getBCELoadBalancer not exist: %s", lb.BlbId) - return "", fmt.Errorf("BLB not exists:%s", lb.BlbId) - } - lb = newlb - glog.V(3).Infof("[%v %v] BLB status is : %s", service.Namespace, service.Name, lb.Status) - } - argsBind := &eip.BindEipArgs{ - Ip: loadBalancerIP, - InstanceId: lb.BlbId, - InstanceType: eip.BLB, - } - glog.V(3).Infof("[%v %v] Bind EIP: %v", service.Namespace, service.Name, argsBind) - glog.V(3).Infof("[%v %v] Bind BLB: %v", service.Namespace, service.Name, lb) - err = bc.clientSet.Eip().BindEip(argsBind) - if err != nil { - glog.V(3).Infof("BindEip error: %v", err) - return "", err - } - lb.PublicIp = loadBalancerIP - pubIP = loadBalancerIP - glog.V(3).Infof("[%v %v] EnsureLoadBalancer: Bind EIP to BLB success.", service.Namespace, service.Name) - } else { // blb already bind eip - if pubIP == loadBalancerIP { // blb bind correct LoadBalancerIP - glog.V(3).Infof("[%v %v] EnsureLoadBalancer: BLB %s already bind EIP %s.", service.Namespace, service.Name, lb.BlbId, pubIP) - } else { // blb not bind correct LoadBalancerIP, need update - glog.V(3).Infof("[%v %v] EnsureLoadBalancer: BLB %s already bind EIP %s, but need updating to %s.", service.Namespace, service.Name, lb.BlbId, pubIP, loadBalancerIP) - // check eip status - argsGet := eip.GetEipsArgs{ - Ip: pubIP, - } - eips, err := bc.clientSet.Eip().GetEips(&argsGet) - if err != nil { - return "", err - } - if len(eips) > 0 { - unbindArgs := eip.EipArgs{ - Ip: pubIP, - } - // just unbind, not delete - err := bc.clientSet.Eip().UnbindEip(&unbindArgs) - if err != nil { - glog.V(3).Infof("Unbind Eip error : %s", err.Error()) - return "", err - } - } - // bind - lb.Status = "unknown" // add here to do loop - for index := 0; (index < 10) && (lb.Status != "available"); index++ { - glog.V(3).Infof("[%v %v] BLB: %s is not available, retry: %d", service.Namespace, service.Name, lb.BlbId, index) - time.Sleep(10 * time.Second) - newlb, exist, err := bc.getBCELoadBalancerById(lb.BlbId) - if err != nil { - glog.V(3).Infof("getBCELoadBalancer error: %s", lb.BlbId) - return "", err - } - if !exist { - glog.V(3).Infof("getBCELoadBalancer not exist: %s", lb.BlbId) - return "", fmt.Errorf("BLB not exists:%s", lb.BlbId) - } - lb = newlb - glog.V(3).Infof("[%v %v] BLB status is : %s", service.Namespace, service.Name, lb.Status) - } - argsBind := &eip.BindEipArgs{ - Ip: loadBalancerIP, - InstanceId: lb.BlbId, - InstanceType: eip.BLB, - } - glog.V(3).Infof("[%v %v] Bind EIP: %v", service.Namespace, service.Name, argsBind) - glog.V(3).Infof("[%v %v] Bind BLB: %v", service.Namespace, service.Name, lb) - err = bc.clientSet.Eip().BindEip(argsBind) - if err != nil { - glog.V(3).Infof("BindEip error: %v", err) - return "", err - } - lb.PublicIp = loadBalancerIP - pubIP = loadBalancerIP - glog.V(3).Infof("[%v %v] EnsureLoadBalancer: Bind EIP to BLB success.", service.Namespace, service.Name) - } - } - return pubIP, nil -} - -func (bc *Baiducloud) getBCELoadBalancer(name string) (lb *blb.LoadBalancer, exists bool, err error) { - args := blb.DescribeLoadBalancersArgs{ - LoadBalancerName: name, - } - lbs, err := bc.clientSet.Blb().DescribeLoadBalancers(&args) - if err != nil { - glog.V(2).Infof("getBCELoadBalancer blb not exists ! %v", args) - return &blb.LoadBalancer{}, false, err - } - if len(lbs) != 1 { - return &blb.LoadBalancer{}, false, nil - } - - return &lbs[0], true, nil -} - -func (bc *Baiducloud) getBCELoadBalancerById(id string) (lb *blb.LoadBalancer, exists bool, err error) { - args := blb.DescribeLoadBalancersArgs{ - LoadBalancerId: id, - } - lbs, err := bc.clientSet.Blb().DescribeLoadBalancers(&args) - if err != nil { - glog.V(2).Infof("getBCELoadBalancer blb %s not exists: %v", args.LoadBalancerId, err) - return &blb.LoadBalancer{}, false, err - } - if len(lbs) != 1 { - return &blb.LoadBalancer{}, false, nil - } - return &lbs[0], true, nil -} - -// This returns a human-readable version of the Service used to tag some resources. -// This is only used for human-readable convenience, and not to filter. -func getServiceName(service *v1.Service) string { - return fmt.Sprintf("%s/%s", service.Namespace, service.Name) -} - -// PortListener describe listener port -type PortListener struct { - Port int - Protocol string - NodePort int32 -} - -func (bc *Baiducloud) reconcileListeners(service *v1.Service, lb *blb.LoadBalancer) error { - expected := make(map[int]PortListener) - // add expected ports - for _, servicePort := range service.Spec.Ports { - expected[int(servicePort.Port)] = PortListener{ - Port: int(servicePort.Port), - Protocol: string(servicePort.Protocol), - NodePort: servicePort.NodePort, - } - } - // delete or update unexpected ports - all, err := bc.getAllListeners(lb) - if err != nil { - return err - } - var deleteList []PortListener - for _, l := range all { - port, ok := expected[l.Port] - if !ok { - // delete listener port - // add to deleteList - deleteList = append(deleteList, l) - } else { - if l != port { - // update listener port - err := bc.updateListener(lb, port) - if err != nil { - return err - } - } - delete(expected, l.Port) - } - } - // delete listener - if len(deleteList) > 0 { - err = bc.deleteListener(lb, deleteList) - if err != nil { - return err - } - } - - // create expected listener - for _, pl := range expected { - err := bc.createListener(lb, pl) - if err != nil { - return err - } - } - return nil -} - -func (bc *Baiducloud) findPortListener(lb *blb.LoadBalancer, port int, proto string) (PortListener, error) { - switch proto { - case "HTTP": - // TODO - case "TCP": - args := blb.DescribeTCPListenerArgs{ - LoadBalancerId: lb.BlbId, - ListenerPort: port, - } - ls, err := bc.clientSet.Blb().DescribeTCPListener(&args) - if err != nil { - return PortListener{}, err - } - if len(ls) < 1 { - return PortListener{}, fmt.Errorf("there is no tcp listener blb:%s port:%d", lb.BlbId, port) - } - return PortListener{ - Port: ls[0].ListenerPort, - NodePort: int32(ls[0].BackendPort), - Protocol: proto, - }, nil - case "HTTPS": - // TODO - case "UDP": - // TODO - } - return PortListener{}, fmt.Errorf("protocol not match: %s", proto) -} - -func (bc *Baiducloud) getAllListeners(lb *blb.LoadBalancer) ([]PortListener, error) { - allListeners := []PortListener{} - // add TCPlisteners - args := blb.DescribeTCPListenerArgs{ - LoadBalancerId: lb.BlbId, - } - ls, err := bc.clientSet.Blb().DescribeTCPListener(&args) - if err != nil { - return nil, err - } - for _, listener := range ls { - allListeners = append(allListeners, PortListener{ - Port: listener.ListenerPort, - Protocol: "TCP", - NodePort: int32(listener.BackendPort), - }) - } - - // add HTTPlisteners HTTPS UDP - // TODO - return allListeners, nil -} - -func (bc *Baiducloud) createListener(lb *blb.LoadBalancer, pl PortListener) error { - switch pl.Protocol { - case "HTTP": - // TODO - case "TCP": - args := blb.CreateTCPListenerArgs{ - LoadBalancerId: lb.BlbId, - ListenerPort: pl.Port, - BackendPort: int(pl.NodePort), - Scheduler: "RoundRobin", - } - err := bc.clientSet.Blb().CreateTCPListener(&args) - if err != nil { - return err - } - return nil - case "HTTPS": - // TODO - case "UDP": - // TODO - } - return fmt.Errorf("CreateListener protocol not match: %s", pl.Protocol) -} - -func (bc *Baiducloud) updateListener(lb *blb.LoadBalancer, pl PortListener) error { - switch pl.Protocol { - case "HTTP": - // TODO - case "TCP": - args := blb.UpdateTCPListenerArgs{ - LoadBalancerId: lb.BlbId, - ListenerPort: pl.Port, - BackendPort: int(pl.NodePort), - Scheduler: "RoundRobin", - } - err := bc.clientSet.Blb().UpdateTCPListener(&args) - if err != nil { - return err - } - return nil - case "HTTPS": - // TODO - case "UDP": - // TODO - } - return fmt.Errorf("updateListener protocol not match: %s", pl.Protocol) -} - -func (bc *Baiducloud) deleteListener(lb *blb.LoadBalancer, pl []PortListener) error { - portList := []int{} - for _, l := range pl { - portList = append(portList, l.Port) - } - args := blb.DeleteListenersArgs{ - LoadBalancerId: lb.BlbId, - PortList: portList, - } - err := bc.clientSet.Blb().DeleteListeners(&args) - if err != nil { - return err - } - return nil -} - -func (bc *Baiducloud) getAllBackendServer(lb *blb.LoadBalancer) ([]blb.BackendServer, error) { - args := blb.DescribeBackendServersArgs{ - LoadBalancerId: lb.BlbId, - } - bs, err := bc.clientSet.Blb().DescribeBackendServers(&args) - if err != nil { - return nil, err - } - return bs, nil -} - -func (bc *Baiducloud) reconcileBackendServers(nodes []*v1.Node, lb *blb.LoadBalancer) error { - expectedServer := make(map[string]string) - for _, node := range nodes { - splitted := strings.Split(node.Spec.ProviderID, "//") - name := splitted[1] - expectedServer[name] = node.ObjectMeta.Name - } - allBS, err := bc.getAllBackendServer(lb) - if err != nil { - return err - } - var removeList []string - // remove unexpected servers - for _, bs := range allBS { - _, exists := expectedServer[bs.InstanceId] - if !exists { - removeList = append(removeList, bs.InstanceId) - } - delete(expectedServer, bs.InstanceId) - } - if len(removeList) > 0 { - args := blb.RemoveBackendServersArgs{ - LoadBalancerId: lb.BlbId, - BackendServerList: removeList, - } - err = bc.clientSet.Blb().RemoveBackendServers(&args) - if err != nil { - return err - } - - } - var addList []blb.BackendServer - // add expected servers - for insID, _ := range expectedServer { - addList = append(addList, blb.BackendServer{ - InstanceId: insID, - Weight: 100, - }) - } - if len(addList) > 0 { - args := blb.AddBackendServersArgs{ - LoadBalancerId: lb.BlbId, - BackendServerList: addList, - } - err = bc.clientSet.Blb().AddBackendServers(&args) - if err != nil { - return err - } - } - return nil -} - -func (bc *Baiducloud) createEIP(args *eip.CreateEipArgs, lb *blb.LoadBalancer) (string, error) { - glog.V(3).Infof("CreateEip: %v", args) - ip, err := bc.clientSet.Eip().CreateEip(args) - if err != nil { - return "", err - } - argsGet := eip.GetEipsArgs{ - Ip: ip, - } - eips, err := bc.clientSet.Eip().GetEips(&argsGet) - if err != nil { - return "", err - } - if len(eips) > 0 { - eipStatus := eips[0].Status - for index := 0; (index < 10) && (eipStatus != "available"); index++ { - glog.V(3).Infof("Eip: %s is not available, retry: %d", ip, index) - time.Sleep(10 * time.Second) - eips, err := bc.clientSet.Eip().GetEips(&argsGet) - if err != nil { - return "", err - } - if len(eips) == 0 { - return "", fmt.Errorf("createEIP failed: CreateEip success but query failed") - } - eipStatus = eips[0].Status - } - glog.V(3).Infof("Eip status is: %s", eipStatus) - } - lb.Status = "unknown" // add here to do loop - for index := 0; (index < 10) && (lb.Status != "available"); index++ { - glog.V(3).Infof("BLB: %s is not available, retry: %d", lb.BlbId, index) - time.Sleep(10 * time.Second) - newlb, exist, err := bc.getBCELoadBalancerById(lb.BlbId) - if err != nil { - glog.V(3).Infof("getBCELoadBalancer error: %s", lb.BlbId) - return "", err - } - if !exist { - glog.V(3).Infof("getBCELoadBalancer not exist: %s", lb.BlbId) - return "", fmt.Errorf("BLB not exists:%s", lb.BlbId) - } - lb = newlb - glog.V(3).Infof("BLB status is : %s", lb.Status) - } - argsBind := &eip.BindEipArgs{ - Ip: ip, - InstanceId: lb.BlbId, - InstanceType: eip.BLB, - } - glog.V(3).Infof("BindEip: %v", argsBind) - glog.V(3).Infof("Bind BLB: %v", lb) - err = bc.clientSet.Eip().BindEip(argsBind) - if err != nil { - glog.V(3).Infof("BindEip error: %v", err) - return ip, err - } - lb.PublicIp = ip - glog.V(3).Infof("createEIP: lb.PublicIp is %s", lb.PublicIp) - return ip, nil -} - -func (bc *Baiducloud) deleteEIP(ip string) error { - argsGet := eip.GetEipsArgs{ - Ip: ip, - } - eips, err := bc.clientSet.Eip().GetEips(&argsGet) - if err != nil { - return err - } - if len(eips) > 0 { - eipStatus := eips[0].Status - for index := 0; (index < 10) && (eipStatus != "available"); index++ { - glog.V(3).Infof("Eip: %s is not available, retry: %d", ip, index) - time.Sleep(10 * time.Second) - eips, err := bc.clientSet.Eip().GetEips(&argsGet) - if err != nil { - return err - } - eipStatus = eips[0].Status - } - } - args := eip.EipArgs{ - Ip: ip, - } - err = bc.clientSet.Eip().DeleteEip(&args) - if err != nil { - return err - } - return nil -} - -func (bc *Baiducloud) waitForLoadBalancer(lb *blb.LoadBalancer) (*blb.LoadBalancer, error) { - lb.Status = "unknown" // add here to do loop - for index := 0; (index < 10) && (lb.Status != "available"); index++ { - glog.V(3).Infof("BLB: %s is not available, retry: %d", lb.BlbId, index) - time.Sleep(10 * time.Second) - newlb, exist, err := bc.getBCELoadBalancerById(lb.BlbId) - if err != nil { - glog.V(3).Infof("getBCELoadBalancer error: %s", lb.BlbId) - return newlb, err - } - if !exist { - glog.V(3).Infof("getBCELoadBalancer not exist: %s", lb.BlbId) - return newlb, fmt.Errorf("BLB not exists:%s", lb.BlbId) - } - lb = newlb - glog.V(3).Infof("BLB status is : %s", lb.Status) - if index == 9 && lb.Status != "available" { - return nil, fmt.Errorf("waitForLoadBalancer failed after retry") - } - } - - return lb, nil -} - -// TODO: 存在很大的优化空间 -// 背景：BLB与NAT子网存在冲突，当用户的集群在NAT子网内时，CCE会创建一个保留子网，类型是通用型，名字是CCE-Reserve，给BLB -// CCE-Reserve 参数： -// (1) 名字: CCE-Reserve -// (2) 可用区：第一台虚机所在可用区 -// (3) CIDR -// IP：第一台虚机所在子网的下一个子网 -// Mask：第一台虚机所在子网的Mask -// (4) VPC：第一台虚机所在VPC -// (5) 类型：通用型 -func (bc *Baiducloud) getVpcInfoForBLB() (string, string, error) { - // get prefer vpc info - ins, err := bc.clientSet.Cce().ListInstances(bc.ClusterID) - if err != nil { - return "", "", err - } - if len(ins) == 0 { - return "", "", fmt.Errorf("getVpcInfoForBLB failed since instance num is zero") - } - vpcId := ins[0].VpcId - subnetId := ins[0].SubnetId - - // check subnet - subnet, err := bc.clientSet.Vpc().DescribeSubnet(subnetId) - if err != nil { - return "", "", fmt.Errorf("DescribeSubnet failed: %v", err) - } - if subnet.SubnetType == "BCC" { - return vpcId, subnetId, nil - } - - // get subnet list and choose preferred one - params := make(map[string]string, 0) - params["vpcId"] = subnet.VpcID - subnets, err := bc.clientSet.Vpc().ListSubnet(params) - if err != nil { - return "", "", fmt.Errorf("ListSubnet failed: %v", err) - } - for _, subnet := range subnets { - if subnet.Name == "系统预定义子网" { - return subnet.VpcID, subnet.SubnetID, nil - } - if subnet.Name == "CCE-Reserve" { - return subnet.VpcID, subnet.SubnetID, nil - } - } - - // create one - currentCidr := subnet.Cidr - for { // loop - _, cidr, err := net.ParseCIDR(currentCidr) - if err != nil { - return "", "", fmt.Errorf("ParseCIDR failed: %v", err) - } - mask, _ := cidr.Mask.Size() - nextCidr, notExist := util.NextSubnet(cidr, mask) - if notExist { - return "", "", fmt.Errorf("NextSubnet failed: %v", err) - } - currentCidr = nextCidr.String() - createSubnetArgs := &vpc.CreateSubnetArgs{ - Name: "CCE-Reserve", - ZoneName: subnet.ZoneName, - Cidr: nextCidr.String(), - VpcID: subnet.VpcID, - SubnetType: "BCC", - } - newSubnetId, err := bc.clientSet.Vpc().CreateSubnet(createSubnetArgs) - if err != nil { - glog.V(3).Infof("CreateSubnet failed: %v, will try again.", err) - time.Sleep(3 * time.Second) - continue - } - return subnet.VpcID, newSubnetId, nil - } -} - -func (bc *Baiducloud) getEipArgsFromAnnotation(serviceAnnotation *ServiceAnnotation) (*eip.CreateEipArgs, error) { - var args *eip.CreateEipArgs - - paymentTiming := serviceAnnotation.ElasticIPPaymentTiming - if len(paymentTiming) == 0 { - paymentTiming = eip.PAYMENTTIMING_POSTPAID // default Postpaid - } - billingMethod := serviceAnnotation.ElasticIPBillingMethod - if len(billingMethod) == 0 { - billingMethod = eip.BILLINGMETHOD_BYTRAFFIC // default ByTraffic - } - bandwidthInMbps := serviceAnnotation.ElasticIPBandwidthInMbps - reservationLength := serviceAnnotation.ElasticIPReservationLength - switch paymentTiming { - case eip.PAYMENTTIMING_PREPAID: - if len(serviceAnnotation.ElasticIPBillingMethod) != 0 { - return nil, fmt.Errorf("when using Prepaid EIP, do not need to set ElasticIPBillingMethod") - } - if bandwidthInMbps == 0 { // not set bandwidthInMbps - bandwidthInMbps = 200 - } else { - if bandwidthInMbps < 1 || bandwidthInMbps > 200 { - return nil, fmt.Errorf("prepaid EIP bandwidthInMbps should in [1, 200]") - } - } - reservationLengthAllowed := []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 12, 24, 36} - rightReservationLength := false - for _, length := range reservationLengthAllowed { - if reservationLength == length { - rightReservationLength = true - } - } - if !rightReservationLength { - return nil, fmt.Errorf("prepaid EIP reservationLength should in [1,2,3,4,5,6,7,8,9,12,24,36]") - } - args = &eip.CreateEipArgs{ - Name: serviceAnnotation.ElasticIPName, - BandwidthInMbps: bandwidthInMbps, - Billing: &eip.Billing{ - PaymentTiming: paymentTiming, - Reservation: &eip.Reservation{ - ReservationLength: reservationLength, - ReservationTimeUnit: "Month", - }, - }, - } - case eip.PAYMENTTIMING_POSTPAID: - switch billingMethod { - case eip.BILLINGMETHOD_BYTRAFFIC: - if bandwidthInMbps == 0 { // not set bandwidthInMbps - bandwidthInMbps = 1000 - } else { - if bandwidthInMbps < 1 || bandwidthInMbps > 1000 { - return nil, fmt.Errorf("postpaid ByTraffic EIP bandwidthInMbps should in [1, 1000]") - } - } - case eip.BILLINGMETHOD_BYBANDWIDTH: - if bandwidthInMbps == 0 { // not set bandwidthInMbps - bandwidthInMbps = 200 - } else { - if bandwidthInMbps < 1 || bandwidthInMbps > 200 { - return nil, fmt.Errorf("postpaid ByBandwidth EIP bandwidthInMbps should in [1, 200]") - } - } - default: - return nil, fmt.Errorf("not support target ElasticIPBillingMethod: %v", billingMethod) - } - args = &eip.CreateEipArgs{ - Name: serviceAnnotation.ElasticIPName, - BandwidthInMbps: bandwidthInMbps, - Billing: &eip.Billing{ - PaymentTiming: paymentTiming, - BillingMethod: billingMethod, - }, - } - default: - return nil, fmt.Errorf("not support target ElasticIPPaymentTiming: %v", paymentTiming) - } - - return args, nil -} - -func (bc *Baiducloud) deleteOldAndCreateNewEip(service *v1.Service, serviceAnnotation *ServiceAnnotation, oldEip string, lb *blb.LoadBalancer) (string, error) { - err := bc.deleteEIP(oldEip) - if err != nil { - return "", err - } - glog.V(2).Infof("[%v %v] EnsureLoadBalancer: createEIP!", service.Namespace, service.Name) - args, err := bc.getEipArgsFromAnnotation(serviceAnnotation) - if err != nil { - glog.Errorf("[%v %v] getEipArgsFromAnnotation failed: %v", service.Namespace, service.Name, err) - return "", err - } - if len(args.Name) == 0 { - args.Name = lb.Name // default EIP name = lb name - } - pubIP, err := bc.createEIP(args, lb) - if err != nil { - if pubIP != "" { - args := eip.EipArgs{ - Ip: pubIP, - } - bc.clientSet.Eip().DeleteEip(&args) - } - return "", err - } - return pubIP, nil -} - -func (bc *Baiducloud) resizeEip(service *v1.Service, serviceAnnotation *ServiceAnnotation, targetEip string) error { - return bc.clientSet.Eip().ResizeEip(&eip.ResizeEipArgs{ - BandwidthInMbps: serviceAnnotation.ElasticIPBandwidthInMbps, - Ip: targetEip, - }) -} +} \ No newline at end of file diff --git a/pkg/cloud-provider/load_balancer_blb.go b/pkg/cloud-provider/load_balancer_blb.go new file mode 100644 index 0000000..b17e886 --- /dev/null +++ b/pkg/cloud-provider/load_balancer_blb.go @@ -0,0 +1,134 @@ +package cloud_provider + +import ( + "context" + "fmt" + "time" + + "github.com/golang/glog" + "k8s.io/api/core/v1" + + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/blb" +) + +func (bc *Baiducloud) ensureBLB(ctx context.Context, clusterName string, service *v1.Service, nodes []*v1.Node, serviceAnnotation *ServiceAnnotation) (*blb.LoadBalancer, error) { + var lb *blb.LoadBalancer + var err error + if len(serviceAnnotation.LoadBalancerId) == 0 { // blb not exist, create one and update annotation + glog.V(3).Infof("[%v %v] EnsureLoadBalancer create blb!", service.Namespace, service.Name) + vpcId, subnetId, err := bc.getVpcInfoForBLB() + if err != nil { + return nil, fmt.Errorf("Can't get VPC info for BLB: %v\n", err) + } + allocateVip := false + if serviceAnnotation.LoadBalancerAllocateVip == "true" { + allocateVip = true + } + args := blb.CreateLoadBalancerArgs{ + Name: bc.ClusterID + "/" + getServiceName(service), + VpcID: vpcId, + SubnetID: subnetId, + Desc: "auto generated by cce:" + bc.ClusterID, + AllocateVip: allocateVip, + } + glog.V(3).Infof("[%v %v] EnsureLoadBalancer create blb args: %v", service.Namespace, service.Name, args) + resp, err := bc.clientSet.Blb().CreateLoadBalancer(&args) + if err != nil { + return nil, err + } + glog.V(3).Infof("[%v %v] EnsureLoadBalancer create blb success, BLB name: %s, BLB id: %s, BLB address: %s.", service.Namespace, service.Name, resp.Name, resp.LoadBalancerId, resp.Address) + argsDesc := blb.DescribeLoadBalancersArgs{ + LoadBalancerId: resp.LoadBalancerId, + } + lbs, err := bc.clientSet.Blb().DescribeLoadBalancers(&argsDesc) + if err != nil { + return nil, err + } + if len(lbs) != 1 { + tryCount := 0 + for { + tryCount ++ + if tryCount > 10 { + return nil, fmt.Errorf("EnsureLoadBalancer create blb success but query get none") + } + glog.V(3).Infof("[%v %v] EnsureLoadBalancer create blb success but query get none, tryCount: ", service.Namespace, service.Name, tryCount) + lbs, err = bc.clientSet.Blb().DescribeLoadBalancers(&argsDesc) + if err != nil { + return nil, err + } + if len(lbs) == 1 { + glog.V(3).Infof("[%v %v] EnsureLoadBalancer create blb success and query get one, tryCount: ", service.Namespace, service.Name, tryCount) + break + } + time.Sleep(10 * time.Second) + } + } + lb = &lbs[0] + if service.Annotations == nil { + service.Annotations = make(map[string]string) + } + service.Annotations[ServiceAnnotationLoadBalancerId] = lb.BlbId + } else { // blb already exist, get info from cloud + var exists bool + lb, exists, err = bc.getBCELoadBalancerById(serviceAnnotation.LoadBalancerId) + if err != nil { + return nil, err + } + if !exists { + return nil, fmt.Errorf("EnsureLoadBalancer getBCELoadBalancerById failed, target blb not exist, blb id: %v", serviceAnnotation.LoadBalancerId) + } + glog.V(3).Infof("[%v %v] EnsureLoadBalancer: blb already exists: %v", service.Namespace, service.Name, lb) + } + lb, err = bc.waitForLoadBalancer(lb) + if err != nil { + return nil, err + } + + // update listener + glog.V(2).Infof("[%v %v] EnsureLoadBalancer: reconcileListeners!", service.Namespace, service.Name) + err = bc.reconcileListeners(service, lb) + if err != nil { + return nil, err + } + lb, err = bc.waitForLoadBalancer(lb) + if err != nil { + return nil, err + } + + // update backend server + glog.V(2).Infof("[%v %v] EnsureLoadBalancer: reconcileBackendServers!", service.Namespace, service.Name) + err = bc.reconcileBackendServers(nodes, lb) + if err != nil { + return nil, err + } + lb, err = bc.waitForLoadBalancer(lb) + if err != nil { + return nil, err + } + + return lb, nil +} + +func (bc *Baiducloud) waitForLoadBalancer(lb *blb.LoadBalancer) (*blb.LoadBalancer, error) { + lb.Status = "unknown" // add here to do loop + for index := 0; (index < 10) && (lb.Status != "available"); index++ { + glog.V(3).Infof("BLB: %s is not available, retry: %d", lb.BlbId, index) + time.Sleep(10 * time.Second) + newlb, exist, err := bc.getBCELoadBalancerById(lb.BlbId) + if err != nil { + glog.V(3).Infof("getBCELoadBalancer error: %s", lb.BlbId) + return newlb, err + } + if !exist { + glog.V(3).Infof("getBCELoadBalancer not exist: %s", lb.BlbId) + return newlb, fmt.Errorf("BLB not exists:%s", lb.BlbId) + } + lb = newlb + glog.V(3).Infof("BLB status is : %s", lb.Status) + if index == 9 && lb.Status != "available" { + return nil, fmt.Errorf("waitForLoadBalancer failed after retry") + } + } + + return lb, nil +} \ No newline at end of file diff --git a/pkg/cloud-provider/load_balancer_blb_backend_server.go b/pkg/cloud-provider/load_balancer_blb_backend_server.go new file mode 100644 index 0000000..2f355cd --- /dev/null +++ b/pkg/cloud-provider/load_balancer_blb_backend_server.go @@ -0,0 +1,88 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloud_provider + +import ( + "strings" + + "k8s.io/api/core/v1" + + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/blb" +) + +func (bc *Baiducloud) reconcileBackendServers(nodes []*v1.Node, lb *blb.LoadBalancer) error { + expectedServer := make(map[string]string) + for _, node := range nodes { + splitted := strings.Split(node.Spec.ProviderID, "//") + name := splitted[1] + expectedServer[name] = node.ObjectMeta.Name + } + allBS, err := bc.getAllBackendServer(lb) + if err != nil { + return err + } + var removeList []string + // remove unexpected servers + for _, bs := range allBS { + _, exists := expectedServer[bs.InstanceId] + if !exists { + removeList = append(removeList, bs.InstanceId) + } + delete(expectedServer, bs.InstanceId) + } + if len(removeList) > 0 { + args := blb.RemoveBackendServersArgs{ + LoadBalancerId: lb.BlbId, + BackendServerList: removeList, + } + err = bc.clientSet.Blb().RemoveBackendServers(&args) + if err != nil { + return err + } + + } + var addList []blb.BackendServer + // add expected servers + for insID, _ := range expectedServer { + addList = append(addList, blb.BackendServer{ + InstanceId: insID, + Weight: 100, + }) + } + if len(addList) > 0 { + args := blb.AddBackendServersArgs{ + LoadBalancerId: lb.BlbId, + BackendServerList: addList, + } + err = bc.clientSet.Blb().AddBackendServers(&args) + if err != nil { + return err + } + } + return nil +} + +func (bc *Baiducloud) getAllBackendServer(lb *blb.LoadBalancer) ([]blb.BackendServer, error) { + args := blb.DescribeBackendServersArgs{ + LoadBalancerId: lb.BlbId, + } + bs, err := bc.clientSet.Blb().DescribeBackendServers(&args) + if err != nil { + return nil, err + } + return bs, nil +} diff --git a/pkg/cloud-provider/load_balancer_blb_listener.go b/pkg/cloud-provider/load_balancer_blb_listener.go new file mode 100644 index 0000000..342a0d8 --- /dev/null +++ b/pkg/cloud-provider/load_balancer_blb_listener.go @@ -0,0 +1,199 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloud_provider + +import ( + "fmt" + + "k8s.io/api/core/v1" + + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/blb" +) + +// PortListener describe listener port +type PortListener struct { + Port int + Protocol string + NodePort int32 +} + +func (bc *Baiducloud) reconcileListeners(service *v1.Service, lb *blb.LoadBalancer) error { + expected := make(map[int]PortListener) + // add expected ports + for _, servicePort := range service.Spec.Ports { + expected[int(servicePort.Port)] = PortListener{ + Port: int(servicePort.Port), + Protocol: string(servicePort.Protocol), + NodePort: servicePort.NodePort, + } + } + // delete or update unexpected ports + all, err := bc.getAllListeners(lb) + if err != nil { + return err + } + var deleteList []PortListener + for _, l := range all { + port, ok := expected[l.Port] + if !ok { + // delete listener port + // add to deleteList + deleteList = append(deleteList, l) + } else { + if l != port { + // update listener port + err := bc.updateListener(lb, port) + if err != nil { + return err + } + } + delete(expected, l.Port) + } + } + // delete listener + if len(deleteList) > 0 { + err = bc.deleteListener(lb, deleteList) + if err != nil { + return err + } + } + + // create expected listener + for _, pl := range expected { + err := bc.createListener(lb, pl) + if err != nil { + return err + } + } + return nil +} + +func (bc *Baiducloud) findPortListener(lb *blb.LoadBalancer, port int, proto string) (PortListener, error) { + switch proto { + case "HTTP": + // TODO + case "TCP": + args := blb.DescribeTCPListenerArgs{ + LoadBalancerId: lb.BlbId, + ListenerPort: port, + } + ls, err := bc.clientSet.Blb().DescribeTCPListener(&args) + if err != nil { + return PortListener{}, err + } + if len(ls) < 1 { + return PortListener{}, fmt.Errorf("there is no tcp listener blb:%s port:%d", lb.BlbId, port) + } + return PortListener{ + Port: ls[0].ListenerPort, + NodePort: int32(ls[0].BackendPort), + Protocol: proto, + }, nil + case "HTTPS": + // TODO + case "UDP": + // TODO + } + return PortListener{}, fmt.Errorf("protocol not match: %s", proto) +} + +func (bc *Baiducloud) getAllListeners(lb *blb.LoadBalancer) ([]PortListener, error) { + allListeners := []PortListener{} + // add TCPlisteners + args := blb.DescribeTCPListenerArgs{ + LoadBalancerId: lb.BlbId, + } + ls, err := bc.clientSet.Blb().DescribeTCPListener(&args) + if err != nil { + return nil, err + } + for _, listener := range ls { + allListeners = append(allListeners, PortListener{ + Port: listener.ListenerPort, + Protocol: "TCP", + NodePort: int32(listener.BackendPort), + }) + } + + // add HTTPlisteners HTTPS UDP + // TODO + return allListeners, nil +} + +func (bc *Baiducloud) createListener(lb *blb.LoadBalancer, pl PortListener) error { + switch pl.Protocol { + case "HTTP": + // TODO + case "TCP": + args := blb.CreateTCPListenerArgs{ + LoadBalancerId: lb.BlbId, + ListenerPort: pl.Port, + BackendPort: int(pl.NodePort), + Scheduler: "RoundRobin", + } + err := bc.clientSet.Blb().CreateTCPListener(&args) + if err != nil { + return err + } + return nil + case "HTTPS": + // TODO + case "UDP": + // TODO + } + return fmt.Errorf("CreateListener protocol not match: %s", pl.Protocol) +} + +func (bc *Baiducloud) updateListener(lb *blb.LoadBalancer, pl PortListener) error { + switch pl.Protocol { + case "HTTP": + // TODO + case "TCP": + args := blb.UpdateTCPListenerArgs{ + LoadBalancerId: lb.BlbId, + ListenerPort: pl.Port, + BackendPort: int(pl.NodePort), + Scheduler: "RoundRobin", + } + err := bc.clientSet.Blb().UpdateTCPListener(&args) + if err != nil { + return err + } + return nil + case "HTTPS": + // TODO + case "UDP": + // TODO + } + return fmt.Errorf("updateListener protocol not match: %s", pl.Protocol) +} + +func (bc *Baiducloud) deleteListener(lb *blb.LoadBalancer, pl []PortListener) error { + portList := []int{} + for _, l := range pl { + portList = append(portList, l.Port) + } + args := blb.DeleteListenersArgs{ + LoadBalancerId: lb.BlbId, + PortList: portList, + } + err := bc.clientSet.Blb().DeleteListeners(&args) + if err != nil { + return err + } + return nil +} \ No newline at end of file diff --git a/pkg/cloud-provider/load_balancer_eip.go b/pkg/cloud-provider/load_balancer_eip.go new file mode 100644 index 0000000..eaf38e8 --- /dev/null +++ b/pkg/cloud-provider/load_balancer_eip.go @@ -0,0 +1,423 @@ +package cloud_provider + +import ( + "context" + "fmt" + "time" + + "github.com/golang/glog" + "k8s.io/api/core/v1" + + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/blb" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/eip" +) + +func (bc *Baiducloud) ensureEIP(ctx context.Context, clusterName string, service *v1.Service, nodes []*v1.Node, serviceAnnotation *ServiceAnnotation, lb *blb.LoadBalancer) (string, error) { + if lb == nil { + return "", fmt.Errorf("[%v %v] EnsureLoadBalancer: ensureEIP need not nil lb", service.Namespace, service.Name) + } + if len(service.Spec.LoadBalancerIP) == 0 { + // not set LoadBalancerIP + return bc.ensureEIPWithNoSpecificIP(ctx, clusterName, service, nodes, serviceAnnotation, lb) + } else { + // set LoadBalancerIP + return bc.ensureEIPWithSpecificIP(ctx, clusterName, service, nodes, serviceAnnotation, lb) + } +} + +func (bc *Baiducloud) ensureEIPWithNoSpecificIP(ctx context.Context, clusterName string, service *v1.Service, nodes []*v1.Node, serviceAnnotation *ServiceAnnotation, lb *blb.LoadBalancer) (string, error) { + pubIP := lb.PublicIp + if len(pubIP) == 0 { // blb not bind eip, mostly case ==> + glog.V(2).Infof("[%v %v] EnsureLoadBalancer: createEIP!", service.Namespace, service.Name) + args, err := bc.getEipArgsFromAnnotation(serviceAnnotation) + if err != nil { + glog.Errorf("[%v %v] getEipArgsFromAnnotation failed: %v", service.Namespace, service.Name, err) + return "", err + } + if len(args.Name) == 0 { + args.Name = lb.Name // default EIP name = lb name + } + pubIP, err = bc.createEIP(args, lb) + if err != nil { + if pubIP != "" { + args := eip.EipArgs{ + Ip: pubIP, + } + bc.clientSet.Eip().DeleteEip(&args) + } + return "", err + } + } else { // blb already bind eip + glog.V(3).Infof("[%v %v] EnsureLoadBalancer: blb's eip already exists, start to ensure...", service.Namespace, service.Name) + argsGet := eip.GetEipsArgs{ + Ip: pubIP, + } + eips, err := bc.clientSet.Eip().GetEips(&argsGet) + if err != nil { + return "", err + } + if len(eips) == 0 { + err = fmt.Errorf("[%v %v] EnsureLoadBalancer: EIP %s not Exist", service.Namespace, service.Name, pubIP) + return "", err + } + targetEip := eips[0] + if (len(serviceAnnotation.ElasticIPPaymentTiming) != 0 && serviceAnnotation.ElasticIPPaymentTiming != targetEip.PaymentTiming) || + (len(serviceAnnotation.ElasticIPBillingMethod) != 0 && serviceAnnotation.ElasticIPBillingMethod != targetEip.BillingMethod) { + glog.V(3).Infof("[%v %v] EnsureLoadBalancer: EIP config change, need delete old eip and create new one", service.Namespace, service.Name) + // TODO + //pubIP, err = bc.deleteOldAndCreateNewEip(service, serviceAnnotation, pubIP, lb) + //if err != nil { + // return "", err + //} + return "", fmt.Errorf("not support change ElasticIP PaymentTiming or ElasticIP BillingMethod, you can delete old and create a new one") + } + if serviceAnnotation.ElasticIPBandwidthInMbps != 0 && serviceAnnotation.ElasticIPBandwidthInMbps != targetEip.BandwidthInMbps { + glog.V(3).Infof("[%v %v] EnsureLoadBalancer: EIP config change, need change ElasticIPBandwidthInMbps", service.Namespace, service.Name) + // just validate args + _, err := bc.getEipArgsFromAnnotation(serviceAnnotation) + if err != nil { + glog.Errorf("[%v %v] Eip Args error: %v", service.Namespace, service.Name, err) + return "", err + } + err = bc.resizeEip(service, serviceAnnotation, pubIP) + if err != nil { + return "", err + } + } + } + return pubIP, nil +} + +func (bc *Baiducloud) ensureEIPWithSpecificIP(ctx context.Context, clusterName string, service *v1.Service, nodes []*v1.Node, serviceAnnotation *ServiceAnnotation, lb *blb.LoadBalancer) (string, error) { + pubIP := lb.PublicIp + loadBalancerIP := service.Spec.LoadBalancerIP + glog.V(3).Infof("[%v %v] EnsureLoadBalancer: Try to bind Custom LoadBalancerIP %s to BLB %s.", service.Namespace, service.Name, loadBalancerIP, lb.BlbId) + if len(pubIP) == 0 { // blb not bind target eip + // check eip status + argsGet := eip.GetEipsArgs{ + Ip: loadBalancerIP, + } + eips, err := bc.clientSet.Eip().GetEips(&argsGet) + if err != nil { + return "", err + } + if len(eips) == 0 { + err = fmt.Errorf("[%v %v] EnsureLoadBalancer: EIP %s not Exist", service.Namespace, service.Name, loadBalancerIP) + return "", err + } else { + eipStatus := eips[0].Status + for index := 0; (index < 10) && (eipStatus != "available"); index++ { + glog.V(3).Infof("[%v %v] Eip: %s is not available, retry: %d", service.Namespace, service.Name, loadBalancerIP, index) + time.Sleep(10 * time.Second) + eips, err := bc.clientSet.Eip().GetEips(&argsGet) + if err != nil { + return "", err + } + eipStatus = eips[0].Status + } + glog.V(3).Infof("Eip final status is: %s", eipStatus) + if eipStatus != "available" { + return "", fmt.Errorf("[%v %v] EnsureLoadBalancer: target eip not available", service.Namespace, service.Name) + } + } + + // bind + lb.Status = "unknown" // add here to do loop + for index := 0; (index < 10) && (lb.Status != "available"); index++ { + glog.V(3).Infof("[%v %v] BLB: %s is not available, retry: %d", service.Namespace, service.Name, lb.BlbId, index) + time.Sleep(10 * time.Second) + newlb, exist, err := bc.getBCELoadBalancerById(lb.BlbId) + if err != nil { + glog.V(3).Infof("getBCELoadBalancer error: %s", lb.BlbId) + return "", err + } + if !exist { + glog.V(3).Infof("getBCELoadBalancer not exist: %s", lb.BlbId) + return "", fmt.Errorf("BLB not exists:%s", lb.BlbId) + } + lb = newlb + glog.V(3).Infof("[%v %v] BLB status is : %s", service.Namespace, service.Name, lb.Status) + } + argsBind := &eip.BindEipArgs{ + Ip: loadBalancerIP, + InstanceId: lb.BlbId, + InstanceType: eip.BLB, + } + glog.V(3).Infof("[%v %v] Bind EIP: %v", service.Namespace, service.Name, argsBind) + glog.V(3).Infof("[%v %v] Bind BLB: %v", service.Namespace, service.Name, lb) + err = bc.clientSet.Eip().BindEip(argsBind) + if err != nil { + glog.V(3).Infof("BindEip error: %v", err) + return "", err + } + lb.PublicIp = loadBalancerIP + pubIP = loadBalancerIP + glog.V(3).Infof("[%v %v] EnsureLoadBalancer: Bind EIP to BLB success.", service.Namespace, service.Name) + } else { // blb already bind eip + if pubIP == loadBalancerIP { // blb bind correct LoadBalancerIP + glog.V(3).Infof("[%v %v] EnsureLoadBalancer: BLB %s already bind EIP %s.", service.Namespace, service.Name, lb.BlbId, pubIP) + } else { // blb not bind correct LoadBalancerIP, need update + glog.V(3).Infof("[%v %v] EnsureLoadBalancer: BLB %s already bind EIP %s, but need updating to %s.", service.Namespace, service.Name, lb.BlbId, pubIP, loadBalancerIP) + // check eip status + argsGet := eip.GetEipsArgs{ + Ip: pubIP, + } + eips, err := bc.clientSet.Eip().GetEips(&argsGet) + if err != nil { + return "", err + } + if len(eips) > 0 { + unbindArgs := eip.EipArgs{ + Ip: pubIP, + } + // just unbind, not delete + err := bc.clientSet.Eip().UnbindEip(&unbindArgs) + if err != nil { + glog.V(3).Infof("Unbind Eip error : %s", err.Error()) + return "", err + } + } + // bind + lb.Status = "unknown" // add here to do loop + for index := 0; (index < 10) && (lb.Status != "available"); index++ { + glog.V(3).Infof("[%v %v] BLB: %s is not available, retry: %d", service.Namespace, service.Name, lb.BlbId, index) + time.Sleep(10 * time.Second) + newlb, exist, err := bc.getBCELoadBalancerById(lb.BlbId) + if err != nil { + glog.V(3).Infof("getBCELoadBalancer error: %s", lb.BlbId) + return "", err + } + if !exist { + glog.V(3).Infof("getBCELoadBalancer not exist: %s", lb.BlbId) + return "", fmt.Errorf("BLB not exists:%s", lb.BlbId) + } + lb = newlb + glog.V(3).Infof("[%v %v] BLB status is : %s", service.Namespace, service.Name, lb.Status) + } + argsBind := &eip.BindEipArgs{ + Ip: loadBalancerIP, + InstanceId: lb.BlbId, + InstanceType: eip.BLB, + } + glog.V(3).Infof("[%v %v] Bind EIP: %v", service.Namespace, service.Name, argsBind) + glog.V(3).Infof("[%v %v] Bind BLB: %v", service.Namespace, service.Name, lb) + err = bc.clientSet.Eip().BindEip(argsBind) + if err != nil { + glog.V(3).Infof("BindEip error: %v", err) + return "", err + } + lb.PublicIp = loadBalancerIP + pubIP = loadBalancerIP + glog.V(3).Infof("[%v %v] EnsureLoadBalancer: Bind EIP to BLB success.", service.Namespace, service.Name) + } + } + return pubIP, nil +} + +func (bc *Baiducloud) createEIP(args *eip.CreateEipArgs, lb *blb.LoadBalancer) (string, error) { + glog.V(3).Infof("CreateEip: %v", args) + ip, err := bc.clientSet.Eip().CreateEip(args) + if err != nil { + return "", err + } + argsGet := eip.GetEipsArgs{ + Ip: ip, + } + eips, err := bc.clientSet.Eip().GetEips(&argsGet) + if err != nil { + return "", err + } + if len(eips) > 0 { + eipStatus := eips[0].Status + for index := 0; (index < 10) && (eipStatus != "available"); index++ { + glog.V(3).Infof("Eip: %s is not available, retry: %d", ip, index) + time.Sleep(10 * time.Second) + eips, err := bc.clientSet.Eip().GetEips(&argsGet) + if err != nil { + return "", err + } + if len(eips) == 0 { + return "", fmt.Errorf("createEIP failed: CreateEip success but query failed") + } + eipStatus = eips[0].Status + } + glog.V(3).Infof("Eip status is: %s", eipStatus) + } + lb.Status = "unknown" // add here to do loop + for index := 0; (index < 10) && (lb.Status != "available"); index++ { + glog.V(3).Infof("BLB: %s is not available, retry: %d", lb.BlbId, index) + time.Sleep(10 * time.Second) + newlb, exist, err := bc.getBCELoadBalancerById(lb.BlbId) + if err != nil { + glog.V(3).Infof("getBCELoadBalancer error: %s", lb.BlbId) + return "", err + } + if !exist { + glog.V(3).Infof("getBCELoadBalancer not exist: %s", lb.BlbId) + return "", fmt.Errorf("BLB not exists:%s", lb.BlbId) + } + lb = newlb + glog.V(3).Infof("BLB status is : %s", lb.Status) + } + argsBind := &eip.BindEipArgs{ + Ip: ip, + InstanceId: lb.BlbId, + InstanceType: eip.BLB, + } + glog.V(3).Infof("BindEip: %v", argsBind) + glog.V(3).Infof("Bind BLB: %v", lb) + err = bc.clientSet.Eip().BindEip(argsBind) + if err != nil { + glog.V(3).Infof("BindEip error: %v", err) + return ip, err + } + lb.PublicIp = ip + glog.V(3).Infof("createEIP: lb.PublicIp is %s", lb.PublicIp) + return ip, nil +} + +func (bc *Baiducloud) deleteEIP(ip string) error { + argsGet := eip.GetEipsArgs{ + Ip: ip, + } + eips, err := bc.clientSet.Eip().GetEips(&argsGet) + if err != nil { + return err + } + if len(eips) > 0 { + eipStatus := eips[0].Status + for index := 0; (index < 10) && (eipStatus != "available"); index++ { + glog.V(3).Infof("Eip: %s is not available, retry: %d", ip, index) + time.Sleep(10 * time.Second) + eips, err := bc.clientSet.Eip().GetEips(&argsGet) + if err != nil { + return err + } + eipStatus = eips[0].Status + } + } + args := eip.EipArgs{ + Ip: ip, + } + err = bc.clientSet.Eip().DeleteEip(&args) + if err != nil { + return err + } + return nil +} + +func (bc *Baiducloud) getEipArgsFromAnnotation(serviceAnnotation *ServiceAnnotation) (*eip.CreateEipArgs, error) { + var args *eip.CreateEipArgs + + paymentTiming := serviceAnnotation.ElasticIPPaymentTiming + if len(paymentTiming) == 0 { + paymentTiming = eip.PAYMENTTIMING_POSTPAID // default Postpaid + } + billingMethod := serviceAnnotation.ElasticIPBillingMethod + if len(billingMethod) == 0 { + billingMethod = eip.BILLINGMETHOD_BYTRAFFIC // default ByTraffic + } + bandwidthInMbps := serviceAnnotation.ElasticIPBandwidthInMbps + reservationLength := serviceAnnotation.ElasticIPReservationLength + switch paymentTiming { + case eip.PAYMENTTIMING_PREPAID: + if len(serviceAnnotation.ElasticIPBillingMethod) != 0 { + return nil, fmt.Errorf("when using Prepaid EIP, do not need to set ElasticIPBillingMethod") + } + if bandwidthInMbps == 0 { // not set bandwidthInMbps + bandwidthInMbps = 200 + } else { + if bandwidthInMbps < 1 || bandwidthInMbps > 200 { + return nil, fmt.Errorf("prepaid EIP bandwidthInMbps should in [1, 200]") + } + } + reservationLengthAllowed := []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 12, 24, 36} + rightReservationLength := false + for _, length := range reservationLengthAllowed { + if reservationLength == length { + rightReservationLength = true + } + } + if !rightReservationLength { + return nil, fmt.Errorf("prepaid EIP reservationLength should in [1,2,3,4,5,6,7,8,9,12,24,36]") + } + args = &eip.CreateEipArgs{ + Name: serviceAnnotation.ElasticIPName, + BandwidthInMbps: bandwidthInMbps, + Billing: &eip.Billing{ + PaymentTiming: paymentTiming, + Reservation: &eip.Reservation{ + ReservationLength: reservationLength, + ReservationTimeUnit: "Month", + }, + }, + } + case eip.PAYMENTTIMING_POSTPAID: + switch billingMethod { + case eip.BILLINGMETHOD_BYTRAFFIC: + if bandwidthInMbps == 0 { // not set bandwidthInMbps + bandwidthInMbps = 1000 + } else { + if bandwidthInMbps < 1 || bandwidthInMbps > 1000 { + return nil, fmt.Errorf("postpaid ByTraffic EIP bandwidthInMbps should in [1, 1000]") + } + } + case eip.BILLINGMETHOD_BYBANDWIDTH: + if bandwidthInMbps == 0 { // not set bandwidthInMbps + bandwidthInMbps = 200 + } else { + if bandwidthInMbps < 1 || bandwidthInMbps > 200 { + return nil, fmt.Errorf("postpaid ByBandwidth EIP bandwidthInMbps should in [1, 200]") + } + } + default: + return nil, fmt.Errorf("not support target ElasticIPBillingMethod: %v", billingMethod) + } + args = &eip.CreateEipArgs{ + Name: serviceAnnotation.ElasticIPName, + BandwidthInMbps: bandwidthInMbps, + Billing: &eip.Billing{ + PaymentTiming: paymentTiming, + BillingMethod: billingMethod, + }, + } + default: + return nil, fmt.Errorf("not support target ElasticIPPaymentTiming: %v", paymentTiming) + } + + return args, nil +} + +func (bc *Baiducloud) deleteOldAndCreateNewEip(service *v1.Service, serviceAnnotation *ServiceAnnotation, oldEip string, lb *blb.LoadBalancer) (string, error) { + err := bc.deleteEIP(oldEip) + if err != nil { + return "", err + } + glog.V(2).Infof("[%v %v] EnsureLoadBalancer: createEIP!", service.Namespace, service.Name) + args, err := bc.getEipArgsFromAnnotation(serviceAnnotation) + if err != nil { + glog.Errorf("[%v %v] getEipArgsFromAnnotation failed: %v", service.Namespace, service.Name, err) + return "", err + } + if len(args.Name) == 0 { + args.Name = lb.Name // default EIP name = lb name + } + pubIP, err := bc.createEIP(args, lb) + if err != nil { + if pubIP != "" { + args := eip.EipArgs{ + Ip: pubIP, + } + bc.clientSet.Eip().DeleteEip(&args) + } + return "", err + } + return pubIP, nil +} + +func (bc *Baiducloud) resizeEip(service *v1.Service, serviceAnnotation *ServiceAnnotation, targetEip string) error { + return bc.clientSet.Eip().ResizeEip(&eip.ResizeEipArgs{ + BandwidthInMbps: serviceAnnotation.ElasticIPBandwidthInMbps, + Ip: targetEip, + }) +} diff --git a/pkg/cloud-provider/load_balancer_util.go b/pkg/cloud-provider/load_balancer_util.go new file mode 100644 index 0000000..68b353f --- /dev/null +++ b/pkg/cloud-provider/load_balancer_util.go @@ -0,0 +1,93 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloud_provider + +import ( + "fmt" + + "github.com/golang/glog" + "k8s.io/api/core/v1" + "k8s.io/kubernetes/pkg/cloudprovider" + + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/blb" +) + +// workaround to support old version, can be removed if not support old version +func (bc *Baiducloud) workAround(service *v1.Service) { + lb, exists, err := bc.getBCELoadBalancer(cloudprovider.GetLoadBalancerName(service)) + if err != nil { + return + } + if !exists { + return + } + if service.Annotations == nil { + service.Annotations = make(map[string]string) + } + // TODO: 不会更新最终Service的annotaion，因为ip没变 + service.Annotations[ServiceAnnotationLoadBalancerId] = lb.BlbId + glog.V(2).Infof("[%v %v] WorkAround for old version, lb: %v", service.Namespace, service.Name, lb) +} + +func (bc *Baiducloud) validateService(service *v1.Service) error { + if len(service.Spec.Ports) == 0 { + return fmt.Errorf("requested load balancer with no ports") + } + for _, port := range service.Spec.Ports { + if port.Protocol != v1.ProtocolTCP && port.Protocol != v1.ProtocolUDP { + return fmt.Errorf("only TCP,UDP LoadBalancer is supported for Baidu CCE") + } + } + return nil +} + +func (bc *Baiducloud) getBCELoadBalancer(name string) (lb *blb.LoadBalancer, exists bool, err error) { + args := blb.DescribeLoadBalancersArgs{ + LoadBalancerName: name, + } + lbs, err := bc.clientSet.Blb().DescribeLoadBalancers(&args) + if err != nil { + glog.V(2).Infof("getBCELoadBalancer blb not exists ! %v", args) + return &blb.LoadBalancer{}, false, err + } + if len(lbs) != 1 { + return &blb.LoadBalancer{}, false, nil + } + + return &lbs[0], true, nil +} + +func (bc *Baiducloud) getBCELoadBalancerById(id string) (lb *blb.LoadBalancer, exists bool, err error) { + args := blb.DescribeLoadBalancersArgs{ + LoadBalancerId: id, + } + lbs, err := bc.clientSet.Blb().DescribeLoadBalancers(&args) + if err != nil { + glog.V(2).Infof("getBCELoadBalancer blb %s not exists: %v", args.LoadBalancerId, err) + return &blb.LoadBalancer{}, false, err + } + if len(lbs) != 1 { + return &blb.LoadBalancer{}, false, nil + } + return &lbs[0], true, nil +} + +// This returns a human-readable version of the Service used to tag some resources. +// This is only used for human-readable convenience, and not to filter. +func getServiceName(service *v1.Service) string { + return fmt.Sprintf("%s/%s", service.Namespace, service.Name) +} diff --git a/pkg/cloud-provider/load_balancer_vpc.go b/pkg/cloud-provider/load_balancer_vpc.go new file mode 100644 index 0000000..1c591f3 --- /dev/null +++ b/pkg/cloud-provider/load_balancer_vpc.go @@ -0,0 +1,105 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloud_provider + +import ( + "fmt" + "net" + "time" + + "github.com/golang/glog" + + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/util" + "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/vpc" +) + +// TODO: 存在很大的优化空间 +// 背景：BLB与NAT子网存在冲突，当用户的集群在NAT子网内时，CCE会创建一个保留子网，类型是通用型，名字是CCE-Reserve，给BLB +// CCE-Reserve 参数： +// (1) 名字: CCE-Reserve +// (2) 可用区：第一台虚机所在可用区 +// (3) CIDR +// IP：第一台虚机所在子网的下一个子网 +// Mask：第一台虚机所在子网的Mask +// (4) VPC：第一台虚机所在VPC +// (5) 类型：通用型 +func (bc *Baiducloud) getVpcInfoForBLB() (string, string, error) { + // get prefer vpc info + ins, err := bc.clientSet.Cce().ListInstances(bc.ClusterID) + if err != nil { + return "", "", err + } + if len(ins) == 0 { + return "", "", fmt.Errorf("getVpcInfoForBLB failed since instance num is zero") + } + vpcId := ins[0].VpcId + subnetId := ins[0].SubnetId + + // check subnet + subnet, err := bc.clientSet.Vpc().DescribeSubnet(subnetId) + if err != nil { + return "", "", fmt.Errorf("DescribeSubnet failed: %v", err) + } + if subnet.SubnetType == "BCC" { + return vpcId, subnetId, nil + } + + // get subnet list and choose preferred one + params := make(map[string]string, 0) + params["vpcId"] = subnet.VpcID + subnets, err := bc.clientSet.Vpc().ListSubnet(params) + if err != nil { + return "", "", fmt.Errorf("ListSubnet failed: %v", err) + } + for _, subnet := range subnets { + if subnet.Name == "系统预定义子网" { + return subnet.VpcID, subnet.SubnetID, nil + } + if subnet.Name == "CCE-Reserve" { + return subnet.VpcID, subnet.SubnetID, nil + } + } + + // create one + currentCidr := subnet.Cidr + for { // loop + _, cidr, err := net.ParseCIDR(currentCidr) + if err != nil { + return "", "", fmt.Errorf("ParseCIDR failed: %v", err) + } + mask, _ := cidr.Mask.Size() + nextCidr, notExist := util.NextSubnet(cidr, mask) + if notExist { + return "", "", fmt.Errorf("NextSubnet failed: %v", err) + } + currentCidr = nextCidr.String() + createSubnetArgs := &vpc.CreateSubnetArgs{ + Name: "CCE-Reserve", + ZoneName: subnet.ZoneName, + Cidr: nextCidr.String(), + VpcID: subnet.VpcID, + SubnetType: "BCC", + } + newSubnetId, err := bc.clientSet.Vpc().CreateSubnet(createSubnetArgs) + if err != nil { + glog.V(3).Infof("CreateSubnet failed: %v, will try again.", err) + time.Sleep(3 * time.Second) + continue + } + return subnet.VpcID, newSubnetId, nil + } +} diff --git a/pkg/cloud-sdk/blb/backendserver.go b/pkg/cloud-sdk/blb/backendserver.go index 960b91b..bc715db 100644 --- a/pkg/cloud-sdk/blb/backendserver.go +++ b/pkg/cloud-sdk/blb/backendserver.go @@ -160,7 +160,7 @@ func (args *UpdateBackendServersArgs) validate() error { return nil } -// UpdateBackendServers update BackendServers +// UpdateBackendServers update BackendServers func (c *Client) UpdateBackendServers(args *UpdateBackendServersArgs) error { err := args.validate() if err != nil { diff --git a/pkg/cloud-sdk/blb/listener.go b/pkg/cloud-sdk/blb/listener.go index 8bd42bc..4df7cfa 100644 --- a/pkg/cloud-sdk/blb/listener.go +++ b/pkg/cloud-sdk/blb/listener.go @@ -199,8 +199,8 @@ type DescribeUDPListenerResponse struct { type UpdateTCPListenerArgs struct { LoadBalancerId string `json:"-"` ListenerPort int `json:"-"` - BackendPort int `json:"backendPort,omitempty"` - Scheduler string `json:"scheduler,omitempty"` + BackendPort int `json:"backendPort"` + Scheduler string `json:"scheduler"` HealthCheckTimeoutInSecond int `json:"healthCheckTimeoutInSecond,omitempty"` HealthCheckInterval int `json:"healthCheckInterval,omitempty"` UnhealthyThreshold int `json:"unhealthyThreshold,omitempty"` @@ -210,7 +210,7 @@ type UpdateTCPListenerArgs struct { // UpdateUDPListenerArgs is the args to update UDPListener type UpdateUDPListenerArgs struct { LoadBalancerId string `json:"-"` - ListenerPort int `json:"listenerPort"` + ListenerPort int `json:"-"` BackendPort int `json:"backendPort"` Scheduler string `json:"scheduler"` HealthCheckTimeoutInSecond int `json:"healthCheckTimeoutInSecond,omitempty"` From 66a25c938242bb004f041548f35293825a3eda5c Mon Sep 17 00:00:00 2001 From: hello2mao Date: Tue, 26 Feb 2019 19:20:20 +0800 Subject: [PATCH 7/7] change doc --- .../service/examples/busybox-udp.yaml | 36 +++++ ...te-vip.yml => nginx-BLB-allocate-vip.yaml} | 0 ...ml => nginx-BLB-support-internal-vpc.yaml} | 0 ... nginx-EIP-Postpaid-ByBandwidth-200M.yaml} | 0 ...> nginx-EIP-Postpaid-ByBandwidth-50M.yaml} | 0 ...> nginx-EIP-Postpaid-ByTraffic-1000M.yaml} | 0 ... => nginx-EIP-Postpaid-ByTraffic-50M.yaml} | 0 ...M-1L.yml => nginx-EIP-Prepaid-10M-1L.yaml} | 0 ...6L.yml => nginx-EIP-Prepaid-200M-36L.yaml} | 0 ...yml => nginx-EIP-with-loadBalancerIP.yaml} | 0 .../examples/{nginx.yml => nginx.yaml} | 0 pkg/cloud-provider/cce.go | 5 + .../load_balancer_blb_listener.go | 134 ++++++++++-------- pkg/cloud-provider/load_balancer_util.go | 17 ++- pkg/cloud-provider/options.go | 6 + pkg/cloud-provider/routes.go | 1 - pkg/cloud-sdk/util/util.go | 2 +- 17 files changed, 137 insertions(+), 64 deletions(-) create mode 100644 docs/controllers/service/examples/busybox-udp.yaml rename docs/controllers/service/examples/{nginx-BLB-allocate-vip.yml => nginx-BLB-allocate-vip.yaml} (100%) rename docs/controllers/service/examples/{nginx-BLB-support-internal-vpc.yml => nginx-BLB-support-internal-vpc.yaml} (100%) rename docs/controllers/service/examples/{nginx-EIP-Postpaid-ByBandwidth-200M.yml => nginx-EIP-Postpaid-ByBandwidth-200M.yaml} (100%) rename docs/controllers/service/examples/{nginx-EIP-Postpaid-ByBandwidth-50M.yml => nginx-EIP-Postpaid-ByBandwidth-50M.yaml} (100%) rename docs/controllers/service/examples/{nginx-EIP-Postpaid-ByTraffic-1000M.yml => nginx-EIP-Postpaid-ByTraffic-1000M.yaml} (100%) rename docs/controllers/service/examples/{nginx-EIP-Postpaid-ByTraffic-50M.yml => nginx-EIP-Postpaid-ByTraffic-50M.yaml} (100%) rename docs/controllers/service/examples/{nginx-EIP-Prepaid-10M-1L.yml => nginx-EIP-Prepaid-10M-1L.yaml} (100%) rename docs/controllers/service/examples/{nginx-EIP-Prepaid-200M-36L.yml => nginx-EIP-Prepaid-200M-36L.yaml} (100%) rename docs/controllers/service/examples/{nginx-EIP-with-loadBalancerIP.yml => nginx-EIP-with-loadBalancerIP.yaml} (100%) rename docs/controllers/service/examples/{nginx.yml => nginx.yaml} (100%) diff --git a/docs/controllers/service/examples/busybox-udp.yaml b/docs/controllers/service/examples/busybox-udp.yaml new file mode 100644 index 0000000..73f32da --- /dev/null +++ b/docs/controllers/service/examples/busybox-udp.yaml @@ -0,0 +1,36 @@ +--- +kind: Service +apiVersion: v1 +metadata: + name: busybox-udp-service +spec: + selector: + app: busybox-udp + type: LoadBalancer + ports: + - name: busybox-udp-port + port: 5300 + targetPort: 5300 + protocol: UDP +--- +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: busybox-udp-deployment +spec: + replicas: 1 + template: + metadata: + labels: + app: busybox-udp + spec: + containers: + - name: busybox + image: busybox + ports: + - containerPort: 5300 + command: + - /bin/sh + - -c + - | + nc -ulp 5300 \ No newline at end of file diff --git a/docs/controllers/service/examples/nginx-BLB-allocate-vip.yml b/docs/controllers/service/examples/nginx-BLB-allocate-vip.yaml similarity index 100% rename from docs/controllers/service/examples/nginx-BLB-allocate-vip.yml rename to docs/controllers/service/examples/nginx-BLB-allocate-vip.yaml diff --git a/docs/controllers/service/examples/nginx-BLB-support-internal-vpc.yml b/docs/controllers/service/examples/nginx-BLB-support-internal-vpc.yaml similarity index 100% rename from docs/controllers/service/examples/nginx-BLB-support-internal-vpc.yml rename to docs/controllers/service/examples/nginx-BLB-support-internal-vpc.yaml diff --git a/docs/controllers/service/examples/nginx-EIP-Postpaid-ByBandwidth-200M.yml b/docs/controllers/service/examples/nginx-EIP-Postpaid-ByBandwidth-200M.yaml similarity index 100% rename from docs/controllers/service/examples/nginx-EIP-Postpaid-ByBandwidth-200M.yml rename to docs/controllers/service/examples/nginx-EIP-Postpaid-ByBandwidth-200M.yaml diff --git a/docs/controllers/service/examples/nginx-EIP-Postpaid-ByBandwidth-50M.yml b/docs/controllers/service/examples/nginx-EIP-Postpaid-ByBandwidth-50M.yaml similarity index 100% rename from docs/controllers/service/examples/nginx-EIP-Postpaid-ByBandwidth-50M.yml rename to docs/controllers/service/examples/nginx-EIP-Postpaid-ByBandwidth-50M.yaml diff --git a/docs/controllers/service/examples/nginx-EIP-Postpaid-ByTraffic-1000M.yml b/docs/controllers/service/examples/nginx-EIP-Postpaid-ByTraffic-1000M.yaml similarity index 100% rename from docs/controllers/service/examples/nginx-EIP-Postpaid-ByTraffic-1000M.yml rename to docs/controllers/service/examples/nginx-EIP-Postpaid-ByTraffic-1000M.yaml diff --git a/docs/controllers/service/examples/nginx-EIP-Postpaid-ByTraffic-50M.yml b/docs/controllers/service/examples/nginx-EIP-Postpaid-ByTraffic-50M.yaml similarity index 100% rename from docs/controllers/service/examples/nginx-EIP-Postpaid-ByTraffic-50M.yml rename to docs/controllers/service/examples/nginx-EIP-Postpaid-ByTraffic-50M.yaml diff --git a/docs/controllers/service/examples/nginx-EIP-Prepaid-10M-1L.yml b/docs/controllers/service/examples/nginx-EIP-Prepaid-10M-1L.yaml similarity index 100% rename from docs/controllers/service/examples/nginx-EIP-Prepaid-10M-1L.yml rename to docs/controllers/service/examples/nginx-EIP-Prepaid-10M-1L.yaml diff --git a/docs/controllers/service/examples/nginx-EIP-Prepaid-200M-36L.yml b/docs/controllers/service/examples/nginx-EIP-Prepaid-200M-36L.yaml similarity index 100% rename from docs/controllers/service/examples/nginx-EIP-Prepaid-200M-36L.yml rename to docs/controllers/service/examples/nginx-EIP-Prepaid-200M-36L.yaml diff --git a/docs/controllers/service/examples/nginx-EIP-with-loadBalancerIP.yml b/docs/controllers/service/examples/nginx-EIP-with-loadBalancerIP.yaml similarity index 100% rename from docs/controllers/service/examples/nginx-EIP-with-loadBalancerIP.yml rename to docs/controllers/service/examples/nginx-EIP-with-loadBalancerIP.yaml diff --git a/docs/controllers/service/examples/nginx.yml b/docs/controllers/service/examples/nginx.yaml similarity index 100% rename from docs/controllers/service/examples/nginx.yml rename to docs/controllers/service/examples/nginx.yaml diff --git a/pkg/cloud-provider/cce.go b/pkg/cloud-provider/cce.go index 6214e1d..f739c3a 100644 --- a/pkg/cloud-provider/cce.go +++ b/pkg/cloud-provider/cce.go @@ -41,6 +41,11 @@ const ProviderName = "cce" // CceUserAgent is prefix of http header UserAgent const CceUserAgent = "cce-k8s:" +// HardSwitch +const ( + EnableUDPLBService = false +) + // Baiducloud defines the main struct type Baiducloud struct { CloudConfig diff --git a/pkg/cloud-provider/load_balancer_blb_listener.go b/pkg/cloud-provider/load_balancer_blb_listener.go index 342a0d8..40bb595 100644 --- a/pkg/cloud-provider/load_balancer_blb_listener.go +++ b/pkg/cloud-provider/load_balancer_blb_listener.go @@ -22,6 +22,7 @@ import ( "k8s.io/api/core/v1" "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/blb" + "github.com/golang/glog" ) // PortListener describe listener port @@ -32,8 +33,8 @@ type PortListener struct { } func (bc *Baiducloud) reconcileListeners(service *v1.Service, lb *blb.LoadBalancer) error { - expected := make(map[int]PortListener) // add expected ports + expected := make(map[int]PortListener) for _, servicePort := range service.Spec.Ports { expected[int(servicePort.Port)] = PortListener{ Port: int(servicePort.Port), @@ -41,6 +42,7 @@ func (bc *Baiducloud) reconcileListeners(service *v1.Service, lb *blb.LoadBalanc NodePort: servicePort.NodePort, } } + // delete or update unexpected ports all, err := bc.getAllListeners(lb) if err != nil { @@ -56,6 +58,7 @@ func (bc *Baiducloud) reconcileListeners(service *v1.Service, lb *blb.LoadBalanc } else { if l != port { // update listener port + glog.V(2).Infof("[%v %v] reconcileListeners: update listener with new config: %v", service.Namespace, service.Name, port) err := bc.updateListener(lb, port) if err != nil { return err @@ -66,6 +69,7 @@ func (bc *Baiducloud) reconcileListeners(service *v1.Service, lb *blb.LoadBalanc } // delete listener if len(deleteList) > 0 { + glog.V(2).Infof("[%v %v] reconcileListeners: delete unexpected listener: %v", service.Namespace, service.Name, deleteList) err = bc.deleteListener(lb, deleteList) if err != nil { return err @@ -73,6 +77,7 @@ func (bc *Baiducloud) reconcileListeners(service *v1.Service, lb *blb.LoadBalanc } // create expected listener + glog.V(2).Infof("[%v %v] reconcileListeners: create expected listener: %v", service.Namespace, service.Name, expected) for _, pl := range expected { err := bc.createListener(lb, pl) if err != nil { @@ -82,62 +87,20 @@ func (bc *Baiducloud) reconcileListeners(service *v1.Service, lb *blb.LoadBalanc return nil } -func (bc *Baiducloud) findPortListener(lb *blb.LoadBalancer, port int, proto string) (PortListener, error) { - switch proto { - case "HTTP": - // TODO - case "TCP": - args := blb.DescribeTCPListenerArgs{ +func (bc *Baiducloud) createListener(lb *blb.LoadBalancer, pl PortListener) error { + switch pl.Protocol { + case "UDP": + args := blb.CreateUDPListenerArgs{ LoadBalancerId: lb.BlbId, - ListenerPort: port, + ListenerPort: pl.Port, + BackendPort: int(pl.NodePort), + Scheduler: "RoundRobin", } - ls, err := bc.clientSet.Blb().DescribeTCPListener(&args) + err := bc.clientSet.Blb().CreateUDPListener(&args) if err != nil { - return PortListener{}, err - } - if len(ls) < 1 { - return PortListener{}, fmt.Errorf("there is no tcp listener blb:%s port:%d", lb.BlbId, port) + return err } - return PortListener{ - Port: ls[0].ListenerPort, - NodePort: int32(ls[0].BackendPort), - Protocol: proto, - }, nil - case "HTTPS": - // TODO - case "UDP": - // TODO - } - return PortListener{}, fmt.Errorf("protocol not match: %s", proto) -} - -func (bc *Baiducloud) getAllListeners(lb *blb.LoadBalancer) ([]PortListener, error) { - allListeners := []PortListener{} - // add TCPlisteners - args := blb.DescribeTCPListenerArgs{ - LoadBalancerId: lb.BlbId, - } - ls, err := bc.clientSet.Blb().DescribeTCPListener(&args) - if err != nil { - return nil, err - } - for _, listener := range ls { - allListeners = append(allListeners, PortListener{ - Port: listener.ListenerPort, - Protocol: "TCP", - NodePort: int32(listener.BackendPort), - }) - } - - // add HTTPlisteners HTTPS UDP - // TODO - return allListeners, nil -} - -func (bc *Baiducloud) createListener(lb *blb.LoadBalancer, pl PortListener) error { - switch pl.Protocol { - case "HTTP": - // TODO + return nil case "TCP": args := blb.CreateTCPListenerArgs{ LoadBalancerId: lb.BlbId, @@ -150,9 +113,9 @@ func (bc *Baiducloud) createListener(lb *blb.LoadBalancer, pl PortListener) erro return err } return nil - case "HTTPS": + case "HTTP": // TODO - case "UDP": + case "HTTPS": // TODO } return fmt.Errorf("CreateListener protocol not match: %s", pl.Protocol) @@ -160,8 +123,18 @@ func (bc *Baiducloud) createListener(lb *blb.LoadBalancer, pl PortListener) erro func (bc *Baiducloud) updateListener(lb *blb.LoadBalancer, pl PortListener) error { switch pl.Protocol { - case "HTTP": - // TODO + case "UDP": + args := blb.UpdateUDPListenerArgs{ + LoadBalancerId: lb.BlbId, + ListenerPort: pl.Port, + BackendPort: int(pl.NodePort), + Scheduler: "RoundRobin", + } + err := bc.clientSet.Blb().UpdateUDPListener(&args) + if err != nil { + return err + } + return nil case "TCP": args := blb.UpdateTCPListenerArgs{ LoadBalancerId: lb.BlbId, @@ -174,16 +147,57 @@ func (bc *Baiducloud) updateListener(lb *blb.LoadBalancer, pl PortListener) erro return err } return nil - case "HTTPS": + case "HTTP": // TODO - case "UDP": + case "HTTPS": // TODO } return fmt.Errorf("updateListener protocol not match: %s", pl.Protocol) } +func (bc *Baiducloud) getAllListeners(lb *blb.LoadBalancer) ([]PortListener, error) { + var allListeners []PortListener + + // add TCPlisteners + describeTCPListenerArgs := blb.DescribeTCPListenerArgs{ + LoadBalancerId: lb.BlbId, + } + tcpListeners, err := bc.clientSet.Blb().DescribeTCPListener(&describeTCPListenerArgs) + if err != nil { + return nil, err + } + for _, listener := range tcpListeners { + allListeners = append(allListeners, PortListener{ + Port: listener.ListenerPort, + Protocol: "TCP", + NodePort: int32(listener.BackendPort), + }) + } + + if EnableUDPLBService { + // add UDPlisteners + describeUDPListenerArgs := blb.DescribeUDPListenerArgs{ + LoadBalancerId: lb.BlbId, + } + udpListeners, err := bc.clientSet.Blb().DescribeUDPListener(&describeUDPListenerArgs) + if err != nil { + return nil, err + } + for _, listener := range udpListeners { + allListeners = append(allListeners, PortListener{ + Port: listener.ListenerPort, + Protocol: "UDP", + NodePort: int32(listener.BackendPort), + }) + } + } + + // TODO: add HTTP,HTTPS + return allListeners, nil +} + func (bc *Baiducloud) deleteListener(lb *blb.LoadBalancer, pl []PortListener) error { - portList := []int{} + var portList []int for _, l := range pl { portList = append(portList, l.Port) } diff --git a/pkg/cloud-provider/load_balancer_util.go b/pkg/cloud-provider/load_balancer_util.go index 68b353f..00bb1a8 100644 --- a/pkg/cloud-provider/load_balancer_util.go +++ b/pkg/cloud-provider/load_balancer_util.go @@ -48,8 +48,21 @@ func (bc *Baiducloud) validateService(service *v1.Service) error { return fmt.Errorf("requested load balancer with no ports") } for _, port := range service.Spec.Ports { - if port.Protocol != v1.ProtocolTCP && port.Protocol != v1.ProtocolUDP { - return fmt.Errorf("only TCP,UDP LoadBalancer is supported for Baidu CCE") + switch port.Protocol { + case "TCP": + continue + case "UDP": + if EnableUDPLBService { + continue + } else { + return fmt.Errorf("UDP is not supported") + } + case "HTTP": + return fmt.Errorf("UDP is not supported") + case "HTTPS": + return fmt.Errorf("UDP is not supported") + default: + return fmt.Errorf("target protocol is not supported: %v", port.Protocol) } } return nil diff --git a/pkg/cloud-provider/options.go b/pkg/cloud-provider/options.go index ba1c130..90614d1 100644 --- a/pkg/cloud-provider/options.go +++ b/pkg/cloud-provider/options.go @@ -32,16 +32,22 @@ const ( ServiceAnnotationLoadBalancerInternalVpc = ServiceAnnotationLoadBalancerPrefix + "internal-vpc" // ServiceAnnotationLoadBalancerAllocateVip is the annotation which indicates BLB with a VIP ServiceAnnotationLoadBalancerAllocateVip = ServiceAnnotationLoadBalancerPrefix + "allocate-vip" + // TODO: // ServiceAnnotationLoadBalancerScheduler is the annotation of load balancer which can be "RoundRobin"/"LeastConnection"/"Hash" ServiceAnnotationLoadBalancerScheduler = ServiceAnnotationLoadBalancerPrefix + "scheduler" + // TODO: // ServiceAnnotationLoadBalancerHealthCheckTimeoutInSecond is the annotation of health check timeout, default 3s, [1, 60] ServiceAnnotationLoadBalancerHealthCheckTimeoutInSecond = ServiceAnnotationLoadBalancerPrefix + "health-check-timeout-in-second" + // TODO: // ServiceAnnotationLoadBalancerHealthCheckInterval is the annotation of health check interval, default 3s, [1, 10] ServiceAnnotationLoadBalancerHealthCheckInterval = ServiceAnnotationLoadBalancerPrefix + "health-check-interval" + // TODO: // ServiceAnnotationLoadBalancerUnhealthyThreshold is the annotation of unhealthy threshold, default 3, [2, 5] ServiceAnnotationLoadBalancerUnhealthyThreshold = ServiceAnnotationLoadBalancerPrefix + "unhealthy-threshold" + // TODO: // ServiceAnnotationLoadBalancerHealthyThreshold is the annotation of healthy threshold, default 3, [2, 5] ServiceAnnotationLoadBalancerHealthyThreshold = ServiceAnnotationLoadBalancerPrefix + "healthy-threshold" + // TODO: // ServiceAnnotationLoadBalancerHealthCheckString is the annotation of health check string ServiceAnnotationLoadBalancerHealthCheckString = ServiceAnnotationLoadBalancerPrefix + "health-check-string" diff --git a/pkg/cloud-provider/routes.go b/pkg/cloud-provider/routes.go index a4e44ac..a1c1ac8 100644 --- a/pkg/cloud-provider/routes.go +++ b/pkg/cloud-provider/routes.go @@ -82,7 +82,6 @@ func (bc *Baiducloud) ListRoutes(ctx context.Context, clusterName string) (route } kubeRoutes = append(kubeRoutes, route) } - glog.V(3).Infof("ListRoutes: %v", kubeRoutes) return kubeRoutes, nil } diff --git a/pkg/cloud-sdk/util/util.go b/pkg/cloud-sdk/util/util.go index c01a99e..6d5c00f 100644 --- a/pkg/cloud-sdk/util/util.go +++ b/pkg/cloud-sdk/util/util.go @@ -540,7 +540,7 @@ func Debug(title, message string) { log.Println("----------------------------DEBUG: start of " + title + "----------------------------") } - glog.V(5).Infoln(message) + glog.V(4).Infoln(message) if title != "" { log.Println("----------------------------DEBUG: end of " + title + "------------------------------\n")