Merge pull request #75 from tianyuansun/fix/issue-71

fix: support subnet-id in annotation and random select vm

docs/controllers/service/annotations.md

@@ -10,6 +10,9 @@ Indicate that the Service only has a BLB and can only be accessed inside the VPC
 ### service.beta.kubernetes.io/cce-load-balancer-allocate-vip: "true"
 Indicate that the BLB for Service has a VIP.**(Baidu Internal Use)**
 
+### service.beta.kubernetes.io/cce-load-balancer-subnet-id: "sbn-25khfnxgfb73"
+Indicate that the BLB for Service will use the Subnet with this id.**(Only used when create Service)**
+
 ## EIP
 
 ### service.beta.kubernetes.io/cce-elastic-ip-payment-timing: ""

docs/controllers/service/examples/nginx-BLB-subnet-id.yaml

@@ -0,0 +1,33 @@
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: nginx-service-blb-subnet-id
+  annotations:
+    service.beta.kubernetes.io/cce-load-balancer-subnet-id: ""
+spec:
+  selector:
+    app: nginx
+  type: LoadBalancer
+  ports:
+  - name: nginx-port
+    port: 80
+    targetPort: 80
+    protocol: TCP
+---
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+  name: nginx-deployment-blb-subnet-id
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx
+        ports:
+        - containerPort: 80

docs/tutorial_zh-CN.md

@@ -8,6 +8,7 @@
     * [3.不分配EIP，即VPC内BLB](#3不分配EIP，即VPC内BLB)
     * [4.UDP-Service](#4UDP-Service)
     * [5.BLB自动分配VIP](#5BLB自动分配VIP)
+    * [6.指定Subnet创建BLB](#6指定Subnet创建BLB)
 
 # 一、使用说明
 本文档会详细介绍如何在CCE下创建类型是**LoadBalancer**的Service。  
@@ -358,3 +359,44 @@ spec:
         - containerPort: 80
 ```
 **注：此VIP只能在百度内网使用，查询VIP请参考BLB的API**
+
+## 6.指定 Subnet 创建 BLB
+通过为 Service 添加 annotations，即 service.beta.kubernetes.io/cce-load-balancer-subnet-id: "sbn-*****"，指定
+BLB 所在的子网。
+
+示例如下。
+
+```yaml
+kind: Service
+apiVersion: v1
+metadata:
+  name: nginx-service-blb-subnet-id
+  annotations:
+    service.beta.kubernetes.io/cce-load-balancer-subnet-id: "sbn-*****"
+spec:
+  selector:
+    app: nginx
+  type: LoadBalancer
+  ports:
+  - name: nginx-port
+    port: 80
+    targetPort: 80
+    protocol: TCP
+---
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+  name: nginx-deployment-blb-subnet-id
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx
+        ports:
+        - containerPort: 80
+```

pkg/cloud-provider/load_balancer_blb.go

@@ -40,10 +40,11 @@ func (bc *Baiducloud) ensureBLB(ctx context.Context, clusterName string, service
 
 	if len(serviceAnnotation.LoadBalancerId) == 0 { // blb not exist, create one and update annotation
 		glog.V(3).Infof("[%v %v] EnsureLoadBalancer create blb!", service.Namespace, service.Name)
-		vpcId, subnetId, err := bc.getVpcInfoForBLB()
+		vpcId, subnetId, err := bc.getVpcInfoForBLB(serviceAnnotation)
 		if err != nil {
 			return nil, fmt.Errorf("Can't get VPC info for BLB: %v\n", err)
 		}
+
 		allocateVip := false
 		if serviceAnnotation.LoadBalancerAllocateVip == "true" {
 			allocateVip = true

pkg/cloud-provider/load_balancer_vpc.go

@@ -18,6 +18,7 @@ package cloud_provider
 
 import (
 	"fmt"
+	"math/rand"
 	"net"
 	"time"
 
@@ -36,25 +37,45 @@ import (
 //		   Mask：第一台虚机所在子网的Mask
 // (4) VPC：第一台虚机所在VPC
 // (5) 类型：通用型
-func (bc *Baiducloud) getVpcInfoForBLB() (string, string, error) {
-	// get prefer vpc info
+func (bc *Baiducloud) getVpcInfoForBLB(serviceAnnotation *ServiceAnnotation) (string, string, error) {
+	// get VPC id
+	vpcId, err := bc.getVpcID()
+	if err != nil {
+		return "", "", fmt.Errorf("Can't get VPC for BLB: %v\n", err)
+	}
+	// user set subnet id in annotation
+	subnetId := serviceAnnotation.LoadBalancerSubnetId
+	if subnetId != "" {
+		glog.V(3).Infof("Find subnetId %v in annotation for BLB", subnetId)
+		subnet, err := bc.clientSet.Vpc().DescribeSubnet(subnetId)
+		if err != nil {
+			return "", "", fmt.Errorf("Can't get subnet with subnetId %v in annotation: %v\n", subnetId, err)
+		}
+		if subnet.SubnetType != "BCC" {
+			return "", "", fmt.Errorf("Can't use subnet with subnetId %v in annotation: subnet type is not BCC\n", subnetId)
+		}
+		glog.V(3).Infof("Use subnet with id %v in annotation for BLB", subnetId)
+		return vpcId, subnetId, nil
+	}
+	// get subnet id from instance
 	ins, err := bc.clientSet.Cce().ListInstances(bc.ClusterID)
 	if err != nil {
 		return "", "", err
 	}
 	if len(ins) == 0 {
 		return "", "", fmt.Errorf("getVpcInfoForBLB failed since instance num is zero")
 	}
-	vpcId := ins[0].VpcId
-	subnetId := ins[0].SubnetId
+	// random select a VM to choose subnet
+	randomVM := ins[rand.Intn(len(ins))]
+	subnetId = randomVM.SubnetId
 
 	// check subnet
 	subnet, err := bc.clientSet.Vpc().DescribeSubnet(subnetId)
 	if err != nil {
 		return "", "", fmt.Errorf("DescribeSubnet failed: %v", err)
 	}
 	if subnet.SubnetType == "BCC" {
-		return vpcId, subnetId, nil
+		return subnet.VpcID, subnetId, nil
 	}
 
 	// get subnet list and choose preferred one
@@ -75,7 +96,12 @@ func (bc *Baiducloud) getVpcInfoForBLB() (string, string, error) {
 
 	// create one
 	currentCidr := subnet.Cidr
+	tryCount := 0
 	for { // loop
+		tryCount++
+		if tryCount > 10 {
+			return "", "", fmt.Errorf("CreateSubnet failed after 10 retries")
+		}
 		_, cidr, err := net.ParseCIDR(currentCidr)
 		if err != nil {
 			return "", "", fmt.Errorf("ParseCIDR failed: %v", err)

pkg/cloud-provider/options.go

@@ -33,6 +33,8 @@ const (
 	ServiceAnnotationLoadBalancerInternalVpc = ServiceAnnotationLoadBalancerPrefix + "internal-vpc"
 	// ServiceAnnotationLoadBalancerAllocateVip is the annotation which indicates BLB with a VIP
 	ServiceAnnotationLoadBalancerAllocateVip = ServiceAnnotationLoadBalancerPrefix + "allocate-vip"
+	//ServiceAnnotationLoadBalancerSubnetId is the annotation which indicates the BCC type subnet the BLB will use
+	ServiceAnnotationLoadBalancerSubnetId = ServiceAnnotationLoadBalancerPrefix + "subnet-id"
 	// TODO:
 	// ServiceAnnotationLoadBalancerScheduler is the annotation of load balancer which can be "RoundRobin"/"LeastConnection"/"Hash"
 	ServiceAnnotationLoadBalancerScheduler = ServiceAnnotationLoadBalancerPrefix + "scheduler"
@@ -86,6 +88,7 @@ type ServiceAnnotation struct {
 	LoadBalancerId                         string
 	LoadBalancerInternalVpc                string
 	LoadBalancerAllocateVip                string
+	LoadBalancerSubnetId                   string
 	LoadBalancerScheduler                  string
 	LoadBalancerHealthCheckTimeoutInSecond int
 	LoadBalancerHealthCheckInterval        int
@@ -133,6 +136,11 @@ func ExtractServiceAnnotation(service *v1.Service) (*ServiceAnnotation, error) {
 		result.LoadBalancerAllocateVip = loadBalancerAllocateVip
 	}
 
+	loadBalancerSubnetId, ok := annotation[ServiceAnnotationLoadBalancerSubnetId]
+	if ok {
+		result.LoadBalancerSubnetId = loadBalancerSubnetId
+	}
+
 	loadBalancerScheduler, ok := annotation[ServiceAnnotationLoadBalancerScheduler]
 	if ok {
 		result.LoadBalancerScheduler = loadBalancerScheduler

pkg/cloud-provider/options_test.go

@@ -0,0 +1,54 @@
+package cloud_provider
+
+import (
+	api "k8s.io/api/core/v1"
+	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"testing"
+)
+
+func buildService() *api.Service {
+	return &api.Service{
+		ObjectMeta: meta_v1.ObjectMeta{
+			Name:      "foo",
+			Namespace: api.NamespaceDefault,
+		},
+		Spec: api.ServiceSpec{
+		},
+	}
+}
+
+func TestExtractServiceAnnotation(t *testing.T) {
+	svc := buildService()
+
+	result, err := ExtractServiceAnnotation(svc)
+	if err != nil {
+		t.Errorf("failed to extract service annotation: %v", err)
+	}
+	if result.LoadBalancerSubnetId != "" {
+		t.Errorf("failed")
+	}
+
+	data := map[string]string{}
+	data[ServiceAnnotationLoadBalancerSubnetId] = ""
+
+	svc.SetAnnotations(data)
+
+	result, err = ExtractServiceAnnotation(svc)
+	if err != nil {
+		t.Errorf("failed to extract service annotation: %v", err)
+	}
+	if result.LoadBalancerSubnetId != "" {
+		t.Errorf("failed")
+	}
+
+	data[ServiceAnnotationLoadBalancerSubnetId] = "abc"
+	svc.SetAnnotations(data)
+
+	result, err = ExtractServiceAnnotation(svc)
+	if err != nil {
+		t.Errorf("failed to extract service annotation: %v", err)
+	}
+	if result.LoadBalancerSubnetId != "abc" {
+		t.Errorf("extrac service annotation wrong")
+	}
+}