Merge pull request #65 from hello2mao/routeRules-conflict-detection

Route rules conflict detection

pkg/cloud-provider/cce.go

@@ -28,8 +28,11 @@ import (
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/tools/record"
 	"k8s.io/kubernetes/pkg/cloudprovider"
 	"k8s.io/kubernetes/pkg/controller"
+	"k8s.io/client-go/kubernetes/scheme"
+	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
 
 	"k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/bce"
 	"k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/clientset"
@@ -46,6 +49,8 @@ type Baiducloud struct {
 	CloudConfig
 	clientSet  clientset.Interface
 	kubeClient kubernetes.Interface
+	eventBroadcaster record.EventBroadcaster
+	eventRecorder    record.EventRecorder
 }
 
 // CloudConfig is the cloud config
@@ -122,6 +127,10 @@ func (bc *Baiducloud) ProviderName() string {
 // to perform housekeeping activities within the cloud provider.
 func (bc *Baiducloud) Initialize(clientBuilder controller.ControllerClientBuilder) {
 	bc.kubeClient = clientBuilder.ClientOrDie(ProviderName)
+	bc.eventBroadcaster = record.NewBroadcaster()
+	bc.eventBroadcaster.StartLogging(glog.Infof)
+	bc.eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: bc.kubeClient.CoreV1().Events("")})
+	bc.eventRecorder = bc.eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "CCM"})
 }
 
 // SetInformers sets the informer on the cloud object.

pkg/cloud-sdk/util/cidr.go → pkg/cloud-provider/cidr.go

@@ -1,4 +1,4 @@
-package util
+package cloud_provider
 
 import (
 	"fmt"

pkg/cloud-provider/cidr_test.go

@@ -0,0 +1,33 @@
+package cloud_provider
+
+import (
+	"net"
+	"testing"
+)
+
+func TestVerifyNoOverlap(t *testing.T) {
+	_, cidrBlock, err := net.ParseCIDR("0.0.0.0/0")
+	if err != nil {
+		t.Error(err)
+	}
+	_, cceCidr, err := net.ParseCIDR("192.168.1.0/24")
+	if err != nil {
+		t.Error(err)
+	}
+	_, customRightCidr, err := net.ParseCIDR("172.16.0.0/29")
+	if err != nil {
+		t.Error(err)
+	}
+	_, customWrongCidr, err := net.ParseCIDR("192.168.1.0/26")
+	if err != nil {
+		t.Error(err)
+	}
+	err = VerifyNoOverlap([]*net.IPNet{cceCidr, customRightCidr,}, cidrBlock)
+	if err != nil {
+		t.Error(err)
+	}
+	err = VerifyNoOverlap([]*net.IPNet{cceCidr, customWrongCidr,}, cidrBlock)
+	if err != nil {
+		t.Log(err)
+	}
+}

pkg/cloud-provider/load_balancer.go

@@ -71,7 +71,7 @@ func (bc *Baiducloud) GetLoadBalancer(ctx context.Context, clusterName string, s
 // parameters as read-only and not modify them.
 // Parameter 'clusterName' is the name of the cluster as presented to kube-controller-manager
 func (bc *Baiducloud) EnsureLoadBalancer(ctx context.Context, clusterName string, service *v1.Service, nodes []*v1.Node) (*v1.LoadBalancerStatus, error) {
-	glog.V(3).Infof("[%v %v] EnsureLoadBalancer(%v, %v, %v, %v, %v, %v, %v)",
+	glog.V(3).Infof("[%v %v] EnsureLoadBalancer(%v, %v, %v, %v, %v)",
 		clusterName, service.Namespace, service.Name, bc.Region, service.Spec.LoadBalancerIP, service.Spec.Ports, service.Annotations)
 	// workaround to support old version, can be removed if not support old version
 	bc.workAround(service)
@@ -166,7 +166,7 @@ func (bc *Baiducloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName
 		if service.Annotations != nil {
 			delete(service.Annotations, ServiceAnnotationLoadBalancerId)
 		}
-		glog.V(3).Infof("[%v %v] EnsureLoadBalancerDeleted: use LoadBalancerInternalVpc, no EIP to delete", service.Namespace, service.Name, lb.Address)
+		glog.V(3).Infof("[%v %v] EnsureLoadBalancerDeleted: use LoadBalancerInternalVpc, no EIP to delete", service.Namespace, service.Name)
 		glog.V(2).Infof("[%v %v] EnsureLoadBalancerDeleted: delete %v FINISH", serviceName, clusterName, serviceName)
 		return nil
 	}

pkg/cloud-provider/load_balancer_vpc.go

@@ -23,7 +23,6 @@ import (
 
 	"github.com/golang/glog"
 
-	"k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/util"
 	"k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/vpc"
 )
 
@@ -82,7 +81,7 @@ func (bc *Baiducloud) getVpcInfoForBLB() (string, string, error) {
 			return "", "", fmt.Errorf("ParseCIDR failed: %v", err)
 		}
 		mask, _ := cidr.Mask.Size()
-		nextCidr, notExist := util.NextSubnet(cidr, mask)
+		nextCidr, notExist := NextSubnet(cidr, mask)
 		if notExist {
 			return "", "", fmt.Errorf("NextSubnet failed: %v", err)
 		}

pkg/cloud-provider/routes.go

@@ -19,13 +19,15 @@ package cloud_provider
 import (
 	"context"
 	"fmt"
+	"net"
 	"strings"
 
 	"github.com/golang/glog"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/kubernetes/pkg/cloudprovider"
 
+	"k8s.io/api/core/v1"
 	"k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/cce"
 	"k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/vpc"
 )
@@ -48,6 +50,10 @@ func (bc *Baiducloud) ListRoutes(ctx context.Context, clusterName string) (route
 	if err != nil {
 		return nil, err
 	}
+
+	// routeTableConflictDetection
+	go bc.routeTableConflictDetection(rs)
+
 	inss, err := bc.clientSet.Cce().ListInstances(bc.ClusterID)
 	if err != nil {
 		return nil, err
@@ -255,3 +261,69 @@ func (bc *Baiducloud) getVpcID() (string, error) {
 	}
 	return bc.VpcID, nil
 }
+
+func (bc *Baiducloud) routeTableConflictDetection(rs []vpc.RouteRule) {
+	glog.V(4).Infof("start routeTable conflict detection.")
+	if len(rs) < 2 {
+		return
+	}
+	var cceRR []vpc.RouteRule
+	var otherRR []vpc.RouteRule
+	for i := 0; i < len(rs); i++ {
+		if strings.Contains(rs[i].Description, "auto generated by cce") {
+			cceRR = append(cceRR, rs[i])
+		} else {
+			otherRR = append(otherRR, rs[i])
+		}
+	}
+	if len(cceRR) == 0 || len(otherRR) == 0 {
+		return
+	}
+	for i := 0; i < len(otherRR); i++ {
+		for j := 0; j < len(cceRR); j++ {
+			if bc.isConflict(otherRR[i], cceRR[j]) {
+				glog.V(4).Infof("RouteTable conflict detected, custom routeRule %v may conflict with cce routeRule %v", otherRR[i], cceRR[j])
+				if bc.eventRecorder != nil {
+					bc.eventRecorder.Eventf(&v1.ObjectReference{
+						Kind:      "VPC",
+						Name:      "RouteTableConflict",
+					}, v1.EventTypeWarning, "RouteTableConflictDetection", "RouteTable conflict detected, custom routeRule %v may conflict with cce routeRule %v", otherRR[i], cceRR[j])
+				}
+			}
+		}
+	}
+}
+
+func (bc *Baiducloud) isConflict(otherRR vpc.RouteRule, cceRR vpc.RouteRule) bool {
+	// rule 1: 用户路由的目标网段 是 CCE实例路由的目标网段 的子网
+	{
+		_, cidrBlock, err := net.ParseCIDR("0.0.0.0/0")
+		if err != nil {
+			glog.Errorf("cidrBlock net.ParseCIDR failed: %v", err)
+			return false
+		}
+		_, cceCidr, err := net.ParseCIDR(cceRR.DestinationAddress)
+		if err != nil {
+			glog.Errorf("cceRR %v net.ParseCIDR failed: %v", cceRR, err)
+			return false
+		}
+		_, otherCidr, err := net.ParseCIDR(otherRR.DestinationAddress)
+		if err != nil {
+			glog.Errorf("otherRR %v net.ParseCIDR failed: %v", otherRR, err)
+			return false
+		}
+		err = VerifyNoOverlap([]*net.IPNet{cceCidr, otherCidr,}, cidrBlock)
+		if err != nil {
+			glog.Errorf("VerifyNoOverlap: %v", err)
+			return true
+		}
+		return false
+	}
+
+	// rule 2: TODO
+	{
+
+	}
+
+	return false
+}