diff --git a/pkg/cloud-provider/cce.go b/pkg/cloud-provider/cce.go index 6214e1d..4ba2887 100644 --- a/pkg/cloud-provider/cce.go +++ b/pkg/cloud-provider/cce.go @@ -28,8 +28,11 @@ import ( "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" "k8s.io/kubernetes/pkg/cloudprovider" "k8s.io/kubernetes/pkg/controller" + "k8s.io/client-go/kubernetes/scheme" + v1core "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/bce" "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/clientset" @@ -46,6 +49,8 @@ type Baiducloud struct { CloudConfig clientSet clientset.Interface kubeClient kubernetes.Interface + eventBroadcaster record.EventBroadcaster + eventRecorder record.EventRecorder } // CloudConfig is the cloud config @@ -122,6 +127,10 @@ func (bc *Baiducloud) ProviderName() string { // to perform housekeeping activities within the cloud provider. func (bc *Baiducloud) Initialize(clientBuilder controller.ControllerClientBuilder) { bc.kubeClient = clientBuilder.ClientOrDie(ProviderName) + bc.eventBroadcaster = record.NewBroadcaster() + bc.eventBroadcaster.StartLogging(glog.Infof) + bc.eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: bc.kubeClient.CoreV1().Events("")}) + bc.eventRecorder = bc.eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "CCM"}) } // SetInformers sets the informer on the cloud object. diff --git a/pkg/cloud-sdk/util/cidr.go b/pkg/cloud-provider/cidr.go similarity index 99% rename from pkg/cloud-sdk/util/cidr.go rename to pkg/cloud-provider/cidr.go index f02a4b0..70e4f1e 100644 --- a/pkg/cloud-sdk/util/cidr.go +++ b/pkg/cloud-provider/cidr.go @@ -1,4 +1,4 @@ -package util +package cloud_provider import ( "fmt" diff --git a/pkg/cloud-provider/cidr_test.go b/pkg/cloud-provider/cidr_test.go new file mode 100644 index 0000000..74c6c8e --- /dev/null +++ b/pkg/cloud-provider/cidr_test.go @@ -0,0 +1,33 @@ +package cloud_provider + +import ( + "net" + "testing" +) + +func TestVerifyNoOverlap(t *testing.T) { + _, cidrBlock, err := net.ParseCIDR("0.0.0.0/0") + if err != nil { + t.Error(err) + } + _, cceCidr, err := net.ParseCIDR("192.168.1.0/24") + if err != nil { + t.Error(err) + } + _, customRightCidr, err := net.ParseCIDR("172.16.0.0/29") + if err != nil { + t.Error(err) + } + _, customWrongCidr, err := net.ParseCIDR("192.168.1.0/26") + if err != nil { + t.Error(err) + } + err = VerifyNoOverlap([]*net.IPNet{cceCidr, customRightCidr,}, cidrBlock) + if err != nil { + t.Error(err) + } + err = VerifyNoOverlap([]*net.IPNet{cceCidr, customWrongCidr,}, cidrBlock) + if err != nil { + t.Log(err) + } +} diff --git a/pkg/cloud-provider/load_balancer.go b/pkg/cloud-provider/load_balancer.go index 25b169d..e74e16d 100644 --- a/pkg/cloud-provider/load_balancer.go +++ b/pkg/cloud-provider/load_balancer.go @@ -71,7 +71,7 @@ func (bc *Baiducloud) GetLoadBalancer(ctx context.Context, clusterName string, s // parameters as read-only and not modify them. // Parameter 'clusterName' is the name of the cluster as presented to kube-controller-manager func (bc *Baiducloud) EnsureLoadBalancer(ctx context.Context, clusterName string, service *v1.Service, nodes []*v1.Node) (*v1.LoadBalancerStatus, error) { - glog.V(3).Infof("[%v %v] EnsureLoadBalancer(%v, %v, %v, %v, %v, %v, %v)", + glog.V(3).Infof("[%v %v] EnsureLoadBalancer(%v, %v, %v, %v, %v)", clusterName, service.Namespace, service.Name, bc.Region, service.Spec.LoadBalancerIP, service.Spec.Ports, service.Annotations) // workaround to support old version, can be removed if not support old version bc.workAround(service) @@ -166,7 +166,7 @@ func (bc *Baiducloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName if service.Annotations != nil { delete(service.Annotations, ServiceAnnotationLoadBalancerId) } - glog.V(3).Infof("[%v %v] EnsureLoadBalancerDeleted: use LoadBalancerInternalVpc, no EIP to delete", service.Namespace, service.Name, lb.Address) + glog.V(3).Infof("[%v %v] EnsureLoadBalancerDeleted: use LoadBalancerInternalVpc, no EIP to delete", service.Namespace, service.Name) glog.V(2).Infof("[%v %v] EnsureLoadBalancerDeleted: delete %v FINISH", serviceName, clusterName, serviceName) return nil } diff --git a/pkg/cloud-provider/load_balancer_vpc.go b/pkg/cloud-provider/load_balancer_vpc.go index 1c591f3..b9e209d 100644 --- a/pkg/cloud-provider/load_balancer_vpc.go +++ b/pkg/cloud-provider/load_balancer_vpc.go @@ -23,7 +23,6 @@ import ( "github.com/golang/glog" - "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/util" "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/vpc" ) @@ -82,7 +81,7 @@ func (bc *Baiducloud) getVpcInfoForBLB() (string, string, error) { return "", "", fmt.Errorf("ParseCIDR failed: %v", err) } mask, _ := cidr.Mask.Size() - nextCidr, notExist := util.NextSubnet(cidr, mask) + nextCidr, notExist := NextSubnet(cidr, mask) if notExist { return "", "", fmt.Errorf("NextSubnet failed: %v", err) } diff --git a/pkg/cloud-provider/routes.go b/pkg/cloud-provider/routes.go index a6d723b..393e168 100644 --- a/pkg/cloud-provider/routes.go +++ b/pkg/cloud-provider/routes.go @@ -19,6 +19,7 @@ package cloud_provider import ( "context" "fmt" + "net" "strings" "github.com/golang/glog" @@ -26,6 +27,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/pkg/cloudprovider" + "k8s.io/api/core/v1" "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/cce" "k8s.io/cloud-provider-baiducloud/pkg/cloud-sdk/vpc" ) @@ -48,6 +50,10 @@ func (bc *Baiducloud) ListRoutes(ctx context.Context, clusterName string) (route if err != nil { return nil, err } + + // routeTableConflictDetection + go bc.routeTableConflictDetection(rs) + inss, err := bc.clientSet.Cce().ListInstances(bc.ClusterID) if err != nil { return nil, err @@ -255,3 +261,69 @@ func (bc *Baiducloud) getVpcID() (string, error) { } return bc.VpcID, nil } + +func (bc *Baiducloud) routeTableConflictDetection(rs []vpc.RouteRule) { + glog.V(4).Infof("start routeTable conflict detection.") + if len(rs) < 2 { + return + } + var cceRR []vpc.RouteRule + var otherRR []vpc.RouteRule + for i := 0; i < len(rs); i++ { + if strings.Contains(rs[i].Description, "auto generated by cce") { + cceRR = append(cceRR, rs[i]) + } else { + otherRR = append(otherRR, rs[i]) + } + } + if len(cceRR) == 0 || len(otherRR) == 0 { + return + } + for i := 0; i < len(otherRR); i++ { + for j := 0; j < len(cceRR); j++ { + if bc.isConflict(otherRR[i], cceRR[j]) { + glog.V(4).Infof("RouteTable conflict detected, custom routeRule %v may conflict with cce routeRule %v", otherRR[i], cceRR[j]) + if bc.eventRecorder != nil { + bc.eventRecorder.Eventf(&v1.ObjectReference{ + Kind: "VPC", + Name: "RouteTableConflict", + }, v1.EventTypeWarning, "RouteTableConflictDetection", "RouteTable conflict detected, custom routeRule %v may conflict with cce routeRule %v", otherRR[i], cceRR[j]) + } + } + } + } +} + +func (bc *Baiducloud) isConflict(otherRR vpc.RouteRule, cceRR vpc.RouteRule) bool { + // rule 1: 用户路由的目标网段 是 CCE实例路由的目标网段 的子网 + { + _, cidrBlock, err := net.ParseCIDR("0.0.0.0/0") + if err != nil { + glog.Errorf("cidrBlock net.ParseCIDR failed: %v", err) + return false + } + _, cceCidr, err := net.ParseCIDR(cceRR.DestinationAddress) + if err != nil { + glog.Errorf("cceRR %v net.ParseCIDR failed: %v", cceRR, err) + return false + } + _, otherCidr, err := net.ParseCIDR(otherRR.DestinationAddress) + if err != nil { + glog.Errorf("otherRR %v net.ParseCIDR failed: %v", otherRR, err) + return false + } + err = VerifyNoOverlap([]*net.IPNet{cceCidr, otherCidr,}, cidrBlock) + if err != nil { + glog.Errorf("VerifyNoOverlap: %v", err) + return true + } + return false + } + + // rule 2: TODO + { + + } + + return false +}