Support dual-stack Services

Signed-off-by: Zhecheng Li <[email protected]>

cmd/cloud-controller-manager/app/controllermanager.go

@@ -309,8 +309,23 @@ func Run(ctx context.Context, c *cloudcontrollerconfig.CompletedConfig, h *contr
 		err   error
 	)
 
+	var ipFamily provider.IPFamily
+	cidrs, isDualStack, err := processCIDRs(c.ComponentConfig.KubeCloudShared.ClusterCIDR)
+	if err != nil || len(cidrs) == 0 {
+		klog.Fatalf("failed to check if it is a dual-stack cluster, cidrs: %v, err: %v", cidrs, err)
+	}
+	if isDualStack {
+		ipFamily = provider.DualStack
+	} else {
+		if cidrs[0].IP.To4() != nil {
+			ipFamily = provider.IPv4
+		} else {
+			ipFamily = provider.IPv6
+		}
+	}
+
 	if c.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile != "" {
-		cloud, err = provider.NewCloudFromConfigFile(ctx, c.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile, true)
+		cloud, err = provider.NewCloudFromConfigFile(ctx, c.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile, true, ipFamily)
 		if err != nil {
 			klog.Fatalf("Cloud provider azure could not be initialized: %v", err)
 		}

cmd/cloud-controller-manager/app/core.go

@@ -157,6 +157,7 @@ func startNodeIpamController(ctx context.Context, controllerContext genericcontr
 	if err != nil {
 		return nil, false, err
 	}
+	klog.V(4).Infof("cluster CIDRs: %v; is dualstack: %v", clusterCIDRs, dualStack)
 
 	// failure: more than one cidr but they are not configured as dual stack
 	if len(clusterCIDRs) > 1 && !dualStack {
@@ -183,7 +184,7 @@ func startNodeIpamController(ctx context.Context, controllerContext genericcontr
 		}
 	}
 
-	// the following checks are triggered if both serviceCIDR and secondaryServiceCIDR are provided
+	// Dual-stack: The following checks are triggered if both serviceCIDR and secondaryServiceCIDR are provided
 	if serviceCIDR != nil && secondaryServiceCIDR != nil {
 		// should be dual stack (from different IPFamilies)
 		dualstackServiceCIDR, err := netutils.IsDualStackCIDRs([]*net.IPNet{serviceCIDR, secondaryServiceCIDR})
@@ -195,8 +196,7 @@ func startNodeIpamController(ctx context.Context, controllerContext genericcontr
 		}
 	}
 
-	nodeCIDRMaskSizeIPv4, nodeCIDRMaskSizeIPv6, err := setNodeCIDRMaskSizesDualStack(completedConfig.NodeIPAMControllerConfig)
-
+	nodeCIDRMaskSizeIPv4, nodeCIDRMaskSizeIPv6, err := setNodeCIDRMaskSizesDualStack(completedConfig.NodeIPAMControllerConfig, dualStack)
 	if err != nil {
 		return nil, false, err
 	}
@@ -224,10 +224,11 @@ func startNodeIpamController(ctx context.Context, controllerContext genericcontr
 // setNodeCIDRMaskSizesDualStack returns the IPv4 and IPv6 node cidr mask sizes to the value provided
 // for --node-cidr-mask-size-ipv4 and --node-cidr-mask-size-ipv6 respectively. If value not provided,
 // then it will return default IPv4 and IPv6 cidr mask sizes.
-func setNodeCIDRMaskSizesDualStack(cfg nodeipamconfig.NodeIPAMControllerConfiguration) (int, int, error) {
+func setNodeCIDRMaskSizesDualStack(cfg nodeipamconfig.NodeIPAMControllerConfiguration, dualstack bool) (int, int, error) {
 	ipv4Mask, ipv6Mask := consts.DefaultNodeMaskCIDRIPv4, consts.DefaultNodeMaskCIDRIPv6
 
-	if cfg.NodeCIDRMaskSize != 0 {
+	// Assume that dual-stack is enabled
+	if dualstack && cfg.NodeCIDRMaskSize != 0 {
 		klog.Warningf("setNodeCIDRMaskSizesDualStack: --node-cidr-mask-size is set to %d, but it would be ignored because the dualstack is enabled", cfg.NodeCIDRMaskSize)
 	}
 

cmd/cloud-controller-manager/app/core_test.go

@@ -59,7 +59,7 @@ func TestSetNodeCIDRMaskSizesDualStack(t *testing.T) {
 				NodeCIDRMaskSizeIPv6: testCase.ipv6Mask,
 			}
 
-			ipv4Mask, ipv6Mask, err := setNodeCIDRMaskSizesDualStack(cfg)
+			ipv4Mask, ipv6Mask, err := setNodeCIDRMaskSizesDualStack(cfg, false)
 			assert.NoError(t, err)
 			assert.Equal(t, testCase.expectedIPV4Mask, ipv4Mask)
 			assert.Equal(t, testCase.expectedIPV6Mask, ipv6Mask)

pkg/consts/consts.go

@@ -201,17 +201,26 @@ var (
 		false: "service.beta.kubernetes.io/azure-load-balancer-ipv4",
 		true:  "service.beta.kubernetes.io/azure-load-balancer-ipv6",
 	}
+	ServiceAnnotationPIPNameDualStack = map[bool]string{
+		false: "service.beta.kubernetes.io/azure-pip-name-ipv4",
+		true:  "service.beta.kubernetes.io/azure-pip-name-ipv6",
+	}
+	ServiceAnnotationPIPPrefixIDDualStack = map[bool]string{
+		false: "service.beta.kubernetes.io/azure-pip-prefix-id-ipv4",
+		true:  "service.beta.kubernetes.io/azure-pip-prefix-id-ipv6",
+	}
 )
 
 // load balancer
 const (
+	// TODO: If PreConfiguredBackendPoolLoadBalancerTypes should account for single IP family separately.
 	// PreConfiguredBackendPoolLoadBalancerTypesNone means that the load balancers are not pre-configured
 	PreConfiguredBackendPoolLoadBalancerTypesNone = ""
-	// PreConfiguredBackendPoolLoadBalancerTypesInternal means that the `internal` load balancers are pre-configured
+	// PreConfiguredBackendPoolLoadBalancerTypesInternal means that the `internal` load balancers are pre-configured. Covers all IP families.
 	PreConfiguredBackendPoolLoadBalancerTypesInternal = "internal"
-	// PreConfiguredBackendPoolLoadBalancerTypesExternal means that the `external` load balancers are pre-configured
+	// PreConfiguredBackendPoolLoadBalancerTypesExternal means that the `external` load balancers are pre-configured. Covers all IP families.
 	PreConfiguredBackendPoolLoadBalancerTypesExternal = "external"
-	// PreConfiguredBackendPoolLoadBalancerTypesAll means that all load balancers are pre-configured
+	// PreConfiguredBackendPoolLoadBalancerTypesAll means that all load balancers are pre-configured. Covers all IP families.
 	PreConfiguredBackendPoolLoadBalancerTypesAll = "all"
 
 	// MaximumLoadBalancerRuleCount is the maximum number of load balancer rules
@@ -352,6 +361,8 @@ const (
 	FrontendIPConfigNameMaxLength = 80
 	// LoadBalancerRuleNameMaxLength is the max length of the load balancing rule
 	LoadBalancerRuleNameMaxLength = 80
+	// IPFamilySuffixLength is the length of suffix length of IP family ("-IPv4", "-IPv6")
+	IPFamilySuffixLength = 5
 
 	// LoadBalancerBackendPoolConfigurationTypeNodeIPConfiguration is the lb backend pool config type node IP configuration
 	LoadBalancerBackendPoolConfigurationTypeNodeIPConfiguration = "nodeIPConfiguration"

pkg/consts/helpers.go

@@ -23,7 +23,6 @@ import (
 	"strings"
 
 	v1 "k8s.io/api/core/v1"
-	"k8s.io/utils/net"
 )
 
 // IsK8sServiceHasHAModeEnabled return if HA Mode is enabled in kubernetes service annotations
@@ -36,10 +35,6 @@ func IsK8sServiceUsingInternalLoadBalancer(service *v1.Service) bool {
 	return expectAttributeInSvcAnnotationBeEqualTo(service.Annotations, ServiceAnnotationLoadBalancerInternal, TrueAnnotationValue)
 }
 
-func IsK8sServiceInternalIPv6(service *v1.Service) bool {
-	return IsK8sServiceUsingInternalLoadBalancer(service) && net.IsIPv6String(service.Spec.ClusterIP)
-}
-
 // IsK8sServiceDisableLoadBalancerFloatingIP return if floating IP in load balancer is disabled in kubernetes service annotations
 func IsK8sServiceDisableLoadBalancerFloatingIP(service *v1.Service) bool {
 	return expectAttributeInSvcAnnotationBeEqualTo(service.Annotations, ServiceAnnotationDisableLoadBalancerFloatingIP, TrueAnnotationValue)

pkg/node/node.go

@@ -38,7 +38,7 @@ func NewIMDSNodeProvider(ctx context.Context) *IMDSNodeProvider {
 	az, err := azureprovider.NewCloud(ctx, bytes.NewBuffer([]byte(`{
 			"useInstanceMetadata": true,
 			"vmType": "vmss"
-		}`)), false)
+		}`)), false, azureprovider.Unknown)
 	if err != nil {
 		klog.Fatalf("Failed to initialize Azure cloud provider: %v", err)
 	}

pkg/node/nodearm.go

@@ -47,7 +47,7 @@ func NewARMNodeProvider(ctx context.Context, cloudConfigFilePath string) *ARMNod
 		}
 		defer configFile.Close()
 
-		az, err = azureprovider.NewCloud(ctx, configFile, false)
+		az, err = azureprovider.NewCloud(ctx, configFile, false, azureprovider.Unknown)
 
 		if err != nil {
 			klog.Fatalf("Failed to initialize Azure cloud provider: %v", err)

pkg/nodeipam/ipam/cidr_allocator.go

@@ -86,7 +86,8 @@ type CIDRAllocatorParams struct {
 	ClusterCIDRs []*net.IPNet
 	// ServiceCIDR is primary service cidr for cluster
 	ServiceCIDR *net.IPNet
-	// SecondaryServiceCIDR is secondary service cidr for cluster
+	// SecondaryServiceCIDR is secondary service cidr for cluster.
+	// It is used in dual-stack clusters and must be of different IP family with ServiceCIDR.
 	SecondaryServiceCIDR *net.IPNet
 	// NodeCIDRMaskSizes is list of node cidr mask sizes
 	NodeCIDRMaskSizes []int

pkg/nodeipam/ipam/cloud_cidr_allocator_test.go

@@ -114,15 +114,16 @@ func TestNewCloudCIDRAllocator(t *testing.T) {
 	allocatorParams := CIDRAllocatorParams{
 		ClusterCIDRs: func() []*net.IPNet {
 			_, clusterCIDRv4, _ := net.ParseCIDR("10.10.0.0/24")
-			return []*net.IPNet{clusterCIDRv4}
+			_, clusterCIDRv6, _ := net.ParseCIDR("fd12:3456:789a:1::/108")
+			return []*net.IPNet{clusterCIDRv4, clusterCIDRv6}
 		}(),
 		ServiceCIDR: func() *net.IPNet {
 			_, clusterCIDRv4, _ := net.ParseCIDR("10.10.0.0/25")
 			return clusterCIDRv4
 		}(),
 		SecondaryServiceCIDR: func() *net.IPNet {
-			_, clusterCIDRv4, _ := net.ParseCIDR("10.10.1.0/25")
-			return clusterCIDRv4
+			_, clusterCIDRv6, _ := net.ParseCIDR("fd12:3456:789a:2::/108")
+			return clusterCIDRv6
 		}(),
 	}
 

pkg/nodeipam/node_ipam_controller.go

@@ -144,6 +144,7 @@ func NewNodeIpamController(
 			klog.Fatal("Controller: Must specify --cluster-cidr if --allocate-node-cidrs is set")
 		}
 
+		// TODO: support ds
 		// TODO: (khenidak) IPv6DualStack beta:
 		// - modify mask to allow flexible masks for IPv4 and IPv6
 		// - for alpha status they are the same

pkg/nodeipam/node_ipam_controller_test.go

@@ -36,7 +36,7 @@ import (
 	"sigs.k8s.io/cloud-provider-azure/pkg/util/controller/testutil"
 )
 
-func newTestNodeIpamController(clusterCIDR []*net.IPNet, serviceCIDR *net.IPNet, secondaryServiceCIDR *net.IPNet, nodeCIDRMaskSizes []int, allocatorType ipam.CIDRAllocatorType) (*Controller, error) {
+func newTestNodeIpamController(clusterCIDRs []*net.IPNet, serviceCIDR *net.IPNet, secondaryServiceCIDR *net.IPNet, nodeCIDRMaskSizes []int, allocatorType ipam.CIDRAllocatorType) (*Controller, error) {
 	clientSet := fake.NewSimpleClientset()
 	fakeNodeHandler := &testutil.FakeNodeHandler{
 		Existing: []*v1.Node{
@@ -55,7 +55,7 @@ func newTestNodeIpamController(clusterCIDR []*net.IPNet, serviceCIDR *net.IPNet,
 	fakeAZ := &providerazure.Cloud{}
 	return NewNodeIpamController(
 		fakeNodeInformer, fakeAZ, clientSet,
-		clusterCIDR, serviceCIDR, secondaryServiceCIDR, nodeCIDRMaskSizes, allocatorType,
+		clusterCIDRs, serviceCIDR, secondaryServiceCIDR, nodeCIDRMaskSizes, allocatorType,
 	)
 }
 

pkg/provider/azure.go

@@ -163,6 +163,7 @@ type Config struct {
 	SystemTags string `json:"systemTags,omitempty" yaml:"systemTags,omitempty"`
 	// Sku of Load Balancer and Public IP. Candidate values are: basic and standard.
 	// If not set, it will be default to basic.
+	// TODO: string to a type
 	LoadBalancerSku string `json:"loadBalancerSku,omitempty" yaml:"loadBalancerSku,omitempty"`
 	// LoadBalancerName determines the specific name of the load balancer user want to use, working with
 	// LoadBalancerResourceGroup
@@ -287,6 +288,15 @@ var (
 	_ cloudprovider.PVLabeler    = (*Cloud)(nil)
 )
 
+type IPFamily string
+
+var (
+	IPv4      IPFamily = "IPv4"
+	IPv6      IPFamily = "IPv6"
+	DualStack IPFamily = "DualStack"
+	Unknown   IPFamily = "Unknown"
+)
+
 // Cloud holds the config and clients
 type Cloud struct {
 	Config
@@ -326,6 +336,7 @@ type Cloud struct {
 
 	// ipv6DualStack allows overriding for unit testing.  It's normally initialized from featuregates
 	ipv6DualStackEnabled bool
+	IPFamily             IPFamily
 	// isSHaredLoadBalancerSynced indicates if the reconcileSharedLoadBalancer has been run
 	isSharedLoadBalancerSynced bool
 	// Lock for access to node caches, includes nodeZones, nodeResourceGroups, and unmanagedNodes.
@@ -380,17 +391,18 @@ type Cloud struct {
 }
 
 // NewCloud returns a Cloud with initialized clients
-func NewCloud(ctx context.Context, configReader io.Reader, callFromCCM bool) (cloudprovider.Interface, error) {
+func NewCloud(ctx context.Context, configReader io.Reader, callFromCCM bool, ipFamily IPFamily) (cloudprovider.Interface, error) {
 	az, err := NewCloudWithoutFeatureGates(ctx, configReader, callFromCCM)
 	if err != nil {
 		return nil, err
 	}
 	az.ipv6DualStackEnabled = true
+	az.IPFamily = ipFamily
 
 	return az, nil
 }
 
-func NewCloudFromConfigFile(ctx context.Context, configFilePath string, calFromCCM bool) (cloudprovider.Interface, error) {
+func NewCloudFromConfigFile(ctx context.Context, configFilePath string, calFromCCM bool, ipFamily IPFamily) (cloudprovider.Interface, error) {
 	var (
 		cloud cloudprovider.Interface
 		err   error
@@ -405,11 +417,11 @@ func NewCloudFromConfigFile(ctx context.Context, configFilePath string, calFromC
 		}
 
 		defer config.Close()
-		cloud, err = NewCloud(ctx, config, calFromCCM)
+		cloud, err = NewCloud(ctx, config, calFromCCM, ipFamily)
 	} else {
 		// Pass explicit nil so plugins can actually check for nil. See
 		// "Why is my nil error value not equal to nil?" in golang.org/doc/faq.
-		cloud, err = NewCloud(ctx, nil, false)
+		cloud, err = NewCloud(ctx, nil, false, ipFamily)
 	}
 
 	if err != nil {

pkg/provider/azure_backoff.go

@@ -381,12 +381,13 @@ func (az *Cloud) ListLB(service *v1.Service) ([]network.LoadBalancer, error) {
 		klog.Errorf("LoadBalancerClient.List(%v) failure with err=%v", rgName, rerr)
 		return nil, rerr.Error()
 	}
-	klog.V(2).Infof("LoadBalancerClient.List(%v) success", rgName)
+	klog.Infof("LoadBalancerClient.List(%v) success", rgName)
 	return allLBs, nil
 }
 
 // CreateOrUpdatePIP invokes az.PublicIPAddressesClient.CreateOrUpdate with exponential backoff retry
 func (az *Cloud) CreateOrUpdatePIP(service *v1.Service, pipResourceGroup string, pip network.PublicIPAddress) error {
+	klog.Infof("DEBUG CreateOrUpdatePIP pipname %s", pointer.StringDeref(pip.Name, ""))
 	ctx, cancel := getContextWithCancel()
 	defer cancel()
 

pkg/provider/azure_fakes.go

@@ -104,6 +104,7 @@ func GetTestCloud(ctrl *gomock.Controller) (az *Cloud) {
 		nodePrivateIPs:           map[string]sets.String{},
 		routeCIDRs:               map[string]string{},
 		eventRecorder:            &record.FakeRecorder{},
+		IPFamily:                 IPv4,
 	}
 	az.DisksClient = mockdiskclient.NewMockInterface(ctrl)
 	az.SnapshotsClient = mocksnapshotclient.NewMockInterface(ctrl)
@@ -126,6 +127,7 @@ func GetTestCloud(ctrl *gomock.Controller) (az *Cloud) {
 	az.pipCache, _ = az.newPIPCache()
 	az.plsCache, _ = az.newPLSCache()
 	az.LoadBalancerBackendPool = NewMockBackendPool(ctrl)
+	az.IPFamily = IPv4
 
 	_ = initDiskControllers(az)