From f525cd499a900ba579a2fb62902fc29db5de4c8b Mon Sep 17 00:00:00 2001 From: Zhecheng Li Date: Thu, 1 Sep 2022 16:54:48 +0800 Subject: [PATCH] Support dual-stack Services Signed-off-by: Zhecheng Li --- .../app/controllermanager.go | 17 +- cmd/cloud-controller-manager/app/core.go | 11 +- cmd/cloud-controller-manager/app/core_test.go | 2 +- pkg/consts/consts.go | 12 +- pkg/consts/helpers.go | 5 - pkg/node/node.go | 2 +- pkg/node/nodearm.go | 2 +- pkg/nodeipam/ipam/cidr_allocator.go | 3 +- .../ipam/cloud_cidr_allocator_test.go | 7 +- pkg/nodeipam/node_ipam_controller.go | 1 + pkg/nodeipam/node_ipam_controller_test.go | 4 +- pkg/provider/azure.go | 20 +- pkg/provider/azure_backoff.go | 3 +- pkg/provider/azure_fakes.go | 2 + pkg/provider/azure_loadbalancer.go | 1103 +++++++++++------ .../azure_loadbalancer_backendpool.go | 550 ++++---- .../azure_loadbalancer_backendpool_test.go | 2 +- pkg/provider/azure_loadbalancer_test.go | 841 ++++++++++--- .../azure_mock_loadbalancer_backendpool.go | 2 +- pkg/provider/azure_standard.go | 176 ++- pkg/provider/azure_standard_test.go | 70 +- pkg/provider/azure_test.go | 456 ++++--- pkg/provider/azure_utils.go | 5 +- pkg/provider/azure_vmss.go | 5 +- pkg/provider/azure_vmssflex.go | 5 +- pkg/provider/azure_wrap.go | 2 +- tests/e2e/network/ensureloadbalancer.go | 511 +++++--- tests/e2e/network/network_security_group.go | 189 ++- tests/e2e/network/node.go | 4 +- tests/e2e/network/private_link_service.go | 50 +- tests/e2e/network/service_annotations.go | 635 +++++++--- tests/e2e/network/standard_lb.go | 8 +- tests/e2e/utils/azure_test_client.go | 4 +- tests/e2e/utils/network_utils.go | 136 +- tests/e2e/utils/service_utils.go | 74 +- tests/e2e/utils/utils.go | 7 + 36 files changed, 3384 insertions(+), 1542 deletions(-) diff --git a/cmd/cloud-controller-manager/app/controllermanager.go b/cmd/cloud-controller-manager/app/controllermanager.go index 748ce504b2..fef0ba7544 100644 --- a/cmd/cloud-controller-manager/app/controllermanager.go +++ b/cmd/cloud-controller-manager/app/controllermanager.go @@ -309,8 +309,23 @@ func Run(ctx context.Context, c *cloudcontrollerconfig.CompletedConfig, h *contr err error ) + var ipFamily provider.IPFamily + cidrs, isDualStack, err := processCIDRs(c.ComponentConfig.KubeCloudShared.ClusterCIDR) + if err != nil || len(cidrs) == 0 { + klog.Fatalf("failed to check if it is a dual-stack cluster, cidrs: %v, err: %v", cidrs, err) + } + if isDualStack { + ipFamily = provider.DualStack + } else { + if cidrs[0].IP.To4() != nil { + ipFamily = provider.IPv4 + } else { + ipFamily = provider.IPv6 + } + } + if c.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile != "" { - cloud, err = provider.NewCloudFromConfigFile(ctx, c.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile, true) + cloud, err = provider.NewCloudFromConfigFile(ctx, c.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile, true, ipFamily) if err != nil { klog.Fatalf("Cloud provider azure could not be initialized: %v", err) } diff --git a/cmd/cloud-controller-manager/app/core.go b/cmd/cloud-controller-manager/app/core.go index b6118ddac2..dd2db605b9 100644 --- a/cmd/cloud-controller-manager/app/core.go +++ b/cmd/cloud-controller-manager/app/core.go @@ -157,6 +157,7 @@ func startNodeIpamController(ctx context.Context, controllerContext genericcontr if err != nil { return nil, false, err } + klog.V(4).Infof("cluster CIDRs: %v; is dualstack: %v", clusterCIDRs, dualStack) // failure: more than one cidr but they are not configured as dual stack if len(clusterCIDRs) > 1 && !dualStack { @@ -183,7 +184,7 @@ func startNodeIpamController(ctx context.Context, controllerContext genericcontr } } - // the following checks are triggered if both serviceCIDR and secondaryServiceCIDR are provided + // Dual-stack: The following checks are triggered if both serviceCIDR and secondaryServiceCIDR are provided if serviceCIDR != nil && secondaryServiceCIDR != nil { // should be dual stack (from different IPFamilies) dualstackServiceCIDR, err := netutils.IsDualStackCIDRs([]*net.IPNet{serviceCIDR, secondaryServiceCIDR}) @@ -195,8 +196,7 @@ func startNodeIpamController(ctx context.Context, controllerContext genericcontr } } - nodeCIDRMaskSizeIPv4, nodeCIDRMaskSizeIPv6, err := setNodeCIDRMaskSizesDualStack(completedConfig.NodeIPAMControllerConfig) - + nodeCIDRMaskSizeIPv4, nodeCIDRMaskSizeIPv6, err := setNodeCIDRMaskSizesDualStack(completedConfig.NodeIPAMControllerConfig, dualStack) if err != nil { return nil, false, err } @@ -224,10 +224,11 @@ func startNodeIpamController(ctx context.Context, controllerContext genericcontr // setNodeCIDRMaskSizesDualStack returns the IPv4 and IPv6 node cidr mask sizes to the value provided // for --node-cidr-mask-size-ipv4 and --node-cidr-mask-size-ipv6 respectively. If value not provided, // then it will return default IPv4 and IPv6 cidr mask sizes. -func setNodeCIDRMaskSizesDualStack(cfg nodeipamconfig.NodeIPAMControllerConfiguration) (int, int, error) { +func setNodeCIDRMaskSizesDualStack(cfg nodeipamconfig.NodeIPAMControllerConfiguration, dualstack bool) (int, int, error) { ipv4Mask, ipv6Mask := consts.DefaultNodeMaskCIDRIPv4, consts.DefaultNodeMaskCIDRIPv6 - if cfg.NodeCIDRMaskSize != 0 { + // Assume that dual-stack is enabled + if dualstack && cfg.NodeCIDRMaskSize != 0 { klog.Warningf("setNodeCIDRMaskSizesDualStack: --node-cidr-mask-size is set to %d, but it would be ignored because the dualstack is enabled", cfg.NodeCIDRMaskSize) } diff --git a/cmd/cloud-controller-manager/app/core_test.go b/cmd/cloud-controller-manager/app/core_test.go index 202a720ed0..069c6e06ed 100644 --- a/cmd/cloud-controller-manager/app/core_test.go +++ b/cmd/cloud-controller-manager/app/core_test.go @@ -59,7 +59,7 @@ func TestSetNodeCIDRMaskSizesDualStack(t *testing.T) { NodeCIDRMaskSizeIPv6: testCase.ipv6Mask, } - ipv4Mask, ipv6Mask, err := setNodeCIDRMaskSizesDualStack(cfg) + ipv4Mask, ipv6Mask, err := setNodeCIDRMaskSizesDualStack(cfg, false) assert.NoError(t, err) assert.Equal(t, testCase.expectedIPV4Mask, ipv4Mask) assert.Equal(t, testCase.expectedIPV6Mask, ipv6Mask) diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index 6fac548c91..9d0adab92c 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -201,12 +201,18 @@ var ( false: "service.beta.kubernetes.io/azure-load-balancer-ipv4", true: "service.beta.kubernetes.io/azure-load-balancer-ipv6", } + ServiceAnnotationPIPNameDualStack = map[bool]string{ + false: "service.beta.kubernetes.io/azure-pip-name-ipv4", + true: "service.beta.kubernetes.io/azure-pip-name-ipv6", + } + ServiceAnnotationPIPPrefixIDDualStack = map[bool]string{ + false: "service.beta.kubernetes.io/azure-pip-prefix-id-ipv4", + true: "service.beta.kubernetes.io/azure-pip-prefix-id-ipv6", + } ) // load balancer const ( - // PreConfiguredBackendPoolLoadBalancerTypesNone means that the load balancers are not pre-configured - PreConfiguredBackendPoolLoadBalancerTypesNone = "" // PreConfiguredBackendPoolLoadBalancerTypesInternal means that the `internal` load balancers are pre-configured PreConfiguredBackendPoolLoadBalancerTypesInternal = "internal" // PreConfiguredBackendPoolLoadBalancerTypesExternal means that the `external` load balancers are pre-configured @@ -352,6 +358,8 @@ const ( FrontendIPConfigNameMaxLength = 80 // LoadBalancerRuleNameMaxLength is the max length of the load balancing rule LoadBalancerRuleNameMaxLength = 80 + // IPFamilySuffixLength is the length of suffix length of IP family ("-IPv4", "-IPv6") + IPFamilySuffixLength = 5 // LoadBalancerBackendPoolConfigurationTypeNodeIPConfiguration is the lb backend pool config type node IP configuration LoadBalancerBackendPoolConfigurationTypeNodeIPConfiguration = "nodeIPConfiguration" diff --git a/pkg/consts/helpers.go b/pkg/consts/helpers.go index b800e9c644..d22006da90 100644 --- a/pkg/consts/helpers.go +++ b/pkg/consts/helpers.go @@ -23,7 +23,6 @@ import ( "strings" v1 "k8s.io/api/core/v1" - "k8s.io/utils/net" ) // IsK8sServiceHasHAModeEnabled return if HA Mode is enabled in kubernetes service annotations @@ -36,10 +35,6 @@ func IsK8sServiceUsingInternalLoadBalancer(service *v1.Service) bool { return expectAttributeInSvcAnnotationBeEqualTo(service.Annotations, ServiceAnnotationLoadBalancerInternal, TrueAnnotationValue) } -func IsK8sServiceInternalIPv6(service *v1.Service) bool { - return IsK8sServiceUsingInternalLoadBalancer(service) && net.IsIPv6String(service.Spec.ClusterIP) -} - // IsK8sServiceDisableLoadBalancerFloatingIP return if floating IP in load balancer is disabled in kubernetes service annotations func IsK8sServiceDisableLoadBalancerFloatingIP(service *v1.Service) bool { return expectAttributeInSvcAnnotationBeEqualTo(service.Annotations, ServiceAnnotationDisableLoadBalancerFloatingIP, TrueAnnotationValue) diff --git a/pkg/node/node.go b/pkg/node/node.go index 11256e3dc2..db20031586 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -38,7 +38,7 @@ func NewIMDSNodeProvider(ctx context.Context) *IMDSNodeProvider { az, err := azureprovider.NewCloud(ctx, bytes.NewBuffer([]byte(`{ "useInstanceMetadata": true, "vmType": "vmss" - }`)), false) + }`)), false, azureprovider.Unknown) if err != nil { klog.Fatalf("Failed to initialize Azure cloud provider: %v", err) } diff --git a/pkg/node/nodearm.go b/pkg/node/nodearm.go index 58dbc5f78a..ea52abc9fa 100644 --- a/pkg/node/nodearm.go +++ b/pkg/node/nodearm.go @@ -47,7 +47,7 @@ func NewARMNodeProvider(ctx context.Context, cloudConfigFilePath string) *ARMNod } defer configFile.Close() - az, err = azureprovider.NewCloud(ctx, configFile, false) + az, err = azureprovider.NewCloud(ctx, configFile, false, azureprovider.Unknown) if err != nil { klog.Fatalf("Failed to initialize Azure cloud provider: %v", err) diff --git a/pkg/nodeipam/ipam/cidr_allocator.go b/pkg/nodeipam/ipam/cidr_allocator.go index c006d3cfda..f78adee449 100644 --- a/pkg/nodeipam/ipam/cidr_allocator.go +++ b/pkg/nodeipam/ipam/cidr_allocator.go @@ -86,7 +86,8 @@ type CIDRAllocatorParams struct { ClusterCIDRs []*net.IPNet // ServiceCIDR is primary service cidr for cluster ServiceCIDR *net.IPNet - // SecondaryServiceCIDR is secondary service cidr for cluster + // SecondaryServiceCIDR is secondary service cidr for cluster. + // It is used in dual-stack clusters and must be of different IP family with ServiceCIDR. SecondaryServiceCIDR *net.IPNet // NodeCIDRMaskSizes is list of node cidr mask sizes NodeCIDRMaskSizes []int diff --git a/pkg/nodeipam/ipam/cloud_cidr_allocator_test.go b/pkg/nodeipam/ipam/cloud_cidr_allocator_test.go index 8709b29724..5bdef3f617 100644 --- a/pkg/nodeipam/ipam/cloud_cidr_allocator_test.go +++ b/pkg/nodeipam/ipam/cloud_cidr_allocator_test.go @@ -114,15 +114,16 @@ func TestNewCloudCIDRAllocator(t *testing.T) { allocatorParams := CIDRAllocatorParams{ ClusterCIDRs: func() []*net.IPNet { _, clusterCIDRv4, _ := net.ParseCIDR("10.10.0.0/24") - return []*net.IPNet{clusterCIDRv4} + _, clusterCIDRv6, _ := net.ParseCIDR("fd12:3456:789a:1::/108") + return []*net.IPNet{clusterCIDRv4, clusterCIDRv6} }(), ServiceCIDR: func() *net.IPNet { _, clusterCIDRv4, _ := net.ParseCIDR("10.10.0.0/25") return clusterCIDRv4 }(), SecondaryServiceCIDR: func() *net.IPNet { - _, clusterCIDRv4, _ := net.ParseCIDR("10.10.1.0/25") - return clusterCIDRv4 + _, clusterCIDRv6, _ := net.ParseCIDR("fd12:3456:789a:2::/108") + return clusterCIDRv6 }(), } diff --git a/pkg/nodeipam/node_ipam_controller.go b/pkg/nodeipam/node_ipam_controller.go index 82e49ca77e..c4b14553aa 100644 --- a/pkg/nodeipam/node_ipam_controller.go +++ b/pkg/nodeipam/node_ipam_controller.go @@ -144,6 +144,7 @@ func NewNodeIpamController( klog.Fatal("Controller: Must specify --cluster-cidr if --allocate-node-cidrs is set") } + // TODO: support ds // TODO: (khenidak) IPv6DualStack beta: // - modify mask to allow flexible masks for IPv4 and IPv6 // - for alpha status they are the same diff --git a/pkg/nodeipam/node_ipam_controller_test.go b/pkg/nodeipam/node_ipam_controller_test.go index 5112cd39db..2293c51241 100644 --- a/pkg/nodeipam/node_ipam_controller_test.go +++ b/pkg/nodeipam/node_ipam_controller_test.go @@ -36,7 +36,7 @@ import ( "sigs.k8s.io/cloud-provider-azure/pkg/util/controller/testutil" ) -func newTestNodeIpamController(clusterCIDR []*net.IPNet, serviceCIDR *net.IPNet, secondaryServiceCIDR *net.IPNet, nodeCIDRMaskSizes []int, allocatorType ipam.CIDRAllocatorType) (*Controller, error) { +func newTestNodeIpamController(clusterCIDRs []*net.IPNet, serviceCIDR *net.IPNet, secondaryServiceCIDR *net.IPNet, nodeCIDRMaskSizes []int, allocatorType ipam.CIDRAllocatorType) (*Controller, error) { clientSet := fake.NewSimpleClientset() fakeNodeHandler := &testutil.FakeNodeHandler{ Existing: []*v1.Node{ @@ -55,7 +55,7 @@ func newTestNodeIpamController(clusterCIDR []*net.IPNet, serviceCIDR *net.IPNet, fakeAZ := &providerazure.Cloud{} return NewNodeIpamController( fakeNodeInformer, fakeAZ, clientSet, - clusterCIDR, serviceCIDR, secondaryServiceCIDR, nodeCIDRMaskSizes, allocatorType, + clusterCIDRs, serviceCIDR, secondaryServiceCIDR, nodeCIDRMaskSizes, allocatorType, ) } diff --git a/pkg/provider/azure.go b/pkg/provider/azure.go index d852c4cc05..f6de9d26fa 100644 --- a/pkg/provider/azure.go +++ b/pkg/provider/azure.go @@ -163,6 +163,7 @@ type Config struct { SystemTags string `json:"systemTags,omitempty" yaml:"systemTags,omitempty"` // Sku of Load Balancer and Public IP. Candidate values are: basic and standard. // If not set, it will be default to basic. + // TODO: string to a type LoadBalancerSku string `json:"loadBalancerSku,omitempty" yaml:"loadBalancerSku,omitempty"` // LoadBalancerName determines the specific name of the load balancer user want to use, working with // LoadBalancerResourceGroup @@ -287,6 +288,15 @@ var ( _ cloudprovider.PVLabeler = (*Cloud)(nil) ) +type IPFamily string + +var ( + IPv4 IPFamily = "IPv4" + IPv6 IPFamily = "IPv6" + DualStack IPFamily = "DualStack" + Unknown IPFamily = "Unknown" +) + // Cloud holds the config and clients type Cloud struct { Config @@ -326,6 +336,7 @@ type Cloud struct { // ipv6DualStack allows overriding for unit testing. It's normally initialized from featuregates ipv6DualStackEnabled bool + IPFamily IPFamily // isSHaredLoadBalancerSynced indicates if the reconcileSharedLoadBalancer has been run isSharedLoadBalancerSynced bool // Lock for access to node caches, includes nodeZones, nodeResourceGroups, and unmanagedNodes. @@ -380,17 +391,18 @@ type Cloud struct { } // NewCloud returns a Cloud with initialized clients -func NewCloud(ctx context.Context, configReader io.Reader, callFromCCM bool) (cloudprovider.Interface, error) { +func NewCloud(ctx context.Context, configReader io.Reader, callFromCCM bool, ipFamily IPFamily) (cloudprovider.Interface, error) { az, err := NewCloudWithoutFeatureGates(ctx, configReader, callFromCCM) if err != nil { return nil, err } az.ipv6DualStackEnabled = true + az.IPFamily = ipFamily return az, nil } -func NewCloudFromConfigFile(ctx context.Context, configFilePath string, calFromCCM bool) (cloudprovider.Interface, error) { +func NewCloudFromConfigFile(ctx context.Context, configFilePath string, calFromCCM bool, ipFamily IPFamily) (cloudprovider.Interface, error) { var ( cloud cloudprovider.Interface err error @@ -405,11 +417,11 @@ func NewCloudFromConfigFile(ctx context.Context, configFilePath string, calFromC } defer config.Close() - cloud, err = NewCloud(ctx, config, calFromCCM) + cloud, err = NewCloud(ctx, config, calFromCCM, ipFamily) } else { // Pass explicit nil so plugins can actually check for nil. See // "Why is my nil error value not equal to nil?" in golang.org/doc/faq. - cloud, err = NewCloud(ctx, nil, false) + cloud, err = NewCloud(ctx, nil, false, ipFamily) } if err != nil { diff --git a/pkg/provider/azure_backoff.go b/pkg/provider/azure_backoff.go index d1a824cd76..2713d249eb 100644 --- a/pkg/provider/azure_backoff.go +++ b/pkg/provider/azure_backoff.go @@ -381,12 +381,13 @@ func (az *Cloud) ListLB(service *v1.Service) ([]network.LoadBalancer, error) { klog.Errorf("LoadBalancerClient.List(%v) failure with err=%v", rgName, rerr) return nil, rerr.Error() } - klog.V(2).Infof("LoadBalancerClient.List(%v) success", rgName) + klog.Infof("LoadBalancerClient.List(%v) success", rgName) return allLBs, nil } // CreateOrUpdatePIP invokes az.PublicIPAddressesClient.CreateOrUpdate with exponential backoff retry func (az *Cloud) CreateOrUpdatePIP(service *v1.Service, pipResourceGroup string, pip network.PublicIPAddress) error { + klog.Infof("DEBUG CreateOrUpdatePIP pipname %s", pointer.StringDeref(pip.Name, "")) ctx, cancel := getContextWithCancel() defer cancel() diff --git a/pkg/provider/azure_fakes.go b/pkg/provider/azure_fakes.go index f749601857..0570bb451d 100644 --- a/pkg/provider/azure_fakes.go +++ b/pkg/provider/azure_fakes.go @@ -104,6 +104,7 @@ func GetTestCloud(ctrl *gomock.Controller) (az *Cloud) { nodePrivateIPs: map[string]sets.String{}, routeCIDRs: map[string]string{}, eventRecorder: &record.FakeRecorder{}, + IPFamily: IPv4, } az.DisksClient = mockdiskclient.NewMockInterface(ctrl) az.SnapshotsClient = mocksnapshotclient.NewMockInterface(ctrl) @@ -126,6 +127,7 @@ func GetTestCloud(ctrl *gomock.Controller) (az *Cloud) { az.pipCache, _ = az.newPIPCache() az.plsCache, _ = az.newPLSCache() az.LoadBalancerBackendPool = NewMockBackendPool(ctrl) + az.IPFamily = IPv4 _ = initDiskControllers(az) diff --git a/pkg/provider/azure_loadbalancer.go b/pkg/provider/azure_loadbalancer.go index 3d839f967f..0c9fad45d2 100644 --- a/pkg/provider/azure_loadbalancer.go +++ b/pkg/provider/azure_loadbalancer.go @@ -48,22 +48,70 @@ import ( "sigs.k8s.io/cloud-provider-azure/pkg/retry" ) +// TODO: remove this +func (az *Cloud) ifIPFamiliesEnabled() (v4Enabled bool, v6Enabled bool) { + if az.IPFamily == DualStack || az.IPFamily == IPv4 { + v4Enabled = true + } + if az.IPFamily == DualStack || az.IPFamily == IPv6 { + v6Enabled = true + } + return +} + +func ifIPFamiliesEnabled1(svc *v1.Service) (v4Enabled bool, v6Enabled bool) { + for _, ipFamily := range svc.Spec.IPFamilies { + if ipFamily == v1.IPv4Protocol { + v4Enabled = true + } else { + v6Enabled = true + } + } + return +} + // getServiceLoadBalancerIP retrieves LB IP from IPv4 annotation, then IPv6 annotation, then service.Spec.LoadBalancerIP. -// TODO: Dual-stack support is not implemented. -func getServiceLoadBalancerIP(service *v1.Service) string { +func getServiceLoadBalancerIP(service *v1.Service, isIPv6 bool) string { if service == nil { return "" } - if ip, ok := service.Annotations[consts.ServiceAnnotationLoadBalancerIPDualStack[false]]; ok && ip != "" { + klog.Infof("DEBUG getServiceLoadBalancerIP annotations %v", service.Annotations) + if ip, ok := service.Annotations[consts.ServiceAnnotationLoadBalancerIPDualStack[isIPv6]]; ok && ip != "" { return ip } + + // Retrieve LB IP from service.Spec.LoadBalancerIP (will be deprecated) + svcLBIP := service.Spec.LoadBalancerIP + if (net.ParseIP(svcLBIP).To4() != nil && !isIPv6) || + (net.ParseIP(svcLBIP).To4() == nil && isIPv6) { + return svcLBIP + } + return "" +} + +func getServiceLoadBalancerIPs(service *v1.Service) []string { + if service == nil { + return []string{} + } + + ips := []string{} + if ip, ok := service.Annotations[consts.ServiceAnnotationLoadBalancerIPDualStack[false]]; ok && ip != "" { + ips = append(ips, ip) + } if ip, ok := service.Annotations[consts.ServiceAnnotationLoadBalancerIPDualStack[true]]; ok && ip != "" { - return ip + ips = append(ips, ip) + } + if len(ips) != 0 { + return ips } - // Retrieve LB IP from service.Spec.LoadBalancerIP (will be deprecated) - return service.Spec.LoadBalancerIP + lbIP := service.Spec.LoadBalancerIP + if lbIP != "" { + ips = append(ips, lbIP) + } + + return ips } // setServiceLoadBalancerIP sets LB IP to a Service @@ -71,31 +119,64 @@ func setServiceLoadBalancerIP(service *v1.Service, ip string) { if service.Annotations == nil { service.Annotations = map[string]string{} } - if net.ParseIP(ip).To4() != nil { - service.Annotations[consts.ServiceAnnotationLoadBalancerIPDualStack[false]] = ip - return + isIPv6 := net.ParseIP(ip).To4() == nil + service.Annotations[consts.ServiceAnnotationLoadBalancerIPDualStack[isIPv6]] = ip +} + +func getServicePIPName(service *v1.Service, isIPv6 bool) string { + if service == nil { + return "" + } + + if name, ok := service.Annotations[consts.ServiceAnnotationPIPNameDualStack[isIPv6]]; ok && name != "" { + return name + } + return service.Annotations[consts.ServiceAnnotationPIPName] +} + +func getServicePIPPrefixID(service *v1.Service, isIPv6 bool) string { + if service == nil { + return "" } - service.Annotations[consts.ServiceAnnotationLoadBalancerIPDualStack[true]] = ip + + if name, ok := service.Annotations[consts.ServiceAnnotationPIPPrefixIDDualStack[isIPv6]]; ok && name != "" { + return name + } + return service.Annotations[consts.ServiceAnnotationPIPPrefixID] } // GetLoadBalancer returns whether the specified load balancer and its components exist, and // if so, what its status is. func (az *Cloud) GetLoadBalancer(ctx context.Context, clusterName string, service *v1.Service) (status *v1.LoadBalancerStatus, exists bool, err error) { + v4Enabled, v6Enabled := ifIPFamiliesEnabled1(service) + // Since public IP is not a part of the load balancer on Azure, // there is a chance that we could orphan public IP resources while we delete the load balancer (kubernetes/kubernetes#80571). // We need to make sure the existence of the load balancer depends on the load balancer resource and public IP resource on Azure. - existsPip := func() bool { + existsPipSingleStack := func(isIPv6 bool) bool { var pips []network.PublicIPAddress - pipName, _, err := az.determinePublicIPName(clusterName, service, &pips) + pipName, _, err := az.determinePublicIPName(clusterName, service, &pips, isIPv6) if err != nil { return false } pipResourceGroup := az.getPublicIPAddressResourceGroup(service) + klog.Infof("DEBUG GetLoadBalancer pipname %s", pipName) _, existsPip, err := az.getPublicIPAddress(pipResourceGroup, pipName, azcache.CacheReadTypeDefault) if err != nil { return false } return existsPip + } + + existsPip := func() bool { + pipExists := true + if v4Enabled && !existsPipSingleStack(false) { + pipExists = false + } + if pipExists && v6Enabled && !existsPipSingleStack(true) { + pipExists = false + } + return pipExists }() existingLBs, err := az.ListLB(service) @@ -142,7 +223,7 @@ func (az *Cloud) reconcileService(ctx context.Context, clusterName string, servi } var pips []network.PublicIPAddress - lbStatus, fipConfig, err := az.getServiceLoadBalancerStatus(service, lb, &pips) + lbStatus, lbIPsNoAdditionalPIPs, fipConfigs, err := az.getServiceLoadBalancerStatus(service, lb, &pips) if err != nil { klog.Errorf("getServiceLoadBalancerStatus(%s) failed: %v", serviceName, err) if !errors.Is(err, ErrorNotVmssInstance) { @@ -150,25 +231,26 @@ func (az *Cloud) reconcileService(ctx context.Context, clusterName string, servi } } - var serviceIP *string - if lbStatus != nil && len(lbStatus.Ingress) > 0 { - serviceIP = &lbStatus.Ingress[0].IP + serviceIPs := []string{} + if lbStatus != nil { + for _, ingress := range lbStatus.Ingress { + serviceIPs = append(serviceIPs, ingress.IP) + } } - - klog.V(2).Infof("reconcileService: reconciling security group for service %q with IP %q, wantLb = true", serviceName, logSafe(serviceIP)) - if _, err := az.reconcileSecurityGroup(clusterName, service, serviceIP, lb.Name, true /* wantLb */); err != nil { + klog.V(2).Infof("reconcileService: reconciling security group for service %q with IPs %q, wantLb = true", serviceName, serviceIPs) + if _, err := az.reconcileSecurityGroup(clusterName, service, &serviceIPs, lb.Name, true /* wantLb */); err != nil { klog.Errorf("reconcileSecurityGroup(%s) failed: %#v", serviceName, err) return nil, err } - if fipConfig != nil { + for _, fipConfig := range fipConfigs { if err := az.reconcilePrivateLinkService(clusterName, service, fipConfig, true /* wantPLS */); err != nil { klog.Errorf("reconcilePrivateLinkService(%s) failed: %#v", serviceName, err) return nil, err } } - updateService := updateServiceLoadBalancerIP(service, pointer.StringDeref(serviceIP, "")) + updateService := updateServiceLoadBalancerIPs(service, *lbIPsNoAdditionalPIPs) flippedService := flipServiceInternalAnnotation(updateService) if _, err := az.reconcileLoadBalancer(clusterName, flippedService, nil, false /* wantLb */); err != nil { klog.Errorf("reconcileLoadBalancer(%s) failed: %#v", serviceName, err) @@ -177,7 +259,7 @@ func (az *Cloud) reconcileService(ctx context.Context, clusterName string, servi // lb is not reused here because the ETAG may be changed in above operations, hence reconcilePublicIP() would get lb again from cache. klog.V(2).Infof("reconcileService: reconciling pip") - if _, err := az.reconcilePublicIP(clusterName, updateService, pointer.StringDeref(lb.Name, ""), true /* wantLb */); err != nil { + if _, err := az.reconcilePublicIPs(clusterName, updateService, pointer.StringDeref(lb.Name, ""), true /* wantLb */); err != nil { klog.Errorf("reconcilePublicIP(%s) failed: %#v", serviceName, err) return nil, err } @@ -296,13 +378,19 @@ func (az *Cloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName stri klog.V(5).InfoS("EnsureLoadBalancerDeleted Finish", "service", serviceName, "cluster", clusterName, "service_spec", service, "error", err) }() - serviceIPToCleanup, err := az.findServiceIPAddress(ctx, clusterName, service) + _, lbStatus, _, err := az.getServiceLoadBalancer(service, clusterName, nil, false, []network.LoadBalancer{}) if err != nil && !retry.HasStatusForbiddenOrIgnoredError(err) { return err } + serviceIPsToCleanup := []string{} + if lbStatus != nil { + for _, ingress := range lbStatus.Ingress { + serviceIPsToCleanup = append(serviceIPsToCleanup, ingress.IP) + } + } - klog.V(2).Infof("EnsureLoadBalancerDeleted: reconciling security group for service %q with IP %q, wantLb = false", serviceName, serviceIPToCleanup) - _, err = az.reconcileSecurityGroup(clusterName, service, &serviceIPToCleanup, nil, false /* wantLb */) + klog.V(2).Infof("EnsureLoadBalancerDeleted: reconciling security group for service %q with IPs %q, wantLb = false", serviceName, serviceIPsToCleanup) + _, err = az.reconcileSecurityGroup(clusterName, service, &serviceIPsToCleanup, nil, false /* wantLb */) if err != nil { return err } @@ -318,9 +406,8 @@ func (az *Cloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName stri return err } - _, err = az.reconcilePublicIP(clusterName, service, "", false /* wantLb */) - if err != nil { - return err + if _, err = az.reconcilePublicIPs(clusterName, service, "", false /* wantLb */); err != nil { + return fmt.Errorf("failed to reconciel PIP: %v", err) } klog.V(2).Infof("Delete service (%s): FINISH", serviceName) @@ -452,87 +539,102 @@ func (az *Cloud) cleanOrphanedLoadBalancer(lb *network.LoadBalancer, existingLBs serviceName := getServiceName(service) isBackendPoolPreConfigured := az.isBackendPoolPreConfigured(service) lbResourceGroup := az.getLoadBalancerResourceGroup() - lbBackendPoolName := getBackendPoolName(clusterName, service) - lbBackendPoolID := az.getBackendPoolID(lbName, lbResourceGroup, lbBackendPoolName) + lbBackendPoolIDs := map[bool]string{ + false: az.getBackendPoolID(lbName, lbResourceGroup, getBackendPoolName(clusterName, false)), + true: az.getBackendPoolID(lbName, lbResourceGroup, getBackendPoolName(clusterName, true)), + } if isBackendPoolPreConfigured { klog.V(2).Infof("cleanOrphanedLoadBalancer(%s, %s, %s): ignore cleanup of dirty lb because the lb is pre-configured", lbName, serviceName, clusterName) - } else { - foundLB := false - for _, existingLB := range existingLBs { - if strings.EqualFold(pointer.StringDeref(lb.Name, ""), pointer.StringDeref(existingLB.Name, "")) { - foundLB = true - break - } - } - if !foundLB { - klog.V(2).Infof("cleanOrphanedLoadBalancer: the LB %s doesn't exist, will not delete it", pointer.StringDeref(lb.Name, "")) - return nil + return nil + } + foundLB := false + for _, existingLB := range existingLBs { + if strings.EqualFold(pointer.StringDeref(lb.Name, ""), pointer.StringDeref(existingLB.Name, "")) { + foundLB = true + break } + } + if !foundLB { + klog.V(2).Infof("cleanOrphanedLoadBalancer: the LB %s doesn't exist, will not delete it", pointer.StringDeref(lb.Name, "")) + return nil + } - // When FrontendIPConfigurations is empty, we need to delete the Azure load balancer resource itself, - // because an Azure load balancer cannot have an empty FrontendIPConfigurations collection - klog.V(2).Infof("cleanOrphanedLoadBalancer(%s, %s, %s): deleting the LB since there are no remaining frontendIPConfigurations", lbName, serviceName, clusterName) + // When FrontendIPConfigurations is empty, we need to delete the Azure load balancer resource itself, + // because an Azure load balancer cannot have an empty FrontendIPConfigurations collection + klog.V(2).Infof("cleanOrphanedLoadBalancer(%s, %s, %s): deleting the LB since there are no remaining frontendIPConfigurations", lbName, serviceName, clusterName) - // Remove backend pools from vmSets. This is required for virtual machine scale sets before removing the LB. - vmSetName := az.mapLoadBalancerNameToVMSet(lbName, clusterName) - if _, ok := az.VMSet.(*availabilitySet); ok { - // do nothing for availability set - lb.BackendAddressPools = nil - } + // Remove backend pools from vmSets. This is required for virtual machine scale sets before removing the LB. + vmSetName := az.mapLoadBalancerNameToVMSet(lbName, clusterName) + if _, ok := az.VMSet.(*availabilitySet); ok { + // do nothing for availability set + lb.BackendAddressPools = nil + } - deleteErr := az.safeDeleteLoadBalancer(*lb, clusterName, vmSetName, service) - if deleteErr != nil { - klog.Warningf("cleanOrphanedLoadBalancer(%s, %s, %s): failed to DeleteLB: %v", lbName, serviceName, clusterName, deleteErr) + if deleteErr := az.safeDeleteLoadBalancer(*lb, clusterName, vmSetName, service); deleteErr != nil { + klog.Warningf("cleanOrphanedLoadBalancer(%s, %s, %s): failed to DeleteLB: %v", lbName, serviceName, clusterName, deleteErr) - rgName, vmssName, parseErr := retry.GetVMSSMetadataByRawError(deleteErr) - if parseErr != nil { - klog.Warningf("cleanOrphanedLoadBalancer(%s, %s, %s): failed to parse error: %v", lbName, serviceName, clusterName, parseErr) - return deleteErr.Error() - } - if rgName == "" || vmssName == "" { - klog.Warningf("cleanOrphanedLoadBalancer(%s, %s, %s): empty rgName or vmssName", lbName, serviceName, clusterName) - return deleteErr.Error() - } + rgName, vmssName, parseErr := retry.GetVMSSMetadataByRawError(deleteErr) + if parseErr != nil { + klog.Warningf("cleanOrphanedLoadBalancer(%s, %s, %s): failed to parse error: %v", lbName, serviceName, clusterName, parseErr) + return deleteErr.Error() + } + if rgName == "" || vmssName == "" { + klog.Warningf("cleanOrphanedLoadBalancer(%s, %s, %s): empty rgName or vmssName", lbName, serviceName, clusterName) + return deleteErr.Error() + } - // if we reach here, it means the VM couldn't be deleted because it is being referenced by a VMSS - if _, ok := az.VMSet.(*ScaleSet); !ok { - klog.Warningf("cleanOrphanedLoadBalancer(%s, %s, %s): unexpected VMSet type, expected VMSS", lbName, serviceName, clusterName) - return deleteErr.Error() - } + // if we reach here, it means the VM couldn't be deleted because it is being referenced by a VMSS + if _, ok := az.VMSet.(*ScaleSet); !ok { + klog.Warningf("cleanOrphanedLoadBalancer(%s, %s, %s): unexpected VMSet type, expected VMSS", lbName, serviceName, clusterName) + return deleteErr.Error() + } - if !strings.EqualFold(rgName, az.ResourceGroup) { - return fmt.Errorf("cleanOrphanedLoadBalancer(%s, %s, %s): the VMSS %s is in the resource group %s, but is referencing the LB in %s", lbName, serviceName, clusterName, vmssName, rgName, az.ResourceGroup) - } + if !strings.EqualFold(rgName, az.ResourceGroup) { + return fmt.Errorf("cleanOrphanedLoadBalancer(%s, %s, %s): the VMSS %s is in the resource group %s, but is referencing the LB in %s", lbName, serviceName, clusterName, vmssName, rgName, az.ResourceGroup) + } - vmssNamesMap := map[string]bool{vmssName: true} - err := az.VMSet.EnsureBackendPoolDeletedFromVMSets(vmssNamesMap, lbBackendPoolID) - if err != nil { - klog.Errorf("cleanOrphanedLoadBalancer(%s, %s, %s): failed to EnsureBackendPoolDeletedFromVMSets: %v", lbName, serviceName, clusterName, err) - return err - } + vmssNamesMap := map[string]bool{vmssName: true} + if err := az.VMSet.EnsureBackendPoolDeletedFromVMSets(vmssNamesMap, lbBackendPoolIDs[false]); err != nil { + klog.Errorf("cleanOrphanedLoadBalancer(%s, %s, %s): failed to EnsureBackendPoolDeletedFromVMSets: %v", lbName, serviceName, clusterName, err) + return err + } + if err := az.VMSet.EnsureBackendPoolDeletedFromVMSets(vmssNamesMap, lbBackendPoolIDs[true]); err != nil { + klog.Errorf("cleanOrphanedLoadBalancer(%s, %s, %s): failed to EnsureBackendPoolDeletedFromVMSets: %v", lbName, serviceName, clusterName, err) + return err + } - deleteErr := az.DeleteLB(service, lbName) - if deleteErr != nil { - klog.Errorf("cleanOrphanedLoadBalancer(%s, %s, %s): failed delete lb for the second time, stop retrying: %v", lbName, serviceName, clusterName, deleteErr) - return deleteErr.Error() - } + if deleteErr := az.DeleteLB(service, lbName); deleteErr != nil { + klog.Errorf("cleanOrphanedLoadBalancer(%s, %s, %s): failed delete lb for the second time, stop retrying: %v", lbName, serviceName, clusterName, deleteErr) + return deleteErr.Error() } - klog.V(10).Infof("cleanOrphanedLoadBalancer(%s, %s, %s): az.DeleteLB finished", lbName, serviceName, clusterName) } + klog.V(10).Infof("cleanOrphanedLoadBalancer(%s, %s, %s): az.DeleteLB finished", lbName, serviceName, clusterName) return nil } // safeDeleteLoadBalancer deletes the load balancer after decoupling it from the vmSet func (az *Cloud) safeDeleteLoadBalancer(lb network.LoadBalancer, clusterName, vmSetName string, service *v1.Service) *retry.Error { - lbBackendPoolID := az.getBackendPoolID(pointer.StringDeref(lb.Name, ""), az.getLoadBalancerResourceGroup(), getBackendPoolName(clusterName, service)) - _, err := az.VMSet.EnsureBackendPoolDeleted(service, lbBackendPoolID, vmSetName, lb.BackendAddressPools, true) - if err != nil { - return retry.NewError(false, fmt.Errorf("safeDeleteLoadBalancer: failed to EnsureBackendPoolDeleted: %w", err)) + deleteBackendPool := func(isIPv6 bool) *retry.Error { + lbBackendPoolID := az.getBackendPoolID(pointer.StringDeref(lb.Name, ""), az.getLoadBalancerResourceGroup(), getBackendPoolName(clusterName, isIPv6)) + if _, err := az.VMSet.EnsureBackendPoolDeleted(service, lbBackendPoolID, vmSetName, lb.BackendAddressPools, true); err != nil { + return retry.NewError(false, fmt.Errorf("safeDeleteLoadBalancer: failed to EnsureBackendPoolDeleted: %w", err)) + } + return nil + } + v4Enabled, v6Enabled := ifIPFamiliesEnabled1(service) + if v4Enabled { + if err := deleteBackendPool(false); err != nil { + return err + } + } + if v6Enabled { + if err := deleteBackendPool(true); err != nil { + return err + } } klog.V(2).Infof("safeDeleteLoadBalancer: deleting LB %s", pointer.StringDeref(lb.Name, "")) - rerr := az.DeleteLB(service, pointer.StringDeref(lb.Name, "")) - if rerr != nil { + if rerr := az.DeleteLB(service, pointer.StringDeref(lb.Name, "")); rerr != nil { return rerr } _ = az.lbCache.Delete(pointer.StringDeref(lb.Name, "")) @@ -683,8 +785,9 @@ func (az *Cloud) getServiceLoadBalancer(service *v1.Service, clusterName string, if isInternalLoadBalancer(&existingLB) != isInternal { continue } - var fipConfig *network.FrontendIPConfiguration - status, fipConfig, err = az.getServiceLoadBalancerStatus(service, &existingLB, &pips) + + var fipConfigs []*network.FrontendIPConfiguration + status, lbIPsNoAdditionalPIPs, fipConfigs, err := az.getServiceLoadBalancerStatus(service, &existingLB, &pips) if err != nil { return nil, nil, false, err } @@ -692,14 +795,16 @@ func (az *Cloud) getServiceLoadBalancer(service *v1.Service, clusterName string, // service is not on this load balancer continue } - klog.V(4).Infof("getServiceLoadBalancer(%s, %s, %v): current lb ip: %s", service.Name, clusterName, wantLb, status.Ingress[0].IP) + klog.V(4).Infof("getServiceLoadBalancer(%s, %s, %v): current lb IPs: %v", service.Name, clusterName, wantLb, lbIPsNoAdditionalPIPs) // select another load balancer instead of returning // the current one if the change is needed if wantLb && az.shouldChangeLoadBalancer(service, pointer.StringDeref(existingLB.Name, ""), clusterName) { - if err := az.removeFrontendIPConfigurationFromLoadBalancer(&existingLB, existingLBs, fipConfig, clusterName, service); err != nil { - klog.Errorf("getServiceLoadBalancer(%s, %s, %v): failed to remove frontend IP configuration from load balancer: %v", service.Name, clusterName, wantLb, err) - return nil, nil, false, err + for _, fipConfig := range fipConfigs { + if err := az.removeFrontendIPConfigurationFromLoadBalancer(&existingLB, existingLBs, fipConfig, clusterName, service); err != nil { + klog.Errorf("getServiceLoadBalancer(%s, %s, %v): failed to remove frontend IP configuration %q from load balancer: %v", service.Name, clusterName, wantLb, *fipConfig.Name, err) + return nil, nil, false, err + } } break } @@ -819,21 +924,23 @@ func (az *Cloud) selectLoadBalancer(clusterName string, service *v1.Service, exi } // pips: a non-nil pointer to a slice of existing PIPs, if the slice being pointed to is nil, listPIP would be called when needed and the slice would be filled -func (az *Cloud) getServiceLoadBalancerStatus(service *v1.Service, lb *network.LoadBalancer, pips *[]network.PublicIPAddress) (status *v1.LoadBalancerStatus, fipConfig *network.FrontendIPConfiguration, err error) { +func (az *Cloud) getServiceLoadBalancerStatus(service *v1.Service, lb *network.LoadBalancer, pips *[]network.PublicIPAddress) (status *v1.LoadBalancerStatus, _ *[]string, fipConfigs []*network.FrontendIPConfiguration, err error) { if lb == nil { klog.V(10).Info("getServiceLoadBalancerStatus: lb is nil") - return nil, nil, nil + return nil, nil, nil, nil } if lb.FrontendIPConfigurations == nil || len(*lb.FrontendIPConfigurations) == 0 { klog.V(10).Info("getServiceLoadBalancerStatus: lb.FrontendIPConfigurations is nil") - return nil, nil, nil + return nil, nil, nil, nil } isInternal := requiresInternalLoadBalancer(service) serviceName := getServiceName(service) + lbIngresses := []v1.LoadBalancerIngress{} + lbIPsNoAdditionalPIPs := []string{} for _, ipConfiguration := range *lb.FrontendIPConfigurations { owns, isPrimaryService, err := az.serviceOwnsFrontendIP(ipConfiguration, service, pips) if err != nil { - return nil, nil, fmt.Errorf("get(%s): lb(%s) - failed to filter frontend IP configs with error: %w", serviceName, pointer.StringDeref(lb.Name, ""), err) + return nil, nil, nil, fmt.Errorf("get(%s): lb(%s) - failed to filter frontend IP configs with error: %w", serviceName, pointer.StringDeref(lb.Name, ""), err) } if owns { klog.V(2).Infof("get(%s): lb(%s) - found frontend IP config, primary service: %v", serviceName, pointer.StringDeref(lb.Name, ""), isPrimaryService) @@ -843,19 +950,19 @@ func (az *Cloud) getServiceLoadBalancerStatus(service *v1.Service, lb *network.L lbIP = ipConfiguration.PrivateIPAddress } else { if ipConfiguration.PublicIPAddress == nil { - return nil, nil, fmt.Errorf("get(%s): lb(%s) - failed to get LB PublicIPAddress is Nil", serviceName, *lb.Name) + return nil, nil, nil, fmt.Errorf("get(%s): lb(%s) - failed to get LB PublicIPAddress is Nil", serviceName, *lb.Name) } pipID := ipConfiguration.PublicIPAddress.ID if pipID == nil { - return nil, nil, fmt.Errorf("get(%s): lb(%s) - failed to get LB PublicIPAddress ID is Nil", serviceName, *lb.Name) + return nil, nil, nil, fmt.Errorf("get(%s): lb(%s) - failed to get LB PublicIPAddress ID is Nil", serviceName, *lb.Name) } pipName, err := getLastSegment(*pipID, "/") if err != nil { - return nil, nil, fmt.Errorf("get(%s): lb(%s) - failed to get LB PublicIPAddress Name from ID(%s)", serviceName, *lb.Name, *pipID) + return nil, nil, nil, fmt.Errorf("get(%s): lb(%s) - failed to get LB PublicIPAddress Name from ID(%s)", serviceName, *lb.Name, *pipID) } pip, existsPip, err := az.getPublicIPAddress(az.getPublicIPAddressResourceGroup(service), pipName, azcache.CacheReadTypeDefault) if err != nil { - return nil, nil, err + return nil, nil, nil, err } if existsPip { lbIP = pip.IPAddress @@ -864,47 +971,48 @@ func (az *Cloud) getServiceLoadBalancerStatus(service *v1.Service, lb *network.L klog.V(2).Infof("getServiceLoadBalancerStatus gets ingress IP %q from frontendIPConfiguration %q for service %q", pointer.StringDeref(lbIP, ""), pointer.StringDeref(ipConfiguration.Name, ""), serviceName) - // set additional public IPs to LoadBalancerStatus, so that kube-proxy would create their iptables rules. - lbIngress := []v1.LoadBalancerIngress{{IP: pointer.StringDeref(lbIP, "")}} - additionalIPs, err := getServiceAdditionalPublicIPs(service) - if err != nil { - return &v1.LoadBalancerStatus{Ingress: lbIngress}, &ipConfiguration, err - } - if len(additionalIPs) > 0 { - for _, pip := range additionalIPs { - lbIngress = append(lbIngress, v1.LoadBalancerIngress{ - IP: pip, - }) - } - } - - return &v1.LoadBalancerStatus{Ingress: lbIngress}, &ipConfiguration, nil + lbIngresses = append(lbIngresses, v1.LoadBalancerIngress{IP: pointer.StringDeref(lbIP, "")}) + lbIPsNoAdditionalPIPs = append(lbIPsNoAdditionalPIPs, pointer.StringDeref(lbIP, "")) + fipConfigs = append(fipConfigs, &ipConfiguration) } } - return nil, nil, nil + // set additional public IPs to LoadBalancerStatus, so that kube-proxy would create their iptables rules. + additionalIPs, err := getServiceAdditionalPublicIPs(service) + if err != nil { + return &v1.LoadBalancerStatus{Ingress: lbIngresses}, &lbIPsNoAdditionalPIPs, fipConfigs, err + } + if len(additionalIPs) > 0 { + for _, pip := range additionalIPs { + lbIngresses = append(lbIngresses, v1.LoadBalancerIngress{ + IP: pip, + }) + } + } + return &v1.LoadBalancerStatus{Ingress: lbIngresses}, &lbIPsNoAdditionalPIPs, fipConfigs, nil } -// pips: a non-nil pointer to a slice of existing PIPs, if the slice being pointed to is nil, listPIP would be called when needed and the slice would be filled -func (az *Cloud) determinePublicIPName(clusterName string, service *v1.Service, pips *[]network.PublicIPAddress) (string, bool, error) { - if name, found := service.Annotations[consts.ServiceAnnotationPIPName]; found && name != "" { +func (az *Cloud) determinePublicIPName(clusterName string, service *v1.Service, pips *[]network.PublicIPAddress, isIPv6 bool) (string, bool, error) { + if name := getServicePIPName(service, isIPv6); name != "" { return name, true, nil } - if ipPrefix, ok := service.Annotations[consts.ServiceAnnotationPIPPrefixID]; ok && ipPrefix != "" { - return az.getPublicIPName(clusterName, service), false, nil + + if id := getServicePIPPrefixID(service, isIPv6); id != "" { + return az.getPublicIPName(clusterName, service, isIPv6), false, nil } pipResourceGroup := az.getPublicIPAddressResourceGroup(service) - loadBalancerIP := getServiceLoadBalancerIP(service) + loadBalancerIP := getServiceLoadBalancerIP(service, isIPv6) // Assume that the service without loadBalancerIP set is a primary service. // If a secondary service doesn't set the loadBalancerIP, it is not allowed to share the IP. if len(loadBalancerIP) == 0 { - return az.getPublicIPName(clusterName, service), false, nil + return az.getPublicIPName(clusterName, service, isIPv6), false, nil } // For the services with loadBalancerIP set, an existing public IP is required, primary // or secondary, or a public IP not found error would be reported. + klog.Infof("DEBUG before findMatchedPIPByLoadBalancerIP %s", loadBalancerIP) pip, err := az.findMatchedPIPByLoadBalancerIP(service, loadBalancerIP, pipResourceGroup, pips) if err != nil { return "", false, err @@ -918,17 +1026,25 @@ func (az *Cloud) determinePublicIPName(clusterName string, service *v1.Service, } // pips: a non-nil pointer to a slice of existing PIPs, if the slice being pointed to is nil, listPIP would be called when needed and the slice would be filled -func (az *Cloud) findMatchedPIPByLoadBalancerIP(service *v1.Service, loadBalancerIP, pipResourceGroup string, pips *[]network.PublicIPAddress) (*network.PublicIPAddress, error) { +func (az *Cloud) ensurePIP(service *v1.Service, pipResourceGroup string, pips *[]network.PublicIPAddress) error { if pips == nil { // this should not happen - return nil, fmt.Errorf("findMatchedPIPByLoadBalancerIP: nil pip list passed") + return fmt.Errorf("nil pip list passed") } else if *pips == nil { pipList, err := az.listPIP(pipResourceGroup) if err != nil { - return nil, err + return err } *pips = pipList } + return nil +} + +func (az *Cloud) findMatchedPIPByLoadBalancerIP(service *v1.Service, loadBalancerIP, pipResourceGroup string, pips *[]network.PublicIPAddress) (*network.PublicIPAddress, error) { + if err := az.ensurePIP(service, pipResourceGroup, pips); err != nil { + return nil, fmt.Errorf("findMatchedPIPByLoadBalancerIP: failed to ensurePIP: %v", err) + } + for _, pip := range *pips { if pip.PublicIPAddressPropertiesFormat.IPAddress != nil && *pip.PublicIPAddressPropertiesFormat.IPAddress == loadBalancerIP { @@ -954,41 +1070,75 @@ func flipServiceInternalAnnotation(service *v1.Service) *v1.Service { return copyService } -func updateServiceLoadBalancerIP(service *v1.Service, serviceIP string) *v1.Service { +func updateServiceLoadBalancerIPs(service *v1.Service, serviceIPs []string) *v1.Service { copyService := service.DeepCopy() - if len(serviceIP) > 0 && copyService != nil { - setServiceLoadBalancerIP(copyService, serviceIP) + if copyService != nil { + for _, serviceIP := range serviceIPs { + if len(serviceIP) == 0 { + continue + } + setServiceLoadBalancerIP(copyService, serviceIP) + } } return copyService } -func (az *Cloud) findServiceIPAddress(ctx context.Context, clusterName string, service *v1.Service) (string, error) { - lbIP := getServiceLoadBalancerIP(service) - if len(lbIP) > 0 { - return lbIP, nil - } - - if len(service.Status.LoadBalancer.Ingress) > 0 && len(service.Status.LoadBalancer.Ingress[0].IP) > 0 { - return service.Status.LoadBalancer.Ingress[0].IP, nil - } - - _, lbStatus, existsLb, err := az.getServiceLoadBalancer(service, clusterName, nil, false, []network.LoadBalancer{}) - if err != nil { - return "", err - } - if !existsLb { - klog.V(2).Infof("Expected to find an IP address for service %s but did not. Assuming it has been removed", service.Name) - return "", nil - } - if len(lbStatus.Ingress) < 1 { - klog.V(2).Infof("Expected to find an IP address for service %s but it had no ingresses. Assuming it has been removed", service.Name) - return "", nil - } - - return lbStatus.Ingress[0].IP, nil -} - -func (az *Cloud) ensurePublicIPExists(service *v1.Service, pipName string, domainNameLabel, clusterName string, shouldPIPExisted, foundDNSLabelAnnotation bool) (*network.PublicIPAddress, error) { +// func (az *Cloud) findServiceIPAddress(ctx context.Context, clusterName string, service *v1.Service) ([]string, error) { +// lbIPs := []string{} +// v4Enabled, v6Enabled := ifIPFamiliesEnabled1(service) + +// // Check Service LB annotation +// ipCount := 0 +// if v4Enabled { +// if ip := getServiceLoadBalancerIP(service, false); ip != "" { +// lbIPs = append(lbIPs, ip) +// } +// ipCount++ +// } +// if v6Enabled { +// if ip := getServiceLoadBalancerIP(service, true); ip != "" { +// lbIPs = append(lbIPs, ip) +// } +// ipCount++ +// } +// if len(lbIPs) == ipCount { +// return lbIPs, nil +// } + +// // Check Service LB ingress +// if ipCount == 1 && len(service.Status.LoadBalancer.Ingress) > 0 && len(service.Status.LoadBalancer.Ingress[0].IP) > 0 { +// lbIPs = []string{service.Status.LoadBalancer.Ingress[0].IP} +// return lbIPs, nil +// } +// if ipCount == 2 && len(service.Status.LoadBalancer.Ingress) > 1 && +// len(service.Status.LoadBalancer.Ingress[0].IP) > 0 && len(service.Status.LoadBalancer.Ingress[1].IP) > 0 { +// lbIPs = []string{service.Status.LoadBalancer.Ingress[0].IP, service.Status.LoadBalancer.Ingress[1].IP} +// return lbIPs, nil +// } + +// // Check Service LB +// _, lbStatus, existsLb, err := az.getServiceLoadBalancer(service, clusterName, nil, false, []network.LoadBalancer{}) +// if err != nil { +// return []string{}, err +// } +// if !existsLb { +// klog.V(2).Infof("Expected to find an IP address for service %s but did not. Assuming it has been removed", service.Name) +// return []string{}, nil +// } +// if (ipCount == 1 && len(lbStatus.Ingress) < 1) || +// (ipCount == 2 && len(lbStatus.Ingress) < 2) { +// klog.V(2).Infof("Expected to find IP addresses for service %s but its ingresses are not enough. Assuming it has been removed", service.Name) +// return []string{}, nil +// } + +// lbIPs = []string{} +// for _, ingress := range lbStatus.Ingress { +// lbIPs = append(lbIPs, ingress.IP) +// } +// return lbIPs, nil +// } + +func (az *Cloud) ensurePublicIPExists(service *v1.Service, pipName string, domainNameLabel, clusterName string, shouldPIPExisted, foundDNSLabelAnnotation, isIPv6 bool) (*network.PublicIPAddress, error) { pipResourceGroup := az.getPublicIPAddressResourceGroup(service) pip, existsPip, err := az.getPublicIPAddress(pipResourceGroup, pipName, azcache.CacheReadTypeDefault) if err != nil { @@ -998,6 +1148,10 @@ func (az *Cloud) ensurePublicIPExists(service *v1.Service, pipName string, domai serviceName := getServiceName(service) var changed bool + ipVersion := network.IPVersionIPv4 + if isIPv6 { + ipVersion = network.IPVersionIPv6 + } if existsPip { // ensure that the service tag is good for managed pips owns, isUserAssignedPIP := serviceOwnsPublicIP(service, &pip, clusterName) @@ -1041,13 +1195,14 @@ func (az *Cloud) ensurePublicIPExists(service *v1.Service, pipName string, domai klog.V(2).Infof("ensurePublicIPExists for service(%s): pip(%s) - updating", serviceName, pointer.StringDeref(pip.Name, "")) if pip.PublicIPAddressPropertiesFormat == nil { pip.PublicIPAddressPropertiesFormat = &network.PublicIPAddressPropertiesFormat{ + PublicIPAddressVersion: ipVersion, PublicIPAllocationMethod: network.IPAllocationMethodStatic, } changed = true } } else { if shouldPIPExisted { - return nil, fmt.Errorf("PublicIP from annotation azure-pip-name=%s for service %s doesn't exist", pipName, serviceName) + return nil, fmt.Errorf("PublicIP from annotation azure-pip-name(-IPv4/-IPv6)=%s for service %s doesn't exist", pipName, serviceName) } changed = true @@ -1062,6 +1217,7 @@ func (az *Cloud) ensurePublicIPExists(service *v1.Service, pipName string, domai } } pip.PublicIPAddressPropertiesFormat = &network.PublicIPAddressPropertiesFormat{ + PublicIPAddressVersion: ipVersion, PublicIPAllocationMethod: network.IPAllocationMethodStatic, IPTags: getServiceIPTagRequestForPublicIP(service).IPTags, } @@ -1077,8 +1233,9 @@ func (az *Cloud) ensurePublicIPExists(service *v1.Service, pipName string, domai pip.Sku = &network.PublicIPAddressSku{ Name: network.PublicIPAddressSkuNameStandard, } - if pipPrefixName, ok := service.Annotations[consts.ServiceAnnotationPIPPrefixID]; ok && pipPrefixName != "" { - pip.PublicIPPrefix = &network.SubResource{ID: pointer.String(pipPrefixName)} + + if id := getServicePIPPrefixID(service, isIPv6); id != "" { + pip.PublicIPPrefix = &network.SubResource{ID: pointer.String(id)} } // skip adding zone info since edge zones doesn't support multiple availability zones. @@ -1141,11 +1298,18 @@ func (az *Cloud) reconcileIPSettings(pip *network.PublicIPAddress, service *v1.S var changed bool serviceName := getServiceName(service) - ipv6 := utilnet.IsIPv6String(service.Spec.ClusterIP) - if ipv6 { - if !strings.EqualFold(string(pip.PublicIPAddressVersion), string(network.IPVersionIPv6)) { + isIPv6 := pip.PublicIPAddressVersion == network.IPVersionIPv6 + var clusterIP string + for _, ip := range service.Spec.ClusterIPs { + if (utilnet.IsIPv6String(ip) && isIPv6) || (!utilnet.IsIPv6String(ip) && !isIPv6) { + clusterIP = ip + } + } + + if isIPv6 { + if !matchingIPVersion(pip.PublicIPAddressVersion, isIPv6) { pip.PublicIPAddressVersion = network.IPVersionIPv6 - klog.V(2).Infof("service(%s): pip(%s) - creating as ipv6 for clusterIP:%v", serviceName, *pip.Name, service.Spec.ClusterIP) + klog.V(2).Infof("service(%s): pip(%s) - creating as IPv6 for clusterIP:%v", serviceName, *pip.Name, clusterIP) changed = true } @@ -1160,9 +1324,9 @@ func (az *Cloud) reconcileIPSettings(pip *network.PublicIPAddress, service *v1.S changed = true } } else { - if !strings.EqualFold(string(pip.PublicIPAddressVersion), string(network.IPVersionIPv4)) { + if !matchingIPVersion(pip.PublicIPAddressVersion, isIPv6) { pip.PublicIPAddressVersion = network.IPVersionIPv4 - klog.V(2).Infof("service(%s): pip(%s) - creating as ipv4 for clusterIP:%v", serviceName, *pip.Name, service.Spec.ClusterIP) + klog.V(2).Infof("service(%s): pip(%s) - creating as IPv4 for clusterIP:%v", serviceName, *pip.Name, clusterIP) changed = true } } @@ -1349,19 +1513,31 @@ func getDomainNameLabel(pip *network.PublicIPAddress) string { } // pips: a non-nil pointer to a slice of existing PIPs, if the slice being pointed to is nil, listPIP would be called when needed and the slice would be filled -func (az *Cloud) isFrontendIPChanged(clusterName string, config network.FrontendIPConfiguration, service *v1.Service, lbFrontendIPConfigName string, pips *[]network.PublicIPAddress) (bool, error) { - isServiceOwnsFrontendIP, isPrimaryService, err := az.serviceOwnsFrontendIP(config, service, pips) +func (az *Cloud) isFrontendIPChanged(clusterName string, config network.FrontendIPConfiguration, service *v1.Service, defaultLBFrontendIPConfigName string, pips *[]network.PublicIPAddress) (bool, error) { + isInternal := requiresInternalLoadBalancer(service) + isIPv6, err := az.ifFIPIPv6(service, &config, pips, isInternal) if err != nil { return false, err } - if isServiceOwnsFrontendIP && isPrimaryService && !strings.EqualFold(pointer.StringDeref(config.Name, ""), lbFrontendIPConfigName) { - return true, nil + klog.Infof("DEBUG isFrontendIPChanged isIPv6: %v", isIPv6) + isServiceOwnsFrontendIP, isPrimaryService, err := az.serviceOwnsFrontendIP(config, service, pips) + if err != nil { + return false, err } - if !strings.EqualFold(pointer.StringDeref(config.Name, ""), lbFrontendIPConfigName) { + lbFrontendIPConfigName := getResourceByIPFamily(defaultLBFrontendIPConfigName, isIPv6) + // klog.Infof("DEBUG isFrontendIPChanged defaultLBFrontendIPConfigName: %s", defaultLBFrontendIPConfigName) + // klog.Infof("DEBUG isFrontendIPChanged lbFrontendIPConfigName: %s", lbFrontendIPConfigName) + // klog.Infof("DEBUG isFrontendIPChanged config.Name: %s", pointer.StringDeref(config.Name)) + if !strings.EqualFold(pointer.StringDeref(config.Name, ""), defaultLBFrontendIPConfigName) && + !strings.EqualFold(pointer.StringDeref(config.Name, ""), lbFrontendIPConfigName) { + if isServiceOwnsFrontendIP && isPrimaryService { + return true, nil + } return false, nil } - loadBalancerIP := getServiceLoadBalancerIP(service) - isInternal := requiresInternalLoadBalancer(service) + + loadBalancerIP := getServiceLoadBalancerIP(service, isIPv6) + klog.Infof("DEBUG isFrontendIPChanged loadBalancerIP: %s", loadBalancerIP) if isInternal { // Judge subnet subnetName := subnet(service) @@ -1379,7 +1555,8 @@ func (az *Cloud) isFrontendIPChanged(clusterName string, config network.Frontend } return loadBalancerIP != "" && !strings.EqualFold(loadBalancerIP, pointer.StringDeref(config.PrivateIPAddress, "")), nil } - pipName, _, err := az.determinePublicIPName(clusterName, service, pips) + pipName, _, err := az.determinePublicIPName(clusterName, service, pips, isIPv6) + klog.Infof("DEBUG isFrontendIPChanged pipName: %s", pipName) if err != nil { return false, err } @@ -1389,8 +1566,11 @@ func (az *Cloud) isFrontendIPChanged(clusterName string, config network.Frontend return false, err } if !existsPip { + klog.Infof("DEBUG isFrontendIPChanged !existsPip") return true, nil } + // klog.Infof("DEBUG isFrontendIPChanged pointer.StringDeref(pip.ID): %s", pointer.StringDeref(pip.ID)) + // klog.Infof("DEBUG isFrontendIPChanged pointer.StringDeref(config.PublicIPAddress.ID): %s", pointer.StringDeref(config.PublicIPAddress.ID)) return config.PublicIPAddress != nil && !strings.EqualFold(pointer.StringDeref(pip.ID, ""), pointer.StringDeref(config.PublicIPAddress.ID, "")), nil } @@ -1509,18 +1689,26 @@ func (az *Cloud) findFrontendIPConfigOfService( fipConfigs *[]network.FrontendIPConfiguration, service *v1.Service, pips *[]network.PublicIPAddress, -) (*network.FrontendIPConfiguration, error) { - for _, config := range *fipConfigs { +) ([]*network.FrontendIPConfiguration, error) { + ownedFIPConfigs := []*network.FrontendIPConfiguration{} + for i := range *fipConfigs { + config := (*fipConfigs)[i] owns, _, err := az.serviceOwnsFrontendIP(config, service, pips) + klog.Infof("DEBUG owns: %s, %v, %v", *config.Name, owns, config.PrivateIPAddressVersion) if err != nil { return nil, err } if owns { - return &config, nil + ownedFIPConfigs = append(ownedFIPConfigs, &config) } } - return nil, nil + return ownedFIPConfigs, nil +} + +// matchingIPVersion checks if IPVersion matches isIPv6. +func matchingIPVersion(ipVersion network.IPVersion, isIPv6 bool) bool { + return (ipVersion == network.IPVersionIPv4 && !isIPv6) || (ipVersion == network.IPVersionIPv6 && isIPv6) } // reconcileLoadBalancer ensures load balancer exists and the frontend ip config is setup. @@ -1530,7 +1718,7 @@ func (az *Cloud) findFrontendIPConfigOfService( func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service, nodes []*v1.Node, wantLb bool) (*network.LoadBalancer, error) { isBackendPoolPreConfigured := az.isBackendPoolPreConfigured(service) serviceName := getServiceName(service) - klog.V(2).Infof("reconcileLoadBalancer for service(%s) - wantLb(%t): started", serviceName, wantLb) + klog.Infof("reconcileLoadBalancer for service(%s) - wantLb(%t): started", serviceName, wantLb) existingLBs, err := az.reconcileSharedLoadBalancer(service, clusterName, nodes) if err != nil { @@ -1546,67 +1734,113 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service, lbName := *lb.Name lbResourceGroup := az.getLoadBalancerResourceGroup() - lbBackendPoolID := az.getBackendPoolID(lbName, az.getLoadBalancerResourceGroup(), getBackendPoolName(clusterName, service)) + lbBackendPoolIDs := map[bool]string{ + false: az.getBackendPoolID(lbName, az.getLoadBalancerResourceGroup(), getBackendPoolName(clusterName, false)), + true: az.getBackendPoolID(lbName, az.getLoadBalancerResourceGroup(), getBackendPoolName(clusterName, true)), + } klog.V(2).Infof("reconcileLoadBalancer for service(%s): lb(%s/%s) wantLb(%t) resolved load balancer name", serviceName, lbResourceGroup, lbName, wantLb) defaultLBFrontendIPConfigName := az.getDefaultFrontendIPConfigName(service) - defaultLBFrontendIPConfigID := az.getFrontendIPConfigID(lbName, lbResourceGroup, defaultLBFrontendIPConfigName) + lbFrontendIPConfigIDs := map[bool]string{ + false: az.getFrontendIPConfigID(lbName, lbResourceGroup, getResourceByIPFamily(defaultLBFrontendIPConfigName, false)), + true: az.getFrontendIPConfigID(lbName, lbResourceGroup, getResourceByIPFamily(defaultLBFrontendIPConfigName, true)), + } dirtyLb := false // reconcile the load balancer's backend pool configuration. if wantLb { + // TODO: reconcile bp is not working to add new ones preConfig, changed, err := az.LoadBalancerBackendPool.ReconcileBackendPools(clusterName, service, lb) if err != nil { return lb, err } if changed { + klog.Infof("DEBUG dirtylb true reconcilebackendpool") dirtyLb = true } isBackendPoolPreConfigured = preConfig } // reconcile the load balancer's frontend IP configurations. - ownedFIPConfig, toDeleteConfigs, changed, err := az.reconcileFrontendIPConfigs(clusterName, service, lb, lbStatus, wantLb, defaultLBFrontendIPConfigName) + var pips []network.PublicIPAddress + ownedFIPConfigs, toDeleteConfigs, changed, err := az.reconcileFrontendIPConfigs(clusterName, service, lb, lbStatus, wantLb, defaultLBFrontendIPConfigName, &pips) if err != nil { return lb, err } if changed { + klog.Infof("DEBUG dirtylb true reconcileFrontendIPConfigs") dirtyLb = true } + // debugPrint(lb, ownedFIPConfigs, "1") + // update probes/rules - if ownedFIPConfig != nil { - if ownedFIPConfig.ID != nil { - defaultLBFrontendIPConfigID = *ownedFIPConfig.ID - } else { + isInternal := requiresInternalLoadBalancer(service) + for _, ownedFIPConfigSingleStack := range ownedFIPConfigs { + if ownedFIPConfigSingleStack == nil || ownedFIPConfigSingleStack.ID == nil { return nil, fmt.Errorf("reconcileLoadBalancer for service (%s)(%t): nil ID for frontend IP config", serviceName, wantLb) } - } - if wantLb { - err = az.checkLoadBalancerResourcesConflicts(lb, defaultLBFrontendIPConfigID, service) + isIPv6, err := az.ifFIPIPv6(service, ownedFIPConfigSingleStack, &pips, isInternal) if err != nil { return nil, err } + lbFrontendIPConfigIDs[isIPv6] = *ownedFIPConfigSingleStack.ID + } + + v4Enabled, v6Enabled := ifIPFamiliesEnabled1(service) + if wantLb && v4Enabled { + if err = az.checkLoadBalancerResourcesConflicts(lb, lbFrontendIPConfigIDs[false], service); err != nil { + return nil, err + } + } + if wantLb && v6Enabled { + if err = az.checkLoadBalancerResourcesConflicts(lb, lbFrontendIPConfigIDs[true], service); err != nil { + return nil, err + } } var expectedProbes []network.Probe var expectedRules []network.LoadBalancingRule - if wantLb { - expectedProbes, expectedRules, err = az.getExpectedLBRules(service, defaultLBFrontendIPConfigID, lbBackendPoolID, lbName) + getExpectedLBRule := func(isIPv6 bool) error { + klog.Infof("DEBUG getExpectedLBRule, lbFrontendIPConfigID: %s, lbBackendPoolID: %s, lbName: %s, isIPv6: %v", lbFrontendIPConfigIDs[isIPv6], lbBackendPoolIDs[isIPv6], lbName, isIPv6) + expectedProbesSingleStack, expectedRulesSingleStack, err := az.getExpectedLBRules(service, lbFrontendIPConfigIDs[isIPv6], lbBackendPoolIDs[isIPv6], lbName, isIPv6) if err != nil { + return err + } + expectedProbes = append(expectedProbes, expectedProbesSingleStack...) + expectedRules = append(expectedRules, expectedRulesSingleStack...) + return nil + } + if wantLb && v4Enabled { + if err := getExpectedLBRule(false); err != nil { + return nil, err + } + } + if wantLb && v6Enabled { + if err := getExpectedLBRule(true); err != nil { return nil, err } } + for _, probe := range expectedProbes { + klog.Infof("DEBUG expected probe name: %s", *probe.Name) + } + for _, rule := range expectedRules { + klog.Infof("DEBUG expected rule name: %s", *rule.Name) + } + if changed := az.reconcileLBProbes(lb, service, serviceName, wantLb, expectedProbes); changed { + klog.Infof("DEBUG dirtylb true reconcileLBProbes") dirtyLb = true } if changed := az.reconcileLBRules(lb, service, serviceName, wantLb, expectedRules); changed { + klog.Infof("DEBUG dirtylb true reconcileLBRules") dirtyLb = true } if changed := az.ensureLoadBalancerTagged(lb); changed { + klog.Infof("DEBUG dirtylb true ensureLoadBalancerTagged") dirtyLb = true } @@ -1665,11 +1899,11 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service, _ = az.lbCache.Delete(lbName) }() - if lb.LoadBalancerPropertiesFormat != nil && lb.BackendAddressPools != nil { - backendPools := *lb.BackendAddressPools - for _, backendPool := range backendPools { - if strings.EqualFold(pointer.StringDeref(backendPool.Name, ""), getBackendPoolName(clusterName, service)) { - if err := az.LoadBalancerBackendPool.EnsureHostsInPool(service, nodes, lbBackendPoolID, vmSetName, clusterName, lbName, backendPool); err != nil { + if lb.LoadBalancerPropertiesFormat != nil && lb.LoadBalancerPropertiesFormat.BackendAddressPools != nil { + for _, backendPool := range *lb.LoadBalancerPropertiesFormat.BackendAddressPools { + isIPv6 := ifBackendPoolIPv6(pointer.StringDeref(backendPool.Name, "")) + if strings.EqualFold(pointer.StringDeref(backendPool.Name, ""), getBackendPoolName(clusterName, isIPv6)) { + if err := az.LoadBalancerBackendPool.EnsureHostsInPool(service, nodes, lbBackendPoolIDs[isIPv6], vmSetName, clusterName, lbName, backendPool, isIPv6); err != nil { return nil, err } } @@ -1681,6 +1915,28 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service, return lb, nil } +func debugPrint(lb *network.LoadBalancer, ownedFIPConfigs []*network.FrontendIPConfiguration, msg string) { + for _, fipc := range ownedFIPConfigs { + if fipc == nil { + continue + } + klog.Infof("DEBUG %s fip config name in ownedFIPConfigs: %v", msg, pointer.StringDeref(fipc.Name, "")) + if fipc.FrontendIPConfigurationPropertiesFormat != nil { + if fipc.FrontendIPConfigurationPropertiesFormat.PublicIPAddress != nil { + klog.Infof("DEBUG %s fip config pip id in ownedFIPConfigs: %v", msg, pointer.StringDeref(fipc.FrontendIPConfigurationPropertiesFormat.PublicIPAddress.ID, "")) + } + } + } + if lb.FrontendIPConfigurations == nil { + return + } + for _, fipc := range *lb.FrontendIPConfigurations { + if fipc.PublicIPAddress != nil { + klog.Infof("DEBUG %s fip pip id: %v", msg, pointer.StringDeref(fipc.PublicIPAddress.ID, "")) + } + } +} + func (az *Cloud) reconcileLBProbes(lb *network.LoadBalancer, service *v1.Service, serviceName string, wantLb bool, expectedProbes []network.Probe) bool { // remove unwanted probes dirtyProbes := false @@ -1740,7 +1996,7 @@ func (az *Cloud) reconcileLBRules(lb *network.LoadBalancer, service *v1.Service, keepRule := false klog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb rule(%s) - considering evicting", serviceName, wantLb, *existingRule.Name) if findRule(expectedRules, existingRule, wantLb) { - klog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb rule(%s) - keeping", serviceName, wantLb, *existingRule.Name) + klog.Infof("reconcileLoadBalancer for service (%s)(%t): lb rule(%s) - keeping", serviceName, wantLb, *existingRule.Name) keepRule = true } if !keepRule { @@ -1771,7 +2027,7 @@ func (az *Cloud) reconcileLBRules(lb *network.LoadBalancer, service *v1.Service, return dirtyRules } -func (az *Cloud) reconcileFrontendIPConfigs(clusterName string, service *v1.Service, lb *network.LoadBalancer, status *v1.LoadBalancerStatus, wantLb bool, defaultLBFrontendIPConfigName string) (*network.FrontendIPConfiguration, []network.FrontendIPConfiguration, bool, error) { +func (az *Cloud) reconcileFrontendIPConfigs(clusterName string, service *v1.Service, lb *network.LoadBalancer, status *v1.LoadBalancerStatus, wantLb bool, defaultLBFrontendIPConfigName string, pips *[]network.PublicIPAddress) ([]*network.FrontendIPConfiguration, []network.FrontendIPConfiguration, bool, error) { var err error lbName := *lb.Name serviceName := getServiceName(service) @@ -1783,13 +2039,11 @@ func (az *Cloud) reconcileFrontendIPConfigs(clusterName string, service *v1.Serv newConfigs = *lb.FrontendIPConfigurations } - // Save pip list so it can be reused in loop - var pips []network.PublicIPAddress - var ownedFIPConfig *network.FrontendIPConfiguration + var ownedFIPConfigs []*network.FrontendIPConfiguration if !wantLb { for i := len(newConfigs) - 1; i >= 0; i-- { config := newConfigs[i] - isServiceOwnsFrontendIP, _, err := az.serviceOwnsFrontendIP(config, service, &pips) + isServiceOwnsFrontendIP, _, err := az.serviceOwnsFrontendIP(config, service, pips) if err != nil { return nil, toDeleteConfigs, false, err } @@ -1821,109 +2075,164 @@ func (az *Cloud) reconcileFrontendIPConfigs(clusterName string, service *v1.Serv } } } else { - var ( - previousZone *[]string - isFipChanged bool - ) + previousZones := map[bool]*[]string{} + isFipChangeds := map[bool]bool{} + + getExpectedOwnedConfigsCount := func(v4Enabled, v6Enabled bool) int { + expectedOwnedConfigsCount := 0 + if v4Enabled { + expectedOwnedConfigsCount++ + } + if v6Enabled { + expectedOwnedConfigsCount++ + } + return expectedOwnedConfigsCount + } + v4Enabled, v6Enabled := ifIPFamiliesEnabled1(service) + expectedOwnedConfigsCount := getExpectedOwnedConfigsCount(v4Enabled, v6Enabled) + for i := len(newConfigs) - 1; i >= 0; i-- { config := newConfigs[i] - isServiceOwnsFrontendIP, _, _ := az.serviceOwnsFrontendIP(config, service, &pips) + klog.Infof("DEBUG current fip name: %v", *config.Name) + isServiceOwnsFrontendIP, _, _ := az.serviceOwnsFrontendIP(config, service, pips) if !isServiceOwnsFrontendIP { klog.V(4).Infof("reconcileFrontendIPConfigs for service (%s): the frontend IP configuration %s does not belong to the service", serviceName, pointer.StringDeref(config.Name, "")) continue } klog.V(4).Infof("reconcileFrontendIPConfigs for service (%s): checking owned frontend IP configuration %s", serviceName, pointer.StringDeref(config.Name, "")) - isFipChanged, err = az.isFrontendIPChanged(clusterName, config, service, defaultLBFrontendIPConfigName, &pips) + isFipChanged, err := az.isFrontendIPChanged(clusterName, config, service, defaultLBFrontendIPConfigName, pips) if err != nil { return nil, toDeleteConfigs, false, err } + isIPv6, err := az.ifFIPIPv6(service, &config, pips, isInternal) + if err != nil { + return nil, toDeleteConfigs, false, err + } + isFipChangeds[isIPv6] = isFipChanged if isFipChanged { klog.V(2).Infof("reconcileLoadBalancer for service (%s)(%t): lb frontendconfig(%s) - dropping", serviceName, wantLb, *config.Name) toDeleteConfigs = append(toDeleteConfigs, newConfigs[i]) newConfigs = append(newConfigs[:i], newConfigs[i+1:]...) dirtyConfigs = true - previousZone = config.Zones + previousZones[isIPv6] = config.Zones + } + expectedOwnedConfigsCount-- + if expectedOwnedConfigsCount == 0 { + break } - break } - ownedFIPConfig, err = az.findFrontendIPConfigOfService(&newConfigs, service, &pips) + var err error + ownedFIPConfigs, err = az.findFrontendIPConfigOfService(&newConfigs, service, pips) if err != nil { return nil, toDeleteConfigs, false, err } - if ownedFIPConfig == nil { - klog.V(4).Infof("ensure(%s): lb(%s) - creating a new frontend IP config", serviceName, lbName) + if len(ownedFIPConfigs) < getExpectedOwnedConfigsCount(v4Enabled, v6Enabled) { + klog.Infof("ensure(%s): lb(%s) - creating new frontend IP configs", serviceName, lbName) // construct FrontendIPConfigurationPropertiesFormat - var fipConfigurationProperties *network.FrontendIPConfigurationPropertiesFormat - if isInternal { - subnetName := subnet(service) - if subnetName == nil { - subnetName = &az.SubnetName - } - subnet, existsSubnet, err := az.getSubnet(az.VnetName, *subnetName) + v4Enabled, v6Enabled := ifIPFamiliesEnabled1(service) + needs := map[bool]bool{} + if v4Enabled { + needs[false] = true + } + if v6Enabled { + needs[true] = true + } + for _, ownedFIPConfig := range ownedFIPConfigs { + isIPv6, err := az.ifFIPIPv6(service, ownedFIPConfig, pips, isInternal) if err != nil { return nil, toDeleteConfigs, false, err } + needs[isIPv6] = false + } - if !existsSubnet { - return nil, toDeleteConfigs, false, fmt.Errorf("ensure(%s): lb(%s) - failed to get subnet: %s/%s", serviceName, lbName, az.VnetName, *subnetName) + for isIPv6, needed := range needs { + if !needed { + continue } - - configProperties := network.FrontendIPConfigurationPropertiesFormat{ - Subnet: &subnet, + var fipConfigurationProperties *network.FrontendIPConfigurationPropertiesFormat + ipVersion := network.IPVersionIPv4 + if isIPv6 { + ipVersion = network.IPVersionIPv6 } + if isInternal { + subnetName := subnet(service) + if subnetName == nil { + subnetName = &az.SubnetName + } + subnet, existsSubnet, err := az.getSubnet(az.VnetName, *subnetName) + if err != nil { + return nil, toDeleteConfigs, false, err + } - if utilnet.IsIPv6String(service.Spec.ClusterIP) { - configProperties.PrivateIPAddressVersion = network.IPVersionIPv6 - } + if !existsSubnet { + return nil, toDeleteConfigs, false, + fmt.Errorf("ensure(%s): lb(%s) - failed to get subnet: %s/%s", serviceName, lbName, az.VnetName, *subnetName) + } + + configProperties := network.FrontendIPConfigurationPropertiesFormat{ + Subnet: &subnet, + PrivateIPAddressVersion: ipVersion, + } - loadBalancerIP := getServiceLoadBalancerIP(service) - if loadBalancerIP != "" { - configProperties.PrivateIPAllocationMethod = network.IPAllocationMethodStatic - configProperties.PrivateIPAddress = &loadBalancerIP - } else if status != nil && len(status.Ingress) > 0 && ipInSubnet(status.Ingress[0].IP, &subnet) { - klog.V(4).Infof("reconcileFrontendIPConfigs for service (%s): keep the original private IP %s", serviceName, status.Ingress[0].IP) - configProperties.PrivateIPAllocationMethod = network.IPAllocationMethodStatic - configProperties.PrivateIPAddress = pointer.String(status.Ingress[0].IP) + loadBalancerIP := getServiceLoadBalancerIP(service, isIPv6) + if loadBalancerIP != "" { + configProperties.PrivateIPAllocationMethod = network.IPAllocationMethodStatic + configProperties.PrivateIPAddress = &loadBalancerIP + } else if status != nil && len(status.Ingress) > 0 && ipInSubnet(status.Ingress[0].IP, &subnet) { + klog.V(4).Infof("reconcileFrontendIPConfigs for service (%s): keep the original private IP %s", serviceName, status.Ingress[0].IP) + configProperties.PrivateIPAllocationMethod = network.IPAllocationMethodStatic + configProperties.PrivateIPAddress = pointer.String(status.Ingress[0].IP) + } else { + // We'll need to call GetLoadBalancer later to retrieve allocated IP. + klog.V(4).Infof("reconcileFrontendIPConfigs for service (%s): dynamically allocate the private IP", serviceName) + configProperties.PrivateIPAllocationMethod = network.IPAllocationMethodDynamic + } + + fipConfigurationProperties = &configProperties } else { - // We'll need to call GetLoadBalancer later to retrieve allocated IP. - klog.V(4).Infof("reconcileFrontendIPConfigs for service (%s): dynamically allocate the private IP", serviceName) - configProperties.PrivateIPAllocationMethod = network.IPAllocationMethodDynamic - } + pipName, shouldPIPExisted, err := az.determinePublicIPName(clusterName, service, pips, isIPv6) + if err != nil { + return nil, toDeleteConfigs, false, err + } + klog.Infof("DEBUG pipName: %v", pipName) + domainNameLabel, found := getPublicIPDomainNameLabel(service) + pip, err := az.ensurePublicIPExists(service, pipName, domainNameLabel, clusterName, shouldPIPExisted, found, isIPv6) + if err != nil { + return nil, toDeleteConfigs, false, err + } - fipConfigurationProperties = &configProperties - } else { - pipName, shouldPIPExisted, err := az.determinePublicIPName(clusterName, service, &pips) - if err != nil { - return nil, toDeleteConfigs, false, err - } - domainNameLabel, found := getPublicIPDomainNameLabel(service) - pip, err := az.ensurePublicIPExists(service, pipName, domainNameLabel, clusterName, shouldPIPExisted, found) - if err != nil { - return nil, toDeleteConfigs, false, err - } - fipConfigurationProperties = &network.FrontendIPConfigurationPropertiesFormat{ - PublicIPAddress: &network.PublicIPAddress{ID: pip.ID}, + pipPropertiesFormat := &network.PublicIPAddressPropertiesFormat{ + PublicIPAddressVersion: ipVersion, + } + + fipConfigurationProperties = &network.FrontendIPConfigurationPropertiesFormat{ + PublicIPAddress: &network.PublicIPAddress{ + ID: pip.ID, + PublicIPAddressPropertiesFormat: pipPropertiesFormat, + }, + } } - } - newConfig := network.FrontendIPConfiguration{ - Name: pointer.String(defaultLBFrontendIPConfigName), - ID: pointer.String(fmt.Sprintf(consts.FrontendIPConfigIDTemplate, az.getNetworkResourceSubscriptionID(), az.ResourceGroup, *lb.Name, defaultLBFrontendIPConfigName)), - FrontendIPConfigurationPropertiesFormat: fipConfigurationProperties, - } + fipConfigName := getResourceByIPFamily(defaultLBFrontendIPConfigName, isIPv6) + newConfig := network.FrontendIPConfiguration{ + Name: pointer.String(fipConfigName), + ID: pointer.String(fmt.Sprintf(consts.FrontendIPConfigIDTemplate, az.getNetworkResourceSubscriptionID(), az.ResourceGroup, *lb.Name, fipConfigName)), + FrontendIPConfigurationPropertiesFormat: fipConfigurationProperties, + } - if isInternal { - if err := az.getFrontendZones(&newConfig, previousZone, isFipChanged, serviceName, defaultLBFrontendIPConfigName); err != nil { - klog.Errorf("reconcileLoadBalancer for service (%s)(%t): failed to getFrontendZones: %s", serviceName, wantLb, err.Error()) - return nil, toDeleteConfigs, false, err + if isInternal { + if err := az.getFrontendZones(&newConfig, previousZones[isIPv6], isFipChangeds[isIPv6], serviceName, fipConfigName); err != nil { + klog.Errorf("reconcileLoadBalancer for service (%s)(%t): failed to getFrontendZones: %s", serviceName, wantLb, err.Error()) + return nil, toDeleteConfigs, false, err + } } + newConfigs = append(newConfigs, newConfig) + klog.V(2).Infof("reconcileLoadBalancer for service (%s)(%t): lb frontendconfig(%s) - adding", serviceName, wantLb, fipConfigName) + dirtyConfigs = true } - newConfigs = append(newConfigs, newConfig) - klog.V(2).Infof("reconcileLoadBalancer for service (%s)(%t): lb frontendconfig(%s) - adding", serviceName, wantLb, defaultLBFrontendIPConfigName) - dirtyConfigs = true } } @@ -1931,7 +2240,7 @@ func (az *Cloud) reconcileFrontendIPConfigs(clusterName string, service *v1.Serv lb.FrontendIPConfigurations = &newConfigs } - return ownedFIPConfig, toDeleteConfigs, dirtyConfigs, err + return ownedFIPConfigs, toDeleteConfigs, dirtyConfigs, err } func (az *Cloud) getFrontendZones( @@ -2273,7 +2582,8 @@ func (az *Cloud) getExpectedLBRules( service *v1.Service, lbFrontendIPConfigID string, lbBackendPoolID string, - lbName string) ([]network.Probe, []network.LoadBalancingRule, error) { + lbName string, + isIPv6 bool) ([]network.Probe, []network.LoadBalancingRule, error) { var expectedRules []network.LoadBalancingRule var expectedProbes []network.Probe @@ -2285,7 +2595,7 @@ func (az *Cloud) getExpectedLBRules( var nodeEndpointHealthprobe *network.Probe if servicehelpers.NeedsHealthCheck(service) { podPresencePath, podPresencePort := servicehelpers.GetServiceHealthCheckPathPort(service) - lbRuleName := az.getLoadBalancerRuleName(service, v1.ProtocolTCP, podPresencePort) + lbRuleName := getResourceByIPFamily(az.getLoadBalancerRuleName(service, v1.ProtocolTCP, podPresencePort), isIPv6) nodeEndpointHealthprobe = &network.Probe{ Name: &lbRuleName, @@ -2306,7 +2616,7 @@ func (az *Cloud) getExpectedLBRules( az.useStandardLoadBalancer() && consts.IsK8sServiceHasHAModeEnabled(service) { - lbRuleName := az.getloadbalancerHAmodeRuleName(service) + lbRuleName := getResourceByIPFamily(az.getloadbalancerHAmodeRuleName(service), isIPv6) klog.V(2).Infof("getExpectedLBRules lb name (%s) rule name (%s)", lbName, lbRuleName) props, err := az.getExpectedHAModeLoadBalancingRuleProperties(service, lbFrontendIPConfigID, lbBackendPoolID) @@ -2346,7 +2656,7 @@ func (az *Cloud) getExpectedLBRules( // generate lb rule for each port defined in svc object for _, port := range service.Spec.Ports { - lbRuleName := az.getLoadBalancerRuleName(service, port.Protocol, port.Port) + lbRuleName := getResourceByIPFamily(az.getLoadBalancerRuleName(service, port.Protocol, port.Port), isIPv6) klog.V(2).Infof("getExpectedLBRules lb name (%s) rule name (%s)", lbName, lbRuleName) isNoLBRuleRequired, err := consts.IsLBRuleOnK8sServicePortDisabled(service.Annotations, port.Port) if err != nil { @@ -2401,6 +2711,7 @@ func (az *Cloud) getExpectedLBRules( props.BackendPort = pointer.Int32(port.NodePort) props.EnableFloatingIP = pointer.Bool(false) } + klog.Infof("DEBUG getExpectedLBRules before append") expectedRules = append(expectedRules, network.LoadBalancingRule{ Name: &lbRuleName, LoadBalancingRulePropertiesFormat: props, @@ -2460,7 +2771,7 @@ func (az *Cloud) getExpectedLoadBalancingRulePropertiesForPort( // Azure ILB does not support secondary IPs as floating IPs on the LB. Therefore, floating IP needs to be turned // off and the rule should point to the nodeIP:nodePort. - if consts.IsK8sServiceInternalIPv6(service) { + if consts.IsK8sServiceUsingInternalLoadBalancer(service) && ifBackendPoolIPv6(lbBackendPoolID) { props.BackendPort = pointer.Int32(servicePort.NodePort) props.EnableFloatingIP = pointer.Bool(false) } @@ -2482,7 +2793,8 @@ func (az *Cloud) getExpectedHAModeLoadBalancingRuleProperties( // This reconciles the Network Security Group similar to how the LB is reconciled. // This entails adding required, missing SecurityRules and removing stale rules. -func (az *Cloud) reconcileSecurityGroup(clusterName string, service *v1.Service, lbIP *string, lbName *string, wantLb bool) (*network.SecurityGroup, error) { +// TODO: dualstack +func (az *Cloud) reconcileSecurityGroup(clusterName string, service *v1.Service, lbIPs *[]string, lbName *string, wantLb bool) (*network.SecurityGroup, error) { serviceName := getServiceName(service) klog.V(5).Infof("reconcileSecurityGroup(%s): START clusterName=%q", serviceName, clusterName) @@ -2500,16 +2812,27 @@ func (az *Cloud) reconcileSecurityGroup(clusterName string, service *v1.Service, return nil, err } - destinationIPAddress := "" - if wantLb && lbIP == nil { + if wantLb && lbIPs == nil { return nil, fmt.Errorf("no load balancer IP for setting up security rules for service %s", service.Name) } - if lbIP != nil { - destinationIPAddress = *lbIP + + destinationIPAddresses := map[bool][]string{} + if lbIPs != nil { + for _, ip := range *lbIPs { + klog.Infof("DEBUG lb ip %q", ip) + if net.ParseIP(ip).To4() != nil { + destinationIPAddresses[false] = append(destinationIPAddresses[false], ip) + } else { + destinationIPAddresses[true] = append(destinationIPAddresses[true], ip) + } + } } - if destinationIPAddress == "" { - destinationIPAddress = "*" + if len(destinationIPAddresses[false]) == 0 { + destinationIPAddresses[false] = []string{"*"} + } + if len(destinationIPAddresses[true]) == 0 { + destinationIPAddresses[true] = []string{"*"} } disableFloatingIP := false @@ -2517,7 +2840,7 @@ func (az *Cloud) reconcileSecurityGroup(clusterName string, service *v1.Service, disableFloatingIP = true } - backendIPAddresses := make([]string, 0) + backendIPAddresses := map[bool][]string{} if wantLb && disableFloatingIP { lb, exist, err := az.getAzureLoadBalancer(pointer.StringDeref(lbName, ""), azcache.CacheReadTypeDefault) if err != nil { @@ -2526,22 +2849,10 @@ func (az *Cloud) reconcileSecurityGroup(clusterName string, service *v1.Service, if !exist { return nil, fmt.Errorf("unable to get lb %s", pointer.StringDeref(lbName, "")) } - backendPrivateIPv4s, backendPrivateIPv6s := az.LoadBalancerBackendPool.GetBackendPrivateIPs(clusterName, service, lb) - backendIPAddresses = backendPrivateIPv4s - if utilnet.IsIPv6String(*lbIP) { - backendIPAddresses = backendPrivateIPv6s - } - } - - additionalIPs, err := getServiceAdditionalPublicIPs(service) - if err != nil { - return nil, fmt.Errorf("unable to get additional public IPs, error=%w", err) - } - - destinationIPAddresses := []string{destinationIPAddress} - if destinationIPAddress != "*" { - destinationIPAddresses = append(destinationIPAddresses, additionalIPs...) + backendIPAddresses[false], backendIPAddresses[true] = az.LoadBalancerBackendPool.GetBackendPrivateIPs(clusterName, service, lb) } + klog.Infof("DEBUG destinationIPAddresses[false] %q", destinationIPAddresses[false]) + klog.Infof("DEBUG destinationIPAddresses[true] %q", destinationIPAddresses[true]) sourceRanges, err := servicehelpers.GetLoadBalancerSourceRanges(service) if err != nil { @@ -2552,25 +2863,52 @@ func (az *Cloud) reconcileSecurityGroup(clusterName string, service *v1.Service, delete(sourceRanges, consts.DefaultLoadBalancerSourceRanges) } - var sourceAddressPrefixes []string + sourceAddressPrefixes := map[bool][]string{} if (sourceRanges == nil || servicehelpers.IsAllowAll(sourceRanges)) && len(serviceTags) == 0 { if !requiresInternalLoadBalancer(service) || len(service.Spec.LoadBalancerSourceRanges) > 0 { - sourceAddressPrefixes = []string{"Internet"} + sourceAddressPrefixes[false] = []string{"Internet"} + sourceAddressPrefixes[true] = []string{"Internet"} } } else { for _, ip := range sourceRanges { - sourceAddressPrefixes = append(sourceAddressPrefixes, ip.String()) + if ip == nil { + continue + } + if net.ParseIP(ip.IP.String()).To4() != nil { + sourceAddressPrefixes[false] = append(sourceAddressPrefixes[false], ip.String()) + } else { + sourceAddressPrefixes[true] = append(sourceAddressPrefixes[true], ip.String()) + } } - sourceAddressPrefixes = append(sourceAddressPrefixes, serviceTags...) + sourceAddressPrefixes[false] = append(sourceAddressPrefixes[false], serviceTags...) + sourceAddressPrefixes[true] = append(sourceAddressPrefixes[true], serviceTags...) } + sourceAddressPrefixesAll := append(sourceAddressPrefixes[false], sourceAddressPrefixes[true]...) + destinationIPAddressesAll := append(destinationIPAddresses[false], destinationIPAddresses[true]...) - expectedSecurityRules, err := az.getExpectedSecurityRules(wantLb, ports, sourceAddressPrefixes, service, destinationIPAddresses, sourceRanges, backendIPAddresses, disableFloatingIP) - if err != nil { - return nil, err + v4Enabled, v6Enabled := ifIPFamiliesEnabled1(service) + expectedSecurityRules := []network.SecurityRule{} + handleSecurityRules := func(isIPv6 bool) error { + expectedSecurityRulesSingleStack, err := az.getExpectedSecurityRules(wantLb, ports, sourceAddressPrefixes[isIPv6], service, destinationIPAddresses[isIPv6], sourceRanges, backendIPAddresses[isIPv6], disableFloatingIP, isIPv6) + if err != nil { + return err + } + expectedSecurityRules = append(expectedSecurityRules, expectedSecurityRulesSingleStack...) + return nil + } + if v4Enabled { + if err := handleSecurityRules(false); err != nil { + return nil, err + } + } + if v6Enabled { + if err := handleSecurityRules(true); err != nil { + return nil, err + } } // update security rules - dirtySg, updatedRules, err := az.reconcileSecurityRules(sg, service, serviceName, wantLb, expectedSecurityRules, ports, sourceAddressPrefixes, destinationIPAddresses) + dirtySg, updatedRules, err := az.reconcileSecurityRules(sg, service, serviceName, wantLb, expectedSecurityRules, ports, sourceAddressPrefixesAll, destinationIPAddressesAll) if err != nil { return nil, err } @@ -2614,11 +2952,11 @@ func (az *Cloud) reconcileSecurityRules(sg network.SecurityGroup, service *v1.Se klog.V(10).Infof("reconcile(%s)(%t): sg rule(%s) - considering evicting", serviceName, wantLb, *existingRule.Name) keepRule := false if findSecurityRule(expectedSecurityRules, existingRule) { - klog.V(10).Infof("reconcile(%s)(%t): sg rule(%s) - keeping", serviceName, wantLb, *existingRule.Name) + klog.Infof("reconcile(%s)(%t): sg rule(%s) - keeping", serviceName, wantLb, *existingRule.Name) keepRule = true } if !keepRule { - klog.V(10).Infof("reconcile(%s)(%t): sg rule(%s) - dropping", serviceName, wantLb, *existingRule.Name) + klog.Infof("reconcile(%s)(%t): sg rule(%s) - dropping", serviceName, wantLb, *existingRule.Name) updatedRules = append(updatedRules[:i], updatedRules[i+1:]...) dirtySg = true } @@ -2630,33 +2968,39 @@ func (az *Cloud) reconcileSecurityRules(sg network.SecurityGroup, service *v1.Se if useSharedSecurityRule(service) && !wantLb { for _, port := range ports { for _, sourceAddressPrefix := range sourceAddressPrefixes { - sharedRuleName := az.getSecurityRuleName(service, port, sourceAddressPrefix) - sharedIndex, sharedRule, sharedRuleFound := findSecurityRuleByName(updatedRules, sharedRuleName) - if !sharedRuleFound { - klog.V(4).Infof("Didn't find shared rule %s for service %s", sharedRuleName, service.Name) - continue - } - if sharedRule.DestinationAddressPrefixes == nil { - klog.V(4).Infof("Didn't find DestinationAddressPrefixes in shared rule for service %s", service.Name) - continue - } - existingPrefixes := *sharedRule.DestinationAddressPrefixes - for _, destinationIPAddress := range destinationIPAddresses { - addressIndex, found := findIndex(existingPrefixes, destinationIPAddress) - if !found { - klog.Warningf("Didn't find destination address %v in shared rule %s for service %s", destinationIPAddress, sharedRuleName, service.Name) - continue + handleSharedSecurityRule := func(isIPv6 bool) { + sharedRuleName := az.getSecurityRuleName(service, port, sourceAddressPrefix, isIPv6) + sharedIndex, sharedRule, sharedRuleFound := findSecurityRuleByName(updatedRules, sharedRuleName) + if !sharedRuleFound { + klog.V(4).Infof("Didn't find shared rule %s for service %s", sharedRuleName, service.Name) } - if len(existingPrefixes) == 1 { - updatedRules = append(updatedRules[:sharedIndex], updatedRules[sharedIndex+1:]...) - } else { - newDestinations := append(existingPrefixes[:addressIndex], existingPrefixes[addressIndex+1:]...) - sharedRule.DestinationAddressPrefixes = &newDestinations - updatedRules[sharedIndex] = sharedRule + if sharedRule.DestinationAddressPrefixes == nil { + klog.V(4).Infof("Didn't find DestinationAddressPrefixes in shared rule for service %s", service.Name) + } + existingPrefixes := *sharedRule.DestinationAddressPrefixes + for _, destinationIPAddress := range destinationIPAddresses { + addressIndex, found := findIndex(existingPrefixes, destinationIPAddress) + if !found { + klog.Warningf("Didn't find destination address %v in shared rule %s for service %s", destinationIPAddress, sharedRuleName, service.Name) + continue + } + if len(existingPrefixes) == 1 { + updatedRules = append(updatedRules[:sharedIndex], updatedRules[sharedIndex+1:]...) + } else { + newDestinations := append(existingPrefixes[:addressIndex], existingPrefixes[addressIndex+1:]...) + sharedRule.DestinationAddressPrefixes = &newDestinations + updatedRules[sharedIndex] = sharedRule + } + dirtySg = true } - dirtySg = true } - + v4Enabled, v6Enabled := ifIPFamiliesEnabled1(service) + if v4Enabled { + handleSharedSecurityRule(false) + } + if v6Enabled { + handleSharedSecurityRule(true) + } } } } @@ -2706,7 +3050,7 @@ func (az *Cloud) reconcileSecurityRules(sg network.SecurityGroup, service *v1.Se return dirtySg, updatedRules, nil } -func (az *Cloud) getExpectedSecurityRules(wantLb bool, ports []v1.ServicePort, sourceAddressPrefixes []string, service *v1.Service, destinationIPAddresses []string, sourceRanges utilnet.IPNetSet, backendIPAddresses []string, disableFloatingIP bool) ([]network.SecurityRule, error) { +func (az *Cloud) getExpectedSecurityRules(wantLb bool, ports []v1.ServicePort, sourceAddressPrefixes []string, service *v1.Service, destinationIPAddresses []string, sourceRanges utilnet.IPNetSet, backendIPAddresses []string, disableFloatingIP, isIPv6 bool) ([]network.SecurityRule, error) { expectedSecurityRules := []network.SecurityRule{} if wantLb { @@ -2723,7 +3067,7 @@ func (az *Cloud) getExpectedSecurityRules(wantLb bool, ports []v1.ServicePort, s } for j := range sourceAddressPrefixes { ix := i*len(sourceAddressPrefixes) + j - securityRuleName := az.getSecurityRuleName(service, port, sourceAddressPrefixes[j]) + securityRuleName := az.getSecurityRuleName(service, port, sourceAddressPrefixes[j], isIPv6) nsgRule := network.SecurityRule{ Name: pointer.String(securityRuleName), SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{ @@ -2760,7 +3104,7 @@ func (az *Cloud) getExpectedSecurityRules(wantLb bool, ports []v1.ServicePort, s if err != nil { return nil, err } - securityRuleName := az.getSecurityRuleName(service, port, "deny_all") + securityRuleName := az.getSecurityRuleName(service, port, "deny_all", isIPv6) nsgRule := network.SecurityRule{ Name: pointer.String(securityRuleName), SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{ @@ -2799,6 +3143,7 @@ func (az *Cloud) shouldUpdateLoadBalancer(clusterName string, service *v1.Servic return existsLb && service.ObjectMeta.DeletionTimestamp == nil && service.Spec.Type == v1.ServiceTypeLoadBalancer, nil } +// TODO: maybe remove this func logSafe(s *string) string { if s == nil { return "(nil)" @@ -3017,11 +3362,54 @@ func (az *Cloud) ensurePIPTagged(service *v1.Service, pip *network.PublicIPAddre return changed } -// This reconciles the PublicIP resources similar to how the LB is reconciled. -func (az *Cloud) reconcilePublicIP(clusterName string, service *v1.Service, lbName string, wantLb bool) (*network.PublicIPAddress, error) { +// reconcilePublicIPs reconciles the PublicIP resources similar to how the LB. +func (az *Cloud) reconcilePublicIPs(clusterName string, service *v1.Service, lbName string, wantLb bool) ([]*network.PublicIPAddress, error) { + pipResourceGroup := az.getPublicIPAddressResourceGroup(service) + + reconciledPIPs := []*network.PublicIPAddress{} + pips, err := az.listPIP(pipResourceGroup) + if err != nil { + return nil, err + } + + pipsV4, pipsV6 := []network.PublicIPAddress{}, []network.PublicIPAddress{} + for _, pip := range pips { + if pip.PublicIPAddressPropertiesFormat == nil || pip.PublicIPAddressPropertiesFormat.PublicIPAddressVersion == "" || + pip.PublicIPAddressPropertiesFormat.PublicIPAddressVersion == network.IPVersionIPv4 { + pipsV4 = append(pipsV4, pip) + } else { + pipsV6 = append(pipsV6, pip) + } + } + + v4Enabled, v6Enabled := ifIPFamiliesEnabled1(service) + if v4Enabled { + reconciledPIP, err := az.reconcilePublicIP(pipsV4, clusterName, service, lbName, wantLb, false) + if err != nil { + return reconciledPIPs, err + } + if reconciledPIP != nil { + reconciledPIPs = append(reconciledPIPs, reconciledPIP) + } + } + if v6Enabled { + reconciledPIP, err := az.reconcilePublicIP(pipsV6, clusterName, service, lbName, wantLb, true) + if err != nil { + return reconciledPIPs, err + } + if reconciledPIP != nil { + reconciledPIPs = append(reconciledPIPs, reconciledPIP) + } + } + return reconciledPIPs, nil +} + +// reconcilePublicIP reconciles the PublicIP resources similar to how the LB is reconciled with the specified IP family. +func (az *Cloud) reconcilePublicIP(pips []network.PublicIPAddress, clusterName string, service *v1.Service, lbName string, wantLb, isIPv6 bool) (*network.PublicIPAddress, error) { isInternal := requiresInternalLoadBalancer(service) serviceName := getServiceName(service) serviceIPTagRequest := getServiceIPTagRequestForPublicIP(service) + pipResourceGroup := az.getPublicIPAddressResourceGroup(service) var ( lb *network.LoadBalancer @@ -3030,15 +3418,8 @@ func (az *Cloud) reconcilePublicIP(clusterName string, service *v1.Service, lbNa shouldPIPExisted bool ) - pipResourceGroup := az.getPublicIPAddressResourceGroup(service) - - pips, err := az.listPIP(pipResourceGroup) - if err != nil { - return nil, err - } - if !isInternal && wantLb { - desiredPipName, shouldPIPExisted, err = az.determinePublicIPName(clusterName, service, &pips) + desiredPipName, shouldPIPExisted, err = az.determinePublicIPName(clusterName, service, &pips, isIPv6) if err != nil { return nil, err } @@ -3052,16 +3433,28 @@ func (az *Cloud) reconcilePublicIP(clusterName string, service *v1.Service, lbNa } discoveredDesiredPublicIP, pipsToBeDeleted, deletedDesiredPublicIP, pipsToBeUpdated, err := az.getPublicIPUpdates( - clusterName, service, pips, wantLb, isInternal, desiredPipName, serviceName, serviceIPTagRequest, shouldPIPExisted) + clusterName, service, pips, wantLb, isInternal, desiredPipName, serviceName, serviceIPTagRequest, shouldPIPExisted, isIPv6) if err != nil { return nil, err } + // printPIPNames := func(pips []*network.PublicIPAddress, msg string) { + // strs := []string{} + // for _, pip := range pips { + // if pip != nil { + // strs = append(strs, *pip.Name) + // } + // } + // klog.Infof("DEBUG %s %v", msg, strs) + // } + // printPIPNames(pipsToBeUpdated, "pipsToBeUpdated") + // printPIPNames(pipsToBeDeleted, "pipsToBeDeleted") + var deleteFuncs, updateFuncs []func() error for _, pip := range pipsToBeUpdated { pipCopy := *pip updateFuncs = append(updateFuncs, func() error { - klog.V(2).Infof("reconcilePublicIP for service(%s): pip(%s) - updating", serviceName, *pip.Name) + klog.Infof("reconcilePublicIP for service(%s): pip(%s), isIPv6(%v) - updating", serviceName, *pip.Name, isIPv6) return az.CreateOrUpdatePIP(service, pipResourceGroup, pipCopy) }) } @@ -3073,7 +3466,7 @@ func (az *Cloud) reconcilePublicIP(clusterName string, service *v1.Service, lbNa for _, pip := range pipsToBeDeleted { pipCopy := *pip deleteFuncs = append(deleteFuncs, func() error { - klog.V(2).Infof("reconcilePublicIP for service(%s): pip(%s) - deleting", serviceName, *pip.Name) + klog.V(2).Infof("reconcilePublicIP for service(%s): pip(%s), isIPv6(%v) - deleting", serviceName, *pip.Name, isIPv6) return az.safeDeletePublicIP(service, pipResourceGroup, &pipCopy, lb) }) } @@ -3087,7 +3480,8 @@ func (az *Cloud) reconcilePublicIP(clusterName string, service *v1.Service, lbNa var pip *network.PublicIPAddress domainNameLabel, found := getPublicIPDomainNameLabel(service) errorIfPublicIPDoesNotExist := shouldPIPExisted && discoveredDesiredPublicIP && !deletedDesiredPublicIP - if pip, err = az.ensurePublicIPExists(service, desiredPipName, domainNameLabel, clusterName, errorIfPublicIPDoesNotExist, found); err != nil { + klog.Infof("DEBUG before ensurePublicIPExists %s %s %v %v", desiredPipName, domainNameLabel, errorIfPublicIPDoesNotExist, found) + if pip, err = az.ensurePublicIPExists(service, desiredPipName, domainNameLabel, clusterName, errorIfPublicIPDoesNotExist, found, isIPv6); err != nil { return nil, err } return pip, nil @@ -3095,6 +3489,7 @@ func (az *Cloud) reconcilePublicIP(clusterName string, service *v1.Service, lbNa return nil, nil } +// getPublicIPUpdates ensures pips' IP family matches isIPv6 but still has logic to check. func (az *Cloud) getPublicIPUpdates( clusterName string, service *v1.Service, @@ -3104,7 +3499,8 @@ func (az *Cloud) getPublicIPUpdates( desiredPipName string, serviceName string, serviceIPTagRequest serviceIPTagRequest, - serviceAnnotationRequestsNamedPublicIP bool, + serviceAnnotationRequestsNamedPublicIP, + isIPv6 bool, ) (bool, []*network.PublicIPAddress, bool, []*network.PublicIPAddress, error) { var ( err error @@ -3115,6 +3511,9 @@ func (az *Cloud) getPublicIPUpdates( ) for i := range pips { pip := pips[i] + if pip.Name == nil { + return false, nil, false, nil, fmt.Errorf("PIP name is empty: %v", pip) + } pipName := *pip.Name // If we've been told to use a specific public ip by the client, let's track whether or not it actually existed @@ -3139,7 +3538,10 @@ func (az *Cloud) getPublicIPUpdates( dirtyPIP = true } } + klog.Infof("DEBUG before shouldReleaseExistingOwnedPublicIP %s %v %v %v %s %v", + pipName, wantLb, isInternal, isUserAssignedPIP, desiredPipName, serviceIPTagRequest) if shouldReleaseExistingOwnedPublicIP(&pip, wantLb, isInternal, isUserAssignedPIP, desiredPipName, serviceIPTagRequest) { + klog.Infof("DEBUG after shouldReleaseExistingOwnedPublicIP") // Then, release the public ip pipsToBeDeleted = append(pipsToBeDeleted, &pip) @@ -3493,8 +3895,9 @@ func serviceOwnsPublicIP(service *v1.Service, pip *network.PublicIPAddress, clus clusterTag := getClusterFromPIPClusterTags(pip.Tags) // if there is no service tag on the pip, it is user-created pip + isIPv6 := pip.PublicIPAddressVersion == network.IPVersionIPv6 if serviceTag == "" { - return strings.EqualFold(pointer.StringDeref(pip.IPAddress, ""), getServiceLoadBalancerIP(service)), true + return strings.EqualFold(pointer.StringDeref(pip.IPAddress, ""), getServiceLoadBalancerIP(service, isIPv6)), true } // if there is service tag on the pip, it is system-created pip @@ -3512,7 +3915,7 @@ func serviceOwnsPublicIP(service *v1.Service, pip *network.PublicIPAddress, clus } else { // if the service is not included in the tags of the system-created pip, check the ip address // this could happen for secondary services - return strings.EqualFold(pointer.StringDeref(pip.IPAddress, ""), getServiceLoadBalancerIP(service)), false + return strings.EqualFold(pointer.StringDeref(pip.IPAddress, ""), getServiceLoadBalancerIP(service, isIPv6)), false } } diff --git a/pkg/provider/azure_loadbalancer_backendpool.go b/pkg/provider/azure_loadbalancer_backendpool.go index 8471f11992..21c8f4756a 100644 --- a/pkg/provider/azure_loadbalancer_backendpool.go +++ b/pkg/provider/azure_loadbalancer_backendpool.go @@ -39,7 +39,7 @@ import ( type BackendPool interface { // EnsureHostsInPool ensures the nodes join the backend pool of the load balancer - EnsureHostsInPool(service *v1.Service, nodes []*v1.Node, backendPoolID, vmSetName, clusterName, lbName string, backendPool network.BackendAddressPool) error + EnsureHostsInPool(service *v1.Service, nodes []*v1.Node, backendPoolID, vmSetName, clusterName, lbName string, backendPool network.BackendAddressPool, isIPv6 bool) error // CleanupVMSetFromBackendPoolByCondition removes nodes of the unwanted vmSet from the lb backend pool. // This is needed in two scenarios: @@ -66,14 +66,45 @@ func newBackendPoolTypeNodeIPConfig(c *Cloud) BackendPool { return &backendPoolTypeNodeIPConfig{c} } -func (bc *backendPoolTypeNodeIPConfig) EnsureHostsInPool(service *v1.Service, nodes []*v1.Node, backendPoolID, vmSetName, clusterName, lbName string, backendPool network.BackendAddressPool) error { +func (bc *backendPoolTypeNodeIPConfig) EnsureHostsInPool(service *v1.Service, nodes []*v1.Node, backendPoolID, vmSetName, clusterName, lbName string, backendPool network.BackendAddressPool, _ bool) error { return bc.VMSet.EnsureHostsInPool(service, nodes, backendPoolID, vmSetName) } +func ifLBBackendPoolsExist(lbBackendPoolNames map[bool]string, bpName *string) (found, isIPv6 bool) { + if strings.EqualFold(pointer.StringDeref(bpName, ""), lbBackendPoolNames[false]) { + isIPv6 = false + found = true + } + if strings.EqualFold(pointer.StringDeref(bpName, ""), lbBackendPoolNames[true]) { + isIPv6 = true + found = true + } + return +} + +func getIPFamiliesFromService(service *v1.Service) (v4Enabled, v6Enabled bool) { + for _, ipFamily := range service.Spec.IPFamilies { + if ipFamily == v1.IPv4Protocol { + v4Enabled = true + } else { + v6Enabled = true + } + } + return +} + func (bc *backendPoolTypeNodeIPConfig) CleanupVMSetFromBackendPoolByCondition(slb *network.LoadBalancer, service *v1.Service, nodes []*v1.Node, clusterName string, shouldRemoveVMSetFromSLB func(string) bool) (*network.LoadBalancer, error) { - lbBackendPoolName := getBackendPoolName(clusterName, service) + v4Enabled, v6Enabled := getIPFamiliesFromService(service) + + lbBackendPoolNames := map[bool]string{ + false: getBackendPoolName(clusterName, false), + true: getBackendPoolName(clusterName, true), + } lbResourceGroup := bc.getLoadBalancerResourceGroup() - lbBackendPoolID := bc.getBackendPoolID(pointer.StringDeref(slb.Name, ""), lbResourceGroup, lbBackendPoolName) + lbBackendPoolIDs := map[bool]string{ + false: bc.getBackendPoolID(pointer.StringDeref(slb.Name, ""), lbResourceGroup, lbBackendPoolNames[false]), + true: bc.getBackendPoolID(pointer.StringDeref(slb.Name, ""), lbResourceGroup, lbBackendPoolNames[true]), + } newBackendPools := make([]network.BackendAddressPool, 0) if slb.LoadBalancerPropertiesFormat != nil && slb.BackendAddressPools != nil { newBackendPools = *slb.BackendAddressPools @@ -81,51 +112,67 @@ func (bc *backendPoolTypeNodeIPConfig) CleanupVMSetFromBackendPoolByCondition(sl vmSetNameToBackendIPConfigurationsToBeDeleted := make(map[string][]network.InterfaceIPConfiguration) for j, bp := range newBackendPools { - if strings.EqualFold(pointer.StringDeref(bp.Name, ""), lbBackendPoolName) { - klog.V(2).Infof("bc.CleanupVMSetFromBackendPoolByCondition: checking the backend pool %s from standard load balancer %s", pointer.StringDeref(bp.Name, ""), pointer.StringDeref(slb.Name, "")) - if bp.BackendAddressPoolPropertiesFormat != nil && bp.BackendIPConfigurations != nil { - for i := len(*bp.BackendIPConfigurations) - 1; i >= 0; i-- { - ipConf := (*bp.BackendIPConfigurations)[i] - ipConfigID := pointer.StringDeref(ipConf.ID, "") - _, vmSetName, err := bc.VMSet.GetNodeNameByIPConfigurationID(ipConfigID) - if err != nil && !errors.Is(err, cloudprovider.InstanceNotFound) { - return nil, err - } + if found, _ := ifLBBackendPoolsExist(lbBackendPoolNames, bp.Name); !found { + continue + } + + klog.V(2).Infof("bc.CleanupVMSetFromBackendPoolByCondition: checking the backend pool %s from standard load balancer %s", pointer.StringDeref(bp.Name, ""), pointer.StringDeref(slb.Name, "")) + if bp.BackendAddressPoolPropertiesFormat != nil && bp.BackendIPConfigurations != nil { + for i := len(*bp.BackendIPConfigurations) - 1; i >= 0; i-- { + ipConf := (*bp.BackendIPConfigurations)[i] + ipConfigID := pointer.StringDeref(ipConf.ID, "") + _, vmSetName, err := bc.VMSet.GetNodeNameByIPConfigurationID(ipConfigID) + if err != nil && !errors.Is(err, cloudprovider.InstanceNotFound) { + return nil, err + } - if shouldRemoveVMSetFromSLB(vmSetName) { - klog.V(2).Infof("bc.CleanupVMSetFromBackendPoolByCondition: found unwanted vmSet %s, decouple it from the LB", vmSetName) - // construct a backendPool that only contains the IP config of the node to be deleted - interfaceIPConfigToBeDeleted := network.InterfaceIPConfiguration{ - ID: pointer.String(ipConfigID), - } - vmSetNameToBackendIPConfigurationsToBeDeleted[vmSetName] = append(vmSetNameToBackendIPConfigurationsToBeDeleted[vmSetName], interfaceIPConfigToBeDeleted) - *bp.BackendIPConfigurations = append((*bp.BackendIPConfigurations)[:i], (*bp.BackendIPConfigurations)[i+1:]...) + if shouldRemoveVMSetFromSLB(vmSetName) { + klog.V(2).Infof("bc.CleanupVMSetFromBackendPoolByCondition: found unwanted vmSet %s, decouple it from the LB", vmSetName) + // construct a backendPool that only contains the IP config of the node to be deleted + interfaceIPConfigToBeDeleted := network.InterfaceIPConfiguration{ + ID: pointer.String(ipConfigID), } + vmSetNameToBackendIPConfigurationsToBeDeleted[vmSetName] = append(vmSetNameToBackendIPConfigurationsToBeDeleted[vmSetName], interfaceIPConfigToBeDeleted) + *bp.BackendIPConfigurations = append((*bp.BackendIPConfigurations)[:i], (*bp.BackendIPConfigurations)[i+1:]...) } } - - newBackendPools[j] = bp - break } + + newBackendPools[j] = bp } for vmSetName := range vmSetNameToBackendIPConfigurationsToBeDeleted { + shouldRefreshLB := false backendIPConfigurationsToBeDeleted := vmSetNameToBackendIPConfigurationsToBeDeleted[vmSetName] - backendpoolToBeDeleted := &[]network.BackendAddressPool{ - { - ID: pointer.String(lbBackendPoolID), - BackendAddressPoolPropertiesFormat: &network.BackendAddressPoolPropertiesFormat{ - BackendIPConfigurations: &backendIPConfigurationsToBeDeleted, + deleteBackendPool := func(isIPv6 bool) error { + backendpoolToBeDeleted := &[]network.BackendAddressPool{ + { + ID: pointer.String(lbBackendPoolIDs[isIPv6]), + BackendAddressPoolPropertiesFormat: &network.BackendAddressPoolPropertiesFormat{ + BackendIPConfigurations: &backendIPConfigurationsToBeDeleted, + }, }, - }, + } + // decouple the backendPool from the node + refresh, err := bc.VMSet.EnsureBackendPoolDeleted(service, lbBackendPoolIDs[isIPv6], vmSetName, backendpoolToBeDeleted, true) + if refresh { + shouldRefreshLB = true + } + return err } - // decouple the backendPool from the node - shouldRefreshLB, err := bc.VMSet.EnsureBackendPoolDeleted(service, lbBackendPoolID, vmSetName, backendpoolToBeDeleted, true) - if err != nil { - return nil, err + if v4Enabled { + if err := deleteBackendPool(false); err != nil { + return nil, err + } } + if v6Enabled { + if err := deleteBackendPool(true); err != nil { + return nil, err + } + } + if shouldRefreshLB { - slb, _, err = bc.getAzureLoadBalancer(pointer.StringDeref(slb.Name, ""), cache.CacheReadTypeForceRefresh) + slb, _, err := bc.getAzureLoadBalancer(pointer.StringDeref(slb.Name, ""), cache.CacheReadTypeForceRefresh) if err != nil { return nil, fmt.Errorf("bc.CleanupVMSetFromBackendPoolByCondition: failed to get load balancer %s, err: %w", pointer.StringDeref(slb.Name, ""), err) } @@ -142,98 +189,109 @@ func (bc *backendPoolTypeNodeIPConfig) ReconcileBackendPools(clusterName string, newBackendPools = *lb.BackendAddressPools } - var foundBackendPool, changed, shouldRefreshLB, isOperationSucceeded, isMigration bool + foundBackendPools := map[bool]bool{} + changed := false + shouldRefreshLB := false + var isOperationSucceeded, isMigration bool lbName := *lb.Name serviceName := getServiceName(service) - lbBackendPoolName := getBackendPoolName(clusterName, service) - lbBackendPoolID := bc.getBackendPoolID(lbName, bc.getLoadBalancerResourceGroup(), lbBackendPoolName) + lbBackendPoolNames := map[bool]string{ + false: getBackendPoolName(clusterName, false), + true: getBackendPoolName(clusterName, true), + } + lbBackendPoolIDs := map[bool]string{ + false: bc.getBackendPoolID(lbName, bc.getLoadBalancerResourceGroup(), lbBackendPoolNames[false]), + true: bc.getBackendPoolID(lbName, bc.getLoadBalancerResourceGroup(), lbBackendPoolNames[true]), + } vmSetName := bc.mapLoadBalancerNameToVMSet(lbName, clusterName) isBackendPoolPreConfigured := bc.isBackendPoolPreConfigured(service) mc := metrics.NewMetricContext("services", "migrate_to_ip_based_backend_pool", bc.ResourceGroup, bc.getNetworkResourceSubscriptionID(), serviceName) + klog.Infof("DEBUG ReconcileBackendPools backendPoolTypeNodeIPConfig") for i := len(newBackendPools) - 1; i >= 0; i-- { bp := newBackendPools[i] - if strings.EqualFold(*bp.Name, lbBackendPoolName) { - klog.V(10).Infof("bc.ReconcileBackendPools for service (%s): lb backendpool - found wanted backendpool. not adding anything", serviceName) - foundBackendPool = true + klog.Infof("DEBUG ReconcileBackendPools backendPoolTypeNodeIPConfig %s", pointer.StringDeref(bp.ID, "")) - // Don't bother to remove unused nodeIPConfiguration if backend pool is pre configured - if isBackendPoolPreConfigured { - break - } + found, isIPv6 := ifLBBackendPoolsExist(lbBackendPoolNames, bp.Name) + if !found { + klog.V(10).Infof("bc.ReconcileBackendPools for service (%s): lb backendpool - found unmanaged backendpool %s", serviceName, pointer.StringDeref(bp.Name, "")) + continue + } + klog.V(10).Infof("bc.ReconcileBackendPools for service (%s): lb backendpool - found wanted backendpool. not adding anything", serviceName) + foundBackendPools[ifBackendPoolIPv6(pointer.StringDeref(bp.Name, ""))] = true - // If the LB backend pool type is configured from nodeIP or podIP - // to nodeIPConfiguration, we need to decouple the VM NICs from the LB - // before attaching nodeIPs/podIPs to the LB backend pool. - if bp.BackendAddressPoolPropertiesFormat != nil && - bp.LoadBalancerBackendAddresses != nil && - len(*bp.LoadBalancerBackendAddresses) > 0 { - isMigration = true - - if removeNodeIPAddressesFromBackendPool(bp, []string{}, true) { - if err := bc.CreateOrUpdateLBBackendPool(lbName, bp); err != nil { - klog.Errorf("bc.ReconcileBackendPools for service (%s): failed to cleanup IP based backend pool %s: %s", serviceName, lbBackendPoolName, err.Error()) - return false, false, fmt.Errorf("bc.ReconcileBackendPools for service (%s): failed to cleanup IP based backend pool %s: %w", serviceName, lbBackendPoolName, err) - } - newBackendPools[i] = bp - lb.BackendAddressPools = &newBackendPools - shouldRefreshLB = true + // Don't bother to remove unused nodeIPConfiguration if backend pool is pre configured + if isBackendPoolPreConfigured { + break + } + + // If the LB backend pool type is configured from nodeIP or podIP + // to nodeIPConfiguration, we need to decouple the VM NICs from the LB + // before attaching nodeIPs/podIPs to the LB backend pool. + if bp.BackendAddressPoolPropertiesFormat != nil && + bp.LoadBalancerBackendAddresses != nil && + len(*bp.LoadBalancerBackendAddresses) > 0 { + isMigration = true + if removeNodeIPAddressesFromBackendPool(bp, []string{}, true) { + if err := bc.CreateOrUpdateLBBackendPool(lbName, bp); err != nil { + klog.Errorf("bc.ReconcileBackendPools for service (%s): failed to cleanup IP based backend pool %s: %s", serviceName, lbBackendPoolNames[isIPv6], err.Error()) + return false, false, fmt.Errorf("bc.ReconcileBackendPools for service (%s): failed to cleanup IP based backend pool %s: %w", serviceName, lbBackendPoolNames[isIPv6], err) } + newBackendPools[i] = bp + lb.BackendAddressPools = &newBackendPools + shouldRefreshLB = true } + } - var backendIPConfigurationsToBeDeleted []network.InterfaceIPConfiguration - if bp.BackendAddressPoolPropertiesFormat != nil && bp.BackendIPConfigurations != nil { - for _, ipConf := range *bp.BackendIPConfigurations { - ipConfID := pointer.StringDeref(ipConf.ID, "") - nodeName, _, err := bc.VMSet.GetNodeNameByIPConfigurationID(ipConfID) - if err != nil { - if errors.Is(err, cloudprovider.InstanceNotFound) { - klog.V(2).Infof("bc.ReconcileBackendPools for service (%s): vm not found for ipConfID %s", serviceName, ipConfID) - backendIPConfigurationsToBeDeleted = append(backendIPConfigurationsToBeDeleted, ipConf) - } else { - return false, false, err - } - } - - // If a node is not supposed to be included in the LB, it - // would not be in the `nodes` slice. We need to check the nodes that - // have been added to the LB's backendpool, find the unwanted ones and - // delete them from the pool. - shouldExcludeLoadBalancer, err := bc.ShouldNodeExcludedFromLoadBalancer(nodeName) - if err != nil { - klog.Errorf("bc.ReconcileBackendPools: ShouldNodeExcludedFromLoadBalancer(%s) failed with error: %v", nodeName, err) + var backendIPConfigurationsToBeDeleted []network.InterfaceIPConfiguration + if bp.BackendAddressPoolPropertiesFormat != nil && bp.BackendIPConfigurations != nil { + for _, ipConf := range *bp.BackendIPConfigurations { + ipConfID := pointer.StringDeref(ipConf.ID, "") + nodeName, _, err := bc.VMSet.GetNodeNameByIPConfigurationID(ipConfID) + if err != nil { + if errors.Is(err, cloudprovider.InstanceNotFound) { + klog.V(2).Infof("bc.ReconcileBackendPools for service (%s): vm not found for ipConfID %s", serviceName, ipConfID) + backendIPConfigurationsToBeDeleted = append(backendIPConfigurationsToBeDeleted, ipConf) + } else { return false, false, err } - if shouldExcludeLoadBalancer { - klog.V(2).Infof("bc.ReconcileBackendPools for service (%s): lb backendpool - found unwanted node %s, decouple it from the LB %s", serviceName, nodeName, lbName) - // construct a backendPool that only contains the IP config of the node to be deleted - backendIPConfigurationsToBeDeleted = append(backendIPConfigurationsToBeDeleted, network.InterfaceIPConfiguration{ID: pointer.String(ipConfID)}) - } } - } - if len(backendIPConfigurationsToBeDeleted) > 0 { - backendpoolToBeDeleted := &[]network.BackendAddressPool{ - { - ID: pointer.String(lbBackendPoolID), - BackendAddressPoolPropertiesFormat: &network.BackendAddressPoolPropertiesFormat{ - BackendIPConfigurations: &backendIPConfigurationsToBeDeleted, - }, - }, - } - // decouple the backendPool from the node - updated, err := bc.VMSet.EnsureBackendPoolDeleted(service, lbBackendPoolID, vmSetName, backendpoolToBeDeleted, false) + + // If a node is not supposed to be included in the LB, it + // would not be in the `nodes` slice. We need to check the nodes that + // have been added to the LB's backendpool, find the unwanted ones and + // delete them from the pool. + shouldExcludeLoadBalancer, err := bc.ShouldNodeExcludedFromLoadBalancer(nodeName) if err != nil { + klog.Errorf("bc.ReconcileBackendPools: ShouldNodeExcludedFromLoadBalancer(%s) failed with error: %v", nodeName, err) return false, false, err } - if updated { - shouldRefreshLB = true + if shouldExcludeLoadBalancer { + klog.V(2).Infof("bc.ReconcileBackendPools for service (%s): lb backendpool - found unwanted node %s, decouple it from the LB %s", serviceName, nodeName, lbName) + // construct a backendPool that only contains the IP config of the node to be deleted + backendIPConfigurationsToBeDeleted = append(backendIPConfigurationsToBeDeleted, network.InterfaceIPConfiguration{ID: pointer.String(ipConfID)}) } } - break - } else { - klog.V(10).Infof("bc.ReconcileBackendPools for service (%s): lb backendpool - found unmanaged backendpool %s", serviceName, *bp.Name) + } + if len(backendIPConfigurationsToBeDeleted) > 0 { + backendpoolToBeDeleted := &[]network.BackendAddressPool{ + { + ID: pointer.String(lbBackendPoolIDs[isIPv6]), + BackendAddressPoolPropertiesFormat: &network.BackendAddressPoolPropertiesFormat{ + BackendIPConfigurations: &backendIPConfigurationsToBeDeleted, + }, + }, + } + // decouple the backendPool from the node + updated, err := bc.VMSet.EnsureBackendPoolDeleted(service, lbBackendPoolIDs[isIPv6], vmSetName, backendpoolToBeDeleted, false) + if err != nil { + return false, false, err + } + if updated { + shouldRefreshLB = true + } } } @@ -244,8 +302,14 @@ func (bc *backendPoolTypeNodeIPConfig) ReconcileBackendPools(clusterName string, } } - if !foundBackendPool { - isBackendPoolPreConfigured = newBackendPool(lb, isBackendPoolPreConfigured, bc.PreConfiguredBackendPoolLoadBalancerTypes, getServiceName(service), getBackendPoolName(clusterName, service)) + for _, ipFamily := range service.Spec.IPFamilies { + if foundBackendPools[ipFamily == v1.IPFamily(network.IPVersionIPv6)] { + continue + } + // TODO: be cautious about isBackendPoolPreConfigured + isBackendPoolPreConfigured = newBackendPool(lb, isBackendPoolPreConfigured, + bc.PreConfiguredBackendPoolLoadBalancerTypes, serviceName, + lbBackendPoolNames[ipFamily == v1.IPv6Protocol]) changed = true } @@ -261,41 +325,47 @@ func (bc *backendPoolTypeNodeIPConfig) ReconcileBackendPools(clusterName string, func (bc *backendPoolTypeNodeIPConfig) GetBackendPrivateIPs(clusterName string, service *v1.Service, lb *network.LoadBalancer) ([]string, []string) { serviceName := getServiceName(service) - lbBackendPoolName := getBackendPoolName(clusterName, service) + lbBackendPoolNames := map[bool]string{ + false: getBackendPoolName(clusterName, false), + true: getBackendPoolName(clusterName, true), + } if lb.LoadBalancerPropertiesFormat == nil || lb.LoadBalancerPropertiesFormat.BackendAddressPools == nil { return nil, nil } backendPrivateIPv4s, backendPrivateIPv6s := sets.NewString(), sets.NewString() for _, bp := range *lb.BackendAddressPools { - if strings.EqualFold(pointer.StringDeref(bp.Name, ""), lbBackendPoolName) { - klog.V(10).Infof("bc.GetBackendPrivateIPs for service (%s): found wanted backendpool %s", serviceName, pointer.StringDeref(bp.Name, "")) - if bp.BackendAddressPoolPropertiesFormat != nil && bp.BackendIPConfigurations != nil { - for _, backendIPConfig := range *bp.BackendIPConfigurations { - ipConfigID := pointer.StringDeref(backendIPConfig.ID, "") - nodeName, _, err := bc.VMSet.GetNodeNameByIPConfigurationID(ipConfigID) - if err != nil { - klog.Errorf("bc.GetBackendPrivateIPs for service (%s): GetNodeNameByIPConfigurationID failed with error: %v", serviceName, err) - continue - } - privateIPsSet, ok := bc.nodePrivateIPs[nodeName] - if !ok { - klog.Warningf("bc.GetBackendPrivateIPs for service (%s): failed to get private IPs of node %s", serviceName, nodeName) - continue - } - privateIPs := privateIPsSet.List() - for _, ip := range privateIPs { - klog.V(2).Infof("bc.GetBackendPrivateIPs for service (%s): lb backendpool - found private IPs %s of node %s", serviceName, ip, nodeName) - if utilnet.IsIPv4String(ip) { - backendPrivateIPv4s.Insert(ip) - } else { - backendPrivateIPv6s.Insert(ip) - } - } + found, _ := ifLBBackendPoolsExist(lbBackendPoolNames, bp.Name) + if !found { + klog.V(10).Infof("bc.GetBackendPrivateIPs for service (%s): found unmanaged backendpool %s", serviceName, pointer.StringDeref(bp.Name, "")) + continue + } + + klog.V(10).Infof("bc.GetBackendPrivateIPs for service (%s): found wanted backendpool %s", serviceName, pointer.StringDeref(bp.Name, "")) + if bp.BackendAddressPoolPropertiesFormat == nil || bp.BackendIPConfigurations == nil { + continue + } + for _, backendIPConfig := range *bp.BackendIPConfigurations { + ipConfigID := pointer.StringDeref(backendIPConfig.ID, "") + nodeName, _, err := bc.VMSet.GetNodeNameByIPConfigurationID(ipConfigID) + if err != nil { + klog.Errorf("bc.GetBackendPrivateIPs for service (%s): GetNodeNameByIPConfigurationID failed with error: %v", serviceName, err) + continue + } + privateIPsSet, ok := bc.nodePrivateIPs[nodeName] + if !ok { + klog.Warningf("bc.GetBackendPrivateIPs for service (%s): failed to get private IPs of node %s", serviceName, nodeName) + continue + } + privateIPs := privateIPsSet.List() + for _, ip := range privateIPs { + klog.V(2).Infof("bc.GetBackendPrivateIPs for service (%s): lb backendpool - found private IPs %s of node %s", serviceName, ip, nodeName) + if utilnet.IsIPv4String(ip) { + backendPrivateIPv4s.Insert(ip) + } else { + backendPrivateIPv6s.Insert(ip) } } - } else { - klog.V(10).Infof("bc.GetBackendPrivateIPs for service (%s): found unmanaged backendpool %s", serviceName, pointer.StringDeref(bp.Name, "")) } } return backendPrivateIPv4s.List(), backendPrivateIPv6s.List() @@ -309,7 +379,7 @@ func newBackendPoolTypeNodeIP(c *Cloud) BackendPool { return &backendPoolTypeNodeIP{c} } -func (bi *backendPoolTypeNodeIP) EnsureHostsInPool(service *v1.Service, nodes []*v1.Node, backendPoolID, vmSetName, clusterName, lbName string, backendPool network.BackendAddressPool) error { +func (bi *backendPoolTypeNodeIP) EnsureHostsInPool(service *v1.Service, nodes []*v1.Node, backendPoolID, vmSetName, clusterName, lbName string, backendPool network.BackendAddressPool, isIPv6 bool) error { vnetResourceGroup := bi.ResourceGroup if len(bi.VnetResourceGroup) > 0 { vnetResourceGroup = bi.VnetResourceGroup @@ -318,7 +388,7 @@ func (bi *backendPoolTypeNodeIP) EnsureHostsInPool(service *v1.Service, nodes [] changed := false numOfAdd := 0 - lbBackendPoolName := getBackendPoolName(clusterName, service) + lbBackendPoolName := getBackendPoolName(clusterName, isIPv6) if strings.EqualFold(pointer.StringDeref(backendPool.Name, ""), lbBackendPoolName) && backendPool.BackendAddressPoolPropertiesFormat != nil { if backendPool.LoadBalancerBackendAddresses == nil { @@ -368,11 +438,11 @@ func (bi *backendPoolTypeNodeIP) EnsureHostsInPool(service *v1.Service, nodes [] continue } - privateIP := getNodePrivateIPAddress(service, node) + privateIP := getNodePrivateIPAddress(service, node, isIPv6) if !existingIPs.Has(privateIP) { name := node.Name if utilnet.IsIPv6String(privateIP) { - name = fmt.Sprintf("%s-ipv6", name) + name = fmt.Sprintf("%s-%s", name, v6Suffix) } klog.V(6).Infof("bi.EnsureHostsInPool: adding %s with ip address %s", name, privateIP) @@ -399,52 +469,58 @@ func (bi *backendPoolTypeNodeIP) EnsureHostsInPool(service *v1.Service, nodes [] } func (bi *backendPoolTypeNodeIP) CleanupVMSetFromBackendPoolByCondition(slb *network.LoadBalancer, service *v1.Service, nodes []*v1.Node, clusterName string, shouldRemoveVMSetFromSLB func(string) bool) (*network.LoadBalancer, error) { - lbBackendPoolName := getBackendPoolName(clusterName, service) + lbBackendPoolNames := map[bool]string{ + false: getBackendPoolName(clusterName, false), + true: getBackendPoolName(clusterName, true), + } newBackendPools := make([]network.BackendAddressPool, 0) if slb.LoadBalancerPropertiesFormat != nil && slb.BackendAddressPools != nil { newBackendPools = *slb.BackendAddressPools } - var updatedPrivateIPs bool + updatedPrivateIPs := map[bool]bool{} for j, bp := range newBackendPools { - if strings.EqualFold(pointer.StringDeref(bp.Name, ""), lbBackendPoolName) { - klog.V(2).Infof("bi.CleanupVMSetFromBackendPoolByCondition: checking the backend pool %s from standard load balancer %s", pointer.StringDeref(bp.Name, ""), pointer.StringDeref(slb.Name, "")) - vmIPsToBeDeleted := sets.NewString() - for _, node := range nodes { - vmSetName, err := bi.VMSet.GetNodeVMSetName(node) - if err != nil { - return nil, err - } + found, isIPv6 := ifLBBackendPoolsExist(lbBackendPoolNames, bp.Name) + if !found { + continue + } - if shouldRemoveVMSetFromSLB(vmSetName) { - privateIP := getNodePrivateIPAddress(service, node) - klog.V(4).Infof("bi.CleanupVMSetFromBackendPoolByCondition: removing ip %s from the backend pool %s", privateIP, lbBackendPoolName) - vmIPsToBeDeleted.Insert(privateIP) - } + klog.V(2).Infof("bi.CleanupVMSetFromBackendPoolByCondition: checking the backend pool %s from standard load balancer %s", pointer.StringDeref(bp.Name, ""), pointer.StringDeref(slb.Name, "")) + vmIPsToBeDeleted := sets.NewString() + for _, node := range nodes { + vmSetName, err := bi.VMSet.GetNodeVMSetName(node) + if err != nil { + return nil, err } - if bp.BackendAddressPoolPropertiesFormat != nil && bp.LoadBalancerBackendAddresses != nil { - for i := len(*bp.LoadBalancerBackendAddresses) - 1; i >= 0; i-- { - if (*bp.LoadBalancerBackendAddresses)[i].LoadBalancerBackendAddressPropertiesFormat != nil && - vmIPsToBeDeleted.Has(pointer.StringDeref((*bp.LoadBalancerBackendAddresses)[i].IPAddress, "")) { - *bp.LoadBalancerBackendAddresses = append((*bp.LoadBalancerBackendAddresses)[:i], (*bp.LoadBalancerBackendAddresses)[i+1:]...) - updatedPrivateIPs = true - } - } + if shouldRemoveVMSetFromSLB(vmSetName) { + privateIP := getNodePrivateIPAddress(service, node, isIPv6) + klog.V(4).Infof("bi.CleanupVMSetFromBackendPoolByCondition: removing ip %s from the backend pool %s", privateIP, lbBackendPoolNames[isIPv6]) + vmIPsToBeDeleted.Insert(privateIP) } + } - newBackendPools[j] = bp - break + if bp.BackendAddressPoolPropertiesFormat != nil && bp.LoadBalancerBackendAddresses != nil { + for i := len(*bp.LoadBalancerBackendAddresses) - 1; i >= 0; i-- { + if (*bp.LoadBalancerBackendAddresses)[i].LoadBalancerBackendAddressPropertiesFormat != nil && + vmIPsToBeDeleted.Has(pointer.StringDeref((*bp.LoadBalancerBackendAddresses)[i].IPAddress, "")) { + *bp.LoadBalancerBackendAddresses = append((*bp.LoadBalancerBackendAddresses)[:i], (*bp.LoadBalancerBackendAddresses)[i+1:]...) + updatedPrivateIPs[isIPv6] = true + } + } } + + newBackendPools[j] = bp } - if updatedPrivateIPs { + for isIPv6 := range updatedPrivateIPs { klog.V(2).Infof("bi.CleanupVMSetFromBackendPoolByCondition: updating lb %s since there are private IP updates", pointer.StringDeref(slb.Name, "")) slb.BackendAddressPools = &newBackendPools for _, backendAddressPool := range *slb.BackendAddressPools { - if strings.EqualFold(lbBackendPoolName, pointer.StringDeref(backendAddressPool.Name, "")) { + if strings.EqualFold(lbBackendPoolNames[isIPv6], pointer.StringDeref(backendAddressPool.Name, "")) { if err := bi.CreateOrUpdateLBBackendPool(pointer.StringDeref(slb.Name, ""), backendAddressPool); err != nil { - return nil, fmt.Errorf("bi.CleanupVMSetFromBackendPoolByCondition: failed to create or update backend pool %s: %w", lbBackendPoolName, err) + return nil, fmt.Errorf("bi.CleanupVMSetFromBackendPoolByCondition: "+ + "failed to create or update backend pool %s: %w", lbBackendPoolNames[isIPv6], err) } } } @@ -459,61 +535,72 @@ func (bi *backendPoolTypeNodeIP) ReconcileBackendPools(clusterName string, servi newBackendPools = *lb.BackendAddressPools } - var foundBackendPool, changed, shouldRefreshLB, isOperationSucceeded, isMigration bool + foundBackendPools := map[bool]bool{} + changed := false + shouldRefreshLB := false + var isOperationSucceeded, isMigration bool lbName := *lb.Name serviceName := getServiceName(service) - lbBackendPoolName := getBackendPoolName(clusterName, service) + lbBackendPoolNames := map[bool]string{ + false: getBackendPoolName(clusterName, false), + true: getBackendPoolName(clusterName, true), + } vmSetName := bi.mapLoadBalancerNameToVMSet(lbName, clusterName) - lbBackendPoolID := bi.getBackendPoolID(pointer.StringDeref(lb.Name, ""), bi.getLoadBalancerResourceGroup(), getBackendPoolName(clusterName, service)) + lbBackendPoolIDs := map[bool]string{ + false: bi.getBackendPoolID(pointer.StringDeref(lb.Name, ""), bi.getLoadBalancerResourceGroup(), lbBackendPoolNames[false]), + true: bi.getBackendPoolID(pointer.StringDeref(lb.Name, ""), bi.getLoadBalancerResourceGroup(), lbBackendPoolNames[true]), + } isBackendPoolPreConfigured := bi.isBackendPoolPreConfigured(service) mc := metrics.NewMetricContext("services", "migrate_to_nic_based_backend_pool", bi.ResourceGroup, bi.getNetworkResourceSubscriptionID(), serviceName) - + klog.Infof("DEBUG ReconcileBackendPools backendPoolTypeNodeIP") var err error for i := len(newBackendPools) - 1; i >= 0; i-- { bp := newBackendPools[i] - if strings.EqualFold(*bp.Name, lbBackendPoolName) { - klog.V(10).Infof("bi.ReconcileBackendPools for service (%s): found wanted backendpool. not adding anything", serviceName) - foundBackendPool = true + klog.Infof("DEBUG ReconcileBackendPools backendPoolTypeNodeIP %s", pointer.StringDeref(bp.ID, "")) + found, isIPv6 := ifLBBackendPoolsExist(lbBackendPoolNames, bp.Name) + if !found { + klog.V(10).Infof("bi.ReconcileBackendPools for service (%s): found unmanaged backendpool %s", serviceName, *bp.Name) + continue + } + klog.V(10).Infof("bi.ReconcileBackendPools for service (%s): found wanted backendpool. not adding anything", serviceName) + foundBackendPools[ifBackendPoolIPv6(pointer.StringDeref(bp.Name, ""))] = true - // Don't bother to remove unused nodeIP if backend pool is pre configured - if isBackendPoolPreConfigured { - break - } + // Don't bother to remove unused nodeIP if backend pool is pre configured + if isBackendPoolPreConfigured { + break + } - // If the LB backend pool type is configured from nodeIPConfiguration - // to nodeIP, we need to decouple the VM NICs from the LB - // before attaching nodeIPs/podIPs to the LB backend pool. - klog.V(2).Infof("bi.ReconcileBackendPools for service (%s) and vmSet (%s): ensuring the LB is decoupled from the VMSet", serviceName, vmSetName) - shouldRefreshLB, err = bi.VMSet.EnsureBackendPoolDeleted(service, lbBackendPoolID, vmSetName, lb.BackendAddressPools, true) - if err != nil { - klog.Errorf("bi.ReconcileBackendPools for service (%s): failed to EnsureBackendPoolDeleted: %s", serviceName, err.Error()) - return false, false, err - } + // If the LB backend pool type is configured from nodeIPConfiguration + // to nodeIP, we need to decouple the VM NICs from the LB + // before attaching nodeIPs/podIPs to the LB backend pool. + klog.V(2).Infof("bi.ReconcileBackendPools for service (%s) and vmSet (%s): ensuring the LB is decoupled from the VMSet", serviceName, vmSetName) + shouldRefreshLB, err = bi.VMSet.EnsureBackendPoolDeleted(service, lbBackendPoolIDs[isIPv6], vmSetName, lb.BackendAddressPools, true) + if err != nil { + klog.Errorf("bi.ReconcileBackendPools for service (%s): failed to EnsureBackendPoolDeleted: %s", serviceName, err.Error()) + return false, false, err + } - var nodeIPAddressesToBeDeleted []string - for nodeName := range bi.excludeLoadBalancerNodes { - for ip := range bi.nodePrivateIPs[nodeName] { - klog.V(2).Infof("bi.ReconcileBackendPools for service (%s): found unwanted node private IP %s, decoupling it from the LB %s", serviceName, ip, lbName) - nodeIPAddressesToBeDeleted = append(nodeIPAddressesToBeDeleted, ip) - } + var nodeIPAddressesToBeDeleted []string + for nodeName := range bi.excludeLoadBalancerNodes { + for ip := range bi.nodePrivateIPs[nodeName] { + klog.V(2).Infof("bi.ReconcileBackendPools for service (%s): found unwanted node private IP %s, decoupling it from the LB %s", serviceName, ip, lbName) + nodeIPAddressesToBeDeleted = append(nodeIPAddressesToBeDeleted, ip) } - if len(nodeIPAddressesToBeDeleted) > 0 { - isMigration = true - - updated := removeNodeIPAddressesFromBackendPool(bp, nodeIPAddressesToBeDeleted, false) - if updated { - (*lb.BackendAddressPools)[i] = bp - if err := bi.CreateOrUpdateLBBackendPool(lbName, bp); err != nil { - return false, false, fmt.Errorf("bi.ReconcileBackendPools for service (%s): lb backendpool - failed to update backend pool %s for load balancer %s: %w", serviceName, lbBackendPoolName, lbName, err) - } - shouldRefreshLB = true + } + if len(nodeIPAddressesToBeDeleted) > 0 { + isMigration = true + + updated := removeNodeIPAddressesFromBackendPool(bp, nodeIPAddressesToBeDeleted, false) + if updated { + (*lb.BackendAddressPools)[i] = bp + if err := bi.CreateOrUpdateLBBackendPool(lbName, bp); err != nil { + return false, false, fmt.Errorf("bi.ReconcileBackendPools for service (%s): lb backendpool - failed to update backend pool %s for load balancer %s: %w", serviceName, lbBackendPoolNames[isIPv6], lbName, err) } + shouldRefreshLB = true } - break - } else { - klog.V(10).Infof("bi.ReconcileBackendPools for service (%s): found unmanaged backendpool %s", serviceName, *bp.Name) } + break } if shouldRefreshLB { @@ -523,8 +610,14 @@ func (bi *backendPoolTypeNodeIP) ReconcileBackendPools(clusterName string, servi } } - if !foundBackendPool { - isBackendPoolPreConfigured = newBackendPool(lb, isBackendPoolPreConfigured, bi.PreConfiguredBackendPoolLoadBalancerTypes, getServiceName(service), getBackendPoolName(clusterName, service)) + for _, ipFamily := range service.Spec.IPFamilies { + if foundBackendPools[ipFamily == v1.IPFamily(network.IPVersionIPv6)] { + continue + } + // TODO: be cautious about isBackendPoolPreConfigured + isBackendPoolPreConfigured = newBackendPool(lb, isBackendPoolPreConfigured, + bi.PreConfiguredBackendPoolLoadBalancerTypes, serviceName, + lbBackendPoolNames[ipFamily == v1.IPv6Protocol]) changed = true } @@ -540,32 +633,37 @@ func (bi *backendPoolTypeNodeIP) ReconcileBackendPools(clusterName string, servi func (bi *backendPoolTypeNodeIP) GetBackendPrivateIPs(clusterName string, service *v1.Service, lb *network.LoadBalancer) ([]string, []string) { serviceName := getServiceName(service) - lbBackendPoolName := getBackendPoolName(clusterName, service) + lbBackendPoolNames := map[bool]string{ + false: getBackendPoolName(clusterName, false), + true: getBackendPoolName(clusterName, true), + } if lb.LoadBalancerPropertiesFormat == nil || lb.LoadBalancerPropertiesFormat.BackendAddressPools == nil { return nil, nil } backendPrivateIPv4s, backendPrivateIPv6s := sets.NewString(), sets.NewString() for _, bp := range *lb.BackendAddressPools { - if strings.EqualFold(pointer.StringDeref(bp.Name, ""), lbBackendPoolName) { - klog.V(10).Infof("bi.GetBackendPrivateIPs for service (%s): found wanted backendpool %s", serviceName, pointer.StringDeref(bp.Name, "")) - if bp.BackendAddressPoolPropertiesFormat != nil && bp.LoadBalancerBackendAddresses != nil { - for _, backendAddress := range *bp.LoadBalancerBackendAddresses { - ipAddress := backendAddress.IPAddress - if ipAddress != nil { - klog.V(2).Infof("bi.GetBackendPrivateIPs for service (%s): lb backendpool - found private IP %q", serviceName, *ipAddress) - if utilnet.IsIPv4String(*ipAddress) { - backendPrivateIPv4s.Insert(*ipAddress) - } else { - backendPrivateIPv6s.Insert(*ipAddress) - } + found, _ := ifLBBackendPoolsExist(lbBackendPoolNames, bp.Name) + if !found { + klog.V(10).Infof("bi.GetBackendPrivateIPs for service (%s): found unmanaged backendpool %s", serviceName, pointer.StringDeref(bp.Name, "")) + continue + } + + klog.V(10).Infof("bi.GetBackendPrivateIPs for service (%s): found wanted backendpool %s", serviceName, pointer.StringDeref(bp.Name, "")) + if bp.BackendAddressPoolPropertiesFormat != nil && bp.LoadBalancerBackendAddresses != nil { + for _, backendAddress := range *bp.LoadBalancerBackendAddresses { + ipAddress := backendAddress.IPAddress + if ipAddress != nil { + klog.V(2).Infof("bi.GetBackendPrivateIPs for service (%s): lb backendpool - found private IP %q", serviceName, *ipAddress) + if utilnet.IsIPv4String(*ipAddress) { + backendPrivateIPv4s.Insert(*ipAddress) } else { - klog.V(4).Infof("bi.GetBackendPrivateIPs for service (%s): lb backendpool - found null private IP") + backendPrivateIPv6s.Insert(*ipAddress) } + } else { + klog.V(4).Infof("bi.GetBackendPrivateIPs for service (%s): lb backendpool - found null private IP") } } - } else { - klog.V(10).Infof("bi.GetBackendPrivateIPs for service (%s): found unmanaged backendpool %s", serviceName, pointer.StringDeref(bp.Name, "")) } } return backendPrivateIPv4s.List(), backendPrivateIPv6s.List() @@ -573,9 +671,10 @@ func (bi *backendPoolTypeNodeIP) GetBackendPrivateIPs(clusterName string, servic func newBackendPool(lb *network.LoadBalancer, isBackendPoolPreConfigured bool, preConfiguredBackendPoolLoadBalancerTypes, serviceName, lbBackendPoolName string) bool { if isBackendPoolPreConfigured { - klog.V(2).Infof("newBackendPool for service (%s)(true): lb backendpool - PreConfiguredBackendPoolLoadBalancerTypes %s has been set but can not find corresponding backend pool, ignoring it", + klog.V(2).Infof("newBackendPool for service (%s)(true): lb backendpool - PreConfiguredBackendPoolLoadBalancerTypes %s has been set but can not find corresponding backend pool %q, ignoring it", serviceName, - preConfiguredBackendPoolLoadBalancerTypes) + preConfiguredBackendPoolLoadBalancerTypes, + lbBackendPoolName) isBackendPoolPreConfigured = false } @@ -586,6 +685,7 @@ func newBackendPool(lb *network.LoadBalancer, isBackendPoolPreConfigured bool, p Name: pointer.String(lbBackendPoolName), BackendAddressPoolPropertiesFormat: &network.BackendAddressPoolPropertiesFormat{}, }) + klog.Infof("DEBUG newBackendPool %q", lbBackendPoolName) return isBackendPoolPreConfigured } diff --git a/pkg/provider/azure_loadbalancer_backendpool_test.go b/pkg/provider/azure_loadbalancer_backendpool_test.go index 6904bb8af9..b02fc23bfa 100644 --- a/pkg/provider/azure_loadbalancer_backendpool_test.go +++ b/pkg/provider/azure_loadbalancer_backendpool_test.go @@ -111,7 +111,7 @@ func TestEnsureHostsInPoolNodeIP(t *testing.T) { } service := getTestService("svc-1", v1.ProtocolTCP, nil, false, 80) - err := bi.EnsureHostsInPool(&service, nodes, "", "", "kubernetes", "kubernetes", backendPool) + err := bi.EnsureHostsInPool(&service, nodes, "", "", "kubernetes", "kubernetes", backendPool, false) assert.NoError(t, err) assert.Equal(t, expectedBackendPool, backendPool) } diff --git a/pkg/provider/azure_loadbalancer_test.go b/pkg/provider/azure_loadbalancer_test.go index 3ccaf15050..2ec1d2032c 100644 --- a/pkg/provider/azure_loadbalancer_test.go +++ b/pkg/provider/azure_loadbalancer_test.go @@ -37,6 +37,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/utils/pointer" "sigs.k8s.io/cloud-provider-azure/pkg/azureclients/loadbalancerclient/mockloadbalancerclient" @@ -161,31 +162,34 @@ func TestGetLoadBalancer(t *testing.T) { }, }, } - ctrl := gomock.NewController(t) - defer ctrl.Finish() - for i, c := range tests { - az := GetTestCloud(ctrl) - mockPIPsClient := az.PublicIPAddressesClient.(*mockpublicipclient.MockInterface) - if c.pipExists { - mockPIPsClient.EXPECT().List(gomock.Any(), "rg").Return([]network.PublicIPAddress{ - { - Name: pointer.String("testCluster-aservice"), - PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ - IPAddress: pointer.String("1.2.3.4"), + for _, c := range tests { + t.Run(c.desc, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + az := GetTestCloud(ctrl) + mockPIPsClient := az.PublicIPAddressesClient.(*mockpublicipclient.MockInterface) + if c.pipExists { + mockPIPsClient.EXPECT().List(gomock.Any(), "rg").Return([]network.PublicIPAddress{ + { + Name: pointer.String("testCluster-aservice"), + PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ + IPAddress: pointer.String("1.2.3.4"), + }, }, - }, - }, nil) - } else { - mockPIPsClient.EXPECT().List(gomock.Any(), "rg").Return([]network.PublicIPAddress{}, nil).MaxTimes(2) - } - mockLBsClient := az.LoadBalancerClient.(*mockloadbalancerclient.MockInterface) - mockLBsClient.EXPECT().List(gomock.Any(), az.Config.ResourceGroup).Return(c.existingLBs, nil) + }, nil) + } else { + mockPIPsClient.EXPECT().List(gomock.Any(), "rg").Return([]network.PublicIPAddress{}, nil).MaxTimes(2) + } + mockLBsClient := az.LoadBalancerClient.(*mockloadbalancerclient.MockInterface) + mockLBsClient.EXPECT().List(gomock.Any(), az.Config.ResourceGroup).Return(c.existingLBs, nil) - status, existsLB, err := az.GetLoadBalancer(context.TODO(), testClusterName, &c.service) - assert.Nil(t, err, "TestCase[%d]: %s", i, c.desc) - assert.Equal(t, c.expectedGotLB, existsLB, "TestCase[%d]: %s", i, c.desc) - assert.Equal(t, c.expectedStatus, status, "TestCase[%d]: %s", i, c.desc) + status, existsLB, err := az.GetLoadBalancer(context.TODO(), testClusterName, &c.service) + assert.Nil(t, err) + assert.Equal(t, c.expectedGotLB, existsLB) + assert.Equal(t, c.expectedStatus, status) + }) } } @@ -742,7 +746,6 @@ func TestEnsureLoadBalancerDeleted(t *testing.T) { setMockEnv(az, ctrl, expectedInterfaces, expectedVirtualMachines, 5) for i, c := range tests { - if c.service.Annotations[consts.ServiceAnnotationLoadBalancerInternal] == "true" { validateTestSubnet(t, az, &c.service) } @@ -2115,7 +2118,7 @@ func TestDeterminePublicIPName(t *testing.T) { { desc: "determinePublicIpName shall get public IP from az.getPublicIPName if no specific " + "loadBalancerIP is given", - expectedIP: "testCluster-atest1", + expectedIP: "testCluster-atest1-IPv4", expectedError: false, }, { @@ -2156,7 +2159,7 @@ func TestDeterminePublicIPName(t *testing.T) { assert.NoError(t, err.Error()) } var pips []network.PublicIPAddress - ip, _, err := az.determinePublicIPName("testCluster", &service, &pips) + ip, _, err := az.determinePublicIPName("testCluster", &service, &pips, false) // TODO: support ds assert.Equal(t, test.expectedIP, ip) assert.Equal(t, test.expectedError, err != nil) }) @@ -2622,7 +2625,7 @@ func TestReconcileLoadBalancerRule(t *testing.T) { service.Annotations[consts.BuildHealthProbeAnnotationKeyForPort(firstPort.Port, consts.HealthProbeParamsRequestPath)] = test.probePath } probe, lbrule, err := az.getExpectedLBRules(&test.service, - "frontendIPConfigID", "backendPoolID", "lbname") + "frontendIPConfigID", "backendPoolID", "lbname", false) // TODO: consider ds if test.expectedErr { assert.Error(t, err) @@ -2829,6 +2832,187 @@ func getTestLoadBalancer(name, rgName, clusterName, identifier *string, service return lb } +func getTestLoadBalancerSingleStack(name, rgName, clusterName, identifier *string, service v1.Service, lbSku string, isIPv6 bool) network.LoadBalancer { + suffix := v4Suffix + bpSuffix := "" + if isIPv6 { + suffix = v6Suffix + bpSuffix = "-IPv6" + } + + caser := cases.Title(language.English) + lb := network.LoadBalancer{ + Name: name, + Sku: &network.LoadBalancerSku{ + Name: network.LoadBalancerSkuName(lbSku), + }, + LoadBalancerPropertiesFormat: &network.LoadBalancerPropertiesFormat{ + FrontendIPConfigurations: &[]network.FrontendIPConfiguration{ + { + Name: pointer.String(*identifier + suffix), + ID: pointer.String("/subscriptions/subscription/resourceGroups/" + *rgName + "/providers/" + + "Microsoft.Network/loadBalancers/" + *name + "/frontendIPConfigurations/" + *identifier + "-" + suffix), + FrontendIPConfigurationPropertiesFormat: &network.FrontendIPConfigurationPropertiesFormat{ + PublicIPAddress: &network.PublicIPAddress{ID: pointer.String("testCluster-aservice1" + "-" + suffix)}, + }, + }, + }, + BackendAddressPools: &[]network.BackendAddressPool{ + {Name: pointer.String(*clusterName + bpSuffix)}, + }, + Probes: &[]network.Probe{ + { + Name: pointer.String(*identifier + "-" + string(service.Spec.Ports[0].Protocol) + + "-" + strconv.Itoa(int(service.Spec.Ports[0].Port)) + "-" + suffix), + ProbePropertiesFormat: &network.ProbePropertiesFormat{ + Port: pointer.Int32(10080), + Protocol: network.ProbeProtocolTCP, + IntervalInSeconds: pointer.Int32(5), + NumberOfProbes: pointer.Int32(2), + }, + }, + }, + LoadBalancingRules: &[]network.LoadBalancingRule{ + { + Name: pointer.String(*identifier + "-" + string(service.Spec.Ports[0].Protocol) + + "-" + strconv.Itoa(int(service.Spec.Ports[0].Port)) + "-" + suffix), + LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{ + Protocol: network.TransportProtocol(caser.String((strings.ToLower(string(service.Spec.Ports[0].Protocol))))), + FrontendIPConfiguration: &network.SubResource{ + ID: pointer.String("/subscriptions/subscription/resourceGroups/" + *rgName + "/providers/" + + "Microsoft.Network/loadBalancers/" + *name + "/frontendIPConfigurations/aservice1-" + suffix), + }, + BackendAddressPool: &network.SubResource{ + ID: pointer.String("/subscriptions/subscription/resourceGroups/" + *rgName + "/providers/" + + "Microsoft.Network/loadBalancers/" + *name + "/backendAddressPools/" + *clusterName + "-" + suffix), + }, + LoadDistribution: network.LoadDistribution("Default"), + FrontendPort: pointer.Int32(service.Spec.Ports[0].Port), + BackendPort: pointer.Int32(service.Spec.Ports[0].Port), + EnableFloatingIP: pointer.Bool(true), + EnableTCPReset: pointer.Bool(strings.EqualFold(lbSku, "standard")), + DisableOutboundSnat: pointer.Bool(false), + IdleTimeoutInMinutes: pointer.Int32(4), + Probe: &network.SubResource{ + ID: pointer.String("/subscriptions/subscription/resourceGroups/" + *rgName + "/providers/Microsoft.Network/loadBalancers/testCluster/probes/aservice1-TCP-80-" + suffix), + }, + }, + }, + }, + }, + } + return lb +} + +func getTestLoadBalancerDualStack(name, rgName, clusterName, identifier *string, service v1.Service, lbSku string) network.LoadBalancer { + caser := cases.Title(language.English) + lb := network.LoadBalancer{ + Name: name, + Sku: &network.LoadBalancerSku{ + Name: network.LoadBalancerSkuName(lbSku), + }, + LoadBalancerPropertiesFormat: &network.LoadBalancerPropertiesFormat{ + FrontendIPConfigurations: &[]network.FrontendIPConfiguration{ + { + Name: pointer.String(*identifier + "-IPv4"), + ID: pointer.String("/subscriptions/subscription/resourceGroups/" + *rgName + "/providers/" + + "Microsoft.Network/loadBalancers/" + *name + "/frontendIPConfigurations/" + *identifier + "-IPv4"), + FrontendIPConfigurationPropertiesFormat: &network.FrontendIPConfigurationPropertiesFormat{ + PublicIPAddress: &network.PublicIPAddress{ID: pointer.String("testCluster-aservice1-IPv4")}, + }, + }, + { + Name: pointer.String(*identifier + "-IPv6"), + ID: pointer.String("/subscriptions/subscription/resourceGroups/" + *rgName + "/providers/" + + "Microsoft.Network/loadBalancers/" + *name + "/frontendIPConfigurations/" + *identifier + "-IPv6"), + FrontendIPConfigurationPropertiesFormat: &network.FrontendIPConfigurationPropertiesFormat{ + PublicIPAddress: &network.PublicIPAddress{ID: pointer.String("testCluster-aservice1-IPv6")}, + }, + }, + }, + BackendAddressPools: &[]network.BackendAddressPool{ + {Name: clusterName}, + {Name: pointer.String(*clusterName + "-IPv6")}, + }, + Probes: &[]network.Probe{ + { + Name: pointer.String(*identifier + "-" + string(service.Spec.Ports[0].Protocol) + + "-" + strconv.Itoa(int(service.Spec.Ports[0].Port)) + "-IPv4"), + ProbePropertiesFormat: &network.ProbePropertiesFormat{ + Port: pointer.Int32(10080), + Protocol: network.ProbeProtocolTCP, + IntervalInSeconds: pointer.Int32(5), + NumberOfProbes: pointer.Int32(2), + }, + }, + { + Name: pointer.String(*identifier + "-" + string(service.Spec.Ports[0].Protocol) + + "-" + strconv.Itoa(int(service.Spec.Ports[0].Port)) + "-IPv6"), + ProbePropertiesFormat: &network.ProbePropertiesFormat{ + Port: pointer.Int32(10080), + Protocol: network.ProbeProtocolTCP, + IntervalInSeconds: pointer.Int32(5), + NumberOfProbes: pointer.Int32(2), + }, + }, + }, + LoadBalancingRules: &[]network.LoadBalancingRule{ + { + Name: pointer.String(*identifier + "-" + string(service.Spec.Ports[0].Protocol) + + "-" + strconv.Itoa(int(service.Spec.Ports[0].Port)) + "-IPv4"), + LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{ + Protocol: network.TransportProtocol(caser.String((strings.ToLower(string(service.Spec.Ports[0].Protocol))))), + FrontendIPConfiguration: &network.SubResource{ + ID: pointer.String("/subscriptions/subscription/resourceGroups/" + *rgName + "/providers/" + + "Microsoft.Network/loadBalancers/" + *name + "/frontendIPConfigurations/aservice1-IPv4"), + }, + BackendAddressPool: &network.SubResource{ + ID: pointer.String("/subscriptions/subscription/resourceGroups/" + *rgName + "/providers/" + + "Microsoft.Network/loadBalancers/" + *name + "/backendAddressPools/" + *clusterName + "-IPv4"), + }, + LoadDistribution: network.LoadDistribution("Default"), + FrontendPort: pointer.Int32(service.Spec.Ports[0].Port), + BackendPort: pointer.Int32(service.Spec.Ports[0].Port), + EnableFloatingIP: pointer.Bool(true), + EnableTCPReset: pointer.Bool(strings.EqualFold(lbSku, "standard")), + DisableOutboundSnat: pointer.Bool(false), + IdleTimeoutInMinutes: pointer.Int32(4), + Probe: &network.SubResource{ + ID: pointer.String("/subscriptions/subscription/resourceGroups/" + *rgName + "/providers/Microsoft.Network/loadBalancers/testCluster/probes/aservice1-TCP-80-IPv4"), + }, + }, + }, + { + Name: pointer.String(*identifier + "-" + string(service.Spec.Ports[0].Protocol) + + "-" + strconv.Itoa(int(service.Spec.Ports[0].Port)) + "-IPv6"), + LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{ + Protocol: network.TransportProtocol(caser.String((strings.ToLower(string(service.Spec.Ports[0].Protocol))))), + FrontendIPConfiguration: &network.SubResource{ + ID: pointer.String("/subscriptions/subscription/resourceGroups/" + *rgName + "/providers/" + + "Microsoft.Network/loadBalancers/" + *name + "/frontendIPConfigurations/aservice1-IPv6"), + }, + BackendAddressPool: &network.SubResource{ + ID: pointer.String("/subscriptions/subscription/resourceGroups/" + *rgName + "/providers/" + + "Microsoft.Network/loadBalancers/" + *name + "/backendAddressPools/" + *clusterName + "-IPv6"), + }, + LoadDistribution: network.LoadDistribution("Default"), + FrontendPort: pointer.Int32(service.Spec.Ports[0].Port), + BackendPort: pointer.Int32(service.Spec.Ports[0].Port), + EnableFloatingIP: pointer.Bool(true), + EnableTCPReset: pointer.Bool(strings.EqualFold(lbSku, "standard")), + DisableOutboundSnat: pointer.Bool(false), + IdleTimeoutInMinutes: pointer.Int32(4), + Probe: &network.SubResource{ + ID: pointer.String("/subscriptions/subscription/resourceGroups/" + *rgName + "/providers/Microsoft.Network/loadBalancers/testCluster/probes/aservice1-TCP-80-IPv6"), + }, + }, + }, + }, + }, + } + return lb +} + func TestReconcileLoadBalancer(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -3386,26 +3570,27 @@ func TestGetServiceLoadBalancerStatus(t *testing.T) { for _, test := range testCases { t.Run(test.desc, func(t *testing.T) { - status, _, err := az.getServiceLoadBalancerStatus(test.service, test.lb, nil) + status, _, _, err := az.getServiceLoadBalancerStatus(test.service, test.lb, nil) assert.Equal(t, test.expectedStatus, status) assert.Equal(t, test.expectedError, err != nil) }) } } -func TestReconcileSecurityGroup(t *testing.T) { +func TestReconcileSecurityGroupCommon(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() testCases := []struct { desc string - lbIP *string + lbIPs *[]string lbName *string service v1.Service existingSgs map[string]network.SecurityGroup expectedSg *network.SecurityGroup wantLb bool expectedError bool + ipFamily IPFamily }{ { desc: "reconcileSecurityGroup shall report error if the sg is shared and no ports in service", @@ -3424,7 +3609,7 @@ func TestReconcileSecurityGroup(t *testing.T) { expectedError: true, }, { - desc: "reconcileSecurityGroup shall report error if wantLb is true and lbIP is nil", + desc: "reconcileSecurityGroup shall report error if wantLb is true and lbIPs is nil", service: getTestService("test1", v1.ProtocolTCP, nil, false, 80), wantLb: true, existingSgs: map[string]network.SecurityGroup{"nsg": {}}, @@ -3437,7 +3622,7 @@ func TestReconcileSecurityGroup(t *testing.T) { expectedSg: &network.SecurityGroup{}, }, { - desc: "reconcileSecurityGroup shall delete unwanted sg if wantLb is false and lbIP is nil", + desc: "reconcileSecurityGroup shall delete unwanted sg if wantLb is false and lbIPs is nil", service: getTestService("test1", v1.ProtocolTCP, nil, false, 80), existingSgs: map[string]network.SecurityGroup{"nsg": { Name: pointer.String("nsg"), @@ -3482,14 +3667,14 @@ func TestReconcileSecurityGroup(t *testing.T) { }, }, }}, - lbIP: pointer.String("1.1.1.1"), + lbIPs: &[]string{"1.1.1.1", "fd00::eef0"}, wantLb: true, expectedSg: &network.SecurityGroup{ Name: pointer.String("nsg"), SecurityGroupPropertiesFormat: &network.SecurityGroupPropertiesFormat{ SecurityRules: &[]network.SecurityRule{ { - Name: pointer.String("atest1-TCP-80-Internet"), + Name: pointer.String("atest1-TCP-80-Internet-IPv4"), SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{ Protocol: network.SecurityRuleProtocol("Tcp"), SourcePortRange: pointer.String("*"), @@ -3501,9 +3686,23 @@ func TestReconcileSecurityGroup(t *testing.T) { Direction: network.SecurityRuleDirection("Inbound"), }, }, + { + Name: pointer.String("atest1-TCP-80-Internet-IPv6"), + SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{ + Protocol: network.SecurityRuleProtocol("Tcp"), + SourcePortRange: pointer.String("*"), + DestinationPortRange: pointer.String("80"), + SourceAddressPrefix: pointer.String("Internet"), + DestinationAddressPrefix: pointer.String("fd00::eef0"), + Access: network.SecurityRuleAccess("Allow"), + Priority: pointer.Int32(501), + Direction: network.SecurityRuleDirection("Inbound"), + }, + }, }, }, }, + ipFamily: DualStack, }, { desc: "reconcileSecurityGroup shall create sgs with correct destinationPrefix for IPv6", @@ -3512,14 +3711,14 @@ func TestReconcileSecurityGroup(t *testing.T) { Name: pointer.String("nsg"), SecurityGroupPropertiesFormat: &network.SecurityGroupPropertiesFormat{}, }}, - lbIP: pointer.String("fd00::eef0"), + lbIPs: &[]string{"fd00::eef0"}, wantLb: true, expectedSg: &network.SecurityGroup{ Name: pointer.String("nsg"), SecurityGroupPropertiesFormat: &network.SecurityGroupPropertiesFormat{ SecurityRules: &[]network.SecurityRule{ { - Name: pointer.String("atest1-TCP-80-Internet"), + Name: pointer.String("atest1-TCP-80-Internet-IPv6"), SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{ Protocol: network.SecurityRuleProtocol("Tcp"), SourcePortRange: pointer.String("*"), @@ -3534,22 +3733,67 @@ func TestReconcileSecurityGroup(t *testing.T) { }, }, }, + ipFamily: IPv6, + }, + { + desc: "reconcileSecurityGroup shall create sgs with correct destinationPrefix for Dual-stack", + service: getTestService("test1", v1.ProtocolTCP, nil, true, 80), + existingSgs: map[string]network.SecurityGroup{"nsg": { + Name: pointer.String("nsg"), + SecurityGroupPropertiesFormat: &network.SecurityGroupPropertiesFormat{}, + }}, + lbIPs: &[]string{"1.1.1.1", "fd00::eef0"}, + wantLb: true, + expectedSg: &network.SecurityGroup{ + Name: pointer.String("nsg"), + SecurityGroupPropertiesFormat: &network.SecurityGroupPropertiesFormat{ + SecurityRules: &[]network.SecurityRule{ + { + Name: pointer.String("atest1-TCP-80-Internet-IPv4"), + SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{ + Protocol: network.SecurityRuleProtocol("Tcp"), + SourcePortRange: pointer.String("*"), + DestinationPortRange: pointer.String("80"), + SourceAddressPrefix: pointer.String("Internet"), + DestinationAddressPrefix: pointer.String("1.1.1.1"), + Access: network.SecurityRuleAccess("Allow"), + Priority: pointer.Int32(500), + Direction: network.SecurityRuleDirection("Inbound"), + }, + }, + { + Name: pointer.String("atest1-TCP-80-Internet-IPv6"), + SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{ + Protocol: network.SecurityRuleProtocol("Tcp"), + SourcePortRange: pointer.String("*"), + DestinationPortRange: pointer.String("80"), + SourceAddressPrefix: pointer.String("Internet"), + DestinationAddressPrefix: pointer.String("fd00::eef0"), + Access: network.SecurityRuleAccess("Allow"), + Priority: pointer.Int32(501), + Direction: network.SecurityRuleDirection("Inbound"), + }, + }, + }, + }, + }, + ipFamily: DualStack, }, { desc: "reconcileSecurityGroup shall create sgs with correct destinationPrefix with additional public IPs", - service: getTestService("test1", v1.ProtocolTCP, map[string]string{consts.ServiceAnnotationAdditionalPublicIPs: "2.3.4.5"}, true, 80), + service: getTestService("test1", v1.ProtocolTCP, map[string]string{consts.ServiceAnnotationAdditionalPublicIPs: "2.3.4.5,fd00::eef1"}, true, 80), existingSgs: map[string]network.SecurityGroup{"nsg": { Name: pointer.String("nsg"), SecurityGroupPropertiesFormat: &network.SecurityGroupPropertiesFormat{}, }}, - lbIP: pointer.String("1.2.3.4"), + lbIPs: &[]string{"1.2.3.4", "fd00::eef0"}, wantLb: true, expectedSg: &network.SecurityGroup{ Name: pointer.String("nsg"), SecurityGroupPropertiesFormat: &network.SecurityGroupPropertiesFormat{ SecurityRules: &[]network.SecurityRule{ { - Name: pointer.String("atest1-TCP-80-Internet"), + Name: pointer.String("atest1-TCP-80-Internet-IPv4"), SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{ Protocol: network.SecurityRuleProtocol("Tcp"), SourcePortRange: pointer.String("*"), @@ -3561,15 +3805,29 @@ func TestReconcileSecurityGroup(t *testing.T) { Direction: network.SecurityRuleDirection("Inbound"), }, }, + { + Name: pointer.String("atest1-TCP-80-Internet-IPv6"), + SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{ + Protocol: network.SecurityRuleProtocol("Tcp"), + SourcePortRange: pointer.String("*"), + DestinationPortRange: pointer.String("80"), + SourceAddressPrefix: pointer.String("Internet"), + DestinationAddressPrefixes: &([]string{"fd00::eef0", "fd00::eef1"}), + Access: network.SecurityRuleAccess("Allow"), + Priority: pointer.Int32(501), + Direction: network.SecurityRuleDirection("Inbound"), + }, + }, }, }, }, + ipFamily: DualStack, }, { desc: "reconcileSecurityGroup shall not create unwanted security rules if there is service tags", service: getTestService("test1", v1.ProtocolTCP, map[string]string{consts.ServiceAnnotationAllowedServiceTag: "tag"}, true, 80), wantLb: true, - lbIP: pointer.String("1.1.1.1"), + lbIPs: &[]string{"1.1.1.1"}, existingSgs: map[string]network.SecurityGroup{"nsg": { Name: pointer.String("nsg"), SecurityGroupPropertiesFormat: &network.SecurityGroupPropertiesFormat{ @@ -3591,7 +3849,7 @@ func TestReconcileSecurityGroup(t *testing.T) { SecurityGroupPropertiesFormat: &network.SecurityGroupPropertiesFormat{ SecurityRules: &[]network.SecurityRule{ { - Name: pointer.String("atest1-TCP-80-tag"), + Name: pointer.String("atest1-TCP-80-tag-IPv4"), SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{ Protocol: network.SecurityRuleProtocol("Tcp"), SourcePortRange: pointer.String("*"), @@ -3614,14 +3872,14 @@ func TestReconcileSecurityGroup(t *testing.T) { Name: pointer.String("nsg"), SecurityGroupPropertiesFormat: &network.SecurityGroupPropertiesFormat{}, }}, - lbIP: pointer.String("1.2.3.4"), + lbIPs: &[]string{"1.2.3.4"}, wantLb: true, expectedSg: &network.SecurityGroup{ Name: pointer.String("nsg"), SecurityGroupPropertiesFormat: &network.SecurityGroupPropertiesFormat{ SecurityRules: &[]network.SecurityRule{ { - Name: pointer.String("shared-TCP-80-Internet"), + Name: pointer.String("shared-TCP-80-Internet-IPv4"), SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{ Protocol: network.SecurityRuleProtocol("Tcp"), SourcePortRange: pointer.String("*"), @@ -3644,7 +3902,7 @@ func TestReconcileSecurityGroup(t *testing.T) { Name: pointer.String("nsg"), SecurityGroupPropertiesFormat: &network.SecurityGroupPropertiesFormat{}, }}, - lbIP: pointer.String("1.2.3.4"), + lbIPs: &[]string{"1.2.3.4"}, lbName: pointer.String("lb"), wantLb: true, expectedSg: &network.SecurityGroup{ @@ -3652,7 +3910,7 @@ func TestReconcileSecurityGroup(t *testing.T) { SecurityGroupPropertiesFormat: &network.SecurityGroupPropertiesFormat{ SecurityRules: &[]network.SecurityRule{ { - Name: pointer.String("atest1-TCP-80-Internet"), + Name: pointer.String("atest1-TCP-80-Internet-IPv4"), SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{ Protocol: network.SecurityRuleProtocol("Tcp"), SourcePortRange: pointer.String("*"), @@ -3670,12 +3928,12 @@ func TestReconcileSecurityGroup(t *testing.T) { }, { desc: "reconcileSecurityGroup shall create sgs with only IPv6 destination addresses for IPv6 services with floating IP disabled", - service: getTestService("test1", v1.ProtocolTCP, map[string]string{consts.ServiceAnnotationDisableLoadBalancerFloatingIP: "true"}, false, 80), + service: getTestService("test1", v1.ProtocolTCP, map[string]string{consts.ServiceAnnotationDisableLoadBalancerFloatingIP: "true"}, true, 80), existingSgs: map[string]network.SecurityGroup{"nsg": { Name: pointer.String("nsg"), SecurityGroupPropertiesFormat: &network.SecurityGroupPropertiesFormat{}, }}, - lbIP: pointer.String("1234::5"), + lbIPs: &[]string{"1234::5"}, lbName: pointer.String("lb"), wantLb: true, expectedSg: &network.SecurityGroup{ @@ -3683,7 +3941,7 @@ func TestReconcileSecurityGroup(t *testing.T) { SecurityGroupPropertiesFormat: &network.SecurityGroupPropertiesFormat{ SecurityRules: &[]network.SecurityRule{ { - Name: pointer.String("atest1-TCP-80-Internet"), + Name: pointer.String("atest1-TCP-80-Internet-IPv6"), SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{ Protocol: network.SecurityRuleProtocol("Tcp"), SourcePortRange: pointer.String("*"), @@ -3698,12 +3956,16 @@ func TestReconcileSecurityGroup(t *testing.T) { }, }, }, + ipFamily: IPv6, }, } for _, test := range testCases { t.Run(test.desc, func(t *testing.T) { az := GetTestCloud(ctrl) + if test.ipFamily != "" { + az.IPFamily = test.ipFamily + } mockSGsClient := az.SecurityGroupsClient.(*mocksecuritygroupclient.MockInterface) mockSGsClient.EXPECT().CreateOrUpdate(gomock.Any(), "rg", gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() if len(test.existingSgs) == 0 { @@ -3720,7 +3982,7 @@ func TestReconcileSecurityGroup(t *testing.T) { mockLBBackendPool.EXPECT().GetBackendPrivateIPs(gomock.Any(), gomock.Any(), gomock.Any()).Return([]string{"1.2.3.4", "5.6.7.8"}, []string{"fc00::1", "fc00::2"}).AnyTimes() mockLBClient.EXPECT().Get(gomock.Any(), "rg", *test.lbName, gomock.Any()).Return(network.LoadBalancer{}, nil) } - sg, err := az.reconcileSecurityGroup("testCluster", &test.service, test.lbIP, test.lbName, test.wantLb) + sg, err := az.reconcileSecurityGroup("testCluster", &test.service, test.lbIPs, test.lbName, test.wantLb) assert.Equal(t, test.expectedSg, sg) assert.Equal(t, test.expectedError, err != nil) }) @@ -3740,13 +4002,13 @@ func TestReconcileSecurityGroupLoadBalancerSourceRanges(t *testing.T) { SecurityRules: &[]network.SecurityRule{}, }, } - lbIP := pointer.String("1.1.1.1") + lbIPs := &[]string{"1.1.1.1"} expectedSg := network.SecurityGroup{ Name: pointer.String("nsg"), SecurityGroupPropertiesFormat: &network.SecurityGroupPropertiesFormat{ SecurityRules: &[]network.SecurityRule{ { - Name: pointer.String("atest1-TCP-80-1.2.3.4_32"), + Name: pointer.String("atest1-TCP-80-1.2.3.4_32-IPv4"), SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{ Protocol: network.SecurityRuleProtocol("Tcp"), SourcePortRange: pointer.String("*"), @@ -3759,7 +4021,7 @@ func TestReconcileSecurityGroupLoadBalancerSourceRanges(t *testing.T) { }, }, { - Name: pointer.String("atest1-TCP-80-deny_all"), + Name: pointer.String("atest1-TCP-80-deny_all-IPv4"), SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{ Protocol: network.SecurityRuleProtocol("Tcp"), SourcePortRange: pointer.String("*"), @@ -3777,7 +4039,7 @@ func TestReconcileSecurityGroupLoadBalancerSourceRanges(t *testing.T) { mockSGClient := az.SecurityGroupsClient.(*mocksecuritygroupclient.MockInterface) mockSGClient.EXPECT().Get(gomock.Any(), az.ResourceGroup, gomock.Any(), gomock.Any()).Return(existingSg, nil) mockSGClient.EXPECT().CreateOrUpdate(gomock.Any(), az.ResourceGroup, gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) - sg, err := az.reconcileSecurityGroup("testCluster", &service, lbIP, nil, true) + sg, err := az.reconcileSecurityGroup("testCluster", &service, lbIPs, nil, true) assert.NoError(t, err) assert.Equal(t, expectedSg, *sg) } @@ -3846,34 +4108,37 @@ func TestSafeDeletePublicIP(t *testing.T) { } } -func TestReconcilePublicIP(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - +// TODO: test upgrade +func TestReconcilePublicIPsCommon(t *testing.T) { deleteUnwantedPIPsAndCreateANewOneclientGet := func(client *mockpublicipclient.MockInterface) { - client.EXPECT().Get(gomock.Any(), "rg", "testCluster-atest1", gomock.Any()).Return(network.PublicIPAddress{ID: pointer.String("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/publicIPAddresses/testCluster-atest1")}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), "rg", "testCluster-atest1-IPv4", gomock.Any()).Return(network.PublicIPAddress{}, &retry.Error{HTTPStatusCode: http.StatusNotFound}).Times(1) + client.EXPECT().Get(gomock.Any(), "rg", "testCluster-atest1-IPv4", gomock.Any()).Return(network.PublicIPAddress{ID: pointer.String("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/publicIPAddresses/testCluster-atest1-IPv4")}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), "rg", "testCluster-atest1-IPv6", gomock.Any()).Return(network.PublicIPAddress{}, &retry.Error{HTTPStatusCode: http.StatusNotFound}).Times(1) + client.EXPECT().Get(gomock.Any(), "rg", "testCluster-atest1-IPv6", gomock.Any()).Return(network.PublicIPAddress{ID: pointer.String("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/publicIPAddresses/testCluster-atest1-IPv6")}, nil).Times(1) + } testCases := []struct { desc string - expectedID string annotations map[string]string existingPIPs []network.PublicIPAddress - expectedPIP *network.PublicIPAddress wantLb bool + expectedIDs []string + expectedPIPs []*network.PublicIPAddress // Consider dual-stack, len(expectedPIPs) <= 2 expectedError bool expectedCreateOrUpdateCount int expectedDeleteCount int expectedClientGet *func(client *mockpublicipclient.MockInterface) + ipFamily IPFamily }{ { - desc: "reconcilePublicIP shall return nil if there's no pip in service", + desc: "reconcilePublicIPs shall return nil if there's no pip in service", wantLb: false, expectedCreateOrUpdateCount: 0, expectedDeleteCount: 0, }, { - desc: "reconcilePublicIP shall return nil if no pip is owned by service", + desc: "reconcilePublicIPs shall return nil if no pip is owned by service", wantLb: false, existingPIPs: []network.PublicIPAddress{ { @@ -3884,25 +4149,37 @@ func TestReconcilePublicIP(t *testing.T) { expectedDeleteCount: 0, }, { - desc: "reconcilePublicIP shall delete unwanted pips and create a new one", + desc: "reconcilePublicIPs shall delete unwanted pips and create a new one for dualstack", wantLb: true, existingPIPs: []network.PublicIPAddress{ { - Name: pointer.String("pip1"), + Name: pointer.String("pip1-IPv4"), Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1")}, PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ IPAddress: pointer.String("1.2.3.4"), }, }, + { + Name: pointer.String("pip1-IPv6"), + Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1")}, + PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ + IPAddress: pointer.String("fd00::eef0"), + }, + }, }, - expectedID: "/subscriptions/subscription/resourceGroups/rg/providers/" + - "Microsoft.Network/publicIPAddresses/testCluster-atest1", - expectedCreateOrUpdateCount: 1, - expectedDeleteCount: 1, + expectedIDs: []string{ + "/subscriptions/subscription/resourceGroups/rg/providers/" + + "Microsoft.Network/publicIPAddresses/testCluster-atest1-IPv4", + "/subscriptions/subscription/resourceGroups/rg/providers/" + + "Microsoft.Network/publicIPAddresses/testCluster-atest1-IPv6", + }, + expectedCreateOrUpdateCount: 2, + expectedDeleteCount: 2, expectedClientGet: &deleteUnwantedPIPsAndCreateANewOneclientGet, + ipFamily: DualStack, }, { - desc: "reconcilePublicIP shall report error if the given PIP name doesn't exist in the resource group", + desc: "reconcilePublicIPs shall report error if the given PIP name doesn't exist in the resource group", wantLb: true, annotations: map[string]string{consts.ServiceAnnotationPIPName: "testPIP"}, existingPIPs: []network.PublicIPAddress{ @@ -3920,141 +4197,242 @@ func TestReconcilePublicIP(t *testing.T) { expectedDeleteCount: 0, }, { - desc: "reconcilePublicIP shall delete unwanted PIP when given the name of desired PIP", - wantLb: true, - annotations: map[string]string{consts.ServiceAnnotationPIPName: "testPIP"}, + desc: "reconcilePublicIPs shall delete unwanted PIP when given the name of desired PIP", + wantLb: true, + annotations: map[string]string{ + consts.ServiceAnnotationPIPNameDualStack[false]: "testPIP-IPv4", + consts.ServiceAnnotationPIPNameDualStack[true]: "testPIP-IPv6", + }, existingPIPs: []network.PublicIPAddress{ { Name: pointer.String("pip1"), Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1")}, PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ - IPAddress: pointer.String("1.2.3.4"), + PublicIPAddressVersion: network.IPVersionIPv4, + IPAddress: pointer.String("1.2.3.4"), }, }, { Name: pointer.String("pip2"), Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1")}, PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ - IPAddress: pointer.String("1.2.3.4"), + PublicIPAddressVersion: network.IPVersionIPv4, + IPAddress: pointer.String("1.2.3.4"), }, }, { - Name: pointer.String("testPIP"), + Name: pointer.String("testPIP-IPv4"), Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1")}, PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ - IPAddress: pointer.String("1.2.3.4"), + PublicIPAddressVersion: network.IPVersionIPv4, + IPAddress: pointer.String("1.2.3.4"), + }, + }, + { + Name: pointer.String("testPIP-IPv6"), + Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1")}, + PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ + PublicIPAddressVersion: network.IPVersionIPv6, + IPAddress: pointer.String("fd00::eef0"), }, }, }, - expectedPIP: &network.PublicIPAddress{ - ID: pointer.String("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/publicIPAddresses/testPIP"), - Name: pointer.String("testPIP"), - Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1")}, - PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ - PublicIPAddressVersion: network.IPVersionIPv4, - IPAddress: pointer.String("1.2.3.4"), + expectedPIPs: []*network.PublicIPAddress{ + { + ID: pointer.String("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/publicIPAddresses/testPIP-IPv4"), + Name: pointer.String("testPIP-IPv4"), + Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1")}, + PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ + PublicIPAddressVersion: network.IPVersionIPv4, + IPAddress: pointer.String("1.2.3.4"), + }, + }, + { + ID: pointer.String("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/publicIPAddresses/testPIP-IPv6"), + Name: pointer.String("testPIP-IPv6"), + Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1")}, + PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ + PublicIPAddressVersion: network.IPVersionIPv6, + PublicIPAllocationMethod: network.IPAllocationMethodDynamic, + IPAddress: pointer.String("fd00::eef0"), + }, }, }, expectedCreateOrUpdateCount: 1, expectedDeleteCount: 2, + ipFamily: DualStack, }, { - desc: "reconcilePublicIP shall not delete unwanted PIP when there are other service references", - wantLb: true, - annotations: map[string]string{consts.ServiceAnnotationPIPName: "testPIP"}, + desc: "reconcilePublicIPs shall not delete unwanted PIP when there are other service references", + wantLb: true, + annotations: map[string]string{ + consts.ServiceAnnotationPIPNameDualStack[false]: "testPIP-IPv4", + consts.ServiceAnnotationPIPNameDualStack[true]: "testPIP-IPv6", + }, existingPIPs: []network.PublicIPAddress{ { - Name: pointer.String("pip1"), + Name: pointer.String("pip1-IPv4"), Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1")}, PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ - IPAddress: pointer.String("1.2.3.4"), + PublicIPAddressVersion: network.IPVersionIPv4, + IPAddress: pointer.String("1.2.3.4"), }, }, { - Name: pointer.String("pip2"), + Name: pointer.String("pip2-IPv4"), Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1,default/test2")}, PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ - IPAddress: pointer.String("1.2.3.4"), + PublicIPAddressVersion: network.IPVersionIPv4, + IPAddress: pointer.String("1.2.3.4"), }, }, { - Name: pointer.String("testPIP"), + Name: pointer.String("pip1-IPv6"), Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1")}, PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ - IPAddress: pointer.String("1.2.3.4"), + PublicIPAddressVersion: network.IPVersionIPv6, + IPAddress: pointer.String("fd00::eef0"), + }, + }, + { + Name: pointer.String("pip2-IPv6"), + Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1,default/test2")}, + PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ + PublicIPAddressVersion: network.IPVersionIPv6, + IPAddress: pointer.String("fd00::eef0"), + }, + }, + { + Name: pointer.String("testPIP-IPv4"), + Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1")}, + PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ + PublicIPAddressVersion: network.IPVersionIPv4, + IPAddress: pointer.String("1.2.3.4"), + }, + }, + { + Name: pointer.String("testPIP-IPv6"), + Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1")}, + PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ + PublicIPAddressVersion: network.IPVersionIPv6, + IPAddress: pointer.String("fd00::eef0"), }, }, }, - expectedPIP: &network.PublicIPAddress{ - ID: pointer.String("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/publicIPAddresses/testPIP"), - Name: pointer.String("testPIP"), - Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1")}, - PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ - PublicIPAddressVersion: network.IPVersionIPv4, - IPAddress: pointer.String("1.2.3.4"), + expectedPIPs: []*network.PublicIPAddress{ + { + ID: pointer.String("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/publicIPAddresses/testPIP-IPv4"), + Name: pointer.String("testPIP-IPv4"), + Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1")}, + PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ + PublicIPAddressVersion: network.IPVersionIPv4, + IPAddress: pointer.String("1.2.3.4"), + }, + }, + { + ID: pointer.String("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/publicIPAddresses/testPIP-IPv6"), + Name: pointer.String("testPIP-IPv6"), + Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1")}, + PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ + PublicIPAddressVersion: network.IPVersionIPv6, + PublicIPAllocationMethod: network.IPAllocationMethodDynamic, + IPAddress: pointer.String("fd00::eef0"), + }, }, }, - expectedCreateOrUpdateCount: 1, - expectedDeleteCount: 1, + expectedCreateOrUpdateCount: 1, // Update testPIP-IPv6 + expectedDeleteCount: 2, + ipFamily: DualStack, }, { - desc: "reconcilePublicIP shall delete unwanted pips and existing pips, when the existing pips IP tags do not match", + desc: "reconcilePublicIPs shall delete unwanted pips and existing pips, when the existing pips IP do not match", wantLb: true, annotations: map[string]string{ - consts.ServiceAnnotationPIPName: "testPIP", - consts.ServiceAnnotationIPTagsForPublicIP: "tag1=tag1value", + consts.ServiceAnnotationPIPNameDualStack[false]: "testPIP-IPv4", + consts.ServiceAnnotationPIPNameDualStack[true]: "testPIP-IPv6", + consts.ServiceAnnotationIPTagsForPublicIP: "tag1=tag1value", }, existingPIPs: []network.PublicIPAddress{ { - Name: pointer.String("pip1"), + Name: pointer.String("pip1-IPv4"), Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1")}, PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ - IPAddress: pointer.String("1.2.3.4"), + PublicIPAddressVersion: network.IPVersionIPv4, + IPAddress: pointer.String("1.2.3.4"), }, }, { - Name: pointer.String("pip2"), + Name: pointer.String("pip2-IPv4"), Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1")}, PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ - IPAddress: pointer.String("1.2.3.4"), + PublicIPAddressVersion: network.IPVersionIPv4, + IPAddress: pointer.String("1.2.3.4"), }, }, { - Name: pointer.String("testPIP"), + Name: pointer.String("testPIP-IPv4"), Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1")}, PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ - IPAddress: pointer.String("1.2.3.4"), + PublicIPAddressVersion: network.IPVersionIPv4, + IPAddress: pointer.String("1.2.3.4"), + }, + }, + { + Name: pointer.String("testPIP-IPv6"), + Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1")}, + PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ + PublicIPAddressVersion: network.IPVersionIPv6, + IPAddress: pointer.String("fd00::eef0"), }, }, }, - expectedPIP: &network.PublicIPAddress{ - ID: pointer.String("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/publicIPAddresses/testPIP"), - Name: pointer.String("testPIP"), - Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1")}, - PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ - PublicIPAddressVersion: network.IPVersionIPv4, - PublicIPAllocationMethod: network.IPAllocationMethodStatic, - IPTags: &[]network.IPTag{ - { - IPTagType: pointer.String("tag1"), - Tag: pointer.String("tag1value"), + expectedPIPs: []*network.PublicIPAddress{ + { + ID: pointer.String("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/publicIPAddresses/testPIP-IPv4"), + Name: pointer.String("testPIP-IPv4"), + Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1")}, + PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ + PublicIPAddressVersion: network.IPVersionIPv4, + PublicIPAllocationMethod: network.IPAllocationMethodStatic, + IPTags: &[]network.IPTag{ + { + IPTagType: pointer.String("tag1"), + Tag: pointer.String("tag1value"), + }, + }, + }, + }, + { + ID: pointer.String("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/publicIPAddresses/testPIP-IPv6"), + Name: pointer.String("testPIP-IPv6"), + Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1")}, + PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ + PublicIPAddressVersion: network.IPVersionIPv6, + PublicIPAllocationMethod: network.IPAllocationMethodDynamic, + IPTags: &[]network.IPTag{ + { + IPTagType: pointer.String("tag1"), + Tag: pointer.String("tag1value"), + }, }, }, }, }, - expectedCreateOrUpdateCount: 1, + expectedCreateOrUpdateCount: 2, expectedDeleteCount: 2, + ipFamily: DualStack, }, { - desc: "reconcilePublicIP shall preserve existing pips, when the existing pips IP tags do match", + desc: "reconcilePublicIPs shall preserve existing pips, when the existing pips IP tags do match", wantLb: true, annotations: map[string]string{ - consts.ServiceAnnotationPIPName: "testPIP", - consts.ServiceAnnotationIPTagsForPublicIP: "tag1=tag1value", + consts.ServiceAnnotationPIPNameDualStack[false]: "testPIP-IPv4", + consts.ServiceAnnotationPIPNameDualStack[true]: "testPIP-IPv6", + consts.ServiceAnnotationIPTagsForPublicIP: "tag1=tag1value", }, existingPIPs: []network.PublicIPAddress{ { - Name: pointer.String("testPIP"), + Name: pointer.String("testPIP-IPv4"), Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1")}, PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ PublicIPAddressVersion: network.IPVersionIPv4, @@ -4068,85 +4446,166 @@ func TestReconcilePublicIP(t *testing.T) { IPAddress: pointer.String("1.2.3.4"), }, }, + { + Name: pointer.String("testPIP-IPv6"), + Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1")}, + PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ + PublicIPAddressVersion: network.IPVersionIPv6, + PublicIPAllocationMethod: network.IPAllocationMethodStatic, + IPTags: &[]network.IPTag{ + { + IPTagType: pointer.String("tag1"), + Tag: pointer.String("tag1value"), + }, + }, + IPAddress: pointer.String("fd00::eef0"), + }, + }, }, - expectedPIP: &network.PublicIPAddress{ - ID: pointer.String("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/publicIPAddresses/testPIP"), - Name: pointer.String("testPIP"), - Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1")}, - PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ - PublicIPAddressVersion: network.IPVersionIPv4, - PublicIPAllocationMethod: network.IPAllocationMethodStatic, - IPTags: &[]network.IPTag{ - { - IPTagType: pointer.String("tag1"), - Tag: pointer.String("tag1value"), + expectedPIPs: []*network.PublicIPAddress{ + { + ID: pointer.String("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/publicIPAddresses/testPIP-IPv4"), + Name: pointer.String("testPIP-IPv4"), + Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1")}, + PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ + PublicIPAddressVersion: network.IPVersionIPv4, + PublicIPAllocationMethod: network.IPAllocationMethodStatic, + IPTags: &[]network.IPTag{ + { + IPTagType: pointer.String("tag1"), + Tag: pointer.String("tag1value"), + }, }, + IPAddress: pointer.String("1.2.3.4"), + }, + }, + { + ID: pointer.String("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/publicIPAddresses/testPIP-IPv6"), + Name: pointer.String("testPIP-IPv6"), + Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1")}, + PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ + PublicIPAddressVersion: network.IPVersionIPv6, + PublicIPAllocationMethod: network.IPAllocationMethodDynamic, + IPTags: &[]network.IPTag{ + { + IPTagType: pointer.String("tag1"), + Tag: pointer.String("tag1value"), + }, + }, + IPAddress: pointer.String("fd00::eef0"), }, - IPAddress: pointer.String("1.2.3.4"), }, }, - expectedCreateOrUpdateCount: 0, + expectedCreateOrUpdateCount: 1, // Update testPIP-IPv6 expectedDeleteCount: 0, + ipFamily: DualStack, }, { - desc: "reconcilePublicIP shall find the PIP by given name and shall not delete the PIP which is not owned by service", - wantLb: true, - annotations: map[string]string{consts.ServiceAnnotationPIPName: "testPIP"}, + desc: "reconcilePublicIPs shall find the PIP by given name and shall not delete the PIP which is not owned by service", + wantLb: true, + annotations: map[string]string{ + consts.ServiceAnnotationPIPNameDualStack[false]: "testPIP-IPv4", + consts.ServiceAnnotationPIPNameDualStack[true]: "testPIP-IPv6", + }, existingPIPs: []network.PublicIPAddress{ { - Name: pointer.String("pip1"), + Name: pointer.String("pip1-IPv4"), PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ - IPAddress: pointer.String("1.2.3.4"), + PublicIPAddressVersion: network.IPVersionIPv4, + IPAddress: pointer.String("1.2.3.4"), }, }, { - Name: pointer.String("pip2"), + Name: pointer.String("pip2-IPv4"), Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1")}, PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ - IPAddress: pointer.String("1.2.3.4"), + PublicIPAddressVersion: network.IPVersionIPv4, + IPAddress: pointer.String("1.2.3.4"), }, }, { - Name: pointer.String("testPIP"), + Name: pointer.String("testPIP-IPv4"), PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ - IPAddress: pointer.String("1.2.3.4"), + PublicIPAddressVersion: network.IPVersionIPv4, + IPAddress: pointer.String("1.2.3.4"), + }, + }, + { + Name: pointer.String("pip2-IPv6"), + Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1")}, + PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ + PublicIPAddressVersion: network.IPVersionIPv6, + IPAddress: pointer.String("fd00::eef0"), + }, + }, + { + Name: pointer.String("testPIP-IPv6"), + PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ + PublicIPAddressVersion: network.IPVersionIPv6, + IPAddress: pointer.String("fd00::eef0"), }, }, }, - expectedPIP: &network.PublicIPAddress{ - ID: pointer.String("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/publicIPAddresses/testPIP"), - Name: pointer.String("testPIP"), - PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ - PublicIPAddressVersion: network.IPVersionIPv4, - IPAddress: pointer.String("1.2.3.4"), + expectedPIPs: []*network.PublicIPAddress{ + { + ID: pointer.String("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/publicIPAddresses/testPIP-IPv4"), + Name: pointer.String("testPIP-IPv4"), + PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ + PublicIPAddressVersion: network.IPVersionIPv4, + IPAddress: pointer.String("1.2.3.4"), + }, + }, + { + ID: pointer.String("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/publicIPAddresses/testPIP-IPv6"), + Name: pointer.String("testPIP-IPv6"), + PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ + PublicIPAddressVersion: network.IPVersionIPv6, + PublicIPAllocationMethod: network.IPAllocationMethodDynamic, + IPAddress: pointer.String("fd00::eef0"), + }, }, }, - expectedCreateOrUpdateCount: 1, - expectedDeleteCount: 1, + expectedCreateOrUpdateCount: 1, // Update testPIP-IPv6 + expectedDeleteCount: 2, + ipFamily: DualStack, }, { - desc: "reconcilePublicIP shall delete the unwanted PIP name from service tag and shall not delete it if there is other reference", + desc: "reconcilePublicIPs shall delete the unwanted PIP name from service tag and shall not delete it if there is other reference", wantLb: false, existingPIPs: []network.PublicIPAddress{ { - Name: pointer.String("pip1"), + Name: pointer.String("pip1-IPv4"), Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1,default/test2")}, PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ IPAddress: pointer.String("1.2.3.4"), }, }, + { + Name: pointer.String("pip1-IPv6"), + Tags: map[string]*string{consts.ServiceTagKey: pointer.String("default/test1,default/test2")}, + PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ + IPAddress: pointer.String("fd00::eef0"), + }, + }, }, - expectedCreateOrUpdateCount: 1, + expectedCreateOrUpdateCount: 2, + ipFamily: DualStack, }, } for _, test := range testCases { t.Run(test.desc, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + deletedPips := make(map[string]bool) savedPips := make(map[string]network.PublicIPAddress) createOrUpdateCount := 0 var m sync.Mutex az := GetTestCloud(ctrl) + if test.ipFamily != "" { + az.IPFamily = test.ipFamily + } mockPIPsClient := az.PublicIPAddressesClient.(*mockpublicipclient.MockInterface) creator := mockPIPsClient.EXPECT().CreateOrUpdate(gomock.Any(), "rg", gomock.Any(), gomock.Any()).AnyTimes() creator.DoAndReturn(func(ctx context.Context, resourceGroupName string, publicIPAddressName string, parameters network.PublicIPAddress) *retry.Error { @@ -4162,6 +4621,7 @@ func TestReconcilePublicIP(t *testing.T) { (*test.expectedClientGet)(mockPIPsClient) } service := getTestService("test1", v1.ProtocolTCP, nil, false, 80) + makeTestServiceDualStack(&service) service.Annotations = test.annotations for _, pip := range test.existingPIPs { savedPips[*pip.Name] = pip @@ -4205,25 +4665,53 @@ func TestReconcilePublicIP(t *testing.T) { m.Unlock() return }) - pip, err := az.reconcilePublicIP("testCluster", &service, "", test.wantLb) + + pips, err := az.reconcilePublicIPs("testCluster", &service, "", test.wantLb) if !test.expectedError { assert.NoError(t, err) } - if test.expectedID != "" { - assert.Equal(t, test.expectedID, pointer.StringDeref(pip.ID, "")) - } else if test.expectedPIP != nil && test.expectedPIP.Name != nil { - assert.Equal(t, *test.expectedPIP.Name, *pip.Name) - - if test.expectedPIP.PublicIPAddressPropertiesFormat != nil { - sortIPTags(test.expectedPIP.PublicIPAddressPropertiesFormat.IPTags) + // Check IDs + if len(test.expectedIDs) != 0 { + ids := []string{} + for _, pip := range pips { + ids = append(ids, pointer.StringDeref(pip.ID, "")) } - - if pip.PublicIPAddressPropertiesFormat != nil { - sortIPTags(pip.PublicIPAddressPropertiesFormat.IPTags) + assert.True(t, compareStrings(test.expectedIDs, ids)) + } + // Check PIPs + if len(test.expectedPIPs) != 0 { + pipsNames := []string{} + for _, pip := range pips { + pipsNames = append(pipsNames, pointer.StringDeref(pip.Name, "")) } + assert.Equal(t, len(test.expectedPIPs), len(pips), pipsNames) + pipsOrdered := []*network.PublicIPAddress{} + if len(test.expectedPIPs) == 1 { + pipsOrdered = append(pipsOrdered, pips[0]) + } else { + // len(test.expectedPIPs) == 2 + if pointer.StringDeref(test.expectedPIPs[0].Name, "") == pointer.StringDeref(pips[0].Name, "") { + pipsOrdered = append(pipsOrdered, pips...) + } else { + pipsOrdered = append(pipsOrdered, pips[1], pips[0]) + } + } + for i := range pipsOrdered { + pip := pipsOrdered[i] + assert.NotNil(t, test.expectedPIPs[i].Name) + assert.NotNil(t, pip.Name) + assert.Equal(t, *test.expectedPIPs[i].Name, *pip.Name) + + if test.expectedPIPs[i].PublicIPAddressPropertiesFormat != nil { + sortIPTags(test.expectedPIPs[i].PublicIPAddressPropertiesFormat.IPTags) + } - assert.Equal(t, test.expectedPIP.PublicIPAddressPropertiesFormat, pip.PublicIPAddressPropertiesFormat) + if pip.PublicIPAddressPropertiesFormat != nil { + sortIPTags(pip.PublicIPAddressPropertiesFormat.IPTags) + } + assert.Equal(t, test.expectedPIPs[i].PublicIPAddressPropertiesFormat, pip.PublicIPAddressPropertiesFormat) + } } assert.Equal(t, test.expectedCreateOrUpdateCount, createOrUpdateCount) assert.Equal(t, test.expectedError, err != nil) @@ -4239,6 +4727,12 @@ func TestReconcilePublicIP(t *testing.T) { } } +func compareStrings(s0, s1 []string) bool { + ss0 := sets.NewString(s0...) + ss1 := sets.NewString(s1...) + return ss0.Equal(ss1) +} + func TestEnsurePublicIPExists(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -4597,7 +5091,7 @@ func TestEnsurePublicIPExists(t *testing.T) { return []network.PublicIPAddress{*basicPIP}, nil }).AnyTimes() - pip, err := az.ensurePublicIPExists(&service, "pip1", test.inputDNSLabel, "", false, test.foundDNSLabelAnnotation) + pip, err := az.ensurePublicIPExists(&service, "pip1", test.inputDNSLabel, "", false, test.foundDNSLabelAnnotation, test.isIPv6) assert.Equal(t, test.expectedError, err != nil, "unexpectedly encountered (or not) error: %v", err) if test.expectedID != "" { assert.Equal(t, test.expectedID, pointer.StringDeref(pip.ID, "")) @@ -4647,7 +5141,7 @@ func TestEnsurePublicIPExistsWithExtendedLocation(t *testing.T) { assert.Nil(t, publicIPAddressParameters.Zones) return nil }).Times(1) - pip, err := az.ensurePublicIPExists(&service, "pip1", "", "", false, false) + pip, err := az.ensurePublicIPExists(&service, "pip1", "", "", false, false, false) assert.NotNil(t, pip, "ensurePublicIPExists shall create a new pip"+ "with extendedLocation if there is no existed pip") assert.Nil(t, err, "ensurePublicIPExists should create a new pip without errors.") @@ -5582,8 +6076,9 @@ func TestReconcileZonesForFrontendIPConfigs(t *testing.T) { zoneClient.EXPECT().GetZones(gomock.Any(), gomock.Any()).Return(map[string][]string{}, tc.getZoneError).MaxTimes(1) cloud.ZoneClient = zoneClient + pips := []network.PublicIPAddress{tc.existingPIP} // TODO: if correct? defaultLBFrontendIPConfigName := cloud.getDefaultFrontendIPConfigName(&tc.service) - _, _, dirty, err := cloud.reconcileFrontendIPConfigs("testCluster", &tc.service, &lb, tc.status, true, defaultLBFrontendIPConfigName) + _, _, dirty, err := cloud.reconcileFrontendIPConfigs("testCluster", &tc.service, &lb, tc.status, true, defaultLBFrontendIPConfigName, &pips) if tc.expectedErr == nil { assert.NoError(t, err) } else { diff --git a/pkg/provider/azure_mock_loadbalancer_backendpool.go b/pkg/provider/azure_mock_loadbalancer_backendpool.go index 9b446ae0b0..958413ee28 100644 --- a/pkg/provider/azure_mock_loadbalancer_backendpool.go +++ b/pkg/provider/azure_mock_loadbalancer_backendpool.go @@ -49,7 +49,7 @@ func (m *MockBackendPool) EXPECT() *MockBackendPoolMockRecorder { } // EnsureHostsInPool mocks base method -func (m *MockBackendPool) EnsureHostsInPool(service *v1.Service, nodes []*v1.Node, backendPoolID, vmSetName, clusterName, lbName string, backendPool network.BackendAddressPool) error { +func (m *MockBackendPool) EnsureHostsInPool(service *v1.Service, nodes []*v1.Node, backendPoolID, vmSetName, clusterName, lbName string, backendPool network.BackendAddressPool, isIPv6 bool) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "EnsureHostsInPool", service, nodes, backendPoolID, vmSetName, clusterName, lbName, backendPool) ret0, _ := ret[0].(error) diff --git a/pkg/provider/azure_standard.go b/pkg/provider/azure_standard.go index 08f5ab5ddf..198d9ad470 100644 --- a/pkg/provider/azure_standard.go +++ b/pkg/provider/azure_standard.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "hash/crc32" + "net" "regexp" "strconv" "strings" @@ -39,7 +40,6 @@ import ( "k8s.io/apimachinery/pkg/util/uuid" cloudprovider "k8s.io/cloud-provider" "k8s.io/klog/v2" - utilnet "k8s.io/utils/net" "k8s.io/utils/pointer" azcache "sigs.k8s.io/cloud-provider-azure/pkg/cache" @@ -54,8 +54,28 @@ var ( nicIDRE = regexp.MustCompile(`(?i)/subscriptions/(?:.*)/resourceGroups/(.+)/providers/Microsoft.Network/networkInterfaces/(.+)/ipConfigurations/(?:.*)`) vmIDRE = regexp.MustCompile(`(?i)/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Compute/virtualMachines/(.+)`) vmasIDRE = regexp.MustCompile(`/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Compute/availabilitySets/(.+)`) + + v4Suffix = "IPv4" + v6Suffix = "IPv6" ) +func getResourceByIPFamily(resource string, isIPv6 bool) string { + if isIPv6 { + return fmt.Sprintf("%s-%s", resource, v6Suffix) + } + return fmt.Sprintf("%s-%s", resource, v4Suffix) +} + +func setResourceByIPFamily(resource string, isIPv6 bool) string { + if !isIPv6 && !strings.HasSuffix(resource, v4Suffix) { + return fmt.Sprintf("%s-%s", resource, v4Suffix) + } + if isIPv6 && !strings.HasSuffix(resource, v6Suffix) { + return fmt.Sprintf("%s-%s", resource, v6Suffix) + } + return resource +} + // returns the full identifier of an availabilitySet func (az *Cloud) getAvailabilitySetID(resourceGroup, availabilitySetName string) string { return fmt.Sprintf( @@ -261,15 +281,20 @@ func isInternalLoadBalancer(lb *network.LoadBalancer) bool { // This means: // clusters moving from IPv4 to dualstack will require no changes // clusters moving from IPv6 to dualstack will require no changes as the IPv4 backend pool will created with -func getBackendPoolName(clusterName string, service *v1.Service) string { - IPv6 := utilnet.IsIPv6String(service.Spec.ClusterIP) - if IPv6 { - return fmt.Sprintf("%v-IPv6", clusterName) +func getBackendPoolName(clusterName string, isIPv6 bool) string { + if isIPv6 { + return fmt.Sprintf("%s-IPv6", clusterName) } return clusterName } +// ifBackendPoolIPv6 checks if a backend pool is of IPv6 according to name/ID. +func ifBackendPoolIPv6(name string) bool { + return strings.HasSuffix(name, "-IPv6") +} + +// TODO: use getResourceByIPFamily in this method. func (az *Cloud) getLoadBalancerRuleName(service *v1.Service, protocol v1.Protocol, port int32) string { prefix := az.getRulePrefix(service) ruleName := fmt.Sprintf("%s-%s-%d", prefix, protocol, port) @@ -280,8 +305,9 @@ func (az *Cloud) getLoadBalancerRuleName(service *v1.Service, protocol v1.Protoc // Load balancer rule name must be less or equal to 80 characters, so excluding the hyphen two segments cannot exceed 79 subnetSegment := *subnet - if len(ruleName)+len(subnetSegment)+1 > consts.LoadBalancerRuleNameMaxLength { - subnetSegment = subnetSegment[:consts.LoadBalancerRuleNameMaxLength-len(ruleName)-1] + maxLength := consts.LoadBalancerRuleNameMaxLength - consts.IPFamilySuffixLength + if len(ruleName)+len(subnetSegment)+1 > maxLength { + subnetSegment = subnetSegment[:maxLength-len(ruleName)-1] } return fmt.Sprintf("%s-%s-%s-%d", prefix, subnetSegment, protocol, port) @@ -291,14 +317,16 @@ func (az *Cloud) getloadbalancerHAmodeRuleName(service *v1.Service) string { return az.getLoadBalancerRuleName(service, service.Spec.Ports[0].Protocol, service.Spec.Ports[0].Port) } -func (az *Cloud) getSecurityRuleName(service *v1.Service, port v1.ServicePort, sourceAddrPrefix string) string { +func (az *Cloud) getSecurityRuleName(service *v1.Service, port v1.ServicePort, sourceAddrPrefix string, isIPv6 bool) string { if useSharedSecurityRule(service) { safePrefix := strings.Replace(sourceAddrPrefix, "/", "_", -1) - return fmt.Sprintf("shared-%s-%d-%s", port.Protocol, port.Port, safePrefix) + name := fmt.Sprintf("shared-%s-%d-%s", port.Protocol, port.Port, safePrefix) + return getResourceByIPFamily(name, isIPv6) } safePrefix := strings.Replace(sourceAddrPrefix, "/", "_", -1) rulePrefix := az.getRulePrefix(service) - return fmt.Sprintf("%s-%s-%d-%s", rulePrefix, port.Protocol, port.Port, safePrefix) + name := fmt.Sprintf("%s-%s-%d-%s", rulePrefix, port.Protocol, port.Port, safePrefix) + return getResourceByIPFamily(name, isIPv6) } // This returns a human-readable version of the Service used to tag some resources. @@ -312,14 +340,16 @@ func (az *Cloud) getRulePrefix(service *v1.Service) string { return az.GetLoadBalancerName(context.TODO(), "", service) } -func (az *Cloud) getPublicIPName(clusterName string, service *v1.Service) string { +func (az *Cloud) getPublicIPName(clusterName string, service *v1.Service, isIPv6 bool) string { pipName := fmt.Sprintf("%s-%s", clusterName, az.GetLoadBalancerName(context.TODO(), clusterName, service)) - if prefixID, ok := service.Annotations[consts.ServiceAnnotationPIPPrefixID]; ok && prefixID != "" { - prefixName, err := getLastSegment(prefixID, "/") + pipName = getResourceByIPFamily(pipName, isIPv6) + klog.Infof("DEBUG getPublicIPName %s %s", pipName, pipName) + if id := getServicePIPPrefixID(service, isIPv6); id != "" { + id, err := getLastSegment(id, "/") if err != nil { return pipName } - pipName = fmt.Sprintf("%s-%s", pipName, prefixName) + pipName = fmt.Sprintf("%s-%s", pipName, id) } return pipName } @@ -329,6 +359,69 @@ func (az *Cloud) serviceOwnsRule(service *v1.Service, rule string) bool { return strings.HasPrefix(strings.ToUpper(rule), strings.ToUpper(prefix)) } +func ifFIPIPVerionSet(fip *network.FrontendIPConfiguration) bool { + set := false + if fip.FrontendIPConfigurationPropertiesFormat != nil { + if fip.FrontendIPConfigurationPropertiesFormat.PublicIPAddress != nil && + fip.FrontendIPConfigurationPropertiesFormat.PublicIPAddress.PublicIPAddressPropertiesFormat != nil { + if fip.FrontendIPConfigurationPropertiesFormat.PublicIPAddress.PublicIPAddressPropertiesFormat.PublicIPAddressVersion == network.IPVersionIPv6 { + set = true + } + if fip.FrontendIPConfigurationPropertiesFormat.PublicIPAddress.PublicIPAddressPropertiesFormat.PublicIPAddressVersion == network.IPVersionIPv4 { + set = true + } + } + if fip.FrontendIPConfigurationPropertiesFormat.PrivateIPAddressVersion == network.IPVersionIPv6 { + set = true + } + if fip.FrontendIPConfigurationPropertiesFormat.PrivateIPAddressVersion == network.IPVersionIPv4 { + set = true + } + } + return set +} + +// ifFIPIPv6 checks if the frontend IP configuration is of IPv6. +func (az *Cloud) ifFIPIPv6(service *v1.Service, fip *network.FrontendIPConfiguration, pips *[]network.PublicIPAddress, isInternal bool) (isIPv6 bool, err error) { + if err := az.ensurePIP(service, az.ResourceGroup, pips); err != nil { + return false, fmt.Errorf("failed to ensure PIP is refreshed: %v", err) + } + if isInternal { + if fip.FrontendIPConfigurationPropertiesFormat != nil { + if fip.FrontendIPConfigurationPropertiesFormat.PrivateIPAddressVersion != "" { + return fip.FrontendIPConfigurationPropertiesFormat.PrivateIPAddressVersion == network.IPVersionIPv6, nil + } + return net.ParseIP(pointer.StringDeref(fip.FrontendIPConfigurationPropertiesFormat.PrivateIPAddress, "")).To4() == nil, nil + } + klog.Errorf("Checking IP Family of frontend IP configuration %q of internal Service but its"+ + " FrontendIPConfigurationPropertiesFormat is nil. It's considered to be IPv4", + pointer.StringDeref(fip.Name, "")) + return + } + var fipPIPID string + if fip.FrontendIPConfigurationPropertiesFormat != nil && fip.FrontendIPConfigurationPropertiesFormat.PublicIPAddress != nil { + fipPIPID = pointer.StringDeref(fip.FrontendIPConfigurationPropertiesFormat.PublicIPAddress.ID, "") + } + // klog.Infof("DEBUG ifFIPIPv6 fipPIPID %v", fipPIPID) + for _, pip := range *pips { + id := pointer.StringDeref(pip.ID, "") + // klog.Infof("DEBUG ifFIPIPv6 pip.ID %v", id) + if fipPIPID != id { + continue + } + if pip.PublicIPAddressPropertiesFormat != nil { + // First check PublicIPAddressVersion, then IPAddress + if pip.PublicIPAddressPropertiesFormat.PublicIPAddressVersion == network.IPVersionIPv6 || + net.ParseIP(pointer.StringDeref(pip.PublicIPAddressPropertiesFormat.IPAddress, "")).To4() == nil { + isIPv6 = true + break + } + } + break + } + return +} + // There are two cases when a service owns the frontend IP config: // 1. The primary service, which means the frontend IP config is created after the creation of the service. // This means the name of the config can be tracked by the service UID. @@ -343,33 +436,36 @@ func (az *Cloud) serviceOwnsFrontendIP(fip network.FrontendIPConfiguration, serv return true, isPrimaryService, nil } - loadBalancerIP := getServiceLoadBalancerIP(service) - if loadBalancerIP == "" { + loadBalancerIPs := getServiceLoadBalancerIPs(service) + if len(loadBalancerIPs) == 0 { // it is a must that the secondary services set the loadBalancer IP return false, isPrimaryService, nil } + // klog.Infof("DEBUG fip lb ips: %s, %s, %v", *fip.Name, loadBalancerIPs, service.Annotations) // for external secondary service the public IP address should be checked if !requiresInternalLoadBalancer(service) { pipResourceGroup := az.getPublicIPAddressResourceGroup(service) - pip, err := az.findMatchedPIPByLoadBalancerIP(service, loadBalancerIP, pipResourceGroup, pips) - if err != nil { - klog.Warningf("serviceOwnsFrontendIP: unexpected error when finding match public IP of the service %s with loadBalancerLP %s: %v", service.Name, loadBalancerIP, err) - return false, isPrimaryService, nil - } - - if pip != nil && - pip.ID != nil && - pip.PublicIPAddressPropertiesFormat != nil && - pip.IPAddress != nil && - fip.FrontendIPConfigurationPropertiesFormat != nil && - fip.FrontendIPConfigurationPropertiesFormat.PublicIPAddress != nil { - if strings.EqualFold(pointer.StringDeref(pip.ID, ""), pointer.StringDeref(fip.PublicIPAddress.ID, "")) { - klog.V(4).Infof("serviceOwnsFrontendIP: found secondary service %s of the frontend IP config %s", service.Name, *fip.Name) + for _, loadBalancerIP := range loadBalancerIPs { + pip, err := az.findMatchedPIPByLoadBalancerIP(service, loadBalancerIP, pipResourceGroup, pips) + if err != nil { + klog.Warningf("serviceOwnsFrontendIP: unexpected error when finding match public IP of the service %s with loadBalancerIP %s: %v", service.Name, loadBalancerIP, err) + return false, isPrimaryService, nil + } - return true, isPrimaryService, nil + if pip != nil && + pip.ID != nil && + pip.PublicIPAddressPropertiesFormat != nil && + pip.PublicIPAddressPropertiesFormat.IPAddress != nil && + fip.FrontendIPConfigurationPropertiesFormat != nil && + fip.FrontendIPConfigurationPropertiesFormat.PublicIPAddress != nil { + if strings.EqualFold(pointer.StringDeref(pip.ID, ""), pointer.StringDeref(fip.PublicIPAddress.ID, "")) { + klog.Infof("serviceOwnsFrontendIP: found secondary service %s of the frontend IP config %s", service.Name, *fip.Name) + return true, isPrimaryService, nil + } } - klog.V(4).Infof("serviceOwnsFrontendIP: the public IP with ID %s is being referenced by other service with public IP address %s", *pip.ID, *pip.IPAddress) + klog.Infof("serviceOwnsFrontendIP: the public IP with ID %s is being referenced by other service with public IP address %s "+ + "OR it is of incorrect IP version", *pip.ID, *pip.IPAddress) } return false, isPrimaryService, nil @@ -380,7 +476,14 @@ func (az *Cloud) serviceOwnsFrontendIP(fip network.FrontendIPConfiguration, serv return false, isPrimaryService, nil } - return strings.EqualFold(*fip.PrivateIPAddress, loadBalancerIP), isPrimaryService, nil + privateIPEquals := false + for _, loadBalancerIP := range loadBalancerIPs { + if strings.EqualFold(*fip.PrivateIPAddress, loadBalancerIP) { + privateIPEquals = true + break + } + } + return privateIPEquals, isPrimaryService, nil } func (az *Cloud) getDefaultFrontendIPConfigName(service *v1.Service) string { @@ -390,8 +493,9 @@ func (az *Cloud) getDefaultFrontendIPConfigName(service *v1.Service) string { ipcName := fmt.Sprintf("%s-%s", baseName, *subnetName) // Azure lb front end configuration name must not exceed 80 characters - if len(ipcName) > consts.FrontendIPConfigNameMaxLength { - ipcName = ipcName[:consts.FrontendIPConfigNameMaxLength] + maxLength := consts.FrontendIPConfigNameMaxLength - consts.IPFamilySuffixLength + if len(ipcName) > maxLength { + ipcName = ipcName[:maxLength] // Cutting the string may result in char like "-" as the string end. // If the last char is not a letter or '_', replace it with "_". if !unicode.IsLetter(rune(ipcName[len(ipcName)-1:][0])) && ipcName[len(ipcName)-1:] != "_" { @@ -929,7 +1033,7 @@ func (as *availabilitySet) EnsureHostInPool(service *v1.Service, nodeName types. } var primaryIPConfig *network.InterfaceIPConfiguration - ipv6 := utilnet.IsIPv6String(service.Spec.ClusterIP) + ipv6 := ifBackendPoolIPv6(backendPoolID) if !as.Cloud.ipv6DualStackEnabled && !ipv6 { primaryIPConfig, err = getPrimaryIPConfig(nic) if err != nil { diff --git a/pkg/provider/azure_standard_test.go b/pkg/provider/azure_standard_test.go index 8a23877831..b9b8b7cd3e 100644 --- a/pkg/provider/azure_standard_test.go +++ b/pkg/provider/azure_standard_test.go @@ -902,22 +902,25 @@ func TestGetBackendPoolName(t *testing.T) { service v1.Service clusterName string expectedPoolName string + isIPv6 bool }{ { name: "GetBackendPoolName should return -IPv6", service: getTestService("test1", v1.ProtocolTCP, nil, true, 80), clusterName: "azure", expectedPoolName: "azure-IPv6", + isIPv6: true, }, { name: "GetBackendPoolName should return ", service: getTestService("test1", v1.ProtocolTCP, nil, false, 80), clusterName: "azure", expectedPoolName: "azure", + isIPv6: false, }, } for _, test := range testcases { - backPoolName := getBackendPoolName(test.clusterName, &test.service) + backPoolName := getBackendPoolName(test.clusterName, test.isIPv6) assert.Equal(t, test.expectedPoolName, backPoolName, test.name) } } @@ -1628,13 +1631,13 @@ func TestStandardEnsureHostsInPool(t *testing.T) { func TestServiceOwnsFrontendIP(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - cloud := GetTestCloud(ctrl) testCases := []struct { desc string existingPIPs []network.PublicIPAddress fip network.FrontendIPConfiguration service *v1.Service + ipFamily IPFamily isOwned bool isPrimary bool expectedErr error @@ -1760,12 +1763,57 @@ func TestServiceOwnsFrontendIP(t *testing.T) { }, service: &v1.Service{ ObjectMeta: meta.ObjectMeta{ - UID: types.UID("secondary"), - Annotations: map[string]string{consts.ServiceAnnotationLoadBalancerIPDualStack[false]: "4.3.2.1"}, + UID: types.UID("secondary"), + Annotations: map[string]string{ + consts.ServiceAnnotationLoadBalancerIPDualStack[false]: "4.3.2.1", + consts.ServiceAnnotationLoadBalancerIPDualStack[true]: "fd00::eef0", + }, }, }, isOwned: true, }, + { + desc: "serviceOwnsFrontendIP should detect the secondary external service dual-stack", + existingPIPs: []network.PublicIPAddress{ + { + ID: pointer.String("pip"), + PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ + PublicIPAddressVersion: network.IPVersionIPv4, + IPAddress: pointer.String("4.3.2.1"), + }, + }, + { + ID: pointer.String("pip1"), + PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ + PublicIPAddressVersion: network.IPVersionIPv6, + IPAddress: pointer.String("fd00::eef0"), + }, + }, + }, + fip: network.FrontendIPConfiguration{ + Name: pointer.String("auid"), + FrontendIPConfigurationPropertiesFormat: &network.FrontendIPConfigurationPropertiesFormat{ + PublicIPAddress: &network.PublicIPAddress{ + PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ + PublicIPAddressVersion: network.IPVersionIPv6, + }, + ID: pointer.String("pip1"), + }, + }, + }, + service: &v1.Service{ + ObjectMeta: meta.ObjectMeta{ + UID: types.UID("secondary"), + Annotations: map[string]string{ + consts.ServiceAnnotationLoadBalancerIPDualStack[false]: "4.3.2.1", + consts.ServiceAnnotationLoadBalancerIPDualStack[true]: "fd00::eef0", + }, + }, + }, + ipFamily: DualStack, + isOwned: true, + isPrimary: false, + }, { desc: "serviceOwnsFrontendIP should detect the secondary internal service", fip: network.FrontendIPConfiguration{ @@ -1788,10 +1836,16 @@ func TestServiceOwnsFrontendIP(t *testing.T) { } for _, test := range testCases { - isOwned, isPrimary, err := cloud.serviceOwnsFrontendIP(test.fip, test.service, &test.existingPIPs) - assert.Equal(t, test.expectedErr, err, test.desc) - assert.Equal(t, test.isOwned, isOwned, test.desc) - assert.Equal(t, test.isPrimary, isPrimary, test.desc) + t.Run(test.desc, func(t *testing.T) { + cloud := GetTestCloud(ctrl) + if test.ipFamily != "" { + cloud.IPFamily = test.ipFamily + } + isOwned, isPrimary, err := cloud.serviceOwnsFrontendIP(test.fip, test.service, &test.existingPIPs) + assert.Equal(t, test.expectedErr, err) + assert.Equal(t, test.isOwned, isOwned) + assert.Equal(t, test.isPrimary, isPrimary) + }) } } diff --git a/pkg/provider/azure_test.go b/pkg/provider/azure_test.go index 077c9e39d6..c3a83ff9db 100644 --- a/pkg/provider/azure_test.go +++ b/pkg/provider/azure_test.go @@ -164,13 +164,42 @@ func setMockEnv(az *Cloud, ctrl *gomock.Controller, expectedInterfaces []network } func setMockPublicIPs(az *Cloud, ctrl *gomock.Controller, serviceCount int) { + mockPIPsClient := mockpublicipclient.NewMockInterface(ctrl) + + v4Enabled, v6Enabled := az.ifIPFamiliesEnabled() + expectedPIPs := []network.PublicIPAddress{} + if v4Enabled { + expectedPIP := setMockPublicIP(az, mockPIPsClient, serviceCount, false) + expectedPIPs = append(expectedPIPs, expectedPIP) + } + if v6Enabled { + expectedPIP := setMockPublicIP(az, mockPIPsClient, serviceCount, true) + expectedPIPs = append(expectedPIPs, expectedPIP) + } + + az.PublicIPAddressesClient = mockPIPsClient + mockPIPsClient.EXPECT().CreateOrUpdate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + mockPIPsClient.EXPECT().List(gomock.Any(), gomock.Not(az.ResourceGroup)).Return(nil, nil).AnyTimes() + mockPIPsClient.EXPECT().Get(gomock.Any(), gomock.Not(az.ResourceGroup), gomock.Any(), gomock.Any()).Return(network.PublicIPAddress{}, &retry.Error{HTTPStatusCode: http.StatusNotFound, RawError: cloudprovider.InstanceNotFound}).AnyTimes() +} + +func setMockPublicIP(az *Cloud, mockPIPsClient *mockpublicipclient.MockInterface, serviceCount int, isIPv6 bool) network.PublicIPAddress { + suffix := v4Suffix + ipVer := network.IPVersionIPv4 + ipAddr := "1.2.3.4" + if isIPv6 { + suffix = v6Suffix + ipVer = network.IPVersionIPv6 + ipAddr = "fd00::eef0" + } + expectedPIP := network.PublicIPAddress{ Name: pointer.String("testCluster-aservicea"), Location: &az.Location, PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ PublicIPAllocationMethod: network.IPAllocationMethodStatic, - PublicIPAddressVersion: network.IPVersionIPv4, - IPAddress: pointer.String("1.2.3.4"), + PublicIPAddressVersion: ipVer, + IPAddress: pointer.String(ipAddr), }, Tags: map[string]*string{ consts.ServiceTagKey: pointer.String("default/servicea"), @@ -182,28 +211,24 @@ func setMockPublicIPs(az *Cloud, ctrl *gomock.Controller, serviceCount int) { ID: pointer.String("testCluster-aservice1"), } - mockPIPsClient := mockpublicipclient.NewMockInterface(ctrl) - az.PublicIPAddressesClient = mockPIPsClient - mockPIPsClient.EXPECT().CreateOrUpdate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() - mockPIPsClient.EXPECT().List(gomock.Any(), gomock.Not(az.ResourceGroup)).Return(nil, nil).AnyTimes() - mockPIPsClient.EXPECT().Get(gomock.Any(), gomock.Not(az.ResourceGroup), gomock.Any(), gomock.Any()).Return(network.PublicIPAddress{}, &retry.Error{HTTPStatusCode: http.StatusNotFound, RawError: cloudprovider.InstanceNotFound}).AnyTimes() - a := 'a' var expectedPIPs []network.PublicIPAddress for i := 1; i <= serviceCount; i++ { - expectedPIP.Name = pointer.String(fmt.Sprintf("testCluster-aservice%d", i)) + expectedPIP.Name = pointer.String(fmt.Sprintf("testCluster-aservice%d-%s", i, suffix)) expectedPIP.Tags[consts.ServiceTagKey] = pointer.String(fmt.Sprintf("default/service%d", i)) - mockPIPsClient.EXPECT().Get(gomock.Any(), az.ResourceGroup, fmt.Sprintf("testCluster-aservice%d", i), gomock.Any()).Return(expectedPIP, nil).AnyTimes() - mockPIPsClient.EXPECT().Delete(gomock.Any(), az.ResourceGroup, fmt.Sprintf("testCluster-aservice%d", i)).Return(nil).AnyTimes() + mockPIPsClient.EXPECT().Get(gomock.Any(), az.ResourceGroup, fmt.Sprintf("testCluster-aservice%d-%s", i, suffix), gomock.Any()).Return(expectedPIP, nil).AnyTimes() + mockPIPsClient.EXPECT().Delete(gomock.Any(), az.ResourceGroup, fmt.Sprintf("testCluster-aservice%d-%s", i, suffix)).Return(nil).AnyTimes() expectedPIPs = append(expectedPIPs, expectedPIP) - expectedPIP.Name = pointer.String(fmt.Sprintf("testCluster-aservice%c", a)) + expectedPIP.Name = pointer.String(fmt.Sprintf("testCluster-aservice%c-%s", a, suffix)) expectedPIP.Tags[consts.ServiceTagKey] = pointer.String(fmt.Sprintf("default/service%c", a)) - mockPIPsClient.EXPECT().Get(gomock.Any(), az.ResourceGroup, fmt.Sprintf("testCluster-aservice%c", a), gomock.Any()).Return(expectedPIP, nil).AnyTimes() - mockPIPsClient.EXPECT().Delete(gomock.Any(), az.ResourceGroup, fmt.Sprintf("testCluster-aservice%c", a)).Return(nil).AnyTimes() + mockPIPsClient.EXPECT().Get(gomock.Any(), az.ResourceGroup, fmt.Sprintf("testCluster-aservice%c-%s", a, suffix), gomock.Any()).Return(expectedPIP, nil).AnyTimes() + mockPIPsClient.EXPECT().Delete(gomock.Any(), az.ResourceGroup, fmt.Sprintf("testCluster-aservice%c-%s", a, suffix)).Return(nil).AnyTimes() expectedPIPs = append(expectedPIPs, expectedPIP) a++ } mockPIPsClient.EXPECT().List(gomock.Any(), az.ResourceGroup).Return(expectedPIPs, nil).AnyTimes() + + return expectedPIP } func setMockSecurityGroup(az *Cloud, ctrl *gomock.Controller, sgs ...*network.SecurityGroup) { @@ -216,6 +241,8 @@ func setMockSecurityGroup(az *Cloud, ctrl *gomock.Controller, sgs ...*network.Se } func setMockLBs(az *Cloud, ctrl *gomock.Controller, expectedLBs *[]network.LoadBalancer, svcName string, lbCount, serviceIndex int, isInternal bool) string { + v4Enabled, v6Enabled := az.ifIPFamiliesEnabled() + lbIndex := (serviceIndex - 1) % lbCount expectedLBName := "" if lbIndex == 0 { @@ -241,43 +268,67 @@ func setMockLBs(az *Cloud, ctrl *gomock.Controller, expectedLBs *[]network.LoadB }, } lb.Name = &expectedLBName - lb.LoadBalancingRules = &[]network.LoadBalancingRule{ - { - Name: pointer.String(fmt.Sprintf("a%s%d-TCP-8081", fullServiceName, serviceIndex)), - }, - } - fips := []network.FrontendIPConfiguration{ - { - Name: pointer.String(fmt.Sprintf("a%s%d", fullServiceName, serviceIndex)), + rules := []network.LoadBalancingRule{} + fips := []network.FrontendIPConfiguration{} + addRuleAndFIP := func(isIPv6 bool) { + suffix := v4Suffix + if isIPv6 { + suffix = v6Suffix + } + rules = append(rules, network.LoadBalancingRule{ + Name: pointer.String(fmt.Sprintf("a%s%d-TCP-8081-%s", fullServiceName, serviceIndex, suffix)), + }) + fips = append(fips, network.FrontendIPConfiguration{ + Name: pointer.String(fmt.Sprintf("a%s%d-%s", fullServiceName, serviceIndex, suffix)), ID: pointer.String("fip"), FrontendIPConfigurationPropertiesFormat: &network.FrontendIPConfigurationPropertiesFormat{ - PrivateIPAllocationMethod: "Dynamic", - PublicIPAddress: &network.PublicIPAddress{ID: pointer.String(fmt.Sprintf("testCluster-a%s%d", fullServiceName, serviceIndex))}, - }, - }, + PrivateIPAllocationMethod: network.IPAllocationMethodDynamic, + PublicIPAddress: &network.PublicIPAddress{ID: pointer.String(fmt.Sprintf("testCluster-a%s%d-%s", fullServiceName, serviceIndex, suffix))}, + }}) + } + if v4Enabled { + addRuleAndFIP(false) } + if v6Enabled { + addRuleAndFIP(true) + } + if isInternal { fips[0].Subnet = &network.Subnet{Name: pointer.String("subnet")} } + lb.LoadBalancingRules = &rules lb.FrontendIPConfigurations = &fips *expectedLBs = append(*expectedLBs, lb) } else { - *(*expectedLBs)[lbIndex].LoadBalancingRules = append(*(*expectedLBs)[lbIndex].LoadBalancingRules, network.LoadBalancingRule{ - Name: pointer.String(fmt.Sprintf("a%s%d-TCP-8081", fullServiceName, serviceIndex)), - }) - fip := network.FrontendIPConfiguration{ - Name: pointer.String(fmt.Sprintf("a%s%d", fullServiceName, serviceIndex)), - ID: pointer.String("fip"), - FrontendIPConfigurationPropertiesFormat: &network.FrontendIPConfigurationPropertiesFormat{ - PrivateIPAllocationMethod: "Dynamic", - PublicIPAddress: &network.PublicIPAddress{ID: pointer.String(fmt.Sprintf("testCluster-a%s%d", fullServiceName, serviceIndex))}, - }, + addRuleAndFIP := func(isIPv6 bool) { + suffix := v4Suffix + if isIPv6 { + suffix = v6Suffix + } + *(*expectedLBs)[lbIndex].LoadBalancingRules = append(*(*expectedLBs)[lbIndex].LoadBalancingRules, network.LoadBalancingRule{ + Name: pointer.String(fmt.Sprintf("a%s%d-TCP-8081-%s", fullServiceName, serviceIndex, suffix)), + }) + + fip := network.FrontendIPConfiguration{ + Name: pointer.String(fmt.Sprintf("a%s%d-%s", fullServiceName, serviceIndex, suffix)), + ID: pointer.String("fip"), + FrontendIPConfigurationPropertiesFormat: &network.FrontendIPConfigurationPropertiesFormat{ + PrivateIPAllocationMethod: network.IPAllocationMethodDynamic, + PublicIPAddress: &network.PublicIPAddress{ID: pointer.String(fmt.Sprintf("testCluster-a%s%d-%s", fullServiceName, serviceIndex, suffix))}, + }, + } + if isInternal { + fip.Subnet = &network.Subnet{Name: pointer.String("subnet")} + } + *(*expectedLBs)[lbIndex].FrontendIPConfigurations = append(*(*expectedLBs)[lbIndex].FrontendIPConfigurations, fip) } - if isInternal { - fip.Subnet = &network.Subnet{Name: pointer.String("subnet")} + if v4Enabled { + addRuleAndFIP(false) + } + if v6Enabled { + addRuleAndFIP(true) } - *(*expectedLBs)[lbIndex].FrontendIPConfigurations = append(*(*expectedLBs)[lbIndex].FrontendIPConfigurations, fip) } mockLBsClient := mockloadbalancerclient.NewMockInterface(ctrl) @@ -716,22 +767,26 @@ func TestReconcileSecurityGroupFromAnyDestinationAddressPrefixToLoadBalancerIP(t defer ctrl.Finish() az := GetTestCloud(ctrl) + az.IPFamily = DualStack svc1 := getTestService("serviceea", v1.ProtocolTCP, nil, false, 80) + makeTestServiceDualStack(&svc1) setServiceLoadBalancerIP(&svc1, "192.168.0.0") + setServiceLoadBalancerIP(&svc1, "fdf8:f535:82e4::53") sg := getTestSecurityGroup(az) setMockSecurityGroup(az, ctrl, sg) // Simulate a pre-Kubernetes 1.8 NSG, where we do not specify the destination address prefix - _, err := az.reconcileSecurityGroup(testClusterName, &svc1, pointer.String(""), nil, true) + _, err := az.reconcileSecurityGroup(testClusterName, &svc1, &[]string{""}, nil, true) if err != nil { t.Errorf("Unexpected error: %q", err) } - sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, pointer.String(getServiceLoadBalancerIP(&svc1)), nil, true) + lbIPs := []string{getServiceLoadBalancerIP(&svc1, false), getServiceLoadBalancerIP(&svc1, true)} + sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, &lbIPs, nil, true) if err != nil { t.Errorf("Unexpected error: %q", err) } - validateSecurityGroup(t, sg, svc1) + validateSecurityGroup(t, az, sg, svc1) } func TestReconcileSecurityGroupDynamicLoadBalancerIP(t *testing.T) { @@ -739,18 +794,19 @@ func TestReconcileSecurityGroupDynamicLoadBalancerIP(t *testing.T) { defer ctrl.Finish() az := GetTestCloud(ctrl) + az.IPFamily = DualStack svc1 := getTestService("servicea", v1.ProtocolTCP, nil, false, 80) setServiceLoadBalancerIP(&svc1, "") sg := getTestSecurityGroup(az) setMockSecurityGroup(az, ctrl, sg) - dynamicallyAssignedIP := "192.168.0.0" - sg, err := az.reconcileSecurityGroup(testClusterName, &svc1, pointer.String(dynamicallyAssignedIP), nil, true) + dynamicallyAssignedIPs := []string{"192.168.0.0", "fdf8:f535:82e4::53"} + sg, err := az.reconcileSecurityGroup(testClusterName, &svc1, &dynamicallyAssignedIPs, nil, true) if err != nil { t.Errorf("unexpected error: %q", err) } - validateSecurityGroup(t, sg, svc1) + validateSecurityGroup(t, az, sg, svc1) } // Test addition of services on an internal LB using both default and explicit subnets. @@ -1236,6 +1292,7 @@ func TestReconcileSecurityGroupNewServiceAddsPort(t *testing.T) { defer ctrl.Finish() az := GetTestCloud(ctrl) + az.IPFamily = DualStack getTestSecurityGroup(az) svc1 := getTestService("service1", v1.ProtocolTCP, nil, false, 80) clusterResources, expectedInterfaces, expectedVirtualMachines := getClusterResources(az, 1, 1) @@ -1246,14 +1303,14 @@ func TestReconcileSecurityGroupNewServiceAddsPort(t *testing.T) { mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, nil).AnyTimes() mockLBBackendPool.EXPECT().EnsureHostsInPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() lb, _ := az.reconcileLoadBalancer(testClusterName, &svc1, clusterResources.nodes, true) - lbStatus, _, _ := az.getServiceLoadBalancerStatus(&svc1, lb, nil) + lbStatus, _, _, _ := az.getServiceLoadBalancerStatus(&svc1, lb, nil) - sg, err := az.reconcileSecurityGroup(testClusterName, &svc1, &lbStatus.Ingress[0].IP, nil, true /* wantLb */) - if err != nil { - t.Errorf("Unexpected error: %q", err) - } + assert.Equal(t, 2, len(lbStatus.Ingress)) + lbIPs := []string{lbStatus.Ingress[0].IP, lbStatus.Ingress[1].IP} + sg, err := az.reconcileSecurityGroup(testClusterName, &svc1, &lbIPs, nil, true /* wantLb */) + assert.NoError(t, err) - validateSecurityGroup(t, sg, svc1) + validateSecurityGroup(t, az, sg, svc1) } func TestReconcileSecurityGroupNewInternalServiceAddsPort(t *testing.T) { @@ -1275,22 +1332,26 @@ func TestReconcileSecurityGroupNewInternalServiceAddsPort(t *testing.T) { mockLBBackendPool.EXPECT().EnsureHostsInPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() lb, _ := az.reconcileLoadBalancer(testClusterName, &svc1, clusterResources.nodes, true) - lbStatus, _, _ := az.getServiceLoadBalancerStatus(&svc1, lb, nil) - sg, err := az.reconcileSecurityGroup(testClusterName, &svc1, &lbStatus.Ingress[0].IP, nil, true /* wantLb */) + lbStatus, _, _, _ := az.getServiceLoadBalancerStatus(&svc1, lb, nil) + sg, err := az.reconcileSecurityGroup(testClusterName, &svc1, &[]string{lbStatus.Ingress[0].IP}, nil, true /* wantLb */) if err != nil { t.Errorf("Unexpected error: %q", err) } - validateSecurityGroup(t, sg, svc1) + validateSecurityGroup(t, az, sg, svc1) } +// TestReconcileSecurityGroupRemoveService keeps service1 but removes service2. func TestReconcileSecurityGroupRemoveService(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() az := GetTestCloud(ctrl) + az.IPFamily = DualStack service1 := getTestService("service1", v1.ProtocolTCP, nil, false, 81) + makeTestServiceDualStack(&service1) service2 := getTestService("service2", v1.ProtocolTCP, nil, false, 82) + makeTestServiceDualStack(&service2) clusterResources, expectedInterfaces, expectedVirtualMachines := getClusterResources(az, 1, 1) setMockEnv(az, ctrl, expectedInterfaces, expectedVirtualMachines, 2, service1, service2) @@ -1298,7 +1359,7 @@ func TestReconcileSecurityGroupRemoveService(t *testing.T) { setMockLBs(az, ctrl, &expectedLBs, "service", 1, 1, true) mockLBClient := az.LoadBalancerClient.(*mockloadbalancerclient.MockInterface) mockLBClient.EXPECT().Get(gomock.Any(), az.ResourceGroup, "testCluster", gomock.Any()).Return( - getTestLoadBalancer(pointer.String(testClusterName), + getTestLoadBalancerDualStack(pointer.String(testClusterName), pointer.String(az.ResourceGroup), pointer.String(testClusterName), pointer.String("aservice1"), @@ -1312,17 +1373,18 @@ func TestReconcileSecurityGroupRemoveService(t *testing.T) { lb, _ := az.reconcileLoadBalancer(testClusterName, &service1, clusterResources.nodes, true) _, _ = az.reconcileLoadBalancer(testClusterName, &service2, clusterResources.nodes, true) - lbStatus, _, _ := az.getServiceLoadBalancerStatus(&service1, lb, nil) + lbStatus, _, _, err := az.getServiceLoadBalancerStatus(&service1, lb, nil) + assert.NoError(t, err) + assert.NotNil(t, lbStatus) sg := getTestSecurityGroup(az, service1, service2) - validateSecurityGroup(t, sg, service1, service2) + validateSecurityGroup(t, az, sg, service1, service2) - sg, err := az.reconcileSecurityGroup(testClusterName, &service1, &lbStatus.Ingress[0].IP, nil, false /* wantLb */) - if err != nil { - t.Errorf("Unexpected error: %q", err) - } + assert.Equal(t, 2, len(lbStatus.Ingress)) + sg, err = az.reconcileSecurityGroup(testClusterName, &service1, &[]string{lbStatus.Ingress[0].IP, lbStatus.Ingress[1].IP}, nil, false /* wantLb */) + assert.NoError(t, err) - validateSecurityGroup(t, sg, service2) + validateSecurityGroup(t, az, sg, service2) } func TestReconcileSecurityGroupRemoveServiceRemovesPort(t *testing.T) { @@ -1351,14 +1413,14 @@ func TestReconcileSecurityGroupRemoveServiceRemovesPort(t *testing.T) { svc, "Standard"), nil).AnyTimes() lb, _ := az.reconcileLoadBalancer(testClusterName, &svc, clusterResources.nodes, true) - lbStatus, _, _ := az.getServiceLoadBalancerStatus(&svc, lb, nil) + lbStatus, _, _, _ := az.getServiceLoadBalancerStatus(&svc, lb, nil) - sg, err := az.reconcileSecurityGroup(testClusterName, &svcUpdated, &lbStatus.Ingress[0].IP, nil, true /* wantLb */) + sg, err := az.reconcileSecurityGroup(testClusterName, &svcUpdated, &[]string{lbStatus.Ingress[0].IP}, nil, true /* wantLb */) if err != nil { t.Errorf("Unexpected error: %q", err) } - validateSecurityGroup(t, sg, svcUpdated) + validateSecurityGroup(t, az, sg, svcUpdated) } func TestReconcileSecurityWithSourceRanges(t *testing.T) { @@ -1382,14 +1444,14 @@ func TestReconcileSecurityWithSourceRanges(t *testing.T) { expectedLBs := make([]network.LoadBalancer, 0) setMockLBs(az, ctrl, &expectedLBs, "service", 1, 1, false) lb, _ := az.reconcileLoadBalancer(testClusterName, &svc, clusterResources.nodes, true) - lbStatus, _, _ := az.getServiceLoadBalancerStatus(&svc, lb, nil) + lbStatus, _, _, _ := az.getServiceLoadBalancerStatus(&svc, lb, nil) - sg, err := az.reconcileSecurityGroup(testClusterName, &svc, &lbStatus.Ingress[0].IP, nil, true /* wantLb */) + sg, err := az.reconcileSecurityGroup(testClusterName, &svc, &[]string{lbStatus.Ingress[0].IP}, nil, true /* wantLb */) if err != nil { t.Errorf("Unexpected error: %q", err) } - validateSecurityGroup(t, sg, svc) + validateSecurityGroup(t, az, sg, svc) } func TestReconcileSecurityGroupEtagMismatch(t *testing.T) { @@ -1430,112 +1492,128 @@ func TestReconcileSecurityGroupEtagMismatch(t *testing.T) { mockLBBackendPool.EXPECT().EnsureHostsInPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() lb, _ := az.reconcileLoadBalancer(testClusterName, &svc1, clusterResources.nodes, true) - lbStatus, _, _ := az.getServiceLoadBalancerStatus(&svc1, lb, nil) + lbStatus, _, _, _ := az.getServiceLoadBalancerStatus(&svc1, lb, nil) - newSG, err := az.reconcileSecurityGroup(testClusterName, &svc1, &lbStatus.Ingress[0].IP, nil, true /* wantLb */) + newSG, err := az.reconcileSecurityGroup(testClusterName, &svc1, &[]string{lbStatus.Ingress[0].IP}, nil, true /* wantLb */) assert.Nil(t, newSG) assert.Error(t, err) assert.Equal(t, expectedError.Error(), err) } -func TestReconcilePublicIPWithNewService(t *testing.T) { +func TestReconcilePublicIPsWithNewService(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() az := GetTestCloud(ctrl) + az.IPFamily = DualStack svc := getTestService("servicea", v1.ProtocolTCP, nil, false, 80, 443) + makeTestServiceDualStack(&svc) setMockPublicIPs(az, ctrl, 1) - pip, err := az.reconcilePublicIP(testClusterName, &svc, "", true /* wantLb*/) + pips, err := az.reconcilePublicIPs(testClusterName, &svc, "", true /* wantLb*/) if err != nil { t.Errorf("Unexpected error: %q", err) } - validatePublicIP(t, pip, &svc, true) + validatePublicIPs(t, pips, &svc, true) - pip2, err := az.reconcilePublicIP(testClusterName, &svc, "", true /* wantLb */) + pips2, err := az.reconcilePublicIPs(testClusterName, &svc, "", true /* wantLb */) if err != nil { t.Errorf("Unexpected error: %q", err) } - validatePublicIP(t, pip2, &svc, true) - if pip.Name != pip2.Name || - pip.PublicIPAddressPropertiesFormat.IPAddress != pip2.PublicIPAddressPropertiesFormat.IPAddress { - t.Errorf("We should get the exact same public ip resource after a second reconcile") + validatePublicIPs(t, pips2, &svc, true) + + pipsNames, pips2Names := []string{}, []string{} + pipsAddrs, pips2Addrs := []string{}, []string{} + for _, pip := range pips { + pipsNames = append(pipsNames, *pip.Name) + pipsAddrs = append(pipsAddrs, pointer.StringDeref(pip.PublicIPAddressPropertiesFormat.IPAddress, "")) + } + for _, pip := range pips2 { + pips2Names = append(pips2Names, *pip.Name) + pips2Addrs = append(pips2Addrs, pointer.StringDeref(pip.PublicIPAddressPropertiesFormat.IPAddress, "")) } + assert.Truef(t, compareStrings(pipsNames, pips2Names) && compareStrings(pipsAddrs, pips2Addrs), + "We should get the exact same public ip resource after a second reconcile") } -func TestReconcilePublicIPRemoveService(t *testing.T) { +func TestReconcilePublicIPsRemoveService(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() az := GetTestCloud(ctrl) + az.IPFamily = DualStack svc := getTestService("servicea", v1.ProtocolTCP, nil, false, 80, 443) + makeTestServiceDualStack(&svc) setMockPublicIPs(az, ctrl, 1) - pip, err := az.reconcilePublicIP(testClusterName, &svc, "", true /* wantLb*/) + pips, err := az.reconcilePublicIPs(testClusterName, &svc, "", true /* wantLb*/) if err != nil { t.Errorf("Unexpected error: %q", err) } - validatePublicIP(t, pip, &svc, true) + validatePublicIPs(t, pips, &svc, true) // Remove the service - pip, err = az.reconcilePublicIP(testClusterName, &svc, "", false /* wantLb */) + pips, err = az.reconcilePublicIPs(testClusterName, &svc, "", false /* wantLb */) if err != nil { t.Errorf("Unexpected error: %q", err) } - validatePublicIP(t, pip, &svc, false) - + validatePublicIPs(t, pips, &svc, false) } -func TestReconcilePublicIPWithInternalService(t *testing.T) { +func TestReconcilePublicIPsWithInternalService(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() az := GetTestCloud(ctrl) + az.IPFamily = DualStack svc := getInternalTestService("servicea", 80, 443) + makeTestServiceDualStack(&svc) setMockPublicIPs(az, ctrl, 1) - pip, err := az.reconcilePublicIP(testClusterName, &svc, "", true /* wantLb*/) + pips, err := az.reconcilePublicIPs(testClusterName, &svc, "", true /* wantLb*/) if err != nil { t.Errorf("Unexpected error: %q", err) } - validatePublicIP(t, pip, &svc, true) + validatePublicIPs(t, pips, &svc, true) } -func TestReconcilePublicIPWithExternalAndInternalSwitch(t *testing.T) { +func TestReconcilePublicIPsWithExternalAndInternalSwitch(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() az := GetTestCloud(ctrl) + az.IPFamily = DualStack svc := getInternalTestService("servicea", 80, 443) + makeTestServiceDualStack(&svc) setMockPublicIPs(az, ctrl, 1) - pip, err := az.reconcilePublicIP(testClusterName, &svc, "", true /* wantLb*/) + pips, err := az.reconcilePublicIPs(testClusterName, &svc, "", true /* wantLb*/) if err != nil { t.Errorf("Unexpected error: %q", err) } - validatePublicIP(t, pip, &svc, true) + validatePublicIPs(t, pips, &svc, true) // Update to external service svcUpdated := getTestService("servicea", v1.ProtocolTCP, nil, false, 80) - pip, err = az.reconcilePublicIP(testClusterName, &svcUpdated, "", true /* wantLb*/) + pips, err = az.reconcilePublicIPs(testClusterName, &svcUpdated, "", true /* wantLb*/) if err != nil { t.Errorf("Unexpected error: %q", err) } - validatePublicIP(t, pip, &svcUpdated, true) + validatePublicIPs(t, pips, &svcUpdated, true) // Update to internal service again - pip, err = az.reconcilePublicIP(testClusterName, &svc, "", true /* wantLb*/) + pips, err = az.reconcilePublicIPs(testClusterName, &svc, "", true /* wantLb*/) if err != nil { t.Errorf("Unexpected error: %q", err) } - validatePublicIP(t, pip, &svc, true) + validatePublicIPs(t, pips, &svc, true) } const networkInterfacesIDTemplate = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/networkInterfaces/%s" @@ -1696,6 +1774,11 @@ func getTestService(identifier string, proto v1.Protocol, annotations map[string return svc } +// TODO: This function should be merged into getTestService() +func makeTestServiceDualStack(svc *v1.Service) { + svc.Spec.ClusterIPs = []string{"10.0.0.2", "fd00::1907"} +} + func getInternalTestService(identifier string, requestedPorts ...int32) v1.Service { return getTestServiceWithAnnotation(identifier, map[string]string{consts.ServiceAnnotationLoadBalancerInternal: consts.TrueAnnotationValue}, requestedPorts...) } @@ -1734,20 +1817,31 @@ func getServiceSourceRanges(service *v1.Service) []string { } func getTestSecurityGroup(az *Cloud, services ...v1.Service) *network.SecurityGroup { - rules := []network.SecurityRule{} + v4Enabled, v6Enabled := az.ifIPFamiliesEnabled() + getRule := func(svc *v1.Service, port v1.ServicePort, src string, isIPv6 bool) network.SecurityRule { + ruleName := az.getSecurityRuleName(svc, port, src, isIPv6) + return network.SecurityRule{ + Name: pointer.String(ruleName), + SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{ + SourceAddressPrefix: pointer.String(src), + DestinationPortRange: pointer.String(fmt.Sprintf("%d", port.Port)), + }, + } + } + rules := []network.SecurityRule{} for i, service := range services { for _, port := range service.Spec.Ports { sources := getServiceSourceRanges(&services[i]) for _, src := range sources { - ruleName := az.getSecurityRuleName(&services[i], port, src) - rules = append(rules, network.SecurityRule{ - Name: pointer.String(ruleName), - SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{ - SourceAddressPrefix: pointer.String(src), - DestinationPortRange: pointer.String(fmt.Sprintf("%d", port.Port)), - }, - }) + if v4Enabled { + rule := getRule(&services[i], port, src, false) + rules = append(rules, rule) + } + if v6Enabled { + rule := getRule(&services[i], port, src, true) + rules = append(rules, rule) + } } } } @@ -1902,6 +1996,12 @@ func describeFIPs(frontendIPs []network.FrontendIPConfiguration) string { return description } +func validatePublicIPs(t *testing.T, pips []*network.PublicIPAddress, service *v1.Service, wantLb bool) { + for _, pip := range pips { + validatePublicIP(t, pip, service, wantLb) + } +} + func validatePublicIP(t *testing.T, publicIP *network.PublicIPAddress, service *v1.Service, wantLb bool) { isInternal := requiresInternalLoadBalancer(service) if isInternal || !wantLb { @@ -1921,17 +2021,11 @@ func validatePublicIP(t *testing.T, publicIP *network.PublicIPAddress, service * } serviceName := getServiceName(service) - if serviceName != *(publicIP.Tags[consts.ServiceTagKey]) { - t.Errorf("Expected publicIP resource has matching tags[%s]", consts.ServiceTagKey) - } - - if publicIP.Tags[consts.ClusterNameKey] == nil { - t.Fatalf("Expected publicIP resource does not have tags[%s]", consts.ClusterNameKey) - } - - if *(publicIP.Tags[consts.ClusterNameKey]) != testClusterName { - t.Errorf("Expected publicIP resource has matching tags[%s]", consts.ClusterNameKey) - } + assert.Equalf(t, serviceName, pointer.StringDeref(publicIP.Tags[consts.ServiceTagKey], ""), + "Expected publicIP resource has matching tags[%s]", consts.ServiceTagKey) + assert.NotNilf(t, publicIP.Tags[consts.ClusterNameKey], "Expected publicIP resource does not have tags[%s]", consts.ClusterNameKey) + assert.Equalf(t, testClusterName, pointer.StringDeref(publicIP.Tags[consts.ClusterNameKey], ""), + "Expected publicIP resource has matching tags[%s]", consts.ClusterNameKey) // We cannot use Service LoadBalancerIP to compare with // Public IP's IPAddress @@ -1990,32 +2084,40 @@ func securityRuleMatches(serviceSourceRange string, servicePort v1.ServicePort, return nil } -func validateSecurityGroup(t *testing.T, securityGroup *network.SecurityGroup, services ...v1.Service) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() +func validateSecurityGroup(t *testing.T, az *Cloud, securityGroup *network.SecurityGroup, services ...v1.Service) { + v4Enabled, v6Enabled := az.ifIPFamiliesEnabled() - az := GetTestCloud(ctrl) seenRules := make(map[string]string) for i, svc := range services { svc := svc for _, wantedRule := range svc.Spec.Ports { sources := getServiceSourceRanges(&services[i]) for _, source := range sources { - wantedRuleName := az.getSecurityRuleName(&services[i], wantedRule, source) - seenRules[wantedRuleName] = wantedRuleName - foundRule := false - for _, actualRule := range *securityGroup.SecurityRules { - if strings.EqualFold(*actualRule.Name, wantedRuleName) { - err := securityRuleMatches(source, wantedRule, getServiceLoadBalancerIP(&svc), actualRule) - if err != nil { - t.Errorf("Found matching security rule %q but properties were incorrect: %v", wantedRuleName, err) + checkRule := func(svc *v1.Service, wantedRule v1.ServicePort, source string, isIPv6 bool) { + wantedRuleName := az.getSecurityRuleName(svc, wantedRule, source, isIPv6) + seenRules[wantedRuleName] = wantedRuleName + + foundRule := false + for _, actualRule := range *securityGroup.SecurityRules { + if strings.EqualFold(*actualRule.Name, wantedRuleName) { + err := securityRuleMatches(source, wantedRule, getServiceLoadBalancerIP(svc, isIPv6), actualRule) + if err != nil { + t.Errorf("Found matching security rule %q but properties were incorrect: %v", wantedRuleName, err) + } + foundRule = true + break } - foundRule = true - break } + if !foundRule { + t.Errorf("Expected security group rule but didn't find it: %q", wantedRuleName) + } + } + + if v4Enabled { + checkRule(&services[i], wantedRule, source, false) } - if !foundRule { - t.Errorf("Expected security group rule but didn't find it: %q", wantedRuleName) + if v6Enabled { + checkRule(&services[i], wantedRule, source, true) } } } @@ -2023,9 +2125,7 @@ func validateSecurityGroup(t *testing.T, securityGroup *network.SecurityGroup, s lenRules := len(*securityGroup.SecurityRules) expectedRuleCount := len(seenRules) - if lenRules != expectedRuleCount { - t.Errorf("Expected the loadbalancer to have %d rules. Found %d.\n", expectedRuleCount, lenRules) - } + assert.Equalf(t, expectedRuleCount, lenRules, "Expected rules: %v", seenRules) } func TestSecurityRulePriorityPicksNextAvailablePriority(t *testing.T) { @@ -2452,12 +2552,12 @@ func TestIfServiceSpecifiesSharedRuleAndRuleDoesNotExistItIsCreated(t *testing.T sg := getTestSecurityGroup(az) setMockSecurityGroup(az, ctrl, sg) - sg, err := az.reconcileSecurityGroup(testClusterName, &svc, pointer.String(getServiceLoadBalancerIP(&svc)), nil, true) + sg, err := az.reconcileSecurityGroup(testClusterName, &svc, &[]string{getServiceLoadBalancerIP(&svc, false)}, nil, true) if err != nil { t.Errorf("Unexpected error: %q", err) } - validateSecurityGroup(t, sg, svc) + validateSecurityGroup(t, az, sg, svc) expectedRuleName := testRuleName _, securityRule, ruleFound := findSecurityRuleByName(*sg.SecurityRules, expectedRuleName) @@ -2511,12 +2611,12 @@ func TestIfServiceSpecifiesSharedRuleAndRuleExistsThenTheServicesPortAndAddressA } setMockSecurityGroup(az, ctrl, sg) - sg, err := az.reconcileSecurityGroup(testClusterName, &svc, pointer.String(getServiceLoadBalancerIP(&svc)), nil, true) + sg, err := az.reconcileSecurityGroup(testClusterName, &svc, &[]string{getServiceLoadBalancerIP(&svc, false)}, nil, true) if err != nil { t.Errorf("Unexpected error: %q", err) } - validateSecurityGroup(t, sg, svc) + validateSecurityGroup(t, az, sg, svc) _, securityRule, ruleFound := findSecurityRuleByName(*sg.SecurityRules, expectedRuleName) if !ruleFound { @@ -2555,18 +2655,18 @@ func TestIfServicesSpecifySharedRuleButDifferentPortsThenSeparateRulesAreCreated sg := getTestSecurityGroup(az) setMockSecurityGroup(az, ctrl, sg) - sg, err := az.reconcileSecurityGroup(testClusterName, &svc1, pointer.String(getServiceLoadBalancerIP(&svc1)), nil, true) + sg, err := az.reconcileSecurityGroup(testClusterName, &svc1, &[]string{getServiceLoadBalancerIP(&svc1, false)}, nil, true) if err != nil { t.Errorf("Unexpected error adding svc1: %q", err) } setMockSecurityGroup(az, ctrl, sg) - sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, pointer.String(getServiceLoadBalancerIP(&svc2)), nil, true) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, &[]string{getServiceLoadBalancerIP(&svc2, false)}, nil, true) if err != nil { t.Errorf("Unexpected error adding svc2: %q", err) } - validateSecurityGroup(t, sg, svc1, svc2) + validateSecurityGroup(t, az, sg, svc1, svc2) _, securityRule1, rule1Found := findSecurityRuleByName(*sg.SecurityRules, testRuleName2) if !rule1Found { @@ -2628,18 +2728,18 @@ func TestIfServicesSpecifySharedRuleButDifferentProtocolsThenSeparateRulesAreCre sg := getTestSecurityGroup(az) setMockSecurityGroup(az, ctrl, sg) - sg, err := az.reconcileSecurityGroup(testClusterName, &svc1, pointer.String(getServiceLoadBalancerIP(&svc1)), nil, true) + sg, err := az.reconcileSecurityGroup(testClusterName, &svc1, &[]string{getServiceLoadBalancerIP(&svc1, false)}, nil, true) if err != nil { t.Errorf("Unexpected error adding svc1: %q", err) } setMockSecurityGroup(az, ctrl, sg) - sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, pointer.String(getServiceLoadBalancerIP(&svc2)), nil, true) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, &[]string{getServiceLoadBalancerIP(&svc2, false)}, nil, true) if err != nil { t.Errorf("Unexpected error adding svc2: %q", err) } - validateSecurityGroup(t, sg, svc1, svc2) + validateSecurityGroup(t, az, sg, svc1, svc2) _, securityRule1, rule1Found := findSecurityRuleByName(*sg.SecurityRules, testRuleName2) if !rule1Found { @@ -2701,18 +2801,18 @@ func TestIfServicesSpecifySharedRuleButDifferentSourceAddressesThenSeparateRules sg := getTestSecurityGroup(az) setMockSecurityGroup(az, ctrl, sg) - sg, err := az.reconcileSecurityGroup(testClusterName, &svc1, pointer.String(getServiceLoadBalancerIP(&svc1)), nil, true) + sg, err := az.reconcileSecurityGroup(testClusterName, &svc1, &[]string{getServiceLoadBalancerIP(&svc1, false)}, nil, true) if err != nil { t.Errorf("Unexpected error adding svc1: %q", err) } setMockSecurityGroup(az, ctrl, sg) - sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, pointer.String(getServiceLoadBalancerIP(&svc2)), nil, true) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, &[]string{getServiceLoadBalancerIP(&svc2, false)}, nil, true) if err != nil { t.Errorf("Unexpected error adding svc2: %q", err) } - validateSecurityGroup(t, sg, svc1, svc2) + validateSecurityGroup(t, az, sg, svc1, svc2) _, securityRule1, rule1Found := findSecurityRuleByName(*sg.SecurityRules, testRuleName2) if !rule1Found { @@ -2776,24 +2876,24 @@ func TestIfServicesSpecifySharedRuleButSomeAreOnDifferentPortsThenRulesAreSepara sg := getTestSecurityGroup(az) setMockSecurityGroup(az, ctrl, sg) - sg, err := az.reconcileSecurityGroup(testClusterName, &svc1, pointer.String(getServiceLoadBalancerIP(&svc1)), nil, true) + sg, err := az.reconcileSecurityGroup(testClusterName, &svc1, &[]string{getServiceLoadBalancerIP(&svc1, false)}, nil, true) if err != nil { t.Errorf("Unexpected error adding svc1: %q", err) } setMockSecurityGroup(az, ctrl, sg) - sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, pointer.String(getServiceLoadBalancerIP(&svc2)), nil, true) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, &[]string{getServiceLoadBalancerIP(&svc2, false)}, nil, true) if err != nil { t.Errorf("Unexpected error adding svc2: %q", err) } setMockSecurityGroup(az, ctrl, sg) - sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, pointer.String(getServiceLoadBalancerIP(&svc3)), nil, true) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, &[]string{getServiceLoadBalancerIP(&svc3, false)}, nil, true) if err != nil { t.Errorf("Unexpected error adding svc3: %q", err) } - validateSecurityGroup(t, sg, svc1, svc2, svc3) + validateSecurityGroup(t, az, sg, svc1, svc2, svc3) _, securityRule13, rule13Found := findSecurityRuleByName(*sg.SecurityRules, testRuleName23) if !rule13Found { @@ -2876,26 +2976,26 @@ func TestIfServiceSpecifiesSharedRuleAndServiceIsDeletedThenTheServicesPortAndAd sg := getTestSecurityGroup(az) setMockSecurityGroup(az, ctrl, sg) - sg, err := az.reconcileSecurityGroup(testClusterName, &svc1, pointer.String(getServiceLoadBalancerIP(&svc1)), nil, true) + sg, err := az.reconcileSecurityGroup(testClusterName, &svc1, &[]string{getServiceLoadBalancerIP(&svc1, false)}, nil, true) if err != nil { t.Errorf("Unexpected error adding svc1: %q", err) } setMockSecurityGroup(az, ctrl, sg) - sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, pointer.String(getServiceLoadBalancerIP(&svc2)), nil, true) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, &[]string{getServiceLoadBalancerIP(&svc2, false)}, nil, true) if err != nil { t.Errorf("Unexpected error adding svc2: %q", err) } - validateSecurityGroup(t, sg, svc1, svc2) + validateSecurityGroup(t, az, sg, svc1, svc2) setMockSecurityGroup(az, ctrl, sg) - sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, pointer.String(getServiceLoadBalancerIP(&svc1)), nil, false) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, &[]string{getServiceLoadBalancerIP(&svc1, false)}, nil, false) if err != nil { t.Errorf("Unexpected error removing svc1: %q", err) } - validateSecurityGroup(t, sg, svc2) + validateSecurityGroup(t, az, sg, svc2) _, securityRule, ruleFound := findSecurityRuleByName(*sg.SecurityRules, expectedRuleName) if !ruleFound { @@ -2939,32 +3039,32 @@ func TestIfSomeServicesShareARuleAndOneIsDeletedItIsRemovedFromTheRightRule(t *t sg := getTestSecurityGroup(az) setMockSecurityGroup(az, ctrl, sg) - sg, err := az.reconcileSecurityGroup(testClusterName, &svc1, pointer.String(getServiceLoadBalancerIP(&svc1)), nil, true) + sg, err := az.reconcileSecurityGroup(testClusterName, &svc1, &[]string{getServiceLoadBalancerIP(&svc1, false)}, nil, true) if err != nil { t.Errorf("Unexpected error adding svc1: %q", err) } setMockSecurityGroup(az, ctrl, sg) - sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, pointer.String(getServiceLoadBalancerIP(&svc2)), nil, true) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, &[]string{getServiceLoadBalancerIP(&svc2, false)}, nil, true) if err != nil { t.Errorf("Unexpected error adding svc2: %q", err) } setMockSecurityGroup(az, ctrl, sg) - sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, pointer.String(getServiceLoadBalancerIP(&svc3)), nil, true) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, &[]string{getServiceLoadBalancerIP(&svc3, false)}, nil, true) if err != nil { t.Errorf("Unexpected error adding svc3: %q", err) } - validateSecurityGroup(t, sg, svc1, svc2, svc3) + validateSecurityGroup(t, az, sg, svc1, svc2, svc3) setMockSecurityGroup(az, ctrl, sg) - sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, pointer.String(getServiceLoadBalancerIP(&svc1)), nil, false) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, &[]string{getServiceLoadBalancerIP(&svc1, false)}, nil, false) if err != nil { t.Errorf("Unexpected error removing svc1: %q", err) } - validateSecurityGroup(t, sg, svc2, svc3) + validateSecurityGroup(t, az, sg, svc2, svc3) _, securityRule13, rule13Found := findSecurityRuleByName(*sg.SecurityRules, testRuleName23) if !rule13Found { @@ -3050,38 +3150,38 @@ func TestIfServiceSpecifiesSharedRuleAndLastServiceIsDeletedThenRuleIsDeleted(t sg := getTestSecurityGroup(az) setMockSecurityGroup(az, ctrl, sg) - sg, err := az.reconcileSecurityGroup(testClusterName, &svc1, pointer.String(getServiceLoadBalancerIP(&svc1)), nil, true) + sg, err := az.reconcileSecurityGroup(testClusterName, &svc1, &[]string{getServiceLoadBalancerIP(&svc1, false)}, nil, true) if err != nil { t.Errorf("Unexpected error adding svc1: %q", err) } setMockSecurityGroup(az, ctrl, sg) - sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, pointer.String(getServiceLoadBalancerIP(&svc2)), nil, true) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, &[]string{getServiceLoadBalancerIP(&svc2, false)}, nil, true) if err != nil { t.Errorf("Unexpected error adding svc2: %q", err) } setMockSecurityGroup(az, ctrl, sg) - sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, pointer.String(getServiceLoadBalancerIP(&svc3)), nil, true) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, &[]string{getServiceLoadBalancerIP(&svc3, false)}, nil, true) if err != nil { t.Errorf("Unexpected error adding svc3: %q", err) } - validateSecurityGroup(t, sg, svc1, svc2, svc3) + validateSecurityGroup(t, az, sg, svc1, svc2, svc3) setMockSecurityGroup(az, ctrl, sg) - sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, pointer.String(getServiceLoadBalancerIP(&svc1)), nil, false) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, &[]string{getServiceLoadBalancerIP(&svc1, false)}, nil, false) if err != nil { t.Errorf("Unexpected error removing svc1: %q", err) } setMockSecurityGroup(az, ctrl, sg) - sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, pointer.String(getServiceLoadBalancerIP(&svc3)), nil, false) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, &[]string{getServiceLoadBalancerIP(&svc3, false)}, nil, false) if err != nil { t.Errorf("Unexpected error removing svc3: %q", err) } - validateSecurityGroup(t, sg, svc2) + validateSecurityGroup(t, az, sg, svc2) _, _, rule13Found := findSecurityRuleByName(*sg.SecurityRules, testRuleName23) if rule13Found { @@ -3143,21 +3243,21 @@ func TestCanCombineSharedAndPrivateRulesInSameGroup(t *testing.T) { testServices := []v1.Service{svc1, svc2, svc3, svc4, svc5} testRuleName23 := testRuleName2 - expectedRuleName4 := az.getSecurityRuleName(&svc4, v1.ServicePort{Port: 4444, Protocol: v1.ProtocolTCP}, "Internet") - expectedRuleName5 := az.getSecurityRuleName(&svc5, v1.ServicePort{Port: 8888, Protocol: v1.ProtocolTCP}, "Internet") + expectedRuleName4 := az.getSecurityRuleName(&svc4, v1.ServicePort{Port: 4444, Protocol: v1.ProtocolTCP}, "Internet", false) + expectedRuleName5 := az.getSecurityRuleName(&svc5, v1.ServicePort{Port: 8888, Protocol: v1.ProtocolTCP}, "Internet", false) sg := getTestSecurityGroup(az) for i, svc := range testServices { svc := svc setMockSecurityGroup(az, ctrl, sg) - sg, err = az.reconcileSecurityGroup(testClusterName, &testServices[i], pointer.String(getServiceLoadBalancerIP(&svc)), nil, true) + sg, err = az.reconcileSecurityGroup(testClusterName, &testServices[i], &[]string{getServiceLoadBalancerIP(&svc, false)}, nil, true) if err != nil { t.Errorf("Unexpected error adding svc%d: %q", i+1, err) } } - validateSecurityGroup(t, sg, svc1, svc2, svc3, svc4, svc5) + validateSecurityGroup(t, az, sg, svc1, svc2, svc3, svc4, svc5) expectedRuleCount := 4 if len(*sg.SecurityRules) != expectedRuleCount { @@ -3226,8 +3326,8 @@ func TestCanCombineSharedAndPrivateRulesInSameGroup(t *testing.T) { if securityRule4.DestinationAddressPrefix == nil { t.Errorf("Expected unshared rule %s to have a destination IP address", expectedRuleName4) } else { - if !strings.EqualFold(*securityRule4.DestinationAddressPrefix, getServiceLoadBalancerIP(&svc4)) { - t.Errorf("Expected unshared rule %s to have a destination %s but had %s", expectedRuleName4, getServiceLoadBalancerIP(&svc4), *securityRule4.DestinationAddressPrefix) + if !strings.EqualFold(*securityRule4.DestinationAddressPrefix, getServiceLoadBalancerIP(&svc4, false)) { + t.Errorf("Expected unshared rule %s to have a destination %s but had %s", expectedRuleName4, getServiceLoadBalancerIP(&svc4, false), *securityRule4.DestinationAddressPrefix) } } @@ -3238,19 +3338,19 @@ func TestCanCombineSharedAndPrivateRulesInSameGroup(t *testing.T) { if securityRule5.DestinationAddressPrefix == nil { t.Errorf("Expected unshared rule %s to have a destination IP address", expectedRuleName5) } else { - if !strings.EqualFold(*securityRule5.DestinationAddressPrefix, getServiceLoadBalancerIP(&svc5)) { - t.Errorf("Expected unshared rule %s to have a destination %s but had %s", expectedRuleName5, getServiceLoadBalancerIP(&svc5), *securityRule5.DestinationAddressPrefix) + if !strings.EqualFold(*securityRule5.DestinationAddressPrefix, getServiceLoadBalancerIP(&svc5, false)) { + t.Errorf("Expected unshared rule %s to have a destination %s but had %s", expectedRuleName5, getServiceLoadBalancerIP(&svc5, false), *securityRule5.DestinationAddressPrefix) } } setMockSecurityGroup(az, ctrl, sg) - sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, pointer.String(getServiceLoadBalancerIP(&svc1)), nil, false) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, &[]string{getServiceLoadBalancerIP(&svc1, false)}, nil, false) if err != nil { t.Errorf("Unexpected error removing svc1: %q", err) } setMockSecurityGroup(az, ctrl, sg) - sg, err = az.reconcileSecurityGroup(testClusterName, &svc5, pointer.String(getServiceLoadBalancerIP(&svc5)), nil, false) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc5, &[]string{getServiceLoadBalancerIP(&svc5, false)}, nil, false) if err != nil { t.Errorf("Unexpected error removing svc5: %q", err) } diff --git a/pkg/provider/azure_utils.go b/pkg/provider/azure_utils.go index aa7aa2c5df..39290cb6aa 100644 --- a/pkg/provider/azure_utils.go +++ b/pkg/provider/azure_utils.go @@ -227,11 +227,10 @@ func getServiceAdditionalPublicIPs(service *v1.Service) ([]string, error) { return result, nil } -func getNodePrivateIPAddress(service *v1.Service, node *v1.Node) string { - isIPV6SVC := utilnet.IsIPv6String(service.Spec.ClusterIP) +func getNodePrivateIPAddress(service *v1.Service, node *v1.Node, isIPv6 bool) string { for _, nodeAddress := range node.Status.Addresses { if strings.EqualFold(string(nodeAddress.Type), string(v1.NodeInternalIP)) && - utilnet.IsIPv6String(nodeAddress.Address) == isIPV6SVC { + utilnet.IsIPv6String(nodeAddress.Address) == isIPv6 { klog.V(6).Infof("getNodePrivateIPAddress: node %s, ip %s", node.Name, nodeAddress.Address) return nodeAddress.Address } diff --git a/pkg/provider/azure_vmss.go b/pkg/provider/azure_vmss.go index ec6a2633c4..60fa77e251 100644 --- a/pkg/provider/azure_vmss.go +++ b/pkg/provider/azure_vmss.go @@ -33,7 +33,6 @@ import ( utilerrors "k8s.io/apimachinery/pkg/util/errors" cloudprovider "k8s.io/cloud-provider" "k8s.io/klog/v2" - utilnet "k8s.io/utils/net" "k8s.io/utils/pointer" azcache "sigs.k8s.io/cloud-provider-azure/pkg/cache" @@ -1143,7 +1142,7 @@ func (ss *ScaleSet) EnsureHostInPool(service *v1.Service, nodeName types.NodeNam } var primaryIPConfiguration *compute.VirtualMachineScaleSetIPConfiguration - ipv6 := utilnet.IsIPv6String(service.Spec.ClusterIP) + ipv6 := ifBackendPoolIPv6(backendPoolID) // Find primary network interface configuration. if !ss.Cloud.ipv6DualStackEnabled && !ipv6 { // Find primary IP configuration. @@ -1317,7 +1316,7 @@ func (ss *ScaleSet) ensureVMSSInPool(service *v1.Service, nodes []*v1.Node, back return err } var primaryIPConfig *compute.VirtualMachineScaleSetIPConfiguration - ipv6 := utilnet.IsIPv6String(service.Spec.ClusterIP) + ipv6 := ifBackendPoolIPv6(backendPoolID) // Find primary network interface configuration. if !ss.Cloud.ipv6DualStackEnabled && !ipv6 { // Find primary IP configuration. diff --git a/pkg/provider/azure_vmssflex.go b/pkg/provider/azure_vmssflex.go index 23d86c40e8..3e930c2d5a 100644 --- a/pkg/provider/azure_vmssflex.go +++ b/pkg/provider/azure_vmssflex.go @@ -31,7 +31,6 @@ import ( utilerrors "k8s.io/apimachinery/pkg/util/errors" cloudprovider "k8s.io/cloud-provider" "k8s.io/klog/v2" - utilnet "k8s.io/utils/net" "k8s.io/utils/pointer" azcache "sigs.k8s.io/cloud-provider-azure/pkg/cache" @@ -511,7 +510,7 @@ func (fs *FlexScaleSet) EnsureHostInPool(service *v1.Service, nodeName types.Nod } var primaryIPConfig *network.InterfaceIPConfiguration - ipv6 := utilnet.IsIPv6String(service.Spec.ClusterIP) + ipv6 := ifBackendPoolIPv6(backendPoolID) if !fs.Cloud.ipv6DualStackEnabled && !ipv6 { primaryIPConfig, err = getPrimaryIPConfig(nic) if err != nil { @@ -662,7 +661,7 @@ func (fs *FlexScaleSet) ensureVMSSFlexInPool(service *v1.Service, nodes []*v1.No return err } var primaryIPConfig *compute.VirtualMachineScaleSetIPConfiguration - ipv6 := utilnet.IsIPv6String(service.Spec.ClusterIP) + ipv6 := ifBackendPoolIPv6(backendPoolID) // Find primary network interface configuration. if !fs.Cloud.ipv6DualStackEnabled && !ipv6 { // Find primary IP configuration. diff --git a/pkg/provider/azure_wrap.go b/pkg/provider/azure_wrap.go index 036053e47b..4d55bce3bd 100644 --- a/pkg/provider/azure_wrap.go +++ b/pkg/provider/azure_wrap.go @@ -85,7 +85,7 @@ func (az *Cloud) getVirtualMachine(nodeName types.NodeName, crt azcache.AzureCac func (az *Cloud) getRouteTable(crt azcache.AzureCacheReadType) (routeTable network.RouteTable, exists bool, err error) { if len(az.RouteTableName) == 0 { - return routeTable, false, fmt.Errorf("Route table name is not configured") + return routeTable, false, fmt.Errorf("route table name is not configured") } cachedRt, err := az.rtCache.GetWithDeepCopy(az.RouteTableName, crt) diff --git a/tests/e2e/network/ensureloadbalancer.go b/tests/e2e/network/ensureloadbalancer.go index f63cb7bbb8..b313db38ac 100644 --- a/tests/e2e/network/ensureloadbalancer.go +++ b/tests/e2e/network/ensureloadbalancer.go @@ -134,8 +134,10 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { service := utils.CreateLoadBalancerServiceManifest(testServiceName, nil, labels, ns.Name, mixedProtocolPorts) _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, "") + ips, err := utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns.Name, testServiceName, []string{}) Expect(err).NotTo(HaveOccurred()) + Expect(len(ips)).NotTo(BeZero()) + ip := ips[0] // ip is used to get LB only By("checking load balancing rules") foundTCP, foundUDP := false, false @@ -158,91 +160,124 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { It("should support BYO public IP", func() { By("creating a public IP with tags") - ipName := basename + "-public-IP" + string(uuid.NewUUID())[0:4] - pip := defaultPublicIPAddress(ipName, tc.IPFamily == utils.IPv6) + ipNameBase := basename + "-public-IP" + string(uuid.NewUUID())[0:4] + v4Enabled, v6Enabled := utils.IfIPFamiliesEnabled(tc.IPFamily) expectedTags := map[string]*string{ "foo": pointer.String("bar"), } - pip.Tags = expectedTags - pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), pip) - Expect(err).NotTo(HaveOccurred()) - Expect(pip.Tags).To(Equal(expectedTags)) - targetIP := pointer.StringDeref(pip.IPAddress, "") - utils.Logf("created pip with address %s", targetIP) + pips := []aznetwork.PublicIPAddress{} + targetIPs := []string{} + ipNames := []string{} + deleteFuncs := []func(){} + + createBYOPIP := func(isIPv6 bool) func() { + ipName := utils.GetNameWithSuffix(ipNameBase, utils.Suffixes[isIPv6]) + ipNames = append(ipNames, ipName) + pip := defaultPublicIPAddress(ipName, isIPv6) + pip.Tags = expectedTags + pips = append(pips, pip) + pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), pip) + Expect(err).NotTo(HaveOccurred()) + deleteFunc := func() { + err := utils.DeletePIPWithRetry(tc, ipName, tc.GetResourceGroup()) + Expect(err).To(BeNil()) + } + Expect(pip.Tags).To(Equal(expectedTags)) + targetIP := pointer.StringDeref(pip.IPAddress, "") + targetIPs = append(targetIPs, targetIP) + utils.Logf("created pip with address %s", targetIP) + return deleteFunc + } + if v4Enabled { + deleteFuncs = append(deleteFuncs, createBYOPIP(false)) + } + if v6Enabled { + deleteFuncs = append(deleteFuncs, createBYOPIP(true)) + } + defer func() { + for _, deleteFunc := range deleteFuncs { + deleteFunc() + } + }() By("creating a service referencing the public IP") service := utils.CreateLoadBalancerServiceManifest(testServiceName, nil, labels, ns.Name, ports) - service = updateServiceLBIP(service, false, targetIP) - _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) + service = updateServiceLBIPs(service, false, targetIPs) + _, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, "") + _, err = utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns.Name, testServiceName, targetIPs) Expect(err).NotTo(HaveOccurred()) - Expect(ip).To(Equal(targetIP)) By("deleting the service") err = utils.DeleteService(cs, ns.Name, testServiceName) Expect(err).NotTo(HaveOccurred()) - By("test if the pip still exists") - pip, err = utils.WaitGetPIP(tc, ipName) - Expect(err).NotTo(HaveOccurred()) - - By("test if the tags are changed") - Expect(pip.Tags).To(Equal(expectedTags)) + for _, ipName := range ipNames { + By("test if the pip still exists") + pip, err := utils.WaitGetPIP(tc, ipName) + Expect(err).NotTo(HaveOccurred()) - By("cleaning up") - err = utils.DeletePIPWithRetry(tc, ipName, "") - Expect(err).NotTo(HaveOccurred()) + By("test if the tags are changed") + Expect(pip.Tags).To(Equal(expectedTags)) + } }) // Public w/o IP -> Public w/ IP It("should support assigning to specific IP when updating public service", func() { - ipName := basename + "-public-none-IP" + string(uuid.NewUUID())[0:4] + ipNameBase := basename + "-public-none-IP" + string(uuid.NewUUID())[0:4] service := utils.CreateLoadBalancerServiceManifest(testServiceName, serviceAnnotationLoadBalancerInternalFalse, labels, ns.Name, ports) _, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) utils.Logf("Successfully created LoadBalancer service " + testServiceName + " in namespace " + ns.Name) - pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), defaultPublicIPAddress(ipName, tc.IPFamily == utils.IPv6)) - Expect(err).NotTo(HaveOccurred()) - targetIP := pointer.StringDeref(pip.IPAddress, "") - utils.Logf("PIP to %s", targetIP) + v4Enabled, v6Enabled := utils.IfIPFamiliesEnabled(tc.IPFamily) + targetIPs := []string{} + deleteFuncs := []func(){} + if v4Enabled { + targetIP, deleteFunc := createPIP(tc, ipNameBase, false) + targetIPs = append(targetIPs, targetIP) + deleteFuncs = append(deleteFuncs, deleteFunc) + } + if v6Enabled { + targetIP, deleteFunc := createPIP(tc, ipNameBase, true) + targetIPs = append(targetIPs, targetIP) + deleteFuncs = append(deleteFuncs, deleteFunc) + } defer func() { - By("Cleaning up") + By("Cleaning up Service") err = utils.DeleteService(cs, ns.Name, testServiceName) Expect(err).NotTo(HaveOccurred()) - err = utils.DeletePIPWithRetry(tc, ipName, "") - Expect(err).NotTo(HaveOccurred()) + for _, deleteFunc := range deleteFuncs { + deleteFunc() + } }() By("Waiting for exposure of the original service without assigned lb IP") - ip1, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, "") + ips1, err := utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns.Name, testServiceName, []string{}) Expect(err).NotTo(HaveOccurred()) - - Expect(ip1).NotTo(Equal(targetIP)) + Expect(utils.CompareStrings(ips1, targetIPs)).To(BeFalse()) By("Updating service to bound to specific public IP") - utils.Logf("will update IP to %s", targetIP) + utils.Logf("will update IPs to %s", targetIPs) service, err = cs.CoreV1().Services(ns.Name).Get(context.TODO(), testServiceName, metav1.GetOptions{}) - service = updateServiceLBIP(service, false, targetIP) + service = updateServiceLBIPs(service, false, targetIPs) _, err = cs.CoreV1().Services(ns.Name).Update(context.TODO(), service, metav1.UpdateOptions{}) Expect(err).NotTo(HaveOccurred()) - ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, targetIP) + _, err = utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns.Name, testServiceName, targetIPs) Expect(err).NotTo(HaveOccurred()) - Expect(ip).To(Equal(targetIP)) }) // Internal w/ IP -> Internal w/ IP It("should support updating internal IP when updating internal service", func() { - ip1, err := utils.SelectAvailablePrivateIP(tc) + ips1, err := utils.SelectAvailablePrivateIPs(tc, tc.IPFamily) Expect(err).NotTo(HaveOccurred()) service := utils.CreateLoadBalancerServiceManifest(testServiceName, serviceAnnotationLoadBalancerInternalTrue, labels, ns.Name, ports) - service = updateServiceLBIP(service, true, ip1) + service = updateServiceLBIPs(service, true, ips1) _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) utils.Logf("Successfully created LoadBalancer service " + testServiceName + " in namespace " + ns.Name) @@ -253,10 +288,9 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { Expect(err).NotTo(HaveOccurred()) }() - By(fmt.Sprintf("Waiting for exposure of internal service with specific IP %q", ip1)) - ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, ip1) + By(fmt.Sprintf("Waiting for exposure of internal service with specific IPs %v", ips1)) + _, err = utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns.Name, testServiceName, ips1) Expect(err).NotTo(HaveOccurred()) - Expect(ip).To(Equal(ip1)) list, errList := cs.CoreV1().Events(ns.Name).List(context.TODO(), metav1.ListOptions{}) Expect(errList).NotTo(HaveOccurred()) utils.Logf("Events list:") @@ -264,46 +298,56 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { utils.Logf("%d. %v", i, event) } - ip2, err := utils.SelectAvailablePrivateIP(tc) + ips2, err := utils.SelectAvailablePrivateIPs(tc, tc.IPFamily) Expect(err).NotTo(HaveOccurred()) By("Updating internal service private IP") - utils.Logf("will update IP to %s", ip2) + utils.Logf("will update IPs to %v", ips2) service, err = cs.CoreV1().Services(ns.Name).Get(context.TODO(), testServiceName, metav1.GetOptions{}) - service = updateServiceLBIP(service, true, ip2) + service = updateServiceLBIPs(service, true, ips2) _, err = cs.CoreV1().Services(ns.Name).Update(context.TODO(), service, metav1.UpdateOptions{}) Expect(err).NotTo(HaveOccurred()) - ip, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, ip2) + _, err = utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns.Name, testServiceName, ips2) Expect(err).NotTo(HaveOccurred()) - Expect(ip).To(Equal(ip2)) }) // internal w/o IP -> public w/ IP It("should support updating an internal service to a public service with assigned IP", func() { - ipName := basename + "-internal-none-public-IP" + string(uuid.NewUUID())[0:4] + ipNameBase := basename + "-internal-none-public-IP" + string(uuid.NewUUID())[0:4] service := utils.CreateLoadBalancerServiceManifest(testServiceName, serviceAnnotationLoadBalancerInternalTrue, labels, ns.Name, ports) _, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) utils.Logf("Successfully created LoadBalancer service " + testServiceName + " in namespace " + ns.Name) - pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), defaultPublicIPAddress(ipName, tc.IPFamily == utils.IPv6)) - Expect(err).NotTo(HaveOccurred()) - targetIP := pointer.StringDeref(pip.IPAddress, "") + targetIPs := []string{} + deleteFuncs := []func(){} + v4Enabled, v6Enabled := utils.IfIPFamiliesEnabled(tc.IPFamily) + if v4Enabled { + targetIP, deleteFunc := createPIP(tc, ipNameBase, false) + targetIPs = append(targetIPs, targetIP) + deleteFuncs = append(deleteFuncs, deleteFunc) + } + if v6Enabled { + targetIP, deleteFunc := createPIP(tc, ipNameBase, true) + targetIPs = append(targetIPs, targetIP) + deleteFuncs = append(deleteFuncs, deleteFunc) + } defer func() { By("Cleaning up") err = utils.DeleteService(cs, ns.Name, testServiceName) Expect(err).NotTo(HaveOccurred()) - err = utils.DeletePIPWithRetry(tc, ipName, "") - Expect(err).NotTo(HaveOccurred()) + for _, deleteFunc := range deleteFuncs { + deleteFunc() + } }() By("Waiting for exposure of the original service without assigned lb private IP") - ip1, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, "") + ips1, err := utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns.Name, testServiceName, []string{}) Expect(err).NotTo(HaveOccurred()) - Expect(ip1).NotTo(Equal(targetIP)) + Expect(utils.CompareStrings(ips1, targetIPs)).To(BeFalse()) list, errList := cs.CoreV1().Events(ns.Name).List(context.TODO(), metav1.ListOptions{}) Expect(errList).NotTo(HaveOccurred()) utils.Logf("Events list:") @@ -312,28 +356,38 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { } By("Updating service to bound to specific public IP") - utils.Logf("will update IP to %s, %v", targetIP, len(targetIP)) + utils.Logf("will update IP to %v", targetIPs) service, err = cs.CoreV1().Services(ns.Name).Get(context.TODO(), testServiceName, metav1.GetOptions{}) - service = updateServiceLBIP(service, false, targetIP) + service = updateServiceLBIPs(service, false, targetIPs) _, err = cs.CoreV1().Services(ns.Name).Update(context.TODO(), service, metav1.UpdateOptions{}) Expect(err).NotTo(HaveOccurred()) - ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, targetIP) + _, err = utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns.Name, testServiceName, targetIPs) Expect(err).NotTo(HaveOccurred()) - Expect(ip).To(Equal(targetIP)) }) It("should have no operation since no change in service when update", Label(utils.TestSuiteLabelSlow), func() { - suffix := string(uuid.NewUUID())[0:4] - ipName := basename + "-public-remain" + suffix - pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), defaultPublicIPAddress(ipName, tc.IPFamily == utils.IPv6)) - Expect(err).NotTo(HaveOccurred()) - targetIP := pointer.StringDeref(pip.IPAddress, "") + suffixBase := string(uuid.NewUUID())[0:4] + ipNameBase := basename + "-public-remain" + suffixBase + + targetIPs := []string{} + deleteFuncs := []func(){} + v4Enabled, v6Enabled := utils.IfIPFamiliesEnabled(tc.IPFamily) + if v4Enabled { + targetIP, deleteFunc := createPIP(tc, ipNameBase, false) + targetIPs = append(targetIPs, targetIP) + deleteFuncs = append(deleteFuncs, deleteFunc) + } + if v6Enabled { + targetIP, deleteFunc := createPIP(tc, ipNameBase, true) + targetIPs = append(targetIPs, targetIP) + deleteFuncs = append(deleteFuncs, deleteFunc) + } service := utils.CreateLoadBalancerServiceManifest(testServiceName, serviceAnnotationLoadBalancerInternalFalse, labels, ns.Name, ports) - service = updateServiceLBIP(service, false, targetIP) - _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) + service = updateServiceLBIPs(service, false, targetIPs) + _, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) utils.Logf("Successfully created LoadBalancer service %s in namespace %s", testServiceName, ns.Name) @@ -341,24 +395,25 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { By("Cleaning up") err = utils.DeleteService(cs, ns.Name, testServiceName) Expect(err).NotTo(HaveOccurred()) - err = utils.DeletePIPWithRetry(tc, ipName, "") - Expect(err).NotTo(HaveOccurred()) + for _, deleteFunc := range deleteFuncs { + deleteFunc() + } }() By("Waiting for exposure of the original service with assigned lb private IP") - targetIP, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, targetIP) + _, err = utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns.Name, testServiceName, targetIPs) Expect(err).NotTo(HaveOccurred()) By("Update without changing the service and wait for a while") - utils.Logf("External IP is now %s", targetIP) + utils.Logf("External IPs are now %v", targetIPs) service, err = cs.CoreV1().Services(ns.Name).Get(context.TODO(), testServiceName, metav1.GetOptions{}) - service.Annotations[consts.ServiceAnnotationDNSLabelName] = "testlabel" + suffix + service.Annotations[consts.ServiceAnnotationDNSLabelName] = "testlabel" + suffixBase utils.Logf(service.Annotations[consts.ServiceAnnotationDNSLabelName]) _, err = cs.CoreV1().Services(ns.Name).Update(context.TODO(), service, metav1.UpdateOptions{}) Expect(err).NotTo(HaveOccurred()) // Wait for 5 minutes, there should return timeout err, since external ip should not change - utils.Logf("External IP should be %s", targetIP) + utils.Logf("External IPs should be %v", targetIPs) err = wait.PollImmediate(10*time.Second, 5*time.Minute, func() (bool, error) { service, err = cs.CoreV1().Services(ns.Name).Get(context.TODO(), testServiceName, metav1.GetOptions{}) if err != nil { @@ -374,24 +429,33 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { utils.Logf("Fail to get ingress, retry it in %s seconds", 10) return false, nil } - if targetIP == ingressList[0].IP { + ingressIPs := []string{} + for _, ingress := range ingressList { + ingressIPs = append(ingressIPs, ingress.IP) + } + if utils.CompareStrings(targetIPs, ingressIPs) { return false, nil } - utils.Logf("External IP changed, unexpected") + utils.Logf("External IP changed unexpectedly to: %v", ingressIPs) return true, nil }) Expect(err).To(Equal(wait.ErrWaitTimeout)) }) - It("should support multiple external services sharing one preset public IP address", func() { - ipName := fmt.Sprintf("%s-public-remain-%s", basename, string(uuid.NewUUID())[0:4]) - pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), defaultPublicIPAddress(ipName, tc.IPFamily == utils.IPv6)) - defer func() { - err = utils.DeletePIPWithRetry(tc, ipName, "") - Expect(err).NotTo(HaveOccurred()) - }() - Expect(err).NotTo(HaveOccurred()) - targetIP := pointer.StringDeref(pip.IPAddress, "") + It("should support multiple external services sharing preset public IP addresses", func() { + ipNameBase := fmt.Sprintf("%s-public-remain-%s", basename, string(uuid.NewUUID())[0:4]) + targetIPs := []string{} + v4Enabled, v6Enabled := utils.IfIPFamiliesEnabled(tc.IPFamily) + if v4Enabled { + targetIP, deleteFunc := createPIP(tc, ipNameBase, false) + targetIPs = append(targetIPs, targetIP) + defer deleteFunc() + } + if v6Enabled { + targetIP, deleteFunc := createPIP(tc, ipNameBase, true) + targetIPs = append(targetIPs, targetIP) + defer deleteFunc() + } serviceNames := []string{} for i := 0; i < 2; i++ { @@ -405,7 +469,7 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { "app": deploymentName, } deployment := createDeploymentManifest(deploymentName, serviceLabels, &tcpPort, nil) - _, err = cs.AppsV1().Deployments(ns.Name).Create(context.TODO(), deployment, metav1.CreateOptions{}) + _, err := cs.AppsV1().Deployments(ns.Name).Create(context.TODO(), deployment, metav1.CreateOptions{}) defer func() { err := cs.AppsV1().Deployments(ns.Name).Delete(context.TODO(), deploymentName, metav1.DeleteOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -421,8 +485,8 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { TargetPort: intstr.FromInt(int(tcpPort)), }} service := utils.CreateLoadBalancerServiceManifest(serviceName, nil, serviceLabels, ns.Name, servicePort) - service = updateServiceLBIP(service, false, targetIP) - _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) + service = updateServiceLBIPs(service, false, targetIPs) + _, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) defer func() { err = utils.DeleteService(cs, ns.Name, serviceName) Expect(err).NotTo(HaveOccurred()) @@ -431,15 +495,15 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { } for _, serviceName := range serviceNames { - ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, targetIP) + _, err := utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns.Name, serviceName, targetIPs) Expect(err).NotTo(HaveOccurred()) - utils.Logf("Successfully created LoadBalancer Service %q in namespace %s with IP %s", serviceName, ns.Name, ip) + utils.Logf("Successfully created LoadBalancer Service %q in namespace %s with IPs: %v", serviceName, ns.Name, targetIPs) } }) - It("should support multiple external services sharing one newly created public IP address", func() { + It("should support multiple external services sharing one newly created public IP addresses", func() { serviceCount := 2 - sharedIP := "" + sharedIPs := []string{} serviceNames := []string{} var serviceLabels map[string]string for i := 0; i < serviceCount; i++ { @@ -468,23 +532,23 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { TargetPort: intstr.FromInt(int(tcpPort)), }} service := utils.CreateLoadBalancerServiceManifest(serviceName, nil, serviceLabels, ns.Name, servicePort) - if sharedIP != "" { - service = updateServiceLBIP(service, false, sharedIP) + if len(sharedIPs) != 0 { + service = updateServiceLBIPs(service, false, sharedIPs) } _, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) // No need to defer delete Services here - ip, err := utils.WaitServiceExposureAndGetIP(cs, ns.Name, serviceName) + ips, err := utils.WaitServiceExposureAndGetIPs(cs, ns.Name, serviceName) Expect(err).NotTo(HaveOccurred()) - if sharedIP == "" { - sharedIP = ip + if len(sharedIPs) == 0 { + sharedIPs = ips } } for _, serviceName := range serviceNames { - _, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, sharedIP) + _, err := utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns.Name, serviceName, sharedIPs) Expect(err).NotTo(HaveOccurred()) - utils.Logf("Successfully created LoadBalancer Service %q in namespace %q with IP %s", serviceName, ns.Name, sharedIP) + utils.Logf("Successfully created LoadBalancer Service %q in namespace %q with IPs %v", serviceName, ns.Name, sharedIPs) } By("Deleting one Service and check if the other service works well") @@ -496,7 +560,7 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { for i := 1; i < serviceCount; i++ { serviceName := serviceNames[i] - _, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, sharedIP) + _, err = utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns.Name, serviceName, sharedIPs) Expect(err).NotTo(HaveOccurred()) } @@ -508,6 +572,11 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { } By("Checking if the public IP has been deleted") + sharedIPsMap := map[string]bool{} + for _, sharedIP := range sharedIPs { + Expect(len(sharedIP)).NotTo(BeZero()) + sharedIPsMap[sharedIP] = true + } err = wait.PollImmediate(5*time.Second, 5*time.Minute, func() (bool, error) { pips, err := tc.ListPublicIPs(tc.GetResourceGroup()) if err != nil { @@ -515,8 +584,8 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { } for _, pip := range pips { - if pip.IPAddress != nil && strings.EqualFold(*pip.IPAddress, sharedIP) { - utils.Logf("the public IP with address %s still exists", sharedIP) + if _, ok := sharedIPsMap[pointer.StringDeref(pip.IPAddress, "")]; ok { + utils.Logf("the public IP with address %s still exists", pointer.StringDeref(pip.IPAddress, "")) return false, nil } } @@ -526,8 +595,8 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { Expect(err).NotTo(HaveOccurred()) }) - It("should support multiple internal services sharing one IP address", func() { - sharedIP := "" + It("should support multiple internal services sharing IP addresses", func() { + sharedIPs := []string{} serviceNames := []string{} for i := 0; i < 2; i++ { serviceLabels := labels @@ -555,8 +624,8 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { TargetPort: intstr.FromInt(int(tcpPort)), }} service := utils.CreateLoadBalancerServiceManifest(serviceName, serviceAnnotationLoadBalancerInternalTrue, serviceLabels, ns.Name, servicePort) - if sharedIP != "" { - service = updateServiceLBIP(service, true, sharedIP) + if len(sharedIPs) != 0 { + service = updateServiceLBIPs(service, true, sharedIPs) } _, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) defer func() { @@ -564,17 +633,17 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { Expect(err).NotTo(HaveOccurred()) }() Expect(err).NotTo(HaveOccurred()) - ip, err := utils.WaitServiceExposureAndGetIP(cs, ns.Name, serviceName) + ips, err := utils.WaitServiceExposureAndGetIPs(cs, ns.Name, serviceName) Expect(err).NotTo(HaveOccurred()) - if sharedIP == "" { - sharedIP = ip + if len(sharedIPs) == 0 { + sharedIPs = ips } } for _, serviceName := range serviceNames { - _, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, sharedIP) + _, err := utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns.Name, serviceName, sharedIPs) Expect(err).NotTo(HaveOccurred()) - utils.Logf("Successfully created LoadBalancer Service %q in namespace %q with IP %s", serviceName, ns.Name, sharedIP) + utils.Logf("Successfully created LoadBalancer Service %q in namespace %q with IPs %v", serviceName, ns.Name, sharedIPs) } }) @@ -592,8 +661,10 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { service := utils.CreateLoadBalancerServiceManifest(testServiceName, nil, labels, ns.Name, ports) _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - publicIP, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, "") + ips, err := utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns.Name, testServiceName, []string{}) Expect(err).NotTo(HaveOccurred()) + Expect(len(ips)).NotTo(BeZero()) + publicIP := ips[0] // publicIP is used to get LB only By("Checking the initial node number in the LB backend pool") lb := getAzureLoadBalancerFromPIP(tc, publicIP, tc.GetResourceGroup(), "") @@ -618,48 +689,62 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { }) It("should support disabling floating IP in load balancer rule with kubernetes service annotations", func() { - By("creating a public IP") - ipName := basename + "-public-IP" + string(uuid.NewUUID())[0:4] - pip := defaultPublicIPAddress(ipName, tc.IPFamily == utils.IPv6) - pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), pip) - Expect(err).NotTo(HaveOccurred()) - targetIP := pointer.StringDeref(pip.IPAddress, "") - utils.Logf("created pip with address %s", targetIP) + ipNameBase := basename + "-public-IP" + string(uuid.NewUUID())[0:4] + v4Enabled, v6Enabled := utils.IfIPFamiliesEnabled(tc.IPFamily) + targetIPs := []string{} + deleteFuncs := []func(){} + if v4Enabled { + targetIP, deleteFunc := createPIP(tc, ipNameBase, false) + targetIPs = append(targetIPs, targetIP) + deleteFuncs = append(deleteFuncs, deleteFunc) + } + if v6Enabled { + targetIP, deleteFunc := createPIP(tc, ipNameBase, true) + targetIPs = append(targetIPs, targetIP) + deleteFuncs = append(deleteFuncs, deleteFunc) + } + defer func() { + By("Clean up PIPs") + for _, deleteFunc := range deleteFuncs { + deleteFunc() + } + }() By("creating a service referencing the public IP") service := utils.CreateLoadBalancerServiceManifest(testServiceName, serviceAnnotationDisableLoadBalancerFloatingIP, labels, ns.Name, ports) - service = updateServiceLBIP(service, false, targetIP) - _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred()) - ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, "") - Expect(err).NotTo(HaveOccurred()) - Expect(ip).To(Equal(targetIP)) - + service = updateServiceLBIPs(service, false, targetIPs) + _, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) defer func() { - By("cleaning up") - err = utils.DeleteService(cs, ns.Name, testServiceName) - Expect(err).NotTo(HaveOccurred()) - err = utils.DeletePIPWithRetry(tc, ipName, "") + By("Clean up Service") + err := utils.DeleteService(cs, ns.Name, testServiceName) Expect(err).NotTo(HaveOccurred()) }() + Expect(err).NotTo(HaveOccurred()) + _, err = utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns.Name, testServiceName, targetIPs) + Expect(err).NotTo(HaveOccurred()) By("testing if floating IP disabled in load balancer rule") - pipFrontendConfigID := getPIPFrontendConfigurationID(tc, ip, tc.GetResourceGroup(), "") + Expect(len(targetIPs)).NotTo(Equal(0)) + pipFrontendConfigID := getPIPFrontendConfigurationID(tc, targetIPs[0], tc.GetResourceGroup(), "") pipFrontendConfigIDSplit := strings.Split(pipFrontendConfigID, "/") Expect(len(pipFrontendConfigIDSplit)).NotTo(Equal(0)) + configID := pipFrontendConfigIDSplit[len(pipFrontendConfigIDSplit)-1] + utils.Logf("PIP frontend config ID %q", configID) - lb := getAzureLoadBalancerFromPIP(tc, ip, tc.GetResourceGroup(), "") - lbRules := lb.LoadBalancingRules + lb := getAzureLoadBalancerFromPIP(tc, targetIPs[0], tc.GetResourceGroup(), "") found := false - for _, lbRule := range *lbRules { - utils.Logf("Checking LB rule %q, may not be the corresponding rule of the Service", *lbRule.Name) + for _, lbRule := range *lb.LoadBalancingRules { + utils.Logf("Checking LB rule %q", *lbRule.Name) lbRuleSplit := strings.Split(*lbRule.Name, "-") Expect(len(lbRuleSplit)).NotTo(Equal(0)) - if pipFrontendConfigIDSplit[len(pipFrontendConfigIDSplit)-1] == lbRuleSplit[0] { - Expect(pointer.BoolDeref(lbRule.EnableFloatingIP, false)).To(BeFalse()) - found = true - break + // id is like xxx-IPv4 or xxx-IPv6 and lbRuleSplit[0] is like xxx. + if !strings.Contains(configID, lbRuleSplit[0]) { + continue } + utils.Logf("%q is the corresponding rule of the Service", *lbRule.Name) + Expect(pointer.BoolDeref(lbRule.EnableFloatingIP, false)).To(BeFalse()) + found = true + break } Expect(found).To(Equal(true)) }) @@ -729,7 +814,9 @@ var _ = Describe("EnsureLoadBalancer should not update any resources when servic annotation[consts.ServiceAnnotationIPTagsForPublicIP] = "RoutingPreference=Internet" } - ip := createAndExposeDefaultServiceWithAnnotation(cs, testServiceName, ns.Name, labels, annotation, ports) + ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, testServiceName, ns.Name, labels, annotation, ports) + Expect(len(ips)).NotTo(BeZero()) + ip := ips[0] // ip is used to get LB only service, err := cs.CoreV1().Services(ns.Name).Get(context.TODO(), testServiceName, metav1.GetOptions{}) defer func() { By("Cleaning up") @@ -743,34 +830,61 @@ var _ = Describe("EnsureLoadBalancer should not update any resources when servic }) It("should respect service with BYO public IP with various configurations", func() { - By("Creating a BYO public IP") - ipName := basename + "-public-IP" + string(uuid.NewUUID())[0:4] - pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), defaultPublicIPAddress(ipName, tc.IPFamily == utils.IPv6)) + By("Creating BYO public IPs") + ipNameBase := basename + "-public-IP" + string(uuid.NewUUID())[0:4] + v4Enabled, v6Enabled := utils.IfIPFamiliesEnabled(tc.IPFamily) + targetIPs := []string{} + deleteFuncs := []func(){} + if v4Enabled { + targetIP, deleteFunc := createPIP(tc, ipNameBase, false) + targetIPs = append(targetIPs, targetIP) + deleteFuncs = append(deleteFuncs, deleteFunc) + } + if v6Enabled { + targetIP, deleteFunc := createPIP(tc, ipNameBase, true) + targetIPs = append(targetIPs, targetIP) + deleteFuncs = append(deleteFuncs, deleteFunc) + } defer func() { - err = utils.DeletePIPWithRetry(tc, ipName, "") - Expect(err).NotTo(HaveOccurred()) + By("Clean up PIPs") + for _, deleteFunc := range deleteFuncs { + deleteFunc() + } }() - Expect(err).NotTo(HaveOccurred()) - targetIP := pointer.StringDeref(pip.IPAddress, "") customHealthProbeConfigPrefix := "service.beta.kubernetes.io/port_" + strconv.Itoa(int(ports[0].Port)) + "_health-probe_" By("Creating a service and expose it") annotation := map[string]string{ - consts.ServiceAnnotationPIPName: ipName, consts.ServiceAnnotationDenyAllExceptLoadBalancerSourceRanges: "true", customHealthProbeConfigPrefix + "interval": "10", customHealthProbeConfigPrefix + "num-of-probe": "6", customHealthProbeConfigPrefix + "request-path": "/healthtz", } + // TODO: After dual-stack implementation finished, update here. + if utils.DualstackSupported { + if v4Enabled { + annotation[consts.ServiceAnnotationPIPNameDualStack[false]] = utils.GetNameWithSuffix(ipNameBase, utils.Suffixes[false]) + } + if v6Enabled { + annotation[consts.ServiceAnnotationPIPNameDualStack[true]] = utils.GetNameWithSuffix(ipNameBase, utils.Suffixes[true]) + } + } else { + annotation[consts.ServiceAnnotationPIPName] = ipNameBase + } service := utils.CreateLoadBalancerServiceManifest(testServiceName, annotation, labels, ns.Name, ports) - service.Spec.LoadBalancerSourceRanges = []string{"0.0.0.0/0"} + service.Spec.LoadBalancerSourceRanges = []string{} + if v4Enabled { + service.Spec.LoadBalancerSourceRanges = append(service.Spec.LoadBalancerSourceRanges, "0.0.0.0/0") + } + if v6Enabled { + service.Spec.LoadBalancerSourceRanges = append(service.Spec.LoadBalancerSourceRanges, "::/0") + } service.Spec.SessionAffinity = "ClientIP" - _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) + _, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, "") + _, err = utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns.Name, testServiceName, targetIPs) Expect(err).NotTo(HaveOccurred()) - Expect(ip).To(Equal(targetIP)) service, err = cs.CoreV1().Services(ns.Name).Get(context.TODO(), testServiceName, metav1.GetOptions{}) defer func() { @@ -781,7 +895,8 @@ var _ = Describe("EnsureLoadBalancer should not update any resources when servic Expect(err).NotTo(HaveOccurred()) By("Update the service and without significant changes and compare etags") - updateServiceAndCompareEtags(tc, cs, ns, service, ip, false) + Expect(len(targetIPs)).NotTo(BeZero()) + updateServiceAndCompareEtags(tc, cs, ns, service, targetIPs[0], false) }) It("should respect service with BYO public IP prefix with various configurations", func() { @@ -789,27 +904,52 @@ var _ = Describe("EnsureLoadBalancer should not update any resources when servic Skip("pip-prefix-id only work with Standard Load Balancer") } - By("Creating a BYO public IP prefix") - prefixName := "prefix" - prefix, err := utils.WaitCreatePIPPrefix(tc, prefixName, tc.GetResourceGroup(), defaultPublicIPPrefix(prefixName, tc.IPFamily == utils.IPv6)) - defer func() { - Expect(utils.DeletePIPPrefixWithRetry(tc, prefixName)).NotTo(HaveOccurred()) - }() - Expect(err).NotTo(HaveOccurred()) - - By("Creating a service and expose it") annotation := map[string]string{ - consts.ServiceAnnotationPIPPrefixID: pointer.StringDeref(prefix.ID, ""), consts.ServiceAnnotationDisableLoadBalancerFloatingIP: "true", consts.ServiceAnnotationSharedSecurityRule: "true", } + By("Creating BYO public IP prefixes") + prefixNameBase := "prefix" + v4Enabled, v6Enabled := utils.IfIPFamiliesEnabled(tc.IPFamily) + createPIPPrefix := func(isIPv6 bool) func() { + prefixName := utils.GetNameWithSuffix(prefixNameBase, utils.Suffixes[isIPv6]) + prefix, err := utils.WaitCreatePIPPrefix(tc, prefixName, tc.GetResourceGroup(), defaultPublicIPPrefix(prefixName, isIPv6)) + deleteFunc := func() { + Expect(utils.DeletePIPPrefixWithRetry(tc, prefixName)).NotTo(HaveOccurred()) + } + Expect(err).NotTo(HaveOccurred()) + + // TODO: After dual-stack implementation finished, update here. + if utils.DualstackSupported { + annotation[consts.ServiceAnnotationPIPPrefixIDDualStack[isIPv6]] = pointer.StringDeref(prefix.ID, "") + } else { + annotation[consts.ServiceAnnotationPIPPrefixID] = pointer.StringDeref(prefix.ID, "") + } + return deleteFunc + } + deleteFuncs := []func(){} + if v4Enabled { + deleteFuncs = append(deleteFuncs, createPIPPrefix(false)) + } + if v6Enabled { + deleteFuncs = append(deleteFuncs, createPIPPrefix(true)) + } + defer func() { + for _, deleteFunc := range deleteFuncs { + deleteFunc() + } + }() + + By("Creating a service and expose it") service := utils.CreateLoadBalancerServiceManifest(testServiceName, annotation, labels, ns.Name, ports) service.Spec.ExternalTrafficPolicy = "Local" - _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) + _, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, "") + ips, err := utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns.Name, testServiceName, []string{}) Expect(err).NotTo(HaveOccurred()) + Expect(len(ips)).NotTo(BeZero()) + ip := ips[0] // ip is used to get LB only service, err = cs.CoreV1().Services(ns.Name).Get(context.TODO(), testServiceName, metav1.GetOptions{}) defer func() { @@ -844,7 +984,9 @@ var _ = Describe("EnsureLoadBalancer should not update any resources when servic consts.ServiceAnnotationLoadBalancerInternalSubnet: subnetName, consts.ServiceAnnotationLoadBalancerEnableHighAvailabilityPorts: "true", } - ip := createAndExposeDefaultServiceWithAnnotation(cs, testServiceName, ns.Name, labels, annotation, ports) + ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, testServiceName, ns.Name, labels, annotation, ports) + Expect(len(ips)).NotTo(BeZero()) + ip := ips[0] // ip is used to get LB only service, err := cs.CoreV1().Services(ns.Name).Get(context.TODO(), testServiceName, metav1.GetOptions{}) defer func() { By("Cleaning up") @@ -870,14 +1012,16 @@ func updateServiceAndCompareEtags(tc *utils.AzureTestClient, cs clientset.Interf utils.Logf("service's annotations: %v", annotation) _, err := cs.CoreV1().Services(ns.Name).Update(context.TODO(), service, metav1.UpdateOptions{}) Expect(err).NotTo(HaveOccurred()) - ip, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, "") + ips, err := utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns.Name, testServiceName, []string{}) + Expect(len(ips)).NotTo(BeZero()) + ip = ips[0] Expect(err).NotTo(HaveOccurred()) utils.Logf("Checking etags are not changed") newLbEtag, newNsgEtag, newPipEtag := getResourceEtags(tc, ip, cloudprovider.DefaultLoadBalancerName(service), isInternal) - Expect(lbEtag).To(Equal(newLbEtag)) - Expect(nsgEtag).To(Equal(newNsgEtag)) - Expect(pipEtag).To(Equal(newPipEtag)) + Expect(lbEtag).To(Equal(newLbEtag), "lb etag") + Expect(nsgEtag).To(Equal(newNsgEtag), "nsg etag") + Expect(pipEtag).To(Equal(newPipEtag), "pip etag") } func createNewSubnet(tc *utils.AzureTestClient, subnetName string) (*network.Subnet, bool) { @@ -898,10 +1042,13 @@ func createNewSubnet(tc *utils.AzureTestClient, subnetName string) (*network.Sub if subnetToReturn == nil { By("Test subnet doesn't exist. Creating a new one...") isNew = true - newSubnetCIDR, err := utils.GetNextSubnetCIDR(vNet, tc.IPFamily) + newSubnetCIDRs, err := utils.GetNextSubnetCIDRs(vNet, tc.IPFamily) Expect(err).NotTo(HaveOccurred()) - newSubnetCIDRStr := newSubnetCIDR.String() - newSubnet, err := tc.CreateSubnet(vNet, &subnetName, &newSubnetCIDRStr, true) + newSubnetCIDRStrs := []string{} + for _, newSubnetCIDR := range newSubnetCIDRs { + newSubnetCIDRStrs = append(newSubnetCIDRStrs, newSubnetCIDR.String()) + } + newSubnet, err := tc.CreateSubnet(vNet, &subnetName, &newSubnetCIDRStrs, true) Expect(err).NotTo(HaveOccurred()) subnetToReturn = &newSubnet } @@ -995,7 +1142,7 @@ func getLBBackendPoolIndex(lb *aznetwork.LoadBalancer) int { return 0 } -func updateServiceLBIP(service *v1.Service, isInternal bool, ip string) (result *v1.Service) { +func updateServiceLBIPs(service *v1.Service, isInternal bool, ips []string) (result *v1.Service) { result = service if result == nil { return @@ -1003,10 +1150,9 @@ func updateServiceLBIP(service *v1.Service, isInternal bool, ip string) (result if result.Annotations == nil { result.Annotations = map[string]string{} } - if net.ParseIP(ip).To4() != nil { - result.Annotations[consts.ServiceAnnotationLoadBalancerIPDualStack[false]] = ip - } else { - result.Annotations[consts.ServiceAnnotationLoadBalancerIPDualStack[true]] = ip + for _, ip := range ips { + isIPv6 := net.ParseIP(ip).To4() == nil + result.Annotations[consts.ServiceAnnotationLoadBalancerIPDualStack[isIPv6]] = ip } if judgeInternal(*service) == isInternal { @@ -1063,3 +1209,16 @@ func defaultPublicIPPrefix(name string, isIPv6 bool) aznetwork.PublicIPPrefix { }, } } + +func createPIP(tc *utils.AzureTestClient, ipNameBase string, isIPv6 bool) (string, func()) { + ipName := utils.GetNameWithSuffix(ipNameBase, utils.Suffixes[isIPv6]) + pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), defaultPublicIPAddress(ipName, isIPv6)) + Expect(err).NotTo(HaveOccurred()) + targetIP := pointer.StringDeref(pip.IPAddress, "") + utils.Logf("Created PIP to %s", targetIP) + return targetIP, func() { + By("Cleaning up PIP") + err = utils.DeletePIPWithRetry(tc, ipName, "") + Expect(err).NotTo(HaveOccurred()) + } +} diff --git a/tests/e2e/network/network_security_group.go b/tests/e2e/network/network_security_group.go index 38cac78be0..74f0aac8a7 100644 --- a/tests/e2e/network/network_security_group.go +++ b/tests/e2e/network/network_security_group.go @@ -90,7 +90,7 @@ var _ = Describe("Network security group", Label(utils.TestSuiteLabelNSG), func( It("should add the rule when expose a service", func() { By("Creating a service and expose it") - ip := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, map[string]string{}, ports) + ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, map[string]string{}, ports) defer func() { By("Cleaning up") err := utils.DeleteService(cs, ns.Name, serviceName) @@ -101,7 +101,7 @@ var _ = Describe("Network security group", Label(utils.TestSuiteLabelNSG), func( port := fmt.Sprintf("%d", serverPort) nsgs, err := tc.GetClusterSecurityGroups() Expect(err).NotTo(HaveOccurred()) - Expect(validateUnsharedSecurityRuleExists(nsgs, ip, port)).To(BeTrue(), "Security rule for service %s not exists", serviceName) + Expect(validateUnsharedSecurityRuleExists(nsgs, ips, port)).To(BeTrue(), "Security rule for service %s not exists", serviceName) By("Validating network security group working") // Use a hostNetwork Pod to validate Service connectivity via the cluster Node's @@ -115,9 +115,12 @@ var _ = Describe("Network security group", Label(utils.TestSuiteLabelNSG), func( Expect(result).To(BeTrue()) Expect(err).NotTo(HaveOccurred()) - By(fmt.Sprintf("Validating External domain name %q", ip)) - err = utils.ValidateServiceConnectivity(ns.Name, agnhostPod, ip, int(ports[0].Port), v1.ProtocolTCP) - Expect(err).NotTo(HaveOccurred(), "Fail to get response from the domain name") + By(fmt.Sprintf("Validating External domain name %q", ips)) + Expect(len(ports)).NotTo(BeZero()) + for _, ip := range ips { + err = utils.ValidateServiceConnectivity(ns.Name, agnhostPod, ip, int(ports[0].Port), v1.ProtocolTCP) + Expect(err).NotTo(HaveOccurred(), "Fail to get response from the domain name %q", ip) + } By("Validate automatically delete the rule, when service is deleted") Expect(utils.DeleteService(cs, ns.Name, serviceName)).NotTo(HaveOccurred()) @@ -126,7 +129,7 @@ var _ = Describe("Network security group", Label(utils.TestSuiteLabelNSG), func( if err != nil { return false, err } - if !validateUnsharedSecurityRuleExists(nsgs, ip, port) { + if !validateUnsharedSecurityRuleExists(nsgs, ips, port) { utils.Logf("Target rule successfully deleted") return true, nil } @@ -140,7 +143,7 @@ var _ = Describe("Network security group", Label(utils.TestSuiteLabelNSG), func( annotation := map[string]string{ consts.ServiceAnnotationSharedSecurityRule: "true", } - ip1 := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports) + ips1 := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, annotation, ports) defer func() { err := utils.DeleteService(cs, ns.Name, serviceName) @@ -148,7 +151,7 @@ var _ = Describe("Network security group", Label(utils.TestSuiteLabelNSG), func( }() serviceName2 := serviceName + "-share" - ip2 := createAndExposeDefaultServiceWithAnnotation(cs, serviceName2, ns.Name, labels, annotation, ports) + ips2 := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName2, ns.Name, labels, annotation, ports) defer func() { By("Cleaning up") err := utils.DeleteService(cs, ns.Name, serviceName2) @@ -160,12 +163,12 @@ var _ = Describe("Network security group", Label(utils.TestSuiteLabelNSG), func( nsgs, err := tc.GetClusterSecurityGroups() Expect(err).NotTo(HaveOccurred()) - ipList := []string{ip1, ip2} + ipList := append(ips1, ips2...) Expect(validateSharedSecurityRuleExists(nsgs, ipList, port)).To(BeTrue(), "Security rule for service %s not exists", serviceName) By("Validate automatically adjust or delete the rule, when service is deleted") Expect(utils.DeleteService(cs, ns.Name, serviceName)).NotTo(HaveOccurred()) - ipList = []string{ip2} + ipList = ips2 Expect(validateSharedSecurityRuleExists(nsgs, ipList, port)).To(BeTrue(), "Security rule should be modified to only contain service %s", serviceName2) Expect(utils.DeleteService(cs, ns.Name, serviceName2)).NotTo(HaveOccurred()) @@ -195,7 +198,7 @@ var _ = Describe("Network security group", Label(utils.TestSuiteLabelNSG), func( utils.Logf("Successfully created LoadBalancer service " + serviceName + " in namespace " + ns.Name) By("Waiting for the service to be exposed") - _, err = utils.WaitServiceExposure(cs, ns.Name, serviceName, "") + _, err = utils.WaitServiceExposure(cs, ns.Name, serviceName, []string{}) Expect(err).NotTo(HaveOccurred()) By("Validating if the corresponding IP prefix existing in nsg") @@ -232,13 +235,13 @@ var _ = Describe("Network security group", Label(utils.TestSuiteLabelNSG), func( Expect(err).NotTo(HaveOccurred()) By("Waiting for the service to expose") - internalIP, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, "") + ips, err := utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns.Name, serviceName, []string{}) Expect(err).NotTo(HaveOccurred()) By("Checking if there is a deny_all rule") nsgs, err := tc.GetClusterSecurityGroups() Expect(err).NotTo(HaveOccurred()) - found := validateDenyAllSecurityRuleExists(nsgs, internalIP) + found := validateDenyAllSecurityRuleExists(nsgs, ips) Expect(found).To(BeFalse()) By("Deleting the service") @@ -259,62 +262,81 @@ var _ = Describe("Network security group", Label(utils.TestSuiteLabelNSG), func( Expect(err).NotTo(HaveOccurred()) hostExecPodIP := hostExecPod.Status.PodIP - mask := 32 - if tc.IPFamily == utils.IPv6 { - mask = 128 + v4Enabled, v6Enabled := utils.IfIPFamiliesEnabled(tc.IPFamily) + maskV4, maskV6 := 32, 128 + allowCIDRs, ipRangesSuffixes := []string{}, []string{} + if v4Enabled { + allowCIDRs = append(allowCIDRs, fmt.Sprintf("%s/%d", hostExecPodIP, maskV4)) + ipRangesSuffixes = append(ipRangesSuffixes, fmt.Sprintf("%s_%d", hostExecPodIP, maskV4)) + } + if v6Enabled { + allowCIDRs = append(allowCIDRs, fmt.Sprintf("%s/%d", hostExecPodIP, maskV6)) + ipRangesSuffixes = append(ipRangesSuffixes, fmt.Sprintf("%s_%d", hostExecPodIP, maskV6)) } - allowCIDR := fmt.Sprintf("%s/%d", hostExecPodIP, mask) - service.Spec.LoadBalancerSourceRanges = []string{allowCIDR} + service.Spec.LoadBalancerSourceRanges = allowCIDRs _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) // Check Service connectivity with the deny-all-except-lb-range ExecAgnhostPod By("Waiting for the service to expose") - internalIP, err = utils.WaitServiceExposureAndGetIP(cs, ns.Name, serviceName) + internalIPs, err := utils.WaitServiceExposureAndGetIPs(cs, ns.Name, serviceName) + Expect(err).NotTo(HaveOccurred()) for _, port := range service.Spec.Ports { - utils.Logf("checking the connectivity of addr %s:%d with protocol %v", internalIP, int(port.Port), port.Protocol) - err := utils.ValidateServiceConnectivity(ns.Name, agnhostPod, internalIP, int(port.Port), port.Protocol) - Expect(err).NotTo(HaveOccurred()) + for _, internalIP := range internalIPs { + utils.Logf("checking the connectivity of addr %s:%d with protocol %v", internalIP, int(port.Port), port.Protocol) + err := utils.ValidateServiceConnectivity(ns.Name, agnhostPod, internalIP, int(port.Port), port.Protocol) + Expect(err).NotTo(HaveOccurred()) + } } - By("Checking if there is a LoadBalancerSourceRanges rule") nsgs, err = tc.GetClusterSecurityGroups() Expect(err).NotTo(HaveOccurred()) - found = validateLoadBalancerSourceRangesRuleExists(nsgs, internalIP, allowCIDR, fmt.Sprintf("%s_%d", hostExecPodIP, mask)) + By("Checking if there is a LoadBalancerSourceRanges rule") + found = validateLoadBalancerSourceRangesRuleExists(nsgs, internalIPs, allowCIDRs, ipRangesSuffixes) Expect(found).To(BeTrue()) By("Checking if there is a deny_all rule") - found = validateDenyAllSecurityRuleExists(nsgs, internalIP) + found = validateDenyAllSecurityRuleExists(nsgs, internalIPs) Expect(found).To(BeTrue()) }) It("should support service annotation `service.beta.kubernetes.io/azure-disable-load-balancer-floating-ip`", func() { - By("Creating a public IP with tags") - ipName := basename + "-public-IP-disable-floating-ip" - pip := defaultPublicIPAddress(ipName, tc.IPFamily == utils.IPv6) - pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), pip) - Expect(err).NotTo(HaveOccurred()) - targetIP := pointer.StringDeref(pip.IPAddress, "") - utils.Logf("created pip with address %s", targetIP) + By("Creating public IPs with tags") + v4Enabled, v6Enabled := utils.IfIPFamiliesEnabled(tc.IPFamily) + ipNameBase := basename + "-public-IP-disable-floating-ip" + targetIPs := []string{} + deleteFuncs := []func(){} + if v4Enabled { + targetIP, deleteFunc := createPIP(tc, ipNameBase, false) + targetIPs = append(targetIPs, targetIP) + deleteFuncs = append(deleteFuncs, deleteFunc) + } + if v6Enabled { + targetIP, deleteFunc := createPIP(tc, ipNameBase, true) + targetIPs = append(targetIPs, targetIP) + deleteFuncs = append(deleteFuncs, deleteFunc) + } + defer func() { + for _, deleteFunc := range deleteFuncs { + deleteFunc() + } + }() By("Creating a test load balancer service with floating IP disabled") annotation := map[string]string{ consts.ServiceAnnotationDisableLoadBalancerFloatingIP: "true", } service := utils.CreateLoadBalancerServiceManifest(serviceName, annotation, labels, ns.Name, ports) - service = updateServiceLBIP(service, false, targetIP) - _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) + service = updateServiceLBIPs(service, false, targetIPs) + _, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, "") + _, err = utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns.Name, serviceName, targetIPs) Expect(err).NotTo(HaveOccurred()) - Expect(ip).To(Equal(targetIP)) defer func() { By("cleaning up") err = utils.DeleteService(cs, ns.Name, serviceName) Expect(err).NotTo(HaveOccurred()) - err = utils.DeletePIPWithRetry(tc, ipName, "") - Expect(err).NotTo(HaveOccurred()) }() By("Checking if the LoadBalancer's public IP is included in the network security rule's DestinationAddressPrefixes") @@ -325,29 +347,41 @@ var _ = Describe("Network security group", Label(utils.TestSuiteLabelNSG), func( continue } - for _, securityRules := range *nsg.SecurityRules { - contains := slices.Contains(*securityRules.DestinationAddressPrefixes, targetIP) + for _, targetIP := range targetIPs { + contains := false + for _, securityRules := range *nsg.SecurityRules { + if slices.Contains(*securityRules.DestinationAddressPrefixes, targetIP) { + contains = true + break + } + } Expect(contains).To(BeFalse()) } } }) - }) -func validateUnsharedSecurityRuleExists(nsgs []aznetwork.SecurityGroup, ip string, port string) bool { +func validateUnsharedSecurityRuleExists(nsgs []aznetwork.SecurityGroup, ips []string, port string) bool { for _, nsg := range nsgs { if nsg.SecurityRules == nil { continue } - for _, securityRule := range *nsg.SecurityRules { - utils.Logf("Checking security rule %q", pointer.StringDeref(securityRule.Name, "")) - if strings.EqualFold(pointer.StringDeref(securityRule.DestinationAddressPrefix, ""), ip) && strings.EqualFold(pointer.StringDeref(securityRule.DestinationPortRange, ""), port) { - utils.Logf("Found target security rule") - return true + for _, ip := range ips { + found := false + for _, securityRule := range *nsg.SecurityRules { + utils.Logf("Checking security rule %q", pointer.StringDeref(securityRule.Name, "")) + if strings.EqualFold(pointer.StringDeref(securityRule.DestinationAddressPrefix, ""), ip) && strings.EqualFold(pointer.StringDeref(securityRule.DestinationPortRange, ""), port) { + utils.Logf("Found one target security rule with IP %q", ip) + found = true + break + } + } + if !found { + return false } } } - return false + return true } func validateSharedSecurityRuleExists(nsgs []aznetwork.SecurityGroup, ips []string, port string) bool { @@ -375,40 +409,63 @@ func validateSharedSecurityRuleExists(nsgs []aznetwork.SecurityGroup, ips []stri return false } -func validateLoadBalancerSourceRangesRuleExists(nsgs []aznetwork.SecurityGroup, ip, sourceAddressPrefix, ipRangesSuffix string) bool { +func validateLoadBalancerSourceRangesRuleExists(nsgs []aznetwork.SecurityGroup, ips, sourceAddressPrefixes, ipRangesSuffixes []string) bool { + if len(sourceAddressPrefixes) != len(ipRangesSuffixes) { + return false + } for _, nsg := range nsgs { if nsg.SecurityRules == nil { continue } - for _, securityRule := range *nsg.SecurityRules { - utils.Logf("Checking security rule %q", pointer.StringDeref(securityRule.Name, "")) - if securityRule.Access == aznetwork.SecurityRuleAccessAllow && - strings.EqualFold(pointer.StringDeref(securityRule.DestinationAddressPrefix, ""), ip) && - strings.HasSuffix(pointer.StringDeref(securityRule.Name, ""), ipRangesSuffix) && - strings.EqualFold(pointer.StringDeref(securityRule.SourceAddressPrefix, ""), sourceAddressPrefix) { - return true + for _, ip := range ips { + found := false + for _, securityRule := range *nsg.SecurityRules { + utils.Logf("Checking security rule %q", pointer.StringDeref(securityRule.Name, "")) + if securityRule.Access == aznetwork.SecurityRuleAccessAllow && + strings.EqualFold(pointer.StringDeref(securityRule.DestinationAddressPrefix, ""), ip) { + for i := range sourceAddressPrefixes { + sourceAddressPrefix := sourceAddressPrefixes[i] + ipRangesSuffix := ipRangesSuffixes[i] + if strings.HasSuffix(pointer.StringDeref(securityRule.Name, ""), ipRangesSuffix) && + strings.EqualFold(pointer.StringDeref(securityRule.SourceAddressPrefix, ""), sourceAddressPrefix) { + found = true + break + } + } + } + } + if !found { + return false } } + } - return false + return true } -func validateDenyAllSecurityRuleExists(nsgs []aznetwork.SecurityGroup, ip string) bool { +func validateDenyAllSecurityRuleExists(nsgs []aznetwork.SecurityGroup, ips []string) bool { for _, nsg := range nsgs { if nsg.SecurityRules == nil { continue } - for _, securityRule := range *nsg.SecurityRules { - utils.Logf("Checking security rule %q", pointer.StringDeref(securityRule.Name, "")) - if securityRule.Access == aznetwork.SecurityRuleAccessDeny && - strings.EqualFold(pointer.StringDeref(securityRule.DestinationAddressPrefix, ""), ip) && - strings.HasSuffix(pointer.StringDeref(securityRule.Name, ""), "deny_all") && - strings.EqualFold(pointer.StringDeref(securityRule.SourceAddressPrefix, ""), "*") { - return true + for _, ip := range ips { + found := false + for _, securityRule := range *nsg.SecurityRules { + utils.Logf("Checking security rule %q", pointer.StringDeref(securityRule.Name, "")) + if securityRule.Access == aznetwork.SecurityRuleAccessDeny && + strings.EqualFold(pointer.StringDeref(securityRule.DestinationAddressPrefix, ""), ip) && + strings.HasSuffix(pointer.StringDeref(securityRule.Name, ""), "deny_all") && + strings.EqualFold(pointer.StringDeref(securityRule.SourceAddressPrefix, ""), "*") { + found = true + break + } + } + if !found { + return false } } } - return false + return true } diff --git a/tests/e2e/network/node.go b/tests/e2e/network/node.go index a0e7f5ede0..27102fff77 100644 --- a/tests/e2e/network/node.go +++ b/tests/e2e/network/node.go @@ -380,7 +380,9 @@ var _ = Describe("Azure nodes", func() { Expect(ok).To(BeTrue()) Expect(nodeRG).NotTo(Equal(rgMaster)) - publicIP := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, map[string]string{}, ports) + ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, map[string]string{}, ports) + Expect(len(ips)).NotTo(BeZero()) + publicIP := ips[0] lb := getAzureLoadBalancerFromPIP(tc, publicIP, rgMaster, rgMaster) utils.Logf("finding NIC of the node %s, assuming it's in the same rg as master", nodeNotInRGMaster.Name) diff --git a/tests/e2e/network/private_link_service.go b/tests/e2e/network/private_link_service.go index 924ef2a4e4..bfcf33bbf0 100644 --- a/tests/e2e/network/private_link_service.go +++ b/tests/e2e/network/private_link_service.go @@ -98,7 +98,9 @@ var _ = Describe("Private link service", Label(utils.TestSuiteLabelPrivateLinkSe } // create service with given annotation and wait it to expose - ip := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports) + ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, annotation, ports) + Expect(len(ips)).NotTo(BeZero()) + ip := ips[0] defer func() { utils.Logf("cleaning up test service %s", serviceName) err := utils.DeleteService(cs, ns.Name, serviceName) @@ -126,7 +128,9 @@ var _ = Describe("Private link service", Label(utils.TestSuiteLabelPrivateLinkSe } // create service with given annotation and wait it to expose - ip := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports) + ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, annotation, ports) + Expect(len(ips)).NotTo(BeZero()) + ip := ips[0] defer func() { utils.Logf("cleaning up test service %s", serviceName) err := utils.DeleteService(cs, ns.Name, serviceName) @@ -161,7 +165,9 @@ var _ = Describe("Private link service", Label(utils.TestSuiteLabelPrivateLinkSe } // create service with given annotation and wait it to expose - ip := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports) + ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, annotation, ports) + Expect(len(ips)).NotTo(BeZero()) + ip := ips[0] defer func() { utils.Logf("cleaning up test service %s", serviceName) err := utils.DeleteService(cs, ns.Name, serviceName) @@ -185,7 +191,9 @@ var _ = Describe("Private link service", Label(utils.TestSuiteLabelPrivateLinkSe } // create service with given annotation and wait it to expose - ip := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports) + ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, annotation, ports) + Expect(len(ips)).NotTo(BeZero()) + ip := ips[0] defer func() { utils.Logf("cleaning up test service %s", serviceName) err := utils.DeleteService(cs, ns.Name, serviceName) @@ -206,7 +214,9 @@ var _ = Describe("Private link service", Label(utils.TestSuiteLabelPrivateLinkSe } // create service with given annotation and wait it to expose - ip := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports) + ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, annotation, ports) + Expect(len(ips)).NotTo(BeZero()) + ip := ips[0] defer func() { utils.Logf("cleaning up test service %s", serviceName) err := utils.DeleteService(cs, ns.Name, serviceName) @@ -214,7 +224,8 @@ var _ = Describe("Private link service", Label(utils.TestSuiteLabelPrivateLinkSe }() utils.Logf("Get Internal IP: %s", ip) - selectedip, err := utils.SelectAvailablePrivateIP(tc) + ips, err := utils.SelectAvailablePrivateIPs(tc, tc.IPFamily) + selectedip := ips[0] Expect(err).NotTo(HaveOccurred()) annotation[consts.ServiceAnnotationPLSIpConfigurationIPAddress] = selectedip utils.Logf("Now update private link service's static ip to %s", selectedip) @@ -226,7 +237,8 @@ var _ = Describe("Private link service", Label(utils.TestSuiteLabelPrivateLinkSe _, err = cs.CoreV1().Services(ns.Name).Update(context.TODO(), service, metav1.UpdateOptions{}) Expect(err).NotTo(HaveOccurred()) - ip, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, "") + ips, err = utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns.Name, serviceName, []string{}) + ip = ips[0] Expect(err).NotTo(HaveOccurred()) // wait and check pls is updated also @@ -253,7 +265,9 @@ var _ = Describe("Private link service", Label(utils.TestSuiteLabelPrivateLinkSe } // create service with given annotation and wait it to expose - ip := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports) + ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, annotation, ports) + Expect(len(ips)).NotTo(BeZero()) + ip := ips[0] defer func() { utils.Logf("cleaning up test service %s", serviceName) err := utils.DeleteService(cs, ns.Name, serviceName) @@ -279,7 +293,9 @@ var _ = Describe("Private link service", Label(utils.TestSuiteLabelPrivateLinkSe } // create service with given annotation and wait it to expose - ip := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports) + ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, annotation, ports) + Expect(len(ips)).NotTo(BeZero()) + ip := ips[0] defer func() { utils.Logf("cleaning up test service %s", serviceName) err := utils.DeleteService(cs, ns.Name, serviceName) @@ -306,7 +322,9 @@ var _ = Describe("Private link service", Label(utils.TestSuiteLabelPrivateLinkSe } // create service with given annotation and wait it to expose - ip := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports) + ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, annotation, ports) + Expect(len(ips)).NotTo(BeZero()) + ip := ips[0] defer func() { utils.Logf("cleaning up test service %s", serviceName) err := utils.DeleteService(cs, ns.Name, serviceName) @@ -339,7 +357,9 @@ var _ = Describe("Private link service", Label(utils.TestSuiteLabelPrivateLinkSe } // create service with given annotation and wait it to expose - ip := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports) + ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, annotation, ports) + Expect(len(ips)).NotTo(BeZero()) + ip := ips[0] defer func() { utils.Logf("cleaning up test service %s", serviceName) err := utils.DeleteService(cs, ns.Name, serviceName) @@ -366,7 +386,9 @@ var _ = Describe("Private link service", Label(utils.TestSuiteLabelPrivateLinkSe consts.ServiceAnnotationPLSIpConfigurationIPAddressCount: strconv.Itoa(ipAddrCount), } svc1 := "service1" - ip := createAndExposeDefaultServiceWithAnnotation(cs, svc1, ns.Name, labels, annotation, ports) + ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, svc1, ns.Name, labels, annotation, ports) + Expect(len(ips)).NotTo(BeZero()) + ip := ips[0] defer func() { err := utils.DeleteService(cs, ns.Name, svc1) Expect(err).NotTo(HaveOccurred()) @@ -398,10 +420,10 @@ var _ = Describe("Private link service", Label(utils.TestSuiteLabelPrivateLinkSe err = utils.DeleteService(cs, ns.Name, svc2) Expect(err).NotTo(HaveOccurred()) }() - service2 = updateServiceLBIP(service2, true, ip) + service2 = updateServiceLBIPs(service2, true, []string{ip}) _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service2, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - _, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, svc2, ip) + _, err = utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns.Name, svc2, []string{ip}) Expect(err).NotTo(HaveOccurred()) utils.Logf("Successfully created %s in namespace %s with IP %s", svc2, ns.Name, ip) diff --git a/tests/e2e/network/service_annotations.go b/tests/e2e/network/service_annotations.go index 4c46fefad4..9b67bb2508 100644 --- a/tests/e2e/network/service_annotations.go +++ b/tests/e2e/network/service_annotations.go @@ -147,7 +147,7 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn consts.ServiceAnnotationDNSLabelName: serviceDomainNamePrefix, } // create service with given annotation and wait it to expose - _ = createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports) + _ = createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, annotation, ports) defer func() { utils.Logf("cleaning up test service %s", serviceName) err := utils.DeleteService(cs, ns.Name, serviceName) @@ -174,24 +174,32 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn err = utils.DeleteService(cs, ns.Name, serviceName) Expect(err).NotTo(HaveOccurred()) - By("Create a PIP") - ipName := fmt.Sprintf("%s-public-IP-%s", basename, uuid.NewUUID()[0:4]) - nsName := ns.Name - rgName := tc.GetResourceGroup() - pip, err := utils.WaitCreatePIP(tc, ipName, rgName, defaultPublicIPAddress(ipName, tc.IPFamily == utils.IPv6)) + By("Create PIPs") + ipNameBase := fmt.Sprintf("%s-public-IP-%s", basename, uuid.NewUUID()[0:4]) + v4Enabled, v6Enabled := utils.IfIPFamiliesEnabled(tc.IPFamily) + pipNames, targetIPs := []string{}, []string{} + deleteFuncs := []func(){} + if v4Enabled { + targetIP, deleteFunc := createPIP(tc, ipNameBase, false) + targetIPs = append(targetIPs, targetIP) + deleteFuncs = append(deleteFuncs, deleteFunc) + } + if v6Enabled { + targetIP, deleteFunc := createPIP(tc, ipNameBase, true) + targetIPs = append(targetIPs, targetIP) + deleteFuncs = append(deleteFuncs, deleteFunc) + } defer func() { - err := utils.DeletePIPWithRetry(tc, ipName, rgName) - Expect(err).NotTo(HaveOccurred()) + for _, deleteFunc := range deleteFuncs { + deleteFunc() + } }() - Expect(err).NotTo(HaveOccurred()) - targetIP := pointer.StringDeref(pip.IPAddress, "") - pipName := pointer.StringDeref(pip.Name, "") - utils.Logf("PIP %q to %q", pipName, targetIP) By("Create a Service which will be deleted with the PIP") + nsName := ns.Name oldServiceName := fmt.Sprintf("%s-old", serviceName) service := utils.CreateLoadBalancerServiceManifest(oldServiceName, annotation, labels, nsName, ports) - service = updateServiceLBIP(service, false, targetIP) + service = updateServiceLBIPs(service, false, targetIPs) // create service with given annotation and wait it to expose _, err = cs.CoreV1().Services(nsName).Create(context.TODO(), service, metav1.CreateOptions{}) @@ -201,7 +209,7 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn Expect(err).NotTo(HaveOccurred()) }() Expect(err).NotTo(HaveOccurred()) - _, err = utils.WaitServiceExposureAndValidateConnectivity(cs, nsName, oldServiceName, targetIP) + _, err = utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, nsName, oldServiceName, targetIPs) Expect(err).NotTo(HaveOccurred()) By("Delete the old Service") @@ -209,9 +217,11 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn Expect(err).NotTo(HaveOccurred()) By("Check if PIP DNS label is deleted") - deleted, err := ifPIPDNSLabelDeleted(tc, pipName) - Expect(err).NotTo(HaveOccurred()) - Expect(deleted).To(BeTrue()) + for _, pipName := range pipNames { + deleted, err := ifPIPDNSLabelDeleted(tc, pipName) + Expect(err).NotTo(HaveOccurred()) + Expect(deleted).To(BeTrue()) + } By("Create a different Service with the same azure-dns-label-name tag") service.Name = serviceName @@ -245,7 +255,7 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn } // create service with given annotation and wait it to expose - _ = createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports) + _ = createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, annotation, ports) defer func() { utils.Logf("cleaning up test service %s", serviceName) err := utils.DeleteService(cs, ns.Name, serviceName) @@ -261,7 +271,7 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn vNet, err := tc.GetClusterVirtualNetwork() Expect(err).NotTo(HaveOccurred()) - var newSubnetCIDR *net.IPNet + var newSubnetCIDRs []*net.IPNet for _, existingSubnet := range *vNet.Subnets { if *existingSubnet.Name != subnetName { continue @@ -269,40 +279,36 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn utils.Logf("Test subnet have existed, skip creating") if existingSubnet.AddressPrefix != nil { // IPv4 picks the only AddressPrefix. - _, newSubnetCIDR, err = net.ParseCIDR(*existingSubnet.AddressPrefix) + _, newSubnetCIDR, err := net.ParseCIDR(*existingSubnet.AddressPrefix) Expect(err).NotTo(HaveOccurred()) + newSubnetCIDRs = append(newSubnetCIDRs, newSubnetCIDR) } else { Expect(existingSubnet.AddressPrefixes).NotTo(BeNil(), "subnet AddressPrefix and AddressPrefixes shouldn't be both nil") - addrPrefixPicked := false + expectedAddrPrefixPickedCount := 1 + if tc.IPFamily == utils.DualStack { + expectedAddrPrefixPickedCount = 2 + } for _, addrPrefix := range *existingSubnet.AddressPrefixes { - var parsedIP net.IP - parsedIP, newSubnetCIDR, err = net.ParseCIDR(addrPrefix) + _, newSubnetCIDR, err := net.ParseCIDR(addrPrefix) Expect(err).NotTo(HaveOccurred(), "failed to parse CIDR %q", addrPrefix) - if tc.IPFamily == utils.DualStack && parsedIP.To4() != nil { - // Dual-stack picks IPv4 prefix. - addrPrefixPicked = true - break - } - if tc.IPFamily == utils.IPv6 && parsedIP.To4() == nil { - // IPv6 picks IPv6 prefix. - addrPrefixPicked = true - break - } + newSubnetCIDRs = append(newSubnetCIDRs, newSubnetCIDR) } - Expect(addrPrefixPicked).To(BeTrue(), "there's no matching AddressPrefixes") + Expect(len(newSubnetCIDRs)).To(Equal(expectedAddrPrefixPickedCount), "incorrect new subnet CIDRs %v", newSubnetCIDRs) } break } - if newSubnetCIDR == nil { + if len(newSubnetCIDRs) == 0 { By("Test subnet doesn't exist. Creating a new one...") - newSubnetCIDR, err = utils.GetNextSubnetCIDR(vNet, tc.IPFamily) + newSubnetCIDRs, err = utils.GetNextSubnetCIDRs(vNet, tc.IPFamily) Expect(err).NotTo(HaveOccurred()) - newSubnetCIDRStr := newSubnetCIDR.String() - By(fmt.Sprintf("Creating a subnet %q", newSubnetCIDRStr)) - _, err = tc.CreateSubnet(vNet, &subnetName, &newSubnetCIDRStr, false) + newSubnetCIDRStrs := []string{} + for _, newSubnetCIDR := range newSubnetCIDRs { + newSubnetCIDRStrs = append(newSubnetCIDRStrs, newSubnetCIDR.String()) + } + _, err = tc.CreateSubnet(vNet, &subnetName, &newSubnetCIDRStrs, false) Expect(err).NotTo(HaveOccurred()) defer func() { utils.Logf("cleaning up test subnet %s", subnetName) @@ -317,17 +323,25 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn } // create service with given annotation and wait it to expose - ip := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports) + ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, annotation, ports) defer func() { utils.Logf("cleaning up test service %s", serviceName) err := utils.DeleteService(cs, ns.Name, serviceName) Expect(err).NotTo(HaveOccurred()) }() - utils.Logf("Get External IP: %s", ip) + utils.Logf("Get External IPs: %v", ips) By("Validating external ip in target subnet") - contains := newSubnetCIDR.Contains(net.ParseIP(ip)) - Expect(contains).To(BeTrue(), "external ip %s is not in the target subnet %s", ip, newSubnetCIDR) + for _, ip := range ips { + contains := false + for _, newSubnetCIDR := range newSubnetCIDRs { + if newSubnetCIDR.Contains(net.ParseIP(ip)) { + contains = true + break + } + } + Expect(contains).To(BeTrue(), "external IP %q is not in the target subnets %q", ip, newSubnetCIDRs) + } }) It("should support service annotation 'service.beta.kubernetes.io/azure-load-balancer-enable-high-availability-ports'", func() { @@ -340,16 +354,18 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn consts.ServiceAnnotationLoadBalancerInternal: "true", } // create service with given annotation and wait it to expose - ip := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports) + ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, annotation, ports) defer func() { utils.Logf("cleaning up test service %s", serviceName) err := utils.DeleteService(cs, ns.Name, serviceName) Expect(err).NotTo(HaveOccurred()) }() + Expect(len(ips)).NotTo(BeZero()) + ip := ips[0] // only to get LB lb := getAzureInternalLoadBalancerFromPrivateIP(tc, ip, "") - Expect(len(*lb.LoadBalancingRules)).To(Equal(1)) + Expect(len(*lb.LoadBalancingRules)).To(Equal(len(ips))) rule := (*lb.LoadBalancingRules)[0] Expect(*rule.FrontendPort).To(Equal(int32(0))) Expect(*rule.BackendPort).To(Equal(int32(0))) @@ -361,15 +377,17 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn } // create service with given annotation and wait it to expose - publicIP := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports) + ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, annotation, ports) defer func() { By("Cleaning up service") err := utils.DeleteService(cs, ns.Name, serviceName) Expect(err).NotTo(HaveOccurred()) }() + Expect(len(ips)).NotTo(BeZero()) + ip := ips[0] // only to get LB // get lb from azure client - lb := getAzureLoadBalancerFromPIP(tc, publicIP, tc.GetResourceGroup(), "") + lb := getAzureLoadBalancerFromPIP(tc, ip, tc.GetResourceGroup(), "") var idleTimeout *int32 for _, rule := range *lb.LoadBalancingRules { @@ -389,15 +407,31 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn defer cleanup(pointer.StringDeref(rg.Name, "")) By("creating test PIP in the test resource group") - testPIPName := "testPIP-" + string(uuid.NewUUID())[0:4] - pip, err := utils.WaitCreatePIP(tc, testPIPName, *rg.Name, defaultPublicIPAddress(testPIPName, tc.IPFamily == utils.IPv6)) - Expect(err).NotTo(HaveOccurred()) + pips := []string{} + testPIPNames := []string{} + v4Enabled, v6Enabled := utils.IfIPFamiliesEnabled(tc.IPFamily) + if v4Enabled { + testPIPName := "testPIP-" + utils.GetNameWithSuffix(string(uuid.NewUUID())[0:4], utils.Suffixes[false]) + pip, err := utils.WaitCreatePIP(tc, testPIPName, *rg.Name, defaultPublicIPAddress(testPIPName, false)) + Expect(err).NotTo(HaveOccurred()) + testPIPNames = append(testPIPNames, testPIPName) + pips = append(pips, pointer.StringDeref(pip.Name, "")) + } + if v6Enabled { + testPIPName := "testPIP-" + utils.GetNameWithSuffix(string(uuid.NewUUID())[0:4], utils.Suffixes[true]) + pip, err := utils.WaitCreatePIP(tc, testPIPName, *rg.Name, defaultPublicIPAddress(testPIPName, true)) + Expect(err).NotTo(HaveOccurred()) + testPIPNames = append(testPIPNames, testPIPName) + pips = append(pips, pointer.StringDeref(pip.Name, "")) + } defer func() { utils.Logf("Cleaning up service and public IP") - err = utils.DeleteService(cs, ns.Name, serviceName) - Expect(err).NotTo(HaveOccurred()) - err = utils.DeletePIPWithRetry(tc, testPIPName, *rg.Name) + err := utils.DeleteService(cs, ns.Name, serviceName) Expect(err).NotTo(HaveOccurred()) + for _, testPIPName := range testPIPNames { + err = utils.DeletePIPWithRetry(tc, testPIPName, *rg.Name) + Expect(err).NotTo(HaveOccurred()) + } }() annotation := map[string]string{ @@ -405,43 +439,56 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn } By("Creating service " + serviceName + " in namespace " + ns.Name) service := utils.CreateLoadBalancerServiceManifest(serviceName, annotation, labels, ns.Name, ports) - service = updateServiceLBIP(service, false, *pip.IPAddress) - _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) + service = updateServiceLBIPs(service, false, pips) + _, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) utils.Logf("Successfully created LoadBalancer service " + serviceName + " in namespace " + ns.Name) //wait and get service's public IP Address By("Waiting service to expose...") - _, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, pointer.StringDeref(pip.IPAddress, "")) + _, err = utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns.Name, serviceName, pips) Expect(err).NotTo(HaveOccurred()) - lb := getAzureLoadBalancerFromPIP(tc, *pip.IPAddress, *rg.Name, "") + Expect(len(pips)).NotTo(BeZero()) + lb := getAzureLoadBalancerFromPIP(tc, pips[0], *rg.Name, "") Expect(lb).NotTo(BeNil()) }) It("should support service annotation `service.beta.kubernetes.io/azure-additional-public-ips`", func() { - By("creating a public IP") - ipName := basename + "-public-IP" + string(uuid.NewUUID())[0:4] - pip := defaultPublicIPAddress(ipName, tc.IPFamily == utils.IPv6) - pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), pip) - Expect(err).NotTo(HaveOccurred()) - additionalPIP := pointer.StringDeref(pip.IPAddress, "") - utils.Logf("created pip with address %s", additionalPIP) + By("creating public IPs") + ipNameBase := basename + "-public-IP" + string(uuid.NewUUID())[0:4] + v4Enabled, v6Enabled := utils.IfIPFamiliesEnabled(tc.IPFamily) + additionalPIPs := []string{} + deleteFuncs := []func(){} + if v4Enabled { + targetIP, deleteFunc := createPIP(tc, ipNameBase, false) + additionalPIPs = append(additionalPIPs, targetIP) + deleteFuncs = append(deleteFuncs, deleteFunc) + } + if v6Enabled { + targetIP, deleteFunc := createPIP(tc, ipNameBase, true) + additionalPIPs = append(additionalPIPs, targetIP) + deleteFuncs = append(deleteFuncs, deleteFunc) + } + defer func() { + for _, deleteFunc := range deleteFuncs { + deleteFunc() + } + }() - By("Exposing service with additional pip") + By("Exposing service with additional pips") + additionalPIPsStr := strings.Join(additionalPIPs, ",") annotation := map[string]string{ - consts.ServiceAnnotationAdditionalPublicIPs: additionalPIP, + consts.ServiceAnnotationAdditionalPublicIPs: additionalPIPsStr, } - ip := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports) + ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, annotation, ports) defer func() { By("cleaning up") err := utils.DeleteService(cs, ns.Name, serviceName) Expect(err).NotTo(HaveOccurred()) - err = utils.DeletePIPWithRetry(tc, ipName, "") - Expect(err).NotTo(HaveOccurred()) }() - err = wait.PollImmediate(10*time.Second, 2*time.Minute, func() (bool, error) { + err := wait.PollImmediate(10*time.Second, 2*time.Minute, func() (bool, error) { service, err := cs.CoreV1().Services(ns.Name).Get(context.TODO(), serviceName, metav1.GetOptions{}) if err != nil { if utils.IsRetryableAPIError(err) { @@ -451,26 +498,49 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn } utils.Logf("Checking additionalPIP in ingress") - foundInIngress := false + foundInIngress := 0 ingressList := service.Status.LoadBalancer.Ingress Expect(ingressList).NotTo(BeNil()) + expectedAdditionalPIPCount := 1 + if tc.IPFamily == utils.DualStack { + expectedAdditionalPIPCount = 2 + } for _, ingress := range ingressList { - if additionalPIP == ingress.IP { - foundInIngress = true - break + for _, additionalPIP := range additionalPIPs { + if additionalPIP == ingress.IP { + foundInIngress++ + break + } } } + if foundInIngress != expectedAdditionalPIPCount { + return false, nil + } utils.Logf("Checking additionalPIP in nsg") - ipList := []string{ip, additionalPIP} + ipList := append(ips, additionalPIPs...) port := fmt.Sprintf("%d", serverPort) nsgs, err := tc.GetClusterSecurityGroups() if err != nil { return false, err } - foundInNsg := validateSharedSecurityRuleExists(nsgs, ipList, port) + utils.Logf("ipList %q", ipList) + ipListV4, ipListV6 := []string{}, []string{} + for _, ip := range ipList { + if net.ParseIP(ip).To4() != nil { + ipListV4 = append(ipListV4, ip) + } else { + ipListV6 = append(ipListV6, ip) + } + } + if v4Enabled && !validateSharedSecurityRuleExists(nsgs, ipListV4, port) { + return false, nil + } + if v6Enabled && !validateSharedSecurityRuleExists(nsgs, ipListV6, port) { + return false, nil + } - return foundInIngress && foundInNsg, nil + return true, nil }) Expect(err).NotTo(HaveOccurred()) }) @@ -486,7 +556,7 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn Expect(err).NotTo(HaveOccurred()) By("Waiting service to expose...") - ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, "") + ips, err := utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns.Name, serviceName, []string{}) Expect(err).NotTo(HaveOccurred()) defer func() { @@ -503,13 +573,15 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn } pips, err := tc.ListPublicIPs(tc.GetResourceGroup()) Expect(err).NotTo(HaveOccurred()) - var targetPIP network.PublicIPAddress + var targetPIPs []network.PublicIPAddress for _, pip := range pips { - if strings.EqualFold(pointer.StringDeref(pip.IPAddress, ""), ip) { - targetPIP = pip - err := waitComparePIPTags(tc, expectedTags, pointer.StringDeref(pip.Name, "")) - Expect(err).NotTo(HaveOccurred()) - break + for _, ip := range ips { + if strings.EqualFold(pointer.StringDeref(pip.IPAddress, ""), ip) { + targetPIPs = append(targetPIPs, pip) + err := waitComparePIPTags(tc, expectedTags, pointer.StringDeref(pip.Name, "")) + Expect(err).NotTo(HaveOccurred()) + break + } } } @@ -527,35 +599,63 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn "e": pointer.String(""), "x": pointer.String("y"), } - err = waitComparePIPTags(tc, expectedTags, pointer.StringDeref(targetPIP.Name, "")) - Expect(err).NotTo(HaveOccurred()) + for _, targetPIP := range targetPIPs { + err = waitComparePIPTags(tc, expectedTags, pointer.StringDeref(targetPIP.Name, "")) + Expect(err).NotTo(HaveOccurred()) + } }) It("should support service annotation `service.beta.kubernetes.io/azure-pip-name`", func() { - By("Creating two test pips") - pipName1 := "pip1" - pip1, err := utils.WaitCreatePIP(tc, pipName1, tc.GetResourceGroup(), defaultPublicIPAddress(pipName1, tc.IPFamily == utils.IPv6)) - defer func() { - By("Cleaning up test PIP") - err := utils.DeletePIPWithRetry(tc, pipName1, tc.GetResourceGroup()) - Expect(err).NotTo(HaveOccurred()) - }() - Expect(err).NotTo(HaveOccurred()) - pipName2 := "pip2" - pip2, err := utils.WaitCreatePIP(tc, pipName2, tc.GetResourceGroup(), defaultPublicIPAddress(pipName2, tc.IPFamily == utils.IPv6)) + By("Creating two test pips or more if DualStack") + v4Enabled, v6Enabled := utils.IfIPFamiliesEnabled(tc.IPFamily) + pipNames1, pipNames2 := map[bool]string{}, map[bool]string{} + pipNamesSlice1, pipNamesSlice2 := []string{}, []string{} + pipNameBase1, pipNameBase2 := "pip1", "pip2" + targetIPs1, targetIPs2 := []string{}, []string{} + deleteFuncs := []func(){} + doPIP := func(isIPv6 bool) { + pipName1 := utils.GetNameWithSuffix(pipNameBase1, utils.Suffixes[isIPv6]) + pipNames1[isIPv6] = pipName1 + pipNamesSlice1 = append(pipNamesSlice1, pipName1) + targetIP1, deleteFunc1 := createPIP(tc, pipNameBase1, isIPv6) + targetIPs1 = append(targetIPs1, targetIP1) + deleteFuncs = append(deleteFuncs, deleteFunc1) + + pipName2 := utils.GetNameWithSuffix(pipNameBase2, utils.Suffixes[isIPv6]) + pipNames2[isIPv6] = pipName2 + pipNamesSlice2 = append(pipNamesSlice2, pipName2) + targetIP2, deleteFunc2 := createPIP(tc, pipNameBase2, isIPv6) + targetIPs2 = append(targetIPs2, targetIP2) + deleteFuncs = append(deleteFuncs, deleteFunc2) + } + if v4Enabled { + doPIP(false) + } + if v6Enabled { + doPIP(true) + } defer func() { - By("Cleaning up test PIP") - err := utils.DeletePIPWithRetry(tc, pipName2, tc.GetResourceGroup()) - Expect(err).NotTo(HaveOccurred()) + for _, deleteFunc := range deleteFuncs { + deleteFunc() + } }() - Expect(err).NotTo(HaveOccurred()) By("Creating a service referring to the first pip") - annotation := map[string]string{ - consts.ServiceAnnotationPIPName: pipName1, + annotation := map[string]string{} + // TODO: dual-stack + if utils.DualstackSupported { + if v4Enabled { + annotation[consts.ServiceAnnotationPIPNameDualStack[false]] = pipNames1[false] + } + if v6Enabled { + annotation[consts.ServiceAnnotationPIPNameDualStack[true]] = pipNames1[true] + } + } else { + annotation[consts.ServiceAnnotationPIPName] = pipNames1[true] } + service := utils.CreateLoadBalancerServiceManifest(serviceName, annotation, labels, ns.Name, ports) - _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) + _, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) defer func() { By("Cleaning up test service") err := utils.DeleteService(cs, ns.Name, serviceName) @@ -564,19 +664,28 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn Expect(err).NotTo(HaveOccurred()) By("Waiting for the service to expose") - ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, "") + _, err = utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns.Name, serviceName, targetIPs1) Expect(err).NotTo(HaveOccurred()) - Expect(ip).To(Equal(pointer.StringDeref(pip1.IPAddress, ""))) By("Updating the service to refer to the second service") service, err = cs.CoreV1().Services(ns.Name).Get(context.TODO(), serviceName, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) - service.Annotations[consts.ServiceAnnotationPIPName] = pipName2 + if utils.DualstackSupported { + if v4Enabled { + service.Annotations[consts.ServiceAnnotationPIPNameDualStack[false]] = pipNames2[false] + } + if v6Enabled { + service.Annotations[consts.ServiceAnnotationPIPNameDualStack[true]] = pipNames2[true] + } + } else { + service.Annotations[consts.ServiceAnnotationPIPName] = pipNames2[true] + } + _, err = cs.CoreV1().Services(ns.Name).Update(context.TODO(), service, metav1.UpdateOptions{}) Expect(err).NotTo(HaveOccurred()) By("Waiting for service IP to be updated") - _, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, pointer.StringDeref(pip2.IPAddress, "")) + _, err = utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns.Name, serviceName, targetIPs2) Expect(err).NotTo(HaveOccurred()) }) @@ -584,33 +693,61 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn if !strings.EqualFold(os.Getenv(utils.LoadBalancerSkuEnv), string(network.PublicIPAddressSkuNameStandard)) { Skip("pip-prefix-id only work with Standard Load Balancer") } + + By("Creating two test PIPPrefixes or more if DualStack") const ( - prefix1Name = "prefix1" - prefix2Name = "prefix2" + prefix1NameBase = "prefix1" + prefix2NameBase = "prefix2" ) + prefixIDs1, prefixIDs2 := map[bool]string{}, map[bool]string{} + prefixNames1, prefixNames2 := map[bool]string{}, map[bool]string{} + deleteFuncs := []func(){} + + v4Enabled, v6Enabled := utils.IfIPFamiliesEnabled(tc.IPFamily) + createPIPPrefix := func(isIPv6 bool) { + prefixName := utils.GetNameWithSuffix(prefix1NameBase, utils.Suffixes[isIPv6]) + prefixNames1[isIPv6] = prefixName + prefix, err := utils.WaitCreatePIPPrefix(tc, prefixName, tc.GetResourceGroup(), defaultPublicIPPrefix(prefixName, isIPv6)) + deleteFuncs = append(deleteFuncs, func() { + Expect(utils.DeletePIPPrefixWithRetry(tc, prefixName)).NotTo(HaveOccurred()) + }) + Expect(err).NotTo(HaveOccurred()) + prefixIDs1[isIPv6] = pointer.StringDeref(prefix.ID, "") - By("Creating two test PIPPrefix") - prefix1, err := utils.WaitCreatePIPPrefix(tc, prefix1Name, tc.GetResourceGroup(), defaultPublicIPPrefix(prefix1Name, tc.IPFamily == utils.IPv6)) - defer func() { - By(fmt.Sprintf("Cleaning up pip-prefix: %s", prefix1Name)) - Expect(utils.DeletePIPPrefixWithRetry(tc, prefix1Name)).NotTo(HaveOccurred()) - }() - Expect(err).NotTo(HaveOccurred()) - - prefix2, err := utils.WaitCreatePIPPrefix(tc, prefix2Name, tc.GetResourceGroup(), defaultPublicIPPrefix(prefix2Name, tc.IPFamily == utils.IPv6)) + prefixName = utils.GetNameWithSuffix(prefix2NameBase, utils.Suffixes[isIPv6]) + prefixNames2[isIPv6] = prefixName + prefix, err = utils.WaitCreatePIPPrefix(tc, prefixName, tc.GetResourceGroup(), defaultPublicIPPrefix(prefixName, isIPv6)) + deleteFuncs = append(deleteFuncs, func() { + Expect(utils.DeletePIPPrefixWithRetry(tc, prefixName)).NotTo(HaveOccurred()) + }) + Expect(err).NotTo(HaveOccurred()) + prefixIDs2[isIPv6] = pointer.StringDeref(prefix.ID, "") + } + if v4Enabled { + createPIPPrefix(false) + } + if v6Enabled { + createPIPPrefix(true) + } defer func() { - By(fmt.Sprintf("Cleaning up pip-prefix: %s", prefix2Name)) - Expect(utils.DeletePIPPrefixWithRetry(tc, prefix2Name)).NotTo(HaveOccurred()) + for _, deleteFunc := range deleteFuncs { + deleteFunc() + } }() - Expect(err).NotTo(HaveOccurred()) By("Creating a service referring to the prefix") { - annotation := map[string]string{ - consts.ServiceAnnotationPIPPrefixID: pointer.StringDeref(prefix1.ID, ""), + annotation := map[string]string{} + for isIPv6, id := range prefixIDs1 { + // TODO: Update after dual-stack implementation finishes + if utils.DualstackSupported { + annotation[consts.ServiceAnnotationPIPPrefixIDDualStack[isIPv6]] = id + } else { + annotation[consts.ServiceAnnotationPIPPrefixID] = id + } } service := utils.CreateLoadBalancerServiceManifest(serviceName, annotation, labels, ns.Name, ports) - _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) + _, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) } @@ -621,38 +758,56 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn By("Waiting for the service to expose") { - ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, "") + ips, err := utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns.Name, serviceName, []string{}) Expect(err).NotTo(HaveOccurred()) - pip, err := utils.WaitGetPIPByPrefix(tc, prefix1Name, true) - Expect(err).NotTo(HaveOccurred()) + for _, ip := range ips { + isIPv6 := net.ParseIP(ip).To4() == nil + pip, err := utils.WaitGetPIPByPrefix(tc, prefixNames1[isIPv6], true) + Expect(err).NotTo(HaveOccurred()) - Expect(pip.IPAddress).NotTo(BeNil()) - Expect(pip.PublicIPPrefix.ID).To(Equal(prefix1.ID)) - Expect(ip).To(Equal(pointer.StringDeref(pip.IPAddress, ""))) + Expect(pip.IPAddress).NotTo(BeNil()) + Expect(pointer.StringDeref(pip.PublicIPPrefix.ID, "")).To(Equal(prefixIDs1[isIPv6])) + Expect(ip).To(Equal(pointer.StringDeref(pip.IPAddress, ""))) + } } By("Updating the service to refer to the second prefix") { service, err := cs.CoreV1().Services(ns.Name).Get(context.TODO(), serviceName, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) - service.Annotations[consts.ServiceAnnotationPIPPrefixID] = pointer.StringDeref(prefix2.ID, "") + for isIPv6, id := range prefixIDs2 { + // TODO: Update after dual-stack implementation finishes + if utils.DualstackSupported { + service.Annotations[consts.ServiceAnnotationPIPPrefixIDDualStack[isIPv6]] = id + } else { + service.Annotations[consts.ServiceAnnotationPIPPrefixID] = id + } + } _, err = cs.CoreV1().Services(ns.Name).Update(context.TODO(), service, metav1.UpdateOptions{}) Expect(err).NotTo(HaveOccurred()) } By("Waiting for service IP to be updated") { - var pip network.PublicIPAddress - // wait until ip created by prefix - pip, err := utils.WaitGetPIPByPrefix(tc, prefix2Name, true) - Expect(err).NotTo(HaveOccurred()) + pipAddrs := []string{} + doPIPByPrefix := func(isIPv6 bool) { + pip, err := utils.WaitGetPIPByPrefix(tc, prefixNames2[isIPv6], true) + Expect(err).NotTo(HaveOccurred()) - Expect(pip.IPAddress).NotTo(BeNil()) - Expect(pip.PublicIPPrefix.ID).To(Equal(prefix2.ID)) + Expect(pip.IPAddress).NotTo(BeNil()) + Expect(pointer.StringDeref(pip.PublicIPPrefix.ID, "")).To(Equal(prefixIDs2[isIPv6])) + pipAddrs = append(pipAddrs, pointer.StringDeref(pip.IPAddress, "")) + } + if v4Enabled { + doPIPByPrefix(false) + } + if v6Enabled { + doPIPByPrefix(true) + } - _, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, pointer.StringDeref(pip.IPAddress, "")) + _, err := utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns.Name, serviceName, pipAddrs) Expect(err).NotTo(HaveOccurred()) } }) @@ -669,33 +824,49 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn } // create service with given annotation and wait it to expose - publicIP := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports) + ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, annotation, ports) defer func() { By("Cleaning up service") err := utils.DeleteService(cs, ns.Name, serviceName) Expect(err).NotTo(HaveOccurred()) }() - pipFrontendConfigID := getPIPFrontendConfigurationID(tc, publicIP, tc.GetResourceGroup(), "") - pipFrontendConfigIDSplit := strings.Split(pipFrontendConfigID, "/") - Expect(len(pipFrontendConfigIDSplit)).NotTo(Equal(0)) + Expect(len(ips)).NotTo(BeZero()) + ids := []string{} + for _, ip := range ips { + pipFrontendConfigID := getPIPFrontendConfigurationID(tc, ip, tc.GetResourceGroup(), "") + pipFrontendConfigIDSplit := strings.Split(pipFrontendConfigID, "/") + Expect(len(pipFrontendConfigIDSplit)).NotTo(Equal(0)) + ids = append(ids, pipFrontendConfigIDSplit[len(pipFrontendConfigIDSplit)-1]) + } var lb *network.LoadBalancer var targetProbes []*network.Probe + expectedTargetProbesCount := 1 + if tc.IPFamily == utils.DualStack { + expectedTargetProbesCount = 2 + } //wait for backend update err := wait.PollImmediate(5*time.Second, 60*time.Second, func() (bool, error) { - lb = getAzureLoadBalancerFromPIP(tc, publicIP, tc.GetResourceGroup(), "") + lb = getAzureLoadBalancerFromPIP(tc, ips[0], tc.GetResourceGroup(), "") targetProbes = []*network.Probe{} for i := range *lb.LoadBalancerPropertiesFormat.Probes { probe := (*lb.LoadBalancerPropertiesFormat.Probes)[i] utils.Logf("One probe of LB is %q", *probe.Name) probeSplit := strings.Split(*probe.Name, "-") Expect(len(probeSplit)).NotTo(Equal(0)) - if pipFrontendConfigIDSplit[len(pipFrontendConfigIDSplit)-1] == probeSplit[0] { - targetProbes = append(targetProbes, &probe) + probeSplitID := probeSplit[0] + if len(probeSplit) > 1 && + (probeSplit[len(probeSplit)-1] == "IPv4" || probeSplit[len(probeSplit)-1] == "IPv6") { + probeSplitID += "-" + probeSplit[len(probeSplit)-1] + } + for _, id := range ids { + if id == probeSplitID { + targetProbes = append(targetProbes, &probe) + } } } - return len(targetProbes) == 1, nil + return len(targetProbes) == expectedTargetProbesCount, nil }) Expect(err).NotTo(HaveOccurred()) @@ -715,8 +886,10 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn utils.Logf("Validating health probe config intervalInSeconds") Expect(*intervalInSeconds).To(Equal(int32(10))) utils.Logf("Validating health probe config protocol") - Expect((len(targetProbes))).To(Equal(1)) - Expect(targetProbes[0].Protocol).To(Equal(network.ProbeProtocolHTTP)) + Expect((len(targetProbes))).To(Equal(expectedTargetProbesCount)) + for _, targetProbe := range targetProbes { + Expect(targetProbe.Protocol).To(Equal(network.ProbeProtocolHTTP)) + } }) It("should generate health probe configs in multi-port scenario", func() { @@ -738,28 +911,35 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn }) // create service with given annotation and wait it to expose - publicIP := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports) + ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, annotation, ports) defer func() { By("Cleaning up service") err := utils.DeleteService(cs, ns.Name, serviceName) Expect(err).NotTo(HaveOccurred()) }() - pipFrontendConfigID := getPIPFrontendConfigurationID(tc, publicIP, tc.GetResourceGroup(), "") - pipFrontendConfigIDSplit := strings.Split(pipFrontendConfigID, "/") - Expect(len(pipFrontendConfigIDSplit)).NotTo(Equal(0)) + Expect(len(ips)).NotTo(BeZero()) + for _, ip := range ips { + pipFrontendConfigID := getPIPFrontendConfigurationID(tc, ip, tc.GetResourceGroup(), "") + pipFrontendConfigIDSplit := strings.Split(pipFrontendConfigID, "/") + Expect(len(pipFrontendConfigIDSplit)).NotTo(Equal(0)) + } var lb *network.LoadBalancer var targetProbes []*network.Probe + expectedTargetProbesCount := 1 + if tc.IPFamily == utils.DualStack { + expectedTargetProbesCount = 2 + } //wait for backend update err := wait.PollImmediate(5*time.Second, 60*time.Second, func() (bool, error) { - lb = getAzureLoadBalancerFromPIP(tc, publicIP, tc.GetResourceGroup(), "") + lb = getAzureLoadBalancerFromPIP(tc, ips[0], tc.GetResourceGroup(), "") targetProbes = []*network.Probe{} for i := range *lb.LoadBalancerPropertiesFormat.Probes { probe := (*lb.LoadBalancerPropertiesFormat.Probes)[i] utils.Logf("One probe of LB is %q", *probe.Name) targetProbes = append(targetProbes, &probe) } - return len(targetProbes) == 1, nil + return len(targetProbes) == expectedTargetProbesCount, nil }) Expect(err).NotTo(HaveOccurred()) @@ -779,49 +959,69 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn utils.Logf("Validating health probe config intervalInSeconds") Expect(*intervalInSeconds).To(Equal(int32(10))) utils.Logf("Validating health probe config protocol") - Expect((len(targetProbes))).To(Equal(1)) - Expect(targetProbes[0].Protocol).To(Equal(network.ProbeProtocolHTTP)) + Expect((len(targetProbes))).To(Equal(expectedTargetProbesCount)) + for _, targetProbe := range targetProbes { + Expect(targetProbe.Protocol).To(Equal(network.ProbeProtocolHTTP)) + } }) // Check if the following annotations are correctly set with Service LB IP // service.beta.kubernetes.io/azure-load-balancer-ipv4 or service.beta.kubernetes.io/azure-load-balancer-ipv6 It("should support service annotation 'service.beta.kubernetes.io/azure-load-balancer-ip'", func() { - pipName := fmt.Sprintf("%s-public-IP%s", basename, string(uuid.NewUUID())[0:4]) - By(fmt.Sprintf("Creating a public IP %q", pipName)) - var pip network.PublicIPAddress - // TODO: dual-stack support - if tc.IPFamily != utils.DualStack { - pip = defaultPublicIPAddress(pipName, tc.IPFamily == utils.IPv6) - } + v4Enabled, v6Enabled := utils.IfIPFamiliesEnabled(tc.IPFamily) rg := tc.GetResourceGroup() - pip, err := utils.WaitCreatePIP(tc, pipName, rg, pip) - Expect(err).NotTo(HaveOccurred()) - defer func() { - utils.Logf("Cleaning up public IP") - err = utils.DeletePIPWithRetry(tc, pipName, rg) - Expect(err).NotTo(HaveOccurred()) - }() - pipAddr := pointer.StringDeref(pip.IPAddress, "") - utils.Logf("Created pip with address %s", pipAddr) annotation := map[string]string{} - // TODO: dual-stack support - if tc.IPFamily == utils.IPv4 { - annotation[consts.ServiceAnnotationLoadBalancerIPDualStack[false]] = pipAddr - } else if tc.IPFamily == utils.IPv6 { - annotation[consts.ServiceAnnotationLoadBalancerIPDualStack[true]] = pipAddr + + createPIPWithIPFamily := func(isIPv6 bool) (string, func()) { + pipName := fmt.Sprintf("%s-public-IP%s", basename, string(uuid.NewUUID())[0:4]) + By(fmt.Sprintf("Creating a public IP %q, isIPv6: %v", pipName, isIPv6)) + pip := defaultPublicIPAddress(pipName, isIPv6) + pip, err := utils.WaitCreatePIP(tc, pipName, rg, pip) + Expect(err).NotTo(HaveOccurred()) + cleanup := func() { + utils.Logf("Cleaning up public IP %q", pipName) + err := utils.DeletePIPWithRetry(tc, pipName, rg) + Expect(err).NotTo(HaveOccurred()) + } + + pipAddr := pointer.StringDeref(pip.IPAddress, "") + utils.Logf("Created pip with address %s", pipAddr) + annotation[consts.ServiceAnnotationLoadBalancerIPDualStack[isIPv6]] = pipAddr + return pipAddr, cleanup + } + + var pipAddrV4, pipAddrV6 string + var cleanPIPV4, cleanPIPV6 func() + if v4Enabled { + pipAddrV4, cleanPIPV4 = createPIPWithIPFamily(false) + defer cleanPIPV4() + } + if v6Enabled { + pipAddrV6, cleanPIPV6 = createPIPWithIPFamily(true) + defer cleanPIPV6() } - By("Creating a Service") - publicIP := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports) + By("Creating Services") + ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, annotation, ports) defer func() { By("Cleaning up service") err := utils.DeleteService(cs, ns.Name, serviceName) Expect(err).NotTo(HaveOccurred()) }() + Expect(len(ips)).To(Equal(2)) + var publicIPv4, publicIPv6 string + if net.ParseIP(ips[0]).To4() != nil { + publicIPv4 = ips[0] + publicIPv6 = ips[1] + } else { + publicIPv4 = ips[1] + publicIPv6 = ips[0] + } By("Check if the Service has the correct address") - Expect(publicIP).To(Equal(pipAddr)) + Expect(publicIPv4).To(Equal(pipAddrV4)) + Expect(publicIPv6).To(Equal(pipAddrV6)) }) }) @@ -1002,16 +1202,21 @@ var _ = Describe("Multi-ports service", Label(utils.TestSuiteLabelMultiPorts), f //wait and get service's public IP Address utils.Logf("Waiting service to expose...") - publicIP, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, "") + ips, err := utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns.Name, serviceName, []string{}) Expect(err).NotTo(HaveOccurred()) + Expect(len(ips)).NotTo(BeZero()) // create service with given annotation and wait it to expose defer func() { By("Cleaning up service and public IP") err := utils.DeleteService(cs, ns.Name, serviceName) Expect(err).NotTo(HaveOccurred()) - err = utils.DeletePIPWithRetry(tc, publicIP, tc.GetResourceGroup()) - Expect(err).NotTo(HaveOccurred()) + for _, ip := range ips { + pip, err := tc.GetPublicIPFromAddress(tc.GetResourceGroup(), ip) + Expect(err).NotTo(HaveOccurred()) + err = utils.DeletePIPWithRetry(tc, pointer.StringDeref(pip.Name, ""), tc.GetResourceGroup()) + Expect(err).NotTo(HaveOccurred()) + } }() By("Changing ExternalTrafficPolicy of the service to Local") @@ -1042,27 +1247,46 @@ var _ = Describe("Multi-ports service", Label(utils.TestSuiteLabelMultiPorts), f }) Expect(retryErr).NotTo(HaveOccurred()) - pipFrontendConfigID := getPIPFrontendConfigurationID(tc, publicIP, tc.GetResourceGroup(), "") - pipFrontendConfigIDSplit := strings.Split(pipFrontendConfigID, "/") - Expect(len(pipFrontendConfigIDSplit)).NotTo(Equal(0)) + ids := []string{} + for _, ip := range ips { + pipFrontendConfigID := getPIPFrontendConfigurationID(tc, ip, tc.GetResourceGroup(), "") + pipFrontendConfigIDSplit := strings.Split(pipFrontendConfigID, "/") + Expect(len(pipFrontendConfigIDSplit)).NotTo(Equal(0)) + ids = append(ids, pipFrontendConfigIDSplit[len(pipFrontendConfigIDSplit)-1]) + } var lb *network.LoadBalancer var targetProbes []*network.Probe + expectedTargetProbesCount := 1 + if tc.IPFamily == utils.DualStack { + expectedTargetProbesCount = 2 + } //wait for backend update + checkPort := func(port int32, targetProbes []*network.Probe) bool { + match := true + for _, targetProbe := range targetProbes { + if *(targetProbe.Port) != port { + match = false + break + } + } + return match + } err = wait.PollImmediate(5*time.Second, 2*time.Minute, func() (bool, error) { - lb = getAzureLoadBalancerFromPIP(tc, publicIP, tc.GetResourceGroup(), "") + lb = getAzureLoadBalancerFromPIP(tc, ips[0], tc.GetResourceGroup(), "") targetProbes = []*network.Probe{} for i := range *lb.LoadBalancerPropertiesFormat.Probes { probe := (*lb.LoadBalancerPropertiesFormat.Probes)[i] utils.Logf("One probe of LB is %q", *probe.Name) probeSplit := strings.Split(*probe.Name, "-") Expect(len(probeSplit)).NotTo(Equal(0)) - if pipFrontendConfigIDSplit[len(pipFrontendConfigIDSplit)-1] == probeSplit[0] { - targetProbes = append(targetProbes, &probe) + for _, id := range ids { + if id == probeSplit[0] { + targetProbes = append(targetProbes, &probe) + } } } - return len(targetProbes) == 1 && - *(targetProbes)[0].Port == service.Spec.HealthCheckNodePort, nil + return len(targetProbes) == expectedTargetProbesCount && checkPort(service.Spec.HealthCheckNodePort, targetProbes), nil }) Expect(err).NotTo(HaveOccurred()) @@ -1082,21 +1306,25 @@ var _ = Describe("Multi-ports service", Label(utils.TestSuiteLabelMultiPorts), f utils.Logf("Successfully updated LoadBalancer service " + serviceName + " in namespace " + ns.Name) //wait for backend update + expectedTargetProbesCount = 2 + if tc.IPFamily == utils.DualStack { + expectedTargetProbesCount = 4 + } err = wait.PollImmediate(5*time.Second, 2*time.Minute, func() (bool, error) { - lb := getAzureLoadBalancerFromPIP(tc, publicIP, tc.GetResourceGroup(), "") + lb := getAzureLoadBalancerFromPIP(tc, ips[0], tc.GetResourceGroup(), "") targetProbes = []*network.Probe{} for i := range *lb.LoadBalancerPropertiesFormat.Probes { probe := (*lb.LoadBalancerPropertiesFormat.Probes)[i] utils.Logf("One probe of LB is %q", *probe.Name) probeSplit := strings.Split(*probe.Name, "-") Expect(len(probeSplit)).NotTo(Equal(0)) - if pipFrontendConfigIDSplit[len(pipFrontendConfigIDSplit)-1] == probeSplit[0] { - targetProbes = append(targetProbes, &probe) + for _, id := range ids { + if id == probeSplit[0] { + targetProbes = append(targetProbes, &probe) + } } } - return len(targetProbes) == 2 && - *(targetProbes)[0].Port != nodeHealthCheckPort && - *(targetProbes)[1].Port != nodeHealthCheckPort, nil + return len(targetProbes) == expectedTargetProbesCount && checkPort(nodeHealthCheckPort, targetProbes), nil }) Expect(err).NotTo(HaveOccurred()) }) @@ -1192,7 +1420,7 @@ func getAzureLoadBalancerFromPIP(tc *utils.AzureTestClient, pip, pipResourceGrou return &lb } -func createAndExposeDefaultServiceWithAnnotation(cs clientset.Interface, serviceName, nsName string, labels, annotation map[string]string, ports []v1.ServicePort) string { +func createAndExposeDefaultServiceWithAnnotation(cs clientset.Interface, ipFamily utils.IPFamily, serviceName, nsName string, labels, annotation map[string]string, ports []v1.ServicePort) []string { utils.Logf("Creating service " + serviceName + " in namespace " + nsName) service := utils.CreateLoadBalancerServiceManifest(serviceName, annotation, labels, nsName, ports) _, err := cs.CoreV1().Services(nsName).Create(context.TODO(), service, metav1.CreateOptions{}) @@ -1201,10 +1429,9 @@ func createAndExposeDefaultServiceWithAnnotation(cs clientset.Interface, service //wait and get service's IP Address utils.Logf("Waiting service to expose...") - publicIP, err := utils.WaitServiceExposureAndValidateConnectivity(cs, nsName, serviceName, "") + ips, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ipFamily, nsName, serviceName, []string{}) Expect(err).NotTo(HaveOccurred()) - - return publicIP + return ips } // createNginxDeploymentManifest returns a default deployment @@ -1277,7 +1504,9 @@ func validateLoadBalancerBackendPools(tc *utils.AzureTestClient, vmssName string //wait and get service's public IP Address By("Waiting for service exposure") - publicIP, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns, serviceName, "") + ips, err := utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns, serviceName, []string{}) + Expect(len(ips)).NotTo(BeZero()) + publicIP := ips[0] Expect(err).NotTo(HaveOccurred()) // Invoking azure network client to get list of public IP Addresses diff --git a/tests/e2e/network/standard_lb.go b/tests/e2e/network/standard_lb.go index 5861566fbb..ca5a78a68f 100644 --- a/tests/e2e/network/standard_lb.go +++ b/tests/e2e/network/standard_lb.go @@ -90,7 +90,9 @@ var _ = Describe("[StandardLoadBalancer] Standard load balancer", func() { } rgName := tc.GetResourceGroup() - publicIP := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, map[string]string{}, ports) + ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, map[string]string{}, ports) + Expect(len(ips)).NotTo(BeZero()) + publicIP := ips[0] lb := getAzureLoadBalancerFromPIP(tc, publicIP, rgName, rgName) nodeList, err := utils.GetAgentNodes(cs) @@ -170,7 +172,9 @@ var _ = Describe("[StandardLoadBalancer] Standard load balancer", func() { } rgName := tc.GetResourceGroup() - publicIP := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, map[string]string{}, ports) + ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, map[string]string{}, ports) + Expect(len(ips)).NotTo(BeZero()) + publicIP := ips[0] lb := getAzureLoadBalancerFromPIP(tc, publicIP, rgName, rgName) Expect(lb.OutboundRules).NotTo(BeNil()) diff --git a/tests/e2e/utils/azure_test_client.go b/tests/e2e/utils/azure_test_client.go index fecffae835..58ffc24ad6 100644 --- a/tests/e2e/utils/azure_test_client.go +++ b/tests/e2e/utils/azure_test_client.go @@ -146,12 +146,12 @@ func (tc *AzureTestClient) CreateSecurityGroupsClient() *aznetwork.SecurityGroup return &aznetwork.SecurityGroupsClient{BaseClient: tc.networkClient} } -// createPublicIPAddressesClient generates virtual network client with the same baseclient as azure test client +// createPublicIPAddressesClient generates public IP addresses client with the same baseclient as azure test client func (tc *AzureTestClient) createPublicIPAddressesClient() *aznetwork.PublicIPAddressesClient { return &aznetwork.PublicIPAddressesClient{BaseClient: tc.networkClient} } -// createPublicIPPrefixesClient generates virtual network client with the same baseclient as azure test client +// createPublicIPPrefixesClient generates public IP prefixes client with the same baseclient as azure test client func (tc *AzureTestClient) createPublicIPPrefixesClient() *aznetwork.PublicIPPrefixesClient { return &aznetwork.PublicIPPrefixesClient{BaseClient: tc.networkClient} } diff --git a/tests/e2e/utils/network_utils.go b/tests/e2e/utils/network_utils.go index a1b7c0d87b..05cc3a45c4 100644 --- a/tests/e2e/utils/network_utils.go +++ b/tests/e2e/utils/network_utils.go @@ -38,9 +38,17 @@ import ( type IPFamily string var ( - IPv4 IPFamily = "ipv4" - IPv6 IPFamily = "ipv6" - DualStack IPFamily = "dualstack" + IPv4 IPFamily = "IPv4" + IPv6 IPFamily = "IPv6" + DualStack IPFamily = "DualStack" + + Suffixes = map[bool]string{ + false: "-IPv4", + true: "-IPv6", + } + + // TODO: After dual-stack implementation finished, update here. + DualstackSupported = true ) // getVirtualNetworkList returns the list of virtual networks in the cluster resource group. @@ -53,6 +61,7 @@ func (azureTestClient *AzureTestClient) getVirtualNetworkList() (result aznetwor if !IsRetryableAPIError(err) { return false, err } + Logf("error when listing virtual network list: %w", err) return false, nil } return true, nil @@ -81,11 +90,11 @@ func (azureTestClient *AzureTestClient) GetClusterVirtualNetwork() (virtualNetwo } // CreateSubnet creates a new subnet in the specified virtual network. -func (azureTestClient *AzureTestClient) CreateSubnet(vnet aznetwork.VirtualNetwork, subnetName *string, prefix *string, waitUntilComplete bool) (network.Subnet, error) { - Logf("creating a new subnet %s, %s", *subnetName, *prefix) +func (azureTestClient *AzureTestClient) CreateSubnet(vnet aznetwork.VirtualNetwork, subnetName *string, prefixes *[]string, waitUntilComplete bool) (network.Subnet, error) { + Logf("creating a new subnet %s, %v", *subnetName, *prefixes) subnetParameter := (*vnet.Subnets)[0] subnetParameter.Name = subnetName - subnetParameter.AddressPrefix = prefix + subnetParameter.AddressPrefixes = prefixes subnetsClient := azureTestClient.createSubnetsClient() _, err := subnetsClient.CreateOrUpdate(context.Background(), azureTestClient.GetResourceGroup(), *vnet.Name, *subnetName, subnetParameter) var subnet network.Subnet @@ -132,15 +141,15 @@ func (azureTestClient *AzureTestClient) DeleteSubnet(vnetName string, subnetName }) } -// GetNextSubnetCIDR obtains a new ip address which has no overlap with existing subnets. -func GetNextSubnetCIDR(vnet aznetwork.VirtualNetwork, ipFamily IPFamily) (*net.IPNet, error) { +// GetNextSubnetCIDRs obtains a new ip address which has no overlap with existing subnets. +func GetNextSubnetCIDRs(vnet aznetwork.VirtualNetwork, ipFamily IPFamily) ([]*net.IPNet, error) { if len(*vnet.AddressSpace.AddressPrefixes) == 0 { return nil, fmt.Errorf("vNet has no prefix") } // Because of Azure vNet limitation, underlying vNet is dual-stack for // those single stack IPv6 clusters. Pods and Services are single stack // IPv6 while Nodes and routes are dual-stack. - vnetCIDR := (*vnet.AddressSpace.AddressPrefixes)[0] + vnetCIDRs := []string{} if ipFamily == IPv6 { for i := range *vnet.AddressSpace.AddressPrefixes { addrPrefix := (*vnet.AddressSpace.AddressPrefixes)[i] @@ -149,10 +158,17 @@ func GetNextSubnetCIDR(vnet aznetwork.VirtualNetwork, ipFamily IPFamily) (*net.I return nil, fmt.Errorf("failed to parse address prefix CIDR: %w", err) } if ip.To4() == nil { - vnetCIDR = addrPrefix + vnetCIDRs = append(vnetCIDRs, addrPrefix) break } } + } else if ipFamily == IPv4 { + vnetCIDRs = append(vnetCIDRs, (*vnet.AddressSpace.AddressPrefixes)[0]) + } else { + for i := range *vnet.AddressSpace.AddressPrefixes { + addrPrefix := (*vnet.AddressSpace.AddressPrefixes)[i] + vnetCIDRs = append(vnetCIDRs, addrPrefix) + } } var existSubnets []string @@ -166,7 +182,15 @@ func GetNextSubnetCIDR(vnet aznetwork.VirtualNetwork, ipFamily IPFamily) (*net.I } existSubnets = append(existSubnets, *subnet.AddressPrefixes...) } - return getNextSubnet(vnetCIDR, existSubnets) + var nextSubnets []*net.IPNet + for _, vnetCIDR := range vnetCIDRs { + nextSubnet, err := getNextSubnet(vnetCIDR, existSubnets) + if err != nil { + return nextSubnets, err + } + nextSubnets = append(nextSubnets, nextSubnet) + } + return nextSubnets, nil } // isCIDRIPv6 checks if the provided CIDR is an IPv6 one. @@ -231,7 +255,7 @@ func CreateLoadBalancerServiceManifest(name string, annotation map[string]string Selector: labels, Ports: ports, Type: "LoadBalancer", - IPFamilyPolicy: &ipFamilyPreferDS, + IPFamilyPolicy: &ipFamilyPreferDS, // TODO: This should be updated if there're single stack Service tests in dual-stack setup. }, } } @@ -394,34 +418,34 @@ func WaitGetPIP(azureTestClient *AzureTestClient, ipName string) (pip aznetwork. return } -// SelectAvailablePrivateIP selects a private IP address in Azure subnet. -func SelectAvailablePrivateIP(tc *AzureTestClient) (string, error) { +// SelectAvailablePrivateIPs selects private IP addresses in Azure subnet. +func SelectAvailablePrivateIPs(tc *AzureTestClient, ipFamily IPFamily) ([]string, error) { vNet, err := tc.GetClusterVirtualNetwork() vNetClient := tc.createVirtualNetworksClient() if err != nil { - return "", err + return []string{}, err } if vNet.Subnets == nil || len(*vNet.Subnets) == 0 { - return "", fmt.Errorf("failed to find a subnet in vNet %s", pointer.StringDeref(vNet.Name, "")) + return []string{}, fmt.Errorf("failed to find a subnet in vNet %s", pointer.StringDeref(vNet.Name, "")) } - subnet := pointer.StringDeref((*vNet.Subnets)[0].AddressPrefix, "") + subnets := []string{pointer.StringDeref((*vNet.Subnets)[0].AddressPrefix, "")} if len(*vNet.Subnets) > 1 { for _, sn := range *vNet.Subnets { // if there is more than one subnet, select the first one we find. if !strings.Contains(*sn.Name, "controlplane") && !strings.Contains(*sn.Name, "control-plane") { if tc.IPFamily == DualStack { - subnet = (*sn.AddressPrefixes)[0] + subnets = []string{(*sn.AddressPrefixes)[0], (*sn.AddressPrefixes)[1]} } else if tc.IPFamily == IPv4 { - subnet = *sn.AddressPrefix + subnets = []string{*sn.AddressPrefix} } else { for i := range *sn.AddressPrefixes { addrPrefix := (*sn.AddressPrefixes)[i] isIPv6, err := isCIDRIPv6(addrPrefix) if err != nil { - return "", err + return []string{}, err } if isIPv6 { - subnet = addrPrefix + subnets = []string{addrPrefix} break } } @@ -430,30 +454,43 @@ func SelectAvailablePrivateIP(tc *AzureTestClient) (string, error) { } } } - ip, _, err := net.ParseCIDR(subnet) - if err != nil { - return "", fmt.Errorf("failed to parse subnet CIDR in vNet %s: %w", pointer.StringDeref(vNet.Name, ""), err) - } - baseIP := ip.To4() - pos := 3 - if tc.IPFamily == IPv6 { - baseIP = ip.To16() - pos = 15 - } - for i := 0; i <= 254; i++ { - baseIP[pos]++ - IP := baseIP.String() - ret, err := vNetClient.CheckIPAddressAvailability(context.Background(), tc.GetResourceGroup(), pointer.StringDeref(vNet.Name, ""), IP) + privateIPs := []string{} + for _, subnet := range subnets { + Logf("Handling subnet %s", subnet) + ip, _, err := net.ParseCIDR(subnet) if err != nil { - // just ignore - continue + return []string{}, fmt.Errorf("failed to parse subnet CIDR in vNet %s: %w", pointer.StringDeref(vNet.Name, ""), err) } - if ret.Available != nil && *ret.Available { - return IP, nil + + baseIP := ip.To4() + pos := 3 + if ip.To4() == nil { + baseIP = ip.To16() + pos = 15 } + for i := 0; i <= 254; i++ { + baseIP[pos]++ + IP := baseIP.String() + ret, err := vNetClient.CheckIPAddressAvailability(context.Background(), tc.GetResourceGroup(), pointer.StringDeref(vNet.Name, ""), IP) + if err != nil { + // just ignore + continue + } + if ret.Available != nil && *ret.Available { + privateIPs = append(privateIPs, IP) + break + } + } + } + expectedPrivateIPCount := 1 + if tc.IPFamily == DualStack { + expectedPrivateIPCount = 2 } - return "", fmt.Errorf("find no availabePrivateIP in subnet CIDR %s", subnet) + if len(privateIPs) != expectedPrivateIPCount { + return []string{}, fmt.Errorf("failed to find all availabePrivateIPs in subnet CIDRs %v, got privateIPs %v", subnets, privateIPs) + } + return privateIPs, nil } // GetPublicIPFromAddress finds public ip according to ip address @@ -604,3 +641,22 @@ func GetClusterServiceIPFamily() (IPFamily, error) { } return DualStack, nil } + +func IfIPFamiliesEnabled(ipFamily IPFamily) (v4Enabled bool, v6Enabled bool) { + if ipFamily == DualStack || ipFamily == IPv4 { + v4Enabled = true + } + if ipFamily == DualStack || ipFamily == IPv6 { + v6Enabled = true + } + return +} + +// GetNameWithSuffix returns resource name with IP family suffix. +// After dual-stack implementation is finished, this function returns name + suffix for all IP families. +func GetNameWithSuffix(name, suffix string) string { + if DualstackSupported { + return name + suffix + } + return name +} diff --git a/tests/e2e/utils/service_utils.go b/tests/e2e/utils/service_utils.go index ba413b6e50..459c3b1d2c 100644 --- a/tests/e2e/utils/service_utils.go +++ b/tests/e2e/utils/service_utils.go @@ -71,40 +71,48 @@ func GetServiceDomainName(prefix string) (ret string) { return } -// WaitServiceExposureAndGetIP returns IP of the Service. -func WaitServiceExposureAndGetIP(cs clientset.Interface, namespace string, name string) (string, error) { +// WaitServiceExposureAndGetIPs returns IPs of the Service. +func WaitServiceExposureAndGetIPs(cs clientset.Interface, namespace string, name string) ([]string, error) { var service *v1.Service var err error - var ip string + ips := []string{} - service, err = WaitServiceExposure(cs, namespace, name, "") + service, err = WaitServiceExposure(cs, namespace, name, []string{}) if err != nil { - return "", err + return ips, err } if service == nil { - return "", errors.New("the service is nil") + return ips, errors.New("the service is nil") } - ip = service.Status.LoadBalancer.Ingress[0].IP + for _, ingress := range service.Status.LoadBalancer.Ingress { + ips = append(ips, ingress.IP) + } - return ip, nil + return ips, nil } -// WaitServiceExposureAndValidateConnectivity returns ip of the service and check the connectivity if it is a public IP -func WaitServiceExposureAndValidateConnectivity(cs clientset.Interface, namespace string, name string, targetIP string) (string, error) { +// WaitServiceExposureAndValidateConnectivity returns IPs of the service and check the connectivity if they are public IPs. +func WaitServiceExposureAndValidateConnectivity(cs clientset.Interface, ipFamily IPFamily, namespace string, name string, targetIPs []string) ([]string, error) { var service *v1.Service var err error - var ip string + var ips []string - service, err = WaitServiceExposure(cs, namespace, name, targetIP) + service, err = WaitServiceExposure(cs, namespace, name, targetIPs) if err != nil { - return "", err + return ips, err } if service == nil { - return "", errors.New("the service is nil") + return ips, errors.New("the service is nil") } - ip = service.Status.LoadBalancer.Ingress[0].IP + if len(service.Status.LoadBalancer.Ingress) == 0 { + return ips, errors.New("service.Status.LoadBalancer.Ingress is empty") + } + ips = append(ips, service.Status.LoadBalancer.Ingress[0].IP) + if len(service.Status.LoadBalancer.Ingress) > 1 { + ips = append(ips, service.Status.LoadBalancer.Ingress[1].IP) + } // Create host exec Pod result, err := CreateHostExecPod(cs, namespace, ExecAgnhostPod) @@ -115,26 +123,28 @@ func WaitServiceExposureAndValidateConnectivity(cs clientset.Interface, namespac } }() if !result || err != nil { - return "", fmt.Errorf("failed to create ExecAgnhostPod, result: %v, error: %w", result, err) + return ips, fmt.Errorf("failed to create ExecAgnhostPod, result: %v, error: %w", result, err) } // TODO: Check if other WaitServiceExposureAndValidateConnectivity() callers with internal Service // should test connectivity as well. for _, port := range service.Spec.Ports { - Logf("checking the connectivity of addr %s:%d with protocol %v", ip, int(port.Port), port.Protocol) - if err := ValidateServiceConnectivity(namespace, ExecAgnhostPod, ip, int(port.Port), port.Protocol); err != nil { - return ip, err + for _, ip := range ips { + Logf("checking the connectivity of address %s %d with protocol %v", ip, int(port.Port), port.Protocol) + if err := ValidateServiceConnectivity(namespace, ExecAgnhostPod, ip, int(port.Port), port.Protocol); err != nil { + return ips, err + } } } - return ip, nil + return ips, nil } // WaitServiceExposure waits for the exposure of the external IP of the service -func WaitServiceExposure(cs clientset.Interface, namespace string, name string, targetIP string) (*v1.Service, error) { +func WaitServiceExposure(cs clientset.Interface, namespace string, name string, targetIPs []string) (*v1.Service, error) { var service *v1.Service var err error - var ip string + var externalIPs []string timeout := serviceTimeout if skuEnv := os.Getenv(LoadBalancerSkuEnv); skuEnv != "" { @@ -153,16 +163,22 @@ func WaitServiceExposure(cs clientset.Interface, namespace string, name string, } IngressList := service.Status.LoadBalancer.Ingress - if len(IngressList) == 0 { - err = fmt.Errorf("Cannot find Ingress in limited time") + if (len(targetIPs) != 0 && len(IngressList) < len(targetIPs)) || + (len(targetIPs) == 0 && len(IngressList) == 0) { + err = fmt.Errorf("cannot find Ingress in limited time") Logf("Fail to find ingress, retry in 10 seconds") return false, nil } - ip = service.Status.LoadBalancer.Ingress[0].IP - if targetIP != "" && !strings.EqualFold(ip, targetIP) { - Logf("expected IP is %s, current IP is %s, retry in 10 seconds", targetIP, ip) - return false, nil + externalIPs = []string{} + for _, ingress := range IngressList { + externalIPs = append(externalIPs, ingress.IP) + } + if len(targetIPs) != 0 { + if !CompareStrings(externalIPs, targetIPs) { + Logf("expected IPs are %v, current IPs are %v, retry in 10 seconds", targetIPs, externalIPs) + return false, nil + } } return true, nil @@ -170,7 +186,7 @@ func WaitServiceExposure(cs clientset.Interface, namespace string, name string, return nil, err } - Logf("Exposure successfully, get external ip: %s", ip) + Logf("Exposure successfully, get external ip: %s", externalIPs) return service, nil } diff --git a/tests/e2e/utils/utils.go b/tests/e2e/utils/utils.go index c2d558cd9c..7bcaabff28 100644 --- a/tests/e2e/utils/utils.go +++ b/tests/e2e/utils/utils.go @@ -27,6 +27,7 @@ import ( apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilnet "k8s.io/apimachinery/pkg/util/net" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -188,3 +189,9 @@ func StringInSlice(s string, list []string) bool { } return false } + +func CompareStrings(s0, s1 []string) bool { + ss0 := sets.NewString(s0...) + ss1 := sets.NewString(s1...) + return ss0.Equal(ss1) +}