From e052e68940cf8b3464cc36e1c86604125b26a232 Mon Sep 17 00:00:00 2001 From: Zhecheng Li Date: Tue, 27 Sep 2022 19:07:04 +0800 Subject: [PATCH] Deprecate LoadBalancerIP with Servie LB IP annotation * Add service.beta.kubernetes.io/azure-load-balancer-ipv4 and service.beta.kubernetes.io/azure-load-balancer-ipv6 * Add an e2e test * Support retrieve Service CIDR in e2e framework Signed-off-by: Zhecheng Li --- pkg/consts/consts.go | 10 ++ pkg/provider/azure_loadbalancer.go | 50 ++++++-- pkg/provider/azure_loadbalancer_test.go | 62 +++++----- pkg/provider/azure_standard.go | 2 +- pkg/provider/azure_standard_test.go | 34 +++--- pkg/provider/azure_test.go | 123 ++++++++++---------- site/content/en/topics/pls-integration.md | 4 +- site/content/en/topics/shared-ip.md | 6 +- tests/e2e/network/ensureloadbalancer.go | 55 +++++---- tests/e2e/network/network_security_group.go | 4 +- tests/e2e/network/private_link_service.go | 2 +- tests/e2e/network/service_annotations.go | 55 ++++++++- tests/e2e/utils/kubectl.go | 4 +- tests/e2e/utils/network_utils.go | 64 ++++++++++ 14 files changed, 319 insertions(+), 156 deletions(-) diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index 4a5a975115..8b33c53fac 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -187,6 +187,16 @@ const ( BackoffJitterDefault = 1.0 ) +// LB variables for dual-stack +var ( + // Service.Spec.LoadBalancerIP has been deprecated and may be removed in a future release. Those two annotations are introduced as alternatives to set IPv4/IPv6 LoadBalancer IPs. + // Refer https://github.com/kubernetes/api/blob/3638040e4063e0f889c129220cd386497f328276/core/v1/types.go#L4459-L4468 for more details. + ServiceAnnotationLoadBalancerIPDualStack = map[bool]string{ + false: "service.beta.kubernetes.io/azure-load-balancer-ipv4", + true: "service.beta.kubernetes.io/azure-load-balancer-ipv6", + } +) + // load balancer const ( // PreConfiguredBackendPoolLoadBalancerTypesNone means that the load balancers are not pre-configured diff --git a/pkg/provider/azure_loadbalancer.go b/pkg/provider/azure_loadbalancer.go index 89c033f502..3b85668b43 100644 --- a/pkg/provider/azure_loadbalancer.go +++ b/pkg/provider/azure_loadbalancer.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "math" + "net" "reflect" "sort" "strconv" @@ -46,6 +47,36 @@ import ( "sigs.k8s.io/cloud-provider-azure/pkg/retry" ) +// getServiceLoadBalancerIP retrieves LB IP from IPv4 annotation, then IPv6 annotation, then service.Spec.LoadBalancerIP. +// TODO: Dual-stack support is not implemented. +func getServiceLoadBalancerIP(service *v1.Service) string { + if service == nil { + return "" + } + + if ip, ok := service.Annotations[consts.ServiceAnnotationLoadBalancerIPDualStack[false]]; ok && ip != "" { + return ip + } + if ip, ok := service.Annotations[consts.ServiceAnnotationLoadBalancerIPDualStack[true]]; ok && ip != "" { + return ip + } + + // Retrieve LB IP from service.Spec.LoadBalancerIP (will be deprecated) + return service.Spec.LoadBalancerIP +} + +// setServiceLoadBalancerIP sets LB IP to a Service +func setServiceLoadBalancerIP(service *v1.Service, ip string) { + if service.Annotations == nil { + service.Annotations = map[string]string{} + } + if net.ParseIP(ip).To4() != nil { + service.Annotations[consts.ServiceAnnotationLoadBalancerIPDualStack[false]] = ip + return + } + service.Annotations[consts.ServiceAnnotationLoadBalancerIPDualStack[true]] = ip +} + // GetLoadBalancer returns whether the specified load balancer and its components exist, and // if so, what its status is. func (az *Cloud) GetLoadBalancer(ctx context.Context, clusterName string, service *v1.Service) (status *v1.LoadBalancerStatus, exists bool, err error) { @@ -863,7 +894,7 @@ func (az *Cloud) determinePublicIPName(clusterName string, service *v1.Service, } pipResourceGroup := az.getPublicIPAddressResourceGroup(service) - loadBalancerIP := service.Spec.LoadBalancerIP + loadBalancerIP := getServiceLoadBalancerIP(service) // Assume that the service without loadBalancerIP set is a primary service. // If a secondary service doesn't set the loadBalancerIP, it is not allowed to share the IP. @@ -921,14 +952,15 @@ func flipServiceInternalAnnotation(service *v1.Service) *v1.Service { func updateServiceLoadBalancerIP(service *v1.Service, serviceIP string) *v1.Service { copyService := service.DeepCopy() if len(serviceIP) > 0 && copyService != nil { - copyService.Spec.LoadBalancerIP = serviceIP + setServiceLoadBalancerIP(copyService, serviceIP) } return copyService } func (az *Cloud) findServiceIPAddress(ctx context.Context, clusterName string, service *v1.Service) (string, error) { - if len(service.Spec.LoadBalancerIP) > 0 { - return service.Spec.LoadBalancerIP, nil + lbIP := getServiceLoadBalancerIP(service) + if len(lbIP) > 0 { + return lbIP, nil } if len(service.Status.LoadBalancer.Ingress) > 0 && len(service.Status.LoadBalancer.Ingress[0].IP) > 0 { @@ -1318,7 +1350,7 @@ func (az *Cloud) isFrontendIPChanged(clusterName string, config network.Frontend if !strings.EqualFold(to.String(config.Name), lbFrontendIPConfigName) { return false, nil } - loadBalancerIP := service.Spec.LoadBalancerIP + loadBalancerIP := getServiceLoadBalancerIP(service) isInternal := requiresInternalLoadBalancer(service) if isInternal { // Judge subnet @@ -1837,7 +1869,7 @@ func (az *Cloud) reconcileFrontendIPConfigs(clusterName string, service *v1.Serv configProperties.PrivateIPAddressVersion = network.IPVersionIPv6 } - loadBalancerIP := service.Spec.LoadBalancerIP + loadBalancerIP := getServiceLoadBalancerIP(service) if loadBalancerIP != "" { configProperties.PrivateIPAllocationMethod = network.IPAllocationMethodStatic configProperties.PrivateIPAddress = &loadBalancerIP @@ -3304,7 +3336,7 @@ func getServiceTags(service *v1.Service) []string { // The pip is user-created if and only if there is no service tags. // The service owns the pip if: // 1. The serviceName is included in the service tags of a system-created pip. -// 2. The service.Spec.LoadBalancerIP matches the IP address of a user-created pip. +// 2. The service LoadBalancerIP matches the IP address of a user-created pip. func serviceOwnsPublicIP(service *v1.Service, pip *network.PublicIPAddress, clusterName string) (bool, bool) { if service == nil || pip == nil { klog.Warningf("serviceOwnsPublicIP: nil service or public IP") @@ -3324,7 +3356,7 @@ func serviceOwnsPublicIP(service *v1.Service, pip *network.PublicIPAddress, clus // if there is no service tag on the pip, it is user-created pip if serviceTag == "" { - return strings.EqualFold(to.String(pip.IPAddress), service.Spec.LoadBalancerIP), true + return strings.EqualFold(to.String(pip.IPAddress), getServiceLoadBalancerIP(service)), true } // if there is service tag on the pip, it is system-created pip @@ -3342,7 +3374,7 @@ func serviceOwnsPublicIP(service *v1.Service, pip *network.PublicIPAddress, clus } else { // if the service is not included in te tags of the system-created pip, check the ip address // this could happen for secondary services - return strings.EqualFold(to.String(pip.IPAddress), service.Spec.LoadBalancerIP), false + return strings.EqualFold(to.String(pip.IPAddress), getServiceLoadBalancerIP(service)), false } } diff --git a/pkg/provider/azure_loadbalancer_test.go b/pkg/provider/azure_loadbalancer_test.go index 5b1dae9b45..f1282f78ec 100644 --- a/pkg/provider/azure_loadbalancer_test.go +++ b/pkg/provider/azure_loadbalancer_test.go @@ -961,7 +961,7 @@ func TestServiceOwnsPublicIP(t *testing.T) { t.Run(c.desc, func(t *testing.T) { service := getTestService(c.serviceName, v1.ProtocolTCP, nil, false, 80) if c.serviceLBIP != "" { - service.Spec.LoadBalancerIP = c.serviceLBIP + setServiceLoadBalancerIP(&service, c.serviceLBIP) } owns, isUserAssignedPIP := serviceOwnsPublicIP(&service, c.pip, c.clusterName) assert.Equal(t, c.expectedOwns, owns, "TestCase[%d]: %s", i, c.desc) @@ -2008,34 +2008,36 @@ func TestIsFrontendIPChanged(t *testing.T) { }, } - for i, test := range testCases { - az := GetTestCloud(ctrl) - mockSubnetsClient := az.SubnetsClient.(*mocksubnetclient.MockInterface) - mockSubnetsClient.EXPECT().Get(gomock.Any(), "rg", "vnet", "testSubnet", "").Return(test.existingSubnet, nil).AnyTimes() - mockSubnetsClient.EXPECT().CreateOrUpdate(gomock.Any(), "rg", "vnet", "testSubnet", test.existingSubnet).Return(nil) - err := az.SubnetsClient.CreateOrUpdate(context.TODO(), "rg", "vnet", "testSubnet", test.existingSubnet) - if err != nil { - t.Fatalf("TestCase[%d] meets unexpected error: %v", i, err) - } - - mockPIPsClient := az.PublicIPAddressesClient.(*mockpublicipclient.MockInterface) - mockPIPsClient.EXPECT().CreateOrUpdate(gomock.Any(), "rg", gomock.Any(), gomock.Any()).Return(nil).AnyTimes() - for _, existingPIP := range test.existingPIPs { - mockPIPsClient.EXPECT().Get(gomock.Any(), "rg", *existingPIP.Name, gomock.Any()).Return(existingPIP, nil).AnyTimes() - err := az.PublicIPAddressesClient.CreateOrUpdate(context.TODO(), "rg", *existingPIP.Name, existingPIP) + for _, test := range testCases { + t.Run(test.desc, func(t *testing.T) { + az := GetTestCloud(ctrl) + mockSubnetsClient := az.SubnetsClient.(*mocksubnetclient.MockInterface) + mockSubnetsClient.EXPECT().Get(gomock.Any(), "rg", "vnet", "testSubnet", "").Return(test.existingSubnet, nil).AnyTimes() + mockSubnetsClient.EXPECT().CreateOrUpdate(gomock.Any(), "rg", "vnet", "testSubnet", test.existingSubnet).Return(nil) + err := az.SubnetsClient.CreateOrUpdate(context.TODO(), "rg", "vnet", "testSubnet", test.existingSubnet) if err != nil { - t.Fatalf("TestCase[%d] meets unexpected error: %v", i, err) + t.Fatal(err) } - } - test.service.Spec.LoadBalancerIP = test.loadBalancerIP - test.service.Annotations[consts.ServiceAnnotationLoadBalancerInternalSubnet] = test.annotations - flag, rerr := az.isFrontendIPChanged("testCluster", test.config, - &test.service, test.lbFrontendIPConfigName, &test.existingPIPs) - if rerr != nil { - fmt.Println(rerr.Error()) - } - assert.Equal(t, test.expectedFlag, flag, "TestCase[%d]: %s", i, test.desc) - assert.Equal(t, test.expectedError, rerr != nil, "TestCase[%d]: %s", i, test.desc) + + mockPIPsClient := az.PublicIPAddressesClient.(*mockpublicipclient.MockInterface) + mockPIPsClient.EXPECT().CreateOrUpdate(gomock.Any(), "rg", gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + for _, existingPIP := range test.existingPIPs { + mockPIPsClient.EXPECT().Get(gomock.Any(), "rg", *existingPIP.Name, gomock.Any()).Return(existingPIP, nil).AnyTimes() + err := az.PublicIPAddressesClient.CreateOrUpdate(context.TODO(), "rg", *existingPIP.Name, existingPIP) + if err != nil { + t.Fatal(err) + } + } + setServiceLoadBalancerIP(&test.service, test.loadBalancerIP) + test.service.Annotations[consts.ServiceAnnotationLoadBalancerInternalSubnet] = test.annotations + flag, rerr := az.isFrontendIPChanged("testCluster", test.config, + &test.service, test.lbFrontendIPConfigName, &test.existingPIPs) + if rerr != nil { + fmt.Println(rerr.Error()) + } + assert.Equal(t, test.expectedFlag, flag) + assert.Equal(t, test.expectedError, rerr != nil) + }) } } @@ -2082,7 +2084,7 @@ func TestDeterminePublicIPName(t *testing.T) { for i, test := range testCases { az := GetTestCloud(ctrl) service := getTestService("test1", v1.ProtocolTCP, nil, false, 80) - service.Spec.LoadBalancerIP = test.loadBalancerIP + setServiceLoadBalancerIP(&service, test.loadBalancerIP) mockPIPsClient := az.PublicIPAddressesClient.(*mockpublicipclient.MockInterface) mockPIPsClient.EXPECT().List(gomock.Any(), "rg").Return(test.existingPIPs, nil).MaxTimes(1) @@ -3055,7 +3057,7 @@ func TestReconcileLoadBalancer(t *testing.T) { clusterResources, expectedInterfaces, expectedVirtualMachines := getClusterResources(az, 3, 3) setMockEnv(az, ctrl, expectedInterfaces, expectedVirtualMachines, 1) - test.service.Spec.LoadBalancerIP = "1.2.3.4" + setServiceLoadBalancerIP(&test.service, "1.2.3.4") err := az.PublicIPAddressesClient.CreateOrUpdate(context.TODO(), "rg", "pipName", network.PublicIPAddress{ Name: to.StringPtr("pipName"), @@ -4623,7 +4625,7 @@ func TestUnbindServiceFromPIP(t *testing.T) { } serviceName := "ns2/svc2" service := getTestService(serviceName, v1.ProtocolTCP, nil, false, 80) - service.Spec.LoadBalancerIP = "1.2.3.4" + setServiceLoadBalancerIP(&service, "1.2.3.4") expectedTags := []map[string]*string{ nil, {consts.ServiceTagKey: to.StringPtr("")}, diff --git a/pkg/provider/azure_standard.go b/pkg/provider/azure_standard.go index 4b47879567..5802eb68f6 100644 --- a/pkg/provider/azure_standard.go +++ b/pkg/provider/azure_standard.go @@ -352,7 +352,7 @@ func (az *Cloud) serviceOwnsFrontendIP(fip network.FrontendIPConfiguration, serv return true, isPrimaryService, nil } - loadBalancerIP := service.Spec.LoadBalancerIP + loadBalancerIP := getServiceLoadBalancerIP(service) if loadBalancerIP == "" { // it is a must that the secondary services set the loadBalancer IP return false, isPrimaryService, nil diff --git a/pkg/provider/azure_standard_test.go b/pkg/provider/azure_standard_test.go index 4d4316b126..6901e3214c 100644 --- a/pkg/provider/azure_standard_test.go +++ b/pkg/provider/azure_standard_test.go @@ -1684,10 +1684,8 @@ func TestServiceOwnsFrontendIP(t *testing.T) { }, service: &v1.Service{ ObjectMeta: meta.ObjectMeta{ - UID: types.UID("secondary"), - }, - Spec: v1.ServiceSpec{ - LoadBalancerIP: "1.2.3.4", + UID: types.UID("secondary"), + Annotations: map[string]string{consts.ServiceAnnotationLoadBalancerIPDualStack[false]: "1.2.3.4"}, }, }, }, @@ -1712,10 +1710,8 @@ func TestServiceOwnsFrontendIP(t *testing.T) { }, service: &v1.Service{ ObjectMeta: meta.ObjectMeta{ - UID: types.UID("secondary"), - }, - Spec: v1.ServiceSpec{ - LoadBalancerIP: "4.3.2.1", + UID: types.UID("secondary"), + Annotations: map[string]string{consts.ServiceAnnotationLoadBalancerIPDualStack[false]: "4.3.2.1"}, }, }, }, @@ -1739,10 +1735,8 @@ func TestServiceOwnsFrontendIP(t *testing.T) { }, service: &v1.Service{ ObjectMeta: meta.ObjectMeta{ - UID: types.UID("secondary"), - }, - Spec: v1.ServiceSpec{ - LoadBalancerIP: "4.3.2.1", + UID: types.UID("secondary"), + Annotations: map[string]string{consts.ServiceAnnotationLoadBalancerIPDualStack[false]: "4.3.2.1"}, }, }, }, @@ -1766,10 +1760,8 @@ func TestServiceOwnsFrontendIP(t *testing.T) { }, service: &v1.Service{ ObjectMeta: meta.ObjectMeta{ - UID: types.UID("secondary"), - }, - Spec: v1.ServiceSpec{ - LoadBalancerIP: "4.3.2.1", + UID: types.UID("secondary"), + Annotations: map[string]string{consts.ServiceAnnotationLoadBalancerIPDualStack[false]: "4.3.2.1"}, }, }, isOwned: true, @@ -1784,11 +1776,11 @@ func TestServiceOwnsFrontendIP(t *testing.T) { }, service: &v1.Service{ ObjectMeta: meta.ObjectMeta{ - UID: types.UID("secondary"), - Annotations: map[string]string{consts.ServiceAnnotationLoadBalancerInternal: "true"}, - }, - Spec: v1.ServiceSpec{ - LoadBalancerIP: "4.3.2.1", + UID: types.UID("secondary"), + Annotations: map[string]string{ + consts.ServiceAnnotationLoadBalancerInternal: "true", + consts.ServiceAnnotationLoadBalancerIPDualStack[false]: "4.3.2.1", + }, }, }, isOwned: true, diff --git a/pkg/provider/azure_test.go b/pkg/provider/azure_test.go index dcb31dc866..391e01b4bf 100644 --- a/pkg/provider/azure_test.go +++ b/pkg/provider/azure_test.go @@ -716,7 +716,7 @@ func TestReconcileSecurityGroupFromAnyDestinationAddressPrefixToLoadBalancerIP(t az := GetTestCloud(ctrl) svc1 := getTestService("serviceea", v1.ProtocolTCP, nil, false, 80) - svc1.Spec.LoadBalancerIP = "192.168.0.0" + setServiceLoadBalancerIP(&svc1, "192.168.0.0") sg := getTestSecurityGroup(az) setMockSecurityGroup(az, ctrl, sg) @@ -726,7 +726,7 @@ func TestReconcileSecurityGroupFromAnyDestinationAddressPrefixToLoadBalancerIP(t if err != nil { t.Errorf("Unexpected error: %q", err) } - sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, true) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, true) if err != nil { t.Errorf("Unexpected error: %q", err) } @@ -739,7 +739,7 @@ func TestReconcileSecurityGroupDynamicLoadBalancerIP(t *testing.T) { az := GetTestCloud(ctrl) svc1 := getTestService("servicea", v1.ProtocolTCP, nil, false, 80) - svc1.Spec.LoadBalancerIP = "" + setServiceLoadBalancerIP(&svc1, "") sg := getTestSecurityGroup(az) setMockSecurityGroup(az, ctrl, sg) @@ -1707,7 +1707,7 @@ func getTestServiceWithAnnotation(identifier string, annotations map[string]stri func getResourceGroupTestService(identifier, resourceGroup, loadBalancerIP string, requestedPorts ...int32) v1.Service { svc := getTestService(identifier, v1.ProtocolTCP, nil, false, requestedPorts...) - svc.Spec.LoadBalancerIP = loadBalancerIP + setServiceLoadBalancerIP(&svc, loadBalancerIP) svc.Annotations[consts.ServiceAnnotationLoadBalancerResourceGroup] = resourceGroup return svc } @@ -1930,7 +1930,7 @@ func validatePublicIP(t *testing.T, publicIP *network.PublicIPAddress, service * t.Errorf("Expected publicIP resource has matching tags[%s]", consts.ClusterNameKey) } - // We cannot use service.Spec.LoadBalancerIP to compare with + // We cannot use Service LoadBalancerIP to compare with // Public IP's IPAddress // Because service properties are updated outside of cloudprovider code } @@ -1994,6 +1994,7 @@ func validateSecurityGroup(t *testing.T, securityGroup *network.SecurityGroup, s az := GetTestCloud(ctrl) seenRules := make(map[string]string) for i, svc := range services { + svc := svc for _, wantedRule := range svc.Spec.Ports { sources := getServiceSourceRanges(&services[i]) for _, source := range sources { @@ -2002,7 +2003,7 @@ func validateSecurityGroup(t *testing.T, securityGroup *network.SecurityGroup, s foundRule := false for _, actualRule := range *securityGroup.SecurityRules { if strings.EqualFold(*actualRule.Name, wantedRuleName) { - err := securityRuleMatches(source, wantedRule, svc.Spec.LoadBalancerIP, actualRule) + err := securityRuleMatches(source, wantedRule, getServiceLoadBalancerIP(&svc), actualRule) if err != nil { t.Errorf("Found matching security rule %q but properties were incorrect: %v", wantedRuleName, err) } @@ -2437,13 +2438,13 @@ func TestIfServiceSpecifiesSharedRuleAndRuleDoesNotExistItIsCreated(t *testing.T az := GetTestCloud(ctrl) svc := getTestService("servicea", v1.ProtocolTCP, nil, false, 80) - svc.Spec.LoadBalancerIP = testIP1 + setServiceLoadBalancerIP(&svc, testIP1) svc.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue sg := getTestSecurityGroup(az) setMockSecurityGroup(az, ctrl, sg) - sg, err := az.reconcileSecurityGroup(testClusterName, &svc, to.StringPtr(svc.Spec.LoadBalancerIP), nil, true) + sg, err := az.reconcileSecurityGroup(testClusterName, &svc, to.StringPtr(getServiceLoadBalancerIP(&svc)), nil, true) if err != nil { t.Errorf("Unexpected error: %q", err) } @@ -2480,7 +2481,7 @@ func TestIfServiceSpecifiesSharedRuleAndRuleExistsThenTheServicesPortAndAddressA az := GetTestCloud(ctrl) svc := getTestService("servicesr", v1.ProtocolTCP, nil, false, 80) - svc.Spec.LoadBalancerIP = testIP1 + setServiceLoadBalancerIP(&svc, testIP1) svc.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue expectedRuleName := testRuleName @@ -2502,7 +2503,7 @@ func TestIfServiceSpecifiesSharedRuleAndRuleExistsThenTheServicesPortAndAddressA } setMockSecurityGroup(az, ctrl, sg) - sg, err := az.reconcileSecurityGroup(testClusterName, &svc, to.StringPtr(svc.Spec.LoadBalancerIP), nil, true) + sg, err := az.reconcileSecurityGroup(testClusterName, &svc, to.StringPtr(getServiceLoadBalancerIP(&svc)), nil, true) if err != nil { t.Errorf("Unexpected error: %q", err) } @@ -2536,22 +2537,22 @@ func TestIfServicesSpecifySharedRuleButDifferentPortsThenSeparateRulesAreCreated az := GetTestCloud(ctrl) svc1 := getTestService("servicesr1", v1.ProtocolTCP, nil, false, 4444) - svc1.Spec.LoadBalancerIP = testIP1 + setServiceLoadBalancerIP(&svc1, testIP1) svc1.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue svc2 := getTestService("servicesr2", v1.ProtocolTCP, nil, false, 8888) - svc2.Spec.LoadBalancerIP = testIP2 + setServiceLoadBalancerIP(&svc2, testIP2) svc2.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue sg := getTestSecurityGroup(az) setMockSecurityGroup(az, ctrl, sg) - _, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, true) + _, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc1: %q", err) } - sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), nil, true) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(getServiceLoadBalancerIP(&svc2)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc2: %q", err) } @@ -2605,11 +2606,12 @@ func TestIfServicesSpecifySharedRuleButDifferentProtocolsThenSeparateRulesAreCre az := GetTestCloud(ctrl) svc1 := getTestService("servicesr1", v1.ProtocolTCP, nil, false, 4444) - svc1.Spec.LoadBalancerIP = testIP1 + + setServiceLoadBalancerIP(&svc1, testIP1) svc1.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue svc2 := getTestService("servicesr2", v1.ProtocolUDP, nil, false, 4444) - svc2.Spec.LoadBalancerIP = testIP1 + setServiceLoadBalancerIP(&svc2, testIP1) svc2.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue testRuleName3 := "shared-UDP-4444-Internet" @@ -2617,12 +2619,12 @@ func TestIfServicesSpecifySharedRuleButDifferentProtocolsThenSeparateRulesAreCre sg := getTestSecurityGroup(az) setMockSecurityGroup(az, ctrl, sg) - _, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, true) + _, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc1: %q", err) } - sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), nil, true) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(getServiceLoadBalancerIP(&svc2)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc2: %q", err) } @@ -2674,12 +2676,12 @@ func TestIfServicesSpecifySharedRuleButDifferentSourceAddressesThenSeparateRules az := GetTestCloud(ctrl) svc1 := getTestService("servicesr1", v1.ProtocolTCP, nil, false, 80) - svc1.Spec.LoadBalancerIP = testIP1 + setServiceLoadBalancerIP(&svc1, testIP1) svc1.Spec.LoadBalancerSourceRanges = []string{"192.168.12.0/24"} svc1.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue svc2 := getTestService("servicesr2", v1.ProtocolTCP, nil, false, 80) - svc2.Spec.LoadBalancerIP = testIP2 + setServiceLoadBalancerIP(&svc2, testIP2) svc2.Spec.LoadBalancerSourceRanges = []string{"192.168.34.0/24"} svc2.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue @@ -2689,12 +2691,12 @@ func TestIfServicesSpecifySharedRuleButDifferentSourceAddressesThenSeparateRules sg := getTestSecurityGroup(az) setMockSecurityGroup(az, ctrl, sg) - _, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, true) + _, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc1: %q", err) } - sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), nil, true) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(getServiceLoadBalancerIP(&svc2)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc2: %q", err) } @@ -2748,32 +2750,32 @@ func TestIfServicesSpecifySharedRuleButSomeAreOnDifferentPortsThenRulesAreSepara az := GetTestCloud(ctrl) svc1 := getTestService("servicesr1", v1.ProtocolTCP, nil, false, 4444) - svc1.Spec.LoadBalancerIP = testIP1 + setServiceLoadBalancerIP(&svc1, testIP1) svc1.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue svc2 := getTestService("servicesr2", v1.ProtocolTCP, nil, false, 8888) - svc2.Spec.LoadBalancerIP = testIP2 + setServiceLoadBalancerIP(&svc2, testIP2) svc2.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue svc3 := getTestService("servicesr3", v1.ProtocolTCP, nil, false, 4444) - svc3.Spec.LoadBalancerIP = testIP3 + setServiceLoadBalancerIP(&svc3, testIP3) svc3.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue testRuleName23 := testRuleName2 sg := getTestSecurityGroup(az) setMockSecurityGroup(az, ctrl, sg) - _, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, true) + _, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc1: %q", err) } - _, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), nil, true) + _, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(getServiceLoadBalancerIP(&svc2)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc2: %q", err) } - sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, to.StringPtr(svc3.Spec.LoadBalancerIP), nil, true) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, to.StringPtr(getServiceLoadBalancerIP(&svc3)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc3: %q", err) } @@ -2849,11 +2851,11 @@ func TestIfServiceSpecifiesSharedRuleAndServiceIsDeletedThenTheServicesPortAndAd az := GetTestCloud(ctrl) svc1 := getTestService("servicesr1", v1.ProtocolTCP, nil, false, 80) - svc1.Spec.LoadBalancerIP = testIP1 + setServiceLoadBalancerIP(&svc1, testIP1) svc1.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue svc2 := getTestService("servicesr2", v1.ProtocolTCP, nil, false, 80) - svc2.Spec.LoadBalancerIP = testIP2 + setServiceLoadBalancerIP(&svc2, testIP2) svc2.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue expectedRuleName := testRuleName @@ -2861,19 +2863,19 @@ func TestIfServiceSpecifiesSharedRuleAndServiceIsDeletedThenTheServicesPortAndAd sg := getTestSecurityGroup(az) setMockSecurityGroup(az, ctrl, sg) - _, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, true) + _, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc1: %q", err) } - sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), nil, true) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(getServiceLoadBalancerIP(&svc2)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc2: %q", err) } validateSecurityGroup(t, sg, svc1, svc2) - sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, false) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, false) if err != nil { t.Errorf("Unexpected error removing svc1: %q", err) } @@ -2907,39 +2909,39 @@ func TestIfSomeServicesShareARuleAndOneIsDeletedItIsRemovedFromTheRightRule(t *t az := GetTestCloud(ctrl) svc1 := getTestService("servicesr1", v1.ProtocolTCP, nil, false, 4444) - svc1.Spec.LoadBalancerIP = testIP1 + setServiceLoadBalancerIP(&svc1, testIP1) svc1.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue svc2 := getTestService("servicesr2", v1.ProtocolTCP, nil, false, 8888) - svc2.Spec.LoadBalancerIP = testIP2 + setServiceLoadBalancerIP(&svc2, testIP2) svc2.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue svc3 := getTestService("servicesr3", v1.ProtocolTCP, nil, false, 4444) - svc3.Spec.LoadBalancerIP = testIP3 + setServiceLoadBalancerIP(&svc3, testIP3) svc3.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue testRuleName23 := testRuleName2 sg := getTestSecurityGroup(az) setMockSecurityGroup(az, ctrl, sg) - _, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, true) + _, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc1: %q", err) } - _, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), nil, true) + _, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(getServiceLoadBalancerIP(&svc2)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc2: %q", err) } - sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, to.StringPtr(svc3.Spec.LoadBalancerIP), nil, true) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, to.StringPtr(getServiceLoadBalancerIP(&svc3)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc3: %q", err) } validateSecurityGroup(t, sg, svc1, svc2, svc3) - sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, false) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, false) if err != nil { t.Errorf("Unexpected error removing svc1: %q", err) } @@ -3015,44 +3017,44 @@ func TestIfServiceSpecifiesSharedRuleAndLastServiceIsDeletedThenRuleIsDeleted(t az := GetTestCloud(ctrl) svc1 := getTestService("servicesr1", v1.ProtocolTCP, nil, false, 4444) - svc1.Spec.LoadBalancerIP = testIP1 + setServiceLoadBalancerIP(&svc1, testIP1) svc1.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue svc2 := getTestService("servicesr2", v1.ProtocolTCP, nil, false, 8888) - svc2.Spec.LoadBalancerIP = testIP2 + setServiceLoadBalancerIP(&svc2, testIP2) svc2.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue svc3 := getTestService("servicesr3", v1.ProtocolTCP, nil, false, 4444) - svc3.Spec.LoadBalancerIP = testIP3 + setServiceLoadBalancerIP(&svc3, testIP3) svc3.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue testRuleName23 := testRuleName2 sg := getTestSecurityGroup(az) setMockSecurityGroup(az, ctrl, sg) - _, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, true) + _, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc1: %q", err) } - _, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), nil, true) + _, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(getServiceLoadBalancerIP(&svc2)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc2: %q", err) } - sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, to.StringPtr(svc3.Spec.LoadBalancerIP), nil, true) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, to.StringPtr(getServiceLoadBalancerIP(&svc3)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc3: %q", err) } validateSecurityGroup(t, sg, svc1, svc2, svc3) - _, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, false) + _, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, false) if err != nil { t.Errorf("Unexpected error removing svc1: %q", err) } - sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, to.StringPtr(svc3.Spec.LoadBalancerIP), nil, false) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, to.StringPtr(getServiceLoadBalancerIP(&svc3)), nil, false) if err != nil { t.Errorf("Unexpected error removing svc3: %q", err) } @@ -3097,23 +3099,23 @@ func TestCanCombineSharedAndPrivateRulesInSameGroup(t *testing.T) { var err error svc1 := getTestService("servicesr1", v1.ProtocolTCP, nil, false, 4444) - svc1.Spec.LoadBalancerIP = testIP1 + setServiceLoadBalancerIP(&svc1, testIP1) svc1.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue svc2 := getTestService("servicesr2", v1.ProtocolTCP, nil, false, 8888) - svc2.Spec.LoadBalancerIP = testIP2 + setServiceLoadBalancerIP(&svc2, testIP2) svc2.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue svc3 := getTestService("servicesr3", v1.ProtocolTCP, nil, false, 4444) - svc3.Spec.LoadBalancerIP = testIP3 + setServiceLoadBalancerIP(&svc3, testIP3) svc3.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue svc4 := getTestService("servicesr4", v1.ProtocolTCP, nil, false, 4444) - svc4.Spec.LoadBalancerIP = "192.168.22.33" + setServiceLoadBalancerIP(&svc4, "192.168.22.33") svc4.Annotations[consts.ServiceAnnotationSharedSecurityRule] = "false" svc5 := getTestService("servicesr5", v1.ProtocolTCP, nil, false, 8888) - svc5.Spec.LoadBalancerIP = "192.168.22.33" + setServiceLoadBalancerIP(&svc5, "192.168.22.33") svc5.Annotations[consts.ServiceAnnotationSharedSecurityRule] = "false" testServices := []v1.Service{svc1, svc2, svc3, svc4, svc5} @@ -3126,7 +3128,8 @@ func TestCanCombineSharedAndPrivateRulesInSameGroup(t *testing.T) { setMockSecurityGroup(az, ctrl, sg) for i, svc := range testServices { - _, err := az.reconcileSecurityGroup(testClusterName, &testServices[i], to.StringPtr(svc.Spec.LoadBalancerIP), nil, true) + svc := svc + _, err := az.reconcileSecurityGroup(testClusterName, &testServices[i], to.StringPtr(getServiceLoadBalancerIP(&svc)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc%d: %q", i+1, err) } @@ -3201,8 +3204,8 @@ func TestCanCombineSharedAndPrivateRulesInSameGroup(t *testing.T) { if securityRule4.DestinationAddressPrefix == nil { t.Errorf("Expected unshared rule %s to have a destination IP address", expectedRuleName4) } else { - if !strings.EqualFold(*securityRule4.DestinationAddressPrefix, svc4.Spec.LoadBalancerIP) { - t.Errorf("Expected unshared rule %s to have a destination %s but had %s", expectedRuleName4, svc4.Spec.LoadBalancerIP, *securityRule4.DestinationAddressPrefix) + if !strings.EqualFold(*securityRule4.DestinationAddressPrefix, getServiceLoadBalancerIP(&svc4)) { + t.Errorf("Expected unshared rule %s to have a destination %s but had %s", expectedRuleName4, getServiceLoadBalancerIP(&svc4), *securityRule4.DestinationAddressPrefix) } } @@ -3213,17 +3216,17 @@ func TestCanCombineSharedAndPrivateRulesInSameGroup(t *testing.T) { if securityRule5.DestinationAddressPrefix == nil { t.Errorf("Expected unshared rule %s to have a destination IP address", expectedRuleName5) } else { - if !strings.EqualFold(*securityRule5.DestinationAddressPrefix, svc5.Spec.LoadBalancerIP) { - t.Errorf("Expected unshared rule %s to have a destination %s but had %s", expectedRuleName5, svc5.Spec.LoadBalancerIP, *securityRule5.DestinationAddressPrefix) + if !strings.EqualFold(*securityRule5.DestinationAddressPrefix, getServiceLoadBalancerIP(&svc5)) { + t.Errorf("Expected unshared rule %s to have a destination %s but had %s", expectedRuleName5, getServiceLoadBalancerIP(&svc5), *securityRule5.DestinationAddressPrefix) } } - _, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, false) + _, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, false) if err != nil { t.Errorf("Unexpected error removing svc1: %q", err) } - sg, err = az.reconcileSecurityGroup(testClusterName, &svc5, to.StringPtr(svc5.Spec.LoadBalancerIP), nil, false) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc5, to.StringPtr(getServiceLoadBalancerIP(&svc5)), nil, false) if err != nil { t.Errorf("Unexpected error removing svc5: %q", err) } diff --git a/site/content/en/topics/pls-integration.md b/site/content/en/topics/pls-integration.md index b641541e2d..7aabc16f39 100644 --- a/site/content/en/topics/pls-integration.md +++ b/site/content/en/topics/pls-integration.md @@ -35,7 +35,7 @@ For more details about each configuration, please refer to [Azure Private Link S ### Creating managed PrivateLinkService -When a `LoadBalancer` typed service is created without the `loadBalancerIP` field specified, an LB frontend IP configuration is created with a dynamically generated IP. If the service has `loadBalancerIP` in its spec, an existing LB frontend IP configuration may be reused if one exists; otherwise a static configuration is created with the specified IP. When a service is created with annotation `service.beta.kubernetes.io/azure-pls-create` set to `true` or updated later with the annotation added, a PLS resource attached to the LB frontend is created in the default resource group or the resource group user set in config file with key `PrivateLinkServiceResourceGroup`. +When a `LoadBalancer` typed service is created without the annotations `service.beta.kubernetes.io/azure-load-balancer-ipv4`, `service.beta.kubernetes.io/azure-load-balancer-ipv6` or field `Service.Spec.LoadBalancerIP` set, an LB frontend IP configuration is created with a dynamically generated IP. If the service has the annotation `service.beta.kubernetes.io/azure-load-balancer-ipv4` or `service.beta.kubernetes.io/azure-load-balancer-ipv6` set, an existing LB frontend IP configuration may be reused if one exists; otherwise a static configuration is created with the specified IP. When a service is created with annotation `service.beta.kubernetes.io/azure-pls-create` set to `true` or updated later with the annotation added, a PLS resource attached to the LB frontend is created in the default resource group or the resource group user set in config file with key `PrivateLinkServiceResourceGroup`. The Kubernetes service creating the PLS is assigned as the owner of the resource. Azure cloud provider tags the PLS with cluster name and service name `kubernetes-owner-service: /`. Only the owner service can later update the properties of the PLS resource. @@ -51,7 +51,7 @@ If there are active PE connections to the PLS, all connections are removed and t ### Sharing managed PrivateLinkService -Multiple Kubernetes services can share the same LB frontend by specifying the same `loadBalancerIP` (for more details, please refer to [Multiple Services Sharing One IP Address](../shared-ip)). Once a PLS is attached to the LB frontend, these services automatically share the PLS. Users can access these services via the same PE but different ports. +Multiple Kubernetes services can share the same LB frontend by specifying the same annotations `service.beta.kubernetes.io/azure-load-balancer-ipv4`, `service.beta.kubernetes.io/azure-load-balancer-ipv6` or field `Service.Spec.LoadBalancerIP` (for more details, please refer to [Multiple Services Sharing One IP Address](../shared-ip)). Once a PLS is attached to the LB frontend, these services automatically share the PLS. Users can access these services via the same PE but different ports. Azure cloud provider tags the service creating the PLS as the owner (`kubernetes-owner-service: /`) and only allows that service to update the configurations of the PLS. If the owner service is deleted or if user wants some other service to take control, user can modify the tag value to a new service in `/` pattern. diff --git a/site/content/en/topics/shared-ip.md b/site/content/en/topics/shared-ip.md index 2322d66405..4b1e3be56e 100644 --- a/site/content/en/topics/shared-ip.md +++ b/site/content/en/topics/shared-ip.md @@ -27,7 +27,7 @@ spec: type: LoadBalancer ``` -Note that the `loadBalancerIP` is not set, or Azure would find a pre-allocated public IP with the address. After obtaining the IP address of the service, you could create other services using this address. +Note that the annotations `service.beta.kubernetes.io/azure-load-balancer-ipv4`, `service.beta.kubernetes.io/azure-load-balancer-ipv6`, field `Service.Spec.LoadBalancerIP` are not set, or Azure would find a pre-allocated public IP with the address. After obtaining the IP address of the service, you could create other services using this address. ```yaml apiVersion: v1 @@ -35,8 +35,8 @@ kind: Service metadata: name: https namespace: default + service.beta.kubernetes.io/azure-load-balancer-ipv4: 1.2.3.4 # the IP address could be the same as it is of `nginx` service spec: - loadBalancerIP: 1.2.3.4 # the IP address could be the same as it is of `nginx` service ports: - port: 443 protocol: TCP @@ -46,7 +46,7 @@ spec: type: LoadBalancer ``` -Note that if you specify the `loadBalancerIP` but there is no corresponding public IP pre-allocated, an error would be reported. +Note that if you specify the annotation `service.beta.kubernetes.io/azure-load-balancer-ipv4` or `service.beta.kubernetes.io/azure-load-balancer-ipv6` but there is no corresponding public IP pre-allocated, an error would be reported. ## DNS diff --git a/tests/e2e/network/ensureloadbalancer.go b/tests/e2e/network/ensureloadbalancer.go index ace96e6d28..7677ef868b 100644 --- a/tests/e2e/network/ensureloadbalancer.go +++ b/tests/e2e/network/ensureloadbalancer.go @@ -19,6 +19,7 @@ package network import ( "context" "fmt" + "net" "os" "strconv" "strings" @@ -158,7 +159,7 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { It("should support BYO public IP", func() { By("creating a public IP with tags") ipName := basename + "-public-IP" + string(uuid.NewUUID())[0:4] - pip := defaultPublicIPAddress(ipName) + pip := defaultPublicIPAddress(ipName, false) expectedTags := map[string]*string{ "foo": to.StringPtr("bar"), } @@ -171,7 +172,7 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { By("creating a service referencing the public IP") service := utils.CreateLoadBalancerServiceManifest(testServiceName, nil, labels, ns.Name, ports) - service = updateServiceBalanceIP(service, false, targetIP) + service = updateServiceLBIP(service, false, targetIP) _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, "") @@ -203,7 +204,7 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { Expect(err).NotTo(HaveOccurred()) utils.Logf("Successfully created LoadBalancer service " + testServiceName + " in namespace " + ns.Name) - pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), defaultPublicIPAddress(ipName)) + pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), defaultPublicIPAddress(ipName, false)) Expect(err).NotTo(HaveOccurred()) targetIP := to.String(pip.IPAddress) utils.Logf("PIP to %s", targetIP) @@ -225,7 +226,7 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { By("Updating service to bound to specific public IP") utils.Logf("will update IP to %s", targetIP) service, err = cs.CoreV1().Services(ns.Name).Get(context.TODO(), testServiceName, metav1.GetOptions{}) - service = updateServiceBalanceIP(service, false, targetIP) + service = updateServiceLBIP(service, false, targetIP) _, err = cs.CoreV1().Services(ns.Name).Update(context.TODO(), service, metav1.UpdateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -241,7 +242,7 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { Expect(err).NotTo(HaveOccurred()) service := utils.CreateLoadBalancerServiceManifest(testServiceName, serviceAnnotationLoadBalancerInternalTrue, labels, ns.Name, ports) - service = updateServiceBalanceIP(service, true, ip1) + service = updateServiceLBIP(service, true, ip1) _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) utils.Logf("Successfully created LoadBalancer service " + testServiceName + " in namespace " + ns.Name) @@ -269,7 +270,7 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { By("Updating internal service private IP") utils.Logf("will update IP to %s", ip2) service, err = cs.CoreV1().Services(ns.Name).Get(context.TODO(), testServiceName, metav1.GetOptions{}) - service = updateServiceBalanceIP(service, true, ip2) + service = updateServiceLBIP(service, true, ip2) _, err = cs.CoreV1().Services(ns.Name).Update(context.TODO(), service, metav1.UpdateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -287,7 +288,7 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { Expect(err).NotTo(HaveOccurred()) utils.Logf("Successfully created LoadBalancer service " + testServiceName + " in namespace " + ns.Name) - pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), defaultPublicIPAddress(ipName)) + pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), defaultPublicIPAddress(ipName, false)) Expect(err).NotTo(HaveOccurred()) targetIP := to.String(pip.IPAddress) @@ -313,7 +314,7 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { By("Updating service to bound to specific public IP") utils.Logf("will update IP to %s, %v", targetIP, len(targetIP)) service, err = cs.CoreV1().Services(ns.Name).Get(context.TODO(), testServiceName, metav1.GetOptions{}) - service = updateServiceBalanceIP(service, false, targetIP) + service = updateServiceLBIP(service, false, targetIP) _, err = cs.CoreV1().Services(ns.Name).Update(context.TODO(), service, metav1.UpdateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -326,12 +327,12 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { It("should have no operation since no change in service when update", Label(utils.TestSuiteLabelSlow), func() { suffix := string(uuid.NewUUID())[0:4] ipName := basename + "-public-remain" + suffix - pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), defaultPublicIPAddress(ipName)) + pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), defaultPublicIPAddress(ipName, false)) Expect(err).NotTo(HaveOccurred()) targetIP := to.String(pip.IPAddress) service := utils.CreateLoadBalancerServiceManifest(testServiceName, serviceAnnotationLoadBalancerInternalFalse, labels, ns.Name, ports) - service = updateServiceBalanceIP(service, false, targetIP) + service = updateServiceLBIP(service, false, targetIP) _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) utils.Logf("Successfully created LoadBalancer service %s in namespace %s", testServiceName, ns.Name) @@ -384,7 +385,7 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { It("should support multiple external services sharing one preset public IP address", func() { ipName := fmt.Sprintf("%s-public-remain-%s", basename, string(uuid.NewUUID())[0:4]) - pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), defaultPublicIPAddress(ipName)) + pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), defaultPublicIPAddress(ipName, false)) defer func() { err = utils.DeletePIPWithRetry(tc, ipName, "") Expect(err).NotTo(HaveOccurred()) @@ -420,7 +421,7 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { TargetPort: intstr.FromInt(int(tcpPort)), }} service := utils.CreateLoadBalancerServiceManifest(serviceName, nil, serviceLabels, ns.Name, servicePort) - service = updateServiceBalanceIP(service, false, targetIP) + service = updateServiceLBIP(service, false, targetIP) _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) defer func() { err = utils.DeleteService(cs, ns.Name, serviceName) @@ -468,7 +469,7 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { }} service := utils.CreateLoadBalancerServiceManifest(serviceName, nil, serviceLabels, ns.Name, servicePort) if sharedIP != "" { - service.Spec.LoadBalancerIP = sharedIP + service = updateServiceLBIP(service, false, sharedIP) } _, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -555,7 +556,7 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { }} service := utils.CreateLoadBalancerServiceManifest(serviceName, serviceAnnotationLoadBalancerInternalTrue, serviceLabels, ns.Name, servicePort) if sharedIP != "" { - service.Spec.LoadBalancerIP = sharedIP + service = updateServiceLBIP(service, true, sharedIP) } _, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) defer func() { @@ -619,7 +620,7 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { It("should support disabling floating IP in load balancer rule with kubernetes service annotations", func() { By("creating a public IP") ipName := basename + "-public-IP" + string(uuid.NewUUID())[0:4] - pip := defaultPublicIPAddress(ipName) + pip := defaultPublicIPAddress(ipName, false) pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), pip) Expect(err).NotTo(HaveOccurred()) targetIP := to.String(pip.IPAddress) @@ -627,7 +628,7 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() { By("creating a service referencing the public IP") service := utils.CreateLoadBalancerServiceManifest(testServiceName, serviceAnnotationDisableLoadBalancerFloatingIP, labels, ns.Name, ports) - service = updateServiceBalanceIP(service, false, targetIP) + service = updateServiceLBIP(service, false, targetIP) _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, "") @@ -743,7 +744,7 @@ var _ = Describe("EnsureLoadBalancer should not update any resources when servic It("should respect service with BYO public IP with various configurations", func() { By("Creating a BYO public IP") ipName := basename + "-public-IP" + string(uuid.NewUUID())[0:4] - pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), defaultPublicIPAddress(ipName)) + pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), defaultPublicIPAddress(ipName, false)) defer func() { err = utils.DeletePIPWithRetry(tc, ipName, "") Expect(err).NotTo(HaveOccurred()) @@ -992,12 +993,20 @@ func getLBBackendPoolIndex(lb *aznetwork.LoadBalancer) int { return 0 } -func updateServiceBalanceIP(service *v1.Service, isInternal bool, ip string) (result *v1.Service) { +func updateServiceLBIP(service *v1.Service, isInternal bool, ip string) (result *v1.Service) { result = service if result == nil { return } - result.Spec.LoadBalancerIP = ip + if result.Annotations == nil { + result.Annotations = map[string]string{} + } + if net.ParseIP(ip).To4() != nil { + result.Annotations[consts.ServiceAnnotationLoadBalancerIPDualStack[false]] = ip + } else { + result.Annotations[consts.ServiceAnnotationLoadBalancerIPDualStack[true]] = ip + } + if judgeInternal(*service) == isInternal { return } @@ -1009,7 +1018,7 @@ func updateServiceBalanceIP(service *v1.Service, isInternal bool, ip string) (re return } -func defaultPublicIPAddress(ipName string) aznetwork.PublicIPAddress { +func defaultPublicIPAddress(ipName string, isIPv6 bool) aznetwork.PublicIPAddress { // The default sku for LoadBalancer and PublicIP is basic. skuName := aznetwork.PublicIPAddressSkuNameBasic if skuEnv := os.Getenv(utils.LoadBalancerSkuEnv); skuEnv != "" { @@ -1017,7 +1026,7 @@ func defaultPublicIPAddress(ipName string) aznetwork.PublicIPAddress { skuName = aznetwork.PublicIPAddressSkuNameStandard } } - return aznetwork.PublicIPAddress{ + pip := aznetwork.PublicIPAddress{ Name: to.StringPtr(ipName), Location: to.StringPtr(os.Getenv(utils.ClusterLocationEnv)), Sku: &aznetwork.PublicIPAddressSku{ @@ -1027,6 +1036,10 @@ func defaultPublicIPAddress(ipName string) aznetwork.PublicIPAddress { PublicIPAllocationMethod: aznetwork.IPAllocationMethodStatic, }, } + if isIPv6 { + pip.PublicIPAddressPropertiesFormat.PublicIPAddressVersion = network.IPVersionIPv6 + } + return pip } func defaultPublicIPPrefix(name string) aznetwork.PublicIPPrefix { diff --git a/tests/e2e/network/network_security_group.go b/tests/e2e/network/network_security_group.go index 53ff9297d6..237c06bbe2 100644 --- a/tests/e2e/network/network_security_group.go +++ b/tests/e2e/network/network_security_group.go @@ -294,7 +294,7 @@ var _ = Describe("Network security group", Label(utils.TestSuiteLabelNSG), func( It("should support service annotation `service.beta.kubernetes.io/azure-disable-load-balancer-floating-ip`", func() { By("Creating a public IP with tags") ipName := basename + "-public-IP-disable-floating-ip" - pip := defaultPublicIPAddress(ipName) + pip := defaultPublicIPAddress(ipName, false) pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), pip) Expect(err).NotTo(HaveOccurred()) targetIP := to.String(pip.IPAddress) @@ -305,7 +305,7 @@ var _ = Describe("Network security group", Label(utils.TestSuiteLabelNSG), func( consts.ServiceAnnotationDisableLoadBalancerFloatingIP: "true", } service := utils.CreateLoadBalancerServiceManifest(serviceName, annotation, labels, ns.Name, ports) - service = updateServiceBalanceIP(service, false, targetIP) + service = updateServiceLBIP(service, false, targetIP) _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, "") diff --git a/tests/e2e/network/private_link_service.go b/tests/e2e/network/private_link_service.go index ca19b863a3..32e5f75215 100644 --- a/tests/e2e/network/private_link_service.go +++ b/tests/e2e/network/private_link_service.go @@ -393,7 +393,7 @@ var _ = Describe("Private link service", Label(utils.TestSuiteLabelPrivateLinkSe err = utils.DeleteService(cs, ns.Name, svc2) Expect(err).NotTo(HaveOccurred()) }() - service2.Spec.LoadBalancerIP = ip + service2 = updateServiceLBIP(service2, true, ip) _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service2, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) _, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, svc2, ip) diff --git a/tests/e2e/network/service_annotations.go b/tests/e2e/network/service_annotations.go index b6687848db..cd335e4e6e 100644 --- a/tests/e2e/network/service_annotations.go +++ b/tests/e2e/network/service_annotations.go @@ -50,6 +50,8 @@ var ( scalesetRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/resourceGroups/(.+)/providers/Microsoft.Compute/virtualMachineScaleSets/(.+)/virtualMachines(?:.*)`) lbNameRE = regexp.MustCompile(`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Network/loadBalancers/(.+)/frontendIPConfigurations(?:.*)`) backendIPConfigurationRE = regexp.MustCompile(`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Compute/virtualMachineScaleSets/(.+)/virtualMachines(?:.*)`) + + ipFamily = utils.IPv4 ) const ( @@ -98,6 +100,9 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn utils.Logf("Waiting for backend pods to be ready") err = utils.WaitPodsToBeReady(cs, ns.Name) Expect(err).NotTo(HaveOccurred()) + + ipFamily, err = utils.GetClusterServiceIPFamily() + Expect(err).NotTo(HaveOccurred()) }) AfterEach(func() { @@ -295,7 +300,7 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn By("creating test PIP in the test resource group") testPIPName := "testPIP-" + string(uuid.NewUUID())[0:4] - pip, err := utils.WaitCreatePIP(tc, testPIPName, *rg.Name, defaultPublicIPAddress(testPIPName)) + pip, err := utils.WaitCreatePIP(tc, testPIPName, *rg.Name, defaultPublicIPAddress(testPIPName, false)) Expect(err).NotTo(HaveOccurred()) defer func() { utils.Logf("Cleaning up service and public IP") @@ -310,7 +315,7 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn } By("Creating service " + serviceName + " in namespace " + ns.Name) service := utils.CreateLoadBalancerServiceManifest(serviceName, annotation, labels, ns.Name, ports) - service.Spec.LoadBalancerIP = *pip.IPAddress + service = updateServiceLBIP(service, false, *pip.IPAddress) _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) utils.Logf("Successfully created LoadBalancer service " + serviceName + " in namespace " + ns.Name) @@ -327,7 +332,7 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn It("should support service annotation `service.beta.kubernetes.io/azure-additional-public-ips`", func() { By("creating a public IP") ipName := basename + "-public-IP" + string(uuid.NewUUID())[0:4] - pip := defaultPublicIPAddress(ipName) + pip := defaultPublicIPAddress(ipName, false) pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), pip) Expect(err).NotTo(HaveOccurred()) additionalPIP := to.String(pip.IPAddress) @@ -386,6 +391,7 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn consts.ServiceAnnotationAzurePIPTags: "a=b,c= d,e =, =f", } service := utils.CreateLoadBalancerServiceManifest(serviceName, annotation, labels, ns.Name, ports) + service.GetAnnotations() _, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -438,7 +444,7 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn It("should support service annotation `service.beta.kubernetes.io/azure-pip-name`", func() { By("Creating two test pips") pipName1 := "pip1" - pip1, err := utils.WaitCreatePIP(tc, pipName1, tc.GetResourceGroup(), defaultPublicIPAddress(pipName1)) + pip1, err := utils.WaitCreatePIP(tc, pipName1, tc.GetResourceGroup(), defaultPublicIPAddress(pipName1, false)) defer func() { By("Cleaning up test PIP") err := utils.DeletePIPWithRetry(tc, pipName1, tc.GetResourceGroup()) @@ -446,7 +452,7 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn }() Expect(err).NotTo(HaveOccurred()) pipName2 := "pip2" - pip2, err := utils.WaitCreatePIP(tc, pipName2, tc.GetResourceGroup(), defaultPublicIPAddress(pipName2)) + pip2, err := utils.WaitCreatePIP(tc, pipName2, tc.GetResourceGroup(), defaultPublicIPAddress(pipName2, false)) defer func() { By("Cleaning up test PIP") err := utils.DeletePIPWithRetry(tc, pipName2, tc.GetResourceGroup()) @@ -622,6 +628,45 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn Expect((len(targetProbes))).To(Equal(1)) Expect(targetProbes[0].Protocol).To(Equal(network.ProbeProtocolHTTP)) }) + + // Check if the following annotations are correctly set with Service LB IP + // service.beta.kubernetes.io/azure-load-balancer-ipv4 or service.beta.kubernetes.io/azure-load-balancer-ipv6 + It("should support service annotation 'service.beta.kubernetes.io/azure-load-balancer-ip'", func() { + pipName := fmt.Sprintf("%s-public-IP%s", basename, string(uuid.NewUUID())[0:4]) + By(fmt.Sprintf("Creating a public IP %q", pipName)) + var pip network.PublicIPAddress + if ipFamily == utils.IPv4 { + pip = defaultPublicIPAddress(pipName, false) + } else if ipFamily == utils.IPv6 { + pip = defaultPublicIPAddress(pipName, true) + } else { + // TODO: dual-stack support + } + pip, err := utils.WaitCreatePIP(tc, pipName, tc.GetResourceGroup(), pip) + Expect(err).NotTo(HaveOccurred()) + pipAddr := to.String(pip.IPAddress) + utils.Logf("Created pip with address %s", pipAddr) + + annotation := map[string]string{} + if ipFamily == utils.IPv4 { + annotation[consts.ServiceAnnotationLoadBalancerIPDualStack[false]] = pipAddr + } else if ipFamily == utils.IPv6 { + annotation[consts.ServiceAnnotationLoadBalancerIPDualStack[true]] = pipAddr + } else { + // TODO: dual-stack support + } + + By("Creating a Service") + publicIP := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports) + defer func() { + By("Cleaning up service") + err := utils.DeleteService(cs, ns.Name, serviceName) + Expect(err).NotTo(HaveOccurred()) + }() + + By("Check if the Service has the correct address") + Expect(publicIP).To(Equal(pipAddr)) + }) }) var _ = Describe("Multiple VMSS", Label(utils.TestSuiteLabelMultiNodePools, utils.TestSuiteLabelVMSS), func() { diff --git a/tests/e2e/utils/kubectl.go b/tests/e2e/utils/kubectl.go index 02042a1050..1fd6efa737 100644 --- a/tests/e2e/utils/kubectl.go +++ b/tests/e2e/utils/kubectl.go @@ -100,7 +100,9 @@ func KubectlCmd(namespace string, args ...string) *exec.Cmd { Logf("Kubernetes configuration file name: %s", filename) defaultArgs = append(defaultArgs, "--"+clientcmd.RecommendedConfigPathFlag+"="+filename) - defaultArgs = append(defaultArgs, fmt.Sprintf("--namespace=%s", namespace)) + if namespace != "" { + defaultArgs = append(defaultArgs, fmt.Sprintf("--namespace=%s", namespace)) + } kubectlArgs := append(defaultArgs, args...) cmd := exec.Command("kubectl", kubectlArgs...) diff --git a/tests/e2e/utils/network_utils.go b/tests/e2e/utils/network_utils.go index aa2547d5c5..f535ad3efd 100644 --- a/tests/e2e/utils/network_utils.go +++ b/tests/e2e/utils/network_utils.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "net" + "regexp" "strings" "time" @@ -32,6 +33,14 @@ import ( "k8s.io/apimachinery/pkg/util/wait" ) +type IPFamily string + +var ( + IPv4 IPFamily = "ipv4" + IPv6 IPFamily = "ipv6" + DualStack IPFamily = "dualstack" +) + // getVirtualNetworkList returns the list of virtual networks in the cluster resource group. func (azureTestClient *AzureTestClient) getVirtualNetworkList() (result aznetwork.VirtualNetworkListResultPage, err error) { Logf("Getting virtual network list") @@ -476,3 +485,58 @@ func (azureTestClient *AzureTestClient) ListPrivateLinkServices(resourceGroupNam return result, nil } + +func retrieveCIDRs(cmd string, reg string) ([]string, error) { + res := make([]string, 2) + stdout, err := RunKubectl("", strings.Split(cmd, " ")...) + if err != nil { + return res, fmt.Errorf("error when running the following kubectl command %q: %v, %s", cmd, err, stdout) + } + re := regexp.MustCompile(reg) + matches := re.FindStringSubmatch(stdout) + if len(matches) == 0 { + return res, fmt.Errorf("cannot retrieve CIDR, unexpected kubectl output: %s", stdout) + } + cidrs := strings.Split(matches[1], ",") + if len(cidrs) == 1 { + _, cidr, err := net.ParseCIDR(cidrs[0]) + if err != nil { + return res, fmt.Errorf("CIDR cannot be parsed: %s", cidrs[0]) + } + if cidr.IP.To4() != nil { + res[0] = cidrs[0] + } else { + res[1] = cidrs[0] + } + } else if len(cidrs) == 2 { + _, cidr, err := net.ParseCIDR(cidrs[0]) + if err != nil { + return res, fmt.Errorf("CIDR cannot be parsed: %s", cidrs[0]) + } + if cidr.IP.To4() != nil { + res[0] = cidrs[0] + res[1] = cidrs[1] + } else { + res[0] = cidrs[1] + res[1] = cidrs[0] + } + } else { + return res, fmt.Errorf("unexpected cluster CIDR: %s", matches[1]) + } + return res, nil +} + +// GetClusterServiceIPFamily gets cluster's Service IPFamily according to Service CIDRs. +func GetClusterServiceIPFamily() (IPFamily, error) { + svcCIDRs, err := retrieveCIDRs("cluster-info dump | grep service-cluster-ip-range", `service-cluster-ip-range=([^"]+)`) + if err != nil { + return "", err + } + if svcCIDRs[0] != "" && svcCIDRs[1] == "" { + return IPv4, nil + } + if svcCIDRs[0] == "" && svcCIDRs[1] != "" { + return IPv6, nil + } + return DualStack, nil +}