From 4cf6ffc46e6bdc774a7cfd5eb66493c7785d220d Mon Sep 17 00:00:00 2001 From: Zhecheng Li Date: Tue, 27 Sep 2022 19:07:04 +0800 Subject: [PATCH] Deprecate LoadBalancerIP with Servie LB IP annotation Signed-off-by: Zhecheng Li --- pkg/consts/consts.go | 10 ++ pkg/provider/azure_loadbalancer.go | 48 +++++++-- pkg/provider/azure_loadbalancer_test.go | 62 +++++------ pkg/provider/azure_standard.go | 2 +- pkg/provider/azure_standard_test.go | 34 +++--- pkg/provider/azure_test.go | 123 +++++++++++----------- site/content/en/topics/pls-integration.md | 4 +- site/content/en/topics/shared-ip.md | 6 +- 8 files changed, 163 insertions(+), 126 deletions(-) diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index 153b2dc03a..640fdb5ac9 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -184,6 +184,16 @@ const ( BackoffJitterDefault = 1.0 ) +// LB variables for dual-stack +var ( + // There's no "service.beta.kubernetes.io/azure-load-balancer" annotation before these 2 new ones are added. + // Service.Spec.LoadBalancerIP was used and it will be deprecated. + ServiceAnnotationLoadBalancerIPDualStack = map[bool]string{ + false: "service.beta.kubernetes.io/azure-load-balancer-ipv4", + true: "service.beta.kubernetes.io/azure-load-balancer-ipv6", + } +) + // load balancer const ( // PreConfiguredBackendPoolLoadBalancerTypesNone means that the load balancers are not pre-configured diff --git a/pkg/provider/azure_loadbalancer.go b/pkg/provider/azure_loadbalancer.go index 706cf31df6..0ffcf93d59 100644 --- a/pkg/provider/azure_loadbalancer.go +++ b/pkg/provider/azure_loadbalancer.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "math" + "net" "reflect" "sort" "strconv" @@ -45,6 +46,34 @@ import ( "sigs.k8s.io/cloud-provider-azure/pkg/retry" ) +// getServiceLoadBalancerIP retrieves LB IP from IPv4 annotation, then IPv6 annotation, then service.Spec.LoadBalancerIP. +func getServiceLoadBalancerIP(service *v1.Service) string { + if service == nil { + return "" + } + + if ip, ok := service.Annotations[consts.ServiceAnnotationLoadBalancerIPDualStack[false]]; ok && ip != "" { + return ip + } + if ip, ok := service.Annotations[consts.ServiceAnnotationLoadBalancerIPDualStack[true]]; ok && ip != "" { + return ip + } + + // Retrieve LB IP from service.Spec.LoadBalancerIP (will be deprecated) + return service.Spec.LoadBalancerIP +} + +// setServiceLoadBalancerIP sets LB IP to a Service +func setServiceLoadBalancerIP(service *v1.Service, ip string) { + if service.Annotations == nil { + service.Annotations = map[string]string{} + } + if net.ParseIP(ip).To4() != nil { + service.Annotations[consts.ServiceAnnotationLoadBalancerIPDualStack[false]] = ip + } + service.Annotations[consts.ServiceAnnotationLoadBalancerIPDualStack[true]] = ip +} + // GetLoadBalancer returns whether the specified load balancer and its components exist, and // if so, what its status is. func (az *Cloud) GetLoadBalancer(ctx context.Context, clusterName string, service *v1.Service) (status *v1.LoadBalancerStatus, exists bool, err error) { @@ -809,7 +838,7 @@ func (az *Cloud) determinePublicIPName(clusterName string, service *v1.Service, } pipResourceGroup := az.getPublicIPAddressResourceGroup(service) - loadBalancerIP := service.Spec.LoadBalancerIP + loadBalancerIP := getServiceLoadBalancerIP(service) // Assume that the service without loadBalancerIP set is a primary service. // If a secondary service doesn't set the loadBalancerIP, it is not allowed to share the IP. @@ -867,14 +896,15 @@ func flipServiceInternalAnnotation(service *v1.Service) *v1.Service { func updateServiceLoadBalancerIP(service *v1.Service, serviceIP string) *v1.Service { copyService := service.DeepCopy() if len(serviceIP) > 0 && copyService != nil { - copyService.Spec.LoadBalancerIP = serviceIP + setServiceLoadBalancerIP(service, serviceIP) } return copyService } func (az *Cloud) findServiceIPAddress(ctx context.Context, clusterName string, service *v1.Service, isInternalLb bool) (string, error) { - if len(service.Spec.LoadBalancerIP) > 0 { - return service.Spec.LoadBalancerIP, nil + lbIP := getServiceLoadBalancerIP(service) + if len(lbIP) > 0 { + return lbIP, nil } if len(service.Status.LoadBalancer.Ingress) > 0 && len(service.Status.LoadBalancer.Ingress[0].IP) > 0 { @@ -1264,7 +1294,7 @@ func (az *Cloud) isFrontendIPChanged(clusterName string, config network.Frontend if !strings.EqualFold(to.String(config.Name), lbFrontendIPConfigName) { return false, nil } - loadBalancerIP := service.Spec.LoadBalancerIP + loadBalancerIP := getServiceLoadBalancerIP(service) isInternal := requiresInternalLoadBalancer(service) if isInternal { // Judge subnet @@ -1783,7 +1813,7 @@ func (az *Cloud) reconcileFrontendIPConfigs(clusterName string, service *v1.Serv configProperties.PrivateIPAddressVersion = network.IPVersionIPv6 } - loadBalancerIP := service.Spec.LoadBalancerIP + loadBalancerIP := getServiceLoadBalancerIP(service) if loadBalancerIP != "" { configProperties.PrivateIPAllocationMethod = network.IPAllocationMethodStatic configProperties.PrivateIPAddress = &loadBalancerIP @@ -3250,7 +3280,7 @@ func getServiceTags(service *v1.Service) []string { // The pip is user-created if and only if there is no service tags. // The service owns the pip if: // 1. The serviceName is included in the service tags of a system-created pip. -// 2. The service.Spec.LoadBalancerIP matches the IP address of a user-created pip. +// 2. The service LoadBalancerIP matches the IP address of a user-created pip. func serviceOwnsPublicIP(service *v1.Service, pip *network.PublicIPAddress, clusterName string) (bool, bool) { if service == nil || pip == nil { klog.Warningf("serviceOwnsPublicIP: nil service or public IP") @@ -3270,7 +3300,7 @@ func serviceOwnsPublicIP(service *v1.Service, pip *network.PublicIPAddress, clus // if there is no service tag on the pip, it is user-created pip if serviceTag == "" { - return strings.EqualFold(to.String(pip.IPAddress), service.Spec.LoadBalancerIP), true + return strings.EqualFold(to.String(pip.IPAddress), getServiceLoadBalancerIP(service)), true } // if there is service tag on the pip, it is system-created pip @@ -3288,7 +3318,7 @@ func serviceOwnsPublicIP(service *v1.Service, pip *network.PublicIPAddress, clus } else { // if the service is not included in te tags of the system-created pip, check the ip address // this could happen for secondary services - return strings.EqualFold(to.String(pip.IPAddress), service.Spec.LoadBalancerIP), false + return strings.EqualFold(to.String(pip.IPAddress), getServiceLoadBalancerIP(service)), false } } diff --git a/pkg/provider/azure_loadbalancer_test.go b/pkg/provider/azure_loadbalancer_test.go index a024c127a8..f1acacc5ed 100644 --- a/pkg/provider/azure_loadbalancer_test.go +++ b/pkg/provider/azure_loadbalancer_test.go @@ -818,7 +818,7 @@ func TestServiceOwnsPublicIP(t *testing.T) { t.Run(c.desc, func(t *testing.T) { service := getTestService(c.serviceName, v1.ProtocolTCP, nil, false, 80) if c.serviceLBIP != "" { - service.Spec.LoadBalancerIP = c.serviceLBIP + setServiceLoadBalancerIP(&service, c.serviceLBIP) } owns, isUserAssignedPIP := serviceOwnsPublicIP(&service, c.pip, c.clusterName) assert.Equal(t, c.expectedOwns, owns, "TestCase[%d]: %s", i, c.desc) @@ -1865,34 +1865,36 @@ func TestIsFrontendIPChanged(t *testing.T) { }, } - for i, test := range testCases { - az := GetTestCloud(ctrl) - mockSubnetsClient := az.SubnetsClient.(*mocksubnetclient.MockInterface) - mockSubnetsClient.EXPECT().Get(gomock.Any(), "rg", "vnet", "testSubnet", "").Return(test.existingSubnet, nil).AnyTimes() - mockSubnetsClient.EXPECT().CreateOrUpdate(gomock.Any(), "rg", "vnet", "testSubnet", test.existingSubnet).Return(nil) - err := az.SubnetsClient.CreateOrUpdate(context.TODO(), "rg", "vnet", "testSubnet", test.existingSubnet) - if err != nil { - t.Fatalf("TestCase[%d] meets unexpected error: %v", i, err) - } - - mockPIPsClient := az.PublicIPAddressesClient.(*mockpublicipclient.MockInterface) - mockPIPsClient.EXPECT().CreateOrUpdate(gomock.Any(), "rg", gomock.Any(), gomock.Any()).Return(nil).AnyTimes() - for _, existingPIP := range test.existingPIPs { - mockPIPsClient.EXPECT().Get(gomock.Any(), "rg", *existingPIP.Name, gomock.Any()).Return(existingPIP, nil).AnyTimes() - err := az.PublicIPAddressesClient.CreateOrUpdate(context.TODO(), "rg", *existingPIP.Name, existingPIP) + for _, test := range testCases { + t.Run(test.desc, func(t *testing.T) { + az := GetTestCloud(ctrl) + mockSubnetsClient := az.SubnetsClient.(*mocksubnetclient.MockInterface) + mockSubnetsClient.EXPECT().Get(gomock.Any(), "rg", "vnet", "testSubnet", "").Return(test.existingSubnet, nil).AnyTimes() + mockSubnetsClient.EXPECT().CreateOrUpdate(gomock.Any(), "rg", "vnet", "testSubnet", test.existingSubnet).Return(nil) + err := az.SubnetsClient.CreateOrUpdate(context.TODO(), "rg", "vnet", "testSubnet", test.existingSubnet) if err != nil { - t.Fatalf("TestCase[%d] meets unexpected error: %v", i, err) + t.Fatal(err) } - } - test.service.Spec.LoadBalancerIP = test.loadBalancerIP - test.service.Annotations[consts.ServiceAnnotationLoadBalancerInternalSubnet] = test.annotations - flag, rerr := az.isFrontendIPChanged("testCluster", test.config, - &test.service, test.lbFrontendIPConfigName, &test.existingPIPs) - if rerr != nil { - fmt.Println(rerr.Error()) - } - assert.Equal(t, test.expectedFlag, flag, "TestCase[%d]: %s", i, test.desc) - assert.Equal(t, test.expectedError, rerr != nil, "TestCase[%d]: %s", i, test.desc) + + mockPIPsClient := az.PublicIPAddressesClient.(*mockpublicipclient.MockInterface) + mockPIPsClient.EXPECT().CreateOrUpdate(gomock.Any(), "rg", gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + for _, existingPIP := range test.existingPIPs { + mockPIPsClient.EXPECT().Get(gomock.Any(), "rg", *existingPIP.Name, gomock.Any()).Return(existingPIP, nil).AnyTimes() + err := az.PublicIPAddressesClient.CreateOrUpdate(context.TODO(), "rg", *existingPIP.Name, existingPIP) + if err != nil { + t.Fatal(err) + } + } + setServiceLoadBalancerIP(&test.service, test.loadBalancerIP) + test.service.Annotations[consts.ServiceAnnotationLoadBalancerInternalSubnet] = test.annotations + flag, rerr := az.isFrontendIPChanged("testCluster", test.config, + &test.service, test.lbFrontendIPConfigName, &test.existingPIPs) + if rerr != nil { + fmt.Println(rerr.Error()) + } + assert.Equal(t, test.expectedFlag, flag) + assert.Equal(t, test.expectedError, rerr != nil) + }) } } @@ -1939,7 +1941,7 @@ func TestDeterminePublicIPName(t *testing.T) { for i, test := range testCases { az := GetTestCloud(ctrl) service := getTestService("test1", v1.ProtocolTCP, nil, false, 80) - service.Spec.LoadBalancerIP = test.loadBalancerIP + setServiceLoadBalancerIP(&service, test.loadBalancerIP) mockPIPsClient := az.PublicIPAddressesClient.(*mockpublicipclient.MockInterface) mockPIPsClient.EXPECT().List(gomock.Any(), "rg").Return(test.existingPIPs, nil).MaxTimes(1) @@ -2912,7 +2914,7 @@ func TestReconcileLoadBalancer(t *testing.T) { clusterResources, expectedInterfaces, expectedVirtualMachines := getClusterResources(az, 3, 3) setMockEnv(az, ctrl, expectedInterfaces, expectedVirtualMachines, 1) - test.service.Spec.LoadBalancerIP = "1.2.3.4" + setServiceLoadBalancerIP(&test.service, "1.2.3.4") err := az.PublicIPAddressesClient.CreateOrUpdate(context.TODO(), "rg", "pipName", network.PublicIPAddress{ Name: to.StringPtr("pipName"), @@ -4440,7 +4442,7 @@ func TestUnbindServiceFromPIP(t *testing.T) { } serviceName := "ns2/svc2" service := getTestService(serviceName, v1.ProtocolTCP, nil, false, 80) - service.Spec.LoadBalancerIP = "1.2.3.4" + setServiceLoadBalancerIP(&service, "1.2.3.4") expectedTags := []map[string]*string{ nil, {consts.ServiceTagKey: to.StringPtr("")}, diff --git a/pkg/provider/azure_standard.go b/pkg/provider/azure_standard.go index 365d8ad542..f72981e6a7 100644 --- a/pkg/provider/azure_standard.go +++ b/pkg/provider/azure_standard.go @@ -352,7 +352,7 @@ func (az *Cloud) serviceOwnsFrontendIP(fip network.FrontendIPConfiguration, serv return true, isPrimaryService, nil } - loadBalancerIP := service.Spec.LoadBalancerIP + loadBalancerIP := getServiceLoadBalancerIP(service) if loadBalancerIP == "" { // it is a must that the secondary services set the loadBalancer IP return false, isPrimaryService, nil diff --git a/pkg/provider/azure_standard_test.go b/pkg/provider/azure_standard_test.go index 643965cbbc..9709dc487f 100644 --- a/pkg/provider/azure_standard_test.go +++ b/pkg/provider/azure_standard_test.go @@ -1684,10 +1684,8 @@ func TestServiceOwnsFrontendIP(t *testing.T) { }, service: &v1.Service{ ObjectMeta: meta.ObjectMeta{ - UID: types.UID("secondary"), - }, - Spec: v1.ServiceSpec{ - LoadBalancerIP: "1.2.3.4", + UID: types.UID("secondary"), + Annotations: map[string]string{consts.ServiceAnnotationLoadBalancerIPDualStack[false]: "1.2.3.4"}, }, }, }, @@ -1712,10 +1710,8 @@ func TestServiceOwnsFrontendIP(t *testing.T) { }, service: &v1.Service{ ObjectMeta: meta.ObjectMeta{ - UID: types.UID("secondary"), - }, - Spec: v1.ServiceSpec{ - LoadBalancerIP: "4.3.2.1", + UID: types.UID("secondary"), + Annotations: map[string]string{consts.ServiceAnnotationLoadBalancerIPDualStack[false]: "4.3.2.1"}, }, }, }, @@ -1739,10 +1735,8 @@ func TestServiceOwnsFrontendIP(t *testing.T) { }, service: &v1.Service{ ObjectMeta: meta.ObjectMeta{ - UID: types.UID("secondary"), - }, - Spec: v1.ServiceSpec{ - LoadBalancerIP: "4.3.2.1", + UID: types.UID("secondary"), + Annotations: map[string]string{consts.ServiceAnnotationLoadBalancerIPDualStack[false]: "4.3.2.1"}, }, }, }, @@ -1766,10 +1760,8 @@ func TestServiceOwnsFrontendIP(t *testing.T) { }, service: &v1.Service{ ObjectMeta: meta.ObjectMeta{ - UID: types.UID("secondary"), - }, - Spec: v1.ServiceSpec{ - LoadBalancerIP: "4.3.2.1", + UID: types.UID("secondary"), + Annotations: map[string]string{consts.ServiceAnnotationLoadBalancerIPDualStack[false]: "4.3.2.1"}, }, }, isOwned: true, @@ -1784,11 +1776,11 @@ func TestServiceOwnsFrontendIP(t *testing.T) { }, service: &v1.Service{ ObjectMeta: meta.ObjectMeta{ - UID: types.UID("secondary"), - Annotations: map[string]string{consts.ServiceAnnotationLoadBalancerInternal: "true"}, - }, - Spec: v1.ServiceSpec{ - LoadBalancerIP: "4.3.2.1", + UID: types.UID("secondary"), + Annotations: map[string]string{ + consts.ServiceAnnotationLoadBalancerInternal: "true", + consts.ServiceAnnotationLoadBalancerIPDualStack[false]: "4.3.2.1", + }, }, }, isOwned: true, diff --git a/pkg/provider/azure_test.go b/pkg/provider/azure_test.go index b821aede76..704d590a37 100644 --- a/pkg/provider/azure_test.go +++ b/pkg/provider/azure_test.go @@ -716,7 +716,7 @@ func TestReconcileSecurityGroupFromAnyDestinationAddressPrefixToLoadBalancerIP(t az := GetTestCloud(ctrl) svc1 := getTestService("serviceea", v1.ProtocolTCP, nil, false, 80) - svc1.Spec.LoadBalancerIP = "192.168.0.0" + setServiceLoadBalancerIP(&svc1, "192.168.0.0") sg := getTestSecurityGroup(az) setMockSecurityGroup(az, ctrl, sg) @@ -726,7 +726,7 @@ func TestReconcileSecurityGroupFromAnyDestinationAddressPrefixToLoadBalancerIP(t if err != nil { t.Errorf("Unexpected error: %q", err) } - sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, true) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, true) if err != nil { t.Errorf("Unexpected error: %q", err) } @@ -739,7 +739,7 @@ func TestReconcileSecurityGroupDynamicLoadBalancerIP(t *testing.T) { az := GetTestCloud(ctrl) svc1 := getTestService("servicea", v1.ProtocolTCP, nil, false, 80) - svc1.Spec.LoadBalancerIP = "" + setServiceLoadBalancerIP(&svc1, "") sg := getTestSecurityGroup(az) setMockSecurityGroup(az, ctrl, sg) @@ -1707,7 +1707,7 @@ func getTestServiceWithAnnotation(identifier string, annotations map[string]stri func getResourceGroupTestService(identifier, resourceGroup, loadBalancerIP string, requestedPorts ...int32) v1.Service { svc := getTestService(identifier, v1.ProtocolTCP, nil, false, requestedPorts...) - svc.Spec.LoadBalancerIP = loadBalancerIP + setServiceLoadBalancerIP(&svc, loadBalancerIP) svc.Annotations[consts.ServiceAnnotationLoadBalancerResourceGroup] = resourceGroup return svc } @@ -1930,7 +1930,7 @@ func validatePublicIP(t *testing.T, publicIP *network.PublicIPAddress, service * t.Errorf("Expected publicIP resource has matching tags[%s]", consts.ClusterNameKey) } - // We cannot use service.Spec.LoadBalancerIP to compare with + // We cannot use Service LoadBalancerIP to compare with // Public IP's IPAddress // Because service properties are updated outside of cloudprovider code } @@ -1994,6 +1994,7 @@ func validateSecurityGroup(t *testing.T, securityGroup *network.SecurityGroup, s az := GetTestCloud(ctrl) seenRules := make(map[string]string) for i, svc := range services { + svc := svc for _, wantedRule := range svc.Spec.Ports { sources := getServiceSourceRanges(&services[i]) for _, source := range sources { @@ -2002,7 +2003,7 @@ func validateSecurityGroup(t *testing.T, securityGroup *network.SecurityGroup, s foundRule := false for _, actualRule := range *securityGroup.SecurityRules { if strings.EqualFold(*actualRule.Name, wantedRuleName) { - err := securityRuleMatches(source, wantedRule, svc.Spec.LoadBalancerIP, actualRule) + err := securityRuleMatches(source, wantedRule, getServiceLoadBalancerIP(&svc), actualRule) if err != nil { t.Errorf("Found matching security rule %q but properties were incorrect: %v", wantedRuleName, err) } @@ -2437,13 +2438,13 @@ func TestIfServiceSpecifiesSharedRuleAndRuleDoesNotExistItIsCreated(t *testing.T az := GetTestCloud(ctrl) svc := getTestService("servicea", v1.ProtocolTCP, nil, false, 80) - svc.Spec.LoadBalancerIP = testIP1 + setServiceLoadBalancerIP(&svc, testIP1) svc.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue sg := getTestSecurityGroup(az) setMockSecurityGroup(az, ctrl, sg) - sg, err := az.reconcileSecurityGroup(testClusterName, &svc, to.StringPtr(svc.Spec.LoadBalancerIP), nil, true) + sg, err := az.reconcileSecurityGroup(testClusterName, &svc, to.StringPtr(getServiceLoadBalancerIP(&svc)), nil, true) if err != nil { t.Errorf("Unexpected error: %q", err) } @@ -2480,7 +2481,7 @@ func TestIfServiceSpecifiesSharedRuleAndRuleExistsThenTheServicesPortAndAddressA az := GetTestCloud(ctrl) svc := getTestService("servicesr", v1.ProtocolTCP, nil, false, 80) - svc.Spec.LoadBalancerIP = testIP1 + setServiceLoadBalancerIP(&svc, testIP1) svc.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue expectedRuleName := testRuleName @@ -2502,7 +2503,7 @@ func TestIfServiceSpecifiesSharedRuleAndRuleExistsThenTheServicesPortAndAddressA } setMockSecurityGroup(az, ctrl, sg) - sg, err := az.reconcileSecurityGroup(testClusterName, &svc, to.StringPtr(svc.Spec.LoadBalancerIP), nil, true) + sg, err := az.reconcileSecurityGroup(testClusterName, &svc, to.StringPtr(getServiceLoadBalancerIP(&svc)), nil, true) if err != nil { t.Errorf("Unexpected error: %q", err) } @@ -2536,22 +2537,22 @@ func TestIfServicesSpecifySharedRuleButDifferentPortsThenSeparateRulesAreCreated az := GetTestCloud(ctrl) svc1 := getTestService("servicesr1", v1.ProtocolTCP, nil, false, 4444) - svc1.Spec.LoadBalancerIP = testIP1 + setServiceLoadBalancerIP(&svc1, testIP1) svc1.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue svc2 := getTestService("servicesr2", v1.ProtocolTCP, nil, false, 8888) - svc2.Spec.LoadBalancerIP = testIP2 + setServiceLoadBalancerIP(&svc2, testIP2) svc2.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue sg := getTestSecurityGroup(az) setMockSecurityGroup(az, ctrl, sg) - _, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, true) + _, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc1: %q", err) } - sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), nil, true) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(getServiceLoadBalancerIP(&svc2)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc2: %q", err) } @@ -2605,11 +2606,12 @@ func TestIfServicesSpecifySharedRuleButDifferentProtocolsThenSeparateRulesAreCre az := GetTestCloud(ctrl) svc1 := getTestService("servicesr1", v1.ProtocolTCP, nil, false, 4444) - svc1.Spec.LoadBalancerIP = testIP1 + + setServiceLoadBalancerIP(&svc1, testIP1) svc1.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue svc2 := getTestService("servicesr2", v1.ProtocolUDP, nil, false, 4444) - svc2.Spec.LoadBalancerIP = testIP1 + setServiceLoadBalancerIP(&svc2, testIP1) svc2.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue testRuleName3 := "shared-UDP-4444-Internet" @@ -2617,12 +2619,12 @@ func TestIfServicesSpecifySharedRuleButDifferentProtocolsThenSeparateRulesAreCre sg := getTestSecurityGroup(az) setMockSecurityGroup(az, ctrl, sg) - _, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, true) + _, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc1: %q", err) } - sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), nil, true) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(getServiceLoadBalancerIP(&svc2)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc2: %q", err) } @@ -2674,12 +2676,12 @@ func TestIfServicesSpecifySharedRuleButDifferentSourceAddressesThenSeparateRules az := GetTestCloud(ctrl) svc1 := getTestService("servicesr1", v1.ProtocolTCP, nil, false, 80) - svc1.Spec.LoadBalancerIP = testIP1 + setServiceLoadBalancerIP(&svc1, testIP1) svc1.Spec.LoadBalancerSourceRanges = []string{"192.168.12.0/24"} svc1.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue svc2 := getTestService("servicesr2", v1.ProtocolTCP, nil, false, 80) - svc2.Spec.LoadBalancerIP = testIP2 + setServiceLoadBalancerIP(&svc2, testIP2) svc2.Spec.LoadBalancerSourceRanges = []string{"192.168.34.0/24"} svc2.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue @@ -2689,12 +2691,12 @@ func TestIfServicesSpecifySharedRuleButDifferentSourceAddressesThenSeparateRules sg := getTestSecurityGroup(az) setMockSecurityGroup(az, ctrl, sg) - _, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, true) + _, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc1: %q", err) } - sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), nil, true) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(getServiceLoadBalancerIP(&svc2)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc2: %q", err) } @@ -2748,32 +2750,32 @@ func TestIfServicesSpecifySharedRuleButSomeAreOnDifferentPortsThenRulesAreSepara az := GetTestCloud(ctrl) svc1 := getTestService("servicesr1", v1.ProtocolTCP, nil, false, 4444) - svc1.Spec.LoadBalancerIP = testIP1 + setServiceLoadBalancerIP(&svc1, testIP1) svc1.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue svc2 := getTestService("servicesr2", v1.ProtocolTCP, nil, false, 8888) - svc2.Spec.LoadBalancerIP = testIP2 + setServiceLoadBalancerIP(&svc2, testIP2) svc2.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue svc3 := getTestService("servicesr3", v1.ProtocolTCP, nil, false, 4444) - svc3.Spec.LoadBalancerIP = testIP3 + setServiceLoadBalancerIP(&svc3, testIP3) svc3.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue testRuleName23 := testRuleName2 sg := getTestSecurityGroup(az) setMockSecurityGroup(az, ctrl, sg) - _, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, true) + _, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc1: %q", err) } - _, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), nil, true) + _, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(getServiceLoadBalancerIP(&svc2)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc2: %q", err) } - sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, to.StringPtr(svc3.Spec.LoadBalancerIP), nil, true) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, to.StringPtr(getServiceLoadBalancerIP(&svc3)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc3: %q", err) } @@ -2849,11 +2851,11 @@ func TestIfServiceSpecifiesSharedRuleAndServiceIsDeletedThenTheServicesPortAndAd az := GetTestCloud(ctrl) svc1 := getTestService("servicesr1", v1.ProtocolTCP, nil, false, 80) - svc1.Spec.LoadBalancerIP = testIP1 + setServiceLoadBalancerIP(&svc1, testIP1) svc1.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue svc2 := getTestService("servicesr2", v1.ProtocolTCP, nil, false, 80) - svc2.Spec.LoadBalancerIP = testIP2 + setServiceLoadBalancerIP(&svc2, testIP2) svc2.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue expectedRuleName := testRuleName @@ -2861,19 +2863,19 @@ func TestIfServiceSpecifiesSharedRuleAndServiceIsDeletedThenTheServicesPortAndAd sg := getTestSecurityGroup(az) setMockSecurityGroup(az, ctrl, sg) - _, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, true) + _, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc1: %q", err) } - sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), nil, true) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(getServiceLoadBalancerIP(&svc2)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc2: %q", err) } validateSecurityGroup(t, sg, svc1, svc2) - sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, false) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, false) if err != nil { t.Errorf("Unexpected error removing svc1: %q", err) } @@ -2907,39 +2909,39 @@ func TestIfSomeServicesShareARuleAndOneIsDeletedItIsRemovedFromTheRightRule(t *t az := GetTestCloud(ctrl) svc1 := getTestService("servicesr1", v1.ProtocolTCP, nil, false, 4444) - svc1.Spec.LoadBalancerIP = testIP1 + setServiceLoadBalancerIP(&svc1, testIP1) svc1.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue svc2 := getTestService("servicesr2", v1.ProtocolTCP, nil, false, 8888) - svc2.Spec.LoadBalancerIP = testIP2 + setServiceLoadBalancerIP(&svc2, testIP2) svc2.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue svc3 := getTestService("servicesr3", v1.ProtocolTCP, nil, false, 4444) - svc3.Spec.LoadBalancerIP = testIP3 + setServiceLoadBalancerIP(&svc3, testIP3) svc3.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue testRuleName23 := testRuleName2 sg := getTestSecurityGroup(az) setMockSecurityGroup(az, ctrl, sg) - _, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, true) + _, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc1: %q", err) } - _, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), nil, true) + _, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(getServiceLoadBalancerIP(&svc2)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc2: %q", err) } - sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, to.StringPtr(svc3.Spec.LoadBalancerIP), nil, true) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, to.StringPtr(getServiceLoadBalancerIP(&svc3)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc3: %q", err) } validateSecurityGroup(t, sg, svc1, svc2, svc3) - sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, false) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, false) if err != nil { t.Errorf("Unexpected error removing svc1: %q", err) } @@ -3015,44 +3017,44 @@ func TestIfServiceSpecifiesSharedRuleAndLastServiceIsDeletedThenRuleIsDeleted(t az := GetTestCloud(ctrl) svc1 := getTestService("servicesr1", v1.ProtocolTCP, nil, false, 4444) - svc1.Spec.LoadBalancerIP = testIP1 + setServiceLoadBalancerIP(&svc1, testIP1) svc1.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue svc2 := getTestService("servicesr2", v1.ProtocolTCP, nil, false, 8888) - svc2.Spec.LoadBalancerIP = testIP2 + setServiceLoadBalancerIP(&svc2, testIP2) svc2.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue svc3 := getTestService("servicesr3", v1.ProtocolTCP, nil, false, 4444) - svc3.Spec.LoadBalancerIP = testIP3 + setServiceLoadBalancerIP(&svc3, testIP3) svc3.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue testRuleName23 := testRuleName2 sg := getTestSecurityGroup(az) setMockSecurityGroup(az, ctrl, sg) - _, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, true) + _, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc1: %q", err) } - _, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), nil, true) + _, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(getServiceLoadBalancerIP(&svc2)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc2: %q", err) } - sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, to.StringPtr(svc3.Spec.LoadBalancerIP), nil, true) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, to.StringPtr(getServiceLoadBalancerIP(&svc3)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc3: %q", err) } validateSecurityGroup(t, sg, svc1, svc2, svc3) - _, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, false) + _, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, false) if err != nil { t.Errorf("Unexpected error removing svc1: %q", err) } - sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, to.StringPtr(svc3.Spec.LoadBalancerIP), nil, false) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, to.StringPtr(getServiceLoadBalancerIP(&svc3)), nil, false) if err != nil { t.Errorf("Unexpected error removing svc3: %q", err) } @@ -3097,23 +3099,23 @@ func TestCanCombineSharedAndPrivateRulesInSameGroup(t *testing.T) { var err error svc1 := getTestService("servicesr1", v1.ProtocolTCP, nil, false, 4444) - svc1.Spec.LoadBalancerIP = testIP1 + setServiceLoadBalancerIP(&svc1, testIP1) svc1.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue svc2 := getTestService("servicesr2", v1.ProtocolTCP, nil, false, 8888) - svc2.Spec.LoadBalancerIP = testIP2 + setServiceLoadBalancerIP(&svc2, testIP2) svc2.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue svc3 := getTestService("servicesr3", v1.ProtocolTCP, nil, false, 4444) - svc3.Spec.LoadBalancerIP = testIP3 + setServiceLoadBalancerIP(&svc3, testIP3) svc3.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue svc4 := getTestService("servicesr4", v1.ProtocolTCP, nil, false, 4444) - svc4.Spec.LoadBalancerIP = "192.168.22.33" + setServiceLoadBalancerIP(&svc4, "192.168.22.33") svc4.Annotations[consts.ServiceAnnotationSharedSecurityRule] = "false" svc5 := getTestService("servicesr5", v1.ProtocolTCP, nil, false, 8888) - svc5.Spec.LoadBalancerIP = "192.168.22.33" + setServiceLoadBalancerIP(&svc5, "192.168.22.33") svc5.Annotations[consts.ServiceAnnotationSharedSecurityRule] = "false" testServices := []v1.Service{svc1, svc2, svc3, svc4, svc5} @@ -3126,7 +3128,8 @@ func TestCanCombineSharedAndPrivateRulesInSameGroup(t *testing.T) { setMockSecurityGroup(az, ctrl, sg) for i, svc := range testServices { - _, err := az.reconcileSecurityGroup(testClusterName, &testServices[i], to.StringPtr(svc.Spec.LoadBalancerIP), nil, true) + svc := svc + _, err := az.reconcileSecurityGroup(testClusterName, &testServices[i], to.StringPtr(getServiceLoadBalancerIP(&svc)), nil, true) if err != nil { t.Errorf("Unexpected error adding svc%d: %q", i+1, err) } @@ -3201,8 +3204,8 @@ func TestCanCombineSharedAndPrivateRulesInSameGroup(t *testing.T) { if securityRule4.DestinationAddressPrefix == nil { t.Errorf("Expected unshared rule %s to have a destination IP address", expectedRuleName4) } else { - if !strings.EqualFold(*securityRule4.DestinationAddressPrefix, svc4.Spec.LoadBalancerIP) { - t.Errorf("Expected unshared rule %s to have a destination %s but had %s", expectedRuleName4, svc4.Spec.LoadBalancerIP, *securityRule4.DestinationAddressPrefix) + if !strings.EqualFold(*securityRule4.DestinationAddressPrefix, getServiceLoadBalancerIP(&svc4)) { + t.Errorf("Expected unshared rule %s to have a destination %s but had %s", expectedRuleName4, getServiceLoadBalancerIP(&svc4), *securityRule4.DestinationAddressPrefix) } } @@ -3213,17 +3216,17 @@ func TestCanCombineSharedAndPrivateRulesInSameGroup(t *testing.T) { if securityRule5.DestinationAddressPrefix == nil { t.Errorf("Expected unshared rule %s to have a destination IP address", expectedRuleName5) } else { - if !strings.EqualFold(*securityRule5.DestinationAddressPrefix, svc5.Spec.LoadBalancerIP) { - t.Errorf("Expected unshared rule %s to have a destination %s but had %s", expectedRuleName5, svc5.Spec.LoadBalancerIP, *securityRule5.DestinationAddressPrefix) + if !strings.EqualFold(*securityRule5.DestinationAddressPrefix, getServiceLoadBalancerIP(&svc5)) { + t.Errorf("Expected unshared rule %s to have a destination %s but had %s", expectedRuleName5, getServiceLoadBalancerIP(&svc5), *securityRule5.DestinationAddressPrefix) } } - _, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, false) + _, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, false) if err != nil { t.Errorf("Unexpected error removing svc1: %q", err) } - sg, err = az.reconcileSecurityGroup(testClusterName, &svc5, to.StringPtr(svc5.Spec.LoadBalancerIP), nil, false) + sg, err = az.reconcileSecurityGroup(testClusterName, &svc5, to.StringPtr(getServiceLoadBalancerIP(&svc5)), nil, false) if err != nil { t.Errorf("Unexpected error removing svc5: %q", err) } diff --git a/site/content/en/topics/pls-integration.md b/site/content/en/topics/pls-integration.md index b641541e2d..677193610f 100644 --- a/site/content/en/topics/pls-integration.md +++ b/site/content/en/topics/pls-integration.md @@ -35,7 +35,7 @@ For more details about each configuration, please refer to [Azure Private Link S ### Creating managed PrivateLinkService -When a `LoadBalancer` typed service is created without the `loadBalancerIP` field specified, an LB frontend IP configuration is created with a dynamically generated IP. If the service has `loadBalancerIP` in its spec, an existing LB frontend IP configuration may be reused if one exists; otherwise a static configuration is created with the specified IP. When a service is created with annotation `service.beta.kubernetes.io/azure-pls-create` set to `true` or updated later with the annotation added, a PLS resource attached to the LB frontend is created in the default resource group or the resource group user set in config file with key `PrivateLinkServiceResourceGroup`. +When a `LoadBalancer` typed service is created without the annotation `service.beta.kubernetes.io/azure-load-balancer-ipv4` or `service.beta.kubernetes.io/azure-load-balancer-ipv6` set, an LB frontend IP configuration is created with a dynamically generated IP. If the service has the annotation `service.beta.kubernetes.io/azure-load-balancer-ipv4` or `service.beta.kubernetes.io/azure-load-balancer-ipv6` set, an existing LB frontend IP configuration may be reused if one exists; otherwise a static configuration is created with the specified IP. When a service is created with annotation `service.beta.kubernetes.io/azure-pls-create` set to `true` or updated later with the annotation added, a PLS resource attached to the LB frontend is created in the default resource group or the resource group user set in config file with key `PrivateLinkServiceResourceGroup`. The Kubernetes service creating the PLS is assigned as the owner of the resource. Azure cloud provider tags the PLS with cluster name and service name `kubernetes-owner-service: /`. Only the owner service can later update the properties of the PLS resource. @@ -51,7 +51,7 @@ If there are active PE connections to the PLS, all connections are removed and t ### Sharing managed PrivateLinkService -Multiple Kubernetes services can share the same LB frontend by specifying the same `loadBalancerIP` (for more details, please refer to [Multiple Services Sharing One IP Address](../shared-ip)). Once a PLS is attached to the LB frontend, these services automatically share the PLS. Users can access these services via the same PE but different ports. +Multiple Kubernetes services can share the same LB frontend by specifying the same annotation `service.beta.kubernetes.io/azure-load-balancer-ipv4` or `service.beta.kubernetes.io/azure-load-balancer-ipv6` (for more details, please refer to [Multiple Services Sharing One IP Address](../shared-ip)). Once a PLS is attached to the LB frontend, these services automatically share the PLS. Users can access these services via the same PE but different ports. Azure cloud provider tags the service creating the PLS as the owner (`kubernetes-owner-service: /`) and only allows that service to update the configurations of the PLS. If the owner service is deleted or if user wants some other service to take control, user can modify the tag value to a new service in `/` pattern. diff --git a/site/content/en/topics/shared-ip.md b/site/content/en/topics/shared-ip.md index 2322d66405..1629883277 100644 --- a/site/content/en/topics/shared-ip.md +++ b/site/content/en/topics/shared-ip.md @@ -27,7 +27,7 @@ spec: type: LoadBalancer ``` -Note that the `loadBalancerIP` is not set, or Azure would find a pre-allocated public IP with the address. After obtaining the IP address of the service, you could create other services using this address. +Note that the annotation `service.beta.kubernetes.io/azure-load-balancer-ipv4` or `service.beta.kubernetes.io/azure-load-balancer-ipv6` is not set, or Azure would find a pre-allocated public IP with the address. After obtaining the IP address of the service, you could create other services using this address. ```yaml apiVersion: v1 @@ -35,8 +35,8 @@ kind: Service metadata: name: https namespace: default + service.beta.kubernetes.io/azure-load-balancer-ipv4: 1.2.3.4 # the IP address could be the same as it is of `nginx` service spec: - loadBalancerIP: 1.2.3.4 # the IP address could be the same as it is of `nginx` service ports: - port: 443 protocol: TCP @@ -46,7 +46,7 @@ spec: type: LoadBalancer ``` -Note that if you specify the `loadBalancerIP` but there is no corresponding public IP pre-allocated, an error would be reported. +Note that if you specify the annotation `service.beta.kubernetes.io/azure-load-balancer-ipv4` or `service.beta.kubernetes.io/azure-load-balancer-ipv6` but there is no corresponding public IP pre-allocated, an error would be reported. ## DNS