diff --git a/tests/e2e/network/ensureloadbalancer.go b/tests/e2e/network/ensureloadbalancer.go index 8dee20befe..f7cc0c262c 100644 --- a/tests/e2e/network/ensureloadbalancer.go +++ b/tests/e2e/network/ensureloadbalancer.go @@ -44,6 +44,15 @@ const ( testServiceName = "servicelb-test" ) +var ( + serviceAnnotationLoadBalancerInternalFalse = map[string]string{ + consts.ServiceAnnotationLoadBalancerInternal: "false", + } + serviceAnnotationLoadBalancerInternalTrue = map[string]string{ + consts.ServiceAnnotationLoadBalancerInternal: "true", + } +) + var _ = Describe("Ensure LoadBalancer", func() { basename := "service-lb" @@ -153,7 +162,7 @@ var _ = Describe("Ensure LoadBalancer", func() { Expect(ip).To(Equal(targetIP)) By("deleting the service") - err = cs.CoreV1().Services(ns.Name).Delete(context.TODO(), testServiceName, metav1.DeleteOptions{}) + err = utils.DeleteService(cs, ns.Name, testServiceName) Expect(err).NotTo(HaveOccurred()) By("test if the pip still exists") @@ -170,12 +179,9 @@ var _ = Describe("Ensure LoadBalancer", func() { // Public w/o IP -> Public w/ IP It("should support assigning to specific IP when updating public service", func() { - annotation := map[string]string{ - consts.ServiceAnnotationLoadBalancerInternal: "false", - } ipName := basename + "-public-none-IP" + string(uuid.NewUUID())[0:4] - service := utils.CreateLoadBalancerServiceManifest(testServiceName, annotation, labels, ns.Name, ports) + service := utils.CreateLoadBalancerServiceManifest(testServiceName, serviceAnnotationLoadBalancerInternalFalse, labels, ns.Name, ports) _, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) utils.Logf("Successfully created LoadBalancer service " + testServiceName + " in namespace " + ns.Name) @@ -187,7 +193,7 @@ var _ = Describe("Ensure LoadBalancer", func() { defer func() { By("Cleaning up") - err = cs.CoreV1().Services(ns.Name).Delete(context.TODO(), testServiceName, metav1.DeleteOptions{}) + err = utils.DeleteService(cs, ns.Name, testServiceName) Expect(err).NotTo(HaveOccurred()) err = utils.DeletePIPWithRetry(tc, ipName, "") Expect(err).NotTo(HaveOccurred()) @@ -214,13 +220,10 @@ var _ = Describe("Ensure LoadBalancer", func() { // Internal w/ IP -> Internal w/ IP It("should support updating internal IP when updating internal service", func() { - annotation := map[string]string{ - consts.ServiceAnnotationLoadBalancerInternal: "true", - } ip1, err := utils.SelectAvailablePrivateIP(tc) Expect(err).NotTo(HaveOccurred()) - service := utils.CreateLoadBalancerServiceManifest(testServiceName, annotation, labels, ns.Name, ports) + service := utils.CreateLoadBalancerServiceManifest(testServiceName, serviceAnnotationLoadBalancerInternalTrue, labels, ns.Name, ports) service = updateServiceBalanceIP(service, true, ip1) _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -228,7 +231,7 @@ var _ = Describe("Ensure LoadBalancer", func() { defer func() { By("Cleaning up") - err = cs.CoreV1().Services(ns.Name).Delete(context.TODO(), testServiceName, metav1.DeleteOptions{}) + err = utils.DeleteService(cs, ns.Name, testServiceName) Expect(err).NotTo(HaveOccurred()) }() By("Waiting for exposure of internal service with specific IP") @@ -259,12 +262,9 @@ var _ = Describe("Ensure LoadBalancer", func() { // internal w/o IP -> public w/ IP It("should support updating an internal service to a public service with assigned IP", func() { - annotation := map[string]string{ - consts.ServiceAnnotationLoadBalancerInternal: "true", - } ipName := basename + "-internal-none-public-IP" + string(uuid.NewUUID())[0:4] - service := utils.CreateLoadBalancerServiceManifest(testServiceName, annotation, labels, ns.Name, ports) + service := utils.CreateLoadBalancerServiceManifest(testServiceName, serviceAnnotationLoadBalancerInternalTrue, labels, ns.Name, ports) _, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) utils.Logf("Successfully created LoadBalancer service " + testServiceName + " in namespace " + ns.Name) @@ -275,7 +275,7 @@ var _ = Describe("Ensure LoadBalancer", func() { defer func() { By("Cleaning up") - err = cs.CoreV1().Services(ns.Name).Delete(context.TODO(), testServiceName, metav1.DeleteOptions{}) + err = utils.DeleteService(cs, ns.Name, testServiceName) Expect(err).NotTo(HaveOccurred()) err = utils.DeletePIPWithRetry(tc, ipName, "") Expect(err).NotTo(HaveOccurred()) @@ -306,24 +306,21 @@ var _ = Describe("Ensure LoadBalancer", func() { }) It("should have no operation since no change in service when update [Slow]", func() { - annotation := map[string]string{ - consts.ServiceAnnotationLoadBalancerInternal: "false", - } suffix := string(uuid.NewUUID())[0:4] ipName := basename + "-public-remain" + suffix pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), defaultPublicIPAddress(ipName)) Expect(err).NotTo(HaveOccurred()) targetIP := to.String(pip.IPAddress) - service := utils.CreateLoadBalancerServiceManifest(testServiceName, annotation, labels, ns.Name, ports) + service := utils.CreateLoadBalancerServiceManifest(testServiceName, serviceAnnotationLoadBalancerInternalFalse, labels, ns.Name, ports) service = updateServiceBalanceIP(service, false, targetIP) _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - utils.Logf("Successfully created LoadBalancer service " + testServiceName + " in namespace " + ns.Name) + utils.Logf("Successfully created LoadBalancer service %s in namespace %s", testServiceName, ns.Name) defer func() { By("Cleaning up") - err = cs.CoreV1().Services(ns.Name).Delete(context.TODO(), testServiceName, metav1.DeleteOptions{}) + err = utils.DeleteService(cs, ns.Name, testServiceName) Expect(err).NotTo(HaveOccurred()) err = utils.DeletePIPWithRetry(tc, ipName, "") Expect(err).NotTo(HaveOccurred()) @@ -351,13 +348,13 @@ var _ = Describe("Ensure LoadBalancer", func() { return false, err } - IngressList := service.Status.LoadBalancer.Ingress - if len(IngressList) == 0 { + ingressList := service.Status.LoadBalancer.Ingress + if len(ingressList) == 0 { err = fmt.Errorf("Cannot find Ingress in limited time") utils.Logf("Fail to get ingress, retry it in %s seconds", 10) return false, nil } - if targetIP == service.Status.LoadBalancer.Ingress[0].IP { + if targetIP == ingressList[0].IP { utils.Logf("External IP is still %s", targetIP) return false, nil } @@ -395,9 +392,9 @@ var _ = Describe("Ensure LoadBalancer", func() { defer func() { By("Cleaning up") - err = cs.CoreV1().Services(ns.Name).Delete(context.TODO(), "service1", metav1.DeleteOptions{}) + err = utils.DeleteService(cs, ns.Name, "service1") Expect(err).NotTo(HaveOccurred()) - err = cs.CoreV1().Services(ns.Name).Delete(context.TODO(), "service2", metav1.DeleteOptions{}) + err = utils.DeleteService(cs, ns.Name, "service2") Expect(err).NotTo(HaveOccurred()) err = utils.DeletePIPWithRetry(tc, ipName, "") Expect(err).NotTo(HaveOccurred()) @@ -425,13 +422,13 @@ var _ = Describe("Ensure LoadBalancer", func() { utils.Logf("Successfully created LoadBalancer service2 in namespace %s with IP %s", ns.Name, ip) By("Deleting one service and check if the other service works well") - err = cs.CoreV1().Services(ns.Name).Delete(context.TODO(), "service1", metav1.DeleteOptions{}) + err = utils.DeleteService(cs, ns.Name, "service1") Expect(err).NotTo(HaveOccurred()) _, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, "service2", ip) Expect(err).NotTo(HaveOccurred()) By("Deleting all services") - err = cs.CoreV1().Services(ns.Name).Delete(context.TODO(), "service2", metav1.DeleteOptions{}) + err = utils.DeleteService(cs, ns.Name, "service2") Expect(err).NotTo(HaveOccurred()) By("Checking if the public IP has been deleted") @@ -454,10 +451,7 @@ var _ = Describe("Ensure LoadBalancer", func() { }) It("should support multiple internal services sharing one IP address", func() { - annotation := map[string]string{ - consts.ServiceAnnotationLoadBalancerInternal: "true", - } - service1 := utils.CreateLoadBalancerServiceManifest("service1", annotation, labels, ns.Name, ports) + service1 := utils.CreateLoadBalancerServiceManifest("service1", serviceAnnotationLoadBalancerInternalTrue, labels, ns.Name, ports) _, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service1, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, "service1", "") @@ -468,7 +462,7 @@ var _ = Describe("Ensure LoadBalancer", func() { Port: testingPort, TargetPort: intstr.FromInt(testingPort), }} - service2 := utils.CreateLoadBalancerServiceManifest("service2", annotation, labels, ns.Name, ports2) + service2 := utils.CreateLoadBalancerServiceManifest("service2", serviceAnnotationLoadBalancerInternalTrue, labels, ns.Name, ports2) service2.Spec.LoadBalancerIP = ip _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service2, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -478,14 +472,15 @@ var _ = Describe("Ensure LoadBalancer", func() { defer func() { By("Cleaning up") - err = cs.CoreV1().Services(ns.Name).Delete(context.TODO(), "service1", metav1.DeleteOptions{}) + err = utils.DeleteService(cs, ns.Name, "service1") Expect(err).NotTo(HaveOccurred()) - err = cs.CoreV1().Services(ns.Name).Delete(context.TODO(), "service2", metav1.DeleteOptions{}) + err = utils.DeleteService(cs, ns.Name, "service2") Expect(err).NotTo(HaveOccurred()) }() }) It("should support node label `node.kubernetes.io/exclude-from-external-load-balancers`", func() { + label := "node.kubernetes.io/exclude-from-external-load-balancers" By("Checking the number of the node pools") nodes, err := utils.GetAgentNodes(cs) Expect(err).NotTo(HaveOccurred()) @@ -509,7 +504,7 @@ var _ = Describe("Ensure LoadBalancer", func() { Expect(len(*lbBackendPoolIPConfigs)).To(Equal(len(nodes))) By("Labeling node") - node, err := utils.LabelNode(cs, &nodes[0], "node.kubernetes.io/exclude-from-external-load-balancers", false) + node, err := utils.LabelNode(cs, &nodes[0], label, false) Expect(err).NotTo(HaveOccurred()) err = waitForNodesInLBBackendPool(tc, publicIP, len(nodes)-1) Expect(err).NotTo(HaveOccurred()) @@ -517,7 +512,7 @@ var _ = Describe("Ensure LoadBalancer", func() { By("Unlabeling node") node, err = utils.GetNode(cs, node.Name) Expect(err).NotTo(HaveOccurred()) - _, err = utils.LabelNode(cs, node, "node.kubernetes.io/exclude-from-external-load-balancers", true) + _, err = utils.LabelNode(cs, node, label, true) Expect(err).NotTo(HaveOccurred()) err = waitForNodesInLBBackendPool(tc, publicIP, len(nodes)) Expect(err).NotTo(HaveOccurred()) @@ -531,7 +526,7 @@ func waitForNodesInLBBackendPool(tc *utils.AzureTestClient, ip string, expectedN if len(*lbBackendPoolIPConfigs) == expectedNum { return true, nil } - utils.Logf("There are %d nodes in the LB backend pool, will retry soon", len(*lbBackendPoolIPConfigs)) + utils.Logf("Number of IP configs: %d in the LB backend pool, will retry soon", len(*lbBackendPoolIPConfigs)) return false, nil }) } diff --git a/tests/e2e/network/service_annotations.go b/tests/e2e/network/service_annotations.go index 8f8d2ed8eb..f56be82b38 100644 --- a/tests/e2e/network/service_annotations.go +++ b/tests/e2e/network/service_annotations.go @@ -265,7 +265,7 @@ var _ = Describe("Service with annotation", func() { pip, err := utils.WaitCreatePIP(tc, testPIPName, *rg.Name, defaultPublicIPAddress(testPIPName)) Expect(err).NotTo(HaveOccurred()) defer func() { - By("Cleaning up service and public IP") + utils.Logf("Cleaning up service and public IP") err = utils.DeleteService(cs, ns.Name, serviceName) Expect(err).NotTo(HaveOccurred()) err = utils.DeletePIPWithRetry(tc, testPIPName, *rg.Name) diff --git a/tests/e2e/utils/node_utils.go b/tests/e2e/utils/node_utils.go index 3a91c9e4a1..28194af453 100644 --- a/tests/e2e/utils/node_utils.go +++ b/tests/e2e/utils/node_utils.go @@ -190,8 +190,7 @@ func DeleteNodes(cs clientset.Interface, names []string) error { //deleteNodes deletes nodes according to names func deleteNode(cs clientset.Interface, name string) error { Logf("Deleting node: %s", name) - zero := int64(0) - if err := cs.CoreV1().Nodes().Delete(context.TODO(), name, metav1.DeleteOptions{GracePeriodSeconds: &zero}); err != nil { + if err := cs.CoreV1().Nodes().Delete(context.TODO(), name, metav1.DeleteOptions{}); err != nil { return err } diff --git a/tests/e2e/utils/service_utils.go b/tests/e2e/utils/service_utils.go index 29e1554fbc..564194e70b 100644 --- a/tests/e2e/utils/service_utils.go +++ b/tests/e2e/utils/service_utils.go @@ -44,9 +44,8 @@ const ( // DeleteService deletes a service func DeleteService(cs clientset.Interface, ns string, serviceName string) error { - zero := int64(0) - err := cs.CoreV1().Services(ns).Delete(context.TODO(), serviceName, metav1.DeleteOptions{GracePeriodSeconds: &zero}) Logf("Deleting service %s in namespace %s", serviceName, ns) + err := cs.CoreV1().Services(ns).Delete(context.TODO(), serviceName, metav1.DeleteOptions{}) if err != nil { return err }