Skip to content

Commit

Permalink
Merge pull request #1236 from lzhecheng/refactor-e2e-svc-node
Browse files Browse the repository at this point in the history 
[e2e] Refactor: Service and Node deletion
  • Loading branch information
@lzhecheng
lzhecheng authored Mar 11, 2022
2 parents 80179be + fefe715 commit 10a510e
Show file tree
Hide file tree
Showing 4 changed files with 37 additions and 44 deletions.
73 changes: 34 additions & 39 deletions tests/e2e/network/ensureloadbalancer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,15 @@ const (
testServiceName = "servicelb-test"
)

var (
serviceAnnotationLoadBalancerInternalFalse = map[string]string{
consts.ServiceAnnotationLoadBalancerInternal: "false",
}
serviceAnnotationLoadBalancerInternalTrue = map[string]string{
consts.ServiceAnnotationLoadBalancerInternal: "true",
}
)

var _ = Describe("Ensure LoadBalancer", func() {
basename := "service-lb"

Expand Down Expand Up @@ -153,7 +162,7 @@ var _ = Describe("Ensure LoadBalancer", func() {
Expect(ip).To(Equal(targetIP))

By("deleting the service")
err = cs.CoreV1().Services(ns.Name).Delete(context.TODO(), testServiceName, metav1.DeleteOptions{})
err = utils.DeleteService(cs, ns.Name, testServiceName)
Expect(err).NotTo(HaveOccurred())

By("test if the pip still exists")
Expand All @@ -170,12 +179,9 @@ var _ = Describe("Ensure LoadBalancer", func() {

// Public w/o IP -> Public w/ IP
It("should support assigning to specific IP when updating public service", func() {
annotation := map[string]string{
consts.ServiceAnnotationLoadBalancerInternal: "false",
}
ipName := basename + "-public-none-IP" + string(uuid.NewUUID())[0:4]

service := utils.CreateLoadBalancerServiceManifest(testServiceName, annotation, labels, ns.Name, ports)
service := utils.CreateLoadBalancerServiceManifest(testServiceName, serviceAnnotationLoadBalancerInternalFalse, labels, ns.Name, ports)
_, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
utils.Logf("Successfully created LoadBalancer service " + testServiceName + " in namespace " + ns.Name)
Expand All @@ -187,7 +193,7 @@ var _ = Describe("Ensure LoadBalancer", func() {

defer func() {
By("Cleaning up")
err = cs.CoreV1().Services(ns.Name).Delete(context.TODO(), testServiceName, metav1.DeleteOptions{})
err = utils.DeleteService(cs, ns.Name, testServiceName)
Expect(err).NotTo(HaveOccurred())
err = utils.DeletePIPWithRetry(tc, ipName, "")
Expect(err).NotTo(HaveOccurred())
Expand All @@ -214,21 +220,18 @@ var _ = Describe("Ensure LoadBalancer", func() {

// Internal w/ IP -> Internal w/ IP
It("should support updating internal IP when updating internal service", func() {
annotation := map[string]string{
consts.ServiceAnnotationLoadBalancerInternal: "true",
}
ip1, err := utils.SelectAvailablePrivateIP(tc)
Expect(err).NotTo(HaveOccurred())

service := utils.CreateLoadBalancerServiceManifest(testServiceName, annotation, labels, ns.Name, ports)
service := utils.CreateLoadBalancerServiceManifest(testServiceName, serviceAnnotationLoadBalancerInternalTrue, labels, ns.Name, ports)
service = updateServiceBalanceIP(service, true, ip1)
_, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
utils.Logf("Successfully created LoadBalancer service " + testServiceName + " in namespace " + ns.Name)

defer func() {
By("Cleaning up")
err = cs.CoreV1().Services(ns.Name).Delete(context.TODO(), testServiceName, metav1.DeleteOptions{})
err = utils.DeleteService(cs, ns.Name, testServiceName)
Expect(err).NotTo(HaveOccurred())
}()
By("Waiting for exposure of internal service with specific IP")
Expand Down Expand Up @@ -259,12 +262,9 @@ var _ = Describe("Ensure LoadBalancer", func() {

// internal w/o IP -> public w/ IP
It("should support updating an internal service to a public service with assigned IP", func() {
annotation := map[string]string{
consts.ServiceAnnotationLoadBalancerInternal: "true",
}
ipName := basename + "-internal-none-public-IP" + string(uuid.NewUUID())[0:4]

service := utils.CreateLoadBalancerServiceManifest(testServiceName, annotation, labels, ns.Name, ports)
service := utils.CreateLoadBalancerServiceManifest(testServiceName, serviceAnnotationLoadBalancerInternalTrue, labels, ns.Name, ports)
_, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
utils.Logf("Successfully created LoadBalancer service " + testServiceName + " in namespace " + ns.Name)
Expand All @@ -275,7 +275,7 @@ var _ = Describe("Ensure LoadBalancer", func() {

defer func() {
By("Cleaning up")
err = cs.CoreV1().Services(ns.Name).Delete(context.TODO(), testServiceName, metav1.DeleteOptions{})
err = utils.DeleteService(cs, ns.Name, testServiceName)
Expect(err).NotTo(HaveOccurred())
err = utils.DeletePIPWithRetry(tc, ipName, "")
Expect(err).NotTo(HaveOccurred())
Expand Down Expand Up @@ -306,24 +306,21 @@ var _ = Describe("Ensure LoadBalancer", func() {
})

It("should have no operation since no change in service when update [Slow]", func() {
annotation := map[string]string{
consts.ServiceAnnotationLoadBalancerInternal: "false",
}
suffix := string(uuid.NewUUID())[0:4]
ipName := basename + "-public-remain" + suffix
pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), defaultPublicIPAddress(ipName))
Expect(err).NotTo(HaveOccurred())
targetIP := to.String(pip.IPAddress)

service := utils.CreateLoadBalancerServiceManifest(testServiceName, annotation, labels, ns.Name, ports)
service := utils.CreateLoadBalancerServiceManifest(testServiceName, serviceAnnotationLoadBalancerInternalFalse, labels, ns.Name, ports)
service = updateServiceBalanceIP(service, false, targetIP)
_, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
utils.Logf("Successfully created LoadBalancer service " + testServiceName + " in namespace " + ns.Name)
utils.Logf("Successfully created LoadBalancer service %s in namespace %s", testServiceName, ns.Name)

defer func() {
By("Cleaning up")
err = cs.CoreV1().Services(ns.Name).Delete(context.TODO(), testServiceName, metav1.DeleteOptions{})
err = utils.DeleteService(cs, ns.Name, testServiceName)
Expect(err).NotTo(HaveOccurred())
err = utils.DeletePIPWithRetry(tc, ipName, "")
Expect(err).NotTo(HaveOccurred())
Expand Down Expand Up @@ -351,13 +348,13 @@ var _ = Describe("Ensure LoadBalancer", func() {
return false, err
}

IngressList := service.Status.LoadBalancer.Ingress
if len(IngressList) == 0 {
ingressList := service.Status.LoadBalancer.Ingress
if len(ingressList) == 0 {
err = fmt.Errorf("Cannot find Ingress in limited time")
utils.Logf("Fail to get ingress, retry it in %s seconds", 10)
return false, nil
}
if targetIP == service.Status.LoadBalancer.Ingress[0].IP {
if targetIP == ingressList[0].IP {
utils.Logf("External IP is still %s", targetIP)
return false, nil
}
Expand Down Expand Up @@ -395,9 +392,9 @@ var _ = Describe("Ensure LoadBalancer", func() {

defer func() {
By("Cleaning up")
err = cs.CoreV1().Services(ns.Name).Delete(context.TODO(), "service1", metav1.DeleteOptions{})
err = utils.DeleteService(cs, ns.Name, "service1")
Expect(err).NotTo(HaveOccurred())
err = cs.CoreV1().Services(ns.Name).Delete(context.TODO(), "service2", metav1.DeleteOptions{})
err = utils.DeleteService(cs, ns.Name, "service2")
Expect(err).NotTo(HaveOccurred())
err = utils.DeletePIPWithRetry(tc, ipName, "")
Expect(err).NotTo(HaveOccurred())
Expand Down Expand Up @@ -425,13 +422,13 @@ var _ = Describe("Ensure LoadBalancer", func() {
utils.Logf("Successfully created LoadBalancer service2 in namespace %s with IP %s", ns.Name, ip)

By("Deleting one service and check if the other service works well")
err = cs.CoreV1().Services(ns.Name).Delete(context.TODO(), "service1", metav1.DeleteOptions{})
err = utils.DeleteService(cs, ns.Name, "service1")
Expect(err).NotTo(HaveOccurred())
_, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, "service2", ip)
Expect(err).NotTo(HaveOccurred())

By("Deleting all services")
err = cs.CoreV1().Services(ns.Name).Delete(context.TODO(), "service2", metav1.DeleteOptions{})
err = utils.DeleteService(cs, ns.Name, "service2")
Expect(err).NotTo(HaveOccurred())

By("Checking if the public IP has been deleted")
Expand All @@ -454,10 +451,7 @@ var _ = Describe("Ensure LoadBalancer", func() {
})

It("should support multiple internal services sharing one IP address", func() {
annotation := map[string]string{
consts.ServiceAnnotationLoadBalancerInternal: "true",
}
service1 := utils.CreateLoadBalancerServiceManifest("service1", annotation, labels, ns.Name, ports)
service1 := utils.CreateLoadBalancerServiceManifest("service1", serviceAnnotationLoadBalancerInternalTrue, labels, ns.Name, ports)
_, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service1, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, "service1", "")
Expand All @@ -468,7 +462,7 @@ var _ = Describe("Ensure LoadBalancer", func() {
Port: testingPort,
TargetPort: intstr.FromInt(testingPort),
}}
service2 := utils.CreateLoadBalancerServiceManifest("service2", annotation, labels, ns.Name, ports2)
service2 := utils.CreateLoadBalancerServiceManifest("service2", serviceAnnotationLoadBalancerInternalTrue, labels, ns.Name, ports2)
service2.Spec.LoadBalancerIP = ip
_, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service2, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
Expand All @@ -478,14 +472,15 @@ var _ = Describe("Ensure LoadBalancer", func() {

defer func() {
By("Cleaning up")
err = cs.CoreV1().Services(ns.Name).Delete(context.TODO(), "service1", metav1.DeleteOptions{})
err = utils.DeleteService(cs, ns.Name, "service1")
Expect(err).NotTo(HaveOccurred())
err = cs.CoreV1().Services(ns.Name).Delete(context.TODO(), "service2", metav1.DeleteOptions{})
err = utils.DeleteService(cs, ns.Name, "service2")
Expect(err).NotTo(HaveOccurred())
}()
})

It("should support node label `node.kubernetes.io/exclude-from-external-load-balancers`", func() {
label := "node.kubernetes.io/exclude-from-external-load-balancers"
By("Checking the number of the node pools")
nodes, err := utils.GetAgentNodes(cs)
Expect(err).NotTo(HaveOccurred())
Expand All @@ -509,15 +504,15 @@ var _ = Describe("Ensure LoadBalancer", func() {
Expect(len(*lbBackendPoolIPConfigs)).To(Equal(len(nodes)))

By("Labeling node")
node, err := utils.LabelNode(cs, &nodes[0], "node.kubernetes.io/exclude-from-external-load-balancers", false)
node, err := utils.LabelNode(cs, &nodes[0], label, false)
Expect(err).NotTo(HaveOccurred())
err = waitForNodesInLBBackendPool(tc, publicIP, len(nodes)-1)
Expect(err).NotTo(HaveOccurred())

By("Unlabeling node")
node, err = utils.GetNode(cs, node.Name)
Expect(err).NotTo(HaveOccurred())
_, err = utils.LabelNode(cs, node, "node.kubernetes.io/exclude-from-external-load-balancers", true)
_, err = utils.LabelNode(cs, node, label, true)
Expect(err).NotTo(HaveOccurred())
err = waitForNodesInLBBackendPool(tc, publicIP, len(nodes))
Expect(err).NotTo(HaveOccurred())
Expand All @@ -531,7 +526,7 @@ func waitForNodesInLBBackendPool(tc *utils.AzureTestClient, ip string, expectedN
if len(*lbBackendPoolIPConfigs) == expectedNum {
return true, nil
}
utils.Logf("There are %d nodes in the LB backend pool, will retry soon", len(*lbBackendPoolIPConfigs))
utils.Logf("Number of IP configs: %d in the LB backend pool, will retry soon", len(*lbBackendPoolIPConfigs))
return false, nil
})
}
Expand Down
2 changes: 1 addition & 1 deletion tests/e2e/network/service_annotations.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -265,7 +265,7 @@ var _ = Describe("Service with annotation", func() {
pip, err := utils.WaitCreatePIP(tc, testPIPName, *rg.Name, defaultPublicIPAddress(testPIPName))
Expect(err).NotTo(HaveOccurred())
defer func() {
By("Cleaning up service and public IP")
utils.Logf("Cleaning up service and public IP")
err = utils.DeleteService(cs, ns.Name, serviceName)
Expect(err).NotTo(HaveOccurred())
err = utils.DeletePIPWithRetry(tc, testPIPName, *rg.Name)
Expand Down
3 changes: 1 addition & 2 deletions tests/e2e/utils/node_utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,8 +190,7 @@ func DeleteNodes(cs clientset.Interface, names []string) error {
//deleteNodes deletes nodes according to names
func deleteNode(cs clientset.Interface, name string) error {
Logf("Deleting node: %s", name)
zero := int64(0)
if err := cs.CoreV1().Nodes().Delete(context.TODO(), name, metav1.DeleteOptions{GracePeriodSeconds: &zero}); err != nil {
if err := cs.CoreV1().Nodes().Delete(context.TODO(), name, metav1.DeleteOptions{}); err != nil {
return err
}

Expand Down
3 changes: 1 addition & 2 deletions tests/e2e/utils/service_utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,8 @@ const (

// DeleteService deletes a service
func DeleteService(cs clientset.Interface, ns string, serviceName string) error {
zero := int64(0)
err := cs.CoreV1().Services(ns).Delete(context.TODO(), serviceName, metav1.DeleteOptions{GracePeriodSeconds: &zero})
Logf("Deleting service %s in namespace %s", serviceName, ns)
err := cs.CoreV1().Services(ns).Delete(context.TODO(), serviceName, metav1.DeleteOptions{})
if err != nil {
return err
}
Expand Down

0 comments on commit 10a510e

Please sign in to comment.