Add e2e to verify round-robin for Service discovery (submariner-io#422)

Add e2e to verify round-robin for Service discovery

Fixes : submariner-io#340

Signed-off-by: Aswin Surayanarayanan <[email protected]>

Makefile

@@ -8,7 +8,7 @@ TARGETS := $(shell ls -p scripts | grep -v -e / -e deploy)
 CLUSTER_SETTINGS_FLAG = --cluster_settings $(DAPPER_SOURCE)/scripts/cluster_settings
 override CLUSTERS_ARGS += $(CLUSTER_SETTINGS_FLAG)
 override DEPLOY_ARGS += $(CLUSTER_SETTINGS_FLAG)
-override E2E_ARGS += cluster1 cluster2
+override E2E_ARGS += cluster1 cluster2 cluster3
 override UNIT_TEST_ARGS += test/e2e
 
 # Process extra flags from the `using=a,b,c` optional flag

scripts/cluster_settings

@@ -1,10 +1,12 @@
 . "${SCRIPTS_DIR}"/lib/source_only
 
 # We need a minimal setup to verify that lighthouse works
-clusters=('cluster1' 'cluster2')
+clusters=('cluster1' 'cluster2' 'cluster3')
 cluster_nodes['cluster1']="control-plane worker worker"
 cluster_nodes['cluster2']="control-plane worker worker"
+cluster_nodes['cluster3']="control-plane worker"
 
-cluster_cni=( ['cluster1']="weave" ['cluster2']="weave" )
+cluster_cni=( ['cluster1']="weave" ['cluster2']="weave" ['cluster3']="weave" )
+
+cluster_subm=( ['cluster1']="true" ['cluster2']="true"  ['cluster3']="true" )
 
-cluster_subm=( ['cluster1']="true" ['cluster2']="true" )

test/e2e/discovery/service_discovery.go

@@ -17,10 +17,12 @@ package discovery
 
 import (
 	"fmt"
+	"math"
 	"strconv"
 	"strings"
 
 	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
 	lhframework "github.com/submariner-io/lighthouse/test/e2e/framework"
 	"github.com/submariner-io/shipyard/test/e2e/framework"
 	corev1 "k8s.io/api/core/v1"
@@ -73,6 +75,12 @@ var _ = Describe("[discovery] Test Service Discovery Across Clusters", func() {
 			RunServiceDiscoveryClusterNameTest(f)
 		})
 	})
+
+	When("a pod tries to resolve a service multiple times", func() {
+		It("should resolve the service from both the clusters in a round robin fashion", func() {
+			RunServiceDiscoveryRoundRobinTest(f)
+		})
+	})
 })
 
 func RunServiceDiscoveryTest(f *lhframework.Framework) {
@@ -351,6 +359,73 @@ func RunServiceDiscoveryClusterNameTest(f *lhframework.Framework) {
 		clusterBName, true)
 }
 
+func RunServiceDiscoveryRoundRobinTest(f *lhframework.Framework) {
+	if len(framework.TestContext.ClusterIDs) < 3 {
+		Skip("Only two clusters are deployed and hence skipping the test")
+		return
+	}
+
+	clusterAName := framework.TestContext.ClusterIDs[framework.ClusterA]
+	clusterBName := framework.TestContext.ClusterIDs[framework.ClusterB]
+	clusterCName := framework.TestContext.ClusterIDs[framework.ClusterC]
+
+	By(fmt.Sprintf("Creating an Nginx Deployment on %q", clusterBName))
+	f.NewNginxDeployment(framework.ClusterB)
+
+	By(fmt.Sprintf("Creating a Nginx Service on %q", clusterBName))
+
+	nginxServiceClusterB := f.NewNginxService(framework.ClusterB)
+
+	f.AwaitGlobalnetIP(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
+	f.NewServiceExport(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
+
+	f.AwaitServiceExportedStatusCondition(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
+
+	By(fmt.Sprintf("Creating an Nginx Deployment on %q", clusterCName))
+	f.NewNginxDeployment(framework.ClusterC)
+
+	By(fmt.Sprintf("Creating a Nginx Service on %q", clusterCName))
+
+	nginxServiceClusterC := f.Framework.NewNginxService(framework.ClusterC)
+
+	f.AwaitGlobalnetIP(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace)
+	f.NewServiceExport(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace)
+
+	By(fmt.Sprintf("Creating a Netshoot Deployment on %q", clusterAName))
+
+	netshootPodList := f.NewNetShootDeployment(framework.ClusterA)
+
+	if svc, err := f.GetService(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace); err == nil {
+		nginxServiceClusterC = svc
+		f.AwaitServiceImportIP(framework.ClusterC, nginxServiceClusterC)
+		f.AwaitEndpointSlices(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace, 2, 2)
+	}
+
+	if svc, err := f.GetService(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace); err == nil {
+		nginxServiceClusterB = svc
+		f.AwaitServiceImportIP(framework.ClusterB, nginxServiceClusterB)
+		f.AwaitEndpointSlices(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 2, 2)
+	}
+
+	var serviceIPList []string
+
+	serviceIPClusterB, ok := nginxServiceClusterB.Annotations[submarinerIpamGlobalIp]
+	if !ok {
+		serviceIPClusterB = nginxServiceClusterB.Spec.ClusterIP
+	}
+
+	serviceIPList = append(serviceIPList, serviceIPClusterB)
+
+	serviceIPClusterC, ok := nginxServiceClusterC.Annotations[submarinerIpamGlobalIp]
+	if !ok {
+		serviceIPClusterC = nginxServiceClusterC.Spec.ClusterIP
+	}
+
+	serviceIPList = append(serviceIPList, serviceIPClusterC)
+
+	verifyRoundRobinWithDig(f.Framework, framework.ClusterA, nginxServiceClusterB.Name, serviceIPList, netshootPodList, checkedDomains)
+}
+
 func verifyServiceIpWithDig(f *framework.Framework, srcCluster, targetCluster framework.ClusterIndex, service *corev1.Service,
 	targetPod *corev1.PodList, domains []string, clusterName string, shouldContain bool) {
 	var serviceIP string
@@ -362,6 +437,7 @@ func verifyServiceIpWithDig(f *framework.Framework, srcCluster, targetCluster fr
 	}
 
 	cmd := []string{"dig", "+short"}
+
 	var clusterDNSName string
 	if clusterName != "" {
 		clusterDNSName = clusterName + "."
@@ -406,6 +482,58 @@ func verifyServiceIpWithDig(f *framework.Framework, srcCluster, targetCluster fr
 	})
 }
 
+func verifyRoundRobinWithDig(f *framework.Framework, srcCluster framework.ClusterIndex, serviceName string, serviceIPList []string,
+	targetPod *corev1.PodList, domains []string) {
+	cmd := []string{"dig", "+short"}
+
+	for i := range domains {
+		cmd = append(cmd, serviceName+"."+f.Namespace+".svc."+domains[i])
+	}
+
+	serviceIPMap := make(map[string]int)
+
+	By(fmt.Sprintf("Executing %q to verify IPs %q for service %q are discoverable in a"+
+		" round-robin fashion", strings.Join(cmd, " "), serviceIPList, serviceName))
+
+	var retIPs []string
+
+	for count := 0; count < 10; count++ {
+		framework.AwaitUntil("verify if service IP is discoverable", func() (interface{}, error) {
+			stdout, _, err := f.ExecWithOptions(framework.ExecOptions{
+				Command:       cmd,
+				Namespace:     f.Namespace,
+				PodName:       targetPod.Items[0].Name,
+				ContainerName: targetPod.Items[0].Spec.Containers[0].Name,
+				CaptureStdout: true,
+				CaptureStderr: true,
+			}, srcCluster)
+			if err != nil {
+				return nil, err
+			}
+
+			return stdout, nil
+		}, func(result interface{}) (bool, string, error) {
+			for _, serviceIP := range serviceIPList {
+				if strings.Contains(result.(string), serviceIP) {
+					serviceIPMap[serviceIP]++
+					retIPs = append(retIPs, serviceIP)
+					break
+				}
+			}
+
+			return true, "", nil
+		})
+	}
+
+	By(fmt.Sprintf("Service IP %q was returned %d times and Service IP %q was returned %d times - "+
+		"verifying the difference between them is within the threshold", serviceIPList[0], serviceIPMap[serviceIPList[0]],
+		serviceIPList[1], serviceIPMap[serviceIPList[1]]))
+
+	Expect(int(math.Abs(float64(serviceIPMap[serviceIPList[0]]-serviceIPMap[serviceIPList[1]]))) < 3).To(BeTrue(),
+		"Service IPs were not returned in proper round-robin fashion: Expected IPs: %v,"+
+			" Returned IPs: %v, IP Counts: %v", serviceIPList, retIPs, serviceIPMap)
+}
+
 func getClusterDomain(f *framework.Framework, cluster framework.ClusterIndex, targetPod *corev1.PodList) string {
 	/*
 		Kubernetes adds --cluster-domain config to all pods' /etc/resolve.conf exactly as follows: