diff --git a/Makefile b/Makefile index be81e55b1..c475c7f76 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ TARGETS := $(shell ls -p scripts | grep -v -e / -e deploy) CLUSTER_SETTINGS_FLAG = --cluster_settings $(DAPPER_SOURCE)/scripts/cluster_settings override CLUSTERS_ARGS += $(CLUSTER_SETTINGS_FLAG) override DEPLOY_ARGS += $(CLUSTER_SETTINGS_FLAG) -override E2E_ARGS += cluster1 cluster2 +override E2E_ARGS += cluster1 cluster2 cluster3 override UNIT_TEST_ARGS += test/e2e # Process extra flags from the `using=a,b,c` optional flag diff --git a/scripts/cluster_settings b/scripts/cluster_settings index 68cba0e73..5f61cbda9 100644 --- a/scripts/cluster_settings +++ b/scripts/cluster_settings @@ -1,10 +1,12 @@ . "${SCRIPTS_DIR}"/lib/source_only # We need a minimal setup to verify that lighthouse works -clusters=('cluster1' 'cluster2') +clusters=('cluster1' 'cluster2' 'cluster3') cluster_nodes['cluster1']="control-plane worker worker" cluster_nodes['cluster2']="control-plane worker worker" +cluster_nodes['cluster3']="control-plane worker" -cluster_cni=( ['cluster1']="weave" ['cluster2']="weave" ) +cluster_cni=( ['cluster1']="weave" ['cluster2']="weave" ['cluster3']="weave" ) + +cluster_subm=( ['cluster1']="true" ['cluster2']="true" ['cluster3']="true" ) -cluster_subm=( ['cluster1']="true" ['cluster2']="true" ) diff --git a/test/e2e/discovery/service_discovery.go b/test/e2e/discovery/service_discovery.go index 56792883d..58f360f6b 100644 --- a/test/e2e/discovery/service_discovery.go +++ b/test/e2e/discovery/service_discovery.go @@ -17,10 +17,12 @@ package discovery import ( "fmt" + "math" "strconv" "strings" . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" lhframework "github.com/submariner-io/lighthouse/test/e2e/framework" "github.com/submariner-io/shipyard/test/e2e/framework" corev1 "k8s.io/api/core/v1" @@ -73,6 +75,12 @@ var _ = Describe("[discovery] Test Service Discovery Across Clusters", func() { RunServiceDiscoveryClusterNameTest(f) }) }) + + When("a pod tries to resolve a service multiple times", func() { + It("should resolve the service from both the clusters in a round robin fashion", func() { + RunServiceDiscoveryRoundRobinTest(f) + }) + }) }) func RunServiceDiscoveryTest(f *lhframework.Framework) { @@ -351,6 +359,73 @@ func RunServiceDiscoveryClusterNameTest(f *lhframework.Framework) { clusterBName, true) } +func RunServiceDiscoveryRoundRobinTest(f *lhframework.Framework) { + if len(framework.TestContext.ClusterIDs) < 3 { + Skip("Only two clusters are deployed and hence skipping the test") + return + } + + clusterAName := framework.TestContext.ClusterIDs[framework.ClusterA] + clusterBName := framework.TestContext.ClusterIDs[framework.ClusterB] + clusterCName := framework.TestContext.ClusterIDs[framework.ClusterC] + + By(fmt.Sprintf("Creating an Nginx Deployment on %q", clusterBName)) + f.NewNginxDeployment(framework.ClusterB) + + By(fmt.Sprintf("Creating a Nginx Service on %q", clusterBName)) + + nginxServiceClusterB := f.NewNginxService(framework.ClusterB) + + f.AwaitGlobalnetIP(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace) + f.NewServiceExport(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace) + + f.AwaitServiceExportedStatusCondition(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace) + + By(fmt.Sprintf("Creating an Nginx Deployment on %q", clusterCName)) + f.NewNginxDeployment(framework.ClusterC) + + By(fmt.Sprintf("Creating a Nginx Service on %q", clusterCName)) + + nginxServiceClusterC := f.Framework.NewNginxService(framework.ClusterC) + + f.AwaitGlobalnetIP(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace) + f.NewServiceExport(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace) + + By(fmt.Sprintf("Creating a Netshoot Deployment on %q", clusterAName)) + + netshootPodList := f.NewNetShootDeployment(framework.ClusterA) + + if svc, err := f.GetService(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace); err == nil { + nginxServiceClusterC = svc + f.AwaitServiceImportIP(framework.ClusterC, nginxServiceClusterC) + f.AwaitEndpointSlices(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace, 2, 2) + } + + if svc, err := f.GetService(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace); err == nil { + nginxServiceClusterB = svc + f.AwaitServiceImportIP(framework.ClusterB, nginxServiceClusterB) + f.AwaitEndpointSlices(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 2, 2) + } + + var serviceIPList []string + + serviceIPClusterB, ok := nginxServiceClusterB.Annotations[submarinerIpamGlobalIp] + if !ok { + serviceIPClusterB = nginxServiceClusterB.Spec.ClusterIP + } + + serviceIPList = append(serviceIPList, serviceIPClusterB) + + serviceIPClusterC, ok := nginxServiceClusterC.Annotations[submarinerIpamGlobalIp] + if !ok { + serviceIPClusterC = nginxServiceClusterC.Spec.ClusterIP + } + + serviceIPList = append(serviceIPList, serviceIPClusterC) + + verifyRoundRobinWithDig(f.Framework, framework.ClusterA, nginxServiceClusterB.Name, serviceIPList, netshootPodList, checkedDomains) +} + func verifyServiceIpWithDig(f *framework.Framework, srcCluster, targetCluster framework.ClusterIndex, service *corev1.Service, targetPod *corev1.PodList, domains []string, clusterName string, shouldContain bool) { var serviceIP string @@ -362,6 +437,7 @@ func verifyServiceIpWithDig(f *framework.Framework, srcCluster, targetCluster fr } cmd := []string{"dig", "+short"} + var clusterDNSName string if clusterName != "" { clusterDNSName = clusterName + "." @@ -406,6 +482,58 @@ func verifyServiceIpWithDig(f *framework.Framework, srcCluster, targetCluster fr }) } +func verifyRoundRobinWithDig(f *framework.Framework, srcCluster framework.ClusterIndex, serviceName string, serviceIPList []string, + targetPod *corev1.PodList, domains []string) { + cmd := []string{"dig", "+short"} + + for i := range domains { + cmd = append(cmd, serviceName+"."+f.Namespace+".svc."+domains[i]) + } + + serviceIPMap := make(map[string]int) + + By(fmt.Sprintf("Executing %q to verify IPs %q for service %q are discoverable in a"+ + " round-robin fashion", strings.Join(cmd, " "), serviceIPList, serviceName)) + + var retIPs []string + + for count := 0; count < 10; count++ { + framework.AwaitUntil("verify if service IP is discoverable", func() (interface{}, error) { + stdout, _, err := f.ExecWithOptions(framework.ExecOptions{ + Command: cmd, + Namespace: f.Namespace, + PodName: targetPod.Items[0].Name, + ContainerName: targetPod.Items[0].Spec.Containers[0].Name, + CaptureStdout: true, + CaptureStderr: true, + }, srcCluster) + if err != nil { + return nil, err + } + + return stdout, nil + }, func(result interface{}) (bool, string, error) { + for _, serviceIP := range serviceIPList { + if strings.Contains(result.(string), serviceIP) { + serviceIPMap[serviceIP]++ + retIPs = append(retIPs, serviceIP) + break + } + } + + return true, "", nil + }) + } + + By(fmt.Sprintf("Service IP %q was returned %d times and Service IP %q was returned %d times - "+ + "verifying the difference between them is within the threshold", serviceIPList[0], serviceIPMap[serviceIPList[0]], + serviceIPList[1], serviceIPMap[serviceIPList[1]])) + + Expect(int(math.Abs(float64(serviceIPMap[serviceIPList[0]]-serviceIPMap[serviceIPList[1]]))) < 3).To(BeTrue(), + "Service IPs were not returned in proper round-robin fashion: Expected IPs: %v,"+ + " Returned IPs: %v, IP Counts: %v", serviceIPList, retIPs, serviceIPMap) +} + func getClusterDomain(f *framework.Framework, cluster framework.ClusterIndex, targetPod *corev1.PodList) string { /* Kubernetes adds --cluster-domain config to all pods' /etc/resolve.conf exactly as follows: