E2e to verify lighthouse return cluster-ip from healthy service

E2e to verify lighthouse return cluster-ip from healthy service Fixes: submariner-io/submariner#1041 Signed-off-by: Aswin Suryanarayanan <[email protected]> (cherry picked from commit 9be744d)
submariner-io · Feb 10, 2021 · efab8aa · efab8aa
1 parent 53ad850
commit efab8aa
Show file tree

Hide file tree

Showing 2 changed files with 134 additions and 0 deletions.
diff --git a/test/e2e/discovery/service_discovery.go b/test/e2e/discovery/service_discovery.go
@@ -66,6 +66,29 @@ var _ = Describe("[discovery] Test Service Discovery Across Clusters", func() {
 			RunServiceDiscoveryRoundRobinTest(f)
 		})
 	})
+
+	When("one of the clusters with a service is not healthy", func() {
+		var healthCheckIP, endpointName string
+
+		BeforeEach(func() {
+			if len(framework.TestContext.ClusterIDs) < 3 {
+				Skip("Only two clusters are deployed and hence skipping the test")
+				return
+			}
+
+			randomIP := "192.168.1.5"
+			endpointName, healthCheckIP = f.GetHealthCheckIPInfo(framework.ClusterC)
+			f.SetHealthCheckIP(framework.ClusterC, randomIP, endpointName)
+		})
+
+		It("should not resolve that cluster's service IP", func() {
+			RunServicesClusterAvailabilityMutliClusterTest(f)
+		})
+
+		AfterEach(func() {
+			f.SetHealthCheckIP(framework.ClusterC, healthCheckIP, endpointName)
+		})
+	})
 })
 
 func RunServiceDiscoveryTest(f *lhframework.Framework) {
@@ -426,6 +449,57 @@ func RunServiceDiscoveryRoundRobinTest(f *lhframework.Framework) {
 	verifyRoundRobinWithDig(f.Framework, framework.ClusterA, nginxServiceClusterB.Name, serviceIPList, netshootPodList, checkedDomains)
 }
 
+func RunServicesClusterAvailabilityMutliClusterTest(f *lhframework.Framework) {
+	clusterBName := framework.TestContext.ClusterIDs[framework.ClusterB]
+	clusterCName := framework.TestContext.ClusterIDs[framework.ClusterC]
+
+	By(fmt.Sprintf("Creating an Nginx Deployment on on %q", clusterBName))
+	f.NewNginxDeployment(framework.ClusterB)
+	By(fmt.Sprintf("Creating a Nginx Service on %q", clusterBName))
+
+	nginxServiceClusterB := f.NewNginxService(framework.ClusterB)
+
+	f.AwaitGlobalnetIP(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
+	f.NewServiceExport(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
+
+	f.AwaitServiceExportedStatusCondition(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
+
+	By(fmt.Sprintf("Creating a Netshoot Deployment on %q", clusterCName))
+
+	netshootPodList := f.NewNetShootDeployment(framework.ClusterA)
+
+	svc, err := f.GetService(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
+	Expect(err).NotTo(HaveOccurred())
+
+	nginxServiceClusterB = svc
+	f.AwaitServiceImportIP(framework.ClusterA, nginxServiceClusterB)
+	f.AwaitEndpointSlices(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1)
+	f.AwaitEndpointSlices(framework.ClusterA, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1)
+
+	By(fmt.Sprintf("Creating an Nginx Deployment on on %q", clusterCName))
+	f.NewNginxDeployment(framework.ClusterC)
+	By(fmt.Sprintf("Creating a Nginx Service on %q", clusterCName))
+
+	nginxServiceClusterC := f.NewNginxService(framework.ClusterC)
+
+	f.AwaitGlobalnetIP(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace)
+	f.NewServiceExport(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace)
+
+	f.AwaitServiceExportedStatusCondition(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace)
+
+	svc, err = f.GetService(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace)
+	Expect(err).NotTo(HaveOccurred())
+
+	nginxServiceClusterC = svc
+	f.AwaitServiceImportIP(framework.ClusterA, nginxServiceClusterC)
+	f.AwaitEndpointSlices(framework.ClusterA, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace, 2, 2)
+
+	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList,
+		checkedDomains, "", true)
+	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterC, nginxServiceClusterC, netshootPodList,
+		checkedDomains, "", false)
+}
+
 func verifyServiceIpWithDig(f *framework.Framework, srcCluster, targetCluster framework.ClusterIndex, service *corev1.Service,
 	targetPod *corev1.PodList, domains []string, clusterName string, shouldContain bool) {
 	var serviceIP string

diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go
@@ -4,6 +4,11 @@ import (
 	"fmt"
 	"strings"
 
+	"k8s.io/client-go/dynamic"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	lhconstants "github.com/submariner-io/lighthouse/pkg/constants"
@@ -34,6 +39,7 @@ type Framework struct {
 }
 
 var MCSClients []*mcsClientset.Clientset
+var EndpointClients []dynamic.ResourceInterface
 
 func init() {
 	framework.AddBeforeSuite(beforeSuite)
@@ -61,6 +67,7 @@ func beforeSuite() {
 
 	for _, restConfig := range framework.RestConfigs {
 		MCSClients = append(MCSClients, createLighthouseClient(restConfig))
+		EndpointClients = append(EndpointClients, createEndpointClientSet(restConfig))
 	}
 
 	framework.By("Fetching ClusterIDs")
@@ -88,6 +95,18 @@ func createLighthouseClient(restConfig *rest.Config) *mcsClientset.Clientset {
 	return clientSet
 }
 
+func createEndpointClientSet(restConfig *rest.Config) dynamic.ResourceInterface {
+	clientSet, err := dynamic.NewForConfig(restConfig)
+	Expect(err).To(Not(HaveOccurred()))
+
+	gvr, _ := schema.ParseResourceArg("endpoints.v1.submariner.io")
+	endpointsClient := clientSet.Resource(*gvr).Namespace("submariner-operator")
+	_, err = endpointsClient.List(metav1.ListOptions{})
+	Expect(err).To(Not(HaveOccurred()))
+
+	return endpointsClient
+}
+
 func (f *Framework) NewServiceExport(cluster framework.ClusterIndex, name, namespace string) *mcsv1a1.ServiceExport {
 	nginxServiceExport := mcsv1a1.ServiceExport{
 		ObjectMeta: metav1.ObjectMeta{
@@ -416,3 +435,44 @@ func (f *Framework) SetNginxStatefulSetReplicas(cluster framework.ClusterIndex,
 
 	return result
 }
+
+func (f *Framework) GetHealthCheckIPInfo(cluster framework.ClusterIndex) (endpointName, healthCheckIP string) {
+	framework.AwaitUntil("Get healthCheckIP", func() (interface{}, error) {
+		unstructuredEndpointList, err := EndpointClients[cluster].List(metav1.ListOptions{})
+		return unstructuredEndpointList, err
+	}, func(result interface{}) (bool, string, error) {
+		unstructuredEndpointList := result.(*unstructured.UnstructuredList)
+		for _, endpoint := range unstructuredEndpointList.Items {
+			By(fmt.Sprintf("Getting the endpoint %s, for cluster %s", endpoint.GetName(), framework.TestContext.ClusterIDs[cluster]))
+
+			if strings.Contains(endpoint.GetName(), framework.TestContext.ClusterIDs[cluster]) {
+				endpointName = endpoint.GetName()
+
+				var found bool
+				var err error
+				healthCheckIP, found, err = unstructured.NestedString(endpoint.Object, "spec", "healthCheckIP")
+
+				if err != nil {
+					return false, "", err
+				}
+
+				if !found {
+					return false, fmt.Sprintf("HealthcheckIP not found in %#v ", endpoint), nil
+				}
+			}
+		}
+		return true, "", nil
+	})
+
+	return endpointName, healthCheckIP
+}
+
+func (f *Framework) SetHealthCheckIP(cluster framework.ClusterIndex, ip, endpointName string) {
+	By(fmt.Sprintf("Setting health check IP cluster %q to %v", framework.TestContext.ClusterIDs[cluster], ip))
+	patch := fmt.Sprintf(`{"spec":{"healthCheckIP":%q}}`, ip)
+
+	framework.AwaitUntil("set healthCheckIP", func() (interface{}, error) {
+		endpoint, err := EndpointClients[cluster].Patch(endpointName, types.MergePatchType, []byte(patch), metav1.PatchOptions{})
+		return endpoint, err
+	}, framework.NoopCheckResult)
+}