E2e to verify lighthouse return cluster-ip from healthy service

Fixes: submariner-io/submariner#1041

Signed-off-by: Aswin Surayanarayanan <[email protected]>

test/e2e/discovery/service_discovery.go

@@ -81,6 +81,12 @@ var _ = Describe("[discovery] Test Service Discovery Across Clusters", func() {
 			RunServiceDiscoveryRoundRobinTest(f)
 		})
 	})
+
+	When("Only one of the clusters with service is healthy", func() {
+		It("should not resolve the service on the unhealthy cluster", func() {
+			RunServicesClusterAvailabilityMutliClusterTest(f)
+		})
+	})
 })
 
 func RunServiceDiscoveryTest(f *lhframework.Framework) {
@@ -426,6 +432,65 @@ func RunServiceDiscoveryRoundRobinTest(f *lhframework.Framework) {
 	verifyRoundRobinWithDig(f.Framework, framework.ClusterA, nginxServiceClusterB.Name, serviceIPList, netshootPodList, checkedDomains)
 }
 
+func RunServicesClusterAvailabilityMutliClusterTest(f *lhframework.Framework) {
+	clusterBName := framework.TestContext.ClusterIDs[framework.ClusterB]
+	clusterCName := framework.TestContext.ClusterIDs[framework.ClusterC]
+
+	By(fmt.Sprintf("Creating an Nginx Deployment on on %q", clusterBName))
+	f.NewNginxDeployment(framework.ClusterB)
+	By(fmt.Sprintf("Creating a Nginx Service on %q", clusterBName))
+
+	nginxServiceClusterB := f.NewNginxService(framework.ClusterB)
+
+	f.AwaitGlobalnetIP(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
+	f.NewServiceExport(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
+
+	f.AwaitServiceExportedStatusCondition(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace)
+
+	By(fmt.Sprintf("Creating a Netshoot Deployment on %q", clusterCName))
+
+	netshootPodList := f.NewNetShootDeployment(framework.ClusterA)
+
+	if svc, err := f.GetService(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace); err == nil {
+		nginxServiceClusterB = svc
+		f.AwaitServiceImportIP(framework.ClusterA, nginxServiceClusterB)
+		f.AwaitEndpointSlices(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1)
+		f.AwaitEndpointSlices(framework.ClusterA, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1)
+	}
+
+	By(fmt.Sprintf("Creating an Nginx Deployment on on %q", clusterCName))
+	f.NewNginxDeployment(framework.ClusterC)
+	By(fmt.Sprintf("Creating a Nginx Service on %q", clusterCName))
+
+	nginxServiceClusterC := f.NewNginxService(framework.ClusterC)
+
+	f.AwaitGlobalnetIP(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace)
+	f.NewServiceExport(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace)
+
+	f.AwaitServiceExportedStatusCondition(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace)
+
+	if svc, err := f.GetService(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace); err == nil {
+		nginxServiceClusterC = svc
+		f.AwaitServiceImportIP(framework.ClusterA, nginxServiceClusterC)
+		f.AwaitEndpointSlices(framework.ClusterA, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace, 2, 2)
+	}
+
+	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList,
+		checkedDomains, "", true)
+	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterC, nginxServiceClusterC, netshootPodList,
+		checkedDomains, "", true)
+
+	randomIP := "192.168.1.5"
+	endpointName, healthCheckIP := f.GetHealthCheckIpInfo(framework.ClusterC)
+	f.SetHealthCheckIp(framework.ClusterC, randomIP, endpointName)
+
+	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList,
+		checkedDomains, "", true)
+	verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterC, nginxServiceClusterC, netshootPodList,
+		checkedDomains, "", false)
+	f.SetHealthCheckIp(framework.ClusterC, healthCheckIP, endpointName)
+}
+
 func verifyServiceIpWithDig(f *framework.Framework, srcCluster, targetCluster framework.ClusterIndex, service *corev1.Service,
 	targetPod *corev1.PodList, domains []string, clusterName string, shouldContain bool) {
 	var serviceIP string

test/e2e/framework/framework.go

@@ -19,6 +19,11 @@ import (
 	"fmt"
 	"strings"
 
+	"k8s.io/client-go/dynamic"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	lhconstants "github.com/submariner-io/lighthouse/pkg/constants"
@@ -49,6 +54,7 @@ type Framework struct {
 }
 
 var MCSClients []*mcsClientset.Clientset
+var EndpointClients []dynamic.ResourceInterface
 
 func init() {
 	framework.AddBeforeSuite(beforeSuite)
@@ -76,6 +82,7 @@ func beforeSuite() {
 
 	for _, restConfig := range framework.RestConfigs {
 		MCSClients = append(MCSClients, createLighthouseClient(restConfig))
+		EndpointClients = append(EndpointClients, createEndpointClientSet(restConfig))
 	}
 
 	framework.DetectGlobalnet()
@@ -88,6 +95,18 @@ func createLighthouseClient(restConfig *rest.Config) *mcsClientset.Clientset {
 	return clientSet
 }
 
+func createEndpointClientSet(restConfig *rest.Config) dynamic.ResourceInterface {
+	clientSet, err := dynamic.NewForConfig(restConfig)
+	Expect(err).To(Not(HaveOccurred()))
+
+	gvr, _ := schema.ParseResourceArg("endpoints.v1.submariner.io")
+	endpointsClient := clientSet.Resource(*gvr).Namespace("submariner-operator")
+	_, err = endpointsClient.List(metav1.ListOptions{})
+	Expect(err).To(Not(HaveOccurred()))
+
+	return endpointsClient
+}
+
 func (f *Framework) NewServiceExport(cluster framework.ClusterIndex, name, namespace string) *mcsv1a1.ServiceExport {
 	nginxServiceExport := mcsv1a1.ServiceExport{
 		ObjectMeta: metav1.ObjectMeta{
@@ -416,3 +435,40 @@ func (f *Framework) SetNginxStatefulSetReplicas(cluster framework.ClusterIndex,
 
 	return result
 }
+
+func (f *Framework) GetHealthCheckIpInfo(cluster framework.ClusterIndex) (endpointName, healthCheckIP string) {
+	unstructuredEndpointList, err := EndpointClients[cluster].List(metav1.ListOptions{})
+
+	Expect(err).ShouldNot(HaveOccurred())
+
+	for _, endpoint := range unstructuredEndpointList.Items {
+		By(fmt.Sprintf("Getting the endpoint %s, for cluster %s", endpoint.GetName(), framework.TestContext.ClusterIDs[cluster]))
+
+		if strings.Contains(endpoint.GetName(), framework.TestContext.ClusterIDs[cluster]) {
+			endpointName = endpoint.GetName()
+			Expect(endpointName).NotTo(BeNil())
+
+			spec, found, err := unstructured.NestedMap(endpoint.Object, "spec")
+
+			Expect(err).ShouldNot(HaveOccurred())
+			Expect(found).NotTo(BeFalse())
+			healthCheckIP, found, err = unstructured.NestedString(spec, "healthCheckIP")
+
+			Expect(err).ShouldNot(HaveOccurred())
+			Expect(found).NotTo(BeFalse())
+		}
+	}
+
+	return endpointName, healthCheckIP
+}
+
+func (f *Framework) SetHealthCheckIp(cluster framework.ClusterIndex, ip, endpointName string) {
+	By(fmt.Sprintf("Setting Health Check Ip to %v in cluster %q", ip, framework.TestContext.ClusterIDs[cluster]))
+	patch := fmt.Sprintf(`{"spec":{"healthCheckIP":%q}}`, ip)
+
+	framework.AwaitUntil("set healthCheckIP", func() (interface{}, error) {
+		endpoint, err := EndpointClients[cluster].Patch(endpointName, types.MergePatchType, []byte(patch), metav1.PatchOptions{})
+		Expect(err).ShouldNot(HaveOccurred())
+		return endpoint, err
+	}, framework.NoopCheckResult)
+}