From 9be744d90c358be8a3ceab1bed0f91de2487b542 Mon Sep 17 00:00:00 2001 From: Aswin Suryanarayanan Date: Fri, 15 Jan 2021 13:32:37 +0530 Subject: [PATCH] E2e to verify lighthouse return cluster-ip from healthy service E2e to verify lighthouse return cluster-ip from healthy service Fixes: submariner-io/submariner#1041 Signed-off-by: Aswin Surayanarayanan --- test/e2e/discovery/service_discovery.go | 74 +++++++++++++++++++++++++ test/e2e/framework/framework.go | 60 ++++++++++++++++++++ 2 files changed, 134 insertions(+) diff --git a/test/e2e/discovery/service_discovery.go b/test/e2e/discovery/service_discovery.go index ff629c887..d4a8b93c3 100644 --- a/test/e2e/discovery/service_discovery.go +++ b/test/e2e/discovery/service_discovery.go @@ -81,6 +81,29 @@ var _ = Describe("[discovery] Test Service Discovery Across Clusters", func() { RunServiceDiscoveryRoundRobinTest(f) }) }) + + When("one of the clusters with a service is not healthy", func() { + var healthCheckIP, endpointName string + + BeforeEach(func() { + if len(framework.TestContext.ClusterIDs) < 3 { + Skip("Only two clusters are deployed and hence skipping the test") + return + } + + randomIP := "192.168.1.5" + endpointName, healthCheckIP = f.GetHealthCheckIPInfo(framework.ClusterC) + f.SetHealthCheckIP(framework.ClusterC, randomIP, endpointName) + }) + + It("should not resolve that cluster's service IP", func() { + RunServicesClusterAvailabilityMutliClusterTest(f) + }) + + AfterEach(func() { + f.SetHealthCheckIP(framework.ClusterC, healthCheckIP, endpointName) + }) + }) }) func RunServiceDiscoveryTest(f *lhframework.Framework) { @@ -441,6 +464,57 @@ func RunServiceDiscoveryRoundRobinTest(f *lhframework.Framework) { verifyRoundRobinWithDig(f.Framework, framework.ClusterA, nginxServiceClusterB.Name, serviceIPList, netshootPodList, checkedDomains) } +func RunServicesClusterAvailabilityMutliClusterTest(f *lhframework.Framework) { + clusterBName := framework.TestContext.ClusterIDs[framework.ClusterB] + clusterCName := framework.TestContext.ClusterIDs[framework.ClusterC] + + By(fmt.Sprintf("Creating an Nginx Deployment on on %q", clusterBName)) + f.NewNginxDeployment(framework.ClusterB) + By(fmt.Sprintf("Creating a Nginx Service on %q", clusterBName)) + + nginxServiceClusterB := f.NewNginxService(framework.ClusterB) + + f.AwaitGlobalnetIP(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace) + f.NewServiceExport(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace) + + f.AwaitServiceExportedStatusCondition(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace) + + By(fmt.Sprintf("Creating a Netshoot Deployment on %q", clusterCName)) + + netshootPodList := f.NewNetShootDeployment(framework.ClusterA) + + svc, err := f.GetService(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace) + Expect(err).NotTo(HaveOccurred()) + + nginxServiceClusterB = svc + f.AwaitServiceImportIP(framework.ClusterA, nginxServiceClusterB) + f.AwaitEndpointSlices(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1) + f.AwaitEndpointSlices(framework.ClusterA, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1) + + By(fmt.Sprintf("Creating an Nginx Deployment on on %q", clusterCName)) + f.NewNginxDeployment(framework.ClusterC) + By(fmt.Sprintf("Creating a Nginx Service on %q", clusterCName)) + + nginxServiceClusterC := f.NewNginxService(framework.ClusterC) + + f.AwaitGlobalnetIP(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace) + f.NewServiceExport(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace) + + f.AwaitServiceExportedStatusCondition(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace) + + svc, err = f.GetService(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace) + Expect(err).NotTo(HaveOccurred()) + + nginxServiceClusterC = svc + f.AwaitServiceImportIP(framework.ClusterA, nginxServiceClusterC) + f.AwaitEndpointSlices(framework.ClusterA, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace, 2, 2) + + verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList, + checkedDomains, "", true) + verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterC, nginxServiceClusterC, netshootPodList, + checkedDomains, "", false) +} + func verifyServiceIpWithDig(f *framework.Framework, srcCluster, targetCluster framework.ClusterIndex, service *corev1.Service, targetPod *corev1.PodList, domains []string, clusterName string, shouldContain bool) { var serviceIP string diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index 3a46047c0..750f6aa64 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -19,6 +19,11 @@ import ( "fmt" "strings" + "k8s.io/client-go/dynamic" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" lhconstants "github.com/submariner-io/lighthouse/pkg/constants" @@ -49,6 +54,7 @@ type Framework struct { } var MCSClients []*mcsClientset.Clientset +var EndpointClients []dynamic.ResourceInterface func init() { framework.AddBeforeSuite(beforeSuite) @@ -76,6 +82,7 @@ func beforeSuite() { for _, restConfig := range framework.RestConfigs { MCSClients = append(MCSClients, createLighthouseClient(restConfig)) + EndpointClients = append(EndpointClients, createEndpointClientSet(restConfig)) } framework.DetectGlobalnet() @@ -88,6 +95,18 @@ func createLighthouseClient(restConfig *rest.Config) *mcsClientset.Clientset { return clientSet } +func createEndpointClientSet(restConfig *rest.Config) dynamic.ResourceInterface { + clientSet, err := dynamic.NewForConfig(restConfig) + Expect(err).To(Not(HaveOccurred())) + + gvr, _ := schema.ParseResourceArg("endpoints.v1.submariner.io") + endpointsClient := clientSet.Resource(*gvr).Namespace("submariner-operator") + _, err = endpointsClient.List(metav1.ListOptions{}) + Expect(err).To(Not(HaveOccurred())) + + return endpointsClient +} + func (f *Framework) NewServiceExport(cluster framework.ClusterIndex, name, namespace string) *mcsv1a1.ServiceExport { nginxServiceExport := mcsv1a1.ServiceExport{ ObjectMeta: metav1.ObjectMeta{ @@ -416,3 +435,44 @@ func (f *Framework) SetNginxStatefulSetReplicas(cluster framework.ClusterIndex, return result } + +func (f *Framework) GetHealthCheckIPInfo(cluster framework.ClusterIndex) (endpointName, healthCheckIP string) { + framework.AwaitUntil("Get healthCheckIP", func() (interface{}, error) { + unstructuredEndpointList, err := EndpointClients[cluster].List(metav1.ListOptions{}) + return unstructuredEndpointList, err + }, func(result interface{}) (bool, string, error) { + unstructuredEndpointList := result.(*unstructured.UnstructuredList) + for _, endpoint := range unstructuredEndpointList.Items { + By(fmt.Sprintf("Getting the endpoint %s, for cluster %s", endpoint.GetName(), framework.TestContext.ClusterIDs[cluster])) + + if strings.Contains(endpoint.GetName(), framework.TestContext.ClusterIDs[cluster]) { + endpointName = endpoint.GetName() + + var found bool + var err error + healthCheckIP, found, err = unstructured.NestedString(endpoint.Object, "spec", "healthCheckIP") + + if err != nil { + return false, "", err + } + + if !found { + return false, fmt.Sprintf("HealthcheckIP not found in %#v ", endpoint), nil + } + } + } + return true, "", nil + }) + + return endpointName, healthCheckIP +} + +func (f *Framework) SetHealthCheckIP(cluster framework.ClusterIndex, ip, endpointName string) { + By(fmt.Sprintf("Setting health check IP cluster %q to %v", framework.TestContext.ClusterIDs[cluster], ip)) + patch := fmt.Sprintf(`{"spec":{"healthCheckIP":%q}}`, ip) + + framework.AwaitUntil("set healthCheckIP", func() (interface{}, error) { + endpoint, err := EndpointClients[cluster].Patch(endpointName, types.MergePatchType, []byte(patch), metav1.PatchOptions{}) + return endpoint, err + }, framework.NoopCheckResult) +}