diff --git a/test/e2e/discovery/service_discovery.go b/test/e2e/discovery/service_discovery.go index 58f360f6b..b3b2c5588 100644 --- a/test/e2e/discovery/service_discovery.go +++ b/test/e2e/discovery/service_discovery.go @@ -81,6 +81,12 @@ var _ = Describe("[discovery] Test Service Discovery Across Clusters", func() { RunServiceDiscoveryRoundRobinTest(f) }) }) + + When("Only one of the clusters with service is healthy", func() { + It("should not resolve the service on the unhealthy cluster", func() { + RunServicesClusterAvailabilityMutliClusterTest(f) + }) + }) }) func RunServiceDiscoveryTest(f *lhframework.Framework) { @@ -426,6 +432,65 @@ func RunServiceDiscoveryRoundRobinTest(f *lhframework.Framework) { verifyRoundRobinWithDig(f.Framework, framework.ClusterA, nginxServiceClusterB.Name, serviceIPList, netshootPodList, checkedDomains) } +func RunServicesClusterAvailabilityMutliClusterTest(f *lhframework.Framework) { + clusterBName := framework.TestContext.ClusterIDs[framework.ClusterB] + clusterCName := framework.TestContext.ClusterIDs[framework.ClusterC] + + By(fmt.Sprintf("Creating an Nginx Deployment on on %q", clusterBName)) + f.NewNginxDeployment(framework.ClusterB) + By(fmt.Sprintf("Creating a Nginx Service on %q", clusterBName)) + + nginxServiceClusterB := f.NewNginxService(framework.ClusterB) + + f.AwaitGlobalnetIP(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace) + f.NewServiceExport(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace) + + f.AwaitServiceExportedStatusCondition(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace) + + By(fmt.Sprintf("Creating a Netshoot Deployment on %q", clusterCName)) + + netshootPodList := f.NewNetShootDeployment(framework.ClusterA) + + if svc, err := f.GetService(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace); err == nil { + nginxServiceClusterB = svc + f.AwaitServiceImportIP(framework.ClusterA, nginxServiceClusterB) + f.AwaitEndpointSlices(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1) + f.AwaitEndpointSlices(framework.ClusterA, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1) + } + + By(fmt.Sprintf("Creating an Nginx Deployment on on %q", clusterCName)) + f.NewNginxDeployment(framework.ClusterC) + By(fmt.Sprintf("Creating a Nginx Service on %q", clusterCName)) + + nginxServiceClusterC := f.NewNginxService(framework.ClusterC) + + f.AwaitGlobalnetIP(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace) + f.NewServiceExport(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace) + + f.AwaitServiceExportedStatusCondition(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace) + + if svc, err := f.GetService(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace); err == nil { + nginxServiceClusterC = svc + f.AwaitServiceImportIP(framework.ClusterA, nginxServiceClusterC) + f.AwaitEndpointSlices(framework.ClusterA, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace, 2, 2) + } + + verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList, + checkedDomains, "", true) + verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterC, nginxServiceClusterC, netshootPodList, + checkedDomains, "", true) + + randomIP := "192.168.1.5" + endpointName, healthCheckIP := f.GetHealthCheckIpInfo(framework.ClusterC) + f.SetHealthCheckIp(framework.ClusterC, randomIP, endpointName) + + verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList, + checkedDomains, "", true) + verifyServiceIpWithDig(f.Framework, framework.ClusterA, framework.ClusterC, nginxServiceClusterC, netshootPodList, + checkedDomains, "", false) + f.SetHealthCheckIp(framework.ClusterC, healthCheckIP, endpointName) +} + func verifyServiceIpWithDig(f *framework.Framework, srcCluster, targetCluster framework.ClusterIndex, service *corev1.Service, targetPod *corev1.PodList, domains []string, clusterName string, shouldContain bool) { var serviceIP string diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index 3a46047c0..60a294c56 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -19,6 +19,11 @@ import ( "fmt" "strings" + "k8s.io/client-go/dynamic" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" lhconstants "github.com/submariner-io/lighthouse/pkg/constants" @@ -49,6 +54,7 @@ type Framework struct { } var MCSClients []*mcsClientset.Clientset +var EndpointClients []dynamic.ResourceInterface func init() { framework.AddBeforeSuite(beforeSuite) @@ -76,6 +82,7 @@ func beforeSuite() { for _, restConfig := range framework.RestConfigs { MCSClients = append(MCSClients, createLighthouseClient(restConfig)) + EndpointClients = append(EndpointClients, createEndpointClientSet(restConfig)) } framework.DetectGlobalnet() @@ -88,6 +95,18 @@ func createLighthouseClient(restConfig *rest.Config) *mcsClientset.Clientset { return clientSet } +func createEndpointClientSet(restConfig *rest.Config) dynamic.ResourceInterface { + clientSet, err := dynamic.NewForConfig(restConfig) + Expect(err).To(Not(HaveOccurred())) + + gvr, _ := schema.ParseResourceArg("endpoints.v1.submariner.io") + endpointsClient := clientSet.Resource(*gvr).Namespace("submariner-operator") + _, err = endpointsClient.List(metav1.ListOptions{}) + Expect(err).To(Not(HaveOccurred())) + + return endpointsClient +} + func (f *Framework) NewServiceExport(cluster framework.ClusterIndex, name, namespace string) *mcsv1a1.ServiceExport { nginxServiceExport := mcsv1a1.ServiceExport{ ObjectMeta: metav1.ObjectMeta{ @@ -416,3 +435,40 @@ func (f *Framework) SetNginxStatefulSetReplicas(cluster framework.ClusterIndex, return result } + +func (f *Framework) GetHealthCheckIpInfo(cluster framework.ClusterIndex) (endpointName, healthCheckIP string) { + unstructuredEndpointList, err := EndpointClients[cluster].List(metav1.ListOptions{}) + + Expect(err).ShouldNot(HaveOccurred()) + + for _, endpoint := range unstructuredEndpointList.Items { + By(fmt.Sprintf("Getting the endpoint %s, for cluster %s", endpoint.GetName(), framework.TestContext.ClusterIDs[cluster])) + + if strings.Contains(endpoint.GetName(), framework.TestContext.ClusterIDs[cluster]) { + endpointName = endpoint.GetName() + Expect(endpointName).NotTo(BeNil()) + + spec, found, err := unstructured.NestedMap(endpoint.Object, "spec") + + Expect(err).ShouldNot(HaveOccurred()) + Expect(found).NotTo(BeFalse()) + healthCheckIP, found, err = unstructured.NestedString(spec, "healthCheckIP") + + Expect(err).ShouldNot(HaveOccurred()) + Expect(found).NotTo(BeFalse()) + } + } + + return endpointName, healthCheckIP +} + +func (f *Framework) SetHealthCheckIp(cluster framework.ClusterIndex, ip, endpointName string) { + By(fmt.Sprintf("Setting Health Check Ip to %v in cluster %q", ip, framework.TestContext.ClusterIDs[cluster])) + patch := fmt.Sprintf(`{"spec":{"healthCheckIP":%q}}`, ip) + + framework.AwaitUntil("set healthCheckIP", func() (interface{}, error) { + endpoint, err := EndpointClients[cluster].Patch(endpointName, types.MergePatchType, []byte(patch), metav1.PatchOptions{}) + Expect(err).ShouldNot(HaveOccurred()) + return endpoint, err + }, framework.NoopCheckResult) +}