diff --git a/Makefile b/Makefile index 4af1583f9..339e3eae7 100644 --- a/Makefile +++ b/Makefile @@ -59,7 +59,7 @@ export-nginx: deploy-latest check-nginx: KUBECONFIG=output/kubeconfigs/kind-config-cluster1 kubectl get serviceexports.multicluster.x-k8s.io -n default nginx-upgrade - KUBECONFIG=output/kubeconfigs/kind-config-cluster2 kubectl get serviceimports.multicluster.x-k8s.io -n submariner-operator nginx-upgrade-default-cluster1 + KUBECONFIG=output/kubeconfigs/kind-config-cluster2 kubectl get serviceimports.multicluster.x-k8s.io -n default nginx-upgrade $(TARGETS): ./scripts/$@ diff --git a/test/e2e/discovery/headless_services.go b/test/e2e/discovery/headless_services.go index 9c02d97b7..60959ff8b 100644 --- a/test/e2e/discovery/headless_services.go +++ b/test/e2e/discovery/headless_services.go @@ -97,7 +97,7 @@ func RunHeadlessDiscoveryTest(f *lhframework.Framework) { clusterBName, false, false, true) f.DeleteServiceExport(framework.ClusterB, nginxHeadlessClusterB.Name, nginxHeadlessClusterB.Namespace) - f.AwaitServiceImportCount(framework.ClusterA, nginxHeadlessClusterB.Name, nginxHeadlessClusterB.Namespace, 0) + f.AwaitAggregatedServiceImport(framework.ClusterA, nginxHeadlessClusterB, 0) f.VerifyIPsWithDig(framework.ClusterA, nginxHeadlessClusterB, netshootPodList, ipList, checkedDomains, "", false) @@ -117,6 +117,7 @@ func RunHeadlessDiscoveryLocalAndRemoteTest(f *lhframework.Framework) { f.NewServiceExport(framework.ClusterB, nginxHeadlessClusterB.Name, nginxHeadlessClusterB.Namespace) f.AwaitServiceExportedStatusCondition(framework.ClusterB, nginxHeadlessClusterB.Name, nginxHeadlessClusterB.Namespace) + f.AwaitAggregatedServiceImport(framework.ClusterA, nginxHeadlessClusterB, 1) By(fmt.Sprintf("Creating an Nginx Deployment on %q", clusterAName)) f.NewNginxDeployment(framework.ClusterA) @@ -126,6 +127,7 @@ func RunHeadlessDiscoveryLocalAndRemoteTest(f *lhframework.Framework) { f.NewServiceExport(framework.ClusterA, nginxHeadlessClusterA.Name, nginxHeadlessClusterA.Namespace) f.AwaitServiceExportedStatusCondition(framework.ClusterA, nginxHeadlessClusterA.Name, nginxHeadlessClusterA.Namespace) + f.AwaitAggregatedServiceImport(framework.ClusterA, nginxHeadlessClusterA, 2) By(fmt.Sprintf("Creating a Netshoot Deployment on %q", clusterAName)) @@ -146,7 +148,7 @@ func RunHeadlessDiscoveryLocalAndRemoteTest(f *lhframework.Framework) { verifyHeadlessSRVRecordsWithDig(f.Framework, framework.ClusterA, nginxHeadlessClusterB, netshootPodList, hostNameListA, checkedDomains, clusterAName, true, false, true) f.DeleteServiceExport(framework.ClusterB, nginxHeadlessClusterB.Name, nginxHeadlessClusterB.Namespace) - f.AwaitServiceImportCount(framework.ClusterA, nginxHeadlessClusterB.Name, nginxHeadlessClusterB.Namespace, 1) + f.AwaitAggregatedServiceImport(framework.ClusterA, nginxHeadlessClusterB, 1) f.VerifyIPsWithDig(framework.ClusterA, nginxHeadlessClusterB, netshootPodList, ipListB, checkedDomains, "", false) diff --git a/test/e2e/discovery/service_discovery.go b/test/e2e/discovery/service_discovery.go index 44839109a..510270ee0 100644 --- a/test/e2e/discovery/service_discovery.go +++ b/test/e2e/discovery/service_discovery.go @@ -137,7 +137,7 @@ func RunServiceDiscoveryTest(f *lhframework.Framework) { Expect(err).NotTo(HaveOccurred()) nginxServiceClusterB = svc - f.AwaitServiceImportIP(framework.ClusterB, framework.ClusterA, nginxServiceClusterB) + f.AwaitAggregatedServiceImport(framework.ClusterA, nginxServiceClusterB, 1) f.AwaitEndpointSlices(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1) f.AwaitEndpointSlices(framework.ClusterA, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1) @@ -150,7 +150,7 @@ func RunServiceDiscoveryTest(f *lhframework.Framework) { false, true) f.DeleteServiceExport(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace) - f.AwaitServiceImportDelete(framework.ClusterA, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace) + f.AwaitAggregatedServiceImport(framework.ClusterA, nginxServiceClusterB, 0) f.DeleteService(framework.ClusterB, nginxServiceClusterB.Name) @@ -200,7 +200,7 @@ func RunServiceDiscoveryLocalTest(f *lhframework.Framework) { Expect(err).NotTo(HaveOccurred()) nginxServiceClusterB = svc - f.AwaitServiceImportIP(framework.ClusterB, framework.ClusterA, nginxServiceClusterB) + f.AwaitAggregatedServiceImport(framework.ClusterA, nginxServiceClusterB, 1) f.AwaitEndpointSlices(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1) f.AwaitEndpointSlices(framework.ClusterA, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1) @@ -208,7 +208,7 @@ func RunServiceDiscoveryLocalTest(f *lhframework.Framework) { "", true) f.DeleteServiceExport(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace) - f.AwaitServiceImportDelete(framework.ClusterA, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace) + f.AwaitAggregatedServiceImport(framework.ClusterA, nginxServiceClusterB, 0) f.DeleteService(framework.ClusterB, nginxServiceClusterB.Name) @@ -236,7 +236,7 @@ func RunServiceExportTest(f *lhframework.Framework) { Expect(err).NotTo(HaveOccurred()) nginxServiceClusterB = svc - f.AwaitServiceImportIP(framework.ClusterB, framework.ClusterA, nginxServiceClusterB) + f.AwaitAggregatedServiceImport(framework.ClusterA, nginxServiceClusterB, 1) f.AwaitEndpointSlices(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1) f.AwaitEndpointSlices(framework.ClusterA, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1) @@ -244,7 +244,7 @@ func RunServiceExportTest(f *lhframework.Framework) { "", true) f.DeleteServiceExport(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace) - f.AwaitServiceImportDelete(framework.ClusterA, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace) + f.AwaitAggregatedServiceImport(framework.ClusterA, nginxServiceClusterB, 0) f.VerifyIPWithDig(framework.ClusterA, nginxServiceClusterB, netshootPodList, checkedDomains, "", "", true) } @@ -271,7 +271,7 @@ func RunServicesPodAvailabilityTest(f *lhframework.Framework) { Expect(err).NotTo(HaveOccurred()) nginxServiceClusterB = svc - f.AwaitServiceImportIP(framework.ClusterB, framework.ClusterA, nginxServiceClusterB) + f.AwaitAggregatedServiceImport(framework.ClusterA, nginxServiceClusterB, 1) f.AwaitEndpointSlices(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1) f.AwaitEndpointSlices(framework.ClusterA, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1) @@ -329,14 +329,14 @@ func RunServicesPodAvailabilityMultiClusterTest(f *lhframework.Framework) { Expect(err).NotTo(HaveOccurred()) nginxServiceClusterC = svc - f.AwaitServiceImportIP(framework.ClusterC, framework.ClusterC, nginxServiceClusterC) + f.AwaitAggregatedServiceImport(framework.ClusterC, nginxServiceClusterC, 2) f.AwaitEndpointSlices(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace, 2, 2) svc, err = f.GetService(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace) Expect(err).NotTo(HaveOccurred()) nginxServiceClusterB = svc - f.AwaitServiceImportIP(framework.ClusterB, framework.ClusterB, nginxServiceClusterB) + f.AwaitAggregatedServiceImport(framework.ClusterB, nginxServiceClusterB, 2) f.AwaitEndpointSlices(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 2, 2) f.VerifyServiceIPWithDig(framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList, checkedDomains, @@ -403,7 +403,7 @@ func RunServiceDiscoveryClusterNameTest(f *lhframework.Framework) { Expect(err).NotTo(HaveOccurred()) nginxServiceClusterA = svc - f.AwaitServiceImportIP(framework.ClusterA, framework.ClusterA, nginxServiceClusterA) + f.AwaitAggregatedServiceImport(framework.ClusterA, nginxServiceClusterA, 2) f.AwaitEndpointSlices(framework.ClusterA, nginxServiceClusterA.Name, nginxServiceClusterA.Namespace, 2, 2) f.VerifyServiceIPWithDig(framework.ClusterA, framework.ClusterA, nginxServiceClusterA, netshootPodList, checkedDomains, @@ -415,7 +415,7 @@ func RunServiceDiscoveryClusterNameTest(f *lhframework.Framework) { Expect(err).NotTo(HaveOccurred()) nginxServiceClusterB = svc - f.AwaitServiceImportIP(framework.ClusterB, framework.ClusterA, nginxServiceClusterB) + f.AwaitAggregatedServiceImport(framework.ClusterA, nginxServiceClusterB, 2) f.AwaitEndpointSlices(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 2, 2) f.VerifyServiceIPWithDig(framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList, checkedDomains, @@ -462,14 +462,14 @@ func RunServiceDiscoveryRoundRobinTest(f *lhframework.Framework) { Expect(err).NotTo(HaveOccurred()) nginxServiceClusterC = svc - f.AwaitServiceImportIP(framework.ClusterC, framework.ClusterC, nginxServiceClusterC) + f.AwaitAggregatedServiceImport(framework.ClusterC, nginxServiceClusterC, 2) f.AwaitEndpointSlices(framework.ClusterC, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace, 2, 2) svc, err = f.GetService(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace) Expect(err).NotTo(HaveOccurred()) nginxServiceClusterB = svc - f.AwaitServiceImportIP(framework.ClusterB, framework.ClusterB, nginxServiceClusterB) + f.AwaitAggregatedServiceImport(framework.ClusterB, nginxServiceClusterB, 2) f.AwaitEndpointSlices(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 2, 2) var serviceIPList []string @@ -507,7 +507,7 @@ func RunServicesClusterAvailabilityMultiClusterTest(f *lhframework.Framework) { Expect(err).NotTo(HaveOccurred()) nginxServiceClusterB = svc - f.AwaitServiceImportIP(framework.ClusterB, framework.ClusterA, nginxServiceClusterB) + f.AwaitAggregatedServiceImport(framework.ClusterA, nginxServiceClusterB, 1) f.AwaitEndpointSlices(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1) f.AwaitEndpointSlices(framework.ClusterA, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1, 1) @@ -525,7 +525,7 @@ func RunServicesClusterAvailabilityMultiClusterTest(f *lhframework.Framework) { Expect(err).NotTo(HaveOccurred()) nginxServiceClusterC = svc - f.AwaitServiceImportIP(framework.ClusterC, framework.ClusterA, nginxServiceClusterC) + f.AwaitAggregatedServiceImport(framework.ClusterA, nginxServiceClusterC, 2) f.AwaitEndpointSlices(framework.ClusterA, nginxServiceClusterC.Name, nginxServiceClusterC.Namespace, 2, 2) f.VerifyServiceIPWithDig(framework.ClusterA, framework.ClusterB, nginxServiceClusterB, netshootPodList, diff --git a/test/e2e/discovery/statefulsets.go b/test/e2e/discovery/statefulsets.go index 4af90f1c8..194635a1c 100644 --- a/test/e2e/discovery/statefulsets.go +++ b/test/e2e/discovery/statefulsets.go @@ -78,7 +78,7 @@ func RunSSDiscoveryTest(f *lhframework.Framework) { verifyEndpointSlices(f, framework.ClusterA, netshootPodList, endpointSlices, nginxServiceClusterB, 1, true, clusterAName) f.DeleteServiceExport(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace) - f.AwaitServiceImportCount(framework.ClusterA, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 0) + f.AwaitAggregatedServiceImport(framework.ClusterA, nginxServiceClusterB, 0) f.AwaitEndpointSlices(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 0, 0) verifyEndpointSlices(f, framework.ClusterA, netshootPodList, endpointSlices, nginxServiceClusterB, 1, false, clusterAName) @@ -124,7 +124,7 @@ func RunSSDiscoveryLocalTest(f *lhframework.Framework) { verifyEndpointSlices(f, framework.ClusterA, netshootPodList, endpointSlices, nginxServiceClusterB, 2, true, clusterAName) f.DeleteServiceExport(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace) - f.AwaitServiceImportCount(framework.ClusterA, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 1) + f.AwaitAggregatedServiceImport(framework.ClusterA, nginxServiceClusterB, 1) verifyCount := 0 @@ -142,7 +142,7 @@ func RunSSDiscoveryLocalTest(f *lhframework.Framework) { Expect(verifyCount).To(Equal(2), "Mismatch in count of IPs to be validated with dig") f.DeleteServiceExport(framework.ClusterA, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace) - f.AwaitServiceImportCount(framework.ClusterA, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 0) + f.AwaitAggregatedServiceImport(framework.ClusterA, nginxServiceClusterB, 0) f.AwaitEndpointSlices(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 0, 0) } @@ -186,7 +186,7 @@ func RunSSPodsAvailabilityTest(f *lhframework.Framework) { } f.DeleteServiceExport(framework.ClusterB, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace) - f.AwaitServiceImportCount(framework.ClusterA, nginxServiceClusterB.Name, nginxServiceClusterB.Namespace, 0) + f.AwaitAggregatedServiceImport(framework.ClusterA, nginxServiceClusterB, 0) } //nolint:unparam // `targetCluster` always receives `framework.ClusterA`. diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index 9b2f12fe8..0360cc7c0 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -31,6 +31,7 @@ import ( appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" @@ -160,7 +161,10 @@ func (f *Framework) AwaitServiceExportedStatusCondition(cluster framework.Cluste } } - return false, fmt.Sprintf("ServiceExport %s condition status not found", constants.ServiceExportSynced), nil + out, _ := json.MarshalIndent(se.Status.Conditions, "", " ") + + return false, fmt.Sprintf("ServiceExport %s condition status not found. Actual: %s", + constants.ServiceExportSynced, out), nil }) } @@ -177,76 +181,37 @@ func (f *Framework) GetService(cluster framework.ClusterIndex, name, namespace s return framework.KubeClients[cluster].CoreV1().Services(namespace).Get(context.TODO(), name, metav1.GetOptions{}) } -func (f *Framework) AwaitServiceImportIP(srcCluster, targetCluster framework.ClusterIndex, svc *v1.Service) *mcsv1a1.ServiceImport { - serviceIP := f.GetServiceIP(srcCluster, svc, false) - - return f.AwaitServiceImportWithIP(targetCluster, svc, serviceIP) -} +func (f *Framework) AwaitAggregatedServiceImport(targetCluster framework.ClusterIndex, svc *v1.Service, clusterCount int) { + By(fmt.Sprintf("Retrieving ServiceImport for %q in ns %q on %q", svc.Name, svc.Namespace, + framework.TestContext.ClusterIDs[targetCluster])) -func (f *Framework) AwaitServiceImportWithIP(targetCluster framework.ClusterIndex, svc *v1.Service, - serviceIP string, -) *mcsv1a1.ServiceImport { - var retServiceImport *mcsv1a1.ServiceImport + si := MCSClients[targetCluster].MulticlusterV1alpha1().ServiceImports(svc.Namespace) - siNamePrefix := svc.Name + "-" + svc.Namespace + "-" - si := MCSClients[targetCluster].MulticlusterV1alpha1().ServiceImports(framework.TestContext.SubmarinerNamespace) - By(fmt.Sprintf("Retrieving ServiceImport for %s on %q", siNamePrefix, framework.TestContext.ClusterIDs[targetCluster])) framework.AwaitUntil("retrieve ServiceImport", func() (interface{}, error) { - return si.List(context.TODO(), metav1.ListOptions{}) - }, func(result interface{}) (bool, string, error) { - siList := result.(*mcsv1a1.ServiceImportList) - if len(siList.Items) < 1 { - return false, fmt.Sprintf("ServiceImport with name prefix %s not found", siNamePrefix), nil - } - for i := range siList.Items { - si := &siList.Items[i] - if strings.HasPrefix(si.Name, siNamePrefix) { - if si.Spec.IPs[0] == serviceIP { - retServiceImport = &siList.Items[i] - return true, "", nil - } - } + obj, err := si.Get(context.TODO(), svc.Name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil, nil //nolint:nilnil // Intentional } - return false, fmt.Sprintf("Failed to find ServiceImport with IP %s", serviceIP), nil - }) - - return retServiceImport -} - -func (f *Framework) AwaitServiceImportDelete(targetCluster framework.ClusterIndex, name, namespace string) { - siNamePrefix := name + "-" + namespace - si := MCSClients[targetCluster].MulticlusterV1alpha1().ServiceImports(framework.TestContext.SubmarinerNamespace) - framework.AwaitUntil("retrieve ServiceImport", func() (interface{}, error) { - return si.List(context.TODO(), metav1.ListOptions{}) + return obj, err }, func(result interface{}) (bool, string, error) { - siList := result.(*mcsv1a1.ServiceImportList) - for i := range siList.Items { - si := &siList.Items[i] - if strings.HasPrefix(si.Name, siNamePrefix) { - return false, fmt.Sprintf("ServiceImport with name prefix %s still exists", siNamePrefix), nil + if clusterCount == 0 { + if result != nil { + return false, "ServiceImport still exists", nil } + + return true, "", nil } - return true, "", nil - }) -} + if result == nil { + return false, "ServiceImport not found", nil + } -func (f *Framework) AwaitServiceImportCount(targetCluster framework.ClusterIndex, name, namespace string, count int) { - labelMap := map[string]string{ - mcsv1a1.LabelServiceName: name, - constants.LabelSourceNamespace: namespace, - } - siListOptions := metav1.ListOptions{ - LabelSelector: labels.Set(labelMap).String(), - } - si := MCSClients[targetCluster].MulticlusterV1alpha1().ServiceImports(framework.TestContext.SubmarinerNamespace) - framework.AwaitUntil("retrieve ServiceImport", func() (interface{}, error) { - return si.List(context.TODO(), siListOptions) - }, func(result interface{}) (bool, string, error) { - siList := result.(*mcsv1a1.ServiceImportList) - if len(siList.Items) != count { - return false, fmt.Sprintf("ServiceImport count was %v instead of %v", len(siList.Items), count), nil + si := result.(*mcsv1a1.ServiceImport) + + if len(si.Status.Clusters) != clusterCount { + return false, fmt.Sprintf("Actual cluster count %d does not match expected %d", + len(si.Status.Clusters), clusterCount), nil } return true, "", nil @@ -438,7 +403,7 @@ func create(f *Framework, cluster framework.ClusterIndex, statefulSet *appsv1.St } func (f *Framework) AwaitEndpointSlices(targetCluster framework.ClusterIndex, name, namespace string, - expSliceCount, expEpCount int, + expSliceCount, expReadyCount int, ) (endpointSliceList *discovery.EndpointSliceList) { ep := framework.KubeClients[targetCluster].DiscoveryV1().EndpointSlices(namespace) labelMap := map[string]string{ @@ -455,22 +420,27 @@ func (f *Framework) AwaitEndpointSlices(targetCluster framework.ClusterIndex, na }, func(result interface{}) (bool, string, error) { endpointSliceList = result.(*discovery.EndpointSliceList) sliceCount := 0 - epCount := 0 + readyCount := 0 for i := range endpointSliceList.Items { es := &endpointSliceList.Items[i] - if name == "" || strings.HasPrefix(es.Name, name) { + if name == "" || es.Labels[mcsv1a1.LabelServiceName] == name { sliceCount++ - epCount += len(es.Endpoints) + + for j := range es.Endpoints { + if es.Endpoints[j].Conditions.Ready == nil || *es.Endpoints[j].Conditions.Ready { + readyCount++ + } + } } } if expSliceCount != anyCount && sliceCount != expSliceCount { - return false, fmt.Sprintf("%d EndpointSlices found when expected %d", len(endpointSliceList.Items), expSliceCount), nil + return false, fmt.Sprintf("%d EndpointSlices found when expected %d", sliceCount, expSliceCount), nil } - if expEpCount != anyCount && epCount != expEpCount { - return false, fmt.Sprintf("%d total Endpoints found when expected %d", epCount, expEpCount), nil + if expReadyCount != anyCount && readyCount != expReadyCount { + return false, fmt.Sprintf("%d ready Endpoints found when expected %d", readyCount, expReadyCount), nil } return true, "", nil