diff --git a/multicluster/test/e2e/framework.go b/multicluster/test/e2e/framework.go index 5887db0c054..53fb2d3c369 100644 --- a/multicluster/test/e2e/framework.go +++ b/multicluster/test/e2e/framework.go @@ -45,11 +45,14 @@ const ( leaderCluster string = "leader-cluster" serviceExportYML string = "serviceexport.yml" + testServerPod string = "test-nginx-pod" + gatewayNodeClientSuffix string = "gateway-client" + regularNodeClientSuffix string = "regular-client" + nameSuffixLength int = 8 nginxImage = "projects.registry.vmware.com/antrea/nginx:1.21.6-alpine" agnhostImage = "agnhost:2.26" - nginxPodName = "test-nginx-pod" ) var provider providers.ProviderInterface diff --git a/multicluster/test/e2e/main_test.go b/multicluster/test/e2e/main_test.go index 1423a565b17..d45f2a78b0e 100644 --- a/multicluster/test/e2e/main_test.go +++ b/multicluster/test/e2e/main_test.go @@ -107,12 +107,10 @@ func TestConnectivity(t *testing.T) { } t.Run("TestMCServiceExport", func(t *testing.T) { - podName := data.setupTestResources(t) - setUpServiceExport(data, t) - t.Run("Case=Connectivity", func(t *testing.T) { testProbeMCService(t, data) }) - t.Run("Case=ANPNoPriority", func(t *testing.T) { testANP(t, data) }) - tearDownServiceExport(data) - data.tearDownTestResources(t, podName) + defer tearDownForServiceExportsTest(t, data) + initializeForServiceExportsTest(t, data) + t.Run("Case=Connectivity", func(t *testing.T) { testMCServiceConnectivity(t, data) }) + t.Run("Case=ANPToServices", func(t *testing.T) { testANPToServices(t, data) }) }) t.Run("TestAntreaPolicy", func(t *testing.T) { diff --git a/multicluster/test/e2e/service_test.go b/multicluster/test/e2e/service_test.go index 7f53216d1e2..9002cda462e 100644 --- a/multicluster/test/e2e/service_test.go +++ b/multicluster/test/e2e/service_test.go @@ -29,10 +29,21 @@ import ( e2euttils "antrea.io/antrea/test/e2e/utils" ) -func (data *MCTestData) setupTestResources(t *testing.T) string { - podName := randName(nginxPodName) +func initializeForServiceExportsTest(t *testing.T, data *MCTestData) { + data.setupServerPodAndService(t) + data.setUpServiceExport(t) + data.setUpClientPodInCluster(t) +} + +func tearDownForServiceExportsTest(t *testing.T, data *MCTestData) { + data.tearDownClientPodInCluster(t) + data.tearDownServiceExport() + data.tearDownServerPodAndService(t) +} + +func (data *MCTestData) setupServerPodAndService(t *testing.T) { createPodAndService := func(clusterName, clusterServiceName string) { - if err := createPodWrapper(t, data, clusterName, multiClusterTestNamespace, podName, "", nginxImage, "nginx", nil, nil, nil, nil, false, nil); err != nil { + if err := createPodWrapper(t, data, clusterName, multiClusterTestNamespace, testServerPod, "", nginxImage, "nginx", nil, nil, nil, nil, false, nil); err != nil { t.Fatalf("Error when creating nginx Pod in cluster %s: %v", clusterName, err) } if _, err := data.createService(clusterName, clusterServiceName, multiClusterTestNamespace, 80, 80, corev1.ProtocolTCP, map[string]string{"app": "nginx"}, false, @@ -40,21 +51,20 @@ func (data *MCTestData) setupTestResources(t *testing.T) string { t.Fatalf("Error when creating Service %s in cluster %s: %v", clusterServiceName, clusterName, err) } } - // Create Pod and Service in west and ease cluster + // Create Pod and Service in west and east cluster createPodAndService(westCluster, westClusterTestService) createPodAndService(eastCluster, eastClusterTestService) - return podName } -func (data *MCTestData) tearDownTestResources(t *testing.T, podName string) { +func (data *MCTestData) tearDownServerPodAndService(t *testing.T) { deleteServiceWrapper(t, testData, westCluster, multiClusterTestNamespace, westClusterTestService) deleteServiceWrapper(t, testData, eastCluster, multiClusterTestNamespace, eastClusterTestService) - deletePodWrapper(t, data, westCluster, multiClusterTestNamespace, podName) - deletePodWrapper(t, data, eastCluster, multiClusterTestNamespace, podName) + deletePodWrapper(t, data, westCluster, multiClusterTestNamespace, testServerPod) + deletePodWrapper(t, data, eastCluster, multiClusterTestNamespace, testServerPod) } -// Deploy service exports in east and west clusters -func setUpServiceExport(data *MCTestData, t *testing.T) { +// Deploy ServiceExports in east and west clusters +func (data *MCTestData) setUpServiceExport(t *testing.T) { if err := data.deployServiceExport(westCluster); err != nil { t.Fatalf("Error when deploy ServiceExport in west cluster: %v", err) } @@ -64,112 +74,71 @@ func setUpServiceExport(data *MCTestData, t *testing.T) { time.Sleep(importServiceDelay) } -func tearDownServiceExport(data *MCTestData) { +func (data *MCTestData) tearDownServiceExport() { data.deleteServiceExport(westCluster) data.deleteServiceExport(eastCluster) } +func (data *MCTestData) setUpClientPodInCluster(t *testing.T) { + data.createClientPodInCluster(t, eastCluster, data.clusterGateways[eastCluster], getClusterGatewayClientPodName(eastCluster)) + data.createClientPodInCluster(t, eastCluster, data.clusterRegularNodes[eastCluster], getClusterRegularClientPodName(eastCluster)) + data.createClientPodInCluster(t, westCluster, data.clusterGateways[westCluster], getClusterGatewayClientPodName(westCluster)) + data.createClientPodInCluster(t, westCluster, data.clusterRegularNodes[westCluster], getClusterRegularClientPodName(westCluster)) +} + +func (data *MCTestData) tearDownClientPodInCluster(t *testing.T) { + deletePodWrapper(t, data, eastCluster, multiClusterTestNamespace, getClusterGatewayClientPodName(eastCluster)) + deletePodWrapper(t, data, eastCluster, multiClusterTestNamespace, getClusterRegularClientPodName(eastCluster)) + deletePodWrapper(t, data, westCluster, multiClusterTestNamespace, getClusterGatewayClientPodName(westCluster)) + deletePodWrapper(t, data, westCluster, multiClusterTestNamespace, getClusterRegularClientPodName(westCluster)) +} + // Try to curl the counter part services in east and west clusters. // If we get status code 200, it means that the resources is exported by the east cluster // and imported by the west cluster. -func testProbeMCService(t *testing.T, data *MCTestData) { - data.testProbeMCService(t) +func testMCServiceConnectivity(t *testing.T, data *MCTestData) { + data.testMCServiceConnectivity(t) } -func testANP(t *testing.T, data *MCTestData) { - data.testANP(t) +func testANPToServices(t *testing.T, data *MCTestData) { + data.testANPToServices(t) } -func (data *MCTestData) testProbeMCService(t *testing.T) { +func (data *MCTestData) testMCServiceConnectivity(t *testing.T) { data.probeMCServiceFromCluster(t, eastCluster, westClusterTestService) data.probeMCServiceFromCluster(t, westCluster, eastClusterTestService) } -func (data *MCTestData) testANP(t *testing.T) { - clientPodName := "test-service-client" - svc, err := data.getService(eastCluster, multiClusterTestNamespace, fmt.Sprintf("antrea-mc-%s", westClusterTestService)) - if err != nil { - t.Fatalf("Error when getting the imported service %s: %v", fmt.Sprintf("antrea-mc-%s", westClusterTestService), err) - } - - eastIP := svc.Spec.ClusterIP - gwClientName := clientPodName + "-gateway" - regularClientName := clientPodName + "-regularnode" - - createEastPod := func(nodeName string, podName string) { - if err := data.createPod(eastCluster, podName, nodeName, multiClusterTestNamespace, "client", agnhostImage, - []string{"sleep", strconv.Itoa(3600)}, nil, nil, nil, false, nil); err != nil { - t.Fatalf("Error when creating client Pod in east cluster: %v", err) - } - t.Logf("Checking Pod status %s in Namespace %s of cluster %s", podName, multiClusterTestNamespace, eastCluster) - _, err := data.podWaitFor(defaultTimeout, eastCluster, podName, multiClusterTestNamespace, func(pod *corev1.Pod) (bool, error) { - return pod.Status.Phase == corev1.PodRunning, nil - }) - if err != nil { - deletePodWrapper(t, data, eastCluster, multiClusterTestNamespace, podName) - t.Fatalf("Error when waiting for Pod '%s' in east cluster: %v", podName, err) - } - } - - // Create a Pod in east cluster's Gateway and verify the MC Service connectivity from it. - createEastPod(data.clusterGateways[eastCluster], gwClientName) - defer deletePodWrapper(t, data, eastCluster, multiClusterTestNamespace, gwClientName) - - t.Logf("Probing Service from client Pod %s in cluster %s", gwClientName, eastCluster) - if err := data.probeServiceFromPodInCluster(eastCluster, gwClientName, "client", multiClusterTestNamespace, eastIP); err != nil { - t.Fatalf("Error when probing Service from client Pod %s in cluster %s, err: %v", gwClientName, eastCluster, err) - } - - // Create a Pod in east cluster's regular Node and verify the MC Service connectivity from it. - createEastPod(data.clusterRegularNodes[eastCluster], regularClientName) - defer deletePodWrapper(t, data, eastCluster, multiClusterTestNamespace, regularClientName) - - t.Logf("Probing Service from client Pod %s in cluster %s", regularClientName, eastCluster) - if err := data.probeServiceFromPodInCluster(eastCluster, regularClientName, "client", multiClusterTestNamespace, eastIP); err != nil { - t.Fatalf("Error when probing Service from client Pod %s in cluster %s, err: %v", regularClientName, eastCluster, err) - } - - // Create a Pod in west cluster and verify the MC Service connectivity from it. - if err := data.createPod(westCluster, clientPodName, "", multiClusterTestNamespace, "client", agnhostImage, - []string{"sleep", strconv.Itoa(3600)}, nil, nil, nil, false, nil); err != nil { - t.Fatalf("Error when creating client Pod in west cluster: %v", err) - } - defer deletePodWrapper(t, data, westCluster, multiClusterTestNamespace, clientPodName) - _, err = data.podWaitFor(defaultTimeout, westCluster, clientPodName, multiClusterTestNamespace, func(pod *corev1.Pod) (bool, error) { - return pod.Status.Phase == corev1.PodRunning, nil - }) +func (data *MCTestData) probeMCServiceFromCluster(t *testing.T, clusterName string, serviceName string) { + svc, err := data.getService(clusterName, multiClusterTestNamespace, fmt.Sprintf("antrea-mc-%s", serviceName)) if err != nil { - t.Fatalf("Error when waiting for Pod '%s' in west cluster: %v", clientPodName, err) + t.Fatalf("Error when getting the imported service %s: %v", fmt.Sprintf("antrea-mc-%s", serviceName), err) } + ip := svc.Spec.ClusterIP + gwClientName := getClusterGatewayClientPodName(clusterName) + regularClientName := getClusterRegularClientPodName(clusterName) - svc, err = data.getService(westCluster, multiClusterTestNamespace, fmt.Sprintf("antrea-mc-%s", eastClusterTestService)) - if err != nil { - t.Fatalf("Error when getting the imported service %s: %v", fmt.Sprintf("antrea-mc-%s", eastClusterTestService), err) + t.Logf("Probing Service from client Pod %s in cluster %s", gwClientName, clusterName) + if err := data.probeServiceFromPodInCluster(clusterName, gwClientName, "client", multiClusterTestNamespace, ip); err != nil { + t.Fatalf("Error when probing Service from client Pod %s in cluster %s, err: %v", gwClientName, clusterName, err) } - westIP := svc.Spec.ClusterIP - if err := data.probeServiceFromPodInCluster(westCluster, clientPodName, "client", multiClusterTestNamespace, westIP); err != nil { - t.Fatalf("Error when probing service from %s, err: %v", westCluster, err) + t.Logf("Probing Service from client Pod %s in cluster %s", regularClientName, clusterName) + if err := data.probeServiceFromPodInCluster(clusterName, regularClientName, "client", multiClusterTestNamespace, ip); err != nil { + t.Fatalf("Error when probing Service from client Pod %s in cluster %s, err: %v", regularClientName, clusterName, err) } - - // Verify that ACNP works fine with new Multicluster Service. - data.verifyMCServiceACNP(t, gwClientName, eastIP) + return } -func (data *MCTestData) probeMCServiceFromCluster(t *testing.T, clusterName string, serviceName string) string { - svc, err := data.getService(clusterName, multiClusterTestNamespace, fmt.Sprintf("antrea-mc-%s", serviceName)) +func (data *MCTestData) testANPToServices(t *testing.T) { + svc, err := data.getService(eastCluster, multiClusterTestNamespace, fmt.Sprintf("antrea-mc-%s", westClusterTestService)) if err != nil { - t.Fatalf("Error when getting the imported service %s: %v", fmt.Sprintf("antrea-mc-%s", serviceName), err) - } - - ip := svc.Spec.ClusterIP - if err := data.probeFromCluster(clusterName, ip); err != nil { - t.Fatalf("Error when probing service from %s", clusterName) + t.Fatalf("Error when getting the imported service %s: %v", fmt.Sprintf("antrea-mc-%s", westClusterTestService), err) } - return ip -} + eastIP := svc.Spec.ClusterIP + eastGwClientName := getClusterGatewayClientPodName(eastCluster) + eastRegularClientName := getClusterRegularClientPodName(eastCluster) -func (data *MCTestData) verifyMCServiceACNP(t *testing.T, clientPodName, eastIP string) { - var err error + // Verify that ACNP ToServices works fine with new Multicluster Service. anpBuilder := &e2euttils.AntreaNetworkPolicySpecBuilder{} anpBuilder = anpBuilder.SetName(multiClusterTestNamespace, "block-west-exported-service"). SetPriority(1.0). @@ -183,11 +152,33 @@ func (data *MCTestData) verifyMCServiceACNP(t *testing.T, clientPodName, eastIP } defer data.deleteANP(eastCluster, multiClusterTestNamespace, anpBuilder.Name) - connectivity := data.probeFromPodInCluster(eastCluster, multiClusterTestNamespace, clientPodName, "client", eastIP, fmt.Sprintf("antrea-mc-%s", westClusterTestService), 80, corev1.ProtocolTCP) + connectivity := data.probeFromPodInCluster(eastCluster, multiClusterTestNamespace, eastGwClientName, "client", eastIP, fmt.Sprintf("antrea-mc-%s", westClusterTestService), 80, corev1.ProtocolTCP) + if connectivity == antreae2e.Error { + t.Errorf("Failure -- could not complete probeFromPodInCluster: %v", err) + } else if connectivity != antreae2e.Dropped { + t.Errorf("Failure -- wrong result from probing exported Service from gateway clientPod after applying toServices AntreaNetworkPolicy. Expected: %v, Actual: %v", antreae2e.Dropped, connectivity) + } + + connectivity = data.probeFromPodInCluster(eastCluster, multiClusterTestNamespace, eastRegularClientName, "client", eastIP, fmt.Sprintf("antrea-mc-%s", westClusterTestService), 80, corev1.ProtocolTCP) if connectivity == antreae2e.Error { t.Errorf("Failure -- could not complete probeFromPodInCluster: %v", err) } else if connectivity != antreae2e.Dropped { - t.Errorf("Failure -- wrong result from probing exported Service after applying toService AntreaNetworkPolicy. Expected: %v, Actual: %v", antreae2e.Dropped, connectivity) + t.Errorf("Failure -- wrong result from probing exported Service from regular clientPod after applying toServices AntreaNetworkPolicy. Expected: %v, Actual: %v", antreae2e.Dropped, connectivity) + } +} + +func (data *MCTestData) createClientPodInCluster(t *testing.T, cluster string, nodeName string, podName string) { + if err := data.createPod(cluster, podName, nodeName, multiClusterTestNamespace, "client", agnhostImage, + []string{"sleep", strconv.Itoa(3600)}, nil, nil, nil, false, nil); err != nil { + t.Fatalf("Error when creating client Pod in cluster '%s': %v", cluster, err) + } + t.Logf("Checking Pod status %s in Namespace %s of cluster %s", podName, multiClusterTestNamespace, cluster) + _, err := data.podWaitFor(defaultTimeout, cluster, podName, multiClusterTestNamespace, func(pod *corev1.Pod) (bool, error) { + return pod.Status.Phase == corev1.PodRunning, nil + }) + if err != nil { + deletePodWrapper(t, data, cluster, multiClusterTestNamespace, podName) + t.Fatalf("Error when waiting for Pod '%s' in cluster '%s': %v", podName, cluster, err) } } @@ -278,12 +269,10 @@ func teardownGateway(t *testing.T, data *MCTestData) { } } -func (data *MCTestData) probeFromCluster(clusterName string, url string) error { - var rc int - var err error - rc, _, _, err = provider.RunCommandOnNode(clusterName, fmt.Sprintf("curl --connect-timeout 5 -s %s", url)) - if err != nil || rc != 0 { - return fmt.Errorf("error when curl the url %s: %v", url, err) - } - return nil +func getClusterGatewayClientPodName(cluster string) string { + return cluster + "-" + gatewayNodeClientSuffix +} + +func getClusterRegularClientPodName(cluster string) string { + return cluster + "-" + regularNodeClientSuffix }