Skip to content

Commit

Permalink
Address comments
Browse files Browse the repository at this point in the history 
Signed-off-by: wgrayson <[email protected]>
  • Loading branch information
@GraysonWu
GraysonWu committed Jul 6, 2022
1 parent d4c7b1f commit 5832421
Show file tree
Hide file tree
Showing 3 changed files with 97 additions and 127 deletions.
23 changes: 3 additions & 20 deletions multicluster/test/e2e/framework.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ package e2e

import (
"fmt"
"math/rand"
"os"
"strings"
"time"
Expand Down Expand Up @@ -45,11 +44,12 @@ const (
leaderCluster string = "leader-cluster"
serviceExportYML string = "serviceexport.yml"

nameSuffixLength int = 8
testServerPod string = "test-nginx-pod"
gatewayNodeClientSuffix string = "gateway-client"
regularNodeClientSuffix string = "regular-client"

nginxImage = "projects.registry.vmware.com/antrea/nginx:1.21.6-alpine"
agnhostImage = "agnhost:2.26"
nginxPodName = "test-nginx-pod"
)

var provider providers.ProviderInterface
Expand Down Expand Up @@ -203,23 +203,6 @@ func (data *MCTestData) podWaitFor(timeout time.Duration, clusterName, name, nam
return nil, fmt.Errorf("clusterName %s not found", clusterName)
}

// A DNS-1123 subdomain must consist of lower case alphanumeric characters
var lettersAndDigits = []rune("abcdefghijklmnopqrstuvwxyz0123456789")

func randSeq(n int) string {
b := make([]rune, n)
for i := range b {
// #nosec G404: random number generator not used for security purposes
randIdx := rand.Intn(len(lettersAndDigits))
b[i] = lettersAndDigits[randIdx]
}
return string(b)
}

func randName(prefix string) string {
return prefix + randSeq(nameSuffixLength)
}

func (data *MCTestData) probeServiceFromPodInCluster(
cluster string,
podName string,
Expand Down
10 changes: 4 additions & 6 deletions multicluster/test/e2e/main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,12 +107,10 @@ func TestConnectivity(t *testing.T) {
}

t.Run("TestMCServiceExport", func(t *testing.T) {
podName := data.setupTestResources(t)
setUpServiceExport(data, t)
t.Run("Case=Connectivity", func(t *testing.T) { testProbeMCService(t, data) })
t.Run("Case=ANPNoPriority", func(t *testing.T) { testANP(t, data) })
tearDownServiceExport(data)
data.tearDownTestResources(t, podName)
defer tearDownForServiceExportsTest(t, data)
initializeForServiceExportsTest(t, data)
t.Run("Case=MCServiceConnectivity", func(t *testing.T) { testMCServiceConnectivity(t, data) })
t.Run("Case=ANPToServices", func(t *testing.T) { testANPToServices(t, data) })
})

t.Run("TestAntreaPolicy", func(t *testing.T) {
Expand Down
191 changes: 90 additions & 101 deletions multicluster/test/e2e/service_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,32 +29,42 @@ import (
e2euttils "antrea.io/antrea/test/e2e/utils"
)

func (data *MCTestData) setupTestResources(t *testing.T) string {
podName := randName(nginxPodName)
func initializeForServiceExportsTest(t *testing.T, data *MCTestData) {
data.setupServerPodAndService(t)
data.setUpServiceExport(t)
data.setUpClientPodInCluster(t)
}

func tearDownForServiceExportsTest(t *testing.T, data *MCTestData) {
data.tearDownClientPodInCluster(t)
data.tearDownServiceExport()
data.tearDownServerPodAndService(t)
}

func (data *MCTestData) setupServerPodAndService(t *testing.T) {
createPodAndService := func(clusterName, clusterServiceName string) {
if err := createPodWrapper(t, data, clusterName, multiClusterTestNamespace, podName, "", nginxImage, "nginx", nil, nil, nil, nil, false, nil); err != nil {
if err := createPodWrapper(t, data, clusterName, multiClusterTestNamespace, testServerPod, "", nginxImage, "nginx", nil, nil, nil, nil, false, nil); err != nil {
t.Fatalf("Error when creating nginx Pod in cluster %s: %v", clusterName, err)
}
if _, err := data.createService(clusterName, clusterServiceName, multiClusterTestNamespace, 80, 80, corev1.ProtocolTCP, map[string]string{"app": "nginx"}, false,
false, corev1.ServiceTypeClusterIP, nil, nil); err != nil {
t.Fatalf("Error when creating Service %s in cluster %s: %v", clusterServiceName, clusterName, err)
}
}
// Create Pod and Service in west and ease cluster
// Create Pod and Service in west and east clusters
createPodAndService(westCluster, westClusterTestService)
createPodAndService(eastCluster, eastClusterTestService)
return podName
}

func (data *MCTestData) tearDownTestResources(t *testing.T, podName string) {
func (data *MCTestData) tearDownServerPodAndService(t *testing.T) {
deleteServiceWrapper(t, testData, westCluster, multiClusterTestNamespace, westClusterTestService)
deleteServiceWrapper(t, testData, eastCluster, multiClusterTestNamespace, eastClusterTestService)
deletePodWrapper(t, data, westCluster, multiClusterTestNamespace, podName)
deletePodWrapper(t, data, eastCluster, multiClusterTestNamespace, podName)
deletePodWrapper(t, data, westCluster, multiClusterTestNamespace, testServerPod)
deletePodWrapper(t, data, eastCluster, multiClusterTestNamespace, testServerPod)
}

// Deploy service exports in east and west clusters
func setUpServiceExport(data *MCTestData, t *testing.T) {
// Deploy ServiceExports in east and west clusters
func (data *MCTestData) setUpServiceExport(t *testing.T) {
if err := data.deployServiceExport(westCluster); err != nil {
t.Fatalf("Error when deploy ServiceExport in west cluster: %v", err)
}
Expand All @@ -64,112 +74,71 @@ func setUpServiceExport(data *MCTestData, t *testing.T) {
time.Sleep(importServiceDelay)
}

func tearDownServiceExport(data *MCTestData) {
func (data *MCTestData) tearDownServiceExport() {
data.deleteServiceExport(westCluster)
data.deleteServiceExport(eastCluster)
}

func (data *MCTestData) setUpClientPodInCluster(t *testing.T) {
data.createClientPodInCluster(t, eastCluster, data.clusterGateways[eastCluster], getClusterGatewayClientPodName(eastCluster))
data.createClientPodInCluster(t, eastCluster, data.clusterRegularNodes[eastCluster], getClusterRegularClientPodName(eastCluster))
data.createClientPodInCluster(t, westCluster, data.clusterGateways[westCluster], getClusterGatewayClientPodName(westCluster))
data.createClientPodInCluster(t, westCluster, data.clusterRegularNodes[westCluster], getClusterRegularClientPodName(westCluster))
}

func (data *MCTestData) tearDownClientPodInCluster(t *testing.T) {
deletePodWrapper(t, data, eastCluster, multiClusterTestNamespace, getClusterGatewayClientPodName(eastCluster))
deletePodWrapper(t, data, eastCluster, multiClusterTestNamespace, getClusterRegularClientPodName(eastCluster))
deletePodWrapper(t, data, westCluster, multiClusterTestNamespace, getClusterGatewayClientPodName(westCluster))
deletePodWrapper(t, data, westCluster, multiClusterTestNamespace, getClusterRegularClientPodName(westCluster))
}

// Try to curl the counter part services in east and west clusters.
// If we get status code 200, it means that the resources is exported by the east cluster
// and imported by the west cluster.
func testProbeMCService(t *testing.T, data *MCTestData) {
data.testProbeMCService(t)
func testMCServiceConnectivity(t *testing.T, data *MCTestData) {
data.testMCServiceConnectivity(t)
}

func testANP(t *testing.T, data *MCTestData) {
data.testANP(t)
func testANPToServices(t *testing.T, data *MCTestData) {
data.testANPToServices(t)
}

func (data *MCTestData) testProbeMCService(t *testing.T) {
func (data *MCTestData) testMCServiceConnectivity(t *testing.T) {
data.probeMCServiceFromCluster(t, eastCluster, westClusterTestService)
data.probeMCServiceFromCluster(t, westCluster, eastClusterTestService)
}

func (data *MCTestData) testANP(t *testing.T) {
clientPodName := "test-service-client"
svc, err := data.getService(eastCluster, multiClusterTestNamespace, fmt.Sprintf("antrea-mc-%s", westClusterTestService))
if err != nil {
t.Fatalf("Error when getting the imported service %s: %v", fmt.Sprintf("antrea-mc-%s", westClusterTestService), err)
}

eastIP := svc.Spec.ClusterIP
gwClientName := clientPodName + "-gateway"
regularClientName := clientPodName + "-regularnode"

createEastPod := func(nodeName string, podName string) {
if err := data.createPod(eastCluster, podName, nodeName, multiClusterTestNamespace, "client", agnhostImage,
[]string{"sleep", strconv.Itoa(3600)}, nil, nil, nil, false, nil); err != nil {
t.Fatalf("Error when creating client Pod in east cluster: %v", err)
}
t.Logf("Checking Pod status %s in Namespace %s of cluster %s", podName, multiClusterTestNamespace, eastCluster)
_, err := data.podWaitFor(defaultTimeout, eastCluster, podName, multiClusterTestNamespace, func(pod *corev1.Pod) (bool, error) {
return pod.Status.Phase == corev1.PodRunning, nil
})
if err != nil {
deletePodWrapper(t, data, eastCluster, multiClusterTestNamespace, podName)
t.Fatalf("Error when waiting for Pod '%s' in east cluster: %v", podName, err)
}
}

// Create a Pod in east cluster's Gateway and verify the MC Service connectivity from it.
createEastPod(data.clusterGateways[eastCluster], gwClientName)
defer deletePodWrapper(t, data, eastCluster, multiClusterTestNamespace, gwClientName)

t.Logf("Probing Service from client Pod %s in cluster %s", gwClientName, eastCluster)
if err := data.probeServiceFromPodInCluster(eastCluster, gwClientName, "client", multiClusterTestNamespace, eastIP); err != nil {
t.Fatalf("Error when probing Service from client Pod %s in cluster %s, err: %v", gwClientName, eastCluster, err)
}

// Create a Pod in east cluster's regular Node and verify the MC Service connectivity from it.
createEastPod(data.clusterRegularNodes[eastCluster], regularClientName)
defer deletePodWrapper(t, data, eastCluster, multiClusterTestNamespace, regularClientName)

t.Logf("Probing Service from client Pod %s in cluster %s", regularClientName, eastCluster)
if err := data.probeServiceFromPodInCluster(eastCluster, regularClientName, "client", multiClusterTestNamespace, eastIP); err != nil {
t.Fatalf("Error when probing Service from client Pod %s in cluster %s, err: %v", regularClientName, eastCluster, err)
}

// Create a Pod in west cluster and verify the MC Service connectivity from it.
if err := data.createPod(westCluster, clientPodName, "", multiClusterTestNamespace, "client", agnhostImage,
[]string{"sleep", strconv.Itoa(3600)}, nil, nil, nil, false, nil); err != nil {
t.Fatalf("Error when creating client Pod in west cluster: %v", err)
}
defer deletePodWrapper(t, data, westCluster, multiClusterTestNamespace, clientPodName)
_, err = data.podWaitFor(defaultTimeout, westCluster, clientPodName, multiClusterTestNamespace, func(pod *corev1.Pod) (bool, error) {
return pod.Status.Phase == corev1.PodRunning, nil
})
func (data *MCTestData) probeMCServiceFromCluster(t *testing.T, clusterName string, serviceName string) {
svc, err := data.getService(clusterName, multiClusterTestNamespace, fmt.Sprintf("antrea-mc-%s", serviceName))
if err != nil {
t.Fatalf("Error when waiting for Pod '%s' in west cluster: %v", clientPodName, err)
t.Fatalf("Error when getting the imported Service %s: %v", fmt.Sprintf("antrea-mc-%s", serviceName), err)
}
ip := svc.Spec.ClusterIP
gwClientName := getClusterGatewayClientPodName(clusterName)
regularClientName := getClusterRegularClientPodName(clusterName)

svc, err = data.getService(westCluster, multiClusterTestNamespace, fmt.Sprintf("antrea-mc-%s", eastClusterTestService))
if err != nil {
t.Fatalf("Error when getting the imported service %s: %v", fmt.Sprintf("antrea-mc-%s", eastClusterTestService), err)
t.Logf("Probing Service from client Pod %s in cluster %s", gwClientName, clusterName)
if err := data.probeServiceFromPodInCluster(clusterName, gwClientName, "client", multiClusterTestNamespace, ip); err != nil {
t.Fatalf("Error when probing Service from client Pod %s in cluster %s, err: %v", gwClientName, clusterName, err)
}
westIP := svc.Spec.ClusterIP
if err := data.probeServiceFromPodInCluster(westCluster, clientPodName, "client", multiClusterTestNamespace, westIP); err != nil {
t.Fatalf("Error when probing service from %s, err: %v", westCluster, err)
t.Logf("Probing Service from client Pod %s in cluster %s", regularClientName, clusterName)
if err := data.probeServiceFromPodInCluster(clusterName, regularClientName, "client", multiClusterTestNamespace, ip); err != nil {
t.Fatalf("Error when probing Service from client Pod %s in cluster %s, err: %v", regularClientName, clusterName, err)
}

// Verify that ACNP works fine with new Multicluster Service.
data.verifyMCServiceACNP(t, gwClientName, eastIP)
return
}

func (data *MCTestData) probeMCServiceFromCluster(t *testing.T, clusterName string, serviceName string) string {
svc, err := data.getService(clusterName, multiClusterTestNamespace, fmt.Sprintf("antrea-mc-%s", serviceName))
func (data *MCTestData) testANPToServices(t *testing.T) {
svc, err := data.getService(eastCluster, multiClusterTestNamespace, fmt.Sprintf("antrea-mc-%s", westClusterTestService))
if err != nil {
t.Fatalf("Error when getting the imported service %s: %v", fmt.Sprintf("antrea-mc-%s", serviceName), err)
}

ip := svc.Spec.ClusterIP
if err := data.probeFromCluster(clusterName, ip); err != nil {
t.Fatalf("Error when probing service from %s", clusterName)
t.Fatalf("Error when getting the imported Service %s: %v", fmt.Sprintf("antrea-mc-%s", westClusterTestService), err)
}
return ip
}
eastIP := svc.Spec.ClusterIP
eastGwClientName := getClusterGatewayClientPodName(eastCluster)
eastRegularClientName := getClusterRegularClientPodName(eastCluster)

func (data *MCTestData) verifyMCServiceACNP(t *testing.T, clientPodName, eastIP string) {
var err error
// Verify that ACNP ToServices works fine with new Multicluster Service.
anpBuilder := &e2euttils.AntreaNetworkPolicySpecBuilder{}
anpBuilder = anpBuilder.SetName(multiClusterTestNamespace, "block-west-exported-service").
SetPriority(1.0).
Expand All @@ -183,11 +152,33 @@ func (data *MCTestData) verifyMCServiceACNP(t *testing.T, clientPodName, eastIP
}
defer data.deleteANP(eastCluster, multiClusterTestNamespace, anpBuilder.Name)

connectivity := data.probeFromPodInCluster(eastCluster, multiClusterTestNamespace, clientPodName, "client", eastIP, fmt.Sprintf("antrea-mc-%s", westClusterTestService), 80, corev1.ProtocolTCP)
connectivity := data.probeFromPodInCluster(eastCluster, multiClusterTestNamespace, eastGwClientName, "client", eastIP, fmt.Sprintf("antrea-mc-%s", westClusterTestService), 80, corev1.ProtocolTCP)
if connectivity == antreae2e.Error {
t.Errorf("Failure -- could not complete probeFromPodInCluster: %v", err)
} else if connectivity != antreae2e.Dropped {
t.Errorf("Failure -- wrong result from probing exported Service from gateway clientPod after applying toServices AntreaNetworkPolicy. Expected: %v, Actual: %v", antreae2e.Dropped, connectivity)
}

connectivity = data.probeFromPodInCluster(eastCluster, multiClusterTestNamespace, eastRegularClientName, "client", eastIP, fmt.Sprintf("antrea-mc-%s", westClusterTestService), 80, corev1.ProtocolTCP)
if connectivity == antreae2e.Error {
t.Errorf("Failure -- could not complete probeFromPodInCluster: %v", err)
} else if connectivity != antreae2e.Dropped {
t.Errorf("Failure -- wrong result from probing exported Service after applying toService AntreaNetworkPolicy. Expected: %v, Actual: %v", antreae2e.Dropped, connectivity)
t.Errorf("Failure -- wrong result from probing exported Service from regular clientPod after applying toServices AntreaNetworkPolicy. Expected: %v, Actual: %v", antreae2e.Dropped, connectivity)
}
}

func (data *MCTestData) createClientPodInCluster(t *testing.T, cluster string, nodeName string, podName string) {
if err := data.createPod(cluster, podName, nodeName, multiClusterTestNamespace, "client", agnhostImage,
[]string{"sleep", strconv.Itoa(3600)}, nil, nil, nil, false, nil); err != nil {
t.Fatalf("Error when creating client Pod in cluster '%s': %v", cluster, err)
}
t.Logf("Checking Pod status %s in Namespace %s of cluster %s", podName, multiClusterTestNamespace, cluster)
_, err := data.podWaitFor(defaultTimeout, cluster, podName, multiClusterTestNamespace, func(pod *corev1.Pod) (bool, error) {
return pod.Status.Phase == corev1.PodRunning, nil
})
if err != nil {
deletePodWrapper(t, data, cluster, multiClusterTestNamespace, podName)
t.Fatalf("Error when waiting for Pod '%s' in cluster '%s': %v", podName, cluster, err)
}
}

Expand Down Expand Up @@ -278,12 +269,10 @@ func teardownGateway(t *testing.T, data *MCTestData) {
}
}

func (data *MCTestData) probeFromCluster(clusterName string, url string) error {
var rc int
var err error
rc, _, _, err = provider.RunCommandOnNode(clusterName, fmt.Sprintf("curl --connect-timeout 5 -s %s", url))
if err != nil || rc != 0 {
return fmt.Errorf("error when curl the url %s: %v", url, err)
}
return nil
func getClusterGatewayClientPodName(cluster string) string {
return cluster + "-" + gatewayNodeClientSuffix
}

func getClusterRegularClientPodName(cluster string) string {
return cluster + "-" + regularNodeClientSuffix
}

0 comments on commit 5832421

Please sign in to comment.