Add E2E test for the ACNP copy-span feature

Signed-off-by: Yang Ding <[email protected]>

ci/jenkins/test-mc.sh

@@ -237,11 +237,12 @@ function deliver_multicluster_controller {
     leader_ip=$(kubectl get nodes -o wide --no-headers=true ${LEADER_CLUSTER_CONFIG} | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 == role {print $6}')
     sed -i "s|<LEADER_CLUSTER_IP>|${leader_ip}|" ./multicluster/test/yamls/east-member-cluster.yml
     sed -i "s|<LEADER_CLUSTER_IP>|${leader_ip}|" ./multicluster/test/yamls/west-member-cluster.yml
+    rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" ./multicluster/test/yamls/test-acnp-copy-span-ns-isolation.yml jenkins@["${leader_ip}"]:"${WORKDIR}"/test-acnp-copy-span-ns-isolation.yml
 
     for kubeconfig in "${membercluter_kubeconfigs[@]}"
     do
        ip=$(kubectl get nodes -o wide --no-headers=true ${EAST_CLUSTER_CONFIG} | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 == role {print $6}')
-       rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" ./multicluster/test/yamls/test-east-serviceexport.yml jenkins@[${ip}]:${WORKDIR}/serviceexport.yml
+       rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" ./multicluster/test/yamls/test-east-serviceexport.yml jenkins@["${ip}"]:"${WORKDIR}"/serviceexport.yml
     done
 }
 

multicluster/test/e2e/main_test.go

@@ -88,3 +88,20 @@ func testMain(m *testing.M) int {
 func TestMain(m *testing.M) {
 	os.Exit(testMain(m))
 }
+
+func TestConnectivity(t *testing.T) {
+	data, err := setupTest(t)
+	if err != nil {
+		t.Fatalf("Error when setting up test: %v", err)
+	}
+	defer teardownTest(t, data)
+
+	t.Run("testServiceExport", func(t *testing.T) {
+		testServiceExport(t, data)
+	})
+	t.Run("testAntreaPolicy", func(t *testing.T) {
+		initializeForPolicyTest(t, data)
+		testMCAntreaPolicy(t, data)
+		tearDownForPolicyTest()
+	})
+}

multicluster/test/e2e/mcs_antreapolicy_test.go

@@ -0,0 +1,161 @@
+// Copyright 2022 Antrea Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+	v1 "k8s.io/api/core/v1"
+
+	antreae2e "antrea.io/antrea/test/e2e"
+)
+
+const (
+	// Provide enough time for policies to be enforced & deleted by the CNI plugin.
+	networkPolicyDelay          = 2000 * time.Millisecond
+	acnpIsolationResourceExport = "test-acnp-copy-span-ns-isolation.yml"
+)
+
+var (
+	allPodsPerCluster                      []antreae2e.Pod
+	perNamespacePods, perClusterNamespaces []string
+	podsByNamespace                        map[string][]antreae2e.Pod
+	clusterK8sUtilsMap                     map[string]*antreae2e.KubernetesUtils
+)
+
+func failOnError(err error, t *testing.T) {
+	if err != nil {
+		log.Errorf("%+v", err)
+		for _, k8sUtils := range clusterK8sUtilsMap {
+			k8sUtils.Cleanup(perClusterNamespaces)
+		}
+		t.Fatalf("test failed: %v", err)
+	}
+}
+
+// initializeForPolicyTest creates three Pods in three test Namespaces for each test cluster.
+func initializeForPolicyTest(t *testing.T, data *MCTestData) {
+	perNamespacePods = []string{"a", "b", "c"}
+	perClusterNamespaces = []string{"x", "y", "z"}
+
+	allPodsPerCluster = []antreae2e.Pod{}
+	podsByNamespace = make(map[string][]antreae2e.Pod)
+	clusterK8sUtilsMap = make(map[string]*antreae2e.KubernetesUtils)
+
+	for _, podName := range perNamespacePods {
+		for _, ns := range perClusterNamespaces {
+			allPodsPerCluster = append(allPodsPerCluster, antreae2e.NewPod(ns, podName))
+			podsByNamespace[ns] = append(podsByNamespace[ns], antreae2e.NewPod(ns, podName))
+		}
+	}
+	for clusterName, d := range data.clusterTestDataMap {
+		k8sUtils, err := antreae2e.NewKubernetesUtils(&d)
+		failOnError(err, t)
+		_, err = k8sUtils.Bootstrap(perClusterNamespaces, perNamespacePods)
+		failOnError(err, t)
+		clusterK8sUtilsMap[clusterName] = k8sUtils
+	}
+}
+
+// tearDownForPolicyTest deletes the test Namespaces specific for policy tests.
+func tearDownForPolicyTest() {
+	for _, k8sUtils := range clusterK8sUtilsMap {
+		k8sUtils.Cleanup(perClusterNamespaces)
+	}
+}
+
+func testMCAntreaPolicy(t *testing.T, data *MCTestData) {
+	data.testAntreaPolicyCopySpanNSIsolation(t)
+}
+
+// testAntreaPolicyCopySpanNSIsolation tests that after applying a ResourceExport of an ACNP
+// for Namespace isolation, strict Namespace isolation is enforced in each of the member clusters.
+func (data *MCTestData) testAntreaPolicyCopySpanNSIsolation(t *testing.T) {
+	setup := func() {
+		err := data.deployACNPResourceExport(acnpIsolationResourceExport)
+		failOnError(err, t)
+	}
+	teardown := func() {
+		err := data.deleteACNPResourceExport(acnpIsolationResourceExport)
+		failOnError(err, t)
+	}
+	reachability := antreae2e.NewReachability(allPodsPerCluster, antreae2e.Dropped)
+	reachability.ExpectAllSelfNamespace(antreae2e.Connected)
+	testStep := &antreae2e.TestStep{
+		Name:         "Port 80",
+		Reachability: reachability,
+		Ports:        []int32{80},
+		Protocol:     v1.ProtocolTCP,
+	}
+	testCaseList := []*antreae2e.TestCase{
+		{
+			Name:  "ACNP strict Namespace isolation for all clusters",
+			Steps: []*antreae2e.TestStep{testStep},
+		},
+	}
+	executeTestsOnAllMemberClusters(t, testCaseList, setup, teardown)
+}
+
+func executeTestsOnAllMemberClusters(t *testing.T, testList []*antreae2e.TestCase, setup, teardown func()) {
+	setup()
+	time.Sleep(networkPolicyDelay)
+	for _, testCase := range testList {
+		log.Infof("running test case %s", testCase.Name)
+		for _, step := range testCase.Steps {
+			log.Infof("running step %s of test case %s", step.Name, testCase.Name)
+			reachability := step.Reachability
+			if reachability != nil {
+				for clusterName, k8sUtils := range clusterK8sUtilsMap {
+					if clusterName == leaderCluster {
+						// skip traffic test for the leader cluster
+						continue
+					}
+					start := time.Now()
+					k8sUtils.Validate(allPodsPerCluster, reachability, step.Ports, step.Protocol)
+					step.Duration = time.Now().Sub(start)
+					_, wrong, _ := step.Reachability.Summary()
+					if wrong != 0 {
+						t.Errorf("failure in cluster %s -- %d wrong results", clusterName, wrong)
+						reachability.PrintSummary(true, true, true)
+					}
+				}
+			}
+		}
+	}
+	teardown()
+}
+
+func (data *MCTestData) deployACNPResourceExport(reFileName string) error {
+	var rc int
+	var err error
+	rc, _, _, err = provider.RunCommandOnNode(leaderCluster, fmt.Sprintf("kubectl apply -f %s", reFileName))
+	if err != nil || rc != 0 {
+		return fmt.Errorf("error when deploying the ACNP ResourceExport in leader cluster: %v", err)
+	}
+	return nil
+}
+
+func (data *MCTestData) deleteACNPResourceExport(reFileName string) error {
+	var rc int
+	var err error
+	rc, _, _, err = provider.RunCommandOnNode(leaderCluster, fmt.Sprintf("kubectl delete -f %s", reFileName))
+	if err != nil || rc != 0 {
+		return fmt.Errorf("error when deleting the ACNP ResourceExport in leader cluster: %v", err)
+	}
+	return nil
+}

multicluster/test/e2e/service_test.go

@@ -27,18 +27,6 @@ import (
 	e2euttils "antrea.io/antrea/test/e2e/utils"
 )
 
-func TestConnectivity(t *testing.T) {
-	data, err := setupTest(t)
-	if err != nil {
-		t.Fatalf("Error when setting up test: %v", err)
-	}
-	defer teardownTest(t, data)
-
-	t.Run("testServiceExport", func(t *testing.T) {
-		testServiceExport(t, data)
-	})
-}
-
 func testServiceExport(t *testing.T, data *MCTestData) {
 	data.testServiceExport(t)
 }

multicluster/test/yamls/test-acnp-copy-span-ns-isolation.yml

@@ -0,0 +1,24 @@
+apiVersion: multicluster.crd.antrea.io/v1alpha1
+kind: ResourceExport
+metadata:
+  name: strict-namespace-isolation-for-test-clusterset
+  namespace: antrea-mcs-ns
+spec:
+  kind: AntreaClusterNetworkPolicy
+  name: strict-namespace-isolation
+  clusternetworkpolicy:
+    priority: 1
+    tier: securityops
+    appliedTo:
+      - namespaceSelector: {}
+    ingress:
+      - action: Pass
+        from:
+          - namespaces:
+              match: Self
+          - podSelector:
+              matchLabels:
+                k8s-app: kube-dns
+      - action: Drop
+        from:
+          - namespaceSelector: {}

test/e2e/antreapolicy_test.go

@@ -104,24 +104,6 @@ func failOnError(err error, t *testing.T) {
 	}
 }
 
-// TestCase is a collection of TestSteps to be tested against.
-type TestCase struct {
-	Name  string
-	Steps []*TestStep
-}
-
-// TestStep is a single unit of testing spec. It includes the policy specs that need to be
-// applied for this test, the port to test traffic on and the expected Reachability matrix.
-type TestStep struct {
-	Name          string
-	Reachability  *Reachability
-	TestResources []metav1.Object
-	Ports         []int32
-	Protocol      v1.Protocol
-	Duration      time.Duration
-	CustomProbes  []*CustomProbe
-}
-
 // podToAddrTestStep is a single unit of testing the connectivity from a Pod to an
 // arbitrary destination address.
 type podToAddrTestStep struct {
@@ -131,19 +113,6 @@ type podToAddrTestStep struct {
 	expectedConnectivity PodConnectivityMark
 }
 
-// CustomProbe will spin up (or update) SourcePod and DestPod such that Add event of Pods
-// can be tested against expected connectivity among those Pods.
-type CustomProbe struct {
-	// Create or update a source Pod.
-	SourcePod CustomPod
-	// Create or update a destination Pod.
-	DestPod CustomPod
-	// Port on which the probe will be made.
-	Port int32
-	// Set the expected connectivity.
-	ExpectConnectivity PodConnectivityMark
-}
-
 func initialize(t *testing.T, data *TestData) {
 	p80 = 80
 	p81 = 81

test/e2e/k8s_util.go

@@ -47,6 +47,37 @@ func NewKubernetesUtils(data *TestData) (*KubernetesUtils, error) {
 	}, nil
 }
 
+// TestCase is a collection of TestSteps to be tested against.
+type TestCase struct {
+	Name  string
+	Steps []*TestStep
+}
+
+// TestStep is a single unit of testing spec. It includes the policy specs that need to be
+// applied for this test, the port to test traffic on and the expected Reachability matrix.
+type TestStep struct {
+	Name          string
+	Reachability  *Reachability
+	TestResources []metav1.Object
+	Ports         []int32
+	Protocol      v1.Protocol
+	Duration      time.Duration
+	CustomProbes  []*CustomProbe
+}
+
+// CustomProbe will spin up (or update) SourcePod and DestPod such that Add event of Pods
+// can be tested against expected connectivity among those Pods.
+type CustomProbe struct {
+	// Create or update a source Pod.
+	SourcePod CustomPod
+	// Create or update a destination Pod.
+	DestPod CustomPod
+	// Port on which the probe will be made.
+	Port int32
+	// Set the expected connectivity.
+	ExpectConnectivity PodConnectivityMark
+}
+
 // GetPodByLabel returns a Pod with the matching Namespace and "pod" label.
 func (k *KubernetesUtils) GetPodByLabel(ns string, name string) (*v1.Pod, error) {
 	pods, err := k.getPodsUncached(ns, "pod", name)