diff --git a/ci/jenkins/test-mc.sh b/ci/jenkins/test-mc.sh index 12181d0ce62..af4cf031042 100755 --- a/ci/jenkins/test-mc.sh +++ b/ci/jenkins/test-mc.sh @@ -236,11 +236,12 @@ function deliver_multicluster_controller { leader_ip=$(kubectl get nodes -o wide --no-headers=true ${LEADER_CLUSTER_CONFIG} | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 == role {print $6}') sed -i "s||${leader_ip}|" ./multicluster/test/yamls/east-member-cluster.yml sed -i "s||${leader_ip}|" ./multicluster/test/yamls/west-member-cluster.yml + rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" ./multicluster/test/yamls/test-acnp-copy-span-ns-isolation.yml jenkins@["${leader_ip}"]:"${WORKDIR}"/test-acnp-copy-span-ns-isolation.yml for kubeconfig in "${membercluter_kubeconfigs[@]}" do ip=$(kubectl get nodes -o wide --no-headers=true ${EAST_CLUSTER_CONFIG} | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 == role {print $6}') - rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" ./multicluster/test/yamls/test-east-serviceexport.yml jenkins@[${ip}]:${WORKDIR}/serviceexport.yml + rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" ./multicluster/test/yamls/test-east-serviceexport.yml jenkins@["${ip}"]:"${WORKDIR}"/serviceexport.yml done } diff --git a/multicluster/test/e2e/antreapolicy_test.go b/multicluster/test/e2e/antreapolicy_test.go new file mode 100644 index 00000000000..70d56ea5565 --- /dev/null +++ b/multicluster/test/e2e/antreapolicy_test.go @@ -0,0 +1,167 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "fmt" + "testing" + "time" + + log "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" + + antreae2e "antrea.io/antrea/test/e2e" +) + +const ( + // Provide enough time for policies to be enforced & deleted by the CNI plugin. + networkPolicyDelay = 2 * time.Second + acnpIsolationResourceExport = "test-acnp-copy-span-ns-isolation.yml" + acnpName = "antrea-mc-strict-namespace-isolation" +) + +var ( + allPodsPerCluster []antreae2e.Pod + perNamespacePods, perClusterNamespaces []string + podsByNamespace map[string][]antreae2e.Pod + clusterK8sUtilsMap map[string]*antreae2e.KubernetesUtils +) + +func failOnError(err error, t *testing.T) { + if err != nil { + log.Errorf("%+v", err) + for _, k8sUtils := range clusterK8sUtilsMap { + k8sUtils.Cleanup(perClusterNamespaces) + } + t.Fatalf("test failed: %v", err) + } +} + +// initializeForPolicyTest creates three Pods in three test Namespaces for each test cluster. +func initializeForPolicyTest(t *testing.T, data *MCTestData) { + perNamespacePods = []string{"a", "b", "c"} + perClusterNamespaces = []string{"x", "y", "z"} + + allPodsPerCluster = []antreae2e.Pod{} + podsByNamespace = make(map[string][]antreae2e.Pod) + clusterK8sUtilsMap = make(map[string]*antreae2e.KubernetesUtils) + + for _, podName := range perNamespacePods { + for _, ns := range perClusterNamespaces { + allPodsPerCluster = append(allPodsPerCluster, antreae2e.NewPod(ns, podName)) + podsByNamespace[ns] = append(podsByNamespace[ns], antreae2e.NewPod(ns, podName)) + } + } + for clusterName := range data.clusterTestDataMap { + d := data.clusterTestDataMap[clusterName] + k8sUtils, err := antreae2e.NewKubernetesUtils(&d) + failOnError(err, t) + _, err = k8sUtils.Bootstrap(perClusterNamespaces, perNamespacePods) + failOnError(err, t) + clusterK8sUtilsMap[clusterName] = k8sUtils + } +} + +// tearDownForPolicyTest deletes the test Namespaces specific for policy tests. +func tearDownForPolicyTest() { + for _, k8sUtils := range clusterK8sUtilsMap { + k8sUtils.Cleanup(perClusterNamespaces) + } +} + +func testMCAntreaPolicy(t *testing.T, data *MCTestData) { + data.testAntreaPolicyCopySpanNSIsolation(t) +} + +// testAntreaPolicyCopySpanNSIsolation tests that after applying a ResourceExport of an ACNP +// for Namespace isolation, strict Namespace isolation is enforced in each of the member clusters. +func (data *MCTestData) testAntreaPolicyCopySpanNSIsolation(t *testing.T) { + setup := func() { + err := data.deployACNPResourceExport(acnpIsolationResourceExport) + failOnError(err, t) + } + teardown := func() { + err := data.deleteACNPResourceExport(acnpIsolationResourceExport) + failOnError(err, t) + } + reachability := antreae2e.NewReachability(allPodsPerCluster, antreae2e.Dropped) + reachability.ExpectAllSelfNamespace(antreae2e.Connected) + testStep := &antreae2e.TestStep{ + Name: "Port 80", + Reachability: reachability, + Ports: []int32{80}, + Protocol: v1.ProtocolTCP, + } + testCaseList := []*antreae2e.TestCase{ + { + Name: "ACNP strict Namespace isolation for all clusters", + Steps: []*antreae2e.TestStep{testStep}, + }, + } + executeTestsOnAllMemberClusters(t, testCaseList, setup, teardown) +} + +func executeTestsOnAllMemberClusters(t *testing.T, testList []*antreae2e.TestCase, setup, teardown func()) { + setup() + time.Sleep(networkPolicyDelay) + for _, testCase := range testList { + log.Infof("Running test case %s", testCase.Name) + for _, step := range testCase.Steps { + log.Infof("Running step %s of test case %s", step.Name, testCase.Name) + reachability := step.Reachability + if reachability != nil { + for clusterName, k8sUtils := range clusterK8sUtilsMap { + if clusterName == leaderCluster { + // skip traffic test for the leader cluster + continue + } + if _, err := k8sUtils.GetACNP(acnpName); err != nil { + t.Errorf("Failed to get ACNP to be replicated in cluster %s", clusterName) + } + start := time.Now() + k8sUtils.Validate(allPodsPerCluster, reachability, step.Ports, step.Protocol) + step.Duration = time.Now().Sub(start) + _, wrong, _ := step.Reachability.Summary() + if wrong != 0 { + t.Errorf("Failure in cluster %s -- %d wrong results", clusterName, wrong) + reachability.PrintSummary(true, true, true) + } + } + } + } + } + teardown() +} + +func (data *MCTestData) deployACNPResourceExport(reFileName string) error { + var rc int + var err error + log.Infof("Creating ResourceExport %s in the leader cluster", reFileName) + rc, _, _, err = provider.RunCommandOnNode(leaderCluster, fmt.Sprintf("kubectl apply -f %s", reFileName)) + if err != nil || rc != 0 { + return fmt.Errorf("error when deploying the ACNP ResourceExport in leader cluster: %v", err) + } + return nil +} + +func (data *MCTestData) deleteACNPResourceExport(reFileName string) error { + var rc int + var err error + rc, _, _, err = provider.RunCommandOnNode(leaderCluster, fmt.Sprintf("kubectl delete -f %s", reFileName)) + if err != nil || rc != 0 { + return fmt.Errorf("error when deleting the ACNP ResourceExport in leader cluster: %v", err) + } + return nil +} diff --git a/multicluster/test/e2e/main_test.go b/multicluster/test/e2e/main_test.go index 2d07fd90dda..de51f8bf44f 100644 --- a/multicluster/test/e2e/main_test.go +++ b/multicluster/test/e2e/main_test.go @@ -88,3 +88,20 @@ func testMain(m *testing.M) int { func TestMain(m *testing.M) { os.Exit(testMain(m)) } + +func TestConnectivity(t *testing.T) { + data, err := setupTest(t) + if err != nil { + t.Fatalf("Error when setting up test: %v", err) + } + defer teardownTest(t, data) + + t.Run("testServiceExport", func(t *testing.T) { + testServiceExport(t, data) + }) + t.Run("testAntreaPolicy", func(t *testing.T) { + initializeForPolicyTest(t, data) + testMCAntreaPolicy(t, data) + tearDownForPolicyTest() + }) +} diff --git a/multicluster/test/e2e/service_test.go b/multicluster/test/e2e/service_test.go index 3304ff7480a..d58c1e9ad8b 100644 --- a/multicluster/test/e2e/service_test.go +++ b/multicluster/test/e2e/service_test.go @@ -27,18 +27,6 @@ import ( e2euttils "antrea.io/antrea/test/e2e/utils" ) -func TestConnectivity(t *testing.T) { - data, err := setupTest(t) - if err != nil { - t.Fatalf("Error when setting up test: %v", err) - } - defer teardownTest(t, data) - - t.Run("testServiceExport", func(t *testing.T) { - testServiceExport(t, data) - }) -} - func testServiceExport(t *testing.T, data *MCTestData) { data.testServiceExport(t) } diff --git a/multicluster/test/yamls/test-acnp-copy-span-ns-isolation.yml b/multicluster/test/yamls/test-acnp-copy-span-ns-isolation.yml new file mode 100644 index 00000000000..81f6811db35 --- /dev/null +++ b/multicluster/test/yamls/test-acnp-copy-span-ns-isolation.yml @@ -0,0 +1,23 @@ +apiVersion: multicluster.crd.antrea.io/v1alpha1 +kind: ResourceExport +metadata: + name: strict-namespace-isolation-for-test-clusterset + namespace: antrea-mcs-ns +spec: + kind: AntreaClusterNetworkPolicy + name: strict-namespace-isolation + clusternetworkpolicy: + priority: 1 + tier: securityops + appliedTo: + - namespaceSelector: # Selects all non-system Namespaces in the cluster + matchExpressions: + - {key: kubernetes.io/metadata.name, operator: NotIn, values: [kube-system]} + ingress: + - action: Pass + from: + - namespaces: + match: Self + - action: Drop + from: + - namespaceSelector: {} diff --git a/test/e2e/antreapolicy_test.go b/test/e2e/antreapolicy_test.go index 45a9ca50190..6f94a78b841 100644 --- a/test/e2e/antreapolicy_test.go +++ b/test/e2e/antreapolicy_test.go @@ -104,24 +104,6 @@ func failOnError(err error, t *testing.T) { } } -// TestCase is a collection of TestSteps to be tested against. -type TestCase struct { - Name string - Steps []*TestStep -} - -// TestStep is a single unit of testing spec. It includes the policy specs that need to be -// applied for this test, the port to test traffic on and the expected Reachability matrix. -type TestStep struct { - Name string - Reachability *Reachability - TestResources []metav1.Object - Ports []int32 - Protocol v1.Protocol - Duration time.Duration - CustomProbes []*CustomProbe -} - // podToAddrTestStep is a single unit of testing the connectivity from a Pod to an // arbitrary destination address. type podToAddrTestStep struct { @@ -131,19 +113,6 @@ type podToAddrTestStep struct { expectedConnectivity PodConnectivityMark } -// CustomProbe will spin up (or update) SourcePod and DestPod such that Add event of Pods -// can be tested against expected connectivity among those Pods. -type CustomProbe struct { - // Create or update a source Pod. - SourcePod CustomPod - // Create or update a destination Pod. - DestPod CustomPod - // Port on which the probe will be made. - Port int32 - // Set the expected connectivity. - ExpectConnectivity PodConnectivityMark -} - func initialize(t *testing.T, data *TestData) { p80 = 80 p81 = 81 diff --git a/test/e2e/k8s_util.go b/test/e2e/k8s_util.go index eb5fa6cc9a7..99afdcd60d6 100644 --- a/test/e2e/k8s_util.go +++ b/test/e2e/k8s_util.go @@ -47,6 +47,37 @@ func NewKubernetesUtils(data *TestData) (*KubernetesUtils, error) { }, nil } +// TestCase is a collection of TestSteps to be tested against. +type TestCase struct { + Name string + Steps []*TestStep +} + +// TestStep is a single unit of testing spec. It includes the policy specs that need to be +// applied for this test, the port to test traffic on and the expected Reachability matrix. +type TestStep struct { + Name string + Reachability *Reachability + TestResources []metav1.Object + Ports []int32 + Protocol v1.Protocol + Duration time.Duration + CustomProbes []*CustomProbe +} + +// CustomProbe will spin up (or update) SourcePod and DestPod such that Add event of Pods +// can be tested against expected connectivity among those Pods. +type CustomProbe struct { + // Create or update a source Pod. + SourcePod CustomPod + // Create or update a destination Pod. + DestPod CustomPod + // Port on which the probe will be made. + Port int32 + // Set the expected connectivity. + ExpectConnectivity PodConnectivityMark +} + // GetPodByLabel returns a Pod with the matching Namespace and "pod" label. func (k *KubernetesUtils) GetPodByLabel(ns string, name string) (*v1.Pod, error) { pods, err := k.getPodsUncached(ns, "pod", name)