From f77ebbd87844705357d2a1956f0e545383177301 Mon Sep 17 00:00:00 2001 From: Lan Luo Date: Thu, 9 Jun 2022 10:53:44 +0800 Subject: [PATCH] Use Service ClusterIPs as MC Service's Endpoints 1. Use Service ClusterIPs instead of Pod IPs as MC Service's Endpoints. The ServiceExport controller will only watch ServiceExport and Service events, and wrap Service's ClusterIPs into a new Endpoint kind of ResourceExport. 2. Includes local Serivce ClusterIP as multi-cluster Service's Endpoints as well. Signed-off-by: Lan Luo --- build/charts/antrea/README.md | 2 +- build/charts/antrea/values.yaml | 2 +- ci/jenkins/test-mc.sh | 52 ++++++-- .../controllers/multicluster/common/helper.go | 26 ++++ .../commonarea/resourceimport_controller.go | 41 +----- .../resourceimport_controller_test.go | 26 ++-- .../multicluster/serviceexport_controller.go | 125 +++--------------- .../serviceexport_controller_test.go | 41 +----- multicluster/hack/mc-integration-test.sh | 6 +- multicluster/test/e2e/service_test.go | 6 +- multicluster/test/integration/README.md | 2 +- .../resourceexport_controller_test.go | 2 +- .../serviceexport_controller_test.go | 85 +++--------- multicluster/test/integration/test_data.go | 4 - 14 files changed, 136 insertions(+), 284 deletions(-) diff --git a/build/charts/antrea/README.md b/build/charts/antrea/README.md index c6e9b828cbc..2124c8b075f 100644 --- a/build/charts/antrea/README.md +++ b/build/charts/antrea/README.md @@ -83,7 +83,7 @@ Kubernetes: `>= 1.16.0-0` | multicast.igmpQueryInterval | string | `"125s"` | The interval at which the antrea-agent sends IGMP queries to Pods. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". | | multicast.multicastInterfaces | list | `[]` | Names of the interfaces on Nodes that are used to forward multicast traffic. | | multicluster.enable | bool | `false` | Enable Antrea Multi-cluster Gateway to support cross-cluster traffic. This feature is supported only with encap mode. | -| multicluster.namespace | string | `""` | The Namespace where Antrea Multi-cluster Controller is running. The default is Antrea Agent Namespace if it's empty. | +| multicluster.namespace | string | `""` | The Namespace where Antrea Multi-cluster Controller is running. The default is antrea-agent's Namespace. | | noSNAT | bool | `false` | Whether or not to SNAT (using the Node IP) the egress traffic from a Pod to the external network. | | nodeIPAM.clusterCIDRs | list | `[]` | CIDR ranges to use when allocating Pod IP addresses. | | nodeIPAM.enable | bool | `false` | Enable Node IPAM in Antrea | diff --git a/build/charts/antrea/values.yaml b/build/charts/antrea/values.yaml index 759b8b2232c..32bd2739fae 100644 --- a/build/charts/antrea/values.yaml +++ b/build/charts/antrea/values.yaml @@ -287,7 +287,7 @@ multicluster: # This feature is supported only with encap mode. enable: false # -- The Namespace where Antrea Multi-cluster Controller is running. - # The default is Antrea Agent Namespace if it's empty. + # The default is antrea-agent's Namespace. namespace: "" testing: diff --git a/ci/jenkins/test-mc.sh b/ci/jenkins/test-mc.sh index 3e209e5f5bd..21e34824b02 100755 --- a/ci/jenkins/test-mc.sh +++ b/ci/jenkins/test-mc.sh @@ -33,8 +33,7 @@ LEADER_CLUSTER_CONFIG="--kubeconfig=$MULTICLUSTER_KUBECONFIG_PATH/leader" EAST_CLUSTER_CONFIG="--kubeconfig=$MULTICLUSTER_KUBECONFIG_PATH/east" WEST_CLUSTER_CONFIG="--kubeconfig=$MULTICLUSTER_KUBECONFIG_PATH/west" ENABLE_MC_GATEWAY=false - -CONTROL_PLANE_NODE_ROLE="control-plane,master" +IS_CONTAINERD=false multicluster_kubeconfigs=($EAST_CLUSTER_CONFIG $LEADER_CLUSTER_CONFIG $WEST_CLUSTER_CONFIG) membercluster_kubeconfigs=($EAST_CLUSTER_CONFIG $WEST_CLUSTER_CONFIG) @@ -160,6 +159,7 @@ function wait_for_antrea_multicluster_pods_ready { } function wait_for_multicluster_controller_ready { + echo "====== Deploying Antrea Multicluster Leader Cluster with ${LEADER_CLUSTER_CONFIG} ======" kubectl create ns antrea-mcs-ns "${LEADER_CLUSTER_CONFIG}" || true kubectl apply -f ./multicluster/test/yamls/manifest.yml "${LEADER_CLUSTER_CONFIG}" kubectl apply -f ./multicluster/build/yamls/antrea-multicluster-leader-global.yml "${LEADER_CLUSTER_CONFIG}" @@ -175,15 +175,17 @@ function wait_for_multicluster_controller_ready { sed -i '/creationTimestamp/d' ./multicluster/test/yamls/leader-access-token.yml sed -i 's/antrea-multicluster-member-access-sa/antrea-multicluster-controller/g' ./multicluster/test/yamls/leader-access-token.yml sed -i 's/antrea-mcs-ns/kube-system/g' ./multicluster/test/yamls/leader-access-token.yml - echo "type: Opaque" >>./multicluster/test/yamls/leader-access-token.yml + echo "type: Opaque" >> ./multicluster/test/yamls/leader-access-token.yml for config in "${membercluster_kubeconfigs[@]}"; do + echo "====== Deploying Antrea Multicluster Member Cluster with ${config} ======" kubectl apply -f ./multicluster/build/yamls/antrea-multicluster-member.yml ${config} kubectl rollout status deployment/antrea-mc-controller -n kube-system ${config} kubectl apply -f ./multicluster/test/yamls/leader-access-token.yml ${config} done + echo "====== ClusterSet Initialization in Leader and Member Clusters ======" kubectl apply -f ./multicluster/test/yamls/east-member-cluster.yml "${EAST_CLUSTER_CONFIG}" kubectl apply -f ./multicluster/test/yamls/west-member-cluster.yml "${WEST_CLUSTER_CONFIG}" kubectl apply -f ./multicluster/test/yamls/clusterset.yml "${LEADER_CLUSTER_CONFIG}" @@ -213,7 +215,11 @@ function deliver_antrea_multicluster { do kubectl get nodes -o wide --no-headers=true ${kubeconfig}| awk '{print $6}' | while read IP; do rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" "${WORKDIR}"/antrea-ubuntu.tar jenkins@[${IP}]:${WORKDIR}/antrea-ubuntu.tar - ssh -o StrictHostKeyChecking=no -n jenkins@${IP} "${CLEAN_STALE_IMAGES}; docker load -i ${WORKDIR}/antrea-ubuntu.tar" || true + if [[ ${IS_CONTAINERD} ]];then + ssh -o StrictHostKeyChecking=no -n jenkins@${IP} "${CLEAN_STALE_IMAGES}; sudo ctr -n=k8s.io images import ${WORKDIR}/antrea-ubuntu.tar" || true + else + ssh -o StrictHostKeyChecking=no -n jenkins@${IP} "${CLEAN_STALE_IMAGES}; docker load -i ${WORKDIR}/antrea-ubuntu.tar" || true + fi done done } @@ -232,13 +238,17 @@ function deliver_multicluster_controller { for kubeconfig in "${multicluster_kubeconfigs[@]}" do - kubectl get nodes -o wide --no-headers=true "${kubeconfig}"| awk '{print $6}' | while read IP; do + kubectl get nodes -o wide --no-headers=true "${kubeconfig}" | awk '{print $6}' | while read IP; do rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" "${WORKDIR}"/antrea-mcs.tar jenkins@[${IP}]:${WORKDIR}/antrea-mcs.tar - ssh -o StrictHostKeyChecking=no -n jenkins@"${IP}" "${CLEAN_STALE_IMAGES}; docker load -i ${WORKDIR}/antrea-mcs.tar" || true + if [[ ${IS_CONTAINERD} ]];then + ssh -o StrictHostKeyChecking=no -n jenkins@"${IP}" "${CLEAN_STALE_IMAGES}; sudo ctr -n=k8s.io images import ${WORKDIR}/antrea-mcs.tar" || true + else + ssh -o StrictHostKeyChecking=no -n jenkins@"${IP}" "${CLEAN_STALE_IMAGES}; docker load -i ${WORKDIR}/antrea-mcs.tar" || true + fi done done - leader_ip=$(kubectl get nodes -o wide --no-headers=true ${LEADER_CLUSTER_CONFIG} | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 == role {print $6}') + leader_ip=$(kubectl get nodes -o wide --no-headers=true ${LEADER_CLUSTER_CONFIG} | awk -v role1="master" -v role2="control-plane" '($3 ~ role1 || $3 ~ role2) {print $6}') sed -i "s||${leader_ip}|" ./multicluster/test/yamls/east-member-cluster.yml sed -i "s||${leader_ip}|" ./multicluster/test/yamls/west-member-cluster.yml rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" ./multicluster/test/yamls/test-acnp-copy-span-ns-isolation.yml jenkins@["${leader_ip}"]:"${WORKDIR}"/test-acnp-copy-span-ns-isolation.yml @@ -248,7 +258,7 @@ function deliver_multicluster_controller { # Remove the longest matched substring '*/' from a string like '--kubeconfig=/var/lib/jenkins/.kube/east' # to get the last element which is the cluster name. cluster=${kubeconfig##*/} - ip=$(kubectl get nodes -o wide --no-headers=true ${kubeconfig} | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 == role {print $6}') + ip=$(kubectl get nodes -o wide --no-headers=true ${kubeconfig} | awk -v role1="master" -v role2="control-plane" '($3 ~ role1 || $3 ~ role2) {print $6}') rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" ./multicluster/test/yamls/test-${cluster}-serviceexport.yml jenkins@["${ip}"]:"${WORKDIR}"/serviceexport.yml done } @@ -262,7 +272,14 @@ function run_multicluster_e2e { export PATH=$GOROOT/bin:$PATH if [[ ${ENABLE_MC_GATEWAY} ]]; then - sed -i.bak -E "s/#[[:space:]]*Multicluster[[:space:]]*:[[:space:]]*[a-z]+[[:space:]]*$/ Multicluster: true/" build/yamls/antrea.yml + cat > build/yamls/chart-values/antrea.yml < /tmp/mc-integration-kubeconfig -kind create cluster --name=antrea-integration-kind --kubeconfig=/tmp/mc-integration-kubeconfig +kind create cluster --name=antrea-integration --kubeconfig=/tmp/mc-integration-kubeconfig sleep 5 export KUBECONFIG=/tmp/mc-integration-kubeconfig kubectl create namespace leader-ns @@ -38,7 +38,7 @@ kubectl apply -f test/integration/cluster-admin.yml if [[ $NO_LOCAL == "true" ]];then # Run go test in a Docker container - container_ip=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' antrea-integration-kind-control-plane) + container_ip=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' antrea-integration-control-plane) sed -i "s|server: https://.*|server: https://${container_ip}:6443|" /tmp/mc-integration-kubeconfig docker run --network kind --privileged --rm \ -w /usr/src/antrea.io/antrea \ diff --git a/multicluster/test/e2e/service_test.go b/multicluster/test/e2e/service_test.go index 1a90fd49d95..578075ef894 100644 --- a/multicluster/test/e2e/service_test.go +++ b/multicluster/test/e2e/service_test.go @@ -186,12 +186,12 @@ func (data *MCTestData) deleteServiceExport(clusterName string) error { // getNodeNamesFromCluster will pick up a Node randomly as the Gateway // and also a regular Node from the specified cluster. -func getNodeNamesFromCluster(nodeName string) (string, string, error) { - rc, output, stderr, err := provider.RunCommandOnNode(nodeName, "kubectl get node -o custom-columns=:metadata.name --no-headers") +func getNodeNamesFromCluster(clusterName string) (string, string, error) { + rc, output, stderr, err := provider.RunCommandOnNode(clusterName, "kubectl get node -o jsonpath='{range .items[*]}{.metadata.name}{\" \"}{end}'") if err != nil || rc != 0 || stderr != "" { return "", "", fmt.Errorf("error when getting Node list: %v, stderr: %s", err, stderr) } - nodes := strings.Split(output, "\n") + nodes := strings.Split(strings.TrimRight(output, " "), " ") gwIdx := rand.Intn(len(nodes)) // #nosec G404: for test only var regularNode string for i, node := range nodes { diff --git a/multicluster/test/integration/README.md b/multicluster/test/integration/README.md index 42b1b77e267..6fb9da365b5 100644 --- a/multicluster/test/integration/README.md +++ b/multicluster/test/integration/README.md @@ -12,7 +12,7 @@ like Service, Endpoints controllers etc. The tests must be run on an real Kubernetes cluster. At the moment, you can simply run `make test-integration` in `antrea/multicluster` folder. It will -create a Kind cluster named `antrea-integration-kind` and execute the integration +create a Kind cluster named `antrea-integration` and execute the integration codes. if you'd like to run the integration test in an existing Kubernetes cluster, you diff --git a/multicluster/test/integration/resourceexport_controller_test.go b/multicluster/test/integration/resourceexport_controller_test.go index 6cb8bcab0dd..8c0f6748743 100644 --- a/multicluster/test/integration/resourceexport_controller_test.go +++ b/multicluster/test/integration/resourceexport_controller_test.go @@ -163,8 +163,8 @@ var _ = Describe("ResourceExport controller", func() { Subsets: []corev1.EndpointSubset{ { Addresses: []corev1.EndpointAddress{ + addr2, addr3, - addr4, }, Ports: epPorts, }, diff --git a/multicluster/test/integration/serviceexport_controller_test.go b/multicluster/test/integration/serviceexport_controller_test.go index 439a2c9e6c2..d01dbc7ced2 100644 --- a/multicluster/test/integration/serviceexport_controller_test.go +++ b/multicluster/test/integration/serviceexport_controller_test.go @@ -53,25 +53,7 @@ var _ = Describe("ServiceExport controller", func() { Namespace: svc.Namespace, Name: svc.Name, } - epNamespacedName := svcNamespacedName - ep := &corev1.Endpoints{ - ObjectMeta: metav1.ObjectMeta{ - Name: "nginx-svc", - Namespace: testNamespace, - }, - Subsets: []corev1.EndpointSubset{ - { - Addresses: []corev1.EndpointAddress{ - addr1, - }, - NotReadyAddresses: []corev1.EndpointAddress{ - addr2, - }, - Ports: epPorts, - }, - }, - } svcExport := &k8smcsv1alpha1.ServiceExport{ ObjectMeta: metav1.ObjectMeta{ Name: "nginx-svc", @@ -85,7 +67,7 @@ var _ = Describe("ServiceExport controller", func() { }, } svcResExportName := LocalClusterID + "-" + svc.Namespace + "-" + svc.Name + "-service" - epResExportName := LocalClusterID + "-" + ep.Namespace + "-" + ep.Name + "-endpoints" + epResExportName := LocalClusterID + "-" + svc.Namespace + "-" + svc.Name + "-endpoints" expectedEpResExport := &mcsv1alpha1.ResourceExport{ ObjectMeta: metav1.ObjectMeta{ @@ -94,8 +76,8 @@ var _ = Describe("ServiceExport controller", func() { }, Spec: mcsv1alpha1.ResourceExportSpec{ ClusterID: LocalClusterID, - Name: ep.Name, - Namespace: ep.Namespace, + Name: svc.Name, + Namespace: svc.Namespace, Kind: common.EndpointsKind, }, } @@ -103,20 +85,7 @@ var _ = Describe("ServiceExport controller", func() { ctx := context.Background() It("Should create ResourceExports when new ServiceExport for ClusterIP Service is created", func() { By("By exposing a ClusterIP type of Service") - expectedEpResExport.Spec.Endpoints = &mcsv1alpha1.EndpointsExport{ - Subsets: []corev1.EndpointSubset{ - { - Addresses: []corev1.EndpointAddress{ - { - IP: "192.168.17.11", - }, - }, - Ports: epPorts, - }, - }, - } Expect(k8sClient.Create(ctx, svc)).Should(Succeed()) - Expect(k8sClient.Create(ctx, ep)).Should(Succeed()) Expect(k8sClient.Create(ctx, svcExport)).Should(Succeed()) var err error latestSvc := &corev1.Service{} @@ -131,6 +100,18 @@ var _ = Describe("ServiceExport controller", func() { Expect(svcResExport.ObjectMeta.Labels["sourceKind"]).Should(Equal("Service")) Expect(svcResExport.Spec.Service.ServiceSpec.ClusterIP).Should(Equal(latestSvc.Spec.ClusterIP)) Expect(len(svcResExport.Spec.Service.ServiceSpec.Ports)).Should(Equal(len(svcPorts))) + expectedEpResExport.Spec.Endpoints = &mcsv1alpha1.EndpointsExport{ + Subsets: []corev1.EndpointSubset{ + { + Addresses: []corev1.EndpointAddress{ + { + IP: latestSvc.Spec.ClusterIP, + }, + }, + Ports: epPorts, + }, + }, + } Eventually(func() bool { err = k8sClient.Get(ctx, types.NamespacedName{Namespace: LeaderNamespace, Name: epResExportName}, epResExport) return err == nil @@ -180,42 +161,6 @@ var _ = Describe("ServiceExport controller", func() { Expect(*conditions[0].Message).Should(Equal("the Service does not exist")) }) - It("Should update existing ResourceExport when corresponding Endpoints has new Endpoint", func() { - By("By update an Endpoint with a new address") - latestEp := &corev1.Endpoints{} - Expect(k8sClient.Get(ctx, epNamespacedName, latestEp)).Should(Succeed()) - addresses := latestEp.Subsets[0].Addresses - addresses = append(addresses, addr3) - latestEp.Subsets[0].Addresses = addresses - Expect(k8sClient.Update(ctx, latestEp)).Should(Succeed()) - time.Sleep(2 * time.Second) - epResExport := &mcsv1alpha1.ResourceExport{} - expectedEpResExport.Spec.Endpoints = &mcsv1alpha1.EndpointsExport{ - Subsets: []corev1.EndpointSubset{ - { - Addresses: []corev1.EndpointAddress{ - { - IP: "192.168.17.11", - }, - { - IP: "192.168.17.13", - }, - }, - Ports: epPorts, - }, - }, - } - - var err error - Eventually(func() bool { - err = k8sClient.Get(ctx, types.NamespacedName{Namespace: LeaderNamespace, Name: epResExportName}, epResExport) - return err == nil - }, timeout, interval).Should(BeTrue()) - Expect(epResExport.ObjectMeta.Labels["sourceKind"]).Should(Equal("Endpoints")) - Expect(epResExport.Spec).Should(Equal(expectedEpResExport.Spec)) - - }) - It("Should delete existing ResourceExport when existing ServiceExport is deleted", func() { By("By remove a ServiceExport resource") err := k8sClient.Delete(ctx, svcExport) diff --git a/multicluster/test/integration/test_data.go b/multicluster/test/integration/test_data.go index c2b6c59cd8f..e5abc57ddd1 100644 --- a/multicluster/test/integration/test_data.go +++ b/multicluster/test/integration/test_data.go @@ -29,10 +29,6 @@ var ( IP: "192.168.17.13", Hostname: "pod3", } - addr4 = corev1.EndpointAddress{ - IP: "192.168.17.14", - Hostname: "pod4", - } epPorts = []corev1.EndpointPort{ { Name: "http",