diff --git a/build/charts/antrea/templates/agent/daemonset.yaml b/build/charts/antrea/templates/agent/daemonset.yaml index 8f766eeaa6d..14f7d30218d 100644 --- a/build/charts/antrea/templates/agent/daemonset.yaml +++ b/build/charts/antrea/templates/agent/daemonset.yaml @@ -59,6 +59,16 @@ spec: operator: NotIn values: - simulator + {{- else if .Values.testing.noAntreaNodeWithLabels.enable }} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + {{- range .Values.testing.noAntreaNodeWithLabels.keys }} + - key: {{ . }} + operator: DoesNotExist + {{- end }} {{- else }} {{- with .Values.agent.affinity }} affinity: diff --git a/build/charts/antrea/templates/controller/deployment.yaml b/build/charts/antrea/templates/controller/deployment.yaml index 7bf34433221..a3964882268 100644 --- a/build/charts/antrea/templates/controller/deployment.yaml +++ b/build/charts/antrea/templates/controller/deployment.yaml @@ -51,6 +51,16 @@ spec: operator: NotIn values: - simulator + {{- else if .Values.testing.noAntreaNodeWithLabels.enable }} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + {{- range .Values.testing.noAntreaNodeWithLabels.keys }} + - key: {{ . }} + operator: DoesNotExist + {{- end }} {{- else }} {{- with .Values.controller.affinity }} affinity: diff --git a/build/charts/antrea/values.yaml b/build/charts/antrea/values.yaml index ce7923d197b..4cc8108ab93 100644 --- a/build/charts/antrea/values.yaml +++ b/build/charts/antrea/values.yaml @@ -439,3 +439,9 @@ testing: coverage: false simulator: enable: false + # -- Don't deploy Antrea on the Nodes labelled with the specific keys. + noAntreaNodeWithLabels: + enable: false + keys: + - no-antrea + diff --git a/ci/kind/kind-setup.sh b/ci/kind/kind-setup.sh index 26a8fae1a1f..5750ada44d6 100755 --- a/ci/kind/kind-setup.sh +++ b/ci/kind/kind-setup.sh @@ -39,6 +39,7 @@ PROMETHEUS=false K8S_VERSION="" KUBE_NODE_IPAM=true DEPLOY_EXTERNAL_SERVER=false +DEPLOY_NO_ANTREA_WORKER=false positional_args=() options=() @@ -50,37 +51,38 @@ function echoerr { } _usage=" -Usage: $0 create CLUSTER_NAME [--pod-cidr POD_CIDR] [--service-cidr SERVICE_CIDR] [--antrea-cni] [--num-workers NUM_WORKERS] [--images IMAGES] [--subnets SUBNETS] [--ip-family ipv4|ipv6|dual] [--k8s-version VERSION] +Usage: $0 create CLUSTER_NAME [--pod-cidr POD_CIDR] [--service-cidr SERVICE_CIDR] [--antrea-cni] [--num-workers NUM_WORKERS] [--images IMAGES] [--subnets SUBNETS] [--ip-family ipv4|ipv6|dual] [--k8s-version VERSION] $0 destroy CLUSTER_NAME $0 help where: - create: create a kind cluster with name CLUSTER_NAME - destroy: delete a kind cluster with name CLUSTER_NAME - --pod-cidr: specifies pod cidr used in kind cluster, kind's default value will be used if empty. - --service-cidr: specifies service clusterip cidr used in kind cluster, kind's default value will be used if empty. - --encap-mode: inter-node pod traffic encap mode, default is encap - --no-proxy: disable Antrea proxy - --no-kube-proxy: disable Kube proxy - --no-kube-node-ipam: disable NodeIPAM in kube-controller-manager + create: create a kind cluster with name CLUSTER_NAME. + destroy: delete a kind cluster with name CLUSTER_NAME. + --pod-cidr: specify pod cidr used in kind cluster, kind's default value will be used if empty. + --service-cidr: specify service clusterip cidr used in kind cluster, kind's default value will be used if empty. + --encap-mode: inter-node pod traffic encap mode, default is encap. + --no-proxy: disable Antrea proxy. + --no-kube-proxy: disable Kube proxy. + --no-kube-node-ipam: disable NodeIPAM in kube-controller-manager. --antrea-cni: install Antrea CNI in Kind cluster; by default the cluster is created without a CNI installed. - --prometheus: create RBAC resources for Prometheus, default is false - --num-workers: specifies number of worker nodes in kind cluster, default is $NUM_WORKERS - --images: specifies images loaded to kind cluster, default is $IMAGES + --prometheus: create RBAC resources for Prometheus, default is false. + --num-workers: specify number of worker nodes in kind cluster, default is $NUM_WORKERS. + --images: specify images loaded to kind cluster, default is $IMAGES. --subnets: a subnet creates a separate Docker bridge network (named 'antrea-') with the assigned subnet. A worker Node will be connected to one of those network. Default is empty: all worker Nodes connected to the default Docker bridge network created by kind. - --vlan-subnets: specifies the subnets of the VLAN to which all Nodes will be connected, in addition to the primary network. + --vlan-subnets: specify the subnets of the VLAN to which all Nodes will be connected, in addition to the primary network. The IP expression of the subnet will be used as the gateway IP. For example, '--vlan-subnets 10.100.100.1/24' means that a VLAN sub-interface will be created on the primary Docker bridge, and it will be assigned the 10.100.100.1/24 address. - --vlan-id: specifies the ID of the VLAN to which all Nodes will be connected, in addition to the primary network. Note, + --vlan-id: specify the ID of the VLAN to which all Nodes will be connected, in addition to the primary network. Note, '--vlan-subnets' and '--vlan-id' must be specified together. --extra-networks: an extra network creates a separate Docker bridge network (named 'antrea-') with the assigned subnet. All worker Nodes will be connected to all the extra networks, in addition to the default Docker bridge network. Note, '--extra-networks' and '--subnets' cannot be specified together. - --ip-family: specifies the ip-family for the kind cluster, default is $IP_FAMILY. - --k8s-version: specifies the Kubernetes version of the kind cluster, kind's default K8s version will be used if empty. + --ip-family: specify the ip-family for the kind cluster, default is $IP_FAMILY. + --k8s-version: specify the Kubernetes version of the kind cluster, kind's default K8s version will be used if empty. --deploy-external-server: deploy a container running as an external server for the cluster. - --all: delete all kind clusters + --deploy-no-antrea-worker: deploy an extra worker node in kind cluster on which Antrea will not be deployed, default is $DEPLOY_NO_ANTREA_WORKER. + --all: delete all kind clusters. --until: delete kind clusters that have been created before the specified duration. " @@ -374,6 +376,10 @@ EOF echo -e "- role: worker" >> $config_file done + if [[ $DEPLOY_NO_ANTREA_WORKER == true ]]; then + echo -e "- role: worker" >> $config_file + fi + # When only the control plane Node is provisioned (no worker Node), # we configure port mappings so that the Antrea Agent and Controller # running on the control plane Node can be easily accessed, including on macOS. @@ -412,6 +418,7 @@ EOF configure_extra_networks configure_vlan_subnets setup_external_server + label_no_antrea_worker load_images if [[ $ANTREA_CNI == true ]]; then @@ -484,6 +491,11 @@ function clean_kind { done } +function label_no_antrea_worker { + no_antrea_node="$(kind get nodes --name $CLUSTER_NAME | grep worker | sort | tail -n1)" + kubectl label node $no_antrea_node no-antrea="" +} + if ! command -v kind &> /dev/null then echoerr "kind could not be found" @@ -574,7 +586,7 @@ while [[ $# -gt 0 ]] shift ;; --num-workers) - add_option "--num-workers" "create" + add_option "--num-workers" "create" NUM_WORKERS="$2" shift 2 ;; @@ -588,6 +600,11 @@ while [[ $# -gt 0 ]] DEPLOY_EXTERNAL_SERVER=true shift ;; + --deploy-no-antrea-worker) + add_option "--deploy-no-antrea-worker" "create" + DEPLOY_NO_ANTREA_WORKER=true + shift + ;; --all) add_option "--all" "destroy" CLUSTER_NAME="*" diff --git a/ci/kind/test-e2e-kind.sh b/ci/kind/test-e2e-kind.sh index 0585116186f..190216a8686 100755 --- a/ci/kind/test-e2e-kind.sh +++ b/ci/kind/test-e2e-kind.sh @@ -35,7 +35,7 @@ _usage="Usage: $0 [--encap-mode ] [--ip-family ] [--coverage] --networkpolicy-evaluation Configures additional NetworkPolicy evaluation level when running e2e tests. --extra-network Creates an extra network that worker Nodes will connect to. Cannot be specified with the hybrid mode. --extra-vlan Creates an subnet-based VLAN that worker Nodes will connect to. - --deploy-external-server Deploy a container running as an external server for the cluster. + --deploy-no-antrea-worker Deploy a worker Node on which Antrea will not be deployed for the cluster. --skip A comma-separated list of keywords, with which tests should be skipped. --coverage Enables measure Antrea code coverage when running e2e tests on kind. --setup-only Only perform setting up the cluster and run test. @@ -80,7 +80,7 @@ flow_visibility=false np_evaluation=false extra_network=false extra_vlan=false -deploy_external_server=false +deploy_no_antrea_worker=false coverage=false skiplist="" setup_only=false @@ -144,8 +144,8 @@ case $key in extra_vlan=true shift ;; - --deploy-external-server) - deploy_external_server=true + --deploy-no-antrea-worker) + deploy_no_antrea_worker=true shift ;; --coverage) @@ -239,6 +239,9 @@ fi if $flow_visibility; then manifest_args="$manifest_args --feature-gates FlowExporter=true,L7FlowExporter=true --extra-helm-values-file $FLOW_VISIBILITY_HELM_VALUES" fi +if $deploy_no_antrea_worker; then + manifest_args="$manifest_args --extra-helm-values testing.noAntreaNodeWithLabels.enable=true,testing.noAntreaNodeWithLabels.keys={no-antrea}" +fi COMMON_IMAGES_LIST=("registry.k8s.io/e2e-test-images/agnhost:2.29" \ "antrea/nginx:1.21.6-alpine" \ @@ -317,6 +320,10 @@ function setup_cluster { # Deploy an external server which could be used when testing Pod-to-External traffic. args="$args --deploy-external-server $vlan_args" + if $deploy_no_antrea_worker; then + args="$args --deploy-no-antrea-worker $vlan_args" + fi + echo "creating test bed with args $args" eval "timeout 600 $TESTBED_CMD create kind $args" }