diff --git a/ci/kind/kind-setup.sh b/ci/kind/kind-setup.sh index e7646ea3b26..714d7ba2238 100755 --- a/ci/kind/kind-setup.sh +++ b/ci/kind/kind-setup.sh @@ -38,7 +38,8 @@ KUBE_PROXY_MODE="iptables" PROMETHEUS=false K8S_VERSION="" KUBE_NODE_IPAM=true -DEPLOY_EXTERNAL_SERVER=false +DEPLOY_EXTERNAL_AGNHOST=false +DEPLOY_EXTERNAL_FRR=false positional_args=() options=() @@ -79,8 +80,9 @@ where: network. Note, '--extra-networks' and '--subnets' cannot be specified together. --ip-family: specify the ip-family for the kind cluster, default is $IP_FAMILY. --k8s-version: specify the Kubernetes version of the kind cluster, kind's default K8s version will be used if empty. - --deploy-external-server: deploy a container running as an external server for the cluster. - --all: delete all kind clusters + --deploy-external-agnhost: deploy a container running agnhost as an external server for the cluster, default is $DEPLOY_EXTERNAL_AGNHOST. + --deploy-external-frr: deploy a container running FRR as an external router for the cluster, default is $DEPLOY_EXTERNAL_FRR. + --all: delete all kind clusters. --until: delete kind clusters that have been created before the specified duration. " @@ -411,7 +413,7 @@ EOF configure_networks configure_extra_networks configure_vlan_subnets - setup_external_server + setup_external_servers load_images if [[ $ANTREA_CNI == true ]]; then @@ -441,7 +443,7 @@ function destroy { else kind delete cluster --name $CLUSTER_NAME fi - destroy_external_server + destroy_external_servers delete_networks delete_vlan_subnets } @@ -456,14 +458,30 @@ function printUnixTimestamp { } function setup_external_server { - if [[ $DEPLOY_EXTERNAL_SERVER == true ]]; then - docker run -d --name antrea-external-server-$RANDOM --network kind -it --rm registry.k8s.io/e2e-test-images/agnhost:2.40 netexec &> /dev/null + if [[ $DEPLOY_EXTERNAL_AGNHOST == true ]]; then + docker run -d --name antrea-external-agnhost-$RANDOM --network kind -it --rm registry.k8s.io/e2e-test-images/agnhost:2.40 netexec &> /dev/null fi + + if [[ $DEPLOY_EXTERNAL_FRR == true ]]; then + docker run -d \ + --name antrea-external-frr-$RANDOM \ + --network kind --cap-add=NET_BIND_SERVICE \ + --cap-add=NET_ADMIN \ + --cap-add=NET_RAW \ + --cap-add=SYS_ADMIN \ + -it \ + --rm \ + frrouting/frr:v8.4.0 \ + bash -c "/bin/sed -i s/bgpd=no/bgpd=yes/g /etc/frr/daemons && /sbin/tini -- /usr/lib/frr/docker-start" &> /dev/null + fi } -function destroy_external_server { - echo "Deleting external server" - cid=$(docker ps -f name="^antrea-external-server" --format '{{.ID}}') +function destroy_external_servers { + echo "Deleting external servers" + cid=$(docker ps -f name="^antrea-external-agnhost" --format '{{.ID}}') + docker rm -f $cid &> /dev/null || true + + cid=$(docker ps -f name="^antrea-external-frr" --format '{{.ID}}') docker rm -f $cid &> /dev/null || true } @@ -583,9 +601,14 @@ while [[ $# -gt 0 ]] K8S_VERSION="$2" shift 2 ;; - --deploy-external-server) - add_option "--deploy-external-server" "create" - DEPLOY_EXTERNAL_SERVER=true + --deploy-external-agnhost) + add_option "--deploy-external-agnhost" "create" + DEPLOY_EXTERNAL_AGNHOST=true + shift + ;; + --deploy-external-frr) + add_option "--deploy-external-frr" "create" + DEPLOY_EXTERNAL_FRR=true shift ;; --all) diff --git a/ci/kind/test-e2e-kind.sh b/ci/kind/test-e2e-kind.sh index 6ae15def137..1e91c95c874 100755 --- a/ci/kind/test-e2e-kind.sh +++ b/ci/kind/test-e2e-kind.sh @@ -24,20 +24,21 @@ function echoerr { _usage="Usage: $0 [--encap-mode ] [--ip-family ] [--coverage] [--help|-h] --encap-mode Traffic encapsulation mode. (default is 'encap'). - --ip-family Configures the ipFamily for the KinD cluster. + --ip-family Configure the ipFamily for the KinD cluster. --feature-gates A comma-separated list of key=value pairs that describe feature gates, e.g. AntreaProxy=true,Egress=false. --run Run only tests matching the regexp. - --proxy-all Enables Antrea proxy with all Service support. + --proxy-all Enable Antrea proxy with all Service support. --no-kube-proxy Don't deploy kube-proxy. --load-balancer-mode LoadBalancer mode. - --node-ipam Enables Antrea NodeIPAM. - --multicast Enables Multicast. + --node-ipam Enable Antrea NodeIPAM. + --multicast Enable Multicast. + --bgp-policy Enable Antrea BGPPolicy. --flow-visibility Only run flow visibility related e2e tests. - --networkpolicy-evaluation Configures additional NetworkPolicy evaluation level when running e2e tests. - --extra-network Creates an extra network that worker Nodes will connect to. Cannot be specified with the hybrid mode. - --extra-vlan Creates an subnet-based VLAN that worker Nodes will connect to. + --networkpolicy-evaluation Configure additional NetworkPolicy evaluation level when running e2e tests. + --extra-network Create an extra network that worker Nodes will connect to. Cannot be specified with the hybrid mode. + --extra-vlan Create an subnet-based VLAN that worker Nodes will connect to. --skip A comma-separated list of keywords, with which tests should be skipped. - --coverage Enables measure Antrea code coverage when running e2e tests on kind. + --coverage Enable measure Antrea code coverage when running e2e tests on kind. --setup-only Only perform setting up the cluster and run test. --cleanup-only Only perform cleaning up the cluster. --test-only Only run test on current cluster. Not set up/clean up the cluster. @@ -77,6 +78,7 @@ no_kube_proxy=false load_balancer_mode="" node_ipam=false multicast=false +bgp_policy=false flow_visibility=false np_evaluation=false extra_network=false @@ -124,6 +126,10 @@ case $key in multicast=true shift ;; + --bgp-policy) + bgp_policy=true + shift + ;; --ip-family) ipfamily="$2" shift 2 @@ -236,6 +242,9 @@ fi if $multicast; then manifest_args="$manifest_args --multicast" fi +if $bgp_policy; then + manifest_args="$manifest_args --feature-gates BGPPolicy=true" +fi if $flow_visibility; then manifest_args="$manifest_args --feature-gates FlowExporter=true,L7FlowExporter=true --extra-helm-values-file $FLOW_VISIBILITY_HELM_VALUES" fi @@ -314,8 +323,12 @@ function setup_cluster { if $extra_network && [[ "$mode" != "hybrid" ]]; then args="$args --extra-networks \"20.20.30.0/24\"" fi - # Deploy an external server which could be used when testing Pod-to-External traffic. - args="$args --deploy-external-server $vlan_args" + # Deploy an external agnhost which could be used when testing Pod-to-External traffic. + args="$args --deploy-external-agnhost $vlan_args" + # Deploy an external FRR which could be used when testing BGPPolicy. + if $bgp_policy; then + args="$args --deploy-external-frr" + fi echo "creating test bed with args $args" eval "timeout 600 $TESTBED_CMD create kind $args" @@ -379,9 +392,15 @@ function run_test { np_evaluation_flag="--networkpolicy-evaluation" fi - external_server_cid=$(docker ps -f name="^antrea-external-server" --format '{{.ID}}') - external_server_ips=$(docker inspect $external_server_cid -f '{{.NetworkSettings.Networks.kind.IPAddress}},{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}') - EXTRA_ARGS="$vlan_args --external-server-ips $external_server_ips" + external_agnhost_cid=$(docker ps -f name="^antrea-external-agnhost" --format '{{.ID}}') + external_agnhost_ips=$(docker inspect $external_agnhost_cid -f '{{.NetworkSettings.Networks.kind.IPAddress}},{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}') + EXTRA_ARGS="$vlan_args --external-agnhost-ips $external_agnhost_ips" + + if $bgp_policy; then + external_frr_cid=$(docker ps -f name="^antrea-external-frr" --format '{{.ID}}') + external_frr_ips=$(docker inspect $external_frr_cid -f '{{.NetworkSettings.Networks.kind.IPAddress}},{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}') + EXTRA_ARGS="$EXTRA_ARGS --external-frr-cid $external_frr_cid --external-frr-ips $external_frr_ips" + fi go test -v -timeout=$timeout $RUN_OPT antrea.io/antrea/test/e2e $flow_visibility_args -provider=kind --logs-export-dir=$ANTREA_LOG_DIR $np_evaluation_flag --skip-cases=$skiplist $coverage_args $EXTRA_ARGS diff --git a/test/e2e/fixtures.go b/test/e2e/fixtures.go index 55297af4d5d..b8aa8674648 100644 --- a/test/e2e/fixtures.go +++ b/test/e2e/fixtures.go @@ -685,7 +685,9 @@ func testMain(m *testing.M) int { flag.StringVar(&testOptions.skipCases, "skip-cases", "", "Key words to skip cases") flag.StringVar(&testOptions.linuxVMs, "linuxVMs", "", "hostname of Linux VMs") flag.StringVar(&testOptions.windowsVMs, "windowsVMs", "", "hostname of Windows VMs") - flag.StringVar(&testOptions.externalServerIPs, "external-server-ips", "", "IP addresses of external server, at most one IP per IP family") + flag.StringVar(&testOptions.externalAgnhostIPs, "external-agnhost-ips", "", "IP addresses of external agnhost, at most one IP per IP family") + flag.StringVar(&testOptions.externalFRRIPs, "external-frr-ips", "", "IP addresses of external FRR, at most one IP per IP family") + flag.StringVar(&testOptions.externalFRRCID, "external-frr-cid", "", "Container ID of external FRR") flag.StringVar(&testOptions.vlanSubnets, "vlan-subnets", "", "IP subnets of the VLAN network the Nodes reside in, at most one subnet per IP family") flag.IntVar(&testOptions.vlanID, "vlan-id", 0, "ID of the VLAN network the Nodes reside in") flag.Parse() diff --git a/test/e2e/framework.go b/test/e2e/framework.go index e752434be2c..0f0530d7d15 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -190,6 +190,10 @@ type ExternalInfo struct { vlanSubnetIPv6 string vlanGatewayIPv6 string vlanID int + + externalFRRIPv4 string + externalFRRIPv6 string + externalFRRCID string } var clusterInfo ClusterInfo @@ -213,9 +217,16 @@ type TestOptions struct { // the home directory of the control-plane Node. Note it doesn't affect the tests that redeploy Antrea themselves. deployAntrea bool - externalServerIPs string - vlanSubnets string - vlanID int + externalAgnhostIPs string + vlanSubnets string + vlanID int + + externalFRRIPs string + // FRR cannot currently be configured remotely over networking. As a result, the e2e tests for BGPPolicy can only + // be run in a Kind cluster, where the FRR container can be configured using Docker exec with the container ID. + // TODO: Introduce a BGP router implementation that can be configured remotely over networking to replace FRR. + // This would allow the e2e tests for BGPPolicy to be run in environments other than just a Kind cluster. + externalFRRCID string } type flowVisibilityTestOptions struct { @@ -498,14 +509,14 @@ func (data *TestData) RunCommandOnNodeExt(nodeName, cmd string, envs map[string] } func (data *TestData) collectExternalInfo() error { - ips := strings.Split(testOptions.externalServerIPs, ",") + ips := strings.Split(testOptions.externalAgnhostIPs, ",") for _, ip := range ips { if ip == "" { continue } parsedIP := net.ParseIP(ip) if parsedIP == nil { - return fmt.Errorf("invalid external server IP %s", ip) + return fmt.Errorf("invalid external agnhost IP %s", ip) } if parsedIP.To4() != nil { externalInfo.externalServerIPv4 = ip @@ -532,6 +543,25 @@ func (data *TestData) collectExternalInfo() error { } } externalInfo.vlanID = testOptions.vlanID + + frrIPs := strings.Split(testOptions.externalFRRIPs, ",") + for _, ip := range frrIPs { + if ip == "" { + continue + } + parsedIP := net.ParseIP(ip) + if parsedIP == nil { + return fmt.Errorf("invalid external FRR IP %s", ip) + } + if parsedIP.To4() != nil { + externalInfo.externalFRRIPv4 = ip + } else { + externalInfo.externalFRRIPv6 = ip + } + } + + externalInfo.externalFRRCID = testOptions.externalFRRCID + return nil }