From 4fdfa49a9788df7d6a806215874fc0ae7d8957e9 Mon Sep 17 00:00:00 2001 From: Pulkit Jain Date: Thu, 20 Jun 2024 12:56:42 +0530 Subject: [PATCH] Fix the cause of failure of multiple kind job runs on same VM. There were two main causes of failure: 1) Overriding of DOCKER_IMG_VERSION in makefile. 2) Overriding of kubeconfig because of which, when we ran multiple e2e tests together it used the same kubeconfig which was present the default location, so multiple jobs used the same cluster and failed. Signed-off-by: Pulkit Jain --- Makefile | 20 +++++++++++--------- ci/jenkins/test.sh | 9 ++++----- test/e2e/providers/kind.go | 14 ++++++-------- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index 47c9fd71c79..9a4f8682e59 100644 --- a/Makefile +++ b/Makefile @@ -37,14 +37,6 @@ TEST_ARGS ?= # If we have stdin we can run interactive so the tests running in docker can be interrupted. INTERACTIVE_ARGS := $(shell [ -t 0 ] && echo "-it") -BUILD_TAG := -ifndef CUSTOM_BUILD_TAG - BUILD_TAG = $(shell build/images/build-tag.sh) -else - BUILD_TAG = $(CUSTOM_BUILD_TAG) - DOCKER_IMG_VERSION = $(CUSTOM_BUILD_TAG) -endif - DOCKER_BUILD_ARGS := ifeq ($(NO_PULL),) DOCKER_BUILD_ARGS += --pull @@ -54,7 +46,6 @@ ifneq ($(NO_CACHE),) endif DOCKER_BUILD_ARGS += --build-arg OVS_VERSION=$(OVS_VERSION) DOCKER_BUILD_ARGS += --build-arg GO_VERSION=$(GO_VERSION) -DOCKER_BUILD_ARGS += --build-arg BUILD_TAG=$(BUILD_TAG) export CGO_ENABLED @@ -63,6 +54,17 @@ all: build include versioning.mk +# This should be located after the include directive for versioning.mk, +# so that we can override DOCKER_IMG_VERSION with CUSTOM_BUILD_TAG. +BUILD_TAG := +ifndef CUSTOM_BUILD_TAG + BUILD_TAG = $(shell build/images/build-tag.sh) +else + BUILD_TAG = $(CUSTOM_BUILD_TAG) + DOCKER_IMG_VERSION = $(CUSTOM_BUILD_TAG) +endif +DOCKER_BUILD_ARGS += --build-arg BUILD_TAG=$(BUILD_TAG) + LDFLAGS += $(VERSION_LDFLAGS) UNAME_S := $(shell uname -s) diff --git a/ci/jenkins/test.sh b/ci/jenkins/test.sh index 0428b1e26cf..45ab4dc49ce 100755 --- a/ci/jenkins/test.sh +++ b/ci/jenkins/test.sh @@ -73,8 +73,8 @@ Run K8s e2e community tests (Conformance & Network Policy) or Antrea e2e tests o --kind-cluster-name Name of the kind Cluster. --proxyall Enable proxyAll to test AntreaProxy. --build-tag Custom build tag for images. - --docker-user Username for Docker account. - --docker-password Password for Docker account." + --docker-user Username for Docker account. + --docker-password Password for Docker account." function print_usage { echoerr "$_usage" @@ -201,7 +201,7 @@ function clean_antrea { for antrea_yml in ${WORKDIR}/*.yml; do kubectl delete -f $antrea_yml --ignore-not-found=true || true done - docker images --format "{{.Repository}}:{{.Tag}}" | grep 'antrea'| xargs -r docker rmi -f || true + docker images --format "{{.Repository}}:{{.Tag}}" | grep '${BUILD_TAG}' | xargs -r docker rmi || true docker images | grep '' | awk '{print $3}' | xargs -r docker rmi || true check_and_cleanup_docker_build_cache } @@ -608,7 +608,6 @@ function deliver_antrea { kind load docker-image antrea/antrea-agent-ubuntu:$BUILD_TAG --name ${KIND_CLUSTER} kind load docker-image antrea/antrea-controller-ubuntu:$BUILD_TAG --name ${KIND_CLUSTER} kind load docker-image antrea/flow-aggregator:latest --name ${KIND_CLUSTER} - kubectl config use-context kind-${KIND_CLUSTER} docker cp ./build/yamls/antrea.yml ${KIND_CLUSTER}-control-plane:/root/antrea.yml elif [[ $TESTBED_TYPE == "jumper" ]]; then kubectl get nodes -o wide --no-headers=true | awk '{print $6}' | while read IP; do @@ -674,7 +673,7 @@ function run_e2e { if [[ $TESTBED_TYPE == "flexible-ipam" ]]; then go test -v antrea.io/antrea/test/e2e --logs-export-dir `pwd`/antrea-test-logs --provider remote -timeout=100m --prometheus --antrea-ipam elif [[ $TESTBED_TYPE == "kind" ]]; then - go test -v antrea.io/antrea/test/e2e --logs-export-dir `pwd`/antrea-test-logs --provider kind -timeout=100m --prometheus + go test -v antrea.io/antrea/test/e2e --logs-export-dir `pwd`/antrea-test-logs --provider kind --kind.kubeconfig ${KUBECONFIG_PATH} -timeout=100m --prometheus elif [[ $TESTBED_TYPE == "kind-flexible-ipam" ]]; then go test -v antrea.io/antrea/test/e2e --logs-export-dir `pwd`/antrea-test-logs --provider kind -timeout=100m --prometheus --antrea-ipam else diff --git a/test/e2e/providers/kind.go b/test/e2e/providers/kind.go index e5bab149cbe..6bb5ddbeda0 100644 --- a/test/e2e/providers/kind.go +++ b/test/e2e/providers/kind.go @@ -15,6 +15,7 @@ package providers import ( + "flag" "fmt" "os" "path" @@ -23,6 +24,8 @@ import ( "antrea.io/antrea/test/e2e/providers/exec" ) +var kindKubeconfigPath = flag.String("kind.kubeconfig", path.Join(homedir, ".kube", "config"), "Path of the kubeconfig of the cluster") + type KindProvider struct { controlPlaneNodeName string } @@ -47,15 +50,10 @@ func (provider *KindProvider) RunCommandOnNodeExt(nodeName, cmd string, envs map } func (provider *KindProvider) GetKubeconfigPath() (string, error) { - homeDir, err := os.UserHomeDir() - if err != nil { - return "", fmt.Errorf("error when retrieving user home directory: %v", err) - } - kubeconfigPath := path.Join(homeDir, ".kube", "config") - if _, err := os.Stat(kubeconfigPath); os.IsNotExist(err) { - return "", fmt.Errorf("Kubeconfig file not found at expected location '%s'", kubeconfigPath) + if _, err := os.Stat(*kindKubeconfigPath); os.IsNotExist(err) { + return "", fmt.Errorf("Kubeconfig file not found at expected location '%s'", *kindKubeconfigPath) } - return kubeconfigPath, nil + return *kindKubeconfigPath, nil } // enableKubectlOnControlPlane copies the Kubeconfig file on the Kind control-plane / control-plane Node to the