From 6bbdb2a65b249e69061e43eae9787817675e46ad Mon Sep 17 00:00:00 2001 From: Lan Date: Wed, 6 Jul 2022 04:37:20 +0800 Subject: [PATCH] Upload multi-cluster e2e coverage to codecov (#3942) 1. Add a new target on Makefile to build multi-cluster controller image with the instrumented binary. 2. Add steps on test-mc.sh to collect coverage files from leader and member cluster controllers and upload them to codecov. Signed-off-by: Lan Luo --- Makefile | 11 +++ ci/jenkins/test-mc.sh | 99 ++++++++++++++++--- ci/jenkins/test-vmc.sh | 4 +- multicluster/Makefile | 4 + .../build/images/Dockerfile.build.coverage | 21 ++++ .../bincover_run_main_test.go | 28 ++++++ .../manager_command_patch_coverage.yaml | 13 +++ .../manager_command_patch_coverage.yaml | 13 +++ multicluster/hack/generate-manifest.sh | 21 +++- 9 files changed, 197 insertions(+), 17 deletions(-) create mode 100644 multicluster/build/images/Dockerfile.build.coverage create mode 100644 multicluster/cmd/multicluster-controller/bincover_run_main_test.go create mode 100644 multicluster/config/overlays/leader-ns/coverage/manager_command_patch_coverage.yaml create mode 100644 multicluster/config/overlays/member/coverage/manager_command_patch_coverage.yaml diff --git a/Makefile b/Makefile index 203208508aa..ea39e90386a 100644 --- a/Makefile +++ b/Makefile @@ -406,6 +406,17 @@ endif docker tag antrea/antrea-mc-controller:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/antrea-mc-controller docker tag antrea/antrea-mc-controller:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/antrea-mc-controller:$(DOCKER_IMG_VERSION) +.PHONY: antrea-mc-controller-coverage +antrea-mc-controller-coverage: + @echo "===> Building antrea/antrea-mc-controller-coverage Docker image <===" +ifneq ($(NO_PULL),) + docker build -t antrea/antrea-mc-controller-coverage:$(DOCKER_IMG_VERSION) -f multicluster/build/images/Dockerfile.build.coverage $(DOCKER_BUILD_ARGS) . +else + docker build --pull -t antrea/antrea-mc-controller-coverage:$(DOCKER_IMG_VERSION) -f multicluster/build/images/Dockerfile.build.coverage $(DOCKER_BUILD_ARGS) . +endif + docker tag antrea/antrea-mc-controller-coverage:$(DOCKER_IMG_VERSION) antrea/antrea-mc-controller-coverage + docker tag antrea/antrea-mc-controller-coverage:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/antrea-mc-controller-coverage + .PHONY: flow-visibility-clickhouse-monitor flow-visibility-clickhouse-monitor: @echo "===> Building antrea/flow-visibility-clickhouse-monitor Docker image <===" diff --git a/ci/jenkins/test-mc.sh b/ci/jenkins/test-mc.sh index 336adfae134..9abb56b84d0 100755 --- a/ci/jenkins/test-mc.sh +++ b/ci/jenkins/test-mc.sh @@ -34,6 +34,8 @@ EAST_CLUSTER_CONFIG="--kubeconfig=$MULTICLUSTER_KUBECONFIG_PATH/east" WEST_CLUSTER_CONFIG="--kubeconfig=$MULTICLUSTER_KUBECONFIG_PATH/west" ENABLE_MC_GATEWAY=false IS_CONTAINERD=false +CODECOV_TOKEN="" +COVERAGE=false multicluster_kubeconfigs=($EAST_CLUSTER_CONFIG $LEADER_CLUSTER_CONFIG $WEST_CLUSTER_CONFIG) membercluster_kubeconfigs=($EAST_CLUSTER_CONFIG $WEST_CLUSTER_CONFIG) @@ -42,7 +44,7 @@ membercluster_kubeconfigs=($EAST_CLUSTER_CONFIG $WEST_CLUSTER_CONFIG) CLEAN_STALE_IMAGES="docker system prune --force --all --filter until=48h" _usage="Usage: $0 [--kubeconfigs-path ] [--workdir ] - [--testcase ] [--mc-gateway] + [--testcase ] [--mc-gateway] [--codecov-token] [--coverage] Run Antrea multi-cluster e2e tests on a remote (Jenkins) Linux Cluster Set. @@ -50,7 +52,9 @@ Run Antrea multi-cluster e2e tests on a remote (Jenkins) Linux Cluster Set. --workdir Home path for Go, vSphere information and antrea_logs during cluster setup. Default is $WORKDIR. --testcase Antrea multi-cluster e2e test cases on a Linux cluster set. --registry The docker registry to use instead of dockerhub. - --mc-gateway Enable Multicluster Gateway." + --mc-gateway Enable Multicluster Gateway. + --codecov-token Token used to upload coverage report(s) to Codecov. + --coverage Run e2e with coverage." function print_usage { echoerr "$_usage" @@ -82,6 +86,14 @@ case $key in ENABLE_MC_GATEWAY=true shift ;; + --codecov-token) + CODECOV_TOKEN="$2" + shift 2 + ;; + --coverage) + COVERAGE=true + shift + ;; -h|--help) print_usage exit 0 @@ -161,10 +173,9 @@ function wait_for_antrea_multicluster_pods_ready { function wait_for_multicluster_controller_ready { echo "====== Deploying Antrea Multicluster Leader Cluster with ${LEADER_CLUSTER_CONFIG} ======" kubectl create ns antrea-multicluster "${LEADER_CLUSTER_CONFIG}" || true - kubectl apply -f ./multicluster/test/yamls/manifest.yml "${LEADER_CLUSTER_CONFIG}" kubectl apply -f ./multicluster/build/yamls/antrea-multicluster-leader-global.yml "${LEADER_CLUSTER_CONFIG}" + kubectl apply -f ./multicluster/test/yamls/leader-manifest.yml "${LEADER_CLUSTER_CONFIG}" kubectl rollout status deployment/antrea-mc-controller -n antrea-multicluster "${LEADER_CLUSTER_CONFIG}" || true - kubectl apply -f ./multicluster/test/yamls/manifest.yml "${LEADER_CLUSTER_CONFIG}" kubectl create -f ./multicluster/test/yamls/leader-access-token-secret.yml "${LEADER_CLUSTER_CONFIG}" || true kubectl get secret -n antrea-multicluster leader-access-token "${LEADER_CLUSTER_CONFIG}" -o yaml > ./multicluster/test/yamls/leader-access-token.yml @@ -180,7 +191,7 @@ function wait_for_multicluster_controller_ready { for config in "${membercluster_kubeconfigs[@]}"; do echo "====== Deploying Antrea Multicluster Member Cluster with ${config} ======" - kubectl apply -f ./multicluster/build/yamls/antrea-multicluster-member.yml ${config} + kubectl apply -f ./multicluster/test/yamls/member-manifest.yml ${config} kubectl rollout status deployment/antrea-mc-controller -n kube-system ${config} kubectl apply -f ./multicluster/test/yamls/leader-access-token.yml ${config} done @@ -191,6 +202,34 @@ function wait_for_multicluster_controller_ready { kubectl apply -f ./multicluster/test/yamls/clusterset.yml "${LEADER_CLUSTER_CONFIG}" } +# We run the function in a subshell with "set -e" to ensure that it exits in +# case of error (e.g. integrity check), no matter the context in which the +# function is called. +function run_codecov { (set -e + flag=$1 + file=$2 + dir=$3 + + rm -f trustedkeys.gpg codecov + # This is supposed to be a one-time step, but there should be no harm in + # getting the key every time. It does not come from the codecov.io + # website. Anyway, this is needed when the VM is re-created for every test. + curl https://keybase.io/codecovsecurity/pgp_keys.asc | gpg --no-default-keyring --keyring trustedkeys.gpg --import + curl -Os https://uploader.codecov.io/latest/linux/codecov + curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM + curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM.sig + + # Check that the sha256 matches the signature + gpgv codecov.SHA256SUM.sig codecov.SHA256SUM + # Then check the integrity of the codecov binary + shasum -a 256 -c codecov.SHA256SUM + + chmod +x codecov + ./codecov -c -t ${CODECOV_TOKEN} -F ${flag} -f ${file} -s ${dir} -C ${GIT_COMMIT} -r antrea-io/antrea + + rm -f trustedkeys.gpg codecov +)} + function deliver_antrea_multicluster { echo "====== Building Antrea for the Following Commit ======" export GO111MODULE=on @@ -231,10 +270,17 @@ function deliver_multicluster_controller { export GOROOT=/usr/local/go export PATH=${GOROOT}/bin:$PATH - export NO_PULL=1;make antrea-mc-controller - - docker save "${DOCKER_REGISTRY}"/antrea/antrea-mc-controller:latest -o "${WORKDIR}"/antrea-mcs.tar - ./multicluster/hack/generate-manifest.sh -l antrea-multicluster > ./multicluster/test/yamls/manifest.yml + if $COVERAGE;then + export NO_PULL=1;make antrea-mc-controller-coverage + docker save "${DOCKER_REGISTRY}"/antrea/antrea-mc-controller-coverage:latest -o "${WORKDIR}"/antrea-mcs.tar + ./multicluster/hack/generate-manifest.sh -l antrea-multicluster -c > ./multicluster/test/yamls/leader-manifest.yml + ./multicluster/hack/generate-manifest.sh -m -c > ./multicluster/test/yamls/member-manifest.yml + else + export NO_PULL=1;make antrea-mc-controller + docker save "${DOCKER_REGISTRY}"/antrea/antrea-mc-controller:latest -o "${WORKDIR}"/antrea-mcs.tar + ./multicluster/hack/generate-manifest.sh -l antrea-multicluster > ./multicluster/test/yamls/leader-manifest.yml + ./multicluster/hack/generate-manifest.sh -m > ./multicluster/test/yamls/member-manifest.yml + fi for kubeconfig in "${multicluster_kubeconfigs[@]}" do @@ -264,7 +310,7 @@ function deliver_multicluster_controller { } function run_multicluster_e2e { - echo "====== Running Multicluster e2e Tests ======" + echo "====== Running Multicluster e2e Tests ======" export GO111MODULE=on export GOPATH=${WORKDIR}/go export GOROOT=/usr/local/go @@ -311,11 +357,12 @@ EOF done set +e - mkdir -p `pwd`/antrea-multicluster-test-logs + CURRENT_DIR=`pwd` + mkdir -p ${CURRENT_DIR}/antrea-multicluster-test-logs if ${ENABLE_MC_GATEWAY};then - go test -v antrea.io/antrea/multicluster/test/e2e --logs-export-dir `pwd`/antrea-multicluster-test-logs --mc-gateway + go test -v antrea.io/antrea/multicluster/test/e2e --logs-export-dir `pwd`/antrea-multicluster-test-logs --mc-gateway else - go test -v antrea.io/antrea/multicluster/test/e2e --logs-export-dir `pwd`/antrea-multicluster-test-logs + go test -v antrea.io/antrea/multicluster/test/e2e --logs-export-dir `pwd`/antrea-multicluster-test-logs fi if [[ "$?" != "0" ]]; then @@ -324,6 +371,22 @@ EOF set -e } +function collect_coverage { + COVERAGE_DIR=$1 + timestamp=$(date +%Y%m%d%H%M%S) + echo "====== Collect Multicluster e2e Tests Coverage Files ======" + for kubeconfig in "${multicluster_kubeconfigs[@]}"; do + namespace="kube-system" + if [[ ${kubeconfig} =~ "leader" ]];then + namespace="antrea-multicluster" + fi + mc_controller_pod_name="$(kubectl get pods --selector=app=antrea,component=antrea-mc-controller -n ${namespace} --no-headers=true ${kubeconfig} | awk '{ print $1 }')" + controller_pid="$(kubectl exec -i $mc_controller_pod_name -n ${namespace} ${kubeconfig} -- pgrep antrea)" + kubectl exec -i $mc_controller_pod_name -n ${namespace} ${kubeconfig} -- kill -SIGINT $controller_pid + kubectl cp ${namespace}/$mc_controller_pod_name:antrea-mc-controller.cov.out ${COVERAGE_DIR}/$mc_controller_pod_name-$timestamp ${kubeconfig} + done +} + trap clean_multicluster EXIT clean_tmp clean_images @@ -341,6 +404,16 @@ if [[ ${TESTCASE} =~ "e2e" ]]; then deliver_antrea_multicluster deliver_multicluster_controller run_multicluster_e2e + if $COVERAGE;then + CURRENT_DIR=`pwd` + rm -rf mc-e2e-coverage + mkdir -p mc-e2e-coverage + collect_coverage ${CURRENT_DIR}/mc-e2e-coverage + # Backup coverage files for later analysis + set +e;find ${DEFAULT_WORKDIR}/mc-e2e-coverage -maxdepth 1 -mtime +1 -type f | xargs -n 1 rm;set -e; # Clean up backup files older than one day. + cp -r mc-e2e-coverage ${DEFAULT_WORKDIR} + run_codecov "e2e-tests" "*antrea-mc*" "${CURRENT_DIR}/mc-e2e-coverage" + fi fi if [[ ${TEST_FAILURE} == true ]]; then diff --git a/ci/jenkins/test-vmc.sh b/ci/jenkins/test-vmc.sh index 58d91a6be74..0a52c9e5c3a 100755 --- a/ci/jenkins/test-vmc.sh +++ b/ci/jenkins/test-vmc.sh @@ -41,7 +41,7 @@ CONTROL_PLANE_NODE_ROLE="master|control-plane" _usage="Usage: $0 [--cluster-name ] [--kubeconfig ] [--workdir ] [--log-mode ] [--testcase ] - [--garbage-collection] [--setup-only] [--cleanup-only] [--coverage] [--test-only] [--registry] + [--garbage-collection] [--setup-only] [--cleanup-only] [--coverage] [--test-only] [--codecov-token] [--registry] Setup a VMC cluster to run K8s e2e community tests (E2e, Conformance, all features Conformance, whole Conformance & Network Policy). @@ -300,7 +300,7 @@ function copy_image { ${SSH_WITH_ANTREA_CI_KEY} -n capv@${IP} "sudo crictl images | grep '' | awk '{print \$3}' | xargs -r crictl rmi" } -# We run the function in a subshell with "set -e" to ensure that it exists in +# We run the function in a subshell with "set -e" to ensure that it exits in # case of error (e.g. integrity check), no matter the context in which the # function is called. function run_codecov { (set -e diff --git a/multicluster/Makefile b/multicluster/Makefile index 2d9dc88705f..80b724e716c 100644 --- a/multicluster/Makefile +++ b/multicluster/Makefile @@ -85,6 +85,10 @@ endif build: fmt vet ## Build manager binary. CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o bin/antrea-mc-controller antrea.io/antrea/multicluster/cmd/... +.PHONY: antrea-mc-instr-binary +antrea-mc-instr-binary: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go test -tags testbincover -covermode count -coverpkg=antrea.io/antrea/multicluster/... -c -o bin/antrea-mc-controller-coverage antrea.io/antrea/multicluster/cmd/... + run: manifests generate fmt vet ## Run a controller from your host. go run ./main.go diff --git a/multicluster/build/images/Dockerfile.build.coverage b/multicluster/build/images/Dockerfile.build.coverage new file mode 100644 index 00000000000..65fa2275fd9 --- /dev/null +++ b/multicluster/build/images/Dockerfile.build.coverage @@ -0,0 +1,21 @@ +ARG GO_VERSION +FROM golang:${GO_VERSION} as antrea-build + +WORKDIR /antrea + +COPY go.mod /antrea/go.mod + +RUN go mod download + +COPY . /antrea + +RUN cd multicluster && make antrea-mc-instr-binary + +FROM ubuntu:20.04 + +LABEL maintainer="Antrea " +LABEL description="The Docker image to deploy the Antrea Multicluster controller with code coverage measurement enabled (used for testing)." + +USER root + +COPY --from=antrea-build /antrea/multicluster/bin/antrea-mc-controller-coverage / diff --git a/multicluster/cmd/multicluster-controller/bincover_run_main_test.go b/multicluster/cmd/multicluster-controller/bincover_run_main_test.go new file mode 100644 index 00000000000..c8eb2554671 --- /dev/null +++ b/multicluster/cmd/multicluster-controller/bincover_run_main_test.go @@ -0,0 +1,28 @@ +// Copyright 2022 Antrea Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build testbincover +// +build testbincover + +package main + +import ( + "testing" + + "github.com/confluentinc/bincover" +) + +func TestBincoverRunMain(t *testing.T) { + bincover.RunTest(main) +} diff --git a/multicluster/config/overlays/leader-ns/coverage/manager_command_patch_coverage.yaml b/multicluster/config/overlays/leader-ns/coverage/manager_command_patch_coverage.yaml new file mode 100644 index 00000000000..7ae00d8492e --- /dev/null +++ b/multicluster/config/overlays/leader-ns/coverage/manager_command_patch_coverage.yaml @@ -0,0 +1,13 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller + namespace: system +spec: + template: + spec: + containers: + - command: ["/bin/sh"] + args: ["-c", "/antrea-mc-controller-coverage -test.run=TestBincoverRunMain -test.coverprofile=antrea-mc-controller.cov.out leader; while true; do sleep 5 & wait $!; done"] + name: antrea-mc-controller + image: projects.registry.vmware.com/antrea/antrea-mc-controller-coverage:latest diff --git a/multicluster/config/overlays/member/coverage/manager_command_patch_coverage.yaml b/multicluster/config/overlays/member/coverage/manager_command_patch_coverage.yaml new file mode 100644 index 00000000000..74aae24e98b --- /dev/null +++ b/multicluster/config/overlays/member/coverage/manager_command_patch_coverage.yaml @@ -0,0 +1,13 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller + namespace: system +spec: + template: + spec: + containers: + - command: ["/bin/sh"] + args: ["-c", "/antrea-mc-controller-coverage -test.run=TestBincoverRunMain -test.coverprofile=antrea-mc-controller.cov.out member; while true; do sleep 5 & wait $!; done"] + name: antrea-mc-controller + image: projects.registry.vmware.com/antrea/antrea-mc-controller-coverage:latest diff --git a/multicluster/hack/generate-manifest.sh b/multicluster/hack/generate-manifest.sh index e3e46eb278a..8b82e35c8de 100755 --- a/multicluster/hack/generate-manifest.sh +++ b/multicluster/hack/generate-manifest.sh @@ -27,6 +27,7 @@ Generate a YAML manifest for Antrea MultiCluster using Kustomize and print it to --leader | -l Generate a per-namespace manifest for a Cluster as leader in a ClusterSet. All resources will be in the given namespace --member | -m Generate a manifest for a Cluster as member in a ClusterSet + --coverage| -c Generate a manifest which supports measuring code coverage --help | -h Print this message and exit Environment variables IMG_NAME and IMG_TAG must be set when release mode is enabled. @@ -43,6 +44,7 @@ function print_help { OVERLAY=member NAMESPACE=antrea-multicluster MODE="" +COVERAGE=false while [[ $# -gt 0 ]] do @@ -52,6 +54,10 @@ case $key in MODE="release" shift ;; + --coverage|-c) + COVERAGE=true + shift + ;; --leader|-l) OVERLAY=leader-ns NAMESPACE="$2" @@ -113,7 +119,7 @@ then cp $KUSTOMIZATION_DIR/overlays/leader-ns/prefix_transformer.yaml . sed -ie "s/antrea-multicluster/$NAMESPACE/g" prefix_transformer.yaml -cat << EOF > kustomization.yaml + cat << EOF > kustomization.yaml namespace: $NAMESPACE bases: @@ -122,8 +128,13 @@ bases: transformers: - prefix_transformer.yaml EOF + + if $COVERAGE; then + cp ../../overlays/$OVERLAY/coverage/manager_command_patch_coverage.yaml . + $KUSTOMIZE edit add patch --path ./manager_command_patch_coverage.yaml + fi else -cat << EOF > kustomization.yaml + cat << EOF > kustomization.yaml bases: - ../overlays/$OVERLAY EOF @@ -134,5 +145,11 @@ if [ "$MODE" == "release" ]; then else $KUSTOMIZE edit set image antrea/antrea-mc-controller=projects.registry.vmware.com/antrea/antrea-mc-controller:latest fi + +if [ "$OVERLAY" == "member" ] && $COVERAGE; then + cp ../overlays/$OVERLAY/coverage/manager_command_patch_coverage.yaml . + $KUSTOMIZE edit add patch --path ./manager_command_patch_coverage.yaml +fi + $KUSTOMIZE build rm -rf $TMP_DIR