Skip to content
This repository has been archived by the owner on Jul 16, 2024. It is now read-only.

Commit

Permalink
Merge pull request #437 from qabpea/pa-3
Browse files Browse the repository at this point in the history 
Added skip filters to make it  possible for ci/cd test succeed
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot authored Jan 28, 2023
2 parents f763e14 + 03b5822 commit 049d930
Show file tree
Hide file tree
Showing 4 changed files with 324 additions and 56 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/e2e_tests.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ jobs:
fi
- name: run e2e tests
run: ./hack/test_e2e.sh -i ${{ env.IP_FAMILY }} -b ${{ env.BACKEND }} -c -n 1
run: ./hack/test_e2e.sh -i ${{ env.IP_FAMILY }} -b ${{ env.BACKEND }} -c -n 1 -S

- name: Export logs
if: always()
Expand Down
160 changes: 116 additions & 44 deletions hack/test_e2e.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,24 +41,55 @@ GINKGO_PROVIDER="local"

source "${SCRIPT_DIR}"/utils.sh
source "${SCRIPT_DIR}"/common.sh
source "${SCRIPT_DIR}"/test_skip_list.sh

function if_error_warning {
###########################################################################
# Description: #
# Validate if previous command failed and show an error msg (if provided) #
# #
# Arguments: #
# $1 - error message if not provided, it will just exit #
# arg1 - error message if not provided, it will just exit #
###########################################################################
if [ "$?" != "0" ]; then
if [ -n "$1" ]; then
RED="\e[31m"
RED="\e[91m"
ENDCOLOR="\e[0m"
echo -e "[ ${RED}FAILED${ENDCOLOR} ] ${1}"
fi
fi
}

function result_message {
###########################################################################
# Description: #
# show [FAILED] in red and a message. #
# #
# Arguments: #
# arg1 - result #
# arg2 - message to output #
###########################################################################
[ $# -eq 2 ]
if_error_exit "result_message: $# Wrong number of arguments to ${FUNCNAME[0]}... args: $#"

local result=${1}
local message="${2}"

if [ -z "${result}" ]; then
echo "result_message() requires a message"
exit 1
fi
RED="\e[91m"
GREEN="\e[92m"
ENDCOLOR="\e[0m"

if [ "${result}" == "0" ] ; then
echo -e "${GREEN}[SUCCESS!] ${message} succeded!!! ${ENDCOLOR}"
else
echo -e "${RED}[FAIL!] ${message} failed!!! ${ENDCOLOR}"
fi
}

function detect_container_engine {
###########################################################################
# Description: #
Expand Down Expand Up @@ -222,7 +253,7 @@ function create_cluster {
;;
esac

info_message -e "\nPreparing to setup ${cluster_name} cluster ..."
info_message "Preparing to setup ${cluster_name} cluster ..."
# create cluster
# create the config file
cat <<EOF > "${artifacts_directory}/kind-config.yaml"
Expand Down Expand Up @@ -448,17 +479,23 @@ function run_tests {
# #
# Arguments: #
# arg1: e2e directory #
# arg2: bin_dir, path to binary directory #
# arg2: bin_dir, path to binary directory #
# arg3: parallel ginkgo tests boolean #
# arg4: ip_family #
# arg5: backend #
# arg6: include specific failed tests #
###########################################################################

[ $# -eq 3 ]
[ $# -eq 6 ]
if_error_exit "exec tests $# Wrong number of arguments to ${FUNCNAME[0]}"

local e2e_dir="${1}"
local bin_dir="${2}"
local parallel="${3}"

local ip_family="${4}"
local backend="${5}"
local include_specific_failed_tests="${6}"

local artifacts_directory="${e2e_dir}/artifacts"

[ -f "${artifacts_directory}/${KUBECONFIG_TESTS}" ]
Expand All @@ -473,7 +510,6 @@ function run_tests {
[ -f "${bin_dir}/ginkgo" ]
if_error_exit "File \"${bin_dir}/ginkgo\" does not exist"


# ginkgo regexes
local ginkgo_skip="${GINKGO_SKIP_TESTS:-}"
local ginkgo_focus=${GINKGO_FOCUS:-"\\[Conformance\\]"}
Expand All @@ -487,13 +523,22 @@ function run_tests {
fi
fi

if [ "${include_specific_failed_tests}" == "false" ] ; then
# find ip_type and backend specific skip sets
skip_set_name="GINKGO_SKIP_${ip_family}_${backend}_TEST"
declare -n skip_set_ref=${skip_set_name}
if ! [ -z "${skip_set_ref}" ] ; then
ginkgo_skip="${ginkgo_skip}|${skip_set_ref}"
fi
fi

# setting this env prevents ginkgo e2e from trying to run provider setup
export KUBERNETES_CONFORMANCE_TEST='y'
# setting these is required to make RuntimeClass tests work ... :/
export KUBE_CONTAINER_RUNTIME=remote
export KUBE_CONTAINER_RUNTIME_ENDPOINT=unix:///run/containerd/containerd.sock
export KUBE_CONTAINER_RUNTIME_NAME=containerd

"${bin_dir}/ginkgo" --nodes="${GINKGO_NUMBER_OF_NODES}" \
--focus="${ginkgo_focus}" \
--skip="${ginkgo_skip}" \
Expand All @@ -504,6 +549,7 @@ function run_tests {
--dump-logs-on-failure="${GINKGO_DUMP_LOGS_ON_FAILURE}" \
--report-dir="${GINKGO_REPORT_DIR}" \
--disable-log-dump="${GINKGO_DISABLE_LOG_DUMP}"
return $?
}

function clean_artifacts {
Expand Down Expand Up @@ -602,8 +648,9 @@ function create_infrastructure_and_run_tests {
# arg7: <ci_mode> #
# arg8: deployment_model #
# arg9: export_metrics #
# arg10: include_specific_failed_tests #
###########################################################################
[ $# -eq 9 ]
[ $# -eq 10 ]
if_error_exit "create and run $# Wrong number of arguments to ${FUNCNAME[0]}"

local e2e_dir="${1}"
Expand All @@ -615,6 +662,8 @@ function create_infrastructure_and_run_tests {
local ci_mode="${7}"
local deployment_model="${8}"
local export_metrics="${9}"
local include_specific_failed_tests="${10}"


local artifacts_directory="${e2e_dir}/artifacts"
local cluster_name="kpng-e2e-${ip_family}-${backend}${suffix}"
Expand Down Expand Up @@ -643,14 +692,16 @@ function create_infrastructure_and_run_tests {
if [ "${backend}" != "not-kpng" ] ; then
install_kpng "${cluster_name}" "${bin_dir}"
fi

local result=0
if ! ${devel_mode} ; then
run_tests "${e2e_dir}" "${bin_dir}" "false"
#need to clean this up
if [ "${ci_mode}" = false ] ; then
clean_artifacts "${e2e_dir}" "${bin_dir}"
fi
run_tests "${e2e_dir}" "${bin_dir}" "false" "${ip_family}" "${backend}" "${include_specific_failed_tests}"
result=$?
#need to clean this up
if [ "${ci_mode}" = false ] ; then
clean_artifacts "${e2e_dir}" "${bin_dir}"
fi
fi
return ${result}
}

function delete_kind_clusters {
Expand All @@ -659,7 +710,7 @@ function delete_kind_clusters {
# delete_kind_clusters #
# #
# Arguments: #
# arg1: bin_dir #
# arg1: bin_dir #
# arg2: ip_family #
# arg3: backend #
# arg4: suffix #
Expand Down Expand Up @@ -735,16 +786,15 @@ function print_reports {
continue
fi

info_message -e "Summary report from cluster \"${i}\" in directory: \"${test_directory}\""
info_message "Summary report from cluster \"${i}\" in directory: \"${test_directory}\""
local output_file="${test_directory}/output.log"
cat "${output_file}" >> "${combined_output_file}"

sed -nE '/Ran[[:space:]]+[[:digit:]]+[[:space:]]+of[[:space:]]+[[:digit:]]/{N;p}' "${output_file}"
done

echo -e "\nOccurrence\tFailure"
grep \"msg\":\"FAILED "${combined_output_file}" |
sed 's/^.*\"FAILED/\t/' | sed 's/\,\"completed\".*//' | sed 's/[ \"]$//' |
grep "\[FAIL\]" "${combined_output_file}" |
sort | uniq -c | sort -nr | sed 's/\,/\n\t\t/g'

rm -f "${combined_output_file}"
Expand All @@ -759,7 +809,7 @@ function main {
# None #
###########################################################################

[ $# -eq 14 ]
[ $# -eq 15 ]
if_error_exit "Wrong number of arguments to ${FUNCNAME[0]}"

# setting up variables
Expand All @@ -777,6 +827,7 @@ function main {
local deployment_model="${12}"
local run_tests_on_existing_cluster="${13}"
local export_metrics="${14}"
local include_specific_failed_tests="${15}"

[ "${cluster_count}" -ge "1" ]
if_error_exit "cluster_count must be larger or equal to one"
Expand All @@ -787,25 +838,27 @@ function main {

if ${erase_clusters} ; then
delete_kind_clusters "${bin_dir}" "${ip_family}" "${backend}" "${suffix}" "${cluster_count}"
exit 1
return 0
fi

if ${print_report} ; then
print_reports "${ip_family}" "${backend}" "${e2e_dir}" "-${suffix}" "${cluster_count}"
exit 1
return 0
fi

echo "+==================================================================+"
echo -e "\t\tStarting KPNG E2E testing"
echo "+==================================================================+"


if [ "${run_tests_on_existing_cluster}" = true ] ; then
run_tests "${e2e_dir}${tmp_suffix}" "${bin_dir}" "false"
run_tests "${e2e_dir}${tmp_suffix}" "${bin_dir}" "false" "${ip_family}" "${backend}" "${include_specific_failed_tests}"
local result=$?
#need to clean this up
if [ "${ci_mode}" = false ] ; then
clean_artifacts "${e2e_dir}${tmp_suffix}"
fi

exit 1
if [ "${ci_mode}" = false ] ; then
clean_artifacts "${e2e_dir}${tmp_suffix}"
fi
return ${result}
fi

# in ci this should fail
Expand Down Expand Up @@ -839,12 +892,14 @@ function main {
done
fi

local result=0
# preparation completed, time to setup infrastructure and run tests
if [ "${cluster_count}" -eq "1" ] ; then
local tmp_suffix=${suffix:+"-${suffix}"}
create_infrastructure_and_run_tests "${e2e_dir}${tmp_suffix}" "${ip_family}" "${backend}" \
create_infrastructure_and_run_tests "${e2e_dir}${tmp_suffix}" "${ip_family}" "${backend}" \
"${bin_dir}" "${tmp_suffix}" "${devel_mode}" "${ci_mode}" \
"${deployment_model}" "${export_metrics}"
"${deployment_model}" "${export_metrics}" "${include_specific_failed_tests}"
result=$?
else
local pids

Expand All @@ -859,24 +914,36 @@ function main {
rm -f "${output_file}"
create_infrastructure_and_run_tests "${e2e_dir}${tmp_suffix}" "${ip_family}" "${backend}" \
"${bin_dir}" "${tmp_suffix}" "${devel_mode}" "${ci_mode}" \
"${deployment_model}" "${export_metrics}" \
"${deployment_model}" "${export_metrics}" "${include_specific_failed_tests}" \
&> "${e2e_dir}${tmp_suffix}/output.log" &
pids[${i}]=$!
done
for pid in ${pids[*]}; do # not possible to use quotes here
wait ${pid}
wait "${pid}"
local tmp=$?
if ! [ ${tmp} -eq 0 ] ; then
result=${tmp}
fi
done
if ! ${devel_mode} ; then
print_reports "${ip_family}" "${backend}" "${e2e_dir}" "-${suffix}" "${cluster_count}"
fi
fi

if ${devel_mode} ; then
echo -e "\n+=====================================================================================+"
echo -e "\t\tDeveloper mode no test run!"
echo -e "+=====================================================================================+"
elif ! ${ci_mode} && ${erase_clusters} ; then
elif ! ${ci_mode}; then
delete_kind_clusters "${bin_dir}" "${ip_family}" "${backend}" "${suffix}" "${cluster_count}"
fi

echo -e "\n+=====================================================================================+"
echo -e "\t\tResult"
echo -e "+=====================================================================================+"

result_message ${result} "test_e2e test suite for \"-i ${ip_family} -b ${backend}\""
return ${result}
}


Expand All @@ -892,23 +959,25 @@ function help {
printf "\n"
printf "Usage: %s [-i ip_family] [-b backend]\n" "$0"
printf "\t-i set ip_family(ipv4/ipv6/dual) name in the e2e test runs.\n"
printf "\t-b set backend (iptables/nft/ipvs/ebpf/not-kpng) name in the e2e test runs. \
printf "\t-b set backend (iptables/nft/ipvs/ebpf/userspacelin/not-kpng/) name in the e2e test runs. \
\"not-kpng\" is used to be able to validate and compare results\n"
printf "\t-c flag allows for ci_mode. Please don't run on local systems.\n"
printf "\t-d devel mode, creates the test env but skip e2e tests. Useful for debugging.\n"
printf "\t-e erase kind clusters.\n"
printf "\t-m set the KPNG deployment model, can either be: \n\
* split-process-per-node [legacy/debug] -> (To run KPNG server + client in separate containers/processes per node)
* single-process-per-node [default] -> (To run KPNG server + client in a single container/process per node)\n"
printf "\t-n number of parallel test clusters.\n"
printf "\t-p flag, only print reports.\n"
printf "\t-s suffix, will be appended to the E2@ directory and kind cluster name (makes it possible to run parallel tests.\n"
printf "\t-t Run tests on existing deployment\n"
printf "\t-B binary directory, specifies the path for the directory where binaries will be installed\n"
printf "\t-D Dockerfile, specifies the path of the Dockerfile to use\n"
printf "\t-E set E2E directory, specifies the path for the E2E directory\n"
printf "\t-m set the KPNG deployment model, can either be: \n\
* split-process-per-node [legacy/debug] -> (To run KPNG server + client in separate containers/processes per node)
* single-process-per-node [default] -> (To run KPNG server + client in a single container/process per node)"
printf "\t-t Run tests on existing deployment\n"
printf "\t-I Include failing \"ip_family and backend\" specific test cases\n"
printf "\t-M Configure kpng to export prometheus metrics\n"
printf "\nExample:\n\t %s -i ipv4 -b iptables\n" "${0}"
printf "\t-S Skip the failing \"ip_family and backend\" specific test cases\n"
printf "\nExample:\n\t %s -i ipv4 -b iptables\n" "${0}\n"
exit 1 # Exit script after printing help
}
tmp_dir=$(dirname "$0")
Expand All @@ -924,8 +993,9 @@ erase_clusters=false
print_report=false
deployment_model="single-process-per-node"
export_metrics=false
include_specific_failed_tests=true

while getopts "i:b:B:cdD:eE:n:ps:mt:M" flag
while getopts "b:cdei:n:mps:t:B:D:E:IMS" flag
do
case "${flag}" in
i ) ip_family="${OPTARG}" ;;
Expand All @@ -935,13 +1005,15 @@ do
e ) erase_clusters=true ;;
n ) cluster_count="${OPTARG}" ;;
p ) print_report=true ;;
s ) suffix="${OPTARG}" ;;
m ) deployment_model="${OPTARG}" ;;
s ) suffix="${OPTARG}" ;;
t ) run_tests_on_existing_cluster=true ;;
B ) bin_dir="${OPTARG}" ;;
D ) dockerfile="${OPTARG}" ;;
E ) e2e_dir="${OPTARG}" ;;
m ) deployment_model="${OPTARG}" ;;
t ) run_tests_on_existing_cluster=true ;;
I ) include_specific_failed_tests=true ;;
M ) export_metrics=true ;;
S ) include_specific_failed_tests=false ;;
? ) help ;; #Print help
esac
done
Expand All @@ -963,7 +1035,7 @@ fi
if [[ -n "${ip_family}" && -n "${backend}" ]]; then
main "${ip_family}" "${backend}" "${ci_mode}" "${e2e_dir}" "${bin_dir}" "${dockerfile}" \
"${suffix}" "${cluster_count}" "${erase_clusters}" "${print_report}" "${devel_mode}" \
"${deployment_model}" "${run_tests_on_existing_cluster}" "${export_metrics}"
"${deployment_model}" "${run_tests_on_existing_cluster}" "${export_metrics}" "${include_specific_failed_tests}"
else
printf "Both of '-i' and '-b' must be specified.\n"
help
Expand Down
Loading

0 comments on commit 049d930

Please sign in to comment.