From 9fef3e3bcf848e01bcddc8ab83552251435c1128 Mon Sep 17 00:00:00 2001 From: Congrool Date: Thu, 30 Sep 2021 12:36:31 +0800 Subject: [PATCH] add e2e tests into github action --- .github/workflows/ci.yaml | 24 +++ Makefile | 3 + hack/kind-config-template.yaml | 11 ++ hack/lib/release-images.sh | 20 ++- hack/local_up_openyurt.sh | 286 +++++++++++++++++++++++++++++++++ hack/run-e2e-tests.sh | 39 +++++ 6 files changed, 380 insertions(+), 3 deletions(-) create mode 100644 hack/kind-config-template.yaml create mode 100644 hack/local_up_openyurt.sh create mode 100644 hack/run-e2e-tests.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 49622456b72..301d9624fbe 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -112,3 +112,27 @@ jobs: restore-keys: ${{ runner.os }}-go- - name: Build All Binaries run: make build + e2e-tests: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + - uses: actions/cache@v2 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: ${{ runner.os }}-go- + + - name: Install Required Commands + run: | + go get sigs.k8s.io/kind@v0.11.1 + curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.20.7/bin/linux/amd64/kubectl && sudo install kubectl /usr/local/bin/kubectl + - name: Local Up Openyurt Cluster With Kind + run: bash hack/local_up_openyurt.sh + + - name: Run e2e Tests + run: make e2e-tests diff --git a/Makefile b/Makefile index cd4256f7920..ecd3c02d60d 100644 --- a/Makefile +++ b/Makefile @@ -62,3 +62,6 @@ clean: e2e: hack/make-rules/build-e2e.sh + +e2e-tests: + bash hack/run-e2e-tests.sh \ No newline at end of file diff --git a/hack/kind-config-template.yaml b/hack/kind-config-template.yaml new file mode 100644 index 00000000000..8d3637dc8ce --- /dev/null +++ b/hack/kind-config-template.yaml @@ -0,0 +1,11 @@ +apiVersion: kind.x-k8s.io/v1alpha4 +kind: Cluster +name: openyurt-e2e-test +nodes: + - role: control-plane + image: |fill image here| + extraMounts: + - hostPath: |fill local bin dir| + containerPath: /root/openyurt + - role: worker + image: |fill image here| \ No newline at end of file diff --git a/hack/lib/release-images.sh b/hack/lib/release-images.sh index 0ca6668c0d5..ae2bc42bb1c 100644 --- a/hack/lib/release-images.sh +++ b/hack/lib/release-images.sh @@ -39,11 +39,25 @@ readonly -a SUPPORTED_ARCH=( readonly SUPPORTED_OS=linux -readonly -a bin_targets=(${WHAT:-${YURT_BIN_TARGETS[@]}}) +readonly -a bin_targets=(${WHAT[@]:-${YURT_BIN_TARGETS[@]}}) readonly -a bin_targets_process_servant=("${bin_targets[@]/yurtctl-servant/yurtctl}") -readonly -a target_arch=(${ARCH:-${SUPPORTED_ARCH[@]}}) +readonly -a target_arch=(${ARCH[@]:-${SUPPORTED_ARCH[@]}}) readonly region=${REGION:-us} +# Parameters +# $1: component name +# $2: arch +function get_image_name { + # If ${GIT_COMMIT} is not at a tag, add commit to the image tag. + if [[ -z $(git tag --points-at ${GIT_COMMIT}) ]]; then + yurt_component_image="${REPO}/$1:${TAG}-$2-$(expr substr ${GIT_COMMIT} 1 7)" + else + yurt_component_image="${REPO}/$1:${TAG}-$2" + fi + + echo ${yurt_component_image} +} + function build_multi_arch_binaries() { local docker_run_opts=( "-i" @@ -123,7 +137,7 @@ ENTRYPOINT ["/usr/local/bin/${binary_name}"] EOF fi - yurt_component_image="${REPO}/${yurt_component_name}:${TAG}-${arch}" + yurt_component_image=$(get_image_name ${yurt_component_name} ${arch}) ln "${binary_path}" "${docker_build_path}/${binary_name}" docker build --no-cache -t "${yurt_component_image}" -f "${docker_file_path}" ${docker_build_path} docker save ${yurt_component_image} > ${YURT_IMAGE_DIR}/${yurt_component_name}-${SUPPORTED_OS}-${arch}.tar diff --git a/hack/local_up_openyurt.sh b/hack/local_up_openyurt.sh new file mode 100644 index 00000000000..1e564c3934c --- /dev/null +++ b/hack/local_up_openyurt.sh @@ -0,0 +1,286 @@ +#!/usr/bin/env bash + +# Copyright 2020 The OpenYurt Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# This shell will create a openyurt cluster locally with kind. The yurt-tunnel will be +# automatically deployed, and the autonomous mode will be active. +# +# It uses the following env variables: +# REGION +# Affect the GOPROXY. You can set it to "cn" to use GOPROXY="https://goproxy.cn". +# Default value is "us", which means using GOPROXY="https://goproxy.io" +# The default value is "us". +# +# KIND_KUBECONFIG +# KIND_KUBECONFIG represents the path to store the kubeconfig file of the cluster +# which is created by this shell. The default value is "$HOME/.kube/config" +# +# NODES_NUM +# NODES_NUM represents the number of nodes to be included in the new-created cluster. +# There are one control-plane node and NODES_NUM-1 worker nodes. Thus, NODES_NUM must +# be greater than 2. The default value is 2. +# +# KUBERNETESVERSION +# It declares the kubernetes version the cluster will use. The format is "1.XX". +# Now only 1.18, 1.19 and 1.20 are supported. The default value is 1.20 +# +# TIMEOUT +# TIMEOUT represents the time to wait for the kind control-plane, yurt-tunnel and +# yurt-server to be ready. If it is not ready after the duration, the shell will exit. +# The default value is 120s. +# +# YURTTUNNEL +# If set YURTTUNNEL=disable, the yurt-tunnel-agent and yurt-tunnel-server will not be +# deployed in the openyurt cluster. +# The default value is "enable". + + +set -x +set -e +set -u + +YURT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" +KIND_KUBECONFIG=${KIND_KUBECONFIG:-${HOME}/.kube/config} + +readonly KIND_NODE_IMAGES=( + kindest/node:v1.18.19@sha256:7af1492e19b3192a79f606e43c35fb741e520d195f96399284515f077b3b622c + kindest/node:v1.19.11@sha256:07db187ae84b4b7de440a73886f008cf903fcf5764ba8106a9fd5243d6f32729 + kindest/node:v1.20.7@sha256:cbeaf907fc78ac97ce7b625e4bf0de16e3ea725daf6b04f930bd14c67c671ff9 +) + +readonly REQUIRED_CMD=( + go + docker + kubectl + kind +) + +readonly BUILD_TARGETS=( + yurthub + yurt-controller-manager + yurtctl + yurt-tunnel-server + yurt-tunnel-agent +) + +readonly LOCAL_ARCH=$(go env GOHOSTARCH) +readonly LOCAL_OS=$(go env GOHOSTOS) +readonly IMAGES_DIR=${YURT_ROOT}/_output/images +readonly CLUSTER_NAME="openyurt-e2e-test" +readonly TIMEOUT=${TIMEOUT:-"120s"} +readonly KUBERNETESVERSION=${KUBERNETESVERSION:-"1.20"} +readonly KIND_CONFIG=${YURT_ROOT}/_output/kind-config-v${KUBERNETESVERSION}.yaml +readonly NODES_NUM=${NODES_NUM:-2} +readonly KIND_CONTEXT="kind-${CLUSTER_NAME}" + +master= +edgenodes= + +# $1 string to escape +function escape_slash { + echo $1 | sed "s/\//\\\\\//g" +} + +function gen_kind_config { + # check if number of nodes is valid + if [[ ${NODES_NUM} -lt 2 ]]; then + echo "NODES_NUM should be greater than 2" + exit -1 + fi + + # check if kubernetes version is valid + local k8s_version=($(echo ${KUBERNETESVERSION} | sed "s/\./ /g")) + local major=${k8s_version[0]} + local minor=${k8s_version[1]} + if [[ ${major} -ne 1 ]] || [[ ${minor} -gt 20 ]] || [[ ${minor} -lt 18 ]]; then + echo "Invalid KUBERNETESVERSION, it should be between 1.18 and 1.20." + exit -1 + fi + + # create and init kind config file + local gen_config_path=${KIND_CONFIG} + cat ${YURT_ROOT}/hack/kind-config-template.yaml > ${gen_config_path} + + # add additional node spec into kind config + for ((count=2; count<${NODES_NUM}; count++)); do + echo -e "\n - role: worker\n image: |fill image here|" >> ${gen_config_path} + done + + # fill name:tag of images and fill bin dir + local bindir=$(escape_slash ${YURT_LOCAL_BIN_DIR}/${LOCAL_OS}/${LOCAL_ARCH}) + local node_image=$(escape_slash $(echo ${KIND_NODE_IMAGES[${minor}-18]})) + sed -i "s/image: |fill image here|$/image: ${node_image}/g + s/- hostPath: |fill local bin dir|/- hostPath: ${bindir}/g" \ + ${gen_config_path} +} + +function install_kind { + echo "Begin to install kind" + GO111MODULE="on" go get sigs.k8s.io/kind@v0.11.1 +} + +function install_docker { + echo "docker should be installed first" + return -1 +} + +function install_kubectl { + echo "kubectl should be installed first" + return -1 +} + +function install_go { + echo "go should be installed first" + return -1 +} + +function preflight { + echo "Preflight Check..." + for bin in "${REQUIRED_CMD[@]}"; do + command -v ${bin} > /dev/null 2>&1 + if [[ $? -ne 0 ]]; then + echo "Cannot find command ${bin}." + install_${bin} + if [[ $? -ne 0 ]]; then + echo "Error occurred, exit" + exit -1 + fi + fi + done +} + +function build_target_binaries_and_images { + echo "Begin to build binaries and images" + + export WHAT=${BUILD_TARGETS[@]} + export ARCH=${LOCAL_ARCH} + + source ${YURT_ROOT}/hack/make-rules/release-images.sh +} + +function kind_load_images { + local postfix="${LOCAL_OS}-${LOCAL_ARCH}.tar" + + for bin in ${BUILD_TARGETS[@]}; do + local imagename="${bin}-${postfix}" + + if [[ "${bin}" = "yurtctl" ]]; then + imagename="yurtctl-servant-${postfix}" + fi + + echo "loading image ${imagename} to nodes" + local nodesarg=$(echo "${master} ${edgenodes[@]}" | sed "s/ /,/g") + kind load image-archive ${IMAGES_DIR}/${imagename} \ + --name ${CLUSTER_NAME} --nodes ${nodesarg} + done +} + +function local_up_cluster { + echo "Creating kubernetes cluster with ${NODES_NUM} nodes" + gen_kind_config + + # output kind config before kind cmd for the convenience of debugging + echo $(cat ${KIND_CONFIG}) + kind create cluster --config ${KIND_CONFIG} + + echo "Waiting for the control-plane ready..." + kubectl wait --for=condition=Ready node/${CLUSTER_NAME}-control-plane --context ${KIND_CONTEXT} --timeout=${TIMEOUT} + master=$(kubectl get node -A -o custom-columns=NAME:.metadata.name --context ${KIND_CONTEXT} | grep control-plane) + edgenodes=$(kubectl get node -A -o custom-columns=NAME:.metadata.name --context ${KIND_CONTEXT} | grep work) +} + +# $1 yurt-tunnel-agent image with the format of "name:tag" +function deploy_yurt_tunnel_agent { + local yurt_tunnel_agent_yaml=${YURT_OUTPUT_DIR}/yurt-tunnel-agent.yaml + # revise the yurt-tunnel-agent.yaml + cat ${YURT_ROOT}/config/setup/yurt-tunnel-agent.yaml | + sed "s/image: openyurt\/yurt-tunnel-agent:latest/image: $(escape_slash ${1})/" | + tee ${yurt_tunnel_agent_yaml} + + kubectl apply -f ${yurt_tunnel_agent_yaml} --context ${KIND_CONTEXT} +} + +# $1 yurt-tunnel-server image with the format of "name:tag" +function deploy_yurt_tunnel_server { + local yurt_tunnel_server_yaml=${YURT_OUTPUT_DIR}/yurt-tunnel-server.yaml + # revise the yurt-tunnel-server.yaml + cat ${YURT_ROOT}/config/setup/yurt-tunnel-server.yaml | + sed "s/image: openyurt\/yurt-tunnel-server:latest/image: $(escape_slash ${1})/; + s/imagePullPolicy: Always/imagePullPolicy: IfNotPresent/" | + tee ${yurt_tunnel_server_yaml} + + kubectl apply -f ${yurt_tunnel_server_yaml} --context ${KIND_CONTEXT} +} + +function deploy_yurt_tunnel { + deploy_yurt_tunnel_agent $(get_image_name "yurt-tunnel-agent" ${LOCAL_ARCH}) + deploy_yurt_tunnel_server $(get_image_name "yurt-tunnel-server" ${LOCAL_ARCH}) +} + +function convert_to_openyurt { + echo "converting to openyurt cluster " + + local yurtctl_dir="/root/openyurt" + # local yurtctl_dir=${YURT_LOCAL_BIN_DIR}/${LOCAL_OS}/${LOCAL_ARCH} + docker exec ${master} \ + ${yurtctl_dir}/yurtctl convert --provider kubeadm --cloud-nodes ${master} \ + --yurthub-image=$(get_image_name "yurthub" ${LOCAL_ARCH}) \ + --yurt-controller-manager-image=$(get_image_name "yurt-controller-manager" ${LOCAL_ARCH}) \ + --yurtctl-servant-image=$(get_image_name "yurtctl-servant" ${LOCAL_ARCH}) + # --yurt-tunnel-server-image=$(get_image_name "yurt-tunnel-server" ${LOCAL_ARCH}) \ + # --yurt-tunnel-agent-image=$(get_image_name "yurt-tunnel-agent" ${LOCAL_ARCH}) + + local nodearg=$(echo ${edgenodes[@]} | sed "s/ /,/g") + docker exec ${master} \ + ${yurtctl_dir}/yurtctl markautonomous -a "${nodearg}" + + if [ ! "${YURTTUNNEL:-"enable"}" = "disable" ]; then + deploy_yurt_tunnel + kubectl rollout status deploy yurt-tunnel-server -n kube-system \ + --timeout=${TIMEOUT} --context ${KIND_CONTEXT} + kubectl rollout status daemonset yurt-tunnel-agent -n kube-system \ + --timeout=${TIMEOUT} --context ${KIND_CONTEXT} + fi +} + +function get_kubeconfig { + mkdir -p ${HOME}/.kube + kind get kubeconfig --name ${CLUSTER_NAME} > ${KIND_KUBECONFIG} +} + +function cleanup { + rm -rf ${YURT_ROOT}/_output + rm -rf ${YURT_ROOT}/dockerbuild + rm -f ${KIND_CONFIG} + kind delete clusters ${CLUSTER_NAME} +} + +function cleanup_on_err { + if [[ $? -ne 0 ]]; then + cleanup + fi +} + + +trap cleanup_on_err EXIT + +cleanup +preflight +build_target_binaries_and_images +local_up_cluster +kind_load_images +convert_to_openyurt +get_kubeconfig \ No newline at end of file diff --git a/hack/run-e2e-tests.sh b/hack/run-e2e-tests.sh new file mode 100644 index 00000000000..bd5b5f577b3 --- /dev/null +++ b/hack/run-e2e-tests.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# Copyright 2020 The OpenYurt Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +set -e +set -u + +YURT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P) +source ${YURT_ROOT}/hack/make-rules/build-e2e.sh + +KUBECONFIG=${KUBECONFIG:-${HOME}/.kube/config} + +# run e2e tests +function run_e2e_tests { + # check kubeconfig + if [ ! -f "${KUBECONFIG}" ]; then + echo "kubeconfig does not exist at ${KUBECONFIG}" + exit -1 + fi + + local target_bin_dir=$(get_binary_dir_with_arch ${YURT_LOCAL_BIN_DIR}) + local e2e_test_file_name=$(basename ${YURT_E2E_TARGETS}) + ${target_bin_dir}/${e2e_test_file_name} -kubeconfig ${KUBECONFIG} +} + +run_e2e_tests \ No newline at end of file