diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5390f9dbe..ac5c259e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,6 +98,10 @@ jobs: uses: actions/setup-go@v4 with: go-version: '1.20' + - name: Clear e2e env + env: + KIND_IMAGE: kindest/node:${{ matrix.k8s }} + run: hack/local-cleanup-kosmos.sh - name: Prepare e2e env env: KIND_IMAGE: kindest/node:${{ matrix.k8s }} diff --git a/hack/cluster.sh b/hack/cluster.sh index 1406104d6..8b4fd5046 100755 --- a/hack/cluster.sh +++ b/hack/cluster.sh @@ -10,7 +10,7 @@ ROOT=$(dirname "${BASH_SOURCE[0]}")/.. REUSE=${REUSE:-false} VERSION=${VERSION:-latest} -CN_ZONE=${CN_ZONE:-false} +CN_ZONE=${CN_ZONE:-true} source "$(dirname "${BASH_SOURCE[0]}")/util.sh" # default cert and key for node server https @@ -23,6 +23,32 @@ fi source "${ROOT}/hack/util.sh" +# pull e2e test image +function prepare_test_image() { + if [ "${CN_ZONE}" == false ]; then + docker pull bitpoke/mysql-operator-orchestrator:v0.6.3 + docker pull bitpoke/mysql-operator:v0.6.3 + docker pull bitpoke/mysql-operator-sidecar-5.7:v0.6.3 + docker pull nginx + docker pull percona:5.7 + docker pull prom/mysqld-exporter:v0.13.0 + else + docker pull docker.m.daocloud.io/bitpoke/mysql-operator-orchestrator:v0.6.3 + docker pull docker.m.daocloud.io/bitpoke/mysql-operator:v0.6.3 + docker pull docker.m.daocloud.io/bitpoke/mysql-operator-sidecar-5.7:v0.6.3 + docker pull docker.m.daocloud.io/nginx + docker pull docker.m.daocloud.io/percona:5.7 + docker pull docker.m.daocloud.io/prom/mysqld-exporter:v0.13.0 + + docker tag docker.m.daocloud.io/bitpoke/mysql-operator-orchestrator:v0.6.3 bitpoke/mysql-operator-orchestrator:v0.6.3 + docker tag docker.m.daocloud.io/bitpoke/mysql-operator:v0.6.3 bitpoke/mysql-operator:v0.6.3 + docker tag docker.m.daocloud.io/bitpoke/mysql-operator-sidecar-5.7:v0.6.3 bitpoke/mysql-operator-sidecar-5.7:v0.6.3 + docker tag docker.m.daocloud.io/nginx nginx + docker tag docker.m.daocloud.io/percona:5.7 percona:5.7 + docker tag docker.m.daocloud.io/prom/mysqld-exporter:v0.13.0 prom/mysqld-exporter:v0.13.0 + fi +} + # prepare e2e cluster function prepare_e2e_cluster() { local -r clustername=$1 @@ -60,16 +86,38 @@ function prepare_e2e_cluster() { # prepare docker image function prepare_docker_image() { - # pull calico image - docker pull calico/apiserver:v3.25.0 - docker pull calico/cni:v3.25.0 - docker pull calico/csi:v3.25.0 - docker pull calico/kube-controllers:v3.25.0 - docker pull calico/node-driver-registrar:v3.25.0 - docker pull calico/node:v3.25.0 - docker pull calico/pod2daemon-flexvol:v3.25.0 - docker pull calico/typha:v3.25.0 - docker pull quay.io/tigera/operator:v1.29.0 + if [ "${CN_ZONE}" == false ]; then + # pull calico image + docker pull calico/apiserver:v3.25.0 + docker pull calico/cni:v3.25.0 + docker pull calico/csi:v3.25.0 + docker pull calico/kube-controllers:v3.25.0 + docker pull calico/node-driver-registrar:v3.25.0 + docker pull calico/node:v3.25.0 + docker pull calico/pod2daemon-flexvol:v3.25.0 + docker pull calico/typha:v3.25.0 + docker pull quay.io/tigera/operator:v1.29.0 + else + docker pull quay.m.daocloud.io/tigera/operator:v1.29.0 + docker pull docker.m.daocloud.io/calico/apiserver:v3.25.0 + docker pull docker.m.daocloud.io/calico/cni:v3.25.0 + docker pull docker.m.daocloud.io/calico/csi:v3.25.0 + docker pull docker.m.daocloud.io/calico/kube-controllers:v3.25.0 + docker pull docker.m.daocloud.io/calico/node-driver-registrar:v3.25.0 + docker pull docker.m.daocloud.io/calico/node:v3.25.0 + docker pull docker.m.daocloud.io/calico/pod2daemon-flexvol:v3.25.0 + docker pull docker.m.daocloud.io/calico/typha:v3.25.0 + + docker tag quay.m.daocloud.io/tigera/operator:v1.29.0 quay.io/tigera/operator:v1.29.0 + docker tag docker.m.daocloud.io/calico/apiserver:v3.25.0 calico/apiserver:v3.25.0 + docker tag docker.m.daocloud.io/calico/cni:v3.25.0 calico/cni:v3.25.0 + docker tag docker.m.daocloud.io/calico/csi:v3.25.0 calico/csi:v3.25.0 + docker tag docker.m.daocloud.io/calico/kube-controllers:v3.25.0 calico/kube-controllers:v3.25.0 + docker tag docker.m.daocloud.io/calico/node-driver-registrar:v3.25.0 calico/node-driver-registrar:v3.25.0 + docker tag docker.m.daocloud.io/calico/node:v3.25.0 calico/node:v3.25.0 + docker tag docker.m.daocloud.io/calico/pod2daemon-flexvol:v3.25.0 calico/pod2daemon-flexvol:v3.25.0 + docker tag docker.m.daocloud.io/calico/typha:v3.25.0 calico/typha:v3.25.0 + fi } #clustername podcidr servicecidr diff --git a/hack/local-cleanup-kosmos.sh b/hack/local-cleanup-kosmos.sh index 400c30f8f..b3aebfffd 100755 --- a/hack/local-cleanup-kosmos.sh +++ b/hack/local-cleanup-kosmos.sh @@ -4,6 +4,8 @@ set -o errexit set -o nounset set -o pipefail +VERSION=${VERSION:-"latest"} + function usage() { echo "Usage:" echo " hack/local-down-kosmos.sh [-k] [-h]" @@ -57,17 +59,16 @@ echo "Remove cluster configs successfully." #step3. remove docker images echo -e "\nStart removing images" -version="v0.2.0" registry="ghcr.io/kosmos-io" images=( -"${registry}/clusterlink-network-manager:${version}" -"${registry}/clusterlink-controller-manager:${version}" -"${registry}/clusterlink-elector:${version}" -"${registry}/clusterlink-operator:${version}" -"${registry}/clusterlink-agent:${version}" -"${registry}/clusterlink-proxy:${version}" -"${registry}/clustertree-cluster-manager:${version}" -"${registry}/scheduler:${version}" +"${registry}/clusterlink-network-manager:${VERSION}" +"${registry}/clusterlink-controller-manager:${VERSION}" +"${registry}/clusterlink-elector:${VERSION}" +"${registry}/clusterlink-operator:${VERSION}" +"${registry}/clusterlink-agent:${VERSION}" +"${registry}/clusterlink-proxy:${VERSION}" +"${registry}/clustertree-cluster-manager:${VERSION}" +"${registry}/scheduler:${VERSION}" ) if [[ "${keep_images}" == "false" ]] ; then for ((i=0;i<${#images[*]};i++)); do diff --git a/hack/local-up-kosmos.sh b/hack/local-up-kosmos.sh index 7dbf93937..91e519fe2 100755 --- a/hack/local-up-kosmos.sh +++ b/hack/local-up-kosmos.sh @@ -49,7 +49,7 @@ MEMBER3_CLUSTER_POD_CIDR="10.236.64.0/18" MEMBER3_CLUSTER_SERVICE_CIDR="10.236.0.0/18" REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. -export VERSION="latest" +VERSION=${VERSION:-"latest"} source "$(dirname "${BASH_SOURCE[0]}")/install_kind_kubectl.sh" source "$(dirname "${BASH_SOURCE[0]}")/cluster.sh" source "$(dirname "${BASH_SOURCE[0]}")/util.sh" @@ -60,7 +60,7 @@ if [[ -z "${HOST_IPADDRESS}" ]]; then util::get_macos_ipaddress # Adapt for macOS HOST_IPADDRESS=${MAC_NIC_IPADDRESS:-} fi -make images GOOS="linux" --directory="${REPO_ROOT}" +make images GOOS="linux" VERSION="$VERSION" --directory="${REPO_ROOT}" make kosmosctl os=$(go env GOOS) diff --git a/hack/rune2e.sh b/hack/rune2e.sh index 0bb5a54d1..9dd53f4dc 100755 --- a/hack/rune2e.sh +++ b/hack/rune2e.sh @@ -21,12 +21,7 @@ source "${ROOT}/cluster.sh" mkdir -p "$ARTIFACTS_PATH" # pull e2e test image -docker pull bitpoke/mysql-operator-orchestrator:v0.6.3 -docker pull bitpoke/mysql-operator:v0.6.3 -docker pull bitpoke/mysql-operator-sidecar-5.7:v0.6.3 -docker pull nginx -docker pull percona:5.7 -docker pull prom/mysqld-exporter:v0.13.0 +prepare_test_image # prepare for e2e test prepare_e2e_cluster "${HOST_CLUSTER_NAME}" & @@ -71,9 +66,10 @@ util::wait_for_condition "mysql operator are ready" \ kubectl --kubeconfig "${REPO_ROOT}/environments/${HOST_CLUSTER_NAME}/kubeconfig" apply -f "${REPO_ROOT}"/test/e2e/deploy/cr util::wait_for_condition "mysql cr are ready" \ - "[ \$(kubectl --kubeconfig ${REPO_ROOT}/environments/${HOST_CLUSTER_NAME}/kubeconfig get pods -n kosmos-e2e --field-selector=status.phase=Running -l app.kubernetes.io/name=mysql --no-headers | wc -l) -eq 2 ]" \ + "[ \$(kubectl --kubeconfig ${REPO_ROOT}/environments/${HOST_CLUSTER_NAME}/kubeconfig get pods -n kosmos-e2e -l app.kubernetes.io/name=mysql |grep \"4/4\"| wc -l) -eq 2 ]" \ 1200 + echo "E2e test of mysql-operator success" # Install ginkgo diff --git a/test/e2e/deploy/cr/cluster-cr.yaml b/test/e2e/deploy/cr/cluster-cr.yaml index fb29e252a..117ab0abf 100644 --- a/test/e2e/deploy/cr/cluster-cr.yaml +++ b/test/e2e/deploy/cr/cluster-cr.yaml @@ -76,9 +76,10 @@ spec: nodeSelectorTerms: - matchExpressions: - key: kubernetes.io/hostname - operator: NotIn + operator: In values: - - cluster-host-control-plane + - kosmos-cluster-member1 + - kosmos-cluster-member2 podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: