reactor code for e2e

Signed-off-by: duanmengkk <[email protected]>
kosmos-io · May 23, 2024 · 76dc952 · 76dc952
1 parent 4c64abc
commit 76dc952
Show file tree

Hide file tree

Showing 6 changed files with 81 additions and 31 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -98,6 +98,10 @@ jobs:
         uses: actions/setup-go@v4
         with:
           go-version: '1.20'
+      - name: Clear e2e env
+        env:
+          KIND_IMAGE: kindest/node:${{ matrix.k8s }}
+        run:  hack/local-cleanup-kosmos.sh
       - name: Prepare e2e env
         env:
           KIND_IMAGE: kindest/node:${{ matrix.k8s }}

diff --git a/hack/cluster.sh b/hack/cluster.sh
@@ -10,7 +10,7 @@ ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 REUSE=${REUSE:-false}
 VERSION=${VERSION:-latest}
 
-CN_ZONE=${CN_ZONE:-false}
+CN_ZONE=${CN_ZONE:-true}
 source "$(dirname "${BASH_SOURCE[0]}")/util.sh"
 
 # default cert and key for node server https
@@ -23,6 +23,32 @@ fi
 
 source "${ROOT}/hack/util.sh"
 
+# pull e2e test image
+function prepare_test_image() {
+  if [ "${CN_ZONE}" == false ]; then
+        docker pull bitpoke/mysql-operator-orchestrator:v0.6.3
+        docker pull bitpoke/mysql-operator:v0.6.3
+        docker pull bitpoke/mysql-operator-sidecar-5.7:v0.6.3
+        docker pull nginx
+        docker pull percona:5.7
+        docker pull prom/mysqld-exporter:v0.13.0
+  else
+        docker pull docker.m.daocloud.io/bitpoke/mysql-operator-orchestrator:v0.6.3
+        docker pull docker.m.daocloud.io/bitpoke/mysql-operator:v0.6.3
+        docker pull docker.m.daocloud.io/bitpoke/mysql-operator-sidecar-5.7:v0.6.3
+        docker pull docker.m.daocloud.io/nginx
+        docker pull docker.m.daocloud.io/percona:5.7
+        docker pull docker.m.daocloud.io/prom/mysqld-exporter:v0.13.0
+
+        docker tag docker.m.daocloud.io/bitpoke/mysql-operator-orchestrator:v0.6.3 bitpoke/mysql-operator-orchestrator:v0.6.3
+        docker tag docker.m.daocloud.io/bitpoke/mysql-operator:v0.6.3 bitpoke/mysql-operator:v0.6.3
+        docker tag docker.m.daocloud.io/bitpoke/mysql-operator-sidecar-5.7:v0.6.3 bitpoke/mysql-operator-sidecar-5.7:v0.6.3
+        docker tag docker.m.daocloud.io/nginx nginx
+        docker tag docker.m.daocloud.io/percona:5.7 percona:5.7
+        docker tag docker.m.daocloud.io/prom/mysqld-exporter:v0.13.0 prom/mysqld-exporter:v0.13.0
+  fi
+}
+
 # prepare e2e cluster
 function prepare_e2e_cluster() {
   local -r clustername=$1
@@ -60,16 +86,38 @@ function prepare_e2e_cluster() {
 
 # prepare docker image
 function prepare_docker_image() {
-  # pull calico image
-  docker pull calico/apiserver:v3.25.0
-  docker pull calico/cni:v3.25.0
-  docker pull calico/csi:v3.25.0
-  docker pull calico/kube-controllers:v3.25.0
-  docker pull calico/node-driver-registrar:v3.25.0
-  docker pull calico/node:v3.25.0
-  docker pull calico/pod2daemon-flexvol:v3.25.0
-  docker pull calico/typha:v3.25.0
-  docker pull quay.io/tigera/operator:v1.29.0
+  if [ "${CN_ZONE}" == false ]; then
+        # pull calico image
+        docker pull calico/apiserver:v3.25.0
+        docker pull calico/cni:v3.25.0
+        docker pull calico/csi:v3.25.0
+        docker pull calico/kube-controllers:v3.25.0
+        docker pull calico/node-driver-registrar:v3.25.0
+        docker pull calico/node:v3.25.0
+        docker pull calico/pod2daemon-flexvol:v3.25.0
+        docker pull calico/typha:v3.25.0
+        docker pull quay.io/tigera/operator:v1.29.0
+  else
+        docker pull quay.m.daocloud.io/tigera/operator:v1.29.0
+        docker pull docker.m.daocloud.io/calico/apiserver:v3.25.0
+        docker pull docker.m.daocloud.io/calico/cni:v3.25.0
+        docker pull docker.m.daocloud.io/calico/csi:v3.25.0
+        docker pull docker.m.daocloud.io/calico/kube-controllers:v3.25.0
+        docker pull docker.m.daocloud.io/calico/node-driver-registrar:v3.25.0
+        docker pull docker.m.daocloud.io/calico/node:v3.25.0
+        docker pull docker.m.daocloud.io/calico/pod2daemon-flexvol:v3.25.0
+        docker pull docker.m.daocloud.io/calico/typha:v3.25.0
+
+        docker tag quay.m.daocloud.io/tigera/operator:v1.29.0 quay.io/tigera/operator:v1.29.0
+        docker tag docker.m.daocloud.io/calico/apiserver:v3.25.0 calico/apiserver:v3.25.0
+        docker tag docker.m.daocloud.io/calico/cni:v3.25.0 calico/cni:v3.25.0
+        docker tag docker.m.daocloud.io/calico/csi:v3.25.0 calico/csi:v3.25.0
+        docker tag docker.m.daocloud.io/calico/kube-controllers:v3.25.0 calico/kube-controllers:v3.25.0
+        docker tag docker.m.daocloud.io/calico/node-driver-registrar:v3.25.0 calico/node-driver-registrar:v3.25.0
+        docker tag docker.m.daocloud.io/calico/node:v3.25.0 calico/node:v3.25.0
+        docker tag docker.m.daocloud.io/calico/pod2daemon-flexvol:v3.25.0 calico/pod2daemon-flexvol:v3.25.0
+        docker tag docker.m.daocloud.io/calico/typha:v3.25.0 calico/typha:v3.25.0
+  fi
 }
 
 #clustername podcidr servicecidr

diff --git a/hack/local-cleanup-kosmos.sh b/hack/local-cleanup-kosmos.sh
@@ -4,6 +4,8 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+VERSION=${VERSION:-"latest"}
+
 function usage() {
     echo "Usage:"
     echo "    hack/local-down-kosmos.sh [-k] [-h]"
@@ -57,17 +59,16 @@ echo "Remove cluster configs successfully."
 
 #step3. remove docker images
 echo -e "\nStart removing images"
-version="v0.2.0"
 registry="ghcr.io/kosmos-io"
 images=(
-"${registry}/clusterlink-network-manager:${version}"
-"${registry}/clusterlink-controller-manager:${version}"
-"${registry}/clusterlink-elector:${version}"
-"${registry}/clusterlink-operator:${version}"
-"${registry}/clusterlink-agent:${version}"
-"${registry}/clusterlink-proxy:${version}"
-"${registry}/clustertree-cluster-manager:${version}"
-"${registry}/scheduler:${version}"
+"${registry}/clusterlink-network-manager:${VERSION}"
+"${registry}/clusterlink-controller-manager:${VERSION}"
+"${registry}/clusterlink-elector:${VERSION}"
+"${registry}/clusterlink-operator:${VERSION}"
+"${registry}/clusterlink-agent:${VERSION}"
+"${registry}/clusterlink-proxy:${VERSION}"
+"${registry}/clustertree-cluster-manager:${VERSION}"
+"${registry}/scheduler:${VERSION}"
 )
 if [[ "${keep_images}" == "false" ]] ; then
   for ((i=0;i<${#images[*]};i++)); do

diff --git a/hack/local-up-kosmos.sh b/hack/local-up-kosmos.sh
@@ -49,7 +49,7 @@ MEMBER3_CLUSTER_POD_CIDR="10.236.64.0/18"
 MEMBER3_CLUSTER_SERVICE_CIDR="10.236.0.0/18"
 
 REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
-export VERSION="latest"
+VERSION=${VERSION:-"latest"}
 source "$(dirname "${BASH_SOURCE[0]}")/install_kind_kubectl.sh"
 source "$(dirname "${BASH_SOURCE[0]}")/cluster.sh"
 source "$(dirname "${BASH_SOURCE[0]}")/util.sh"
@@ -60,7 +60,7 @@ if [[ -z "${HOST_IPADDRESS}" ]]; then
   util::get_macos_ipaddress # Adapt for macOS
   HOST_IPADDRESS=${MAC_NIC_IPADDRESS:-}
 fi
-make images GOOS="linux" --directory="${REPO_ROOT}"
+make images GOOS="linux" VERSION="$VERSION" --directory="${REPO_ROOT}"
 
 make kosmosctl
 os=$(go env GOOS)

diff --git a/hack/rune2e.sh b/hack/rune2e.sh
@@ -21,12 +21,7 @@ source "${ROOT}/cluster.sh"
 mkdir -p "$ARTIFACTS_PATH"
 
 # pull e2e test image
-docker pull bitpoke/mysql-operator-orchestrator:v0.6.3
-docker pull bitpoke/mysql-operator:v0.6.3
-docker pull bitpoke/mysql-operator-sidecar-5.7:v0.6.3
-docker pull nginx
-docker pull percona:5.7
-docker pull prom/mysqld-exporter:v0.13.0
+prepare_test_image
 
 # prepare for e2e test
 prepare_e2e_cluster "${HOST_CLUSTER_NAME}" &
@@ -71,9 +66,10 @@ util::wait_for_condition "mysql operator are ready" \
 kubectl --kubeconfig "${REPO_ROOT}/environments/${HOST_CLUSTER_NAME}/kubeconfig" apply -f "${REPO_ROOT}"/test/e2e/deploy/cr
 
 util::wait_for_condition "mysql cr are ready" \
-  "[ \$(kubectl --kubeconfig ${REPO_ROOT}/environments/${HOST_CLUSTER_NAME}/kubeconfig get pods -n kosmos-e2e --field-selector=status.phase=Running -l app.kubernetes.io/name=mysql --no-headers | wc -l) -eq 2 ]" \
+  "[ \$(kubectl --kubeconfig ${REPO_ROOT}/environments/${HOST_CLUSTER_NAME}/kubeconfig get pods -n kosmos-e2e -l app.kubernetes.io/name=mysql |grep \"4/4\"| wc -l) -eq 2 ]" \
   1200
 
+
 echo "E2e test of mysql-operator success"
 
 # Install ginkgo

diff --git a/test/e2e/deploy/cr/cluster-cr.yaml b/test/e2e/deploy/cr/cluster-cr.yaml
@@ -76,9 +76,10 @@ spec:
           nodeSelectorTerms:
             - matchExpressions:
                 - key: kubernetes.io/hostname
-                  operator: NotIn
+                  operator: In
                   values:
-                    - cluster-host-control-plane
+                    - kosmos-cluster-member1
+                    - kosmos-cluster-member2
       podAntiAffinity:
         requiredDuringSchedulingIgnoredDuringExecution:
           - labelSelector: