kosmos-io · duanmengkk · Mar 15, 2024 · Mar 13, 2024
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -9,7 +9,7 @@ concurrency:
 jobs:
   verify:
     name: verify
-    runs-on: self-hosted
+    runs-on: [self-hosted, ecs]
     if: ${{ always() }}
     env:
       #GOPATH: ${{ github.workspace }}
@@ -38,7 +38,7 @@ jobs:
   build:
     name: build
     needs: verify
-    runs-on: self-hosted
+    runs-on: [self-hosted, ecs]
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
@@ -54,7 +54,7 @@ jobs:
   test:
     name: Unit test
     needs: build
-    runs-on: self-hosted
+    runs-on: [self-hosted, ecs]
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
@@ -68,7 +68,14 @@ jobs:
     needs: build
     env:
       GOPROXY: "https://goproxy.cn,direct"
-    runs-on: self-hosted
+    runs-on: [self-hosted, ecs]
+    strategy:
+      fail-fast: false
+      matrix:
+        # Here support the latest three minor releases of Kubernetes, this can be considered to be roughly
+        # the same as the End of Life of the Kubernetes release: https://kubernetes.io/releases/
+        # Please remember to update the CI Schedule Workflow when we add a new version.
+        k8s: [ v1.27.2, v1.28.0, v1.29.0 ]
     steps:
       # Free up disk space on Ubuntu
       - name: Free Disk Space (Ubuntu)
@@ -92,12 +99,22 @@ jobs:
         with:
           go-version: '1.20'
       - name: Prepare e2e env
-        run: ./hack/prepare-e2e.sh
+        env:
+          KIND_IMAGE: kindest/node:${{ matrix.k8s }}
+        run:  hack/local-up-kosmos.sh
       - name: Run e2e test
-        run: ./hack/rune2e.sh
+        env:
+          ARTIFACTS_PATH: ${{ github.workspace }}/e2e-test/${{ matrix.k8s }}/
+        run: hack/rune2e.sh
       - name: Upload logs
         uses: actions/upload-artifact@v3
-        if: failure()
+        if: always()
         with:
-          name: kosmos-e2e-logs-${{ github.run_id }}
-          path: ${{ github.workspace }}/e2e-test/logs-*
+          name: kosmos-e2e-logs-${{ github.run_id }}_${{ matrix.k8s }}
+          path: ${{ github.workspace }}/e2e-test/${{ matrix.k8s }}/
+      - name: upload environments logs
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: kosmos_environments_log_${{ matrix.k8s }}
+          path: ${{ github.workspace }}/environments
diff --git a/examples/nginx-demo.yaml b/examples/nginx-demo.yaml
@@ -0,0 +1,25 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+  labels:
+    app: nginx
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      tolerations:
+        - key: kosmos.io/node
+          operator: Equal
+          value: "true"
+      containers:
+        - name: nginx
+          image: nginx:1.14.2
+          ports:
+            - containerPort: 80
diff --git a/hack/cluster.sh b/hack/cluster.sh
diff --git a/hack/clustertemplete/kindconfig b/hack/clustertemplete/kindconfig
@@ -17,4 +17,5 @@ networking:
   ipFamily: __IP_FAMILY__
   disableDefaultCNI: true # disable kindnet
   podSubnet: __POD_CIDR__
-  serviceSubnet: __SERVICE_CIDR__
+  serviceSubnet: __SERVICE_CIDR__
+  apiServerAddress: __HOST_IPADDRESS__
diff --git a/hack/local-cleanup-kosmos.sh b/hack/local-cleanup-kosmos.sh
@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+function usage() {
+    echo "Usage:"
+    echo "    hack/local-down-kosmos.sh [-k] [-h]"
+    echo "Args:"
+    echo "    k: keep the local images"
+    echo "    h: print help information"
+}
+
+keep_images="false"
+while getopts 'kh' OPT; do
+    case $OPT in
+        k) keep_images="true";;
+        h)
+          usage
+          exit 0
+          ;;
+        ?)
+          usage
+          exit 1
+          ;;
+    esac
+done
+HOST_CLUSTER_NAME=${HOST_CLUSTER_NAME:-"cluster-host"}
+MEMBER_CLUSTER_1_NAME=${MEMBER_CLUSTER_1_NAME:-"cluster-member1"}
+MEMBER_CLUSTER_2_NAME=${MEMBER_CLUSTER_2_NAME:-"cluster-member2"}
+MEMBER_CLUSTER_3_NAME=${MEMBER_CLUSTER_3_NAME:-"cluster-member3"}
+#step1 remove kind clusters
+echo -e "\nStart removing kind clusters"
+kind delete cluster --name "${HOST_CLUSTER_NAME}"
+kind delete cluster --name "${MEMBER_CLUSTER_1_NAME}"
+kind delete cluster --name "${MEMBER_CLUSTER_2_NAME}"
+kind delete cluster --name "${MEMBER_CLUSTER_3_NAME}"
+echo "Remove kind clusters successfully."
+
+ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+CLUSTER_DIR="${ROOT}/environments"
+source "${ROOT}/hack/cluster.sh"
+
+#step2. remove kubeconfig
+echo -e "\nStart removing kubeconfig, kindconfig, cailcoconfig"
+HOST_CLUSTER_CONFIG=${HOST_CLUSTER_CONFIG:-"${CLUSTER_DIR}/${HOST_CLUSTER_NAME}"}
+MEMBER1_CLUSTER_CONFIG=${MEMBER_CLUSTER_CONFIG:-"${CLUSTER_DIR}/${MEMBER_CLUSTER_1_NAME}"}
+MEMBER2_CLUSTER_CONFIG=${MEMBER_CLUSTER_CONFIG:-"${CLUSTER_DIR}/${MEMBER_CLUSTER_2_NAME}"}
+MEMBER3_CLUSTER_CONFIG=${MEMBER_CLUSTER_CONFIG:-"${CLUSTER_DIR}/${MEMBER_CLUSTER_3_NAME}"}
+delete_cluster "${HOST_CLUSTER_CONFIG}" "${HOST_CLUSTER_CONFIG}"
+delete_cluster "${MEMBER1_CLUSTER_CONFIG}" "${MEMBER1_CLUSTER_CONFIG}"
+delete_cluster "${MEMBER2_CLUSTER_CONFIG}" "${MEMBER2_CLUSTER_CONFIG}"
+delete_cluster "${MEMBER3_CLUSTER_CONFIG}" "${MEMBER3_CLUSTER_CONFIG}"
+
+echo "Remove cluster configs successfully."
+
+#step3. remove docker images
+echo -e "\nStart removing images"
+version="v0.2.0"
+registry="ghcr.io/kosmos-io"
+images=(
+"${registry}/clusterlink-network-manager:${version}"
+"${registry}/clusterlink-controller-manager:${version}"
+"${registry}/clusterlink-elector:${version}"
+"${registry}/clusterlink-operator:${version}"
+"${registry}/clusterlink-agent:${version}"
+"${registry}/clusterlink-proxy:${version}"
+"${registry}/clustertree-cluster-manager:${version}"
+"${registry}/scheduler:${version}"
+)
+if [[ "${keep_images}" == "false" ]] ; then
+  for ((i=0;i<${#images[*]};i++)); do
+    docker rmi ${images[i]} || true
+  done
+  echo "Remove images successfully."
+else
+  echo "Skip removing images as required."
+fi
+
+echo -e "\nLocal Kosmos is removed successfully."
diff --git a/hack/local-down-clusterlink.sh b/hack/local-down-clusterlink.sh
diff --git a/hack/local-up-clusterlink.sh b/hack/local-up-clusterlink.sh
diff --git a/hack/local-up-kosmos.sh b/hack/local-up-kosmos.sh
@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+
+function usage() {
+    echo "Usage:"
+    echo "    hack/local-up-kosmos.sh [HOST_IPADDRESS] [-h]"
+    echo "Args:"
+    echo "    HOST_IPADDRESS: (required) if you want to export clusters' API server port to specific IP address"
+    echo "    h: print help information"
+}
+
+while getopts 'h' OPT; do
+    case $OPT in
+        h)
+          usage
+          exit 0
+          ;;
+        ?)
+          usage
+          exit 1
+          ;;
+    esac
+done
+
+
+KUBECONFIG_PATH=${KUBECONFIG_PATH:-"${HOME}/.kube"}
+export KUBECONFIG=$KUBECONFIG_PATH/"config"
+
+KIND_IMAGE=${KIND_IMAGE:-"kindest/node:v1.27.2"}
+HOST_IPADDRESS=${1:-}
+HOST_CLUSTER_NAME="cluster-host"
+HOST_CLUSTER_POD_CIDR="10.233.64.0/18"
+HOST_CLUSTER_SERVICE_CIDR="10.233.0.0/18"
+
+MEMBER1_CLUSTER_NAME="cluster-member1"
+MEMBER1_CLUSTER_POD_CIDR="10.234.64.0/18"
+MEMBER1_CLUSTER_SERVICE_CIDR="10.234.0.0/18"
+
+MEMBER2_CLUSTER_NAME="cluster-member2"
+MEMBER2_CLUSTER_POD_CIDR="10.235.64.0/18"
+MEMBER2_CLUSTER_SERVICE_CIDR="10.235.0.0/18"
+
+MEMBER3_CLUSTER_NAME="cluster-member3"
+MEMBER3_CLUSTER_POD_CIDR="10.236.64.0/18"
+MEMBER3_CLUSTER_SERVICE_CIDR="10.236.0.0/18"
+
+REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+export VERSION="latest"
+source "$(dirname "${BASH_SOURCE[0]}")/install_kind_kubectl.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/cluster.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/util.sh"
+
+#step1. create host cluster and member clusters in parallel
+# host IP address: script parameter ahead of macOS IP
+if [[ -z "${HOST_IPADDRESS}" ]]; then
+  util::get_macos_ipaddress # Adapt for macOS
+  HOST_IPADDRESS=${MAC_NIC_IPADDRESS:-}
+fi
+make images GOOS="linux" --directory="${REPO_ROOT}"
+
+make kosmosctl
+os=$(go env GOOS)
+arch=$(go env GOARCH)
+export PATH=$PATH:"${REPO_ROOT}"/_output/bin/"$os"/"$arch"
+
+# prepare docker image
+prepare_docker_image
+
+#cluster cluster concurrent backend
+create_cluster "${KIND_IMAGE}" "$HOST_IPADDRESS" $HOST_CLUSTER_NAME $HOST_CLUSTER_POD_CIDR $HOST_CLUSTER_SERVICE_CIDR &
+create_cluster "${KIND_IMAGE}" "$HOST_IPADDRESS" $MEMBER1_CLUSTER_NAME $MEMBER1_CLUSTER_POD_CIDR $MEMBER1_CLUSTER_SERVICE_CIDR false &
+create_cluster "${KIND_IMAGE}" "$HOST_IPADDRESS" $MEMBER2_CLUSTER_NAME $MEMBER2_CLUSTER_POD_CIDR $MEMBER2_CLUSTER_SERVICE_CIDR false &
+create_cluster "${KIND_IMAGE}" "$HOST_IPADDRESS" $MEMBER3_CLUSTER_NAME $MEMBER3_CLUSTER_POD_CIDR $MEMBER3_CLUSTER_SERVICE_CIDR false &
+
+# wait for finish
+wait
+
+#deploy cluster concurrent backend
+deploy_cluster_by_ctl $HOST_CLUSTER_NAME "${REPO_ROOT}/environments/${HOST_CLUSTER_NAME}/kubeconfig" "${REPO_ROOT}/environments/${HOST_CLUSTER_NAME}/kubeconfig-nodeIp" &
+load_cluster_images $MEMBER1_CLUSTER_NAME &
+load_cluster_images $MEMBER2_CLUSTER_NAME &
+load_cluster_images $MEMBER3_CLUSTER_NAME &
+
+# wait for finish
+wait
+
+#join cluster
+join_cluster_by_ctl $HOST_CLUSTER_NAME $MEMBER1_CLUSTER_NAME "${REPO_ROOT}/environments/${HOST_CLUSTER_NAME}" "${REPO_ROOT}/environments/${MEMBER1_CLUSTER_NAME}"
+join_cluster_by_ctl $HOST_CLUSTER_NAME $MEMBER2_CLUSTER_NAME "${REPO_ROOT}/environments/${HOST_CLUSTER_NAME}" "${REPO_ROOT}/environments/${MEMBER2_CLUSTER_NAME}"
+join_cluster_by_ctl $HOST_CLUSTER_NAME $MEMBER3_CLUSTER_NAME "${REPO_ROOT}/environments/${HOST_CLUSTER_NAME}" "${REPO_ROOT}/environments/${MEMBER3_CLUSTER_NAME}"
+
+#add leafnode test taint
+addTaint $HOST_CLUSTER_NAME $MEMBER3_CLUSTER_NAME