Update test scripts for Cluster

README.md

@@ -396,12 +396,14 @@ Check out [CONTRIBUTING](https://github.com/cloud-barista/cb-tumblebug/blob/main
       ├── deploy-dragonfly-docker.sh  # MCIS에 CB-Dragonfly 자동 배포 및 환경 자동 설정      
       ├── clean-all.sh  # 모든 오브젝트를 생성의 역순으로 삭제
       ├── create-cluster-only.sh # testSet에 지정된 멀티 클라우드 인프라를 대상으로 K8s 클러스터를 생성
+      ├── get-cluster.sh # testSet에 지정된 멀티 클라우드 인프라를 대상으로 K8s 클러스터 정보를 얻음
       ├── clean-cluster-only.sh # testSet에 지정된 멀티 클라우드 인프라를 대상으로 K8s 클러스터를 삭제
+      ├── force-clean-cluster-only.sh # testSet에 지정된 멀티 클라우드 인프라를 대상으로 K8s 클러스터를 강제 삭제
       ├── add-nodegroup.sh # 생성된 K8s 클러스터에 신규 노드그룹을 생성
       ├── remove-nodegroup.sh # 생성된 K8s 클러스터에 생성된 신규 노드그룹을 삭제
       ├── set-nodegroup-autoscaling.sh # 생성된 노드그룹의 오토스케일링 설정값을 off로 변경
       ├── change-nodegroup-autoscalesize.sh # 생성된 노드그룹의 오토스케일 크기를 변경
-      ├── deploy-wavescope-to-cluster.sh # 생성된 K8s 클러스터에 weavescope를 배포
+      ├── deploy-weavescope-to-cluster.sh # 생성된 K8s 클러스터에 weavescope를 배포
       └── executionStatus  # 수행이 진행된 테스트 로그 (testAll 수행시 정보가 추가되며, cleanAll 수행시 정보가 제거됨. 진행중인 작업 확인 가능)
 
   ```
@@ -463,13 +465,15 @@ Check out [CONTRIBUTING](https://github.com/cloud-barista/cb-tumblebug/blob/main
   - K8s 클러스터 테스트
     - `initMultiCloudEnv.sh`를 사전 실행함을 가정
     - `./create-mcir-ns-cloud.sh -n tb -f ../testSet.env`  # K8s 클러스터 생성에 필요한 MCIR 생성
-    - `./create-cluster-only.sh -n tb -f ../testSet.env -x 1`  # K8s 클러스터를 생성(-x 최대노드수)
-    - `./add-nodegroup.sh -n tb -f ../testSet.env -x 1`  # K8s 클러스터에 신규 노드 그룹 추가
-    - `./change-nodegroup-autoscalesize.sh -n tb -f ../testSet.env -x 1`  # 신규 노드 그룹의 autoscale size를 기존+1로 변경
+    - `./create-cluster-only.sh -n tb -f ../testSet.env -x 1 -z 1`  # K8s 클러스터를 생성(-x 최대노드수 -z 노드그룹 및 클러스터 추가 이름)
+    - `./get-cluster.sh -n tb -f ../testSet.env -z 1`  # K8s 클러스터 정보 얻기
+    - `./add-nodegroup.sh -n tb -f ../testSet.env -x 1 -z 1`  # K8s 클러스터에 신규 노드 그룹 추가
+    - `./change-nodegroup-autoscalesize.sh -n tb -f ../testSet.env -x 1 -z 1`  # 신규 노드 그룹의 autoscale size를 기존+1로 변경
     - `./deploy-weavescope-to-cluster.sh -n tb -f ../testSet.env -y n`  # 생성한 클러스터에 weascope를 배포
-    - `./set-nodegroup-autoscaling.sh -n tb -f ../testSet.env`  # 신규 노드그룹의 autosacling 설정값을 off로 변경
-    - `./remove-nodegroup.sh -n tb -f ../testSet.env`  # 신규 생성한 노드그룹을 삭제
-    - `./clean-cluster-only.sh -n tb -f ../testSet.env`  # 생성한 K8s 클러스터를 삭제
+    - `./set-nodegroup-autoscaling.sh -n tb -f ../testSet.env -z 1`  # 신규 노드그룹의 autosacling 설정값을 off로 변경
+    - `./remove-nodegroup.sh -n tb -f ../testSet.env -z 1`  # 신규 생성한 노드그룹을 삭제
+    - `./clean-cluster-only.sh -n tb -f ../testSet.env -z 1`  # 생성한 K8s 클러스터를 삭제
+    - `./force-clean-cluster-only.sh -n tb -f ../testSet.env -z 1`  # 생성한 K8s 클러스터를 삭제가 되지 않는 경우 강제 삭제
     - `./clean-mcir-ns-cloud.h -n tb -f ../testSet.env`  # 생성한 MCIR 삭제
 
     

src/testclient/scripts/1.configureSpider/register-cloud-interactive.sh

@@ -140,7 +140,7 @@ EOF
 }
 
 echo "####################################################################"
-echo "## 1. Register Cloud Inforation"
+echo "## 1. Register Cloud Information"
 echo "####################################################################"
 
 SCRIPT_DIR=`dirname ${BASH_SOURCE[0]-$0}`

src/testclient/scripts/13.cluster/add-nodegroup.sh

@@ -18,18 +18,17 @@ else
         RootDiskSize="${DISK_SIZE[$INDEX,$REGION]}"
 fi
 
-if [ "$CSP" == "azure" ]; then
-	NODEGROUPNAME="new${INDEX}${REGION}"
-	NODEIMAGEID=""
-else
-	#NODEGROUPNAME="${CONN_CONFIG[$INDEX,$REGION]}-${POSTFIX}"
-	NODEGROUPNAME="new${INDEX}${REGION}"
-	NODEIMAGEID="${CONN_CONFIG[$INDEX,$REGION]}-${POSTFIX}"
-fi 
+NUMVM=${OPTION01:-1}
+CLUSTERID_ADD=${OPTION03:-1}
 
-CLUSTERID=${CLUSTERID_PREFIX}${INDEX}${REGION}
+CLUSTERID=${CLUSTERID_PREFIX}${INDEX}${REGION}${CLUSTERID_ADD}
 
-NUMVM=${OPTION01:-1}
+NODEGROUPNAME="ng${INDEX}${REGION}${CLUSTERID_ADD}"
+if [ -n "${CONTAINER_IMAGE_NAME[$INDEX,$REGION]}" ]; then
+	NODEIMAGEID="k8s-${CONN_CONFIG[$INDEX,$REGION]}-${POSTFIX}"
+else
+	NODEIMAGEID="${CONN_CONFIG[$INDEX,$REGION]}-${POSTFIX}"
+fi
 
 DesiredNodeSize=$NUMVM
 MinNodeSize="1"
@@ -41,6 +40,8 @@ echo "NSID=${NSID}"
 echo "INDEX=${INDEX}"
 echo "REGION=${REGION}"
 echo "POSTFIX=${POSTFIX}"
+echo "NAME=${NODEGROUPNAME}"
+echo "IMAGEID=${NODEIMAGEID}"
 echo "RootDiskType=${RootDiskType}"
 echo "RootDiskSize=${RootDiskSize}"
 echo "DesiredNodeSize=${DesiredNodeSize}"

src/testclient/scripts/13.cluster/change-autoscalesize.sh

@@ -6,15 +6,11 @@ echo "####################################################################"
 
 source ../init.sh
 
-if [ "$CSP" == "azure" ]; then
-	NODEGROUPNAME="new${INDEX}${REGION}"
-else
-	NODEGROUPNAME="new${INDEX}${REGION}"
-fi 
-
-CLUSTERID=${CLUSTERID_PREFIX}${INDEX}${REGION}
-
 NUMVM=${OPTION01:-1}
+CLUSTERID_ADD=${OPTION03:-1}
+
+CLUSTERID=${CLUSTERID_PREFIX}${INDEX}${REGION}${CLUSTERID_ADD}
+NODEGROUPNAME="ng${INDEX}${REGION}${CLUSTERID_ADD}"
 
 DesiredNodeSize=$(($NUMVM+1))
 MinNodeSize=$(($NUMVM+1))

src/testclient/scripts/13.cluster/create-cluster.sh

@@ -18,24 +18,24 @@ else
         RootDiskSize="${DISK_SIZE[$INDEX,$REGION]}"
 fi
 
-# Set NODEGROUPNAME, NODEIMAGEID for each CSP
-if [ "$CSP" == "azure" ]; then
-	VERSION=${OPTION02:-1.25.11}
-	NODEGROUPNAME="ng${INDEX}${REGION}"
-	NODEIMAGEID=""
-elif [ "$CSP" == "tencent" ]; then
-	VERSION=${OPTION02:-1.26.1}
-	NODEGROUPNAME="${CONN_CONFIG[$INDEX,$REGION]}-${POSTFIX}"
-	NODEIMAGEID=""
+NODEGROUPNAME="ng${INDEX}${REGION}${CLUSTERID_ADD}"
+if [ -n "${CONTAINER_IMAGE_NAME[$INDEX,$REGION]}" ]; then
+	NODEIMAGEID="k8s-${CONN_CONFIG[$INDEX,$REGION]}-${POSTFIX}"
 else
-	NODEGROUPNAME="${CONN_CONFIG[$INDEX,$REGION]}-${POSTFIX}"
 	NODEIMAGEID="${CONN_CONFIG[$INDEX,$REGION]}-${POSTFIX}"
-fi 
+fi
 
-CLUSTERID=${CLUSTERID_PREFIX}${INDEX}${REGION}
+if [ -n "${K8S_VERSION[$INDEX,$REGION]}" ]; then
+	VERSION=${K8S_VERSION[$INDEX,$REGION]}
+else
+	echo "You need to specify K8S_VERION[\$IX,\$IY] in conf.env!!!"
+	exit
+fi
 
 NUMVM=${OPTION01:-1}
-VERSION=${OPTION02:-1.25.11}
+CLUSTERID_ADD=${OPTION03:-1}
+
+CLUSTERID=${CLUSTERID_PREFIX}${INDEX}${REGION}${CLUSTERID_ADD}
 
 DesiredNodeSize=$NUMVM
 MinNodeSize="1"
@@ -47,17 +47,21 @@ echo "NSID=${NSID}"
 echo "INDEX=${INDEX}"
 echo "REGION=${REGION}"
 echo "POSTFIX=${POSTFIX}"
+echo "NAME=${NODEGROUPNAME}"
+echo "IMAGEID=${NODEIMAGEID}"
 echo "RootDiskType=${RootDiskType}"
 echo "RootDiskSize=${RootDiskSize}"
 echo "DesiredNodeSize=${DesiredNodeSize}"
 echo "MinNodeSize=${MinNodeSize}"
 echo "MaxNodeSize=${MaxNodeSize}"
+echo "VERSION=${VERSION}"
 echo "CLUSTERID=${CLUSTERID}"
 echo "===================================================================="
 
 # Set NodeGroupList for Type-I and Type-II CSP
 # https://github.com/cloud-barista/cb-spider/wiki/Provider-Managed-Kubernetes-and-Driver-API#3-%EB%93%9C%EB%9D%BC%EC%9D%B4%EB%B2%84-%EA%B0%9C%EB%B0%9C-%EB%85%B8%ED%8A%B8
-if [ "$CSP" == "tencent"  ]|| [ "$CSP" == "alibaba" ]; then # Type-I CSP
+
+if [ "${ClusterType[$INDEX]}" == "type1"  ]; then # Type-I CSP
     	NODEGROUPLIST=""
 else # Type-II CSP
 	NODEGROUPLIST=$(cat <<-END

src/testclient/scripts/13.cluster/delete-cluster.sh

@@ -6,7 +6,9 @@ echo "####################################################################"
 
 source ../init.sh
 
-CLUSTERID=${CLUSTERID_PREFIX}${INDEX}${REGION}
+CLUSTERID_ADD=${OPTION03:-1}
+
+CLUSTERID=${CLUSTERID_PREFIX}${INDEX}${REGION}${CLUSTERID_ADD}
 
 echo "NSID: "${NSID}
 echo "CLUSTERID=${CLUSTERID}"

src/testclient/scripts/13.cluster/force-delete-cluster.sh

@@ -6,7 +6,9 @@ echo "####################################################################"
 
 source ../init.sh
 
-CLUSTERID=${CLUSTERID_PREFIX}${INDEX}${REGION}
+CLUSTERID_ADD=${OPTION03:-1}
+
+CLUSTERID=${CLUSTERID_PREFIX}${INDEX}${REGION}${CLUSTERID_ADD}
 
 echo "NSID: "${NSID}
 echo "CLUSTERID=${CLUSTERID}"

src/testclient/scripts/13.cluster/remove-nodegroup-with-force.sh → src/testclient/scripts/13.cluster/force-remove-nodegroup.sh

@@ -6,14 +6,10 @@ echo "####################################################################"
 
 source ../init.sh
 
-if [ "$CSP" == "azure" ]; then
-	NODEGROUPNAME="new${INDEX}${REGION}"
-else
-	#NODEGROUPNAME="${CONN_CONFIG[$INDEX,$REGION]}-${POSTFIX}"
-	NODEGROUPNAME="new${INDEX}${REGION}"
-fi 
+CLUSTERID_ADD=${OPTION03:-1}
 
-CLUSTERID=${CLUSTERID_PREFIX}${INDEX}${REGION}
+CLUSTERID=${CLUSTERID_PREFIX}${INDEX}${REGION}${CLUSTERID_ADD}
+NODEGROUPNAME="ng${INDEX}${REGION}${CLUSTERID_ADD}"
 
 echo "NSID: "${NSID}
 echo "CLUSTERID=${CLUSTERID}"

src/testclient/scripts/13.cluster/get-cluster.sh

@@ -6,7 +6,9 @@ echo "####################################################################"
 
 source ../init.sh
 
-CLUSTERID=${CLUSTERID_PREFIX}${INDEX}${REGION}
+CLUSTERID_ADD=${OPTION03:-1}
+
+CLUSTERID=${CLUSTERID_PREFIX}${INDEX}${REGION}${CLUSTERID_ADD}
 
 echo "NSID: "${NSID}
 echo "CLUSTERID=${CLUSTERID}"

src/testclient/scripts/13.cluster/set-autoscaling.sh

@@ -6,14 +6,10 @@ echo "####################################################################"
 
 source ../init.sh
 
-if [ "$CSP" == "azure" ]; then
-	NODEGROUPNAME="new${INDEX}${REGION}"
-else
-	#NODEGROUPNAME="${CONN_CONFIG[$INDEX,$REGION]}-${POSTFIX}"
-	NODEGROUPNAME="new${INDEX}${REGION}"
-fi 
+CLUSTERID_ADD=${OPTION03:-1}
 
-CLUSTERID=${CLUSTERID_PREFIX}${INDEX}${REGION}
+CLUSTERID=${CLUSTERID_PREFIX}${INDEX}${REGION}${CLUSTERID_ADD}
+NODEGROUPNAME="ng${INDEX}${REGION}${CLUSTERID_ADD}"
 
 echo "NSID: "${NSID}
 echo "CLUSTERID=${CLUSTERID}"

src/testclient/scripts/13.cluster/spider-delete-cluster-with-force.sh

@@ -6,7 +6,9 @@ echo "####################################################################"
 
 source ../init.sh
 
-CLUSTERID=${CLUSTERID_PREFIX}${INDEX}${REGION}
+CLUSTERID_ADD=${OPTION03:-1}
+
+CLUSTERID=${CLUSTERID_PREFIX}${INDEX}${REGION}${CLUSTERID_ADD}
 
 echo "NSID: "${NSID}
 echo "CLUSTERID=${CLUSTERID}"

src/testclient/scripts/13.cluster/spider-delete-cluster.sh

@@ -6,7 +6,9 @@ echo "####################################################################"
 
 source ../init.sh
 
-CLUSTERID=${CLUSTERID_PREFIX}${INDEX}${REGION}
+CLUSTERID_ADD=${OPTION03:-1}
+
+CLUSTERID=${CLUSTERID_PREFIX}${INDEX}${REGION}${CLUSTERID_ADD}
 
 echo "NSID: "${NSID}
 echo "CLUSTERID=${CLUSTERID}"

src/testclient/scripts/13.cluster/spider-get-cluster.sh

@@ -6,7 +6,9 @@ echo "####################################################################"
 
 source ../init.sh
 
-CLUSTERID=${CLUSTERID_PREFIX}${INDEX}${REGION}
+CLUSTERID_ADD=${OPTION03:-1}
+
+CLUSTERID=${CLUSTERID_PREFIX}${INDEX}${REGION}${CLUSTERID_ADD}
 
 echo "NSID: "${NSID}
 echo "CLUSTERID=${CLUSTERID}"

src/testclient/scripts/13.cluster/status-cluster.sh

@@ -9,8 +9,9 @@ exit 1
 
 source ../init.sh
 
+CLUSTERID_ADD=${OPTION03:-1}
 
-CLUSTERID=${CLUSTERID_PREFIX}${INDEX}${REGION}
+CLUSTERID=${CLUSTERID_PREFIX}${INDEX}${REGION}${CLUSTERID_ADD}
 
 echo "NSID: "${NSID}
 echo "CLUSTERID=${CLUSTERID}"

src/testclient/scripts/13.cluster/upgrade-cluster.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+echo "####################################################################"
+echo "## 13. CLUSTER: Upgrade"
+echo "####################################################################"
+
+source ../init.sh
+
+<<COMMENT
+if [ "$CSP" == "azure" ]; then
+	VERSION=${OPTION02:-1.25.11}
+elif [ "$CSP" == "tencent" ]; then
+	#VERSION=${OPTION02:-1.20.6}
+	VERSION=${OPTION02:-1.22.5}
+	#VERSION=${OPTION02:-1.24.4}
+	#VERSION=${OPTION02:-1.26.1}
+elif [ "$CSP" == "alibaba" ]; then
+	#VERSION=${OPTION02:-1.24.6-aliyun.1}
+	#VERSION=${OPTION02:-1.26.3-aliyun.1}
+	VERSION=${OPTION02:-1.28.3-aliyun.1}
+elif [ "$CSP" == "nhncloud" ]; then
+	#VERSION=${OPTION02:-1.24.3}
+	#VERSION=${OPTION02:-1.25.4}
+	#VERSION=${OPTION02:-1.26.3}
+	VERSION=${OPTION02:-v1.27.3}
+else
+	VERSION=${OPTION02:-1.25.11}
+fi 
+COMMENT
+
+if [ -n "${K8S_UPGRADE_VERSION[$INDEX,$REGION]}" ]; then
+	VERSION=${K8S_UPGRADE_VERSION[$INDEX,$REGION]}
+else
+	echo "You need to specify K8S_UPGRADE_VERION[\$IX,\$IY] in conf.env!!!"
+	exit
+fi
+
+CLUSTERID_ADD=${OPTION03:-1}
+
+CLUSTERID=${CLUSTERID_PREFIX}${INDEX}${REGION}${CLUSTERID_ADD}
+
+echo "NSID: "${NSID}
+echo "CLUSTERID=${CLUSTERID}"
+
+resp=$(
+	curl -H "${AUTH}" -sX PUT http://$TumblebugServer/tumblebug/ns/$NSID/cluster/${CLUSTERID}/upgrade -H 'Content-Type: application/json' -d @- <<EOF
+	{
+		"version": "${VERSION}"
+	}
+EOF
+    ); echo ${resp} | jq ''
+    echo ""
+

src/testclient/scripts/6.image/registerImageWithId.sh

@@ -10,11 +10,28 @@ function CallTB() {
 			"name": "${CONN_CONFIG[$INDEX,$REGION]}-${POSTFIX}",
 			"cspImageId": "${IMAGE_NAME[$INDEX,$REGION]}",
 			"description": "Canonical, Ubuntu, 18.04 LTS, amd64 bionic",
-            "guestOS": "Ubuntu"
+			"guestOS": "Ubuntu"
 		}
 EOF
-    ); echo ${resp} | jq ''
-    echo ""
+	); echo ${resp} | jq ''
+	echo ""
+
+	if [ -n "${CONTAINER_IMAGE_NAME[$INDEX,$REGION]}" ]; then
+	    echo "- Register K8s node image in ${MCIRRegionName}"
+
+	    resp=$(
+	    curl -H "${AUTH}" -sX POST http://$TumblebugServer/tumblebug/ns/$NSID/resources/image?action=registerWithId -H 'Content-Type: application/json' -d @- <<EOF
+		    { 
+			    "connectionName": "${CONN_CONFIG[$INDEX,$REGION]}", 
+			    "name": "k8s-${CONN_CONFIG[$INDEX,$REGION]}-${POSTFIX}",
+			    "cspImageId": "${CONTAINER_IMAGE_NAME[$INDEX,$REGION]}",
+			    "description": "${CONTAINER_IMAGE_TYPE[$INDEX,$REGION]}",
+			    "guestOS": "Ubuntu"
+		    }
+EOF
+	    ); echo ${resp} | jq ''
+	fi
+
 }
 
 #function registerImageWithId() {
@@ -35,8 +52,10 @@ EOF
             for ((cspj = 1; cspj <= INDEXY; cspj++)); do
                 echo "[$cspi,$cspj] ${RegionName[$cspi,$cspj]}"
 
-				MCIRRegionName=${RegionName[$cspi,$cspj]}
+				MCIRRegionName=${CONN_CONFIG[$cspi,$cspj]}
 
+				INDEX=$cspi
+				REGION=$cspj
 				CallTB
 
 			done

src/testclient/scripts/6.image/unregister-image.sh

@@ -4,6 +4,10 @@ function CallTB() {
 	echo "- Unregister image in ${MCIRRegionName}"
 
 	curl -H "${AUTH}" -sX DELETE http://$TumblebugServer/tumblebug/ns/$NSID/resources/image/${CONN_CONFIG[$INDEX,$REGION]}-${POSTFIX} | jq ''
+
+	if [ -n "${CONTAINER_IMAGE_NAME[$INDEX,$REGION]}" ]; then
+		curl -H "${AUTH}" -sX DELETE http://$TumblebugServer/tumblebug/ns/$NSID/resources/image/k8s-${CONN_CONFIG[$INDEX,$REGION]}-${POSTFIX} | jq ''
+	fi
 }
 
 #function unregister_image() {