From da7d60bbd949aa1c8839936440998c38af99ee3d Mon Sep 17 00:00:00 2001
From: chen hui <huchen@vmware.com>
Date: Tue, 15 Feb 2022 09:32:05 +0000
Subject: [PATCH 01/14] More automation in getting_started.md guide

Signed-off-by: chen hui <huchen@vmware.com>
---
 scripts/getting_started.sh | 583 +++++++++++++++++++++++++++++++++++++
 1 file changed, 583 insertions(+)
 create mode 100755 scripts/getting_started.sh

diff --git a/scripts/getting_started.sh b/scripts/getting_started.sh
new file mode 100755
index 000000000..cde400ddd
--- /dev/null
+++ b/scripts/getting_started.sh
@@ -0,0 +1,583 @@
+#!/bin/bash
+
+function isCmdInstalled() {
+    cmd=$1
+    ${cmd} version 2>&1 | grep -q "not found"
+    returnCode=$?  
+    if [ ${returnCode} -eq 0 ] ; then
+        echo "$cmd not found"
+    else
+        echo "$cmd found"
+    fi
+    return ${returnCode}
+}
+
+
+function exitIfNot(){
+    code=$1
+    expectedCode=$2
+    prompt=$3
+    if [ ${code} == ${expectedCode} ]; then
+        echo ${prompt}
+        exit
+    fi
+}
+
+function runCmd(){
+    cmd=$1
+    expectCode=$2
+    promptBeforeCmd=$3
+    PromptAfterCmdSuc=$4
+    promptAfterCmdFail=$5
+
+    if [ -n "${promptBeforeCmd}" ]; then
+        echo ${promptBeforeCmd}
+    fi
+
+    # run command
+    ${cmd}
+    returnCode=$?
+    if [ ${returnCode} -ne ${expectCode} ]; then
+        if [ -n "${promptAfterCmdFail}" ]; then
+            echo ${promptAfterCmdFail}
+        fi
+        echo "${cmd} failed, exit...."
+        exit
+    fi
+
+    if [ -n "${PromptAfterCmdSuc}" ]; then
+        echo ${PromptAfterCmdSuc}
+    fi
+}
+
+function runCmdUntil() {
+    cmd=$1
+    expectCode=$2
+    waitTime=$3
+    maxRunTimes=$4
+    promptBeforeCmd=$5
+    PromptAfterCmdSuc=$6
+    promptAfterCmdFail=$7
+
+    echo "huchen: cmd is ${cmd}"
+
+    if [ -n "${promptBeforeCmd}" ]; then
+        echo ${promptBeforeCmd}
+    fi
+    for((i=1;i<=${maxRunTimes};i++));  
+    do   
+        ${cmd}
+        returnCode=$?
+        if [ ${returnCode} -ne ${expectCode} ]; then
+            sleep ${waitTime}
+            continue
+        else   
+            if [ -n "${PromptAfterCmdSuc}" ]; then
+                echo ${PromptAfterCmdSuc}
+            fi
+            return
+        fi
+        
+    done
+    if [ -n "${promptAfterCmdFail}" ]; then
+        echo ${promptAfterCmdFail}
+    fi
+    exit
+}
+
+
+
+
+
+function isIPOccupied() {
+    ip=$1
+    occupiedIPList=$2
+    echo "Checking if ${ip} is occupied"
+
+    echo "huchen: occupiedIPList is ${occupiedIPList}"
+    IFS=$'\n'
+    for occupiedIPWithMask in ${occupiedIPList}
+    do
+        echo "huchen: occupiedIPWithMask is ${occupiedIPWithMask}"
+        occupiedIP=`echo $occupiedIPWithMask | sed -En 's/^(.*)\/([0-9]{1,2})/\1/p'`
+        echo "huchen: ip is ${ip}, occupiedIP is ${occupiedIP}"
+        if [ "${ip}" == "${occupiedIP}" ]; then
+             return 1
+        fi
+    done
+    echo "${ip} is not occupied"
+    return 0
+}
+
+
+function num2IP() {
+    num=$1
+    a=$((num>>24))
+    b=$((num>>16&0xff))
+    c=$((num>>8&0xff))
+    d=$((num&0xff))
+    echo "$a.$b.$c.$d"
+}  
+
+function binary2IP() {
+    num=`echo "ibase=2;  $1" | bc`
+    returnValue=$(num2IP $num)
+    echo ${returnValue}
+}
+
+
+function calcControlPlaneIP() {
+    subNet=`docker network inspect kind | jq -r 'map(.IPAM.Config[].Subnet) []'`
+    occupiedIPList=`docker network inspect kind | jq -r 'map(.Containers[].IPv4Address) []'`
+
+    for line in ${subNet}
+    do
+        echo ${line} | grep ":"
+        if [ $? -ne 0 ]; then
+            IPMask=${line}
+            break
+        fi
+    done
+
+    echo "IPMask is ${subNet}"
+    echo "occupiedIPList:${occupiedIPList}"
+
+    ip=`echo $IPMask | sed -En 's/^(.*)\/([0-9]{1,2})/\1/p'`
+    ipSubNetBit=`echo $IPMask | sed -En 's/^(.*)\/([0-9]{1,2})/\2/p'`
+    ipSubHostBit=$[32-${ipSubNetBit}]
+
+    IFS=.
+    hexIP=`for str in ${ip}; do printf "%02X" $str; done`
+    binaryIP=`echo "ibase=16; obase=2; ${hexIP}" | bc`
+
+    ipSubNet=`echo ${binaryIP:0:${ipSubNetBit}}`
+
+    full0="00000000000000000000000000000010"
+    full1="11111111111111111111111111111110"
+
+    minIPBinary=${ipSubNet}`echo ${full0:${ipSubNetBit}}`
+    maxIPBinary=${ipSubNet}`echo ${full1:${ipSubNetBit}}`
+
+    minIPint=`echo "ibase=2; ${minIPBinary}"|bc`
+    maxIPint=`echo "ibase=2; ${maxIPBinary}"|bc`
+
+
+    for((i=${minIPint};i<=${maxIPint};i++));  
+    do
+        ip=$(num2IP ${i})
+
+        isIPOccupied "${ip}" "${occupiedIPList}"
+        if [ $? == 0 ]; then
+            CONTROL_PLANE_ENDPOINT_IP=${ip}
+            echo "Available IP is ${CONTROL_PLANE_ENDPOINT_IP}"
+            return
+        fi
+    done
+}
+
+function configClusterctl() {
+    # Write clusterctl.yaml
+    writeByoh=0
+    writeCertManager=0
+    clusterCtlYamlFile="${HOME}/.cluster-api/clusterctl.yaml"
+    if [ ! -f "${clusterCtlYamlFile}" ]; then
+        writeByoh=1
+        writeCertManager=1
+        touch ${clusterCtlYamlFile}
+    else
+        grep -q "byoh" ${clusterCtlYamlFile}
+        if [ $? -ne 0 ] ; then
+            writeByoh=1
+        fi
+        grep -q "cert-manager" ${clusterCtlYamlFile}
+        if [ $? -ne 0 ] ; then
+            writeCertManager=1
+        fi
+    fi 
+
+    # if cert-manager missing,it report error: Error: action failed after 10 attempts: failed to get cert-manager object /, Kind=, /: Object 'Kind' is missing in 'unstructured object has no kind'
+    if [ ${writeCertManager} -eq 1 ] ; then
+        cat>>${clusterCtlYamlFile}<<EOF
+cert-manager:
+  url: "https://github.com/cert-manager/cert-manager/releases/latest/cert-manager.yaml"
+EOF
+    fi
+
+    if [ ${writeCertManager} -eq 1 ] ; then
+        cat>>${clusterCtlYamlFile}<<EOF
+providers:
+  - name: byoh
+    url: https://github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/releases/latest/infrastructure-components.yaml
+    type: InfrastructureProvider
+EOF
+    fi
+
+    clusterctl config repositories | grep -q byoh
+    if [ $? -ne 0 ] ; then
+        echo "Config clusterctl failed..."
+        exit
+    fi
+}
+
+function runByohAgent(){
+    index=$1
+    byohBinaryFile=$2
+    manageClusterConfFile=$3
+
+    runCmd "docker cp ${byohBinaryFile} host${index}:/byoh-hostagent" 0 "Copying agent binary to byoh container: host${index}..."
+    runCmd "docker cp ${manageClusterConfFile} host${index}:/management-cluster.conf" 0 "Copying kubeconfig to byoh container: host${index}..."
+    docker exec -d host$i sh -c "chmod +x /byoh-hostagent && /byoh-hostagent --kubeconfig management-cluster.conf --skip-installation > /agent.log 2>&1"
+    if [ $? -ne 0 ] ; then
+        echo "Starting the host${index} agent..."
+        exit
+    fi
+
+    
+    maxRunTimes=10
+    waitTime=1
+    for((i=1;i<=${maxRunTimes};i++));  
+    do   
+        kubectl get byohosts host${index} | grep -v NAME | grep -q host${index}
+        if [ $? -ne 0 ]; then
+            sleep ${waitTime}
+            continue
+        else   
+            echo "byohost object(host${index}) is created successfully..."
+            return
+        fi
+    done
+
+    echo "Error: byohost object(host${index}) is created failed..."
+    exit
+
+}
+
+function installDocker() {
+    cmdName="docker"
+
+    ## check  if denpency is installed before
+    isCmdInstalled "${cmdName}"
+
+    if [ $? -ne 0 ]; then
+        return
+    fi
+
+    ## install it if it not installed
+    echo "Installing docker...."
+    runCmd "sudo apt update" 0
+    runCmd "sudo apt-get install -y apt-transport-https ca-certificates curl software-properties-common" 0
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+    sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu  $(lsb_release -cs)  stable"
+    runCmd "sudo apt-get install -y docker-ce" 0
+
+    ## check  if denpency is installed successfully
+    isCmdInstalled  "${cmdName}"
+    exitIfNot $? 0 "Installing ${cmdName} failed, exit..."
+
+
+    sudo systemctl enable docker
+
+}
+
+function enableDocker() {
+
+    runCmd "sudo systemctl start docker" 0
+    runCmd "sudo systemctl enable docker" 0
+
+    sudo systemctl status docker | grep -q "active (running)"
+    if [ $? -ne 0 ]; then
+        echo "Enable docker failed"
+        exit 
+    fi
+    echo "Enable docker success"
+
+    # Make sure current user has permission for docker, do this if not Create the docker group.
+    docker ps 2>&1 | grep -q "connect: permission denied"
+    if [ $? -eq 0 ]; then
+
+        grep -q "docker" /etc/group 
+        if [ $? -ne 0 ]; then
+            runCmd "sudo groupadd docker" 0
+        else
+            echo "group 'docker' already exists"
+        fi
+        USER=`whoami`
+        # Add your user to the docker group.
+        runCmd "sudo usermod -aG docker ${USER}" 0 "Add ${USER} to docker group"
+        echo "You would need to log out and log back in so that your group membership is re-evaluated. Rerun this script after that."
+        exit
+    fi
+    echo "current user has permission for docker"
+}
+
+function installKind() {
+
+    cmdName="kind"
+     ## check  if denpency is installed before
+    isCmdInstalled "${cmdName}"
+
+    ## install denpency if it not installed
+    if [ $? -eq 0 ] ; then
+        echo "Installing ${cmdName}..."
+        curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.11.1/kind-linux-amd64 && sudo install kind /usr/local/bin/kind
+        ## check  if denpency is installed successfully
+        isCmdInstalled  "${cmdName}"
+        exitIfNot $? 0 "Installing ${cmdName} failed, exit..."
+    fi
+}
+
+function installClusterctl() {
+
+    cmdName="clusterctl"
+     ## check  if denpency is installed before
+    isCmdInstalled "${cmdName}"
+
+    ## install denpency if it not installed
+    if [ $? -eq 0 ] ; then
+        echo "Installing ${cmdName}..."
+        curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.0/clusterctl-linux-amd64 -o clusterctl && sudo install clusterctl /usr/local/bin/clusterctl
+        ## check  if denpency is installed successfully
+        isCmdInstalled  "${cmdName}"
+        exitIfNot $? 0 "Installing ${cmdName} failed, exit..."
+    fi
+}
+
+
+function intallDependencies(){
+    installDocker
+    enableDocker
+    installKind
+    installClusterctl
+    commonInstall jq "sudo apt install -y jq"
+    commonInstall kubectl "sudo snap install kubectl --classic"
+    commonInstall make "sudo apt install -y build-essential"
+    commonInstall go "sudo snap install go --classic"
+}
+
+function commonInstall(){
+    cmdName=$1
+    installCmd=$2
+
+    ## check  if denpency is installed before
+    isCmdInstalled "${cmdName}"
+
+    ## install denpency if it not installed
+    if [ $? -eq 0 ] ; then
+        runCmd "${installCmd}" 0 "Installing ${cmdName}..."
+
+        ## check  if denpency is installed successfully
+        isCmdInstalled  "${cmdName}"
+        exitIfNot $? 0 "Installing ${cmdName} failed, exit..."
+    fi
+}
+
+function createKindCluster(){
+    echo  "Creating kind cluster..."
+
+    msg=$(kind create cluster 2>&1)
+    if [ $? -ne 0 ]; then
+        echo "Create kind cluster failed"
+        echo $msg | grep -q "You have reached your pull rate limit"
+        if [ $? -eq 0 ]; then 
+            echo "Suggestion: you can use \"docker login\" to avoid such an error."
+            exit
+        fi
+    else
+        echo "Create kind cluster successfully"
+    fi
+}
+
+function downloadByohCode(){
+    runCmd "rm -rf cluster-api-provider-bringyourownhost" 0 " Cleaning byoh code..."
+    msg=$(git clone git@github.com:vmware-tanzu/cluster-api-provider-bringyourownhost.git 2>&1)
+    if [ $? -ne 0 ]; then
+        echo "Downloading byoh code..."
+        echo $msg | grep "Please make sure you have the correct access rights"
+        if [ $? -eq 0 ]; then 
+            echo "Suggestion: Add an public key of this machine into \"SSH and GPG keys\" of your github setting"
+            exit
+        fi
+    else
+        echo "Download byoh code successfully"
+    fi
+}
+
+export PATH=/snap/bin:${PATH}
+intallDependencies 
+ 
+runCmd "sudo swapoff -a" 0 "Turning off swap..."
+swapMsg=$(sudo swapon -s)
+if [ -n "${swapMsg}" ]; then
+    echo "Please turn off swap first."
+    exit
+fi
+ 
+# check if cluster "kind" is already exited
+clusterName=`kind get clusters`
+if [ "${clusterName}" != "kind" ]; then
+    #runCmd "kind create cluster" 0 "Creating kind cluster..."
+    createKindCluster
+fi
+
+configClusterctl
+
+
+# check if init it before
+kubectl get pods --all-namespaces | grep -q byoh-controller-manager
+if [ $? -ne 0 ]; then
+    runCmd "clusterctl init --infrastructure byoh" 0 " Transforming the Kubernetes cluster into a management cluster..."
+else
+    echo "clusterctl init --infrastructure byoh before"
+fi
+
+#check if byoh image is existed
+docker images | grep "byoh-dev/node" | grep -q "v1.22.3"
+if [ $? -ne 0 ]; then
+    #runCmd "rm -rf cluster-api-provider-bringyourownhost" 0 " Cleaning byoh code..."
+    #runCmd "git clone git@github.com:vmware-tanzu/cluster-api-provider-bringyourownhost.git" 0 " Downloading byoh code..."
+    downloadByohCode
+    runCmd "cd cluster-api-provider-bringyourownhost" 0
+    # The origin one will report error:   Could not connect to apt.kubernetes.io:443 (10.25.207.164), connection timed out [IP: 10.25.207.164 443]
+    echo "deb http://packages.cloud.google.com/apt/ kubernetes-xenial main" > test/e2e/kubernetes.list
+    runCmd "make prepare-byoh-docker-host-image-dev" 0  "Making a byoh image: byoh-dev/node:v1.22.3 ..."
+    runCmd "cd -" 0
+else
+    echo "byoh image \"byoh-dev/node:v1.22.3\" existed."
+fi
+
+#check if byoh binary is existed
+byohBinaryFile="/tmp/byoh-hostagent-linux-amd64"
+if [ ! -f "${byohBinaryFile}" ]; then
+    runCmd "wget https://github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/releases/download/v0.1.0/byoh-hostagent-linux-amd64 -P /tmp" 0  "Downloading a byoh binary..."
+else
+    echo "${byohBinaryFile} existed."
+fi
+
+
+manageClusterConfFile="${HOME}/.kube/management-cluster.conf"
+if [ ! -f "${manageClusterConfFile}" ]; then
+    cp ${HOME}/.kube/config ${manageClusterConfFile}
+else
+    echo "${manageClusterConfFile} existed."
+fi
+
+KIND_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' kind-control-plane) 
+
+grep ${KIND_IP} ${manageClusterConfFile} | grep -q 6443
+if [ $? -ne 0 ]; then
+    sed -i 's/    server\:.*/    server\: https\:\/\/'"${KIND_IP}"'\:6443/g' ${manageClusterConfFile}
+else
+    echo "Already modified ${manageClusterConfFile} before"
+fi
+
+
+## Register BYOH host to management cluster
+for i in {1..2}
+do
+  #Check if container "host$i" is already created.
+  docker ps -a | grep -q "host${i}"
+  if [ $? -ne 0 ]; then
+    runCmd "docker run --detach --tty --hostname host${i} --name host${i} --privileged --security-opt seccomp=unconfined --tmpfs /tmp --tmpfs /run --volume /var --volume /lib/modules:/lib/modules:ro --network kind byoh-dev/node:v1.22.3" 0 "Starting byoh container: host${i}..."
+    runByohAgent ${i} ${byohBinaryFile} ${manageClusterConfFile}
+  else
+    echo "Container \"host${i}\" is already created"
+
+    # check if docker status is valid
+    status=`docker container inspect -f '{{.State.Status}}' host${i}`
+    if [ "${status}" != "running" ]; then
+        echo "Error: Status of Container \"host${i}\" is ${status}, suggest to remove it, then rerun this script"
+        exit
+    fi
+
+    # check if byoh process running in this container
+    docker exec host${i} sh -c "ps aux" | grep -q byoh-hostagent
+    if [ $? -ne 0 ]; then
+        runByohAgent ${i} ${byohBinaryFile} ${manageClusterConfFile}
+    else
+        echo "byoh process is already ran in this container"
+    fi
+  fi
+done
+
+# Find a available IP for control plane endpoint
+CONTROL_PLANE_ENDPOINT_IP=""
+calcControlPlaneIP
+if [ -z "${CONTROL_PLANE_ENDPOINT_IP}" ]; then
+    echo "Can't get an available IP for control plane endpoint, exit...."
+    exit
+fi
+echo "CONTROL_PLANE_ENDPOINT_IP is ${CONTROL_PLANE_ENDPOINT_IP}"
+
+clusterYamlFile="/tmp/cluster-yaml"
+CONTROL_PLANE_ENDPOINT_IP=${CONTROL_PLANE_ENDPOINT_IP} clusterctl generate cluster byoh-cluster --infrastructure byoh --kubernetes-version v1.22.3 --control-plane-machine-count 1  --worker-machine-count 1 --flavor docker > "${clusterYamlFile}"
+if [ $? -ne 0 ]; then
+    echo "Generate ${clusterYamlFile} failed, exiting..."
+    exit
+fi
+
+
+echo "Creating the workload cluster..."
+kubectl apply -f ${clusterYamlFile} 
+if [ $? -ne 0 ]; then
+    echo "Create the workload cluster failed"
+    exit
+fi
+
+echo "Retrieving the kubeconfig of workload cluster..."
+
+maxRunTimes=10
+waitTime=1
+kubeConfigFile=/tmp/byoh-cluster-kubeconfig
+for((i=1;i<=${maxRunTimes};i++));  
+do   
+    kubectl get secret/byoh-cluster-kubeconfig 2>&1 | grep -q "not found"
+    if [ $? -eq 0 ]; then
+        sleep ${waitTime}
+        continue
+    else 
+        kubectl get secret/byoh-cluster-kubeconfig -o json | jq -r .data.value | base64 --decode  > ${kubeConfigFile}
+        echo "Retrieve the kubeconfig of workload cluster successfully"
+        break
+    fi
+done
+
+
+echo "Applying a CNI for network..."
+
+# Sometimes work cluster is not entirely ready, it reports error: Unable to connect to the server: dial tcp 172.18.0.5:6443: connect: no route to host
+maxRunTimes=10
+waitTime=5
+cniSucc=0
+for((i=1;i<=${maxRunTimes};i++));  
+do   
+    KUBECONFIG=${kubeConfigFile} kubectl apply -f https://docs.projectcalico.org/v3.20/manifests/calico.yaml
+    if [ $? -ne 0 ]; then
+        sleep ${waitTime}
+        continue
+    else 
+        echo "Applya CNI for network successfully"
+        cniSucc=1
+        break
+    fi
+done
+
+
+if [ ${cniSucc} -eq 0 ]; then
+    echo "Apply a CNI for network failed"
+    exit
+fi
+
+KUBECONFIG=${kubeConfigFile} kubectl get nodes | grep host
+if [ $? -eq 0 ]; then
+    echo "SUCCESS"
+else
+    echo "FAIL"
+fi
+
+
+
+
+
+
+
+

From b51060f2d50cd0cb6649bdb64f9860433ce79bdd Mon Sep 17 00:00:00 2001
From: huchen2021 <85480625+huchen2021@users.noreply.github.com>
Date: Fri, 18 Feb 2022 10:20:19 +0800
Subject: [PATCH 02/14] Update scripts/getting_started.sh

Co-authored-by: Anusha Hegde <anushah@vmware.com>
---
 scripts/getting_started.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/getting_started.sh b/scripts/getting_started.sh
index cde400ddd..00f78dd13 100755
--- a/scripts/getting_started.sh
+++ b/scripts/getting_started.sh
@@ -255,7 +255,7 @@ function runByohAgent(){
 function installDocker() {
     cmdName="docker"
 
-    ## check  if denpency is installed before
+    ## check  if dependency is present
     isCmdInstalled "${cmdName}"
 
     if [ $? -ne 0 ]; then

From 39fba820a1dcd8e2ec9d571fb72465ea7b4d80b6 Mon Sep 17 00:00:00 2001
From: chen hui <huchen@vmware.com>
Date: Fri, 18 Feb 2022 04:53:08 +0000
Subject: [PATCH 03/14] comment from anusha

Signed-off-by: chen hui <huchen@vmware.com>
---
 {scripts => hack}/getting_started.sh | 68 ++++++++--------------------
 1 file changed, 18 insertions(+), 50 deletions(-)
 rename {scripts => hack}/getting_started.sh (94%)

diff --git a/scripts/getting_started.sh b/hack/getting_started.sh
similarity index 94%
rename from scripts/getting_started.sh
rename to hack/getting_started.sh
index 00f78dd13..6a8dae98f 100755
--- a/scripts/getting_started.sh
+++ b/hack/getting_started.sh
@@ -1,5 +1,17 @@
 #!/bin/bash
 
+<<COMMENT
+ssh-copy-id <alias-name-for-this-jump-box>
+> sudo bash
+> chmod +w /etc/sudoers
+> vi /etc/sudoers
+<your-user-name>  ALL=(ALL) NOPASSWD:ALL
+> exit
+ssh-keygen -t rsa
+docker login
+
+COMMENT
+
 function isCmdInstalled() {
     cmd=$1
     ${cmd} version 2>&1 | grep -q "not found"
@@ -12,7 +24,6 @@ function isCmdInstalled() {
     return ${returnCode}
 }
 
-
 function exitIfNot(){
     code=$1
     expectedCode=$2
@@ -85,10 +96,6 @@ function runCmdUntil() {
     exit
 }
 
-
-
-
-
 function isIPOccupied() {
     ip=$1
     occupiedIPList=$2
@@ -178,32 +185,18 @@ function calcControlPlaneIP() {
 function configClusterctl() {
     # Write clusterctl.yaml
     writeByoh=0
-    writeCertManager=0
     clusterCtlYamlFile="${HOME}/.cluster-api/clusterctl.yaml"
     if [ ! -f "${clusterCtlYamlFile}" ]; then
         writeByoh=1
-        writeCertManager=1
         touch ${clusterCtlYamlFile}
     else
         grep -q "byoh" ${clusterCtlYamlFile}
         if [ $? -ne 0 ] ; then
             writeByoh=1
         fi
-        grep -q "cert-manager" ${clusterCtlYamlFile}
-        if [ $? -ne 0 ] ; then
-            writeCertManager=1
-        fi
     fi 
 
-    # if cert-manager missing,it report error: Error: action failed after 10 attempts: failed to get cert-manager object /, Kind=, /: Object 'Kind' is missing in 'unstructured object has no kind'
-    if [ ${writeCertManager} -eq 1 ] ; then
-        cat>>${clusterCtlYamlFile}<<EOF
-cert-manager:
-  url: "https://github.com/cert-manager/cert-manager/releases/latest/cert-manager.yaml"
-EOF
-    fi
-
-    if [ ${writeCertManager} -eq 1 ] ; then
+    if [ ${writeByoh} -eq 1 ] ; then
         cat>>${clusterCtlYamlFile}<<EOF
 providers:
   - name: byoh
@@ -249,7 +242,6 @@ function runByohAgent(){
 
     echo "Error: byohost object(host${index}) is created failed..."
     exit
-
 }
 
 function installDocker() {
@@ -273,14 +265,10 @@ function installDocker() {
     ## check  if denpency is installed successfully
     isCmdInstalled  "${cmdName}"
     exitIfNot $? 0 "Installing ${cmdName} failed, exit..."
-
-
-    sudo systemctl enable docker
-
+    runCmd "sudo systemctl enable docker" 0
 }
 
 function enableDocker() {
-
     runCmd "sudo systemctl start docker" 0
     runCmd "sudo systemctl enable docker" 0
 
@@ -311,7 +299,6 @@ function enableDocker() {
 }
 
 function installKind() {
-
     cmdName="kind"
      ## check  if denpency is installed before
     isCmdInstalled "${cmdName}"
@@ -327,7 +314,6 @@ function installKind() {
 }
 
 function installClusterctl() {
-
     cmdName="clusterctl"
      ## check  if denpency is installed before
     isCmdInstalled "${cmdName}"
@@ -335,14 +321,13 @@ function installClusterctl() {
     ## install denpency if it not installed
     if [ $? -eq 0 ] ; then
         echo "Installing ${cmdName}..."
-        curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.0/clusterctl-linux-amd64 -o clusterctl && sudo install clusterctl /usr/local/bin/clusterctl
+        curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.1/clusterctl-linux-amd64 -o clusterctl && sudo install clusterctl /usr/local/bin/clusterctl
         ## check  if denpency is installed successfully
         isCmdInstalled  "${cmdName}"
         exitIfNot $? 0 "Installing ${cmdName} failed, exit..."
     fi
 }
 
-
 function intallDependencies(){
     installDocker
     enableDocker
@@ -421,7 +406,6 @@ fi
 
 configClusterctl
 
-
 # check if init it before
 kubectl get pods --all-namespaces | grep -q byoh-controller-manager
 if [ $? -ne 0 ]; then
@@ -453,13 +437,9 @@ else
     echo "${byohBinaryFile} existed."
 fi
 
-
 manageClusterConfFile="${HOME}/.kube/management-cluster.conf"
-if [ ! -f "${manageClusterConfFile}" ]; then
-    cp ${HOME}/.kube/config ${manageClusterConfFile}
-else
-    echo "${manageClusterConfFile} existed."
-fi
+cp -f ${HOME}/.kube/config ${manageClusterConfFile}
+
 
 KIND_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' kind-control-plane) 
 
@@ -470,7 +450,6 @@ else
     echo "Already modified ${manageClusterConfFile} before"
 fi
 
-
 ## Register BYOH host to management cluster
 for i in {1..2}
 do
@@ -515,7 +494,6 @@ if [ $? -ne 0 ]; then
     exit
 fi
 
-
 echo "Creating the workload cluster..."
 kubectl apply -f ${clusterYamlFile} 
 if [ $? -ne 0 ]; then
@@ -541,7 +519,6 @@ do
     fi
 done
 
-
 echo "Applying a CNI for network..."
 
 # Sometimes work cluster is not entirely ready, it reports error: Unable to connect to the server: dial tcp 172.18.0.5:6443: connect: no route to host
@@ -561,7 +538,6 @@ do
     fi
 done
 
-
 if [ ${cniSucc} -eq 0 ]; then
     echo "Apply a CNI for network failed"
     exit
@@ -572,12 +548,4 @@ if [ $? -eq 0 ]; then
     echo "SUCCESS"
 else
     echo "FAIL"
-fi
-
-
-
-
-
-
-
-
+fi
\ No newline at end of file

From c4dbcf306312696ec03bae785b25ab728ad59b13 Mon Sep 17 00:00:00 2001
From: chen hui <huchen@vmware.com>
Date: Tue, 22 Feb 2022 09:09:36 +0000
Subject: [PATCH 04/14] comment from anusha

Signed-off-by: chen hui <huchen@vmware.com>
---
 hack/getting_started.sh | 610 +++++++++++++++++++++-------------------
 1 file changed, 320 insertions(+), 290 deletions(-)

diff --git a/hack/getting_started.sh b/hack/getting_started.sh
index 6a8dae98f..e2a60738f 100755
--- a/hack/getting_started.sh
+++ b/hack/getting_started.sh
@@ -1,19 +1,6 @@
 #!/bin/bash
-
-<<COMMENT
-ssh-copy-id <alias-name-for-this-jump-box>
-> sudo bash
-> chmod +w /etc/sudoers
-> vi /etc/sudoers
-<your-user-name>  ALL=(ALL) NOPASSWD:ALL
-> exit
-ssh-keygen -t rsa
-docker login
-
-COMMENT
-
 function isCmdInstalled() {
-    cmd=$1
+    local cmd=$1
     ${cmd} version 2>&1 | grep -q "not found"
     returnCode=$?  
     if [ ${returnCode} -eq 0 ] ; then
@@ -25,21 +12,21 @@ function isCmdInstalled() {
 }
 
 function exitIfNot(){
-    code=$1
-    expectedCode=$2
-    prompt=$3
+    local code=$1
+    local expectedCode=$2
+    local prompt=$3
     if [ ${code} == ${expectedCode} ]; then
         echo ${prompt}
-        exit
+        exit 1
     fi
 }
 
 function runCmd(){
-    cmd=$1
-    expectCode=$2
-    promptBeforeCmd=$3
-    PromptAfterCmdSuc=$4
-    promptAfterCmdFail=$5
+    local cmd=$1
+    local expectCode=$2
+    local promptBeforeCmd=$3
+    local PromptAfterCmdSuc=$4
+    local promptAfterCmdFail=$5
 
     if [ -n "${promptBeforeCmd}" ]; then
         echo ${promptBeforeCmd}
@@ -53,7 +40,7 @@ function runCmd(){
             echo ${promptAfterCmdFail}
         fi
         echo "${cmd} failed, exit...."
-        exit
+        exit 1
     fi
 
     if [ -n "${PromptAfterCmdSuc}" ]; then
@@ -61,64 +48,36 @@ function runCmd(){
     fi
 }
 
-function runCmdUntil() {
-    cmd=$1
-    expectCode=$2
-    waitTime=$3
-    maxRunTimes=$4
-    promptBeforeCmd=$5
-    PromptAfterCmdSuc=$6
-    promptAfterCmdFail=$7
-
-    echo "huchen: cmd is ${cmd}"
-
-    if [ -n "${promptBeforeCmd}" ]; then
-        echo ${promptBeforeCmd}
-    fi
-    for((i=1;i<=${maxRunTimes};i++));  
-    do   
-        ${cmd}
-        returnCode=$?
-        if [ ${returnCode} -ne ${expectCode} ]; then
-            sleep ${waitTime}
-            continue
-        else   
-            if [ -n "${PromptAfterCmdSuc}" ]; then
-                echo ${PromptAfterCmdSuc}
-            fi
-            return
-        fi
-        
-    done
-    if [ -n "${promptAfterCmdFail}" ]; then
-        echo ${promptAfterCmdFail}
-    fi
-    exit
-}
-
 function isIPOccupied() {
-    ip=$1
-    occupiedIPList=$2
+    local ip=$1
+    local occupiedIPList=$2
     echo "Checking if ${ip} is occupied"
 
-    echo "huchen: occupiedIPList is ${occupiedIPList}"
     IFS=$'\n'
     for occupiedIPWithMask in ${occupiedIPList}
     do
-        echo "huchen: occupiedIPWithMask is ${occupiedIPWithMask}"
         occupiedIP=`echo $occupiedIPWithMask | sed -En 's/^(.*)\/([0-9]{1,2})/\1/p'`
-        echo "huchen: ip is ${ip}, occupiedIP is ${occupiedIP}"
         if [ "${ip}" == "${occupiedIP}" ]; then
              return 1
         fi
     done
+
+    # should not use the gateway address
+    for gateway in ${gateways}
+    do 
+        if [ "${gateway}" == "${ip}" ]; then
+            return 1
+        fi
+    done
+
     echo "${ip} is not occupied"
     return 0
 }
 
 
+
 function num2IP() {
-    num=$1
+    local num=$1
     a=$((num>>24))
     b=$((num>>16&0xff))
     c=$((num>>8&0xff))
@@ -132,9 +91,9 @@ function binary2IP() {
     echo ${returnValue}
 }
 
-
 function calcControlPlaneIP() {
     subNet=`docker network inspect kind | jq -r 'map(.IPAM.Config[].Subnet) []'`
+    gateways=`docker network inspect kind | jq -r 'map(.IPAM.Config[].Gateway) []'`
     occupiedIPList=`docker network inspect kind | jq -r 'map(.Containers[].IPv4Address) []'`
 
     for line in ${subNet}
@@ -148,6 +107,7 @@ function calcControlPlaneIP() {
 
     echo "IPMask is ${subNet}"
     echo "occupiedIPList:${occupiedIPList}"
+    echo "gateways:${gateways}"
 
     ip=`echo $IPMask | sed -En 's/^(.*)\/([0-9]{1,2})/\1/p'`
     ipSubNetBit=`echo $IPMask | sed -En 's/^(.*)\/([0-9]{1,2})/\2/p'`
@@ -159,7 +119,7 @@ function calcControlPlaneIP() {
 
     ipSubNet=`echo ${binaryIP:0:${ipSubNetBit}}`
 
-    full0="00000000000000000000000000000010"
+    full0="00000000000000000000000000000001"
     full1="11111111111111111111111111111110"
 
     minIPBinary=${ipSubNet}`echo ${full0:${ipSubNetBit}}`
@@ -168,24 +128,30 @@ function calcControlPlaneIP() {
     minIPint=`echo "ibase=2; ${minIPBinary}"|bc`
     maxIPint=`echo "ibase=2; ${maxIPBinary}"|bc`
 
-
     for((i=${minIPint};i<=${maxIPint};i++));  
     do
         ip=$(num2IP ${i})
-
-        isIPOccupied "${ip}" "${occupiedIPList}"
+ 
+        isIPOccupied "${ip}" "${occupiedIPList}" "${gateways}"
         if [ $? == 0 ]; then
-            CONTROL_PLANE_ENDPOINT_IP=${ip}
-            echo "Available IP is ${CONTROL_PLANE_ENDPOINT_IP}"
+            controlPlaneEndPointIp=${ip}
+            echo "CONTROL_PLANE_ENDPOINT_IP is ${controlPlaneEndPointIp}"
             return
         fi
     done
+
+    echo "Can't get an available IP for control plane endpoint, exit...."
+    exit 1
+    
 }
 
-function configClusterctl() {
+function installByohProvider() {
+    local writeByoh=0
+    local clusterCtlYamlFile="${HOME}/.cluster-api/clusterctl.yaml"
+    local maxRunTimes=40
+    local waitTime=20
+
     # Write clusterctl.yaml
-    writeByoh=0
-    clusterCtlYamlFile="${HOME}/.cluster-api/clusterctl.yaml"
     if [ ! -f "${clusterCtlYamlFile}" ]; then
         writeByoh=1
         touch ${clusterCtlYamlFile}
@@ -208,44 +174,29 @@ EOF
     clusterctl config repositories | grep -q byoh
     if [ $? -ne 0 ] ; then
         echo "Config clusterctl failed..."
-        exit
+        exit 1
     fi
-}
-
-function runByohAgent(){
-    index=$1
-    byohBinaryFile=$2
-    manageClusterConfFile=$3
-
-    runCmd "docker cp ${byohBinaryFile} host${index}:/byoh-hostagent" 0 "Copying agent binary to byoh container: host${index}..."
-    runCmd "docker cp ${manageClusterConfFile} host${index}:/management-cluster.conf" 0 "Copying kubeconfig to byoh container: host${index}..."
-    docker exec -d host$i sh -c "chmod +x /byoh-hostagent && /byoh-hostagent --kubeconfig management-cluster.conf --skip-installation > /agent.log 2>&1"
-    if [ $? -ne 0 ] ; then
-        echo "Starting the host${index} agent..."
-        exit
-    fi
-
     
-    maxRunTimes=10
-    waitTime=1
+    runCmd "clusterctl init --infrastructure byoh" 0 " Transforming the Kubernetes cluster into a management cluster..."
+
+    # Waiting for byoh provider is totally ready
     for((i=1;i<=${maxRunTimes};i++));  
-    do   
-        kubectl get byohosts host${index} | grep -v NAME | grep -q host${index}
-        if [ $? -ne 0 ]; then
-            sleep ${waitTime}
-            continue
-        else   
-            echo "byohost object(host${index}) is created successfully..."
+    do  
+        byohStatus=$(kubectl get pods --all-namespaces | grep byoh-controller-manager | awk '{print $4}')
+        if [ "${byohStatus}" == "Running" ] ; then
+            echo "Byoh provider is ready"
             return
+        else
+            echo "Waiting for byoh-provider to be ready..."
+            sleep ${waitTime}
         fi
     done
-
-    echo "Error: byohost object(host${index}) is created failed..."
+    echo "Waiting too long for byoh provider, something may wrong with it."
     exit
 }
 
 function installDocker() {
-    cmdName="docker"
+    local cmdName="docker"
 
     ## check  if dependency is present
     isCmdInstalled "${cmdName}"
@@ -275,31 +226,14 @@ function enableDocker() {
     sudo systemctl status docker | grep -q "active (running)"
     if [ $? -ne 0 ]; then
         echo "Enable docker failed"
-        exit 
+        exit 1
     fi
     echo "Enable docker success"
-
-    # Make sure current user has permission for docker, do this if not Create the docker group.
-    docker ps 2>&1 | grep -q "connect: permission denied"
-    if [ $? -eq 0 ]; then
-
-        grep -q "docker" /etc/group 
-        if [ $? -ne 0 ]; then
-            runCmd "sudo groupadd docker" 0
-        else
-            echo "group 'docker' already exists"
-        fi
-        USER=`whoami`
-        # Add your user to the docker group.
-        runCmd "sudo usermod -aG docker ${USER}" 0 "Add ${USER} to docker group"
-        echo "You would need to log out and log back in so that your group membership is re-evaluated. Rerun this script after that."
-        exit
-    fi
-    echo "current user has permission for docker"
 }
 
 function installKind() {
-    cmdName="kind"
+    local cmdName="kind"
+
      ## check  if denpency is installed before
     isCmdInstalled "${cmdName}"
 
@@ -314,7 +248,8 @@ function installKind() {
 }
 
 function installClusterctl() {
-    cmdName="clusterctl"
+    local cmdName="clusterctl"
+
      ## check  if denpency is installed before
     isCmdInstalled "${cmdName}"
 
@@ -328,20 +263,9 @@ function installClusterctl() {
     fi
 }
 
-function intallDependencies(){
-    installDocker
-    enableDocker
-    installKind
-    installClusterctl
-    commonInstall jq "sudo apt install -y jq"
-    commonInstall kubectl "sudo snap install kubectl --classic"
-    commonInstall make "sudo apt install -y build-essential"
-    commonInstall go "sudo snap install go --classic"
-}
-
 function commonInstall(){
-    cmdName=$1
-    installCmd=$2
+    local cmdName=$1
+    local installCmd=$2
 
     ## check  if denpency is installed before
     isCmdInstalled "${cmdName}"
@@ -356,196 +280,302 @@ function commonInstall(){
     fi
 }
 
+function intallDependencies(){
+    installDocker
+    enableDocker
+    installKind
+    installClusterctl
+    commonInstall jq "sudo apt install -y jq"
+    commonInstall kubectl "sudo snap install kubectl --classic"
+    commonInstall make "sudo apt install -y build-essential"
+    commonInstall go "sudo snap install go --classic"
+}
+
 function createKindCluster(){
     echo  "Creating kind cluster..."
 
-    msg=$(kind create cluster 2>&1)
+    kind create cluster --name ${managerClusterName}
     if [ $? -ne 0 ]; then
         echo "Create kind cluster failed"
-        echo $msg | grep -q "You have reached your pull rate limit"
-        if [ $? -eq 0 ]; then 
-            echo "Suggestion: you can use \"docker login\" to avoid such an error."
-            exit
-        fi
+        exit 1
     else
         echo "Create kind cluster successfully"
     fi
 }
 
-function downloadByohCode(){
-    runCmd "rm -rf cluster-api-provider-bringyourownhost" 0 " Cleaning byoh code..."
-    msg=$(git clone git@github.com:vmware-tanzu/cluster-api-provider-bringyourownhost.git 2>&1)
-    if [ $? -ne 0 ]; then
-        echo "Downloading byoh code..."
-        echo $msg | grep "Please make sure you have the correct access rights"
-        if [ $? -eq 0 ]; then 
-            echo "Suggestion: Add an public key of this machine into \"SSH and GPG keys\" of your github setting"
-            exit
+function cleanUp(){
+    kind delete clusters ${managerClusterName}
+    docker ps -a | grep host
+    for dockerId in `docker ps -a | grep host | awk '{print $1}'`
+    do
+        docker rm -f ${dockerId}
+    done
+}
+
+function readArgs() {
+    TEMP=`getopt -o nm:c: --long cni,md:,cp:`
+    if [ $? != 0 ] ; then 
+        echo "Terminating..." >&2 
+        exit 1 
+    fi
+    # Note the quotes around `$TEMP': they are essential!
+    while true ; do
+        case "$1" in
+            -n|--cni) 
+                defaultCni=1
+                shift 
+                ;;
+            -m|--md) 
+                workerNums=$2
+                shift 2 
+                ;;
+            -c|--cp) 
+                controlPlaneNums=$2
+                shift 2 
+                ;;
+            *) 
+                break 
+                ;;
+        esac
+    done
+
+    byohNums=$[${workerNums}+${controlPlaneNums}]
+
+    echo "defaultCni=${defaultCni}, controlPlaneNums=${controlPlaneNums}, workerNums=${workerNums}"
+}
+
+function installCNI(){
+    local maxRunTimes=40
+    local waitTime=20
+    local cniSucc=0
+    local i=1
+
+    # Sometimes work cluster is not entirely ready, it reports error: Unable to connect to the server: dial tcp 172.18.0.5:6443: connect: no route to host
+    echo "Applying a CNI for network..."
+    for((i=1;i<=${maxRunTimes};i++));  
+    do
+        KUBECONFIG=${kubeConfigFile} kubectl apply -f https://docs.projectcalico.org/v3.20/manifests/calico.yaml
+        if [ $? -ne 0 ]; then
+            sleep ${waitTime}
+            continue
+        else 
+            echo "Apply CNI for network successfully"
+            cniSucc=1
+            break
         fi
-    else
-        echo "Download byoh code successfully"
+    done
+
+    if [ ${cniSucc} -eq 0 ]; then
+        echo "Apply a CNI for network failed"
+        exit 1
     fi
 }
 
-export PATH=/snap/bin:${PATH}
-intallDependencies 
- 
-runCmd "sudo swapoff -a" 0 "Turning off swap..."
-swapMsg=$(sudo swapon -s)
-if [ -n "${swapMsg}" ]; then
-    echo "Please turn off swap first."
+function retrieveKubeConfig() {
+    local maxRunTimes=10
+    local waitTime=1
+    
+    echo "Retrieving the kubeconfig of workload cluster..."
+    for((i=1;i<=${maxRunTimes};i++));  
+    do   
+        kubectl get secret/${workerClusterName}-kubeconfig 2>&1 | grep -q "not found"
+        if [ $? -eq 0 ]; then
+            sleep ${waitTime}
+            continue
+        else 
+            kubectl get secret/${workerClusterName}-kubeconfig -o json | jq -r .data.value | base64 --decode  > ${kubeConfigFile}
+            echo "Retrieve the kubeconfig of workload cluster successfully"
+            return
+        fi
+    done
+
+    echo "Retrieve the kubeconfig of workload cluster failed"
     exit
-fi
- 
-# check if cluster "kind" is already exited
-clusterName=`kind get clusters`
-if [ "${clusterName}" != "kind" ]; then
-    #runCmd "kind create cluster" 0 "Creating kind cluster..."
-    createKindCluster
-fi
+}
 
-configClusterctl
 
-# check if init it before
-kubectl get pods --all-namespaces | grep -q byoh-controller-manager
-if [ $? -ne 0 ]; then
-    runCmd "clusterctl init --infrastructure byoh" 0 " Transforming the Kubernetes cluster into a management cluster..."
-else
-    echo "clusterctl init --infrastructure byoh before"
-fi
+function checkNodeStatus() {
+    local maxRunTimes=40
+    local waitTime=30
+    local i=1
+    local j=1
+    local ready=0
 
-#check if byoh image is existed
-docker images | grep "byoh-dev/node" | grep -q "v1.22.3"
-if [ $? -ne 0 ]; then
-    #runCmd "rm -rf cluster-api-provider-bringyourownhost" 0 " Cleaning byoh code..."
-    #runCmd "git clone git@github.com:vmware-tanzu/cluster-api-provider-bringyourownhost.git" 0 " Downloading byoh code..."
-    downloadByohCode
-    runCmd "cd cluster-api-provider-bringyourownhost" 0
-    # The origin one will report error:   Could not connect to apt.kubernetes.io:443 (10.25.207.164), connection timed out [IP: 10.25.207.164 443]
-    echo "deb http://packages.cloud.google.com/apt/ kubernetes-xenial main" > test/e2e/kubernetes.list
-    runCmd "make prepare-byoh-docker-host-image-dev" 0  "Making a byoh image: byoh-dev/node:v1.22.3 ..."
-    runCmd "cd -" 0
-else
-    echo "byoh image \"byoh-dev/node:v1.22.3\" existed."
-fi
+    for((i=1;i<=${byohNums};i++)); 
+    do
+        ready=0
+        for((j=1;j<=${maxRunTimes};j++));  
+        do   
+            KUBECONFIG=${kubeConfigFile} kubectl get nodes host${i} | grep -q "not found"
+            if [ $? -eq 0 ]; then
+                sleep ${waitTime}
+                continue
+            fi
+            if [ ${defaultCni} -eq 1 ]; then
+                status=`KUBECONFIG=${kubeConfigFile} kubectl get nodes host${i} | grep -v NAME | awk '{print $2}'`
+                if [ "${status}" != "Ready" ]; then
+                    sleep ${waitTime}
+                    continue
+                fi
+            fi
+            ready=1
+            echo "node \"host${i}\" is ready"
+            break
+            
+        done
 
-#check if byoh binary is existed
-byohBinaryFile="/tmp/byoh-hostagent-linux-amd64"
-if [ ! -f "${byohBinaryFile}" ]; then
-    runCmd "wget https://github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/releases/download/v0.1.0/byoh-hostagent-linux-amd64 -P /tmp" 0  "Downloading a byoh binary..."
-else
-    echo "${byohBinaryFile} existed."
-fi
+        if [ ${ready} -eq 0 ]; then
+            echo "FAIL! node \"host${i}\" is not ready"
+            exit 1
+        fi
+    done
+}
 
-manageClusterConfFile="${HOME}/.kube/management-cluster.conf"
-cp -f ${HOME}/.kube/config ${manageClusterConfFile}
 
+function prepareImageAndBinary() {
+    local  isImageExisted=0
 
-KIND_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' kind-control-plane) 
+    runCmd "cd ${reposDir}" 0
 
-grep ${KIND_IP} ${manageClusterConfFile} | grep -q 6443
-if [ $? -ne 0 ]; then
-    sed -i 's/    server\:.*/    server\: https\:\/\/'"${KIND_IP}"'\:6443/g' ${manageClusterConfFile}
-else
-    echo "Already modified ${manageClusterConfFile} before"
-fi
+    # Build byoh image
+    # Check if byoh image is existed
+    for line in `docker images | grep "${byohImageName}" | grep "${byohImageTag}" | awk '{print $1":"$2}'`
+    do
+        if [ ${line} == "${byohImageName}:${byohImageTag}" ]; then
+            isImageExisted=1
+            break
+        fi
+    done
 
-## Register BYOH host to management cluster
-for i in {1..2}
-do
-  #Check if container "host$i" is already created.
-  docker ps -a | grep -q "host${i}"
-  if [ $? -ne 0 ]; then
-    runCmd "docker run --detach --tty --hostname host${i} --name host${i} --privileged --security-opt seccomp=unconfined --tmpfs /tmp --tmpfs /run --volume /var --volume /lib/modules:/lib/modules:ro --network kind byoh-dev/node:v1.22.3" 0 "Starting byoh container: host${i}..."
-    runByohAgent ${i} ${byohBinaryFile} ${manageClusterConfFile}
-  else
-    echo "Container \"host${i}\" is already created"
-
-    # check if docker status is valid
-    status=`docker container inspect -f '{{.State.Status}}' host${i}`
-    if [ "${status}" != "running" ]; then
-        echo "Error: Status of Container \"host${i}\" is ${status}, suggest to remove it, then rerun this script"
-        exit
+    if [ ${isImageExisted} -eq 0 ]; then
+        # The origin one will report error:   Could not connect to apt.kubernetes.io:443 (10.25.207.164), connection timed out [IP: 10.25.207.164 443]
+        echo "deb http://packages.cloud.google.com/apt/ kubernetes-xenial main" > ${reposDir}/test/e2e/kubernetes.list
+        runCmd "make prepare-byoh-docker-host-image" 0  "Making a byoh image: ${byohImageName}:${byohImageTag} ..."
+        
+    else
+        echo "byoh image \"${byohImageName}:${byohImageTag}\" existed."
     fi
 
-    # check if byoh process running in this container
-    docker exec host${i} sh -c "ps aux" | grep -q byoh-hostagent
-    if [ $? -ne 0 ]; then
-        runByohAgent ${i} ${byohBinaryFile} ${manageClusterConfFile}
+    # Build byoh binary
+    # Check if byoh binary is existed
+    if [ ! -f ${byohBinaryFile} ]; then
+        runCmd "make host-agent-binaries" 0  "Making byoh binary: ${byohBinaryFile} ..."
     else
-        echo "byoh process is already ran in this container"
+        echo "byoh binary \"${byohBinaryFile}\" existed."
     fi
-  fi
-done
 
-# Find a available IP for control plane endpoint
-CONTROL_PLANE_ENDPOINT_IP=""
-calcControlPlaneIP
-if [ -z "${CONTROL_PLANE_ENDPOINT_IP}" ]; then
-    echo "Can't get an available IP for control plane endpoint, exit...."
-    exit
-fi
-echo "CONTROL_PLANE_ENDPOINT_IP is ${CONTROL_PLANE_ENDPOINT_IP}"
+    runCmd "cd -" 0
 
-clusterYamlFile="/tmp/cluster-yaml"
-CONTROL_PLANE_ENDPOINT_IP=${CONTROL_PLANE_ENDPOINT_IP} clusterctl generate cluster byoh-cluster --infrastructure byoh --kubernetes-version v1.22.3 --control-plane-machine-count 1  --worker-machine-count 1 --flavor docker > "${clusterYamlFile}"
-if [ $? -ne 0 ]; then
-    echo "Generate ${clusterYamlFile} failed, exiting..."
-    exit
-fi
 
-echo "Creating the workload cluster..."
-kubectl apply -f ${clusterYamlFile} 
-if [ $? -ne 0 ]; then
-    echo "Create the workload cluster failed"
-    exit
-fi
+    # Download byoh binary
+    #rm -f ${byohBinaryFile}
+    #runCmd "wget https://github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/releases/download/v0.1.0/byoh-hostagent-linux-amd64 -P /tmp" 0  "Downloading a byoh binary..."
+}
 
-echo "Retrieving the kubeconfig of workload cluster..."
+function bringUpByoHost(){
+    local i=1
+    local t=1
+    local maxRunTimes=10
+    local waitTime=1
+    local KIND_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ${managerClusterName}-control-plane) 
+    local ok=0
+    
+    cp -f ${HOME}/.kube/config ${manageClusterConfFile}
+    sed -i 's/    server\:.*/    server\: https\:\/\/'"${KIND_IP}"'\:6443/g' ${manageClusterConfFile}
+    
+    for (( i=1; i<=${byohNums}; i++ ))
+    do
+        runCmd "docker run --detach --tty --hostname host${i} --name host${i} --privileged --security-opt seccomp=unconfined --tmpfs /tmp --tmpfs /run --volume /var --volume /lib/modules:/lib/modules:ro --network kind ${byohImageName}:${byohImageTag}" 0 "Starting byoh container: host${i}..."
 
-maxRunTimes=10
-waitTime=1
-kubeConfigFile=/tmp/byoh-cluster-kubeconfig
-for((i=1;i<=${maxRunTimes};i++));  
-do   
-    kubectl get secret/byoh-cluster-kubeconfig 2>&1 | grep -q "not found"
-    if [ $? -eq 0 ]; then
-        sleep ${waitTime}
-        continue
-    else 
-        kubectl get secret/byoh-cluster-kubeconfig -o json | jq -r .data.value | base64 --decode  > ${kubeConfigFile}
-        echo "Retrieve the kubeconfig of workload cluster successfully"
-        break
-    fi
-done
+        runCmd "docker cp ${byohBinaryFile} host${i}:/byoh-hostagent" 0 "Copying agent binary to byoh container: host${i}..."
+        runCmd "docker cp ${manageClusterConfFile} host${i}:/management-cluster.conf" 0 "Copying kubeconfig to byoh container: host${i}..."
+        
+        echo "Starting the host${i} agent..."
+        docker exec -d host${i} sh -c "chmod +x /byoh-hostagent && /byoh-hostagent --kubeconfig /management-cluster.conf > /agent.log 2>&1"
+
+        ok=0
+        for((t=1;t<=${maxRunTimes};t++));  
+        do
+            kubectl get byohosts host${i} | grep -q "not found" 2>/dev/null
+            if [ $? -eq 0 ]; then
+                sleep ${waitTime}
+                continue
+            else   
+                echo "byohost object(host${i}) is created successfully..."
+                ok=1
+                break
+            fi
+        done
+        if [ $ok -eq 0 ]; then
+            echo "Error: byohost object(host${i}) is created failed..."
+            exit 1
+        fi
+    done
+}
 
-echo "Applying a CNI for network..."
+function createWorkCluster() {
+    local clusterYamlFile="/tmp/cluster-yaml"
 
-# Sometimes work cluster is not entirely ready, it reports error: Unable to connect to the server: dial tcp 172.18.0.5:6443: connect: no route to host
-maxRunTimes=10
-waitTime=5
-cniSucc=0
-for((i=1;i<=${maxRunTimes};i++));  
-do   
-    KUBECONFIG=${kubeConfigFile} kubectl apply -f https://docs.projectcalico.org/v3.20/manifests/calico.yaml
+    # Find a available IP for control plane endpoint
+    calcControlPlaneIP
+    
+    CONTROL_PLANE_ENDPOINT_IP=${controlPlaneEndPointIp} clusterctl generate cluster ${workerClusterName} --infrastructure byoh --kubernetes-version ${kubernetesVersion} --control-plane-machine-count ${controlPlaneNums}  --worker-machine-count ${workerNums} --flavor docker > "${clusterYamlFile}"
     if [ $? -ne 0 ]; then
-        sleep ${waitTime}
-        continue
-    else 
-        echo "Applya CNI for network successfully"
-        cniSucc=1
-        break
+        echo "Generate ${clusterYamlFile} failed, exiting..."
+        exit 1
     fi
-done
 
-if [ ${cniSucc} -eq 0 ]; then
-    echo "Apply a CNI for network failed"
-    exit
+    echo "Creating the workload cluster..."
+    kubectl apply -f ${clusterYamlFile} 
+    if [ $? -ne 0 ]; then
+        echo "Create the workload cluster failed"
+        exit 1
+    fi
+}
+
+function swapOff() {
+    runCmd "sudo swapoff -a" 0 "Turning off swap..."
+    swapMsg=$(sudo swapon -s)
+    if [ -n "${swapMsg}" ]; then
+        echo "Please turn off swap first."
+        exit
+    fi
+}
+
+
+export PATH=/snap/bin:${PATH}
+byohImageName="byoh/node"
+byohImageTag="v1.22.3"
+managerClusterName="kind-byoh"
+workerClusterName="worker-byoh"
+controlPlaneEndPointIp=""
+workerNums=1
+controlPlaneNums=1
+byohNums=2
+defaultCni=0
+manageClusterConfFile="${HOME}/.kube/management-cluster.conf"
+kubeConfigFile=/tmp/byoh-cluster-kubeconfig
+reposDir=$(dirname $0)/../
+byohBinaryFile=${reposDir}/bin/byoh-hostagent-linux-amd64
+
+
+if [ -z "${KUBERNETES_VERSION}" ]; then
+    kubernetesVersion="v1.22.3"
+fi
+
+readArgs $@
+swapOff
+intallDependencies 
+cleanUp
+createKindCluster
+installByohProvider
+prepareImageAndBinary
+bringUpByoHost
+createWorkCluster
+retrieveKubeConfig
+
+if [ ${defaultCni} -eq 1 ]; then
+    installCNI
 fi
 
-KUBECONFIG=${kubeConfigFile} kubectl get nodes | grep host
-if [ $? -eq 0 ]; then
-    echo "SUCCESS"
-else
-    echo "FAIL"
-fi
\ No newline at end of file
+checkNodeStatus
\ No newline at end of file

From a45399614817a7aeab2bf11a4af40ffa2472056c Mon Sep 17 00:00:00 2001
From: chen hui <huchen@vmware.com>
Date: Tue, 22 Feb 2022 09:14:47 +0000
Subject: [PATCH 05/14] comment from anusha

Signed-off-by: chen hui <huchen@vmware.com>
---
 hack/getting_started.sh | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/hack/getting_started.sh b/hack/getting_started.sh
index e2a60738f..826d12dfd 100755
--- a/hack/getting_started.sh
+++ b/hack/getting_started.sh
@@ -518,7 +518,7 @@ function createWorkCluster() {
 
     # Find a available IP for control plane endpoint
     calcControlPlaneIP
-    
+
     CONTROL_PLANE_ENDPOINT_IP=${controlPlaneEndPointIp} clusterctl generate cluster ${workerClusterName} --infrastructure byoh --kubernetes-version ${kubernetesVersion} --control-plane-machine-count ${controlPlaneNums}  --worker-machine-count ${workerNums} --flavor docker > "${clusterYamlFile}"
     if [ $? -ne 0 ]; then
         echo "Generate ${clusterYamlFile} failed, exiting..."
@@ -578,4 +578,8 @@ if [ ${defaultCni} -eq 1 ]; then
     installCNI
 fi
 
-checkNodeStatus
\ No newline at end of file
+checkNodeStatus
+
+if [ ${defaultCni} -eq 0 ]; then
+    echo "Byoh cluster \"${workerClusterName}\" is successfully created, next step is to apply a CNI of your choice."
+fi
\ No newline at end of file

From 7c8c54d5516c9d743bb04ddb4a138720b992e21b Mon Sep 17 00:00:00 2001
From: chen hui <huchen@vmware.com>
Date: Wed, 23 Feb 2022 05:07:34 +0000
Subject: [PATCH 06/14] comment from anusha

Signed-off-by: chen hui <huchen@vmware.com>
---
 hack/getting_started.sh | 55 ++++++++++++++++++++++++-----------------
 1 file changed, 32 insertions(+), 23 deletions(-)

diff --git a/hack/getting_started.sh b/hack/getting_started.sh
index 826d12dfd..d08d860de 100755
--- a/hack/getting_started.sh
+++ b/hack/getting_started.sh
@@ -304,11 +304,12 @@ function createKindCluster(){
 }
 
 function cleanUp(){
+    local i=1
+
     kind delete clusters ${managerClusterName}
-    docker ps -a | grep host
-    for dockerId in `docker ps -a | grep host | awk '{print $1}'`
+    for ((i=1;i<=${byohNums};i++)); 
     do
-        docker rm -f ${dockerId}
+        docker rm -f host${i}
     done
 }
 
@@ -340,8 +341,6 @@ function readArgs() {
     done
 
     byohNums=$[${workerNums}+${controlPlaneNums}]
-
-    echo "defaultCni=${defaultCni}, controlPlaneNums=${controlPlaneNums}, workerNums=${workerNums}"
 }
 
 function installCNI(){
@@ -393,7 +392,6 @@ function retrieveKubeConfig() {
     exit
 }
 
-
 function checkNodeStatus() {
     local maxRunTimes=40
     local waitTime=30
@@ -433,21 +431,12 @@ function checkNodeStatus() {
 
 
 function prepareImageAndBinary() {
-    local  isImageExisted=0
-
     runCmd "cd ${reposDir}" 0
 
     # Build byoh image
     # Check if byoh image is existed
-    for line in `docker images | grep "${byohImageName}" | grep "${byohImageTag}" | awk '{print $1":"$2}'`
-    do
-        if [ ${line} == "${byohImageName}:${byohImageTag}" ]; then
-            isImageExisted=1
-            break
-        fi
-    done
-
-    if [ ${isImageExisted} -eq 0 ]; then
+    image=$(docker images ${byohImageName}:${byohImageTag} | grep -v REPOSITORY)
+    if [ -z "${image}" ]; then
         # The origin one will report error:   Could not connect to apt.kubernetes.io:443 (10.25.207.164), connection timed out [IP: 10.25.207.164 443]
         echo "deb http://packages.cloud.google.com/apt/ kubernetes-xenial main" > ${reposDir}/test/e2e/kubernetes.list
         runCmd "make prepare-byoh-docker-host-image" 0  "Making a byoh image: ${byohImageName}:${byohImageTag} ..."
@@ -465,11 +454,6 @@ function prepareImageAndBinary() {
     fi
 
     runCmd "cd -" 0
-
-
-    # Download byoh binary
-    #rm -f ${byohBinaryFile}
-    #runCmd "wget https://github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/releases/download/v0.1.0/byoh-hostagent-linux-amd64 -P /tmp" 0  "Downloading a byoh binary..."
 }
 
 function bringUpByoHost(){
@@ -543,6 +527,31 @@ function swapOff() {
 }
 
 
+function askForProceed() {
+    local warning='
+#####################################################################################################
+
+** WARNING **
+This modifys system settings - and do **NOT** revert them at the end of the test.
+
+It locally will change the following host config
+- disable swap, but it can revert back if rebooting vm
+- use "sudo apt-get update" command to download package information from all configured sources.
+- install docker, and enable it as service if not
+- install kind, clusterctl, jq, kubectl, build-essential and go, if not
+- create a kind cluster as manager cluster, byoh clustr as worker cluster
+
+#####################################################################################################'
+
+    echo "${warning}"
+	read -p "Do you want to proceed [Y/N]?" REPLY; 
+	if [[ ${REPLY} != "Y" && ${REPLY} != "y" ]]; then 
+        echo "Aborting..."
+        exit 1
+    fi
+}
+
+
 export PATH=/snap/bin:${PATH}
 byohImageName="byoh/node"
 byohImageTag="v1.22.3"
@@ -558,12 +567,12 @@ kubeConfigFile=/tmp/byoh-cluster-kubeconfig
 reposDir=$(dirname $0)/../
 byohBinaryFile=${reposDir}/bin/byoh-hostagent-linux-amd64
 
-
 if [ -z "${KUBERNETES_VERSION}" ]; then
     kubernetesVersion="v1.22.3"
 fi
 
 readArgs $@
+askForProceed
 swapOff
 intallDependencies 
 cleanUp

From 58d0a8235851da973a9f4f45499b2a8ef835931a Mon Sep 17 00:00:00 2001
From: chen hui <huchen@vmware.com>
Date: Wed, 23 Feb 2022 06:10:10 +0000
Subject: [PATCH 07/14] fix exit code

Signed-off-by: chen hui <huchen@vmware.com>
---
 hack/getting_started.sh | 30 ++++++++++++++----------------
 1 file changed, 14 insertions(+), 16 deletions(-)

diff --git a/hack/getting_started.sh b/hack/getting_started.sh
index d08d860de..991a5887e 100755
--- a/hack/getting_started.sh
+++ b/hack/getting_started.sh
@@ -131,7 +131,7 @@ function calcControlPlaneIP() {
     for((i=${minIPint};i<=${maxIPint};i++));  
     do
         ip=$(num2IP ${i})
- 
+
         isIPOccupied "${ip}" "${occupiedIPList}" "${gateways}"
         if [ $? == 0 ]; then
             controlPlaneEndPointIp=${ip}
@@ -142,7 +142,7 @@ function calcControlPlaneIP() {
 
     echo "Can't get an available IP for control plane endpoint, exit...."
     exit 1
-    
+
 }
 
 function installByohProvider() {
@@ -176,7 +176,7 @@ EOF
         echo "Config clusterctl failed..."
         exit 1
     fi
-    
+
     runCmd "clusterctl init --infrastructure byoh" 0 " Transforming the Kubernetes cluster into a management cluster..."
 
     # Waiting for byoh provider is totally ready
@@ -192,7 +192,7 @@ EOF
         fi
     done
     echo "Waiting too long for byoh provider, something may wrong with it."
-    exit
+    exit 1
 }
 
 function installDocker() {
@@ -373,7 +373,7 @@ function installCNI(){
 function retrieveKubeConfig() {
     local maxRunTimes=10
     local waitTime=1
-    
+
     echo "Retrieving the kubeconfig of workload cluster..."
     for((i=1;i<=${maxRunTimes};i++));  
     do   
@@ -389,7 +389,7 @@ function retrieveKubeConfig() {
     done
 
     echo "Retrieve the kubeconfig of workload cluster failed"
-    exit
+    exit 1
 }
 
 function checkNodeStatus() {
@@ -419,7 +419,7 @@ function checkNodeStatus() {
             ready=1
             echo "node \"host${i}\" is ready"
             break
-            
+
         done
 
         if [ ${ready} -eq 0 ]; then
@@ -440,7 +440,7 @@ function prepareImageAndBinary() {
         # The origin one will report error:   Could not connect to apt.kubernetes.io:443 (10.25.207.164), connection timed out [IP: 10.25.207.164 443]
         echo "deb http://packages.cloud.google.com/apt/ kubernetes-xenial main" > ${reposDir}/test/e2e/kubernetes.list
         runCmd "make prepare-byoh-docker-host-image" 0  "Making a byoh image: ${byohImageName}:${byohImageTag} ..."
-        
+
     else
         echo "byoh image \"${byohImageName}:${byohImageTag}\" existed."
     fi
@@ -463,17 +463,17 @@ function bringUpByoHost(){
     local waitTime=1
     local KIND_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ${managerClusterName}-control-plane) 
     local ok=0
-    
+
     cp -f ${HOME}/.kube/config ${manageClusterConfFile}
     sed -i 's/    server\:.*/    server\: https\:\/\/'"${KIND_IP}"'\:6443/g' ${manageClusterConfFile}
-    
+
     for (( i=1; i<=${byohNums}; i++ ))
     do
         runCmd "docker run --detach --tty --hostname host${i} --name host${i} --privileged --security-opt seccomp=unconfined --tmpfs /tmp --tmpfs /run --volume /var --volume /lib/modules:/lib/modules:ro --network kind ${byohImageName}:${byohImageTag}" 0 "Starting byoh container: host${i}..."
 
         runCmd "docker cp ${byohBinaryFile} host${i}:/byoh-hostagent" 0 "Copying agent binary to byoh container: host${i}..."
         runCmd "docker cp ${manageClusterConfFile} host${i}:/management-cluster.conf" 0 "Copying kubeconfig to byoh container: host${i}..."
-        
+
         echo "Starting the host${i} agent..."
         docker exec -d host${i} sh -c "chmod +x /byoh-hostagent && /byoh-hostagent --kubeconfig /management-cluster.conf > /agent.log 2>&1"
 
@@ -522,7 +522,7 @@ function swapOff() {
     swapMsg=$(sudo swapon -s)
     if [ -n "${swapMsg}" ]; then
         echo "Please turn off swap first."
-        exit
+        exit 1
     fi
 }
 
@@ -530,17 +530,14 @@ function swapOff() {
 function askForProceed() {
     local warning='
 #####################################################################################################
-
 ** WARNING **
 This modifys system settings - and do **NOT** revert them at the end of the test.
-
 It locally will change the following host config
 - disable swap, but it can revert back if rebooting vm
 - use "sudo apt-get update" command to download package information from all configured sources.
 - install docker, and enable it as service if not
 - install kind, clusterctl, jq, kubectl, build-essential and go, if not
 - create a kind cluster as manager cluster, byoh clustr as worker cluster
-
 #####################################################################################################'
 
     echo "${warning}"
@@ -591,4 +588,5 @@ checkNodeStatus
 
 if [ ${defaultCni} -eq 0 ]; then
     echo "Byoh cluster \"${workerClusterName}\" is successfully created, next step is to apply a CNI of your choice."
-fi
\ No newline at end of file
+fi
+

From 786271314ef64141c188b7ad3785ed7d899b610b Mon Sep 17 00:00:00 2001
From: huchen2021 <85480625+huchen2021@users.noreply.github.com>
Date: Thu, 24 Feb 2022 10:48:13 +0800
Subject: [PATCH 08/14] Update hack/getting_started.sh

Co-authored-by: Anusha Hegde <anushah@vmware.com>
---
 hack/getting_started.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hack/getting_started.sh b/hack/getting_started.sh
index 991a5887e..2d73500a2 100755
--- a/hack/getting_started.sh
+++ b/hack/getting_started.sh
@@ -213,7 +213,7 @@ function installDocker() {
     sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu  $(lsb_release -cs)  stable"
     runCmd "sudo apt-get install -y docker-ce" 0
 
-    ## check  if denpency is installed successfully
+    ## check  if dependency is installed successfully
     isCmdInstalled  "${cmdName}"
     exitIfNot $? 0 "Installing ${cmdName} failed, exit..."
     runCmd "sudo systemctl enable docker" 0

From b5e0cc7c601fb339a1fea2dc1a1f017272716a1e Mon Sep 17 00:00:00 2001
From: huchen2021 <85480625+huchen2021@users.noreply.github.com>
Date: Thu, 24 Feb 2022 10:48:21 +0800
Subject: [PATCH 09/14] Update hack/getting_started.sh

Co-authored-by: Anusha Hegde <anushah@vmware.com>
---
 hack/getting_started.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hack/getting_started.sh b/hack/getting_started.sh
index 2d73500a2..3302d7146 100755
--- a/hack/getting_started.sh
+++ b/hack/getting_started.sh
@@ -234,7 +234,7 @@ function enableDocker() {
 function installKind() {
     local cmdName="kind"
 
-     ## check  if denpency is installed before
+     ## check  if dependency is installed before
     isCmdInstalled "${cmdName}"
 
     ## install denpency if it not installed

From cac396821f2c70599caea233e71945762b0ec050 Mon Sep 17 00:00:00 2001
From: chen hui <huchen@vmware.com>
Date: Thu, 24 Feb 2022 03:28:10 +0000
Subject: [PATCH 10/14] comment from anusha

Signed-off-by: chen hui <huchen@vmware.com>
---
 hack/getting_started.sh | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/hack/getting_started.sh b/hack/getting_started.sh
index 3302d7146..415a4fc9b 100755
--- a/hack/getting_started.sh
+++ b/hack/getting_started.sh
@@ -266,10 +266,8 @@ function installClusterctl() {
 function commonInstall(){
     local cmdName=$1
     local installCmd=$2
-
     ## check  if denpency is installed before
     isCmdInstalled "${cmdName}"
-
     ## install denpency if it not installed
     if [ $? -eq 0 ] ; then
         runCmd "${installCmd}" 0 "Installing ${cmdName}..."
@@ -314,7 +312,7 @@ function cleanUp(){
 }
 
 function readArgs() {
-    TEMP=`getopt -o nm:c: --long cni,md:,cp:`
+    TEMP=`getopt -o nm:c:k: --long cni,md:,cp:,kv:`
     if [ $? != 0 ] ; then 
         echo "Terminating..." >&2 
         exit 1 
@@ -334,6 +332,10 @@ function readArgs() {
                 controlPlaneNums=$2
                 shift 2 
                 ;;
+            -k|--kv) 
+                kubernetesVersion=$2
+                shift 2 
+                ;;
             *) 
                 break 
                 ;;
@@ -497,7 +499,7 @@ function bringUpByoHost(){
     done
 }
 
-function createWorkCluster() {
+function createWorkloadCluster() {
     local clusterYamlFile="/tmp/cluster-yaml"
 
     # Find a available IP for control plane endpoint
@@ -527,7 +529,7 @@ function swapOff() {
 }
 
 
-function askForProceed() {
+function userConfirmation() {
     local warning='
 #####################################################################################################
 ** WARNING **
@@ -563,13 +565,10 @@ manageClusterConfFile="${HOME}/.kube/management-cluster.conf"
 kubeConfigFile=/tmp/byoh-cluster-kubeconfig
 reposDir=$(dirname $0)/../
 byohBinaryFile=${reposDir}/bin/byoh-hostagent-linux-amd64
-
-if [ -z "${KUBERNETES_VERSION}" ]; then
-    kubernetesVersion="v1.22.3"
-fi
+kubernetesVersion="v1.22.3"
 
 readArgs $@
-askForProceed
+userConfirmation
 swapOff
 intallDependencies 
 cleanUp
@@ -577,7 +576,7 @@ createKindCluster
 installByohProvider
 prepareImageAndBinary
 bringUpByoHost
-createWorkCluster
+createWorkloadCluster
 retrieveKubeConfig
 
 if [ ${defaultCni} -eq 1 ]; then

From b21f8550d89ed178a8a2935bf2f28601b621199b Mon Sep 17 00:00:00 2001
From: chen hui <huchen@vmware.com>
Date: Fri, 25 Feb 2022 07:10:43 +0000
Subject: [PATCH 11/14] comment from dharmjit

Signed-off-by: chen hui <huchen@vmware.com>
---
 hack/getting_started.sh | 41 ++++++-----------------------------------
 1 file changed, 6 insertions(+), 35 deletions(-)

diff --git a/hack/getting_started.sh b/hack/getting_started.sh
index 415a4fc9b..d006b9d3f 100755
--- a/hack/getting_started.sh
+++ b/hack/getting_started.sh
@@ -1,14 +1,14 @@
 #!/bin/bash
 function isCmdInstalled() {
     local cmd=$1
-    ${cmd} version 2>&1 | grep -q "not found"
-    returnCode=$?  
-    if [ ${returnCode} -eq 0 ] ; then
+    which ${cmd}
+    if [ $? -eq 1 ] ; then
         echo "$cmd not found"
+        return 0
     else
         echo "$cmd found"
+        return 1
     fi
-    return ${returnCode}
 }
 
 function exitIfNot(){
@@ -146,39 +146,10 @@ function calcControlPlaneIP() {
 }
 
 function installByohProvider() {
-    local writeByoh=0
-    local clusterCtlYamlFile="${HOME}/.cluster-api/clusterctl.yaml"
     local maxRunTimes=40
     local waitTime=20
 
-    # Write clusterctl.yaml
-    if [ ! -f "${clusterCtlYamlFile}" ]; then
-        writeByoh=1
-        touch ${clusterCtlYamlFile}
-    else
-        grep -q "byoh" ${clusterCtlYamlFile}
-        if [ $? -ne 0 ] ; then
-            writeByoh=1
-        fi
-    fi 
-
-    if [ ${writeByoh} -eq 1 ] ; then
-        cat>>${clusterCtlYamlFile}<<EOF
-providers:
-  - name: byoh
-    url: https://github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/releases/latest/infrastructure-components.yaml
-    type: InfrastructureProvider
-EOF
-    fi
-
-    clusterctl config repositories | grep -q byoh
-    if [ $? -ne 0 ] ; then
-        echo "Config clusterctl failed..."
-        exit 1
-    fi
-
     runCmd "clusterctl init --infrastructure byoh" 0 " Transforming the Kubernetes cluster into a management cluster..."
-
     # Waiting for byoh provider is totally ready
     for((i=1;i<=${maxRunTimes};i++));  
     do  
@@ -439,10 +410,11 @@ function prepareImageAndBinary() {
     # Check if byoh image is existed
     image=$(docker images ${byohImageName}:${byohImageTag} | grep -v REPOSITORY)
     if [ -z "${image}" ]; then
+        cp -f ${reposDir}/test/e2e/kubernetes.list ${reposDir}/test/e2e/kubernetes.list.bak
         # The origin one will report error:   Could not connect to apt.kubernetes.io:443 (10.25.207.164), connection timed out [IP: 10.25.207.164 443]
         echo "deb http://packages.cloud.google.com/apt/ kubernetes-xenial main" > ${reposDir}/test/e2e/kubernetes.list
         runCmd "make prepare-byoh-docker-host-image" 0  "Making a byoh image: ${byohImageName}:${byohImageTag} ..."
-
+        mv -f ${reposDir}/test/e2e/kubernetes.list.bak ${reposDir}/test/e2e/kubernetes.list
     else
         echo "byoh image \"${byohImageName}:${byohImageTag}\" existed."
     fi
@@ -550,7 +522,6 @@ It locally will change the following host config
     fi
 }
 
-
 export PATH=/snap/bin:${PATH}
 byohImageName="byoh/node"
 byohImageTag="v1.22.3"

From 2f724999cee88166d1ea9a8f5577fde7a742f5a3 Mon Sep 17 00:00:00 2001
From: chen hui <huchen@vmware.com>
Date: Fri, 25 Feb 2022 07:23:09 +0000
Subject: [PATCH 12/14] comment from anusha

Signed-off-by: chen hui <huchen@vmware.com>
---
 hack/getting_started.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/hack/getting_started.sh b/hack/getting_started.sh
index d006b9d3f..378d62cd7 100755
--- a/hack/getting_started.sh
+++ b/hack/getting_started.sh
@@ -153,8 +153,9 @@ function installByohProvider() {
     # Waiting for byoh provider is totally ready
     for((i=1;i<=${maxRunTimes};i++));  
     do  
-        byohStatus=$(kubectl get pods --all-namespaces | grep byoh-controller-manager | awk '{print $4}')
-        if [ "${byohStatus}" == "Running" ] ; then
+        #byohStatus=$(kubectl get pods --all-namespaces | grep byoh-controller-manager | awk '{print $4}')
+        replicas=$(kubectl get deployment byoh-controller-manager -n byoh-system -o json | jq .status.readyReplicas)
+        if [ "${replicas}" == "1" ] ; then
             echo "Byoh provider is ready"
             return
         else

From c151bf9265fdc3651ce93c782f68e1407aa72241 Mon Sep 17 00:00:00 2001
From: chen hui <huchen@vmware.com>
Date: Fri, 25 Feb 2022 07:34:00 +0000
Subject: [PATCH 13/14] string is more stable than integer

Signed-off-by: chen hui <huchen@vmware.com>
---
 hack/getting_started.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/hack/getting_started.sh b/hack/getting_started.sh
index 378d62cd7..98d59f073 100755
--- a/hack/getting_started.sh
+++ b/hack/getting_started.sh
@@ -293,7 +293,7 @@ function readArgs() {
     while true ; do
         case "$1" in
             -n|--cni) 
-                defaultCni=1
+                defaultCni="1"
                 shift 
                 ;;
             -m|--md) 
@@ -383,7 +383,7 @@ function checkNodeStatus() {
                 sleep ${waitTime}
                 continue
             fi
-            if [ ${defaultCni} -eq 1 ]; then
+            if [ "${defaultCni}" == "1" ]; then
                 status=`KUBECONFIG=${kubeConfigFile} kubectl get nodes host${i} | grep -v NAME | awk '{print $2}'`
                 if [ "${status}" != "Ready" ]; then
                     sleep ${waitTime}
@@ -532,7 +532,7 @@ controlPlaneEndPointIp=""
 workerNums=1
 controlPlaneNums=1
 byohNums=2
-defaultCni=0
+defaultCni="0"
 manageClusterConfFile="${HOME}/.kube/management-cluster.conf"
 kubeConfigFile=/tmp/byoh-cluster-kubeconfig
 reposDir=$(dirname $0)/../
@@ -551,13 +551,13 @@ bringUpByoHost
 createWorkloadCluster
 retrieveKubeConfig
 
-if [ ${defaultCni} -eq 1 ]; then
+if [ "${defaultCni}" == "1" ]; then
     installCNI
 fi
 
 checkNodeStatus
 
-if [ ${defaultCni} -eq 0 ]; then
+if [ "${defaultCni}" == "0" ]; then
     echo "Byoh cluster \"${workerClusterName}\" is successfully created, next step is to apply a CNI of your choice."
 fi
 

From c098ace00ea9a3d56a5d216241e70e7e9e7a8d7c Mon Sep 17 00:00:00 2001
From: chen hui <huchen@vmware.com>
Date: Tue, 1 Mar 2022 03:14:15 +0000
Subject: [PATCH 14/14] comment from anusha

Signed-off-by: chen hui <huchen@vmware.com>
---
 hack/getting_started.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/hack/getting_started.sh b/hack/getting_started.sh
index 98d59f073..bb57b3b5f 100755
--- a/hack/getting_started.sh
+++ b/hack/getting_started.sh
@@ -297,11 +297,11 @@ function readArgs() {
                 shift 
                 ;;
             -m|--md) 
-                workerNums=$2
+                workerCount=$2
                 shift 2 
                 ;;
             -c|--cp) 
-                controlPlaneNums=$2
+                controlPlaneCount=$2
                 shift 2 
                 ;;
             -k|--kv) 
@@ -314,7 +314,7 @@ function readArgs() {
         esac
     done
 
-    byohNums=$[${workerNums}+${controlPlaneNums}]
+    byohNums=$[${workerCount}+${controlPlaneCount}]
 }
 
 function installCNI(){
@@ -478,7 +478,7 @@ function createWorkloadCluster() {
     # Find a available IP for control plane endpoint
     calcControlPlaneIP
 
-    CONTROL_PLANE_ENDPOINT_IP=${controlPlaneEndPointIp} clusterctl generate cluster ${workerClusterName} --infrastructure byoh --kubernetes-version ${kubernetesVersion} --control-plane-machine-count ${controlPlaneNums}  --worker-machine-count ${workerNums} --flavor docker > "${clusterYamlFile}"
+    CONTROL_PLANE_ENDPOINT_IP=${controlPlaneEndPointIp} clusterctl generate cluster ${workerClusterName} --infrastructure byoh --kubernetes-version ${kubernetesVersion} --control-plane-machine-count ${controlPlaneCount}  --worker-machine-count ${workerCount} --flavor docker > "${clusterYamlFile}"
     if [ $? -ne 0 ]; then
         echo "Generate ${clusterYamlFile} failed, exiting..."
         exit 1
@@ -529,8 +529,8 @@ byohImageTag="v1.22.3"
 managerClusterName="kind-byoh"
 workerClusterName="worker-byoh"
 controlPlaneEndPointIp=""
-workerNums=1
-controlPlaneNums=1
+workerCount=1
+controlPlaneCount=1
 byohNums=2
 defaultCni="0"
 manageClusterConfFile="${HOME}/.kube/management-cluster.conf"