Add IT in pipeline

.ci/pipeline_integration_test

@@ -0,0 +1,196 @@
+############################################## <Color> ##############################################
+RED='\033[0;31m'
+NC='\033[0m' # No Color
+############################################## </Color> ##############################################
+
+mkdir -p dev
+
+logs_path=.ci/controllers-test/logs
+TEST_RESULT=
+cli_path=/cc/utils/cli.py
+num_of_existing_nodes=1
+
+#these variables are accessed in test/integration/controller so prefixed by ${SOURCE_PATH} for absolute path
+declare CONTROL_KUBECONFIG=${SOURCE_PATH}/dev/control_kubeconfig.yaml
+declare TARGET_KUBECONFIG=${SOURCE_PATH}/dev/target_kubeconfig.yaml
+
+export CONTROL_KUBECONFIG
+export TARGET_KUBECONFIG
+export MACHINECLASS_V1=${SOURCE_PATH}/dev/v1machineclass_converted.yaml
+export MACHINE_CONTROLLER_MANAGER_DEPLOYMENT_NAME="machine-controller-manager"
+
+############################################## <Helper fn> ##############################################
+
+function hf_num_of_objects() {
+    output=$(kubectl --kubeconfig=dev/control_kubeconfig.yaml get "$1" | grep machine.sapcloud.io 2>&1)
+
+    if [ -z "$output" ]; then
+       return 0
+    fi
+
+    object_count=$(echo "$output" | wc -l)
+
+    return "$object_count"
+}
+
+function hf_num_of_ready_nodes() {
+    output=$(kubectl --kubeconfig=dev/target_kubeconfig.yaml get "$1" 2>&1)
+
+    ready_count=$(echo "$output" | tr " " "\n" | grep ^Ready -c)
+
+    return $((ready_count-num_of_existing_nodes))
+}
+
+function hf_wait_on() {
+    function_name=$1
+    function_param=$2
+    count_to_match=$3
+    seconds_to_wait=$4
+    iteration_count=$(($seconds_to_wait/30))
+
+    while
+        "$function_name" "$function_param"
+        ret=$?
+        [[ $ret -ne $count_to_match ]]
+    do
+        sleep 30
+        ((iteration_count--))
+
+        # Exit script when timeout occurs
+        if [ $iteration_count -le 0 ]; then
+            printf "\tFailed: Timeout occured while waiting for operation. Exiting Test to avoid further conflicts.\n"
+            printf "\tWas Executing function: %s, %s\n" $function_name $function_param
+            printf "${RED}There is another PR running its integration test on the clusters. Waiting Timed Out. Kindly re-run the tests.${NC}\n"
+            exit 1
+        fi
+
+    done
+}
+
+############################################## </Helper fn> ##############################################
+
+
+############################################## <Initialization> ##############################################
+
+function setup_ginkgo() {
+    echo "Installing Ginkgo..."
+    GO111MODULE=off go get -u github.com/onsi/ginkgo/ginkgo
+    ginkgo version
+    echo "Successfully installed Ginkgo."
+}
+
+function fetch_control_kubeconfig() {
+    ${cli_path} config attribute --cfg-type kubernetes --cfg-name mcm-ci-ali-oot-control --key kubeconfig  > dev/control_kubeconfig.yaml
+}
+
+function fetch_target_kubeconfig() {
+    ${cli_path} config attribute --cfg-type kubernetes --cfg-name mcm-ci-ali-oot-target --key kubeconfig > dev/target_kubeconfig.yaml
+}
+
+function fetch_machine_class() {
+    #bringing machineclass from secret server
+    ${cli_path} config attribute --cfg-type kubernetes --cfg-name mcm-ci-ali-oot-target --key machineClass > ${SOURCE_PATH}/dev/v1machineclass.json
+
+    #convert json to yaml, so that machineclass can be parsed during integration-test
+    yq e -P ${SOURCE_PATH}/dev/v1machineclass.json > ${SOURCE_PATH}/dev/v1machineclass_converted.yaml
+    yq -i e '.providerSpec.keyPairName |= null' ${SOURCE_PATH}/dev/v1machineclass_converted.yaml
+}
+
+function setup_environment() {
+    printf "\n\t\t\t----- Setup Test Environment --------\n"
+
+    #installing yq
+    printf "\nDownloading and installing yq\n"
+    curl -LO https://github.com/mikefarah/yq/releases/download/v4.13.3/yq_linux_amd64
+    chmod +x ./yq_linux_amd64
+    mv ./yq_linux_amd64 /usr/local/bin/yq
+    printf "Successfully installed yq\n"
+
+    # install kubectl
+    printf "\nDownloading and installing kubectl\n"
+    curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.16.0/bin/linux/amd64/kubectl
+    chmod +x ./kubectl
+    mv ./kubectl /usr/local/bin/kubectl
+    printf "Successfully installed kubectl\n"
+
+    #install ginkgo
+    if ! [ -x "$(command -v ginkgo)" ]; then
+    setup_ginkgo
+    fi
+
+    #fetching kubeconfigs and machineClass from secret_server
+    fetch_control_kubeconfig
+    fetch_target_kubeconfig
+    fetch_machine_class
+}
+
+function check_cluster_state() {
+    printf "\t\t\t----- Checking Test Environment -------\n"
+
+    printf "\nChecking existance of machine crds\n"
+    # Wait 60mins for any existing PRs to cleanup machine crds, as crd cleanup is last step.
+    hf_wait_on "hf_num_of_objects" crd 0 3600
+    printf "No machine crds in control test cluster\n"
+
+    printf "\nChecking existance of node objects\n"
+    # Wait 60mins for any existing PRs to cleanup nodes
+    hf_wait_on "hf_num_of_ready_nodes" nodes 0 3600
+    printf "No additional node objects in target test cluster\n"
+
+    #wait in case some orphan resources are terminating
+    sleep 30
+
+    printf "\nCluster state looks clean\n"
+    printf "\t\t\t----- Checking Test Environment DONE -------\n"
+}
+
+############################################## </Initialization> ##############################################
+
+############################################## <Modules> ########################################################
+
+function run_integration_tests() {
+    echo "Starting integration tests..."
+    set +e
+
+    ginkgo -v -mod=vendor test/integration/controller
+    TEST_RESULT=$?
+
+    set -e
+
+    if [ ${TEST_RESULT} -ne 0 ]; then
+        printf "\n\t\t\t${RED}Integration tests failed. Kindly check your PR${NC}\n"
+    else
+        printf "Done with integration test\n"
+    fi
+}
+
+function print_controller_logs {
+    printf "\n\t\t\t----- Start of MCM Logs -----------\n"
+    cat $logs_path/mcm_process.log
+    printf "\n\t\t\t----- End of MCM Logs   ----------\n\n"
+
+    printf "\n\t\t\t----- Start of MC Logs -----------\n"
+    cat $logs_path/mc_process.log
+    printf "\n\t\t\t----- End of MC Logs   ----------\n\n"
+}
+
+############################################## </Modules> ########################################################
+
+
+############################################## <Main> ########################################################
+
+printf "\n\t\t\t----- Start of Test Script -----------\n"
+setup_environment
+#if cluster state is not clean then don't run the tests
+check_cluster_state
+result=$?
+if [ ${result} -ne 0 ]; then
+exit $result
+fi
+run_integration_tests
+print_controller_logs
+printf "\n\t\t\t----- End of Test Script -----------\n"
+
+exit $TEST_RESULT
+
+############################################## </Main> ########################################################

.ci/test

@@ -53,12 +53,12 @@ else
   echo ">>>>> Finished executing unit tests"
 fi
 
-#if [[ "${SKIP_INTEGRATION_TESTS}" != "" ]]; then
-#  echo ">>>>> Skipping integration tests"
-#else
-#  echo ">>>>> Invoking intergration tests"
-#  .ci/pipeline_integration_test
-#  echo ">>>>> Finished executing integration tests"
-#fi
+if [[ "${SKIP_INTEGRATION_TESTS}" != "" ]]; then
+  echo ">>>>> Skipping integration tests"
+else
+  echo ">>>>> Invoking intergration tests"
+  .ci/pipeline_integration_test
+  echo ">>>>> Finished executing integration tests"
+fi
 
 echo "CI tests have passed successfully"

Makefile

@@ -19,20 +19,20 @@ IMAGE_TAG           := $(shell cat VERSION)
 PROVIDER_NAME       := alicloud
 PROJECT_NAME        := gardener
 CONTROL_NAMESPACE   := default
-CONTROL_KUBECONFIG  := dev/target-kubeconfig.yaml
+CONTROL_KUBECONFIG  := dev/control-kubeconfig.yaml
 TARGET_KUBECONFIG   := dev/target-kubeconfig.yaml
 
 # Below ones are used in tests
 MACHINECLASS_V1 	:= dev/machineclassv1.yaml
 MACHINECLASS_V2 	:=
 MCM_IMAGE			:=
 MC_IMAGE			:=
-# MCM_IMAGE			:= eu.gcr.io/gardener-project/gardener/machine-controller-manager:v0.39.0
-# MC_IMAGE			:= $(IMAGE_REPOSITORY):v0.7.0
+# MCM_IMAGE			:= eu.gcr.io/gardener-project/gardener/machine-controller-manager:v0.46.0
+# MC_IMAGE			:= $(IMAGE_REPOSITORY):v0.6.0
 LEADER_ELECT 	    := "true"
 # If Integration Test Suite is to be run locally against clusters then export the below variable
 # with MCM deployment name in the cluster
-MACHINE_CONTROLLER_MANAGER_DEPLOYMENT_NAME := machine-controller-manager
+# MACHINE_CONTROLLER_MANAGER_DEPLOYMENT_NAME := machine-controller-manager
 
 #########################################
 # Rules for running helper scripts

kubernetes/deployment.yaml

@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2021 SAP SE or an SAP affiliate company and Gardener contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+
 # Sample deployment file, used to run Machine Controller Manager on your cluster
 
 apiVersion: apps/v1 # Version may change based on kubernetes version
@@ -6,26 +10,29 @@ metadata:
   name: machine-controller-manager
 spec:
   replicas: 1
+  selector:
+    matchLabels:
+      role: machine-controller-manager
   template:
     metadata:
       labels:
-        app: machine-controller-manager
+        role: machine-controller-manager
     spec:
       containers:
       - name: machine-controller-manager
-        image:eu.gcr.io/gardener-project/gardener/machine-controller-manager:v0.28.0-dev-793b105c41adfc434e07107decaf1ac67fc1fd3f
+        image: eu.gcr.io/gardener-project/gardener/machine-controller-manager:v0.46.0
         imagePullPolicy: Always
         command:
           - ./machine-controller-manager
-          - --target-kubeconfig=$(TARGET_KUBECONFIG) # Mandatory Parameter - Filepath to the target cluster's kubeconfig where node objects are expected to join.
-          - --control-kubeconfig=$(CONTROL_KUBECONFIG) # Optional Parameter - Default value is same as target-kubeconfig - Filepath to the control cluster's kubeconfig where machine objects would be created. Optionally you could also use "inClusterConfig" when pod is running inside control kubeconfig.
-          - --namespace=$(CONTROL_NAMESPACE) # Optional Parameter - Default value for namespace is 'default' - The control namespace where the controller watches for it's machine objects.
+          - --target-kubeconfig=/var/lib/machine-controller-manager/kubeconfig #$(TARGET_KUBECONFIG) Mandatory Parameter - Filepath to the target cluster's kubeconfig where node objects are expected to join.
+          - --control-kubeconfig=inClusterConfig #(CONTROL_KUBECONFIG) Optional Parameter - Default value is same as target-kubeconfig - Filepath to the control cluster's kubeconfig where machine objects would be created. Optionally you could also use "inClusterConfig" when pod is running inside control kubeconfig.
+          #- --namespace=$(CONTROL_NAMESPACE) # Optional Parameter - Default value for namespace is 'default' - The control namespace where the controller watches for it's machine objects.
           - --safety-up=2 # Optional Parameter - Default value 2 - The number of excess machine objects permitted for any machineSet/machineDeployment beyond its expected number of replicas based on desired and max-surge, we call this the upper-limit. When this upper-limit is reached, the objects are frozen until the number of objects reduce. upper-limit = desired + maxSurge (if applicable) + safetyUp.
           - --safety-down=1 # Optional Parameter - Default value 1 - Upper-limit minus safety-down value gives the lower-limit. This is the limits below which any temporarily frozen machineSet/machineDeployment object is unfrozen. lower-limit = desired + maxSurge (if applicable) + safetyUp - safetyDown.
           - --machine-drain-timeout=5m # Optional Parameter - Timeout (in time) used while draining of machine before deletion, beyond which MCM forcefully deletes machine.
           - --machine-health-timeout=10m  # Optional Parameter - Default value 10mins - Timeout (in time) used while joining (during creation) or re-joining (in case of temporary health issues) of machine before it is declared as failed.
-          - --machine-safety-orphan-vms-period=30 # Optional Parameter - Default value 30mins - Time period (in time) used to poll for orphan VMs by safety controller.
-          - --machine-safety-overshooting-period=1 # Optional Parameter - Default value 1min - Time period (in time) used to poll for overshooting of machine objects backing a machineSet by safety controller.
+          - --machine-safety-orphan-vms-period=30m # Optional Parameter - Default value 30mins - Time period (in time) used to poll for orphan VMs by safety controller.
+          - --machine-safety-overshooting-period=1m # Optional Parameter - Default value 1min - Time period (in time) used to poll for overshooting of machine objects backing a machineSet by safety controller.
           - --node-conditions=ReadonlyFilesystem,KernelDeadlock,DiskPressure # List of comma-separated/case-sensitive node-conditions which when set to True will change machine to a failed state after MachineHealthTimeout duration. It may further be replaced with a new machine if the machine is backed by a machine-set object.
           - --v=3
         livenessProbe:
@@ -38,17 +45,21 @@ spec:
           periodSeconds: 10
           successThreshold: 1
           timeoutSeconds: 5
+        volumeMounts:
+        - mountPath: /var/lib/machine-controller-manager
+          name: machine-controller-manager
+          readOnly: true
       - command:
         - ./machine-controller
-        - --control-kubeconfig=$(TARGET_KUBECONFIG) # Mandatory Parameter - Filepath to the target cluster's kubeconfig where node objects are expected to join.
-        - --target-kubeconfig=$(CONTROL_KUBECONFIG) # Optional Parameter - Default value is same as target-kubeconfig - Filepath to the control cluster's kubeconfig where machine objects would be created. Optionally you could also use "inClusterConfig" when pod is running inside control kubeconfig.
-        - --namespace=$(CONTROL_NAMESPACE) # Optional Parameter - Default value for namespace is 'default' - The control namespace where the controller watches for it's machine objects.
+        - --control-kubeconfig=inClusterConfig  #$(CONTROL_KUBECONFIG) # Optional Parameter - Default value is same as target-kubeconfig - Filepath to the control cluster's kubeconfig where machine objects would be created. Optionally you could also use "inClusterConfig" when pod is running inside control kubeconfig.
+        - --target-kubeconfig=/var/lib/machine-controller-manager/kubeconfig  #$(TARGET_KUBECONFIG) # Mandatory Parameter - Filepath to the target cluster's kubeconfig where node objects are expected to join.
+        #- --namespace=$(CONTROL_NAMESPACE) # Optional Parameter - Default value for namespace is 'default' - The control namespace where the controller watches for it's machine objects.
         - --machine-drain-timeout=5m # Optional Parameter - Timeout (in time) used while draining of machine before deletion, beyond which MCM forcefully deletes machine.
         - --machine-health-timeout=10m  # Optional Parameter - Default value 10mins - Timeout (in time) used while joining (during creation) or re-joining (in case of temporary health issues) of machine before it is declared as failed.
         - --machine-safety-orphan-vms-period=30m # Optional Parameter - Default value 30mins - Time period (in time) used to poll for orphan VMs by safety controller.
         - --node-conditions=ReadonlyFilesystem,KernelDeadlock,DiskPressure # List of comma-separated/case-sensitive node-conditions which when set to True will change machine to a failed state after MachineHealthTimeout duration. It may further be replaced with a new machine if the machine is backed by a machine-set object.
         - --v=3
-        image: gcr.io/gardener-project/gardener/machine-controller-manager-provider-aws:0.1.0-dev-47bc8cf5b02affba97bfb7b0e57202947d397b4c
+        image: eu.gcr.io/gardener-project/gardener/machine-controller-manager-provider-alicloud:v0.6.0
         imagePullPolicy: IfNotPresent
         livenessProbe:
           failureThreshold: 3
@@ -72,10 +83,19 @@ spec:
           requests:
             cpu: 50m
             memory: 64Mi
+        volumeMounts:
+        - mountPath: /var/lib/machine-controller-manager
+          name: machine-controller-manager
+          readOnly: true
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
         securityContext: {}
         serviceAccount: machine-controller-manager
         serviceAccountName: machine-controller-manager
       dnsPolicy: ClusterFirst
       restartPolicy: Always
+      volumes:
+      - name: machine-controller-manager
+        secret:
+          defaultMode: 420
+          secretName: machine-controller-manager