From ee9341bb6d9aff9052aa875e27403e4c5776f53d Mon Sep 17 00:00:00 2001 From: Rishabh Patel Date: Mon, 8 Aug 2022 17:34:52 +0530 Subject: [PATCH] adding IT in pipeline --- .ci/pipeline_integration_test | 195 ++++++++++++++++++++++++++++++++++ .ci/test | 14 +-- Makefile | 12 +-- kubernetes/deployment.yaml | 42 ++++++-- 4 files changed, 239 insertions(+), 24 deletions(-) create mode 100644 .ci/pipeline_integration_test diff --git a/.ci/pipeline_integration_test b/.ci/pipeline_integration_test new file mode 100644 index 00000000..a9d12458 --- /dev/null +++ b/.ci/pipeline_integration_test @@ -0,0 +1,195 @@ +############################################## ############################################## +RED='\033[0;31m' +NC='\033[0m' # No Color +############################################## ############################################## + +mkdir -p dev + +logs_path=.ci/controllers-test/logs +TEST_RESULT= +cli_path=/cc/utils/cli.py +num_of_existing_nodes=1 + +#these variables are accessed in test/integration/controller so prefixed by ${SOURCE_PATH} for absolute path +declare CONTROL_KUBECONFIG=${SOURCE_PATH}/dev/control_kubeconfig.yaml +declare TARGET_KUBECONFIG=${SOURCE_PATH}/dev/target_kubeconfig.yaml + +export CONTROL_KUBECONFIG +export TARGET_KUBECONFIG +export MACHINECLASS_V1=${SOURCE_PATH}/dev/v1machineclass_converted.yaml +export MACHINE_CONTROLLER_MANAGER_DEPLOYMENT_NAME="machine-controller-manager" + +############################################## ############################################## + +function hf_num_of_objects() { + output=$(kubectl --kubeconfig=dev/control_kubeconfig.yaml get "$1" | grep machine.sapcloud.io 2>&1) + + if [ -z "$output" ]; then + return 0 + fi + + object_count=$(echo "$output" | wc -l) + + return "$object_count" +} + +function hf_num_of_ready_nodes() { + output=$(kubectl --kubeconfig=dev/target_kubeconfig.yaml get "$1" 2>&1) + + ready_count=$(echo "$output" | tr " " "\n" | grep ^Ready -c) + + return $((ready_count-num_of_existing_nodes)) +} + +function hf_wait_on() { + function_name=$1 + function_param=$2 + count_to_match=$3 + seconds_to_wait=$4 + iteration_count=$(($seconds_to_wait/30)) + + while + "$function_name" "$function_param" + ret=$? + [[ $ret -ne $count_to_match ]] + do + sleep 30 + ((iteration_count--)) + + # Exit script when timeout occurs + if [ $iteration_count -le 0 ]; then + printf "\tFailed: Timeout occured while waiting for operation. Exiting Test to avoid further conflicts.\n" + printf "\tWas Executing function: %s, %s\n" $function_name $function_param + printf "${RED}There is another PR running its integration test on the clusters. Waiting Timed Out. Kindly re-run the tests.${NC}\n" + exit 1 + fi + + done +} + +############################################## ############################################## + + +############################################## ############################################## + +function setup_ginkgo() { + echo "Installing Ginkgo..." + GO111MODULE=off go get -u github.com/onsi/ginkgo/ginkgo + ginkgo version + echo "Successfully installed Ginkgo." +} + +function fetch_control_kubeconfig() { + ${cli_path} config attribute --cfg-type kubernetes --cfg-name mcm-ci-ali-oot-control --key kubeconfig > dev/control_kubeconfig.yaml +} + +function fetch_target_kubeconfig() { + ${cli_path} config attribute --cfg-type kubernetes --cfg-name mcm-ci-ali-oot-target --key kubeconfig > dev/target_kubeconfig.yaml +} + +function fetch_machine_class() { + #bringing machineclass from secret server + ${cli_path} config attribute --cfg-type kubernetes --cfg-name mcm-ci-ali-oot-target --key machineClass > ${SOURCE_PATH}/dev/v1machineclass.json + + #convert json to yaml, so that machineclass can be parsed during integration-test + yq e -P ${SOURCE_PATH}/dev/v1machineclass.json > ${SOURCE_PATH}/dev/v1machineclass_converted.yaml +} + +function setup_environment() { + printf "\n\t\t\t----- Setup Test Environment --------\n" + + #installing yq + printf "\nDownloading and installing yq\n" + curl -LO https://github.com/mikefarah/yq/releases/download/v4.13.3/yq_linux_amd64 + chmod +x ./yq_linux_amd64 + mv ./yq_linux_amd64 /usr/local/bin/yq + printf "Successfully installed yq\n" + + # install kubectl + printf "\nDownloading and installing kubectl\n" + curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.16.0/bin/linux/amd64/kubectl + chmod +x ./kubectl + mv ./kubectl /usr/local/bin/kubectl + printf "Successfully installed kubectl\n" + + #install ginkgo + if ! [ -x "$(command -v ginkgo)" ]; then + setup_ginkgo + fi + + #fetching kubeconfigs and machineClass from secret_server + fetch_control_kubeconfig + fetch_target_kubeconfig + fetch_machine_class +} + +function check_cluster_state() { + printf "\t\t\t----- Checking Test Environment -------\n" + + printf "\nChecking existance of machine crds\n" + # Wait 60mins for any existing PRs to cleanup machine crds, as crd cleanup is last step. + hf_wait_on "hf_num_of_objects" crd 0 3600 + printf "No machine crds in control test cluster\n" + + printf "\nChecking existance of node objects\n" + # Wait 60mins for any existing PRs to cleanup nodes + hf_wait_on "hf_num_of_ready_nodes" nodes 0 3600 + printf "No additional node objects in target test cluster\n" + + #wait in case some orphan resources are terminating + sleep 30 + + printf "\nCluster state looks clean\n" + printf "\t\t\t----- Checking Test Environment DONE -------\n" +} + +############################################## ############################################## + +############################################## ######################################################## + +function run_integration_tests() { + echo "Starting integration tests..." + set +e + + ginkgo -v -mod=vendor test/integration/controller + TEST_RESULT=$? + + set -e + + if [ ${TEST_RESULT} -ne 0 ]; then + printf "\n\t\t\t${RED}Integration tests failed. Kindly check you PR${NC}\n" + else + printf "Done with integration test\n" + fi +} + +function print_controller_logs { + printf "\n\t\t\t----- Start of MCM Logs -----------\n" + cat $logs_path/mcm_process.log + printf "\n\t\t\t----- End of MCM Logs ----------\n\n" + + printf "\n\t\t\t----- Start of MC Logs -----------\n" + cat $logs_path/mc_process.log + printf "\n\t\t\t----- End of MC Logs ----------\n\n" +} + +############################################## ######################################################## + + +##############################################
######################################################## + +printf "\n\t\t\t----- Start of Test Script -----------\n" +setup_environment +#if cluster state is not clean then don't run the tests +check_cluster_state +result=$? +if [ ${result} -ne 0 ]; then +exit $result +fi +run_integration_tests +print_controller_logs +printf "\n\t\t\t----- End of Test Script -----------\n" + +exit $TEST_RESULT + +##############################################
######################################################## \ No newline at end of file diff --git a/.ci/test b/.ci/test index c894a848..5552fee8 100755 --- a/.ci/test +++ b/.ci/test @@ -53,12 +53,12 @@ else echo ">>>>> Finished executing unit tests" fi -#if [[ "${SKIP_INTEGRATION_TESTS}" != "" ]]; then -# echo ">>>>> Skipping integration tests" -#else -# echo ">>>>> Invoking intergration tests" -# .ci/pipeline_integration_test -# echo ">>>>> Finished executing integration tests" -#fi +if [[ "${SKIP_INTEGRATION_TESTS}" != "" ]]; then + echo ">>>>> Skipping integration tests" +else + echo ">>>>> Invoking intergration tests" + .ci/pipeline_integration_test + echo ">>>>> Finished executing integration tests" +fi echo "CI tests have passed successfully" \ No newline at end of file diff --git a/Makefile b/Makefile index ea28d538..db62ac0e 100644 --- a/Makefile +++ b/Makefile @@ -19,20 +19,20 @@ IMAGE_TAG := $(shell cat VERSION) PROVIDER_NAME := alicloud PROJECT_NAME := gardener CONTROL_NAMESPACE := default -CONTROL_KUBECONFIG := dev/target-kubeconfig.yaml -TARGET_KUBECONFIG := dev/target-kubeconfig.yaml +CONTROL_KUBECONFIG := ./dev/kubeconfigs/kubeconfig_control.yaml +TARGET_KUBECONFIG := ./dev/kubeconfigs/kubeconfig_target.yaml # Below ones are used in tests -MACHINECLASS_V1 := dev/machineclassv1.yaml +MACHINECLASS_V1 := ./dev/machineclassv1.yaml MACHINECLASS_V2 := MCM_IMAGE := MC_IMAGE := -# MCM_IMAGE := eu.gcr.io/gardener-project/gardener/machine-controller-manager:v0.39.0 -# MC_IMAGE := $(IMAGE_REPOSITORY):v0.7.0 +# MCM_IMAGE := eu.gcr.io/gardener-project/gardener/machine-controller-manager:v0.46.0 +# MC_IMAGE := $(IMAGE_REPOSITORY):v0.6.0 LEADER_ELECT := "true" # If Integration Test Suite is to be run locally against clusters then export the below variable # with MCM deployment name in the cluster -MACHINE_CONTROLLER_MANAGER_DEPLOYMENT_NAME := machine-controller-manager +#MACHINE_CONTROLLER_MANAGER_DEPLOYMENT_NAME := machine-controller-manager ######################################### # Rules for running helper scripts diff --git a/kubernetes/deployment.yaml b/kubernetes/deployment.yaml index 509667da..8199f99d 100644 --- a/kubernetes/deployment.yaml +++ b/kubernetes/deployment.yaml @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2021 SAP SE or an SAP affiliate company and Gardener contributors +# +# SPDX-License-Identifier: Apache-2.0 + # Sample deployment file, used to run Machine Controller Manager on your cluster apiVersion: apps/v1 # Version may change based on kubernetes version @@ -6,26 +10,29 @@ metadata: name: machine-controller-manager spec: replicas: 1 + selector: + matchLabels: + role: machine-controller-manager template: metadata: labels: - app: machine-controller-manager + role: machine-controller-manager spec: containers: - name: machine-controller-manager - image:eu.gcr.io/gardener-project/gardener/machine-controller-manager:v0.28.0-dev-793b105c41adfc434e07107decaf1ac67fc1fd3f + image: eu.gcr.io/gardener-project/gardener/machine-controller-manager:v0.46.0 imagePullPolicy: Always command: - ./machine-controller-manager - - --target-kubeconfig=$(TARGET_KUBECONFIG) # Mandatory Parameter - Filepath to the target cluster's kubeconfig where node objects are expected to join. - - --control-kubeconfig=$(CONTROL_KUBECONFIG) # Optional Parameter - Default value is same as target-kubeconfig - Filepath to the control cluster's kubeconfig where machine objects would be created. Optionally you could also use "inClusterConfig" when pod is running inside control kubeconfig. - - --namespace=$(CONTROL_NAMESPACE) # Optional Parameter - Default value for namespace is 'default' - The control namespace where the controller watches for it's machine objects. + - --target-kubeconfig=/var/lib/machine-controller-manager/kubeconfig #$(TARGET_KUBECONFIG) Mandatory Parameter - Filepath to the target cluster's kubeconfig where node objects are expected to join. + - --control-kubeconfig=inClusterConfig #(CONTROL_KUBECONFIG) Optional Parameter - Default value is same as target-kubeconfig - Filepath to the control cluster's kubeconfig where machine objects would be created. Optionally you could also use "inClusterConfig" when pod is running inside control kubeconfig. + #- --namespace=$(CONTROL_NAMESPACE) # Optional Parameter - Default value for namespace is 'default' - The control namespace where the controller watches for it's machine objects. - --safety-up=2 # Optional Parameter - Default value 2 - The number of excess machine objects permitted for any machineSet/machineDeployment beyond its expected number of replicas based on desired and max-surge, we call this the upper-limit. When this upper-limit is reached, the objects are frozen until the number of objects reduce. upper-limit = desired + maxSurge (if applicable) + safetyUp. - --safety-down=1 # Optional Parameter - Default value 1 - Upper-limit minus safety-down value gives the lower-limit. This is the limits below which any temporarily frozen machineSet/machineDeployment object is unfrozen. lower-limit = desired + maxSurge (if applicable) + safetyUp - safetyDown. - --machine-drain-timeout=5m # Optional Parameter - Timeout (in time) used while draining of machine before deletion, beyond which MCM forcefully deletes machine. - --machine-health-timeout=10m # Optional Parameter - Default value 10mins - Timeout (in time) used while joining (during creation) or re-joining (in case of temporary health issues) of machine before it is declared as failed. - - --machine-safety-orphan-vms-period=30 # Optional Parameter - Default value 30mins - Time period (in time) used to poll for orphan VMs by safety controller. - - --machine-safety-overshooting-period=1 # Optional Parameter - Default value 1min - Time period (in time) used to poll for overshooting of machine objects backing a machineSet by safety controller. + - --machine-safety-orphan-vms-period=30m # Optional Parameter - Default value 30mins - Time period (in time) used to poll for orphan VMs by safety controller. + - --machine-safety-overshooting-period=1m # Optional Parameter - Default value 1min - Time period (in time) used to poll for overshooting of machine objects backing a machineSet by safety controller. - --node-conditions=ReadonlyFilesystem,KernelDeadlock,DiskPressure # List of comma-separated/case-sensitive node-conditions which when set to True will change machine to a failed state after MachineHealthTimeout duration. It may further be replaced with a new machine if the machine is backed by a machine-set object. - --v=3 livenessProbe: @@ -38,17 +45,21 @@ spec: periodSeconds: 10 successThreshold: 1 timeoutSeconds: 5 + volumeMounts: + - mountPath: /var/lib/machine-controller-manager + name: machine-controller-manager + readOnly: true - command: - ./machine-controller - - --control-kubeconfig=$(TARGET_KUBECONFIG) # Mandatory Parameter - Filepath to the target cluster's kubeconfig where node objects are expected to join. - - --target-kubeconfig=$(CONTROL_KUBECONFIG) # Optional Parameter - Default value is same as target-kubeconfig - Filepath to the control cluster's kubeconfig where machine objects would be created. Optionally you could also use "inClusterConfig" when pod is running inside control kubeconfig. - - --namespace=$(CONTROL_NAMESPACE) # Optional Parameter - Default value for namespace is 'default' - The control namespace where the controller watches for it's machine objects. + - --control-kubeconfig=inClusterConfig #$(CONTROL_KUBECONFIG) # Optional Parameter - Default value is same as target-kubeconfig - Filepath to the control cluster's kubeconfig where machine objects would be created. Optionally you could also use "inClusterConfig" when pod is running inside control kubeconfig. + - --target-kubeconfig=/var/lib/machine-controller-manager/kubeconfig #$(TARGET_KUBECONFIG) # Mandatory Parameter - Filepath to the target cluster's kubeconfig where node objects are expected to join. + #- --namespace=$(CONTROL_NAMESPACE) # Optional Parameter - Default value for namespace is 'default' - The control namespace where the controller watches for it's machine objects. - --machine-drain-timeout=5m # Optional Parameter - Timeout (in time) used while draining of machine before deletion, beyond which MCM forcefully deletes machine. - --machine-health-timeout=10m # Optional Parameter - Default value 10mins - Timeout (in time) used while joining (during creation) or re-joining (in case of temporary health issues) of machine before it is declared as failed. - --machine-safety-orphan-vms-period=30m # Optional Parameter - Default value 30mins - Time period (in time) used to poll for orphan VMs by safety controller. - --node-conditions=ReadonlyFilesystem,KernelDeadlock,DiskPressure # List of comma-separated/case-sensitive node-conditions which when set to True will change machine to a failed state after MachineHealthTimeout duration. It may further be replaced with a new machine if the machine is backed by a machine-set object. - --v=3 - image: gcr.io/gardener-project/gardener/machine-controller-manager-provider-aws:0.1.0-dev-47bc8cf5b02affba97bfb7b0e57202947d397b4c + image: eu.gcr.io/gardener-project/gardener/machine-controller-manager-provider-alicloud:v0.6.0 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 3 @@ -72,6 +83,10 @@ spec: requests: cpu: 50m memory: 64Mi + volumeMounts: + - mountPath: /var/lib/machine-controller-manager + name: machine-controller-manager + readOnly: true terminationMessagePath: /dev/termination-log terminationMessagePolicy: File securityContext: {} @@ -79,3 +94,8 @@ spec: serviceAccountName: machine-controller-manager dnsPolicy: ClusterFirst restartPolicy: Always + volumes: + - name: machine-controller-manager + secret: + defaultMode: 420 + secretName: machine-controller-manager