Enhance IT pipeline (#64)

* Enhance IT pipeline * Corrected secret name
gardener · Dec 20, 2023 · 5b9a941 · 5b9a941
1 parent f8b9ed9
commit 5b9a941
Showing 1 changed file with 89 additions and 11 deletions.
diff --git a/.ci/pipeline_integration_test b/.ci/pipeline_integration_test
@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
 ############################################## <Color> ##############################################
 RED='\033[0;31m'
 NC='\033[0m' # No Color
@@ -9,13 +11,19 @@ logs_path=.ci/controllers-test/logs
 TEST_RESULT=
 cli_path=/cc/utils/cli.py
 num_of_existing_nodes=1
+TEST_SHOOT=ali-ci-it
+CREDENTIALS_SECRET_NAME=shoot-operator-ali-team
+GARDEN_CORE_NAMESPACE=garden-core
+KUBECONFIG_EXPIRY=10800
 
 #these variables are accessed in test/integration/controller so prefixed by ${SOURCE_PATH} for absolute path
 declare CONTROL_KUBECONFIG=${SOURCE_PATH}/dev/control_kubeconfig.yaml
 declare TARGET_KUBECONFIG=${SOURCE_PATH}/dev/target_kubeconfig.yaml
+declare TEST_CREDS_SECRET=test-mc-secret
 
 export CONTROL_KUBECONFIG
 export TARGET_KUBECONFIG
+export TEST_CREDS_SECRET
 export MACHINECLASS_V1=${SOURCE_PATH}/dev/v1machineclass_converted.yaml
 export MACHINE_CONTROLLER_MANAGER_DEPLOYMENT_NAME="machine-controller-manager"
 
@@ -79,20 +87,79 @@ function setup_ginkgo() {
     echo "Successfully installed Ginkgo."
 }
 
-function fetch_control_kubeconfig() {
-    ${cli_path} config attribute --cfg-type kubernetes --cfg-name mcm-ci-ali-oot-control --key kubeconfig  > dev/control_kubeconfig.yaml
+# Fetch service account kubeconfig
+function fetch_service_account_kubeconfig() {
+    ${cli_path} config attribute --cfg-type kubernetes --cfg-name mcm-ci-garden-dev-virtual --key kubeconfig  > dev/sa_kubeconfig.yaml
+}
+
+
+function request_kubeconfig() {
+    kubectl --kubeconfig=dev/sa_kubeconfig.yaml create -f <(printf '{"spec":{"expirationSeconds":%s}}' $KUBECONFIG_EXPIRY) --raw /apis/core.gardener.cloud/v1beta1/namespaces/"$1"/shoots/"$2"/adminkubeconfig | jq -r ".status.kubeconfig" | base64 -d > "$3"
+}
+
+# Fetch *-ci-it cluster kubeconfig
+function fetch_oot_cluster_kubeconfig() {
+    NAMESPACE=garden-mcm-ci
+    request_kubeconfig $NAMESPACE $TEST_SHOOT "dev/target_kubeconfig.yaml"
+    request_kubeconfig $NAMESPACE $TEST_SHOOT "dev/control_kubeconfig.yaml"
+}
+
+function fetch_userdata(){
+    NAMESPACE='garden'
+    kubectl --kubeconfig=dev/sa_kubeconfig.yaml get shoot $TEST_SHOOT -oyaml > dev/shoot.yaml
+    SEED_NAME=$(yq eval '.status.seedName' dev/shoot.yaml)
+    request_kubeconfig $NAMESPACE $SEED_NAME "dev/seed_kubeconfig.yaml"
+    USERDATA_SECRET_NAME=$(kubectl --kubeconfig=dev/seed_kubeconfig.yaml get secrets -n shoot--mcm-ci--$TEST_SHOOT -o custom-columns=Name:.metadata.name --no-headers | grep shoot--mcm-ci)
+    kubectl --kubeconfig=dev/seed_kubeconfig.yaml get secret $USERDATA_SECRET_NAME -n shoot--mcm-ci--$TEST_SHOOT -oyaml > dev/userData.yaml
+    userData=$(yq eval '.data.userData' dev/userData.yaml)
+    echo $userData
 }
 
-function fetch_target_kubeconfig() {
-    ${cli_path} config attribute --cfg-type kubernetes --cfg-name mcm-ci-ali-oot-target --key kubeconfig > dev/target_kubeconfig.yaml
+function fetch_machine_class(){
+    MCC_NAME=$(kubectl --kubeconfig=dev/seed_kubeconfig.yaml get mcc -n shoot--mcm-ci--$TEST_SHOOT -o custom-columns=Name:.metadata.name --no-headers | grep shoot--mcm-ci)
+    kubectl --kubeconfig=dev/seed_kubeconfig.yaml get mcc $MCC_NAME -n shoot--mcm-ci--$TEST_SHOOT -oyaml > ${SOURCE_PATH}/dev/v1machineclass_converted.yaml
+    yq eval 'del(.metadata.creationTimestamp) | del(.metadata.finalizers) | del(.metadata.generation) | del(.metadata.resourceVersion) | del(.metadata.uid) | del(.credentialsSecretRef)'  dev/v1machineclass_converted.yaml -i
+    yq eval '.metadata.name = "test-mc-v1" | .metadata.namespace = "default" | .secretRef.name = env(TEST_CREDS_SECRET) | .secretRef.namespace = "default"' dev/v1machineclass_converted.yaml -i
 }
 
-function fetch_machine_class() {
-    #bringing machineclass from secret server
-    ${cli_path} config attribute --cfg-type kubernetes --cfg-name mcm-ci-ali-oot-target --key machineClass > ${SOURCE_PATH}/dev/v1machineclass.json
+function fetch_credentials(){
+   kubectl --kubeconfig=dev/sa_kubeconfig.yaml get secret $CREDENTIALS_SECRET_NAME -n $GARDEN_CORE_NAMESPACE -o yaml > dev/credentials.yaml
+   accessKeyID=$(yq eval '.data.accessKeyID' dev/credentials.yaml)
+   accessKeySecret=$(yq eval '.data.accessKeySecret' dev/credentials.yaml)
+   echo $accessKeyID
+   echo $accessKeySecret
+}
+
+function create_test_mc_secret() {
+    result=$(fetch_credentials)
+    accessKeyID=$(echo "$result" | head -n 1)
+    accessKeySecret=$(echo "$result" | tail -n 1)
+    userData=$(fetch_userdata)
+    rm -f dev/credentials.yaml
+    rm -f dev/userData.yaml
+secret_yaml=$(cat <<EOF
+apiVersion: v1
+data:
+  accessKeyID: $accessKeyID
+  accessKeySecret: $accessKeySecret
+  userData: $userData
+kind: Secret
+metadata:
+  name: $TEST_CREDS_SECRET
+  namespace: default
+type: Opaque
+EOF
+)
+    echo "$secret_yaml" | kubectl --kubeconfig=$CONTROL_KUBECONFIG apply -f -
+}
+
+function delete_test_mc_secret() {
+    kubectl --kubeconfig=$CONTROL_KUBECONFIG delete secret $TEST_CREDS_SECRET
+}
 
-    #convert json to yaml, so that machineclass can be parsed during integration-test
-    yq e -P ${SOURCE_PATH}/dev/v1machineclass.json > ${SOURCE_PATH}/dev/v1machineclass_converted.yaml
+function  cleanup_environment() {
+    printf "\n\t\t\t----- Deleting the test_mc_secret --------\n"
+    delete_test_mc_secret
 }
 
 function setup_environment() {
@@ -118,15 +185,25 @@ function setup_environment() {
     fi
 
     #fetching kubeconfigs and machineClass from secret_server
-    fetch_control_kubeconfig
-    fetch_target_kubeconfig
+    fetch_service_account_kubeconfig
+    fetch_oot_cluster_kubeconfig
+    create_test_mc_secret
+    echo "test-mc-secret created successfully"
     fetch_machine_class
+    echo "machineclass fetched successfully"
+
+}
+
+function fetch_existing_nodes() {
+    kubectl --kubeconfig=$TARGET_KUBECONFIG get nodes --no-headers | wc -l
 }
 
 function check_cluster_state() {
     printf "\t\t\t----- Checking Test Environment -------\n"
 
     printf "\nChecking existance of machine crds\n"
+    num_of_existing_nodes=$(($(fetch_existing_nodes)))
+    printf "\nexisting_default_nodes=$num_of_existing_nodes\n"
     # Wait 60mins for any existing PRs to cleanup machine crds, as crd cleanup is last step.
     hf_wait_on "hf_num_of_objects" crd 0 3600
     printf "No machine crds in control test cluster\n"
@@ -187,6 +264,7 @@ if [ ${result} -ne 0 ]; then
 exit $result
 fi
 run_integration_tests
+cleanup_environment
 print_controller_logs
 printf "\n\t\t\t----- End of Test Script -----------\n"