diff --git a/.github/workflows/kubernetes-tests.yml b/.github/workflows/kubernetes-tests.yml index 102742ffedd..581d0415082 100644 --- a/.github/workflows/kubernetes-tests.yml +++ b/.github/workflows/kubernetes-tests.yml @@ -255,7 +255,29 @@ jobs: - name: Test PolicyBinding CRD and sts call on kind run: | "${GITHUB_WORKSPACE}/testing/test-policy-binding.sh" - + test-decommission: + timeout-minutes: 30 + needs: + - operator + runs-on: ${{ matrix.os }} + strategy: + matrix: + go-version: [ 1.21.x ] + os: [ ubuntu-latest ] + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: ${{ matrix.go-version }} + - uses: actions/cache@v3 + name: Operator Binary Cache + with: + path: | + ./minio-operator + key: ${{ runner.os }}-binary-${{ github.run_id }} + - name: Decommission Test + run: | + "${GITHUB_WORKSPACE}/testing/decommission-test.sh" helm: timeout-minutes: 30 # The type of runner that the job will run on @@ -321,3 +343,4 @@ jobs: chmod +x kubectl mv kubectl /usr/local/bin "${GITHUB_WORKSPACE}/testing/check-helm.sh" + diff --git a/testing/common.sh b/testing/common.sh index 3381de002de..a43c0d578de 100644 --- a/testing/common.sh +++ b/testing/common.sh @@ -47,6 +47,7 @@ die() { try() { "$@" || die "cannot $*"; } function setup_kind() { + echo "setup_kind():" if [ "$TEST_FLOOR" = "true" ]; then try kind create cluster --config "${SCRIPT_DIR}/kind-config-floor.yaml" else @@ -67,8 +68,471 @@ function install_cert_manager() { try kubectl wait -n cert-manager --for=condition=ready pod -l app=webhook --timeout=120s } -function install_operator() { +# removes the pool from the provided tenant. +# usage: remove_decommissioned_pool +function remove_decommissioned_pool() { + + TENANT_NAME=$1 + NAMESPACE=$2 + + # While there is a conflict, let's retry + RESULT=1 + while [ "$RESULT" == "1" ]; do + sleep 10 # wait for new tenant spec to be ready + get_tenant_spec "$TENANT_NAME" "$NAMESPACE" + # modify it + yq eval 'del(.spec.pools[0])' ~/tenant.yaml >~/new-tenant.yaml + # replace it + RESULT=$(kubectl_replace ~/new-tenant.yaml) + done + +} + +# waits for n resources to come up. +# usage: wait_for_n_resources +function wait_for_n_resources() { + NUMBER_OF_RESOURCES=$3 + waitdone=0 + totalwait=0 + DEFAULT_WAIT_TIME=300 # 300 SECONDS + if [ -z "$4" ]; then + echo "No requested waiting time hence using default value" + else + echo "Requested specific waiting time" + DEFAULT_WAIT_TIME=$4 + fi + echo "* Waiting for ${NUMBER_OF_RESOURCES} pods to come up; wait_time: ${DEFAULT_WAIT_TIME};" + command_to_wait="kubectl -n $1 get pods --field-selector=status.phase=Running --no-headers --ignore-not-found=true" + if [ "$2" ]; then + command_to_wait="kubectl -n $1 get pods --field-selector=status.phase=Running --no-headers --ignore-not-found=true -l $2" + fi + echo "* Waiting on: $command_to_wait" + + while true; do + # xargs to trim whitespaces from bash variable. + waitdone=$($command_to_wait | wc -l | xargs) + + echo " " + echo " " + echo "##############################" + echo "To show visibility in all pods" + echo "##############################" + kubectl get pods -A + echo " " + echo " " + echo " " + + if [ "$waitdone" -ne 0 ]; then + if [ "$NUMBER_OF_RESOURCES" == "$waitdone" ]; then + break + fi + fi + sleep 5 + totalwait=$((totalwait + 10)) + if [ "$totalwait" -gt "$DEFAULT_WAIT_TIME" ]; then + echo "* Unable to get resource after 10 minutes, exiting." + try false + fi + done +} + +# waits for n tenant pods to come up. +# usage: wait_for_n_tenant_pods +function wait_for_n_tenant_pods() { + NUMBER_OF_RESOURCES=$1 + NAMESPACE=$2 + TENANT_NAME=$3 + echo "wait_for_n_tenant_pods(): * Waiting for ${NUMBER_OF_RESOURCES} '${TENANT_NAME}' tenant pods in ${NAMESPACE} namespace" + wait_for_n_resources "$NAMESPACE" "v1.min.io/tenant=$TENANT_NAME" "$NUMBER_OF_RESOURCES" 600 + echo "Waiting for the tenant pods to be ready (5m timeout)" + try kubectl wait --namespace "$NAMESPACE" \ + --for=condition=ready pod \ + --selector v1.min.io/tenant="$TENANT_NAME" \ + --timeout=300s +} + +# copies the script to the pod. +# usage: copy_script_to_pod