diff --git a/.github/workflows/kubernetes-tests.yml b/.github/workflows/kubernetes-tests.yml index 38d41ccaed9..13f7b200c84 100644 --- a/.github/workflows/kubernetes-tests.yml +++ b/.github/workflows/kubernetes-tests.yml @@ -147,6 +147,28 @@ jobs: - name: Prometheus test on Kind run: | "${GITHUB_WORKSPACE}/testing/deploy-tenant.sh" + test-tenant-upgrade: + runs-on: ${{ matrix.os }} + needs: + - logsearch + - operator + - lint + - gotest + - getdeps + - govet + strategy: + matrix: + go-version: [ 1.18.x ] + os: [ ubuntu-latest ] + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go-version }} + - name: Tenant upgrade test on Kind + run: | + "${GITHUB_WORKSPACE}/testing/deploy-tenant-upgrade.sh" test-prometheus: runs-on: ${{ matrix.os }} needs: diff --git a/testing/common.sh b/testing/common.sh index ae18e2ee52b..269f6e37c05 100644 --- a/testing/common.sh +++ b/testing/common.sh @@ -100,6 +100,54 @@ function install_operator() { echo "end - get data to verify proper image is being used" } +function install_operator_version() { + # Obtain release + version="$1" + if [ -z "$version" ] + then + version=$(curl https://api.github.com/repos/minio/operator/releases/latest | jq --raw-output '.tag_name | "\(.[1:])"') + fi + echo "Target operator release: $version" + sudo curl -#L "https://github.com/minio/operator/releases/download/v${version}/kubectl-minio_${version}_${OS}_${ARCH}" -o /usr/local/bin/kubectl-minio + sudo chmod +x /usr/local/bin/kubectl-minio + + # Initialize the MinIO Kubernetes Operator + kubectl minio init + + # Verify installation of the plugin + echo "Installed operator release: $(kubectl minio version)" + + if [ "$1" = "helm" ]; then + echo "key, value for pod selector in helm test" + key=app.kubernetes.io/name + value=operator + else + echo "key, value for pod selector in kustomize test" + key=name + value=minio-operator + fi + + # Reusing the wait for both, Kustomize and Helm + echo "Waiting for k8s api" + sleep 10 + + kubectl get ns + + kubectl -n minio-operator get deployments + kubectl -n minio-operator get pods + + echo "Waiting for Operator Pods to come online (2m timeout)" + try kubectl wait --namespace minio-operator \ + --for=condition=ready pod \ + --selector $key=$value \ + --timeout=120s + + echo "start - get data to verify proper image is being used" + kubectl get pods --namespace minio-operator + kubectl describe pods -n minio-operator | grep Image + echo "end - get data to verify proper image is being used" +} + function destroy_kind() { kind delete cluster } diff --git a/testing/deploy-tenant-upgrade.sh b/testing/deploy-tenant-upgrade.sh new file mode 100755 index 00000000000..ddda6d8dbb4 --- /dev/null +++ b/testing/deploy-tenant-upgrade.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +# Copyright (C) 2022, MinIO, Inc. +# +# This code is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3, +# as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License, version 3, +# along with this program. If not, see + +# This script requires: kubectl, kind + +SCRIPT_DIR=$(dirname "$0") + +export SCRIPT_DIR + +source "${SCRIPT_DIR}/common.sh" + +lower_version="$1" +upper_version="$2" +namespace="tenant-lite" +tenant="storage-lite" +bucket="data" +dummy="dummy.data" +localport="9000" +alias="minios3" + +# Announce test +function announce_test() { + local lower_text + local upper_text + if [ -n "$lower_version" ] + then + lower_text=$lower_version; + else + lower_text="latest Operator release"; + fi + + if [ -n "$upper_version" ] + then + upper_text=$upper_version; + else + upper_text="current branch of Operator"; + fi + + echo "## Testing upgrade of Operator from $lower_text to $upper_text ##" +} + +# Port forward +function port_forward() { + totalwait=0 + echo 'Validating tenant pods are ready to serve' + for pod in `kubectl --namespace $namespace --selector=v1.min.io/tenant=$tenant get pod -o json | jq '.items[] | select(.metadata.name|contains("'$tenant'"))| .metadata.name' | sed 's/"//g'`; do + while true; do + if kubectl --namespace $namespace logs pod/$pod | grep --quiet 'All MinIO sub-systems initialized successfully'; then + echo "$pod is ready to serve" && break + fi + sleep 5 + totalwait=$((totalwait + 5)) + if [ "$totalwait" -gt 305 ]; then + echo "Unable to validate pods after 5 minutes, exiting." + try false + fi + done + done + + echo "Killing any current port-forward" + for pid in $(lsof -i :$localport | awk '{print $2}' | uniq | grep -o '[0-9]*') + do + if [ -n "$pid" ] + then + kill -9 $pid + echo "Killed previous port-forward process using port $localport: $pid" + fi + done + + echo "Establishing port-forward" + kubectl port-forward service/$tenant-hl -n $namespace $localport & + + echo 'start - wait for port-forward to be completed' + sleep 15 + echo 'end - wait for port-forward to be completed' +} + +# Preparing tenant for bucket manipulation +# shellcheck disable=SC2317 +function bootstrap_tenant() { + port_forward + + # Obtain root credentials + TENANT_CONFIG_SECRET=$(kubectl -n $namespace get tenants $tenant -o jsonpath="{.spec.configuration.name}") + USER=$(kubectl -n $namespace get secrets "$TENANT_CONFIG_SECRET" -o go-template='{{index .data "config.env"|base64decode }}' | grep 'export MINIO_ROOT_USER="' | sed -e 's/export MINIO_ROOT_USER="//g' | sed -e 's/"//g') + PASSWORD=$(kubectl -n $namespace get secrets "$TENANT_CONFIG_SECRET" -o go-template='{{index .data "config.env"|base64decode }}' | grep 'export MINIO_ROOT_PASSWORD="' | sed -e 's/export MINIO_ROOT_PASSWORD="//g' | sed -e 's/"//g') + + echo "Creating alias with user ${USER}" + mc alias set $alias https://localhost:$localport ${USER} ${PASSWORD} --insecure + + echo "Creating bucket on tenant" + mc mb $alias/$bucket --insecure +} + +# Upload dummy data to tenant bucket +function upload_dummy_data() { + port_forward + + echo "Uploading dummy data to tenant bucket" + cp ${SCRIPT_DIR}/deploy-tenant-upgrade.sh ${SCRIPT_DIR}/$dummy + mc cp ${SCRIPT_DIR}/$dummy $alias/$bucket/$dummy --insecure +} + +# Download dummy data from tenant bucket +function download_dummy_data() { + port_forward + + echo "Download dummy data from tenant bucket" + mc cp $alias/$bucket/$dummy ${SCRIPT_DIR}/$dummy --insecure + + if cmp "${SCRIPT_DIR}/deploy-tenant-upgrade.sh" "${SCRIPT_DIR}/$dummy"; then + echo "Operator upgrade test complete; no issue found" + else + echo "Operator upgrade test failed" + try false + fi +} + +function main() { + announce_test + + destroy_kind + + setup_kind + + if [ -n "$lower_version" ] + then + # Test specific version of operator + install_operator_version $lower_version + else + # Test latest release + install_operator_version + fi + + install_tenant + + bootstrap_tenant + + upload_dummy_data + + if [ -n "$upper_version" ] + then + # Test specific version of operator + install_operator_version $upper_version + else + # Test current branch + install_operator + fi + + check_tenant_status tenant-lite storage-lite + + download_dummy_data + + destroy_kind +} + +main "$@"