Skip to content

Commit

Permalink
Upgrade operator test (#1368)
Browse files Browse the repository at this point in the history
* Upgrade operator test (#1)

Test operator upgrade from version x to version y

* Upgrade operator test (#2)

Test operator upgrade from version x to version y

* Upgrade operator test (#3)

Announce versions involved in the test
  • Loading branch information
allanrogerr authored Dec 13, 2022
1 parent 6badba3 commit dd39a12
Show file tree
Hide file tree
Showing 3 changed files with 239 additions and 0 deletions.
22 changes: 22 additions & 0 deletions .github/workflows/kubernetes-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,28 @@ jobs:
- name: Prometheus test on Kind
run: |
"${GITHUB_WORKSPACE}/testing/deploy-tenant.sh"
test-tenant-upgrade:
runs-on: ${{ matrix.os }}
needs:
- logsearch
- operator
- lint
- gotest
- getdeps
- govet
strategy:
matrix:
go-version: [ 1.18.x ]
os: [ ubuntu-latest ]

steps:
- uses: actions/checkout@v2
- uses: actions/setup-go@v2
with:
go-version: ${{ matrix.go-version }}
- name: Tenant upgrade test on Kind
run: |
"${GITHUB_WORKSPACE}/testing/deploy-tenant-upgrade.sh"
test-prometheus:
runs-on: ${{ matrix.os }}
needs:
Expand Down
48 changes: 48 additions & 0 deletions testing/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,54 @@ function install_operator() {
echo "end - get data to verify proper image is being used"
}

function install_operator_version() {
# Obtain release
version="$1"
if [ -z "$version" ]
then
version=$(curl https://api.github.com/repos/minio/operator/releases/latest | jq --raw-output '.tag_name | "\(.[1:])"')
fi
echo "Target operator release: $version"
sudo curl -#L "https://github.com/minio/operator/releases/download/v${version}/kubectl-minio_${version}_${OS}_${ARCH}" -o /usr/local/bin/kubectl-minio
sudo chmod +x /usr/local/bin/kubectl-minio

# Initialize the MinIO Kubernetes Operator
kubectl minio init

# Verify installation of the plugin
echo "Installed operator release: $(kubectl minio version)"

if [ "$1" = "helm" ]; then
echo "key, value for pod selector in helm test"
key=app.kubernetes.io/name
value=operator
else
echo "key, value for pod selector in kustomize test"
key=name
value=minio-operator
fi

# Reusing the wait for both, Kustomize and Helm
echo "Waiting for k8s api"
sleep 10

kubectl get ns

kubectl -n minio-operator get deployments
kubectl -n minio-operator get pods

echo "Waiting for Operator Pods to come online (2m timeout)"
try kubectl wait --namespace minio-operator \
--for=condition=ready pod \
--selector $key=$value \
--timeout=120s

echo "start - get data to verify proper image is being used"
kubectl get pods --namespace minio-operator
kubectl describe pods -n minio-operator | grep Image
echo "end - get data to verify proper image is being used"
}

function destroy_kind() {
kind delete cluster
}
Expand Down
169 changes: 169 additions & 0 deletions testing/deploy-tenant-upgrade.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
#!/usr/bin/env bash
# Copyright (C) 2022, MinIO, Inc.
#
# This code is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3,
# as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License, version 3,
# along with this program. If not, see <http://www.gnu.org/licenses/>

# This script requires: kubectl, kind

SCRIPT_DIR=$(dirname "$0")

export SCRIPT_DIR

source "${SCRIPT_DIR}/common.sh"

lower_version="$1"
upper_version="$2"
namespace="tenant-lite"
tenant="storage-lite"
bucket="data"
dummy="dummy.data"
localport="9000"
alias="minios3"

# Announce test
function announce_test() {
local lower_text
local upper_text
if [ -n "$lower_version" ]
then
lower_text=$lower_version;
else
lower_text="latest Operator release";
fi

if [ -n "$upper_version" ]
then
upper_text=$upper_version;
else
upper_text="current branch of Operator";
fi

echo "## Testing upgrade of Operator from $lower_text to $upper_text ##"
}

# Port forward
function port_forward() {
totalwait=0
echo 'Validating tenant pods are ready to serve'
for pod in `kubectl --namespace $namespace --selector=v1.min.io/tenant=$tenant get pod -o json | jq '.items[] | select(.metadata.name|contains("'$tenant'"))| .metadata.name' | sed 's/"//g'`; do
while true; do
if kubectl --namespace $namespace logs pod/$pod | grep --quiet 'All MinIO sub-systems initialized successfully'; then
echo "$pod is ready to serve" && break
fi
sleep 5
totalwait=$((totalwait + 5))
if [ "$totalwait" -gt 305 ]; then
echo "Unable to validate pods after 5 minutes, exiting."
try false
fi
done
done

echo "Killing any current port-forward"
for pid in $(lsof -i :$localport | awk '{print $2}' | uniq | grep -o '[0-9]*')
do
if [ -n "$pid" ]
then
kill -9 $pid
echo "Killed previous port-forward process using port $localport: $pid"
fi
done

echo "Establishing port-forward"
kubectl port-forward service/$tenant-hl -n $namespace $localport &

echo 'start - wait for port-forward to be completed'
sleep 15
echo 'end - wait for port-forward to be completed'
}

# Preparing tenant for bucket manipulation
# shellcheck disable=SC2317
function bootstrap_tenant() {
port_forward

# Obtain root credentials
TENANT_CONFIG_SECRET=$(kubectl -n $namespace get tenants $tenant -o jsonpath="{.spec.configuration.name}")
USER=$(kubectl -n $namespace get secrets "$TENANT_CONFIG_SECRET" -o go-template='{{index .data "config.env"|base64decode }}' | grep 'export MINIO_ROOT_USER="' | sed -e 's/export MINIO_ROOT_USER="//g' | sed -e 's/"//g')
PASSWORD=$(kubectl -n $namespace get secrets "$TENANT_CONFIG_SECRET" -o go-template='{{index .data "config.env"|base64decode }}' | grep 'export MINIO_ROOT_PASSWORD="' | sed -e 's/export MINIO_ROOT_PASSWORD="//g' | sed -e 's/"//g')

echo "Creating alias with user ${USER}"
mc alias set $alias https://localhost:$localport ${USER} ${PASSWORD} --insecure

echo "Creating bucket on tenant"
mc mb $alias/$bucket --insecure
}

# Upload dummy data to tenant bucket
function upload_dummy_data() {
port_forward

echo "Uploading dummy data to tenant bucket"
cp ${SCRIPT_DIR}/deploy-tenant-upgrade.sh ${SCRIPT_DIR}/$dummy
mc cp ${SCRIPT_DIR}/$dummy $alias/$bucket/$dummy --insecure
}

# Download dummy data from tenant bucket
function download_dummy_data() {
port_forward

echo "Download dummy data from tenant bucket"
mc cp $alias/$bucket/$dummy ${SCRIPT_DIR}/$dummy --insecure

if cmp "${SCRIPT_DIR}/deploy-tenant-upgrade.sh" "${SCRIPT_DIR}/$dummy"; then
echo "Operator upgrade test complete; no issue found"
else
echo "Operator upgrade test failed"
try false
fi
}

function main() {
announce_test

destroy_kind

setup_kind

if [ -n "$lower_version" ]
then
# Test specific version of operator
install_operator_version $lower_version
else
# Test latest release
install_operator_version
fi

install_tenant

bootstrap_tenant

upload_dummy_data

if [ -n "$upper_version" ]
then
# Test specific version of operator
install_operator_version $upper_version
else
# Test current branch
install_operator
fi

check_tenant_status tenant-lite storage-lite

download_dummy_data

destroy_kind
}

main "$@"

0 comments on commit dd39a12

Please sign in to comment.