add a kinder e2e test

test/e2e/test-kinder.sh

@@ -0,0 +1,134 @@
+#!/usr/bin/env bash
+
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script tests manually started storage migration via the migration
+# initializer binary.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+MIGRATORROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"/../..
+REGISTRY=""
+VERSION=""
+
+failure() {
+  local lineno=$1
+  local msg=$2
+  echo "Failed at $lineno: $msg"
+}
+
+trap 'failure ${LINENO} "$BASH_COMMAND"' ERR
+
+SCRIPT_PATH="$(dirname "$(readlink -f "$0")")"
+KINDER_ROOT_PATH="$(readlink -f "${SCRIPT_PATH}/../../../kubeadm/kinder/")"
+
+# build kinder
+pushd "${KINDER_ROOT_PATH}"
+echo "Building kinder..."
+GO111MODULE=on go build
+popd
+
+# TODO(roycaihw): automate or document how to build the altered image
+docker pull roycaihw/kindest-node:test-sv-crd
+
+CLUSTER_NAME="kind-test-e2e"
+kinder create cluster --image roycaihw/kindest-node:test-sv-crd --control-plane-nodes 3 --name "${CLUSTER_NAME}" --worker-nodes 0
+kinder do kubeadm-init --name "${CLUSTER_NAME}"
+export KUBECONFIG="$(kinder get kubeconfig-path --name="${CLUSTER_NAME}")"
+
+# Sanity check.
+kubectl version
+
+# build and deploy the migrator
+
+pushd "${MIGRATORROOT}"
+# TODO(roycaihw): switch to gcloud for the controller images
+export REGISTRY="roycaihw"
+echo "REGISTRY=${REGISTRY}"
+commit=$(git rev-parse --short HEAD)
+export VERSION="v${commit}"
+echo "VERSION=${VERSION}"
+make local-manifests
+make push-all
+popd
+
+kinder do kubeadm-join --name "${CLUSTER_NAME}" --loglevel=debug
+
+pushd "${MIGRATORROOT}/manifests.local"
+kubectl apply -f storage_migration_crd.yaml
+kubectl apply -f storage_state_crd.yaml
+kubectl apply -f namespace-rbac.yaml
+kubectl apply -f trigger.yaml
+kubectl apply -f migrator.yaml
+popd
+
+kinder do kubeadm-upgrade --upgrade-version v1.20.0-beta.2.67+b0adc0a51238ee --only-node "${CLUSTER_NAME}"-control-plane-1 --name "${CLUSTER_NAME}" --loglevel=debug
+
+# TODO: use worker node and stop force restarting controller when we figure out
+# the kinder iptable routing, or use a node label selector to force assigning
+# pod to master 1 (so we don't need to re-apply after upgrading master 2 and 3)
+
+pushd "${MIGRATORROOT}/manifests.local"
+kubectl delete -f trigger.yaml
+kubectl apply -f trigger.yaml
+kubectl delete -f migrator.yaml
+kubectl apply -f migrator.yaml
+popd
+
+if kubectl get storageversionmigrations | grep customresourcedefinitions.apiextensions.k8s.io; then
+  echo "unexpected migrating crd after upgrade master 1: $CRDNAME"
+  exit 1
+fi
+
+kinder do kubeadm-upgrade --upgrade-version v1.20.0-beta.2.67+b0adc0a51238ee --only-node "${CLUSTER_NAME}"-control-plane-2 --name "${CLUSTER_NAME}" --loglevel=debug
+
+pushd "${MIGRATORROOT}/manifests.local"
+kubectl delete -f trigger.yaml
+kubectl apply -f trigger.yaml
+kubectl delete -f migrator.yaml
+kubectl apply -f migrator.yaml
+popd
+
+if kubectl get storageversionmigrations | grep customresourcedefinitions.apiextensions.k8s.io; then
+  echo "unexpected migrating crd after upgrade master 2: $CRDNAME"
+  exit 1
+fi
+
+kinder do kubeadm-upgrade --upgrade-version v1.20.0-beta.2.67+b0adc0a51238ee --only-node "${CLUSTER_NAME}"-control-plane-3 --name "${CLUSTER_NAME}" --loglevel=debug
+
+pushd "${MIGRATORROOT}/manifests.local"
+kubectl delete -f trigger.yaml
+kubectl apply -f trigger.yaml
+kubectl delete -f migrator.yaml
+kubectl apply -f migrator.yaml
+popd
+
+sleep 180
+wait-for-migration
+
+CRDNAME=$(kubectl get storageversionmigrations | grep customresourcedefinitions.apiextensions.k8s.io | grep -o '^\S*')
+VERSION=$(kubectl get storagestates customresourcedefinitions.apiextensions.k8s.io -o jsonpath='{.status.currentStorageVersionHash}')
+if [ "$VERSION" != "apiextensions.k8s.io/v1" ]; then
+  echo "has wrong version after upgrade master 3: $VERSION"
+  exit 1
+fi
+
+# TODO: evaluate object in etcd
+echo "succeeded"
+
+kinder do kubeadm-reset --name "${CLUSTER_NAME}" --loglevel=debug
+kinder delete cluster --name "${CLUSTER_NAME}" --loglevel=debug