Add e2e test for upgrading from 1.0.x

tests/actions.go

@@ -505,7 +505,7 @@ func (oa *operatorActions) InstallCRDOrDie(info *OperatorConfig) {
 func (oa *operatorActions) DeployOperator(info *OperatorConfig) error {
 	klog.Infof("deploying tidb-operator %s", info.ReleaseName)
 
-	if info.Tag != "e2e" {
+	if info.Tag != "e2e" && !strings.HasPrefix(info.Tag, "v1.0.") {
 		if err := oa.cloneOperatorRepo(); err != nil {
 			return err
 		}

tests/e2e/tidbcluster/serial.go

@@ -18,6 +18,8 @@ import (
 	"encoding/json"
 	"fmt"
 	_ "net/http/pprof"
+	"os"
+	"sort"
 	"strconv"
 	"time"
 
@@ -1008,4 +1010,147 @@ var _ = ginkgo.Describe("[tidb-operator][Serial]", func() {
 			klog.Info("success to check auto scale-in tidb to 2 replicas")
 		})
 	})
+
+	ginkgo.Context("[Verify: Upgrading Operator from 1.0.6", func() {
+		var oa tests.OperatorActions
+		var ocfg *tests.OperatorConfig
+		var version string
+
+		ginkgo.BeforeEach(func() {
+			version = os.Getenv("RELEASED_VERSION")
+			if len(version) < 1 {
+				version = "v1.0.6"
+			}
+			ocfg = &tests.OperatorConfig{
+				Namespace:   ns,
+				ReleaseName: "operator",
+				Tag:         version,
+				Image:       fmt.Sprintf("pingcap/tidb-operator:%s", version),
+			}
+			oa = tests.NewOperatorActions(cli, c, asCli, aggrCli, apiExtCli, tests.DefaultPollInterval, ocfg, e2econfig.TestConfig, nil, fw, f)
+			ginkgo.By("Installing CRDs")
+			oa.CleanCRDOrDie()
+			tests.DeployReleasedCRDOrDie(version)
+			ginkgo.By("Installing tidb-operator")
+			oa.CleanOperatorOrDie(ocfg)
+			oa.DeployOperatorOrDie(ocfg)
+		})
+
+		ginkgo.AfterEach(func() {
+			ginkgo.By("Uninstall tidb-operator")
+			oa.CleanOperatorOrDie(ocfg)
+			ginkgo.By("Uninstalling CRDs")
+			tests.CleanReleasedCRDOrDie(version)
+		})
+
+		ginkgo.It("Deploy TidbCluster and Upgrade Operator", func() {
+			tcName := "tidbcluster"
+			cluster := newTidbClusterConfig(e2econfig.TestConfig, ns, tcName, "", "")
+			cluster.Resources["pd.replicas"] = "3"
+			cluster.Resources["tikv.replicas"] = "3"
+			cluster.Resources["tidb.replicas"] = "2"
+			cluster.Monitor = false
+			cluster.OperatorTag = version
+			oa.DeployTidbClusterOrDie(&cluster)
+			oa.CheckTidbClusterStatusOrDie(&cluster)
+
+			getPodUids := func(ls string) ([]string, error) {
+				listOptions := metav1.ListOptions{
+					LabelSelector: ls,
+				}
+				podList, err := c.CoreV1().Pods(ns).List(listOptions)
+				if err != nil {
+					return nil, err
+				}
+				var uids []string
+				for _, pod := range podList.Items {
+					uids = append(uids, string(pod.GetUID()))
+				}
+				sort.Strings(uids)
+				return uids, nil
+			}
+
+			tc, err := cli.PingcapV1alpha1().TidbClusters(ns).Get(tcName, metav1.GetOptions{})
+			framework.ExpectNoError(err, "failed to get tc")
+
+			pdUids, err := getPodUids(labels.SelectorFromSet(label.New().Instance(tcName).PD().Labels()).String())
+			if err != nil {
+				framework.ExpectNoError(err, "failed to get pd pods uids")
+			}
+			tikvUids, err := getPodUids(labels.SelectorFromSet(label.New().Instance(tcName).TiKV().Labels()).String())
+			if err != nil {
+				framework.ExpectNoError(err, "failed to get tikv pods uids")
+			}
+			tidbUids, err := getPodUids(labels.SelectorFromSet(label.New().Instance(tcName).TiDB().Labels()).String())
+			if err != nil {
+				framework.ExpectNoError(err, "failed to get tidb pods uids")
+			}
+
+			// Upgrade CRD / Operator to current version
+			ocfg.Tag = cfg.OperatorTag
+			ocfg.Image = cfg.OperatorImage
+			oa.InstallCRDOrDie(ocfg)
+			oa.UpgradeOperatorOrDie(ocfg)
+			err = wait.Poll(5*time.Second, 10*time.Minute, func() (done bool, err error) {
+
+				newTc, err := cli.PingcapV1alpha1().TidbClusters(ns).Get(tcName, metav1.GetOptions{})
+				if err != nil {
+					return false, nil
+				}
+				// wait tidb to be updated
+				if tc.Status.TiDB.StatefulSet.CurrentRevision == newTc.Status.TiDB.StatefulSet.CurrentRevision {
+					return false, nil
+				}
+				// wait tidb finish updating
+				if newTc.Status.TiDB.StatefulSet.CurrentRevision != newTc.Status.TiDB.StatefulSet.UpdateRevision {
+					return false, nil
+				}
+
+				tidbNewUids, err := getPodUids(labels.SelectorFromSet(label.New().Instance(tcName).TiDB().Labels()).String())
+				if err != nil {
+					return false, nil
+				}
+
+				if len(tidbNewUids) != len(tidbUids) {
+					return false, fmt.Errorf("tidb replicas has changed")
+				}
+				// Confirm the tidb has Upgraded
+				for i := 0; i < len(tidbNewUids); i++ {
+					if tidbNewUids[i] == tidbUids[i] {
+						return false, nil
+					}
+				}
+
+				pdNewUids, err := getPodUids(labels.SelectorFromSet(label.New().Instance(tcName).PD().Labels()).String())
+				if err != nil {
+					return false, nil
+				}
+				if len(pdNewUids) != len(pdUids) {
+					return false, fmt.Errorf("pd replicas has changed")
+				}
+				// Confirm the pd hasn't changed
+				for i := 0; i < len(pdNewUids); i++ {
+					if pdNewUids[i] != pdUids[i] {
+						return false, fmt.Errorf("pd pods have been changed after upgrading operator")
+					}
+				}
+
+				tikvNewUids, err := getPodUids(labels.SelectorFromSet(label.New().Instance(tcName).TiKV().Labels()).String())
+				if err != nil {
+					return false, nil
+				}
+				if len(tikvNewUids) != len(tikvUids) {
+					return false, fmt.Errorf("tikv replicas has changed")
+				}
+				// Confirm the tikv hasn't changed
+				for i := 0; i < len(tikvNewUids); i++ {
+					if tikvNewUids[i] != tikvUids[i] {
+						return false, fmt.Errorf("tikv pods have been changed after upgrading operator")
+					}
+				}
+				return true, nil
+			})
+			framework.ExpectNoError(err, "Failed to check TidbCluster Status After Upgrading Operator")
+		})
+	})
 })

tests/images/e2e/Dockerfile

@@ -2,6 +2,7 @@ FROM debian:buster-slim
 
 ENV KUBECTL_VERSION=v1.12.2
 ENV HELM_VERSION=v2.9.1
+ENV RELEASED_VERSION=v1.0.6
 
 RUN apt-get update && \
     apt-get install -y ca-certificates curl git openssl default-mysql-client unzip
@@ -29,5 +30,21 @@ ADD bin/webhook /usr/local/bin/
 ADD bin/blockwriter /usr/local/bin/
 ADD bin/apiserver /usr/local/bin/
 
+RUN mkdir /charts/${RELEASED_VERSION}
+
+RUN curl -L https://github.com/pingcap/tidb-operator/releases/download/${RELEASED_VERSION}/tidb-operator-chart-${RELEASED_VERSION}.tgz \
+    -o tidb-operator-chart-${RELEASED_VERSION}.tgz && \
+    tar -zxvf tidb-operator-chart-${RELEASED_VERSION}.tgz && \
+    mv tidb-operator /charts/${RELEASED_VERSION}/tidb-operator && \
+    rm -rf tidb-operator && \
+    rm tidb-operator-chart-${RELEASED_VERSION}.tgz
+
+RUN curl -L https://github.com/pingcap/tidb-operator/releases/download/${RELEASED_VERSION}/tidb-cluster-chart-${RELEASED_VERSION}.tgz \
+    -o tidb-cluster-chart-${RELEASED_VERSION}.tgz && \
+    tar -zxvf tidb-cluster-chart-${RELEASED_VERSION}.tgz && \
+    mv tidb-cluster /charts/${RELEASED_VERSION}/tidb-cluster && \
+    rm -rf tidb-cluster && \
+    rm tidb-cluster-chart-${RELEASED_VERSION}.tgz
+
 ADD entrypoint.sh /usr/local/bin
 ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]

tests/util.go

@@ -17,17 +17,18 @@ import (
 	"bytes"
 	"fmt"
 	"math/rand"
+	"os/exec"
 	"text/template"
 	"time"
 
 	"github.com/pingcap/tidb-operator/tests/slack"
-
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/klog"
 )
 
 // Keep will keep the fun running in the period, otherwise the fun return error
@@ -299,3 +300,21 @@ func waitForComponentStatus(c kubernetes.Interface, component string, statusType
 func IntPtr(i int) *int {
 	return &i
 }
+
+func DeployReleasedCRDOrDie(version string) {
+	cmd := fmt.Sprintf(`kubectl apply -f https://raw.githubusercontent.com/pingcap/tidb-operator/%s/manifests/crd.yaml`, version)
+	klog.Info(cmd)
+	res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
+	if err != nil {
+		klog.Fatalf(fmt.Sprintf("failed to deploy crd: %v, %s", err, string(res)))
+	}
+}
+
+func CleanReleasedCRDOrDie(version string) {
+	cmd := fmt.Sprintf(`kubectl delete -f https://raw.githubusercontent.com/pingcap/tidb-operator/%s/manifests/crd.yaml`, version)
+	klog.Info(cmd)
+	res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
+	if err != nil {
+		klog.Fatalf(fmt.Sprintf("failed to clean crd: %v, %s", err, string(res)))
+	}
+}