Skip to content
This repository has been archived by the owner on Jan 20, 2022. It is now read-only.

Commit

Permalink
Better etcd-version handling
Browse files Browse the repository at this point in the history
We move etcd version handling to its own package, and we are now more
tolerant about reading from patch versions that we don't have (we
adopt the same-or-next minor rule that etcd uses).  Also on etcd
cluster adoption we bump it to the next supported version so that we
can come up even if the user was previously running an unsupported
version.
  • Loading branch information
justinsb committed Jul 31, 2019
1 parent 8a739a7 commit 0708fe6
Show file tree
Hide file tree
Showing 13 changed files with 174 additions and 48 deletions.
2 changes: 1 addition & 1 deletion pkg/controller/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,11 @@ go_library(
"//pkg/contextutil:go_default_library",
"//pkg/etcd:go_default_library",
"//pkg/etcdclient:go_default_library",
"//pkg/etcdversions:go_default_library",
"//pkg/locking:go_default_library",
"//pkg/pki:go_default_library",
"//pkg/privateapi:go_default_library",
"//pkg/urls:go_default_library",
"//vendor/github.com/blang/semver:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/github.com/golang/protobuf/proto:go_default_library",
],
Expand Down
38 changes: 2 additions & 36 deletions pkg/controller/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import (
"sync"
"time"

"github.com/blang/semver"
"github.com/golang/glog"
"github.com/golang/protobuf/proto"
protoetcd "kope.io/etcd-manager/pkg/apis/etcd"
Expand All @@ -23,6 +22,7 @@ import (
"kope.io/etcd-manager/pkg/contextutil"
"kope.io/etcd-manager/pkg/etcd"
"kope.io/etcd-manager/pkg/etcdclient"
"kope.io/etcd-manager/pkg/etcdversions"
"kope.io/etcd-manager/pkg/locking"
"kope.io/etcd-manager/pkg/pki"
"kope.io/etcd-manager/pkg/privateapi"
Expand Down Expand Up @@ -417,7 +417,7 @@ func (m *EtcdController) run(ctx context.Context) (bool, error) {
glog.Infof("mismatched version for peer %v: want %q, have %q", peer.peer, clusterSpec.EtcdVersion, peer.info.EtcdState.EtcdVersion)
versionMismatch = append(versionMismatch, peer)

if !upgradeInPlaceSupported(peer.info.EtcdState.EtcdVersion, clusterSpec.EtcdVersion) {
if !etcdversions.UpgradeInPlaceSupported(peer.info.EtcdState.EtcdVersion, clusterSpec.EtcdVersion) {
// TODO: Automatic intermediate upgrades? 3.1 -> 3.2 -> 3.3 ?
if canUpgradeInPlace {
glog.Infof("can't do in-place upgrade from %q -> %q", peer.info.EtcdState.EtcdVersion, clusterSpec.EtcdVersion)
Expand Down Expand Up @@ -1009,37 +1009,3 @@ func (m *EtcdController) verifyEtcdVersion(clusterSpec *protoetcd.ClusterSpec) (

return false, fmt.Errorf("can't act as leader for unknown etcd version %q", clusterSpec.EtcdVersion)
}

func upgradeInPlaceSupported(fromVersion, toVersion string) bool {
fromSemver, err := semver.ParseTolerant(fromVersion)
if err != nil {
glog.Warningf("unknown version format: %q", fromVersion)
return false
}

toSemver, err := semver.ParseTolerant(toVersion)
if err != nil {
glog.Warningf("unknown version format: %q", toVersion)
return false
}

if fromSemver.Major == 3 && toSemver.Major == 3 {
if fromSemver.Minor == 1 && toSemver.Minor == 1 {
return true
}
if fromSemver.Minor == 1 && toSemver.Minor == 2 {
return true
}
if fromSemver.Minor == 2 && toSemver.Minor == 2 {
return true
}
if fromSemver.Minor == 2 && toSemver.Minor == 3 {
return true
}
if fromSemver.Minor == 3 && toSemver.Minor == 3 {
return true
}
}

return false
}
1 change: 1 addition & 0 deletions pkg/etcd/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ go_library(
"//pkg/contextutil:go_default_library",
"//pkg/dns:go_default_library",
"//pkg/etcdclient:go_default_library",
"//pkg/etcdversions:go_default_library",
"//pkg/legacy:go_default_library",
"//pkg/pki:go_default_library",
"//pkg/privateapi:go_default_library",
Expand Down
11 changes: 7 additions & 4 deletions pkg/etcd/restore.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
protoetcd "kope.io/etcd-manager/pkg/apis/etcd"
"kope.io/etcd-manager/pkg/backup"
"kope.io/etcd-manager/pkg/etcdclient"
"kope.io/etcd-manager/pkg/etcdversions"
"kope.io/etcd-manager/pkg/pki"
)

Expand Down Expand Up @@ -112,10 +113,12 @@ func RunEtcdFromBackup(backupStore backup.Store, backupName string, basedir stri

etcdVersion := backupInfo.EtcdVersion
// A few known-safe restore-from-backup options
switch etcdVersion {
case "3.1.11":
glog.Warningf("restoring backup from etcd 3.1.11, will restore with 3.1.12")
etcdVersion = "3.1.12"
{
restoreWith := etcdversions.EtcdVersionForRestore(etcdVersion)
if restoreWith != "" && restoreWith != etcdVersion {
glog.Warningf("restoring backup from etcd %q, will restore with %q", etcdVersion, restoreWith)
etcdVersion = restoreWith
}
}

binDir, err := BindirForEtcdVersion(etcdVersion, "etcd")
Expand Down
12 changes: 12 additions & 0 deletions pkg/etcdversions/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")

go_library(
name = "go_default_library",
srcs = ["mappings.go"],
importpath = "kope.io/etcd-manager/pkg/etcdversions",
visibility = ["//visibility:public"],
deps = [
"//vendor/github.com/blang/semver:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
],
)
133 changes: 133 additions & 0 deletions pkg/etcdversions/mappings.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
package etcdversions

import (
"fmt"

"github.com/blang/semver"
"github.com/golang/glog"
)

// By declaring the versions here, using constants, we likely force a compilation error
// on an inconsistent update

const (
Version_2_2_1 = "2.2.1"
Version_3_1_12 = "3.1.12"
Version_3_2_18 = "3.2.18"
Version_3_2_24 = "3.2.24"
Version_3_3_10 = "3.3.10"
Version_3_3_13 = "3.3.13"
)

var AllEtcdVersions = []string{
Version_2_2_1,
Version_3_1_12,
Version_3_2_18,
Version_3_2_24,
Version_3_3_10,
Version_3_3_13,
}

func UpgradeInPlaceSupported(fromVersion, toVersion string) bool {
fromSemver, err := semver.ParseTolerant(fromVersion)
if err != nil {
glog.Warningf("unknown version format: %q", fromVersion)
return false
}

toSemver, err := semver.ParseTolerant(toVersion)
if err != nil {
glog.Warningf("unknown version format: %q", toVersion)
return false
}

if fromSemver.Major == 3 && toSemver.Major == 3 {
if fromSemver.Minor == 0 && toSemver.Minor == 0 {
return true
}
if fromSemver.Minor == 0 && toSemver.Minor == 1 {
return true
}
if fromSemver.Minor == 1 && toSemver.Minor == 1 {
return true
}
if fromSemver.Minor == 1 && toSemver.Minor == 2 {
return true
}
if fromSemver.Minor == 2 && toSemver.Minor == 2 {
return true
}
if fromSemver.Minor == 2 && toSemver.Minor == 3 {
return true
}
if fromSemver.Minor == 3 && toSemver.Minor == 3 {
return true
}
}

return false
}

func EtcdVersionForAdoption(fromVersion string) string {
fromSemver, err := semver.ParseTolerant(fromVersion)
if err != nil {
glog.Warningf("unknown version format: %q", fromVersion)
return ""
}

family := fmt.Sprintf("%d.%d", fromSemver.Major, fromSemver.Minor)
switch family {
case "2.2":
return Version_2_2_1
case "3.0":
return Version_3_1_12
case "3.1":
return Version_3_1_12
case "3.2":
if fromSemver.Patch <= 18 {
return Version_3_2_18
} else {
return Version_3_2_24
}
case "3.3":
if fromSemver.Patch <= 10 {
return Version_3_3_10
} else {
return Version_3_3_13
}
default:
return ""
}
}

func EtcdVersionForRestore(fromVersion string) string {
fromSemver, err := semver.ParseTolerant(fromVersion)
if err != nil {
glog.Warningf("unknown version format: %q", fromVersion)
return ""
}

family := fmt.Sprintf("%d.%d", fromSemver.Major, fromSemver.Minor)
switch family {
case "2.2":
return Version_2_2_1
case "3.0":
return Version_3_1_12
case "3.1":
return Version_3_1_12
case "3.2":
if fromSemver.Patch <= 18 {
return Version_3_2_18
} else {
return Version_3_2_24
}
case "3.3":
if fromSemver.Patch <= 10 {
return Version_3_3_10
} else {
return Version_3_3_13
}
default:
return ""
}
}
1 change: 1 addition & 0 deletions pkg/legacy/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ go_library(
deps = [
"//pkg/apis/etcd:go_default_library",
"//pkg/commands:go_default_library",
"//pkg/etcdversions:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/gopkg.in/yaml.v2:go_default_library",
],
Expand Down
7 changes: 7 additions & 0 deletions pkg/legacy/adopt.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
yaml "gopkg.in/yaml.v2"
protoetcd "kope.io/etcd-manager/pkg/apis/etcd"
"kope.io/etcd-manager/pkg/commands"
"kope.io/etcd-manager/pkg/etcdversions"
)

type legacyManifest struct {
Expand Down Expand Up @@ -226,6 +227,12 @@ func ImportExistingEtcd(baseDir string, etcdNodeConfiguration *protoetcd.EtcdNod
return nil, fmt.Errorf("error renaming directory %s -> %s: %v", legacyDir, newDataDir, err)
}

adoptAs := etcdversions.EtcdVersionForAdoption(state.EtcdVersion)
if adoptAs != "" && adoptAs != state.EtcdVersion {
glog.Warningf("adopting from etcd %q, will adopt with %q", state.EtcdVersion, adoptAs)
state.EtcdVersion = adoptAs
}

glog.Infof("imported etcd state: %v", state.String())
return state, nil
}
Expand Down
1 change: 1 addition & 0 deletions test/integration/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ go_test(
deps = [
"//pkg/apis/etcd:go_default_library",
"//pkg/etcd:go_default_library",
"//pkg/etcdversions:go_default_library",
"//test/integration/harness:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
],
Expand Down
5 changes: 2 additions & 3 deletions test/integration/clusterformation_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (

"github.com/golang/glog"
protoetcd "kope.io/etcd-manager/pkg/apis/etcd"
"kope.io/etcd-manager/pkg/etcdversions"
"kope.io/etcd-manager/test/integration/harness"
)

Expand All @@ -17,10 +18,8 @@ func init() {
flag.Parse()
}

var AllEtcdVersions = []string{"2.2.1", "3.1.12", "3.2.18", "3.2.24", "3.3.10", "3.3.13"}

func TestClusterWithOneMember(t *testing.T) {
for _, etcdVersion := range AllEtcdVersions {
for _, etcdVersion := range etcdversions.AllEtcdVersions {
t.Run("etcdVersion="+etcdVersion, func(t *testing.T) {
ctx := context.TODO()
ctx, cancel := context.WithTimeout(ctx, time.Second*30)
Expand Down
3 changes: 2 additions & 1 deletion test/integration/etcdinstalled_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,11 @@ import (
"testing"

"kope.io/etcd-manager/pkg/etcd"
"kope.io/etcd-manager/pkg/etcdversions"
)

func TestEtcdInstalled(t *testing.T) {
for _, etcdVersion := range AllEtcdVersions {
for _, etcdVersion := range etcdversions.AllEtcdVersions {
t.Run("etcdVersion="+etcdVersion, func(t *testing.T) {
{
bindir, err := etcd.BindirForEtcdVersion(etcdVersion, "etcd")
Expand Down
3 changes: 2 additions & 1 deletion test/integration/resize_cluster_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,12 @@ import (

"github.com/golang/glog"
protoetcd "kope.io/etcd-manager/pkg/apis/etcd"
"kope.io/etcd-manager/pkg/etcdversions"
"kope.io/etcd-manager/test/integration/harness"
)

func TestResizeCluster(t *testing.T) {
for _, etcdVersion := range AllEtcdVersions {
for _, etcdVersion := range etcdversions.AllEtcdVersions {
t.Run("etcdVersion="+etcdVersion, func(t *testing.T) {
ctx := context.TODO()
ctx, cancel := context.WithTimeout(ctx, time.Second*180)
Expand Down
5 changes: 3 additions & 2 deletions test/integration/upgradedowngrade_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,13 @@ import (

"github.com/golang/glog"
protoetcd "kope.io/etcd-manager/pkg/apis/etcd"
"kope.io/etcd-manager/pkg/etcdversions"
"kope.io/etcd-manager/test/integration/harness"
)

func TestUpgradeDowngrade(t *testing.T) {
for _, fromVersion := range AllEtcdVersions {
for _, toVersion := range AllEtcdVersions {
for _, fromVersion := range etcdversions.AllEtcdVersions {
for _, toVersion := range etcdversions.AllEtcdVersions {
if fromVersion == toVersion {
continue
}
Expand Down

0 comments on commit 0708fe6

Please sign in to comment.