diff --git a/.ci/build b/.ci/build index 69a57d01c..f08bf75c8 100755 --- a/.ci/build +++ b/.ci/build @@ -18,16 +18,22 @@ set -e # For the build step concourse will set the following environment variables: # SOURCE_PATH - path to component repository root directory. # BINARY_PATH - path to an existing (empty) directory to place build results into. +if [[ $(uname) == 'Darwin' ]]; then + READLINK_BIN="greadlink" +else + READLINK_BIN="readlink" +fi + if [[ -z "${SOURCE_PATH}" ]]; then - export SOURCE_PATH="$(readlink -f $(dirname ${0})/..)" + export SOURCE_PATH="$(${READLINK_BIN} -f $(dirname ${0})/..)" else - export SOURCE_PATH="$(readlink -f "${SOURCE_PATH}")" + export SOURCE_PATH="$(${READLINK_BIN} -f "${SOURCE_PATH}")" fi if [[ -z "${BINARY_PATH}" ]]; then export BINARY_PATH="${SOURCE_PATH}/bin" else - export BINARY_PATH="$(readlink -f "${BINARY_PATH}")/bin" + export BINARY_PATH="$(${READLINK_BIN} -f "${BINARY_PATH}")/bin" fi VCS="github.com" @@ -40,7 +46,7 @@ cd "${SOURCE_PATH}" ############################################################################### -VERSION_FILE="$(readlink -f "${SOURCE_PATH}/VERSION")" +VERSION_FILE="$(${READLINK_BIN} -f "${SOURCE_PATH}/VERSION")" VERSION="$(cat "${VERSION_FILE}")" GIT_SHA=$(git rev-parse --short HEAD || echo "GitNotFound") diff --git a/.ci/integration_test b/.ci/integration_test index a448a95bc..76b5cc574 100755 --- a/.ci/integration_test +++ b/.ci/integration_test @@ -16,11 +16,16 @@ set -e # For the test step concourse will set the following environment variables: # SOURCE_PATH - path to component repository root directory. +if [[ $(uname) == 'Darwin' ]]; then + READLINK_BIN="greadlink" +else + READLINK_BIN="readlink" +fi if [[ -z "${SOURCE_PATH}" ]]; then - export SOURCE_PATH="$(readlink -f "$(dirname ${0})/..")" + export SOURCE_PATH="$(${READLINK_BIN} -f "$(dirname ${0})/..")" else - export SOURCE_PATH="$(readlink -f "${SOURCE_PATH}")" + export SOURCE_PATH="$(${READLINK_BIN} -f "${SOURCE_PATH}")" fi VCS="github.com" @@ -28,7 +33,7 @@ ORGANIZATION="gardener" PROJECT="etcd-backup-restore" REPOSITORY=${VCS}/${ORGANIZATION}/${PROJECT} URL=https://${REPOSITORY}.git -VERSION_FILE="$(readlink -f "${SOURCE_PATH}/VERSION")" +VERSION_FILE="$(${READLINK_BIN} -f "${SOURCE_PATH}/VERSION")" VERSION="$(cat "${VERSION_FILE}")" TEST_ID_PREFIX="etcdbr-test" TM_TEST_ID_PREFIX="etcdbr-tm-test" diff --git a/.ci/unit_test b/.ci/unit_test index a58757816..c6681f2dc 100755 --- a/.ci/unit_test +++ b/.ci/unit_test @@ -16,10 +16,16 @@ set -e # For the test step concourse will set the following environment variables: # SOURCE_PATH - path to component repository root directory. +if [[ $(uname) == 'Darwin' ]]; then + READLINK_BIN="greadlink" +else + READLINK_BIN="readlink" +fi + if [[ -z "${SOURCE_PATH}" ]]; then - export SOURCE_PATH="$(readlink -f "$(dirname ${0})/..")" + export SOURCE_PATH="$(${READLINK_BIN} -f "$(dirname ${0})/..")" else - export SOURCE_PATH="$(readlink -f "${SOURCE_PATH}")" + export SOURCE_PATH="$(${READLINK_BIN} -f "${SOURCE_PATH}")" fi VCS="github.com" diff --git a/pkg/initializer/initializer.go b/pkg/initializer/initializer.go index ca01dbae6..3b861b0f6 100644 --- a/pkg/initializer/initializer.go +++ b/pkg/initializer/initializer.go @@ -92,7 +92,8 @@ func NewInitializer(options *restorer.RestoreOptions, snapstoreConfig *snapstore // bootstrapping a new data directory or if restoration failed func (e *EtcdInitializer) restoreCorruptData() (bool, error) { logger := e.Logger - dataDir := e.Config.RestoreOptions.Config.RestoreDataDir + tempRestoreOptions := *(e.Config.RestoreOptions.DeepCopy()) + dataDir := tempRestoreOptions.Config.RestoreDataDir if e.Config.SnapstoreConfig == nil || len(e.Config.SnapstoreConfig.Provider) == 0 { logger.Warnf("No snapstore storage provider configured.") @@ -116,9 +117,8 @@ func (e *EtcdInitializer) restoreCorruptData() (bool, error) { return e.restoreWithEmptySnapstore() } - e.Config.RestoreOptions.BaseSnapshot = *baseSnap - e.Config.RestoreOptions.DeltaSnapList = deltaSnapList - tempRestoreOptions := *e.Config.RestoreOptions + tempRestoreOptions.BaseSnapshot = *baseSnap + tempRestoreOptions.DeltaSnapList = deltaSnapList tempRestoreOptions.Config.RestoreDataDir = fmt.Sprintf("%s.%s", tempRestoreOptions.Config.RestoreDataDir, "part") if err := e.removeDir(tempRestoreOptions.Config.RestoreDataDir); err != nil { diff --git a/pkg/snapshot/restorer/types.go b/pkg/snapshot/restorer/types.go index 4cf272cfc..5f242150b 100755 --- a/pkg/snapshot/restorer/types.go +++ b/pkg/snapshot/restorer/types.go @@ -15,6 +15,7 @@ package restorer import ( + "net/url" "time" "github.com/coreos/etcd/clientv3" @@ -41,6 +42,7 @@ type Restorer struct { } // RestoreOptions hold all snapshot restore related fields +// Note: Please ensure DeepCopy and DeepCopyInto are properly implemented. type RestoreOptions struct { Config *RestorationConfig ClusterURLs types.URLsMap @@ -51,6 +53,7 @@ type RestoreOptions struct { } // RestorationConfig holds the restoration configuration. +// Note: Please ensure DeepCopy and DeepCopyInto are properly implemented. type RestorationConfig struct { InitialCluster string `json:"initialCluster"` InitialClusterToken string `json:"initialClusterToken,omitempty"` @@ -83,3 +86,95 @@ type applierInfo struct { EventsFilePath string SnapIndex int } + +// DeepCopyInto copies the structure deeply from in to out. +func (in *RestoreOptions) DeepCopyInto(out *RestoreOptions) { + *out = *in + if in.Config != nil { + in, out := &in.Config, &out.Config + *out = new(RestorationConfig) + (*in).DeepCopyInto(*out) + } + if in.ClusterURLs != nil { + in, out := &in.ClusterURLs, &out.ClusterURLs + *out = make(types.URLsMap) + for k := range *in { + if (*in)[k] != nil { + (*out)[k] = DeepCopyURLs((*in)[k]) + } + } + } + if in.PeerURLs != nil { + out.PeerURLs = DeepCopyURLs(in.PeerURLs) + } + if in.DeltaSnapList != nil { + out.DeltaSnapList = DeepCopySnapList(in.DeltaSnapList) + } +} + +// DeepCopyURLs returns a deeply copy +func DeepCopyURLs(in types.URLs) types.URLs { + out := make(types.URLs, len(in)) + for i, u := range in { + out[i] = *(DeepCopyURL(&u)) + } + return out +} + +// DeepCopyURL returns a deeply copy +func DeepCopyURL(in *url.URL) *url.URL { + var out = new(url.URL) + *out = *in + if in.User != nil { + in, out := &in.User, &out.User + *out = new (url.Userinfo) + *out = *in + } + return out +} + +// DeepCopySnapList returns a deep copy +func DeepCopySnapList(in snapstore.SnapList) snapstore.SnapList { + out := make(snapstore.SnapList, len(in)) + for i, v := range in { + if v != nil { + var cpv = *v + out[i] = &cpv + } + } + return out +} + +// DeepCopy returns a deeply copied structure. +func (in *RestoreOptions) DeepCopy() *RestoreOptions { + if in == nil { + return nil + } + + out := new(RestoreOptions) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto copies the structure deeply from in to out. +func (in *RestorationConfig) DeepCopyInto(out *RestorationConfig) { + *out = *in + if in.InitialAdvertisePeerURLs != nil { + in, out := &in.InitialAdvertisePeerURLs, &out.InitialAdvertisePeerURLs + *out = make([]string, len(*in)) + for i, v := range *in { + (*out)[i] = v + } + } +} + +// DeepCopy returns a deeply copied structure. +func (in *RestorationConfig) DeepCopy() *RestorationConfig { + if in == nil { + return nil + } + + out := new(RestorationConfig) + in.DeepCopyInto(out) + return out +} \ No newline at end of file diff --git a/pkg/snapshot/restorer/types_test.go b/pkg/snapshot/restorer/types_test.go new file mode 100644 index 000000000..38dd19668 --- /dev/null +++ b/pkg/snapshot/restorer/types_test.go @@ -0,0 +1,218 @@ +// Copyright (c) 2018 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package restorer_test + +import ( + "fmt" + "net/url" + "time" + + "github.com/coreos/etcd/pkg/types" + "github.com/gardener/etcd-backup-restore/pkg/snapstore" + . "github.com/gardener/etcd-backup-restore/pkg/snapshot/restorer" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("restorer types", func() { + var ( + makeRestorationConfig = func(s string, b bool, i int) *RestorationConfig { + return &RestorationConfig{ + InitialCluster: s, + InitialClusterToken: s, + RestoreDataDir: s, + InitialAdvertisePeerURLs: []string{ s, s }, + Name: s, + SkipHashCheck: b, + MaxFetchers: uint(i), + EmbeddedEtcdQuotaBytes: int64(i), + } + } + makeSnap = func(s string, i int, t time.Time, b bool) snapstore.Snapshot { + return snapstore.Snapshot{ + Kind: s, + StartRevision: int64(i), + LastRevision: int64(i), + CreatedOn: t, + SnapDir: s, + SnapName: s, + IsChunk: b, + } + } + makeSnapList = func(s string, i int, t time.Time, b bool) snapstore.SnapList { + var s1, s2 = makeSnap(s, i, t, b), makeSnap(s, i, t, b) + return snapstore.SnapList{ &s1, &s2 } + } + makeURL = func(s string, b bool) url.URL { + return url.URL{ + Scheme: s, + Opaque: s, + User: url.UserPassword(s, s), + Host: s, + Path: s, + RawPath: s, + ForceQuery: b, + Fragment: s, + } + } + makeURLs = func(s string, b bool) types.URLs { + return types.URLs{ makeURL(s, b), makeURL(s, b) } + } + makeURLsMap = func(s string, b bool) types.URLsMap { + var out = types.URLsMap{} + for _, v := range []int{1, 2} { + out[fmt.Sprintf("%s-%d", s, v)] = makeURLs(s, b) + } + return out + } + makeRestoreOptions = func(s string, i int, t time.Time, b bool) *RestoreOptions { + return &RestoreOptions{ + Config: makeRestorationConfig(s, b, i), + ClusterURLs: makeURLsMap(s, b), + PeerURLs: makeURLs(s, b), + BaseSnapshot: makeSnap(s, i, t, b), + DeltaSnapList: makeSnapList(s, i, t, b), + } + } + ) + + Describe("RestorationConfig", func() { + var ( + makeA = func() *RestorationConfig { return makeRestorationConfig("a", false, 1) } + makeB = func() *RestorationConfig { return makeRestorationConfig("b", true, 2) } + ) + Describe("DeepCopyInto", func() { + It("new out", func() { + var a, in, out = makeA(), makeA(), new(RestorationConfig) + in.DeepCopyInto(out) + Expect(out).To(Equal(in)) + Expect(out).ToNot(BeIdenticalTo(in)) + Expect(in).To(Equal(a)) + Expect(in).ToNot(BeIdenticalTo(a)) + }) + It("existing out", func() { + var a, in, b, out = makeA(), makeA(), makeB(), makeB() + in.DeepCopyInto(out) + Expect(out).To(Equal(in)) + Expect(out).ToNot(BeIdenticalTo(in)) + Expect(in).To(Equal(a)) + Expect(in).ToNot(BeIdenticalTo(a)) + Expect(out).ToNot(Equal(b)) + }) + }) + Describe("DeepCopy", func() { + It("out", func() { + var a, in = makeA(), makeA() + var out = in.DeepCopy() + Expect(out).ToNot(BeNil()) + Expect(out).To(Equal(in)) + Expect(out).ToNot(BeIdenticalTo(in)) + Expect(in).To(Equal(a)) + Expect(in).ToNot(BeIdenticalTo(a)) + }) + }) + }) + + Describe("SnapList", func() { + var ( + now = time.Now() + makeA = func() snapstore.SnapList { return makeSnapList("a", 1, now, false) } + ) + Describe("DeepCopySnapList", func() { + It("out", func() { + var a, in = makeA(), makeA() + var out = DeepCopySnapList(in) + Expect(out).ToNot(BeNil()) + Expect(out).To(Equal(in)) + Expect(out).ToNot(BeIdenticalTo(in)) + Expect(in).To(Equal(a)) + Expect(in).ToNot(BeIdenticalTo(a)) + }) + }) + }) + + Describe("URL", func() { + var ( + makeA = func() *url.URL { var u = makeURL("a", false); return &u } + ) + Describe("DeepCopyURL", func() { + It("out", func() { + var a, in = makeA(), makeA() + var out = DeepCopyURL(in) + Expect(out).ToNot(BeNil()) + Expect(out).To(Equal(in)) + Expect(out).ToNot(BeIdenticalTo(in)) + Expect(in).To(Equal(a)) + Expect(in).ToNot(BeIdenticalTo(a)) + }) + }) + }) + + Describe("URLs", func() { + var ( + makeA = func() types.URLs { return makeURLs("a", false) } + ) + Describe("DeepCopyURLs", func() { + It("out", func() { + var a, in = makeA(), makeA() + var out = DeepCopyURLs(in) + Expect(out).ToNot(BeNil()) + Expect(out).To(Equal(in)) + Expect(out).ToNot(BeIdenticalTo(in)) + Expect(in).To(Equal(a)) + Expect(in).ToNot(BeIdenticalTo(a)) + }) + }) + }) + + Describe("RestoreOptions", func() { + var ( + now = time.Now() + makeA = func() *RestoreOptions { return makeRestoreOptions("a", 1, now, false) } + makeB = func() *RestoreOptions { return makeRestoreOptions("b", 2, now.Add(-1*time.Hour), true) } + ) + Describe("DeepCopyInto", func() { + It("new out", func() { + var a, in, out = makeA(), makeA(), new(RestoreOptions) + in.DeepCopyInto(out) + Expect(out).To(Equal(in)) + Expect(out).ToNot(BeIdenticalTo(in)) + Expect(in).To(Equal(a)) + Expect(in).ToNot(BeIdenticalTo(a)) + }) + It("existing out", func() { + var a, in, b, out = makeA(), makeA(), makeB(), makeB() + in.DeepCopyInto(out) + Expect(out).To(Equal(in)) + Expect(out).ToNot(BeIdenticalTo(in)) + Expect(in).To(Equal(a)) + Expect(in).ToNot(BeIdenticalTo(a)) + Expect(out).ToNot(Equal(b)) + }) + }) + Describe("DeepCopy", func() { + It("out", func() { + var a, in = makeA(), makeA() + var out = in.DeepCopy() + Expect(out).ToNot(BeNil()) + Expect(out).To(Equal(in)) + Expect(out).ToNot(BeIdenticalTo(in)) + Expect(in).To(Equal(a)) + Expect(in).ToNot(BeIdenticalTo(a)) + }) + }) + }) + +}) \ No newline at end of file diff --git a/test/e2e/integration/cloud_backup_test.go b/test/e2e/integration/cloud_backup_test.go index c349889e7..675038836 100644 --- a/test/e2e/integration/cloud_backup_test.go +++ b/test/e2e/integration/cloud_backup_test.go @@ -350,34 +350,40 @@ var _ = Describe("CloudBackup", func() { // Stop etcd. cmdEtcd.StopProcess() time.Sleep(10 * time.Second) + // Corrupt directory - dataDir := os.Getenv("ETCD_DATA_DIR") - dbFilePath := filepath.Join(dataDir, "member", "snap", "db") - logger.Infof("db file: %v", dbFilePath) - file, err := os.Create(dbFilePath) - defer file.Close() - fileWriter := bufio.NewWriter(file) - Expect(err).ShouldNot(HaveOccurred()) - fileWriter.Write([]byte("corrupt file..")) - fileWriter.Flush() - status, err = getEtcdBrServerStatus() - Expect(err).ShouldNot(HaveOccurred()) - Expect(status).Should(Equal("New")) - // Curl request to etcdbrctl server to initialize data directory. - _, err = initializeDataDir() - Expect(err).ShouldNot(HaveOccurred()) - for status, err = getEtcdBrServerStatus(); status == "Progress"; status, err = getEtcdBrServerStatus() { - logger.Infof("Etcdbr server status: %v", status) - time.Sleep(1 * time.Second) - } - Expect(err).ShouldNot(HaveOccurred()) - // Start etcd. - cmdEtcd, etcdErrChan = startEtcd() - go func() { - err := <-*etcdErrChan + testDataCorruptionRestoration := func() { + dataDir := os.Getenv("ETCD_DATA_DIR") + dbFilePath := filepath.Join(dataDir, "member", "snap", "db") + logger.Infof("db file: %v", dbFilePath) + file, err := os.Create(dbFilePath) + defer file.Close() + fileWriter := bufio.NewWriter(file) Expect(err).ShouldNot(HaveOccurred()) - }() - time.Sleep(10 * time.Second) + fileWriter.Write([]byte("corrupt file..")) + fileWriter.Flush() + status, err = getEtcdBrServerStatus() + Expect(err).ShouldNot(HaveOccurred()) + Expect(status).Should(Equal("New")) + // Curl request to etcdbrctl server to initialize data directory. + _, err = initializeDataDir() + Expect(err).ShouldNot(HaveOccurred()) + for status, err = getEtcdBrServerStatus(); status == "Progress"; status, err = getEtcdBrServerStatus() { + logger.Infof("Etcdbr server status: %v", status) + time.Sleep(1 * time.Second) + } + Expect(err).ShouldNot(HaveOccurred()) + // Start etcd. + cmdEtcd, etcdErrChan = startEtcd() + go func() { + err := <-*etcdErrChan + Expect(err).ShouldNot(HaveOccurred()) + }() + time.Sleep(10 * time.Second) + } + for i := 0; i < 3; i++ { // 3 consecutive restorations + testDataCorruptionRestoration() + } }) AfterEach(func() { cmdEtcd.StopProcess() diff --git a/test/e2e/integrationcluster/backup_test.go b/test/e2e/integrationcluster/backup_test.go index e40bd19b3..d9ab4a1a8 100644 --- a/test/e2e/integrationcluster/backup_test.go +++ b/test/e2e/integrationcluster/backup_test.go @@ -272,47 +272,52 @@ var _ = Describe("Backup", func() { Context("when data is corrupt", func() { It("should restore data from latest snapshot", func() { - cmd := "rm -rf /var/etcd/data/new.etcd/member" - stdout, stderr, err := executeRemoteCommand(kubeconfigPath, releaseNamespace, podName, "backup-restore", cmd) - Expect(err).ShouldNot(HaveOccurred()) - Expect(stderr).Should(BeEmpty()) - Expect(stdout).Should(BeEmpty()) - - podClient := typedClient.CoreV1().Pods(releaseNamespace) - err = podClient.Delete(context.TODO(), podName, metav1.DeleteOptions{}) - Expect(err).ShouldNot(HaveOccurred()) - time.Sleep(time.Duration(time.Second * 5)) - - logger.Infof("waiting for %s pod to be running", podName) - err = waitForPodToBeRunning(typedClient, podName, releaseNamespace) - Expect(err).ShouldNot(HaveOccurred()) - logger.Infof("waiting for %s endpoint to be ready", etcdEndpointName) - err = waitForEndpointPortsToBeReady(typedClient, etcdEndpointName, releaseNamespace, []int32{etcdClientPort}) - Expect(err).ShouldNot(HaveOccurred()) - logger.Infof("waiting for %s endpoint to be ready", backupEndpointName) - err = waitForEndpointPortsToBeReady(typedClient, backupEndpointName, releaseNamespace, []int32{backupClientPort}) - Expect(err).ShouldNot(HaveOccurred()) - logger.Infof("pod %s and endpoints %s, %s ready", podName, etcdEndpointName, backupEndpointName) - - cmd = fmt.Sprintf("curl http://localhost:%d/initialization/status -s", backupClientPort) - stdout, stderr, err = executeRemoteCommand(kubeconfigPath, releaseNamespace, podName, "backup-restore", cmd) - Expect(err).ShouldNot(HaveOccurred()) - Expect(stdout).Should(Equal("New")) - - cmd = "ETCDCTL_API=3 etcdctl get init-3" - stdout, stderr, err = executeRemoteCommand(kubeconfigPath, releaseNamespace, podName, "etcd", cmd) - Expect(err).ShouldNot(HaveOccurred()) - Expect(stderr).Should(BeEmpty()) - lines := strings.Split(stdout, "\n") - Expect(len(lines)).Should(Equal(2)) - Expect(lines[0]).Should(Equal("init-3")) - Expect(lines[1]).Should(Equal("val-3")) - - cmd = "ETCDCTL_API=3 etcdctl get init-4" - stdout, stderr, err = executeRemoteCommand(kubeconfigPath, releaseNamespace, podName, "etcd", cmd) - Expect(err).ShouldNot(HaveOccurred()) - Expect(stderr).Should(BeEmpty()) - Expect(stdout).Should(BeEmpty()) + testDataCorruptionRestoration := func() { + cmd := "rm -rf /var/etcd/data/new.etcd/member" + stdout, stderr, err := executeRemoteCommand(kubeconfigPath, releaseNamespace, podName, "backup-restore", cmd) + Expect(err).ShouldNot(HaveOccurred()) + Expect(stderr).Should(BeEmpty()) + Expect(stdout).Should(BeEmpty()) + + podClient := typedClient.CoreV1().Pods(releaseNamespace) + err = podClient.Delete(context.TODO(), podName, metav1.DeleteOptions{}) + Expect(err).ShouldNot(HaveOccurred()) + time.Sleep(time.Duration(time.Second * 5)) + + logger.Infof("waiting for %s pod to be running", podName) + err = waitForPodToBeRunning(typedClient, podName, releaseNamespace) + Expect(err).ShouldNot(HaveOccurred()) + logger.Infof("waiting for %s endpoint to be ready", etcdEndpointName) + err = waitForEndpointPortsToBeReady(typedClient, etcdEndpointName, releaseNamespace, []int32{etcdClientPort}) + Expect(err).ShouldNot(HaveOccurred()) + logger.Infof("waiting for %s endpoint to be ready", backupEndpointName) + err = waitForEndpointPortsToBeReady(typedClient, backupEndpointName, releaseNamespace, []int32{backupClientPort}) + Expect(err).ShouldNot(HaveOccurred()) + logger.Infof("pod %s and endpoints %s, %s ready", podName, etcdEndpointName, backupEndpointName) + + cmd = fmt.Sprintf("curl http://localhost:%d/initialization/status -s", backupClientPort) + stdout, stderr, err = executeRemoteCommand(kubeconfigPath, releaseNamespace, podName, "backup-restore", cmd) + Expect(err).ShouldNot(HaveOccurred()) + Expect(stdout).Should(Equal("New")) + + cmd = "ETCDCTL_API=3 etcdctl get init-3" + stdout, stderr, err = executeRemoteCommand(kubeconfigPath, releaseNamespace, podName, "etcd", cmd) + Expect(err).ShouldNot(HaveOccurred()) + Expect(stderr).Should(BeEmpty()) + lines := strings.Split(stdout, "\n") + Expect(len(lines)).Should(Equal(2)) + Expect(lines[0]).Should(Equal("init-3")) + Expect(lines[1]).Should(Equal("val-3")) + + cmd = "ETCDCTL_API=3 etcdctl get init-4" + stdout, stderr, err = executeRemoteCommand(kubeconfigPath, releaseNamespace, podName, "etcd", cmd) + Expect(err).ShouldNot(HaveOccurred()) + Expect(stderr).Should(BeEmpty()) + Expect(stdout).Should(BeEmpty()) + } + for i := 0; i < 3; i++ { // 3 consecutive restorations + testDataCorruptionRestoration() + } }) }) })