From bf01c49591f88df9be7c0ba885fb0005fa9be95a Mon Sep 17 00:00:00 2001 From: aaronfern Date: Thu, 6 May 2021 12:18:07 +0530 Subject: [PATCH] Added tests for compaction backward compatibility --- pkg/compactor/compactor_suite_test.go | 4 +- pkg/snapshot/restorer/restorer_suite_test.go | 5 +- pkg/snapshot/restorer/restorer_test.go | 441 ++++++++++++++++- pkg/snapshot/snapshotter/snapshotter_test.go | 489 ++++++++++++++++++- test/utils/utils.go | 4 +- 5 files changed, 924 insertions(+), 19 deletions(-) diff --git a/pkg/compactor/compactor_suite_test.go b/pkg/compactor/compactor_suite_test.go index d18814a39..843eee65f 100644 --- a/pkg/compactor/compactor_suite_test.go +++ b/pkg/compactor/compactor_suite_test.go @@ -24,6 +24,7 @@ import ( "time" "github.com/gardener/etcd-backup-restore/pkg/compressor" + brtypes "github.com/gardener/etcd-backup-restore/pkg/types" "github.com/gardener/etcd-backup-restore/test/utils" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -83,7 +84,8 @@ var _ = SynchronizedBeforeSuite(func() []byte { compressionConfig := compressor.NewCompressorConfig() compressionConfig.Enabled = true compressionConfig.CompressionPolicy = "gzip" - err = utils.RunSnapshotter(logger, testSnapshotDir, deltaSnapshotPeriod, endpoints, ctx.Done(), true, compressionConfig) + snapstoreConfig := brtypes.SnapstoreConfig{Container: testSnapshotDir, Provider: "Local"} + err = utils.RunSnapshotter(logger, snapstoreConfig, deltaSnapshotPeriod, endpoints, ctx.Done(), true, compressionConfig) Expect(err).ShouldNot(HaveOccurred()) // Wait unitil the populator finishes with populating ETCD diff --git a/pkg/snapshot/restorer/restorer_suite_test.go b/pkg/snapshot/restorer/restorer_suite_test.go index 848f80fdb..b8e386232 100644 --- a/pkg/snapshot/restorer/restorer_suite_test.go +++ b/pkg/snapshot/restorer/restorer_suite_test.go @@ -23,6 +23,7 @@ import ( "time" "github.com/gardener/etcd-backup-restore/pkg/compressor" + brtypes "github.com/gardener/etcd-backup-restore/pkg/types" "github.com/gardener/etcd-backup-restore/test/utils" . "github.com/onsi/ginkgo" @@ -78,12 +79,12 @@ var _ = SynchronizedBeforeSuite(func() []byte { ctx := utils.ContextWithWaitGroupFollwedByGracePeriod(populatorCtx, wg, deltaSnapshotPeriod+2*time.Second) compressionConfig := compressor.NewCompressorConfig() - err = utils.RunSnapshotter(logger, snapstoreDir, deltaSnapshotPeriod, endpoints, ctx.Done(), true, compressionConfig) + snapstoreConfig := brtypes.SnapstoreConfig{Container: snapstoreDir, Provider: "Local"} + err = utils.RunSnapshotter(logger, snapstoreConfig, deltaSnapshotPeriod, endpoints, ctx.Done(), true, compressionConfig) Expect(err).ShouldNot(HaveOccurred()) keyTo = resp.KeyTo return data - }, func(data []byte) {}) var _ = SynchronizedAfterSuite(func() {}, cleanUp) diff --git a/pkg/snapshot/restorer/restorer_test.go b/pkg/snapshot/restorer/restorer_test.go index 0d51c2634..6a47a156c 100644 --- a/pkg/snapshot/restorer/restorer_test.go +++ b/pkg/snapshot/restorer/restorer_test.go @@ -16,8 +16,11 @@ package restorer_test import ( "context" + "fmt" + "io/ioutil" "os" "path" + "strings" "sync" "time" @@ -26,6 +29,7 @@ import ( "github.com/gardener/etcd-backup-restore/pkg/snapstore" brtypes "github.com/gardener/etcd-backup-restore/pkg/types" "github.com/gardener/etcd-backup-restore/test/utils" + "github.com/sirupsen/logrus" "go.etcd.io/etcd/pkg/types" . "github.com/gardener/etcd-backup-restore/pkg/snapshot/restorer" @@ -33,6 +37,12 @@ import ( . "github.com/onsi/gomega" ) +const ( + allSnapsInV1 = "allSnapsInV1" + fullSnapInV1 = "fullSnapInV1" + fullSnapInV2 = "fullSnapInV2" +) + var _ = Describe("Running Restorer", func() { var ( store brtypes.SnapStore @@ -277,7 +287,8 @@ var _ = Describe("Running Restorer", func() { logger.Infoln("Starting snapshotter with basesnapshot set to false") ssrCtx := utils.ContextWithWaitGroupFollwedByGracePeriod(testCtx, wg, 2) compressionConfig := compressor.NewCompressorConfig() - err = utils.RunSnapshotter(logger, snapstoreDir, deltaSnapshotPeriod, endpoints, ssrCtx.Done(), startWithFullSnapshot, compressionConfig) + snapstoreConfig := brtypes.SnapstoreConfig{Container: snapstoreDir, Provider: "Local"} + err = utils.RunSnapshotter(logger, snapstoreConfig, deltaSnapshotPeriod, endpoints, ssrCtx.Done(), startWithFullSnapshot, compressionConfig) Expect(err).ShouldNot(HaveOccurred()) etcd.Server.Stop() etcd.Close() @@ -322,7 +333,8 @@ var _ = Describe("Running Restorer", func() { defer cancelPopulator() ssrCtx := utils.ContextWithWaitGroupFollwedByGracePeriod(testCtx, wg, time.Second) compressionConfig := compressor.NewCompressorConfig() - err = utils.RunSnapshotter(logger, snapstoreDir, deltaSnapshotPeriod, endpoints, ssrCtx.Done(), true, compressionConfig) + snapstoreConfig := brtypes.SnapstoreConfig{Container: snapstoreDir, Provider: "Local"} + err = utils.RunSnapshotter(logger, snapstoreConfig, deltaSnapshotPeriod, endpoints, ssrCtx.Done(), true, compressionConfig) Expect(err).ShouldNot(HaveOccurred()) etcd.Server.Stop() etcd.Close() @@ -360,7 +372,8 @@ var _ = Describe("Running Restorer", func() { defer cancelPopulator() ssrCtx := utils.ContextWithWaitGroupFollwedByGracePeriod(testCtx, wg, time.Second) compressionConfig := compressor.NewCompressorConfig() - err = utils.RunSnapshotter(logger, snapstoreDir, deltaSnapshotPeriod, endpoints, ssrCtx.Done(), true, compressionConfig) + snapstoreConfig := brtypes.SnapstoreConfig{Container: snapstoreDir, Provider: "Local"} + err = utils.RunSnapshotter(logger, snapstoreConfig, deltaSnapshotPeriod, endpoints, ssrCtx.Done(), true, compressionConfig) Expect(err).ShouldNot(HaveOccurred()) etcd.Close() @@ -406,7 +419,8 @@ var _ = Describe("Running Restorer", func() { defer cancelPopulator() ssrCtx := utils.ContextWithWaitGroupFollwedByGracePeriod(testCtx, wg, 2*time.Second) compressionConfig := compressor.NewCompressorConfig() - err = utils.RunSnapshotter(logger, snapstoreDir, deltaSnapshotPeriod, endpoints, ssrCtx.Done(), true, compressionConfig) + snapstoreConfig := brtypes.SnapstoreConfig{Container: snapstoreDir, Provider: "Local"} + err = utils.RunSnapshotter(logger, snapstoreConfig, deltaSnapshotPeriod, endpoints, ssrCtx.Done(), true, compressionConfig) Expect(err).ShouldNot(HaveOccurred()) etcd.Close() @@ -444,7 +458,8 @@ var _ = Describe("Running Restorer", func() { logger.Infoln("Starting snapshotter while loading is happening") compressionConfig := compressor.NewCompressorConfig() - err = utils.RunSnapshotter(logger, snapstoreDir, deltaSnapshotPeriod, endpoints, ssrCtx.Done(), true, compressionConfig) + snapstoreConfig := brtypes.SnapstoreConfig{Container: snapstoreDir, Provider: "Local"} + err = utils.RunSnapshotter(logger, snapstoreConfig, deltaSnapshotPeriod, endpoints, ssrCtx.Done(), true, compressionConfig) Expect(err).ShouldNot(HaveOccurred()) time.Sleep(time.Duration(5 * time.Second)) @@ -493,7 +508,8 @@ var _ = Describe("Running Restorer", func() { compressionConfig := compressor.NewCompressorConfig() compressionConfig.Enabled = false ctx, cancel := context.WithTimeout(testCtx, time.Duration(2*time.Second)) - err = utils.RunSnapshotter(logger, snapstoreDir, deltaSnapshotPeriod, endpoints, ctx.Done(), true, compressionConfig) + snapstoreConfig := brtypes.SnapstoreConfig{Container: snapstoreDir, Provider: "Local"} + err = utils.RunSnapshotter(logger, snapstoreConfig, deltaSnapshotPeriod, endpoints, ctx.Done(), true, compressionConfig) Expect(err).ShouldNot(HaveOccurred()) cancel() @@ -507,7 +523,8 @@ var _ = Describe("Running Restorer", func() { compressionConfig.Enabled = true compressionConfig.CompressionPolicy = "lzw" ctx, cancel = context.WithTimeout(testCtx, time.Duration(2*time.Second)) - err = utils.RunSnapshotter(logger, snapstoreDir, deltaSnapshotPeriod, endpoints, ctx.Done(), false, compressionConfig) + snapstoreConfig = brtypes.SnapstoreConfig{Container: snapstoreDir, Provider: "Local"} + err = utils.RunSnapshotter(logger, snapstoreConfig, deltaSnapshotPeriod, endpoints, ctx.Done(), false, compressionConfig) Expect(err).ShouldNot(HaveOccurred()) cancel() @@ -520,7 +537,8 @@ var _ = Describe("Running Restorer", func() { compressionConfig = compressor.NewCompressorConfig() compressionConfig.Enabled = true ctx, cancel = context.WithTimeout(testCtx, time.Duration(2*time.Second)) - err = utils.RunSnapshotter(logger, snapstoreDir, deltaSnapshotPeriod, endpoints, ctx.Done(), false, compressionConfig) + snapstoreConfig = brtypes.SnapstoreConfig{Container: snapstoreDir, Provider: "Local"} + err = utils.RunSnapshotter(logger, snapstoreConfig, deltaSnapshotPeriod, endpoints, ctx.Done(), false, compressionConfig) Expect(err).ShouldNot(HaveOccurred()) cancel() @@ -534,7 +552,8 @@ var _ = Describe("Running Restorer", func() { compressionConfig.Enabled = true compressionConfig.CompressionPolicy = "zlib" ctx, cancel = context.WithTimeout(testCtx, time.Duration(2*time.Second)) - err = utils.RunSnapshotter(logger, snapstoreDir, deltaSnapshotPeriod, endpoints, ctx.Done(), false, compressionConfig) + snapstoreConfig = brtypes.SnapstoreConfig{Container: snapstoreDir, Provider: "Local"} + err = utils.RunSnapshotter(logger, snapstoreConfig, deltaSnapshotPeriod, endpoints, ctx.Done(), false, compressionConfig) Expect(err).ShouldNot(HaveOccurred()) cancel() @@ -575,7 +594,8 @@ var _ = Describe("Running Restorer", func() { compressionConfig := compressor.NewCompressorConfig() compressionConfig.Enabled = true ctx, cancel := context.WithTimeout(testCtx, time.Duration(2*time.Second)) - err = utils.RunSnapshotter(logger, snapstoreDir, deltaSnapshotPeriod, endpoints, ctx.Done(), true, compressionConfig) + snapstoreConfig := brtypes.SnapstoreConfig{Container: snapstoreDir, Provider: "Local"} + err = utils.RunSnapshotter(logger, snapstoreConfig, deltaSnapshotPeriod, endpoints, ctx.Done(), true, compressionConfig) Expect(err).ShouldNot(HaveOccurred()) cancel() @@ -589,7 +609,8 @@ var _ = Describe("Running Restorer", func() { compressionConfig.Enabled = true compressionConfig.CompressionPolicy = "lzw" ctx, cancel = context.WithTimeout(testCtx, time.Duration(2*time.Second)) - err = utils.RunSnapshotter(logger, snapstoreDir, deltaSnapshotPeriod, endpoints, ctx.Done(), false, compressionConfig) + snapstoreConfig = brtypes.SnapstoreConfig{Container: snapstoreDir, Provider: "Local"} + err = utils.RunSnapshotter(logger, snapstoreConfig, deltaSnapshotPeriod, endpoints, ctx.Done(), false, compressionConfig) Expect(err).ShouldNot(HaveOccurred()) cancel() @@ -603,7 +624,8 @@ var _ = Describe("Running Restorer", func() { compressionConfig.Enabled = true compressionConfig.CompressionPolicy = "zlib" ctx, cancel = context.WithTimeout(testCtx, time.Duration(2*time.Second)) - err = utils.RunSnapshotter(logger, snapstoreDir, deltaSnapshotPeriod, endpoints, ctx.Done(), false, compressionConfig) + snapstoreConfig = brtypes.SnapstoreConfig{Container: snapstoreDir, Provider: "Local"} + err = utils.RunSnapshotter(logger, snapstoreConfig, deltaSnapshotPeriod, endpoints, ctx.Done(), false, compressionConfig) Expect(err).ShouldNot(HaveOccurred()) cancel() @@ -615,7 +637,8 @@ var _ = Describe("Running Restorer", func() { // start the Snapshotter with compression not enabled to take delta snapshot. compressionConfig = compressor.NewCompressorConfig() ctx, cancel = context.WithTimeout(testCtx, time.Duration(2*time.Second)) - err = utils.RunSnapshotter(logger, snapstoreDir, deltaSnapshotPeriod, endpoints, ctx.Done(), false, compressionConfig) + snapstoreConfig = brtypes.SnapstoreConfig{Container: snapstoreDir, Provider: "Local"} + err = utils.RunSnapshotter(logger, snapstoreConfig, deltaSnapshotPeriod, endpoints, ctx.Done(), false, compressionConfig) Expect(err).ShouldNot(HaveOccurred()) cancel() @@ -645,6 +668,287 @@ var _ = Describe("Running Restorer", func() { }) + Describe("For scenarios involving both old as well as updated directory structures being present", func() { + var ( + store brtypes.SnapStore + deltaSnapshotPeriod time.Duration + endpoints []string + restorationConfig *brtypes.RestorationConfig + ) + + BeforeEach(func() { + cleanUp() //Cleans etcd and backup store for these tests + deltaSnapshotPeriod = time.Second + etcd, err = utils.StartEmbeddedEtcd(testCtx, etcdDir, logger) + Expect(err).ShouldNot(HaveOccurred()) + endpoints = []string{etcd.Clients[0].Addr().String()} + + store, err = snapstore.GetSnapstore(&brtypes.SnapstoreConfig{Container: snapstoreDir, Provider: "Local", Prefix: "v2"}) + Expect(err).ShouldNot(HaveOccurred()) + + restorationConfig = &brtypes.RestorationConfig{ + RestoreDataDir: etcdDir, + InitialClusterToken: restoreClusterToken, + InitialCluster: restoreCluster, + Name: restoreName, + InitialAdvertisePeerURLs: restorePeerURLs, + SkipHashCheck: skipHashCheck, + MaxFetchers: maxFetchers, + MaxCallSendMsgSize: maxCallSendMsgSize, + MaxRequestBytes: maxRequestBytes, + MaxTxnOps: maxTxnOps, + EmbeddedEtcdQuotaBytes: embeddedEtcdQuotaBytes, + AutoCompactionMode: autoCompactionMode, + AutoCompactionRetention: autoCompactionRetention, + } + }) + + // Test to check backward compatibility of restorer + // Tests restorer behaviour when local database has to be restored from snapstore with old (v1) as well as updated (v2) directory structures + // TODO: Consider removing when backward compatibility no longer needed + Context("With snapshots in v1 as well as v2 dir", func() { + It("should restore from v2 dir snapshots", func() { + memberPath := path.Join(etcdDir, "member") + + //Take snapshots for v1 dir + compressionConfig := compressor.NewCompressorConfig() + ctx, cancel := context.WithTimeout(testCtx, time.Duration(2*time.Second)) + err = takeInvalidV1Snaps(logger, snapstoreDir, deltaSnapshotPeriod, endpoints, ctx.Done(), compressionConfig) + Expect(err).ShouldNot(HaveOccurred()) + cancel() + + //Take snapshots for v2 dir + //Add data into etcd + resp := &utils.EtcdDataPopulationResponse{} + utils.PopulateEtcd(testCtx, logger, endpoints, 0, keyTo, resp) + Expect(resp.Err).ShouldNot(HaveOccurred()) + //Take a full snapshot + ctx, cancel = context.WithTimeout(testCtx, time.Duration(2*time.Second)) + snapstoreConfig := brtypes.SnapstoreConfig{Container: snapstoreDir, Provider: "Local", Prefix: "v2"} + err = utils.RunSnapshotter(logger, snapstoreConfig, deltaSnapshotPeriod, endpoints, ctx.Done(), true, compressionConfig) + Expect(err).ShouldNot(HaveOccurred()) + cancel() + + //Add data into etcd + utils.PopulateEtcd(testCtx, logger, endpoints, 0, keyTo, resp) + Expect(resp.Err).ShouldNot(HaveOccurred()) + //Take a delta snapshot + ctx, cancel = context.WithTimeout(testCtx, time.Duration(2*time.Second)) + snapstoreConfig = brtypes.SnapstoreConfig{Container: snapstoreDir, Provider: "Local", Prefix: "v2"} + err = utils.RunSnapshotter(logger, snapstoreConfig, deltaSnapshotPeriod, endpoints, ctx.Done(), false, compressionConfig) + Expect(err).ShouldNot(HaveOccurred()) + cancel() + + //Add data into etcd + utils.PopulateEtcd(testCtx, logger, endpoints, 0, keyTo, resp) + Expect(resp.Err).ShouldNot(HaveOccurred()) + //Take a delta snapshot + ctx, cancel = context.WithTimeout(testCtx, time.Duration(2*time.Second)) + snapstoreConfig = brtypes.SnapstoreConfig{Container: snapstoreDir, Provider: "Local", Prefix: "v2"} + err = utils.RunSnapshotter(logger, snapstoreConfig, deltaSnapshotPeriod, endpoints, ctx.Done(), false, compressionConfig) + Expect(err).ShouldNot(HaveOccurred()) + cancel() + + // remove the member dir + err = os.RemoveAll(memberPath) + Expect(err).ShouldNot(HaveOccurred()) + + baseSnapshot, deltaSnapList, err = miscellaneous.GetLatestFullSnapshotAndDeltaSnapList(store) + Expect(err).ShouldNot(HaveOccurred()) + + rstr = NewRestorer(store, logger) + + restoreOpts := brtypes.RestoreOptions{ + Config: restorationConfig, + BaseSnapshot: baseSnapshot, + DeltaSnapList: deltaSnapList, + ClusterURLs: clusterUrlsMap, + PeerURLs: peerUrls, + } + + //Restore + err = rstr.RestoreAndStopEtcd(restoreOpts) + Expect(err).ShouldNot(HaveOccurred()) + err = utils.CheckDataConsistency(testCtx, restoreOpts.Config.RestoreDataDir, keyTo, logger) + Expect(err).ShouldNot(HaveOccurred()) + }) + }) + + //Test to check backward compatibility of restorer + //Tests restorer behaviour when local database has to be restored from snapstore with only old (v1) directory structures + //TODO: Consider removing when backward compatibility no longer needed + Context("With snapshots in v1 dir only", func() { + It("should restore from v1 dir", func() { + memberPath := path.Join(etcdDir, "member") + + //Take snapshots for v1 dir + compressionConfig := compressor.NewCompressorConfig() + ctx, cancel := context.WithTimeout(testCtx, time.Duration(2*time.Second)) + err = takeValidV1Snaps(logger, snapstoreDir, deltaSnapshotPeriod, endpoints, ctx.Done(), compressionConfig, allSnapsInV1) + cancel() + + // remove the member dir + err = os.RemoveAll(memberPath) + Expect(err).ShouldNot(HaveOccurred()) + + baseSnapshot, deltaSnapList, err = miscellaneous.GetLatestFullSnapshotAndDeltaSnapList(store) + Expect(err).ShouldNot(HaveOccurred()) + + rstr = NewRestorer(store, logger) + + restoreOpts := brtypes.RestoreOptions{ + Config: restorationConfig, + BaseSnapshot: baseSnapshot, + DeltaSnapList: deltaSnapList, + ClusterURLs: clusterUrlsMap, + PeerURLs: peerUrls, + } + + //Restore + err = rstr.RestoreAndStopEtcd(restoreOpts) + Expect(err).ShouldNot(HaveOccurred()) + err = utils.CheckDataConsistency(testCtx, restoreOpts.Config.RestoreDataDir, keyTo, logger) + Expect(err).ShouldNot(HaveOccurred()) + }) + }) + + //Test to check backward compatibility of restorer + //Tests restorer behaviour when local database has to be restored from snapstore with only updated (v2) directory structures + //TODO: Consider removing when backward compatibility no longer needed + Context("With snapshots in v2 dir only", func() { + It("should restore from v2 dir snapshots", func() { + memberPath := path.Join(etcdDir, "member") + compressionConfig := compressor.NewCompressorConfig() + + //Snapshots for the v2 dir + //Add data into etcd + resp := &utils.EtcdDataPopulationResponse{} + utils.PopulateEtcd(testCtx, logger, endpoints, 0, keyTo, resp) + Expect(resp.Err).ShouldNot(HaveOccurred()) + //Take a full snapshot + ctx, cancel := context.WithTimeout(testCtx, time.Duration(2*time.Second)) + snapstoreConfig := brtypes.SnapstoreConfig{Container: snapstoreDir, Provider: "Local", Prefix: "v2"} + err = utils.RunSnapshotter(logger, snapstoreConfig, deltaSnapshotPeriod, endpoints, ctx.Done(), true, compressionConfig) + Expect(err).ShouldNot(HaveOccurred()) + cancel() + + //Add data to etcd + utils.PopulateEtcd(testCtx, logger, endpoints, 0, keyTo, resp) + Expect(resp.Err).ShouldNot(HaveOccurred()) + //Take delta snapshot + ctx, cancel = context.WithTimeout(testCtx, time.Duration(2*time.Second)) + snapstoreConfig = brtypes.SnapstoreConfig{Container: snapstoreDir, Provider: "Local", Prefix: "v2"} + err = utils.RunSnapshotter(logger, snapstoreConfig, deltaSnapshotPeriod, endpoints, ctx.Done(), false, compressionConfig) + Expect(err).ShouldNot(HaveOccurred()) + cancel() + + //Add data into etcd + utils.PopulateEtcd(testCtx, logger, endpoints, 0, keyTo, resp) + Expect(resp.Err).ShouldNot(HaveOccurred()) + //Take delta snapshot + ctx, cancel = context.WithTimeout(testCtx, time.Duration(2*time.Second)) + snapstoreConfig = brtypes.SnapstoreConfig{Container: snapstoreDir, Provider: "Local", Prefix: "v2"} + err = utils.RunSnapshotter(logger, snapstoreConfig, deltaSnapshotPeriod, endpoints, ctx.Done(), false, compressionConfig) + Expect(err).ShouldNot(HaveOccurred()) + cancel() + + // remove the member dir + err = os.RemoveAll(memberPath) + Expect(err).ShouldNot(HaveOccurred()) + + baseSnapshot, deltaSnapList, err = miscellaneous.GetLatestFullSnapshotAndDeltaSnapList(store) + Expect(err).ShouldNot(HaveOccurred()) + + rstr = NewRestorer(store, logger) + + restoreOpts := brtypes.RestoreOptions{ + Config: restorationConfig, + BaseSnapshot: baseSnapshot, + DeltaSnapList: deltaSnapList, + ClusterURLs: clusterUrlsMap, + PeerURLs: peerUrls, + } + + //Restore + err = rstr.RestoreAndStopEtcd(restoreOpts) + Expect(err).ShouldNot(HaveOccurred()) + err = utils.CheckDataConsistency(testCtx, restoreOpts.Config.RestoreDataDir, keyTo, logger) + Expect(err).ShouldNot(HaveOccurred()) + }) + }) + + Context("With first few snapshots in v1 dir and some more incr snapshots are in v2 dir", func() { + It("should restore from v1 dir and v2 dir", func() { + memberPath := path.Join(etcdDir, "member") + + //Take snapshots for v1 dir + compressionConfig := compressor.NewCompressorConfig() + ctx, cancel := context.WithTimeout(testCtx, time.Duration(2*time.Second)) + err = takeValidV1Snaps(logger, snapstoreDir, deltaSnapshotPeriod, endpoints, ctx.Done(), compressionConfig, fullSnapInV1) + cancel() + + // remove the member dir + err = os.RemoveAll(memberPath) + Expect(err).ShouldNot(HaveOccurred()) + + baseSnapshot, deltaSnapList, err = miscellaneous.GetLatestFullSnapshotAndDeltaSnapList(store) + Expect(err).ShouldNot(HaveOccurred()) + + rstr = NewRestorer(store, logger) + + restoreOpts := brtypes.RestoreOptions{ + Config: restorationConfig, + BaseSnapshot: baseSnapshot, + DeltaSnapList: deltaSnapList, + ClusterURLs: clusterUrlsMap, + PeerURLs: peerUrls, + } + + //Restore + err = rstr.RestoreAndStopEtcd(restoreOpts) + Expect(err).ShouldNot(HaveOccurred()) + err = utils.CheckDataConsistency(testCtx, restoreOpts.Config.RestoreDataDir, 400, logger) + Expect(err).ShouldNot(HaveOccurred()) + }) + }) + + Context("With first few snapshots in v2 dir and some more incr snapshots are in v1 dir", func() { + It("should restore from v1 dir and v2 dir", func() { + memberPath := path.Join(etcdDir, "member") + + //Take snapshots for v1 dir + compressionConfig := compressor.NewCompressorConfig() + ctx, cancel := context.WithTimeout(testCtx, time.Duration(2*time.Second)) + err = takeValidV1Snaps(logger, snapstoreDir, deltaSnapshotPeriod, endpoints, ctx.Done(), compressionConfig, fullSnapInV2) + cancel() + + // remove the member dir + err = os.RemoveAll(memberPath) + Expect(err).ShouldNot(HaveOccurred()) + + baseSnapshot, deltaSnapList, err = miscellaneous.GetLatestFullSnapshotAndDeltaSnapList(store) + Expect(err).ShouldNot(HaveOccurred()) + + rstr = NewRestorer(store, logger) + + restoreOpts := brtypes.RestoreOptions{ + Config: restorationConfig, + BaseSnapshot: baseSnapshot, + DeltaSnapList: deltaSnapList, + ClusterURLs: clusterUrlsMap, + PeerURLs: peerUrls, + } + + //Restore + err = rstr.RestoreAndStopEtcd(restoreOpts) + Expect(err).ShouldNot(HaveOccurred()) + err = utils.CheckDataConsistency(testCtx, restoreOpts.Config.RestoreDataDir, 400, logger) + Expect(err).ShouldNot(HaveOccurred()) + }) + }) + + }) + }) // corruptEtcdDir corrupts the etcd directory by deleting it @@ -654,3 +958,114 @@ func corruptEtcdDir() error { } return os.RemoveAll(etcdDir) } + +//takeValidV1Snaps saves valid snaps in the v1 prefix dir of snapstore so that restorer could restore from them +//TODO: Consider removing when backward compatibility no longer needed +func takeValidV1Snaps(logger *logrus.Entry, container string, deltaSnapshotPeriod time.Duration, endpoints []string, stopCh <-chan struct{}, compressionConfig *compressor.CompressionConfig, mode string) error { + //Here we run the snapshotter to take snapshots. The snapshotter by default stores the snaps in the v2 directory. + //We then move those snaps into the v1 dir under a 'Backup-xxxxxx' dir + + //Snapshots for the v2 dir + //Add data into etcd + resp := &utils.EtcdDataPopulationResponse{} + utils.PopulateEtcd(testCtx, logger, endpoints, 0, 100, resp) + Expect(resp.Err).ShouldNot(HaveOccurred()) + //Take a full snapshot. + ctx, cancel := context.WithTimeout(testCtx, time.Duration(2*time.Second)) + snapstoreConfig := brtypes.SnapstoreConfig{Container: snapstoreDir, Provider: "Local", Prefix: "v2"} + err = utils.RunSnapshotter(logger, snapstoreConfig, deltaSnapshotPeriod, endpoints, ctx.Done(), true, compressionConfig) + Expect(err).ShouldNot(HaveOccurred()) + cancel() + + //Add data into etcd + utils.PopulateEtcd(testCtx, logger, endpoints, 100, 200, resp) + Expect(resp.Err).ShouldNot(HaveOccurred()) + //Take delta snapshot. + ctx, cancel = context.WithTimeout(testCtx, time.Duration(2*time.Second)) + snapstoreConfig = brtypes.SnapstoreConfig{Container: snapstoreDir, Provider: "Local", Prefix: "v2"} + err = utils.RunSnapshotter(logger, snapstoreConfig, deltaSnapshotPeriod, endpoints, ctx.Done(), false, compressionConfig) + Expect(err).ShouldNot(HaveOccurred()) + cancel() + + //Add data to etcd + utils.PopulateEtcd(testCtx, logger, endpoints, 200, 300, resp) + Expect(resp.Err).ShouldNot(HaveOccurred()) + //Take delta snapshot. + ctx, cancel = context.WithTimeout(testCtx, time.Duration(2*time.Second)) + snapstoreConfig = brtypes.SnapstoreConfig{Container: snapstoreDir, Provider: "Local", Prefix: "v2"} + err = utils.RunSnapshotter(logger, snapstoreConfig, deltaSnapshotPeriod, endpoints, ctx.Done(), false, compressionConfig) + Expect(err).ShouldNot(HaveOccurred()) + cancel() + + //Add data to etcd + utils.PopulateEtcd(testCtx, logger, endpoints, 300, 400, resp) + Expect(resp.Err).ShouldNot(HaveOccurred()) + //Take delta snapshot. + ctx, cancel = context.WithTimeout(testCtx, time.Duration(2*time.Second)) + snapstoreConfig = brtypes.SnapstoreConfig{Container: snapstoreDir, Provider: "Local", Prefix: "v2"} + err = utils.RunSnapshotter(logger, snapstoreConfig, deltaSnapshotPeriod, endpoints, ctx.Done(), false, compressionConfig) + Expect(err).ShouldNot(HaveOccurred()) + cancel() + + //Move snaps from v2 dir to a v1 dir + //Create v1/Backup-xxxxxx dir + var curTime = time.Now().Unix() + err = os.MkdirAll(path.Join(path.Join(snapstoreDir, "v1"), fmt.Sprintf("Backup-%d", curTime)), 0755) + Expect(err).ShouldNot(HaveOccurred()) + //Move contents from v2 to v1/Backup-xxxxxx + files, err := ioutil.ReadDir(path.Join(snapstoreDir, "v2")) + Expect(err).ShouldNot(HaveOccurred()) + oldPath := path.Join(snapstoreDir, "v2") + newPath := path.Join(path.Join(snapstoreDir, "v1"), fmt.Sprintf("Backup-%d", curTime)) + + if mode == allSnapsInV1 { + //For the case where all snapshots should be in v1 dir, so we can delete v2 dir + for _, f := range files { + err = os.Rename(path.Join(oldPath, f.Name()), path.Join(newPath, f.Name())) + Expect(err).ShouldNot(HaveOccurred()) + } + + //Delete v2 dir + err = os.RemoveAll(path.Join(snapstoreDir, "v2")) + Expect(err).ShouldNot(HaveOccurred()) + } else if mode == fullSnapInV1 { + for _, f := range files[:len(files)-1] { + //For the case where full snap are in v1 dir and some incr snaps are in v2 dir + err = os.Rename(path.Join(oldPath, f.Name()), path.Join(newPath, f.Name())) + Expect(err).ShouldNot(HaveOccurred()) + } + } else { + //For the case where full snaps are in v2 dir and some incr snaps are in v1 + f := files[len(files)-1] + err = os.Rename(path.Join(oldPath, f.Name()), path.Join(newPath, f.Name())) + Expect(err).ShouldNot(HaveOccurred()) + + } + return nil +} + +//takeInvalidV1Snaps saves an invalid snap in the v1 prefix dir of the snapstore so that restorer can't restore from it +//TODO: Consider removing when backward compatibility no longer needed +func takeInvalidV1Snaps(logger *logrus.Entry, container string, deltaSnapshotPeriod time.Duration, endpoints []string, stopCh <-chan struct{}, compressionConfig *compressor.CompressionConfig) error { + //V1 snapstore object + store, err := snapstore.GetSnapstore(&brtypes.SnapstoreConfig{Container: container, Provider: "Local", Prefix: "v1"}) + if err != nil { + return err + } + + //Take a full snapshot + var curTime = time.Now().Unix() + var kind = brtypes.SnapshotKindFull + snap := brtypes.Snapshot{ + Kind: kind, + CreatedOn: time.Now(), + StartRevision: 0, + LastRevision: 1, + SnapDir: fmt.Sprintf("Backup-%d", curTime), + } + snap.GenerateSnapshotName() + store.Save(snap, ioutil.NopCloser(strings.NewReader(fmt.Sprintf("dummy-snapshot-content for snap created on %s", snap.CreatedOn)))) + Expect(err).ShouldNot(HaveOccurred()) + + return nil +} diff --git a/pkg/snapshot/snapshotter/snapshotter_test.go b/pkg/snapshot/snapshotter/snapshotter_test.go index 3b64eb33f..17c68a5ec 100644 --- a/pkg/snapshot/snapshotter/snapshotter_test.go +++ b/pkg/snapshot/snapshotter/snapshotter_test.go @@ -20,8 +20,9 @@ import ( "io/ioutil" "path" "strconv" - "strings" "sync" + + "strings" "time" "github.com/gardener/etcd-backup-restore/pkg/compressor" @@ -30,11 +31,18 @@ import ( "github.com/gardener/etcd-backup-restore/pkg/snapstore" brtypes "github.com/gardener/etcd-backup-restore/pkg/types" "github.com/gardener/etcd-backup-restore/pkg/wrappers" + "github.com/gardener/etcd-backup-restore/test/utils" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) +const ( + mixed = "mixed" + snapsInV1 = "V1" + snapsInV2 = "V2" +) + var _ = Describe("Snapshotter", func() { var ( store brtypes.SnapStore @@ -452,6 +460,140 @@ var _ = Describe("Snapshotter", func() { } }) + //Test to check backward compatibility of garbage collector + //Checks garbage collector behaviour (in exponential config) when both v1 and v2 directories are present + //TODO: Consider removing when backward compatibility no longer needed + It("should garbage collect exponentially from both v1 and v2 dir structures (backward compatible)", func() { + logger.Infoln("creating expected output") + + // Prepare expected resultant snapshot list + var ( + now = time.Now().UTC() + store = prepareStoreForBackwardCompatibleGC(now, "gc_exponential_backward_compatible.bkp") + snapTime = time.Date(now.Year(), now.Month(), now.Day()-35, 0, -30, 0, 0, now.Location()) + expectedSnapList = brtypes.SnapList{} + ) + + expectedSnapList = prepareExpectedSnapshotsList(snapTime, now, mixed) + + //start test + snapshotterConfig := &brtypes.SnapshotterConfig{ + FullSnapshotSchedule: schedule, + DeltaSnapshotPeriod: wrappers.Duration{Duration: 10 * time.Second}, + DeltaSnapshotMemoryLimit: brtypes.DefaultDeltaSnapMemoryLimit, + GarbageCollectionPeriod: wrappers.Duration{Duration: garbageCollectionPeriod}, + GarbageCollectionPolicy: brtypes.GarbageCollectionPolicyExponential, + MaxBackups: maxBackups, + } + + ssr, err := NewSnapshotter(logger, snapshotterConfig, store, etcdConnectionConfig, compressionConfig) + Expect(err).ShouldNot(HaveOccurred()) + + gcCtx, cancel := context.WithTimeout(testCtx, testTimeout) + defer cancel() + ssr.RunGarbageCollector(gcCtx.Done()) + + list, err := store.List() + Expect(err).ShouldNot(HaveOccurred()) + Expect(len(list)).Should(Equal(len(expectedSnapList))) + + for index, snap := range list { + Expect(snap.CreatedOn).Should(Equal(expectedSnapList[index].CreatedOn)) + Expect(snap.Kind).Should(Equal(expectedSnapList[index].Kind)) + Expect(snap.SnapDir).Should(Equal(expectedSnapList[index].SnapDir)) + } + }) + + //Test to check backward compatibility of garbage collector + //Tests garbage collector behaviour (in exponential config) when only v1 directory is present + //TODO: Consider removing when backward compatibility no longer needed + It("should garbage collect exponentially with only v1 dir structure present (backward compatible test)", func() { + logger.Infoln("creating expected output") + + // Prepare expected resultant snapshot list + var ( + now = time.Now().UTC() + //store = prepareBackwardCompatibleStoreInV1ForGC(now, "gc_exponential_backward_compatiblev1.bkp") + store = prepareStoreForGarbageCollectionInPrefix(now, "gc_exponential_backward_compatiblev1.bkp", "v1") + snapTime = time.Date(now.Year(), now.Month(), now.Day()-35, 0, -30, 0, 0, now.Location()) + expectedSnapList = brtypes.SnapList{} + ) + + expectedSnapList = prepareExpectedSnapshotsList(snapTime, now, snapsInV1) + + //start test + snapshotterConfig := &brtypes.SnapshotterConfig{ + FullSnapshotSchedule: schedule, + DeltaSnapshotPeriod: wrappers.Duration{Duration: 10 * time.Second}, + DeltaSnapshotMemoryLimit: brtypes.DefaultDeltaSnapMemoryLimit, + GarbageCollectionPeriod: wrappers.Duration{Duration: garbageCollectionPeriod}, + GarbageCollectionPolicy: brtypes.GarbageCollectionPolicyExponential, + MaxBackups: maxBackups, + } + + ssr, err := NewSnapshotter(logger, snapshotterConfig, store, etcdConnectionConfig, compressionConfig) + Expect(err).ShouldNot(HaveOccurred()) + + gcCtx, cancel := context.WithTimeout(testCtx, testTimeout) + defer cancel() + ssr.RunGarbageCollector(gcCtx.Done()) + + list, err := store.List() + Expect(err).ShouldNot(HaveOccurred()) + Expect(len(list)).Should(Equal(len(expectedSnapList))) + + for index, snap := range list { + Expect(snap.CreatedOn).Should(Equal(expectedSnapList[index].CreatedOn)) + Expect(snap.Kind).Should(Equal(expectedSnapList[index].Kind)) + Expect(snap.SnapDir).Should(Equal(expectedSnapList[index].SnapDir)) + } + }) + + //Test to check backward compatibility of garbage collector + //Tests garbage collector behaviour (in exponential config) when only v2 directory is present + //TODO: Consider removing when backward compatibility no longer needed + It("should garbage collect exponentially with only v2 dir structure (backward compatible test)", func() { + logger.Infoln("creating expected output") + + // Prepare expected resultant snapshot list + var ( + now = time.Now().UTC() + //store = prepareBackwardCompatibleStoreInV2ForGC(now, "gc_exponential_backward_compatiblev2.bkp") + store = prepareStoreForGarbageCollectionInPrefix(now, "gc_exponential_backward_compatiblev2.bkp", "v2") + snapTime = time.Date(now.Year(), now.Month(), now.Day()-35, 0, -30, 0, 0, now.Location()) + expectedSnapList = brtypes.SnapList{} + ) + + expectedSnapList = prepareExpectedSnapshotsList(snapTime, now, snapsInV2) + + //start test + snapshotterConfig := &brtypes.SnapshotterConfig{ + FullSnapshotSchedule: schedule, + DeltaSnapshotPeriod: wrappers.Duration{Duration: 10 * time.Second}, + DeltaSnapshotMemoryLimit: brtypes.DefaultDeltaSnapMemoryLimit, + GarbageCollectionPeriod: wrappers.Duration{Duration: garbageCollectionPeriod}, + GarbageCollectionPolicy: brtypes.GarbageCollectionPolicyExponential, + MaxBackups: maxBackups, + } + + ssr, err := NewSnapshotter(logger, snapshotterConfig, store, etcdConnectionConfig, compressionConfig) + Expect(err).ShouldNot(HaveOccurred()) + + gcCtx, cancel := context.WithTimeout(testCtx, testTimeout) + defer cancel() + ssr.RunGarbageCollector(gcCtx.Done()) + + list, err := store.List() + Expect(err).ShouldNot(HaveOccurred()) + Expect(len(list)).Should(Equal(len(expectedSnapList))) + + for index, snap := range list { + Expect(snap.CreatedOn).Should(Equal(expectedSnapList[index].CreatedOn)) + Expect(snap.Kind).Should(Equal(expectedSnapList[index].Kind)) + Expect(snap.SnapDir).Should(Equal(expectedSnapList[index].SnapDir)) + } + }) + It("should garbage collect limitBased", func() { now := time.Now().UTC() store := prepareStoreForGarbageCollection(now, "garbagecollector_limit_based.bkp") @@ -489,6 +631,92 @@ var _ = Describe("Snapshotter", func() { } } }) + + // Test to check backward compatibility of garbage collector + // Tests garbage collector behaviour (in limit based config) when both v1 and v2 directories are present + // TODO: Consider removing when backward compatibility no longer needed + It("should garbage collect limitBased from both v1 and v2 dir structures (backward compatibility test)", func() { + now := time.Now().UTC() + store := prepareStoreForBackwardCompatibleGC(now, "gc_limit_based_backward_compatible.bkp") + snapshotterConfig := &brtypes.SnapshotterConfig{ + FullSnapshotSchedule: schedule, + DeltaSnapshotPeriod: wrappers.Duration{Duration: 10 * time.Second}, + DeltaSnapshotMemoryLimit: brtypes.DefaultDeltaSnapMemoryLimit, + GarbageCollectionPeriod: wrappers.Duration{Duration: garbageCollectionPeriod}, + GarbageCollectionPolicy: brtypes.GarbageCollectionPolicyLimitBased, + MaxBackups: maxBackups, + } + + ssr, err := NewSnapshotter(logger, snapshotterConfig, store, etcdConnectionConfig, compressionConfig) + Expect(err).ShouldNot(HaveOccurred()) + + gcCtx, cancel := context.WithTimeout(testCtx, testTimeout) + defer cancel() + ssr.RunGarbageCollector(gcCtx.Done()) + + list, err := store.List() + Expect(err).ShouldNot(HaveOccurred()) + + validateLimitBasedSnapshots(list, maxBackups, snapsInV2) + }) + + //Test to check backward compatibility of garbage collector + //Tests garbage collector behaviour (in limit based config) when only v1 directory is present + //TODO: Consider removing when backward compatibility no longer needed + It("should garbage collect limitBased with only v1 dir structure present (backward compatible test)", func() { + now := time.Now().UTC() + //store := prepareBackwardCompatibleStoreInV1ForGC(now, "gc_limit_based_backward_compatiblev1.bkp") + store := prepareStoreForGarbageCollectionInPrefix(now, "gc_limit_based_backward_compatiblev1.bkp", "v1") + snapshotterConfig := &brtypes.SnapshotterConfig{ + FullSnapshotSchedule: schedule, + DeltaSnapshotPeriod: wrappers.Duration{Duration: 10 * time.Second}, + DeltaSnapshotMemoryLimit: brtypes.DefaultDeltaSnapMemoryLimit, + GarbageCollectionPeriod: wrappers.Duration{Duration: garbageCollectionPeriod}, + GarbageCollectionPolicy: brtypes.GarbageCollectionPolicyLimitBased, + MaxBackups: maxBackups, + } + + ssr, err := NewSnapshotter(logger, snapshotterConfig, store, etcdConnectionConfig, compressionConfig) + Expect(err).ShouldNot(HaveOccurred()) + + gcCtx, cancel := context.WithTimeout(testCtx, testTimeout) + defer cancel() + ssr.RunGarbageCollector(gcCtx.Done()) + + list, err := store.List() + Expect(err).ShouldNot(HaveOccurred()) + + validateLimitBasedSnapshots(list, maxBackups, snapsInV1) + }) + + //Test to check backward compatibility of garbage collector + //Tests garbage collector behaviour (in limit based config) when only v2 directory is present + //TODO: Consider removing when backward compatibility no longer needed + It("should garbage collect limitBased with only v2 dir structure present (backward compatible test)", func() { + now := time.Now().UTC() + //store := prepareBackwardCompatibleStoreInV2ForGC(now, "gc_limit_based_backward_compatiblev2.bkp") + store := prepareStoreForGarbageCollectionInPrefix(now, "gc_limit_based_backward_compatiblev2.bkp", "v2") + snapshotterConfig := &brtypes.SnapshotterConfig{ + FullSnapshotSchedule: schedule, + DeltaSnapshotPeriod: wrappers.Duration{Duration: 10 * time.Second}, + DeltaSnapshotMemoryLimit: brtypes.DefaultDeltaSnapMemoryLimit, + GarbageCollectionPeriod: wrappers.Duration{Duration: garbageCollectionPeriod}, + GarbageCollectionPolicy: brtypes.GarbageCollectionPolicyLimitBased, + MaxBackups: maxBackups, + } + + ssr, err := NewSnapshotter(logger, snapshotterConfig, store, etcdConnectionConfig, compressionConfig) + Expect(err).ShouldNot(HaveOccurred()) + + gcCtx, cancel := context.WithTimeout(testCtx, testTimeout) + defer cancel() + ssr.RunGarbageCollector(gcCtx.Done()) + + list, err := store.List() + Expect(err).ShouldNot(HaveOccurred()) + + validateLimitBasedSnapshots(list, maxBackups, snapsInV2) + }) }) }) }) @@ -522,3 +750,262 @@ func prepareStoreForGarbageCollection(forTime time.Time, storeContainer string) } return store } + +// prepareStoreForBackwardCompatibleGC populates the store with dummy snapshots in the old as well as updated drectory structures for backward compatible garbage collection tests +//Tied up with backward compatibility tests +//TODO: Consider removing when backward compatibility no longer needed +func prepareStoreForBackwardCompatibleGC(forTime time.Time, storeContainer string) brtypes.SnapStore { + var ( + snapTimev2 = time.Date(forTime.Year(), forTime.Month(), forTime.Day()-18, 0, 0, 0, 0, forTime.Location()) + snapTimev1 = time.Date(forTime.Year(), forTime.Month(), forTime.Day()-36, 0, 0, 0, 0, forTime.Location()) + count = 0 + noOfDeltaSnapshots = 3 + dir = "" + ) + fmt.Println("setting up garbage collection test") + // Prepare snapshot directory + storev1, err := snapstore.GetSnapstore(&brtypes.SnapstoreConfig{Container: path.Join(outputDir, storeContainer), Prefix: "v1"}) + Expect(err).ShouldNot(HaveOccurred()) + storev2, err := snapstore.GetSnapstore(&brtypes.SnapstoreConfig{Container: path.Join(outputDir, storeContainer), Prefix: "v2"}) + Expect(err).ShouldNot(HaveOccurred()) + for forTime.Sub(snapTimev1) > forTime.Sub(snapTimev2) { + var kind = brtypes.SnapshotKindDelta + if count == 0 { + kind = brtypes.SnapshotKindFull + dir = fmt.Sprintf("Backup-%d", snapTimev1.Unix()) + } + count = (count + 1) % noOfDeltaSnapshots + snapv1 := brtypes.Snapshot{ + Kind: kind, + CreatedOn: snapTimev1, + StartRevision: 0, + LastRevision: 1001, + SnapDir: dir, + } + snapv1.GenerateSnapshotName() + snapTimev1 = snapTimev1.Add(time.Duration(time.Minute * 10)) + storev1.Save(snapv1, ioutil.NopCloser(strings.NewReader(fmt.Sprintf("dummy-snapshot-content for snap created on %s", snapv1.CreatedOn)))) + } + + count = 0 + for forTime.Sub(snapTimev2) >= 0 { + var kind = brtypes.SnapshotKindDelta + if count == 0 { + kind = brtypes.SnapshotKindFull + } + count = (count + 1) % noOfDeltaSnapshots + snapv2 := brtypes.Snapshot{ + Kind: kind, + CreatedOn: snapTimev2, + StartRevision: 0, + LastRevision: 1001, + } + snapv2.GenerateSnapshotName() + snapTimev2 = snapTimev2.Add(time.Duration(time.Minute * 10)) + storev2.Save(snapv2, ioutil.NopCloser(strings.NewReader(fmt.Sprintf("dummy-snapshot-content for snap created on %s", snapv2.CreatedOn)))) + } + return storev2 +} + +func prepareStoreForGarbageCollectionInPrefix(forTime time.Time, storeContainer string, storePrefix string) brtypes.SnapStore { + var ( + snapTimev1 = time.Date(forTime.Year(), forTime.Month(), forTime.Day()-36, 0, 0, 0, 0, forTime.Location()) + count = 0 + noOfDeltaSnapshots = 3 + dir = "" + ) + fmt.Println("setting up garbage collection test") + // Prepare snapshot directory + var storev1 brtypes.SnapStore + if storePrefix == "v1" { + storev1, err = snapstore.GetSnapstore(&brtypes.SnapstoreConfig{Container: path.Join(outputDir, storeContainer), Prefix: "v1"}) + Expect(err).ShouldNot(HaveOccurred()) + } + storev2, err := snapstore.GetSnapstore(&brtypes.SnapstoreConfig{Container: path.Join(outputDir, storeContainer), Prefix: "v2"}) + Expect(err).ShouldNot(HaveOccurred()) + for forTime.Sub(snapTimev1) > 0 { + var kind = brtypes.SnapshotKindDelta + if count == 0 { + kind = brtypes.SnapshotKindFull + dir = fmt.Sprintf("Backup-%d", snapTimev1.Unix()) + } + count = (count + 1) % noOfDeltaSnapshots + var snapv1 brtypes.Snapshot + if storePrefix == "v1" { + snapv1 = brtypes.Snapshot{ + Kind: kind, + CreatedOn: snapTimev1, + StartRevision: 0, + LastRevision: 1001, + SnapDir: dir, + } + snapv1.GenerateSnapshotName() + snapTimev1 = snapTimev1.Add(time.Duration(time.Minute * 10)) + storev1.Save(snapv1, ioutil.NopCloser(strings.NewReader(fmt.Sprintf("dummy-snapshot-content for snap created on %s", snapv1.CreatedOn)))) + } else { + snapv1 = brtypes.Snapshot{ + Kind: kind, + CreatedOn: snapTimev1, + StartRevision: 0, + LastRevision: 1001, + } + snapv1.GenerateSnapshotName() + snapTimev1 = snapTimev1.Add(time.Duration(time.Minute * 10)) + storev2.Save(snapv1, ioutil.NopCloser(strings.NewReader(fmt.Sprintf("dummy-snapshot-content for snap created on %s", snapv1.CreatedOn)))) + } + } + + return storev2 +} + +func prepareExpectedSnapshotsList(snapTime time.Time, now time.Time, mode string) brtypes.SnapList { + var expectedSnapList brtypes.SnapList + var dir string + + // weekly snapshot + for i := 1; i <= 4; i++ { + snapTime = snapTime.Add(time.Duration(time.Hour * 24 * 7)) + if (mode == snapsInV1) || (mode == mixed && (i == 1 || i == 2)) { + dir = fmt.Sprintf("Backup-%d", snapTime.Unix()) + } else { + dir = "" + } + snap := &brtypes.Snapshot{ + Kind: brtypes.SnapshotKindFull, + CreatedOn: snapTime, + StartRevision: 0, + LastRevision: 1001, + SnapDir: dir, + } + snap.GenerateSnapshotName() + expectedSnapList = append(expectedSnapList, snap) + } + fmt.Println("Weekly snapshot list prepared") + fmt.Printf("len: %d", len(expectedSnapList)) + + // daily snapshot + for i := 1; i <= 7; i++ { + snapTime = snapTime.Add(time.Duration(time.Hour * 24)) + if mode == snapsInV1 { + dir = fmt.Sprintf("Backup-%d", snapTime.Unix()) + } else { + dir = "" + } + snap := &brtypes.Snapshot{ + Kind: brtypes.SnapshotKindFull, + CreatedOn: snapTime, + StartRevision: 0, + LastRevision: 1001, + SnapDir: dir, + } + snap.GenerateSnapshotName() + expectedSnapList = append(expectedSnapList, snap) + } + fmt.Println("Daily snapshot list prepared") + + // hourly snapshot + snapTime = snapTime.Add(time.Duration(time.Hour)) + for now.Truncate(time.Hour).Sub(snapTime) > 0 { + if mode == snapsInV1 { + dir = fmt.Sprintf("Backup-%d", snapTime.Unix()) + } else { + dir = "" + } + snap := &brtypes.Snapshot{ + Kind: brtypes.SnapshotKindFull, + CreatedOn: snapTime, + StartRevision: 0, + LastRevision: 1001, + SnapDir: dir, + } + snap.GenerateSnapshotName() + expectedSnapList = append(expectedSnapList, snap) + snapTime = snapTime.Add(time.Duration(time.Hour)) + } + fmt.Println("Hourly snapshot list prepared") + + // current hour + snapTime = now.Truncate(time.Hour) + if mode == snapsInV1 { + dir = fmt.Sprintf("Backup-%d", snapTime.Unix()) + } else { + dir = "" + } + snap := &brtypes.Snapshot{ + Kind: brtypes.SnapshotKindFull, + CreatedOn: snapTime, + StartRevision: 0, + LastRevision: 1001, + SnapDir: dir, + } + snap.GenerateSnapshotName() + expectedSnapList = append(expectedSnapList, snap) + snapTime = snapTime.Add(time.Duration(time.Minute * 30)) + for now.Sub(snapTime) >= 0 { + if mode == snapsInV1 { + dir = fmt.Sprintf("Backup-%d", snapTime.Unix()) + } else { + dir = "" + } + snap := &brtypes.Snapshot{ + Kind: brtypes.SnapshotKindFull, + CreatedOn: snapTime, + StartRevision: 0, + LastRevision: 1001, + SnapDir: dir, + } + snap.GenerateSnapshotName() + expectedSnapList = append(expectedSnapList, snap) + snapTime = snapTime.Add(time.Duration(time.Minute * 30)) + } + fmt.Println("Current hour full snapshot list prepared") + + // delta snapshots + snapTime = snapTime.Add(time.Duration(-time.Minute * 30)) + snapTime = snapTime.Add(time.Duration(time.Minute * 10)) + for now.Sub(snapTime) >= 0 { + snap := &brtypes.Snapshot{ + Kind: brtypes.SnapshotKindDelta, + CreatedOn: snapTime, + StartRevision: 0, + LastRevision: 1001, + SnapDir: dir, + } + snap.GenerateSnapshotName() + expectedSnapList = append(expectedSnapList, snap) + snapTime = snapTime.Add(time.Duration(time.Minute * 10)) + } + fmt.Println("Incremental snapshot list prepared") + + return expectedSnapList +} + +//validateLimitBasedSnapshots verifies whether the snapshot list after being garbage collected using the limit-based configuration is a valid snapshot list +func validateLimitBasedSnapshots(list brtypes.SnapList, maxBackups uint, mode string) { + incr := false + fullSnapCount := 0 + for _, snap := range list { + if incr == false { + if snap.Kind == brtypes.SnapshotKindDelta { + //Indicates that no full snapshot can occur after a incr snapshot in an already garbage collected list + incr = true + } else { + //Number of full snapshots in garbage collected list cannot be more than the maxBackups configuration + fullSnapCount++ + Expect(fullSnapCount).Should(BeNumerically("<=", maxBackups)) + } + if mode == snapsInV2 { + Expect(snap.SnapDir).Should(Equal("")) + } else if mode == snapsInV1 { + Expect(snap.SnapDir).Should(ContainSubstring("Backup")) + } + } else { + Expect(snap.Kind).Should(Equal(brtypes.SnapshotKindDelta)) + if mode == snapsInV2 { + Expect(snap.SnapDir).Should(Equal("")) + } else if mode == snapsInV1 { + Expect(snap.SnapDir).Should(ContainSubstring("Backup")) + } + } + } +} diff --git a/test/utils/utils.go b/test/utils/utils.go index 0baabfc19..06fd2a8e7 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -184,8 +184,8 @@ func ContextWithWaitGroupFollwedByGracePeriod(parent context.Context, wg *sync.W } // RunSnapshotter creates a snapshotter object and runs it for a duration specified by 'snapshotterDurationSeconds' -func RunSnapshotter(logger *logrus.Entry, container string, deltaSnapshotPeriod time.Duration, endpoints []string, stopCh <-chan struct{}, startWithFullSnapshot bool, compressionConfig *compressor.CompressionConfig) error { - store, err := snapstore.GetSnapstore(&brtypes.SnapstoreConfig{Container: container, Provider: "Local"}) +func RunSnapshotter(logger *logrus.Entry, snapstoreConfig brtypes.SnapstoreConfig, deltaSnapshotPeriod time.Duration, endpoints []string, stopCh <-chan struct{}, startWithFullSnapshot bool, compressionConfig *compressor.CompressionConfig) error { + store, err := snapstore.GetSnapstore(&snapstoreConfig) if err != nil { return err }