Added compactor subcommand.

.gitignore

@@ -22,6 +22,7 @@ test/output
 test/e2e_test_data
 default.bkp*
 default.etcd*
+compctedsnap.bkp*
 
 # IDE config
 .vscode

cmd/compact.go

@@ -0,0 +1,62 @@
+// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/gardener/etcd-backup-restore/pkg/compactor"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+// NewCompactCommand compacts the ETCD instance
+func NewCompactCommand(ctx context.Context) *cobra.Command {
+	opts := newCompactOptions()
+	// compactCmd represents the restore command
+	compactCmd := &cobra.Command{
+		Use:   "compact",
+		Short: "compacts multiple incremental snapshots in etcd backup into a single full snapshot",
+		Long:  fmt.Sprintf(`Compacts an existing backup stored in snapshot store.`),
+		Run: func(cmd *cobra.Command, args []string) {
+			/* Compact operation
+			- Restore from all the latest snapshots (Base + Delta).
+			- Compact the newly created embedded ETCD instance.
+			- Defragment
+			- Save the snapshot
+			*/
+			logger := logrus.New()
+
+			options, store, err := BuildRestoreOptionsAndStore(opts)
+			if err != nil {
+				return
+			}
+
+			cp := compactor.NewCompactor(store, logrus.NewEntry(logger))
+			res, err := cp.Compact(options, opts.needDefragmentation)
+			if err != nil {
+				logger.Fatalf("Failed to restore snapshot: %v", err)
+				return
+			}
+			// logger.Infof("Compacted snapshot is in: %v", filepath.Join(opts.snapstoreConfig.Container, res.Snapshot.SnapDir, res.Snapshot.SnapName))
+			logger.Infof("Compacted snapshot is in: %v", res.Path)
+
+		},
+	}
+
+	opts.addFlags(compactCmd.Flags())
+	return compactCmd
+}

cmd/initializer.go

@@ -20,7 +20,7 @@ import (
 
 	"github.com/gardener/etcd-backup-restore/pkg/initializer"
 	"github.com/gardener/etcd-backup-restore/pkg/initializer/validator"
-	"github.com/gardener/etcd-backup-restore/pkg/snapshot/restorer"
+	brtypes "github.com/gardener/etcd-backup-restore/pkg/types"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"go.etcd.io/etcd/pkg/types"
@@ -64,7 +64,7 @@ func NewInitializeCommand(ctx context.Context) *cobra.Command {
 				logger.Fatal("validation-mode can only be one of these values [full/sanity]")
 			}
 
-			restoreOptions := &restorer.RestoreOptions{
+			restoreOptions := &brtypes.RestoreOptions{
 				Config:      opts.restorerOptions.restorationConfig,
 				ClusterURLs: clusterUrlsMap,
 				PeerURLs:    peerUrls,

cmd/miscellaneous.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2018 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file.
+// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -15,9 +15,14 @@
 package cmd
 
 import (
+	"fmt"
 	"runtime"
 
+	"github.com/gardener/etcd-backup-restore/pkg/miscellaneous"
+	"github.com/gardener/etcd-backup-restore/pkg/snapstore"
+	brtypes "github.com/gardener/etcd-backup-restore/pkg/types"
 	ver "github.com/gardener/etcd-backup-restore/pkg/version"
+	"go.etcd.io/etcd/pkg/types"
 )
 
 func printVersionInfo() {
@@ -26,3 +31,47 @@ func printVersionInfo() {
 	logger.Infof("Go Version: %s", runtime.Version())
 	logger.Infof("Go OS/Arch: %s/%s", runtime.GOOS, runtime.GOARCH)
 }
+
+// BuildRestoreOptionsAndStore forms the RestoreOptions and Store object
+func BuildRestoreOptionsAndStore(opts restoreOpts) (*brtypes.RestoreOptions, snapstore.SnapStore, error) {
+	if err := opts.validate(); err != nil {
+		logger.Fatalf("failed to validate the options: %v", err)
+		return nil, nil, err
+	}
+
+	opts.complete()
+
+	clusterUrlsMap, err := types.NewURLsMap(opts.getRestorationConfig().InitialCluster)
+	if err != nil {
+		logger.Fatalf("failed creating url map for restore cluster: %v", err)
+	}
+
+	peerUrls, err := types.NewURLs(opts.getRestorationConfig().InitialAdvertisePeerURLs)
+	if err != nil {
+		logger.Fatalf("failed parsing peers urls for restore cluster: %v", err)
+	}
+
+	store, err := snapstore.GetSnapstore(opts.getSnapstoreConfig())
+	if err != nil {
+		logger.Fatalf("failed to create restore snapstore from configured storage provider: %v", err)
+	}
+
+	logger.Info("Finding latest set of snapshot to recover from...")
+	baseSnap, deltaSnapList, err := miscellaneous.GetLatestFullSnapshotAndDeltaSnapList(store)
+	if err != nil {
+		logger.Fatalf("failed to get latest snapshot: %v", err)
+	}
+
+	if baseSnap == nil {
+		logger.Infof("No base snapshot found. Will do nothing.")
+		return nil, nil, fmt.Errorf("No base snapshot found")
+	}
+
+	return &brtypes.RestoreOptions{
+		Config:        opts.getRestorationConfig(),
+		BaseSnapshot:  baseSnap,
+		DeltaSnapList: deltaSnapList,
+		ClusterURLs:   clusterUrlsMap,
+		PeerURLs:      peerUrls,
+	}, store, nil
+}

cmd/options.go

@@ -26,9 +26,9 @@ import (
 	"github.com/gardener/etcd-backup-restore/pkg/snapstore"
 
 	"github.com/gardener/etcd-backup-restore/pkg/initializer/validator"
-	"github.com/gardener/etcd-backup-restore/pkg/snapshot/restorer"
 
 	"github.com/gardener/etcd-backup-restore/pkg/server"
+	brtypes "github.com/gardener/etcd-backup-restore/pkg/types"
 	"github.com/ghodss/yaml"
 	"github.com/sirupsen/logrus"
 	flag "github.com/spf13/pflag"
@@ -127,19 +127,83 @@ func (c *initializerOptions) complete() {
 	c.restorerOptions.complete()
 }
 
+type restoreOpts interface {
+	getRestorationConfig() *brtypes.RestorationConfig
+	getSnapstoreConfig() *snapstore.Config
+	validate() error
+	complete()
+}
+
+type compactOptions struct {
+	*restorerOptions
+	needDefragmentation bool
+}
+
+// newCompactOptions returns the validation config.
+func newCompactOptions() *compactOptions {
+	return &compactOptions{
+		restorerOptions: &restorerOptions{
+			restorationConfig: brtypes.NewRestorationConfig(),
+			snapstoreConfig:   snapstore.NewSnapstoreConfig(),
+		},
+		needDefragmentation: true,
+	}
+}
+
+func (c *compactOptions) getRestorationConfig() *brtypes.RestorationConfig {
+	return c.restorationConfig
+}
+
+func (c *compactOptions) getSnapstoreConfig() *snapstore.Config {
+	return c.snapstoreConfig
+}
+
+// AddFlags adds the flags to flagset.
+func (c *compactOptions) addFlags(fs *flag.FlagSet) {
+	c.restorationConfig.AddFlags(fs)
+	c.snapstoreConfig.AddFlags(fs)
+	fs.BoolVar(&c.needDefragmentation, "defragment", c.needDefragmentation, "defragment after compaction")
+}
+
+// Validate validates the config.
+func (c *compactOptions) validate() error {
+	if err := c.snapstoreConfig.Validate(); err != nil {
+		return err
+	}
+
+	if err := c.snapstoreConfig.Validate(); err != nil {
+		return err
+	}
+
+	return c.restorationConfig.Validate()
+}
+
+// complete completes the config.
+func (c *compactOptions) complete() {
+	c.snapstoreConfig.Complete()
+}
+
 type restorerOptions struct {
-	restorationConfig *restorer.RestorationConfig
+	restorationConfig *brtypes.RestorationConfig
 	snapstoreConfig   *snapstore.Config
 }
 
 // newRestorerOptions returns the validation config.
 func newRestorerOptions() *restorerOptions {
 	return &restorerOptions{
-		restorationConfig: restorer.NewRestorationConfig(),
+		restorationConfig: brtypes.NewRestorationConfig(),
 		snapstoreConfig:   snapstore.NewSnapstoreConfig(),
 	}
 }
 
+func (c *restorerOptions) getRestorationConfig() *brtypes.RestorationConfig {
+	return c.restorationConfig
+}
+
+func (c *restorerOptions) getSnapstoreConfig() *snapstore.Config {
+	return c.snapstoreConfig
+}
+
 // AddFlags adds the flags to flagset.
 func (c *restorerOptions) addFlags(fs *flag.FlagSet) {
 	c.restorationConfig.AddFlags(fs)

cmd/restore.go

@@ -18,12 +18,9 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/gardener/etcd-backup-restore/pkg/miscellaneous"
 	"github.com/gardener/etcd-backup-restore/pkg/snapshot/restorer"
-	"github.com/gardener/etcd-backup-restore/pkg/snapstore"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"go.etcd.io/etcd/pkg/types"
 )
 
 // NewRestoreCommand returns the command to restore
@@ -40,48 +37,13 @@ func NewRestoreCommand(ctx context.Context) *cobra.Command {
 			- Restore etcd data diretory from full snapshot.
 			*/
 			logger := logrus.New()
-			if err := opts.validate(); err != nil {
-				logger.Fatalf("failed to validate the options: %v", err)
-				return
-			}
-
-			opts.complete()
-
-			clusterUrlsMap, err := types.NewURLsMap(opts.restorationConfig.InitialCluster)
-			if err != nil {
-				logger.Fatalf("failed creating url map for restore cluster: %v", err)
-			}
-
-			peerUrls, err := types.NewURLs(opts.restorationConfig.InitialAdvertisePeerURLs)
-			if err != nil {
-				logger.Fatalf("failed parsing peers urls for restore cluster: %v", err)
-			}
-
-			store, err := snapstore.GetSnapstore(opts.snapstoreConfig)
-			if err != nil {
-				logger.Fatalf("failed to create snapstore from configured storage provider: %v", err)
-			}
 
-			logger.Info("Finding latest set of snapshot to recover from...")
-			baseSnap, deltaSnapList, err := miscellaneous.GetLatestFullSnapshotAndDeltaSnapList(store)
+			options, store, err := BuildRestoreOptionsAndStore(opts)
 			if err != nil {
-				logger.Fatalf("failed to get latest snapshot: %v", err)
-			}
-			if baseSnap == nil {
-				logger.Infof("No snapshot found. Will do nothing.")
 				return
 			}
 
 			rs := restorer.NewRestorer(store, logrus.NewEntry(logger))
-
-			options := &restorer.RestoreOptions{
-				Config:        opts.restorationConfig,
-				BaseSnapshot:  *baseSnap,
-				DeltaSnapList: deltaSnapList,
-				ClusterURLs:   clusterUrlsMap,
-				PeerURLs:      peerUrls,
-			}
-
 			if err := rs.Restore(*options); err != nil {
 				logger.Fatalf("Failed to restore snapshot: %v", err)
 				return

cmd/root.go

@@ -38,6 +38,7 @@ from previously taken snapshot.`,
 	RootCmd.Flags().BoolVarP(&version, "version", "v", false, "print version info")
 	RootCmd.AddCommand(NewSnapshotCommand(ctx),
 		NewRestoreCommand(ctx),
+		NewCompactCommand(ctx),
 		NewInitializeCommand(ctx),
 		NewServerCommand(ctx))
 	return RootCmd