Skip to content

Commit

Permalink
Support config file for configration
Browse files Browse the repository at this point in the history
Signed-off-by: Swapnil Mhamane <[email protected]>
  • Loading branch information
Swapnil Mhamane committed Dec 10, 2019
1 parent 82fe3db commit 16cecba
Show file tree
Hide file tree
Showing 41 changed files with 1,624 additions and 1,353 deletions.
2 changes: 2 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ BUILD_DIR := build
BIN_DIR := bin
COVERPROFILE := test/output/coverprofile.out

.DEFAULT_GOAL := build-local

.PHONY: revendor
revendor:
@env GO111MODULE=on go mod vendor -v
Expand Down
56 changes: 19 additions & 37 deletions cmd/initializer.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,41 +17,45 @@ package cmd
import (
"context"
"fmt"
"path"

"github.com/coreos/etcd/pkg/types"
"github.com/gardener/etcd-backup-restore/pkg/initializer"
"github.com/gardener/etcd-backup-restore/pkg/initializer/validator"
"github.com/gardener/etcd-backup-restore/pkg/snapshot/restorer"
"github.com/gardener/etcd-backup-restore/pkg/snapstore"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)

// NewInitializeCommand returns the command to initialize etcd by validating the data
// directory and restoring from cloud store if needed.
func NewInitializeCommand(ctx context.Context) *cobra.Command {

opts := newInitializerOptions()
// restoreCmd represents the restore command
initializeCmd := &cobra.Command{
Use: "initialize",
Short: "initialize an etcd instance.",
Long: fmt.Sprintf(`Initializes an etcd instance. Data directory is checked for corruption and restored in case of corruption.`),
Run: func(cmd *cobra.Command, args []string) {
var mode validator.Mode
logger := logrus.New()
if err := opts.validate(); err != nil {
logger.Fatalf("failed to validate the options: %v", err)
return
}

clusterUrlsMap, err := types.NewURLsMap(restoreCluster)
opts.complete()

clusterUrlsMap, err := types.NewURLsMap(opts.restorerOptions.restorationConfig.InitialCluster)
if err != nil {
logger.Fatalf("failed creating url map for restore cluster: %v", err)
}

peerUrls, err := types.NewURLs(restorePeerURLs)
peerUrls, err := types.NewURLs(opts.restorerOptions.restorationConfig.InitialAdvertisePeerURLs)
if err != nil {
logger.Fatalf("failed parsing peers urls for restore cluster: %v", err)
}

switch validator.Mode(validationMode) {
var mode validator.Mode
switch validator.Mode(opts.validatorOptions.ValidationMode) {
case validator.Full:
mode = validator.Full
case validator.Sanity:
Expand All @@ -60,41 +64,19 @@ func NewInitializeCommand(ctx context.Context) *cobra.Command {
logger.Fatal("validation-mode can only be one of these values [full/sanity]")
}

options := &restorer.RestoreOptions{
RestoreDataDir: path.Clean(restoreDataDir),
Name: restoreName,
ClusterURLs: clusterUrlsMap,
PeerURLs: peerUrls,
ClusterToken: restoreClusterToken,
SkipHashCheck: skipHashCheck,
MaxFetchers: restoreMaxFetchers,
EmbeddedEtcdQuotaBytes: embeddedEtcdQuotaBytes,
restoreOptions := &restorer.RestoreOptions{
Config: opts.restorerOptions.restorationConfig,
ClusterURLs: clusterUrlsMap,
PeerURLs: peerUrls,
}

var snapstoreConfig *snapstore.Config
if storageProvider != "" {
snapstoreConfig = &snapstore.Config{
Provider: storageProvider,
Container: storageContainer,
Prefix: path.Join(storagePrefix, backupFormatVersion),
MaxParallelChunkUploads: maxParallelChunkUploads,
TempDir: snapstoreTempDir,
}
}
etcdInitializer := initializer.NewInitializer(options, snapstoreConfig, logger)
err = etcdInitializer.Initialize(mode, failBelowRevision)
if err != nil {
etcdInitializer := initializer.NewInitializer(restoreOptions, opts.restorerOptions.snapstoreConfig, logger)
if err := etcdInitializer.Initialize(mode, opts.validatorOptions.FailBelowRevision); err != nil {
logger.Fatalf("initializer failed. %v", err)
}
},
}
initializeEtcdFlags(initializeCmd)
initializeSnapstoreFlags(initializeCmd)
initializeValidatorFlags(initializeCmd)
initializeCmd.Flags().Int64Var(&failBelowRevision, "experimental-fail-below-revision", 0, "minimum required etcd revision, below which validation fails")
return initializeCmd
}

func initializeValidatorFlags(cmd *cobra.Command) {
cmd.Flags().StringVar(&validationMode, "validation-mode", string(validator.Full), "mode to do data initialization[full/sanity]")
opts.addFlags(initializeCmd.Flags())
return initializeCmd
}
10 changes: 0 additions & 10 deletions cmd/miscellaneous.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,18 +18,8 @@ import (
"runtime"

ver "github.com/gardener/etcd-backup-restore/pkg/version"
"github.com/spf13/cobra"
)

// initializeSnapstoreFlags adds the snapstore related flags to <cmd>
func initializeSnapstoreFlags(cmd *cobra.Command) {
cmd.Flags().StringVar(&storageProvider, "storage-provider", "", "snapshot storage provider")
cmd.Flags().StringVar(&storageContainer, "store-container", "", "container which will be used as snapstore")
cmd.Flags().StringVar(&storagePrefix, "store-prefix", "", "prefix or directory inside container under which snapstore is created")
cmd.Flags().IntVar(&maxParallelChunkUploads, "max-parallel-chunk-uploads", 5, "maximum number of parallel chunk uploads allowed ")
cmd.Flags().StringVar(&snapstoreTempDir, "snapstore-temp-directory", "/tmp", "temporary directory for processing")
}

func printVersionInfo() {
logger.Infof("etcd-backup-restore Version: %s", ver.Version)
logger.Infof("Git SHA: %s", ver.GitSHA)
Expand Down
228 changes: 228 additions & 0 deletions cmd/options.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,228 @@
// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
"context"
"io/ioutil"

"github.com/gardener/etcd-backup-restore/pkg/snapshot/snapshotter"

"github.com/gardener/etcd-backup-restore/pkg/etcdutil"

"github.com/gardener/etcd-backup-restore/pkg/snapstore"

"github.com/gardener/etcd-backup-restore/pkg/initializer/validator"
"github.com/gardener/etcd-backup-restore/pkg/snapshot/restorer"

"github.com/gardener/etcd-backup-restore/pkg/server"
"github.com/ghodss/yaml"
"github.com/sirupsen/logrus"
flag "github.com/spf13/pflag"
)

type serverOptions struct {
ConfigFile string
Version bool
LogLevel uint32
Logger *logrus.Logger
Config *server.BackupRestoreComponentConfig
}

// newServerOptions returns a new Options object.
func newServerOptions() *serverOptions {
logger := logrus.New()
logger.SetLevel(logrus.InfoLevel)

return &serverOptions{
LogLevel: 4,
Version: false,
Config: server.NewBackupRestoreComponentConfig(),
Logger: logger,
}
}

func (o *serverOptions) validate() error {
return o.Config.Validate()
}

func (o *serverOptions) addFlags(fs *flag.FlagSet) {
fs.StringVar(&o.ConfigFile, "config-file", o.ConfigFile, "path to the configuration file")
fs.Uint32Var(&o.LogLevel, "log-level", o.LogLevel, "verbosity level of logs")
o.Config.AddFlags(fs)
}

func (o *serverOptions) complete() {
o.Config.Complete()
o.Logger.SetLevel(logrus.Level(o.LogLevel))
}

func (o *serverOptions) loadConfigFromFile() error {
if len(o.ConfigFile) != 0 {
data, err := ioutil.ReadFile(o.ConfigFile)
if err != nil {
return err
}
config := server.NewBackupRestoreComponentConfig()
if err := yaml.Unmarshal(data, config); err != nil {
return err
}
o.Config = config
}
// TODO: Overwrite config with flags
return nil
}

func (o *serverOptions) run(ctx context.Context) error {
brServer, err := server.NewBackupRestoreServer(o.Logger, o.Config)
if err != nil {
return err
}
return brServer.Run(ctx)
}

type initializerOptions struct {
validatorOptions *validatorOptions
restorerOptions *restorerOptions
}

// newInitializerOptions returns the validation config.
func newInitializerOptions() *initializerOptions {
return &initializerOptions{
validatorOptions: newValidatorOptions(),
restorerOptions: newRestorerOptions(),
}
}

// AddFlags adds the flags to flagset.
func (c *initializerOptions) addFlags(fs *flag.FlagSet) {
c.validatorOptions.addFlags(fs)
c.restorerOptions.addFlags(fs)
}

// Validate validates the config.
func (c *initializerOptions) validate() error {
if err := c.validatorOptions.validate(); err != nil {
return err
}

return c.restorerOptions.validate()
}

// Complete completes the config.
func (c *initializerOptions) complete() {
c.restorerOptions.complete()
}

type restorerOptions struct {
restorationConfig *restorer.RestorationConfig
snapstoreConfig *snapstore.Config
}

// newRestorerOptions returns the validation config.
func newRestorerOptions() *restorerOptions {
return &restorerOptions{
restorationConfig: restorer.NewRestorationConfig(),
snapstoreConfig: snapstore.NewSnapstoreConfig(),
}
}

// AddFlags adds the flags to flagset.
func (c *restorerOptions) addFlags(fs *flag.FlagSet) {
c.restorationConfig.AddFlags(fs)
c.snapstoreConfig.AddFlags(fs)
}

// Validate validates the config.
func (c *restorerOptions) validate() error {
if err := c.snapstoreConfig.Validate(); err != nil {
return err
}

return c.restorationConfig.Validate()
}

// complete completes the config.
func (c *restorerOptions) complete() {
c.snapstoreConfig.Complete()
}

type validatorOptions struct {
ValidationMode string `json:"validationMode,omitempty"`
FailBelowRevision int64 `json:"experimentalFailBelowRevision,omitempty"`
}

// newValidatorOptions returns the validation config.
func newValidatorOptions() *validatorOptions {
return &validatorOptions{
ValidationMode: string(validator.Full),
}
}

// AddFlags adds the flags to flagset.
func (c *validatorOptions) addFlags(fs *flag.FlagSet) {
fs.StringVar(&c.ValidationMode, "validation-mode", string(c.ValidationMode), "mode to do data initialization[full/sanity]")
fs.Int64Var(&c.FailBelowRevision, "experimental-fail-below-revision", c.FailBelowRevision, "minimum required etcd revision, below which validation fails")
}

// Validate validates the config.
func (c *validatorOptions) validate() error {
return nil
}

type snapshotterOptions struct {
etcdConnectionConfig *etcdutil.EtcdConnectionConfig
snapstoreConfig *snapstore.Config
snapshotterConfig *snapshotter.Config
defragmentationSchedule string
}

// newSnapshotterOptions returns the snapshotter options.
func newSnapshotterOptions() *snapshotterOptions {
return &snapshotterOptions{
etcdConnectionConfig: etcdutil.NewEtcdConnectionConfig(),
snapstoreConfig: snapstore.NewSnapstoreConfig(),
snapshotterConfig: snapshotter.NewSnapshotterConfig(),
defragmentationSchedule: "0 0 */3 * *",
}
}

// AddFlags adds the flags to flagset.
func (c *snapshotterOptions) addFlags(fs *flag.FlagSet) {
c.etcdConnectionConfig.AddFlags(fs)
c.snapstoreConfig.AddFlags(fs)
c.snapshotterConfig.AddFlags(fs)

// Miscellaneous
fs.StringVar(&c.defragmentationSchedule, "defragmentation-schedule", c.defragmentationSchedule, "schedule to defragment etcd data directory")
}

// Validate validates the config.
func (c *snapshotterOptions) validate() error {
if err := c.snapstoreConfig.Validate(); err != nil {
return err
}

if err := c.snapshotterConfig.Validate(); err != nil {
return err
}

return c.etcdConnectionConfig.Validate()
}

// complete completes the config.
func (c *snapshotterOptions) complete() {
c.snapstoreConfig.Complete()
}
Loading

0 comments on commit 16cecba

Please sign in to comment.