diff --git a/pkg/cmds/cli/cli.go b/pkg/cmds/cli/cli.go index bc7112aab..34bb1af35 100644 --- a/pkg/cmds/cli/cli.go +++ b/pkg/cmds/cli/cli.go @@ -4,6 +4,7 @@ import ( "path/filepath" cs "github.com/appscode/stash/client/clientset/versioned" + docker_image "github.com/appscode/stash/pkg/docker" "github.com/spf13/cobra" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -13,8 +14,8 @@ import ( ) const ( - cliScratchDir = "/tmp/stash-cli/scratch" cliSecretDir = "/tmp/stash-cli/secret" + cliConfigDir = "/tmp/stash-cli/config" ) type stashCLIController struct { @@ -23,6 +24,14 @@ type stashCLIController struct { stashClient cs.Interface } +var ( + image = docker_image.Docker{ + Registry: docker_image.ACRegistry, + Image: docker_image.ImageStash, + Tag: "latest", // TODO: update default release tag + } +) + func NewCLICmd() *cobra.Command { var cmd = &cobra.Command{ Use: "cli", diff --git a/pkg/cmds/cli/download.go b/pkg/cmds/cli/download.go index 5da95e6bc..065b0f0d4 100644 --- a/pkg/cmds/cli/download.go +++ b/pkg/cmds/cli/download.go @@ -4,23 +4,29 @@ import ( "fmt" "io/ioutil" "os" + "os/exec" + "os/user" "path/filepath" "github.com/appscode/go/flags" "github.com/appscode/go/log" + "github.com/appscode/stash/pkg/cmds/docker" "github.com/appscode/stash/pkg/restic" "github.com/appscode/stash/pkg/util" "github.com/spf13/cobra" + core "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func NewDownloadCmd() *cobra.Command { var ( - kubeConfig string - repositoryName string - namespace string - restoreOpt = restic.RestoreOptions{ - SourceHost: restic.DefaultHost, + kubeConfig string + repositoryName string + namespace string + localDestination string + restoreOpt = restic.RestoreOptions{ + SourceHost: restic.DefaultHost, + Destination: docker.DestinationDir, } ) @@ -42,55 +48,37 @@ func NewDownloadCmd() *cobra.Command { if err != nil { return err } - // unlock local backend if repository.Spec.Backend.Local != nil { return fmt.Errorf("can't restore from repository with local backend") } - - // get source repository secret + // get repository secret secret, err := c.kubeClient.CoreV1().Secrets(namespace).Get(repository.Spec.Backend.StorageSecretName, metav1.GetOptions{}) if err != nil { return err } - // cleanup whole scratch/secret dir at the end - defer os.RemoveAll(cliScratchDir) - defer os.RemoveAll(cliSecretDir) - - // write repository secrets in a temp dir - if err := os.MkdirAll(cliSecretDir, 0755); err != nil { - return err - } - for key, value := range secret.Data { - if err := ioutil.WriteFile(filepath.Join(cliSecretDir, key), value, 0755); err != nil { - return err - } - } - // configure restic wrapper extraOpt := util.ExtraOptions{ - SecretDir: cliSecretDir, + SecretDir: docker.SecretDir, EnableCache: false, - ScratchDir: cliScratchDir, + ScratchDir: docker.ScratchDir, } setupOpt, err := util.SetupOptionsForRepository(*repository, extraOpt) if err != nil { - return fmt.Errorf("setup option for repository fail") + return fmt.Errorf("setup option for repository failed") } - resticWrapper, err := restic.NewResticWrapper(setupOpt) - if err != nil { + + // write secret and config + // cleanup whole config/secret dir at the end + defer os.RemoveAll(cliSecretDir) + defer os.RemoveAll(cliConfigDir) + if err = prepareDockerVolumeForRestore(*secret, setupOpt, restoreOpt); err != nil { return err } - // if destination flag not specified, restore in current directory - if restoreOpt.Destination == "" { - restoreOpt.Destination, err = os.Getwd() - if err != nil { - return err - } - } - // run restore - if _, err = resticWrapper.RunRestore(restoreOpt); err != nil { + + // run restore inside docker + if err = runRestoreViaDocker(localDestination); err != nil { return err } log.Infof("Repository %s/%s restored in path %s", namespace, repositoryName, restoreOpt.Destination) @@ -101,12 +89,65 @@ func NewDownloadCmd() *cobra.Command { cmd.Flags().StringVar(&kubeConfig, "kubeconfig", kubeConfig, "Path of the Kube config file.") cmd.Flags().StringVar(&repositoryName, "repository", repositoryName, "Name of the Repository.") cmd.Flags().StringVar(&namespace, "namespace", "default", "Namespace of the Repository.") + cmd.Flags().StringVar(&localDestination, "destination", localDestination, "Destination path where snapshot will be restored.") - cmd.Flags().StringVar(&restoreOpt.Destination, "destination", restoreOpt.Destination, "Destination path where snapshot will be restored.") cmd.Flags().StringVar(&restoreOpt.SourceHost, "host", restoreOpt.SourceHost, "Name of the source host machine") cmd.Flags().StringSliceVar(&restoreOpt.RestoreDirs, "directories", restoreOpt.RestoreDirs, "List of directories to be restored") - // TODO: only allow a single snapshot ? cmd.Flags().StringSliceVar(&restoreOpt.Snapshots, "snapshots", restoreOpt.Snapshots, "List of snapshots to be restored") + cmd.Flags().StringVar(&image.Registry, "docker-registry", image.Registry, "Docker image registry for unlock job") + cmd.Flags().StringVar(&image.Tag, "image-tag", image.Tag, "Stash image tag for unlock job") + return cmd } + +func prepareDockerVolumeForRestore(secret core.Secret, setupOpt restic.SetupOptions, restoreOpt restic.RestoreOptions) error { + // write repository secrets + if err := os.MkdirAll(cliSecretDir, 0755); err != nil { + return err + } + for key, value := range secret.Data { + if err := ioutil.WriteFile(filepath.Join(cliSecretDir, key), value, 0755); err != nil { + return err + } + } + // write restic options + err := docker.WriteSetupOptionToFile(&setupOpt, filepath.Join(cliConfigDir, docker.SetupOptionsFile)) + if err != nil { + return err + } + return docker.WriteRestoreOptionToFile(&restoreOpt, filepath.Join(cliConfigDir, docker.RestoreOptionsFile)) +} + +func runRestoreViaDocker(localDestination string) error { + // get current user + currentUser, err := user.Current() + if err != nil { + return err + } + // if destination flag is not specified, restore in current directory + if localDestination == "" { + if localDestination, err = os.Getwd(); err != nil { + return err + } + } + // create local destination dir + if err := os.MkdirAll(localDestination, 0755); err != nil { + return err + } + args := []string{ + "run", + "--rm", + "-u", currentUser.Uid, + "-v", cliConfigDir + ":" + docker.ConfigDir, + "-v", cliSecretDir + ":" + docker.SecretDir, + "-v", localDestination + ":" + docker.DestinationDir, + image.ToContainerImage(), + "docker", + "download-snapshots", + } + log.Infoln("Running docker with args:", args) + out, err := exec.Command("docker", args...).CombinedOutput() + log.Infoln("Output:", string(out)) + return err +} diff --git a/pkg/cmds/cli/unlock_repository.go b/pkg/cmds/cli/unlock_repository.go index edce25880..e8d8874fb 100644 --- a/pkg/cmds/cli/unlock_repository.go +++ b/pkg/cmds/cli/unlock_repository.go @@ -4,6 +4,8 @@ import ( "fmt" "io/ioutil" "os" + "os/exec" + "os/user" "path/filepath" "github.com/appscode/go/flags" @@ -11,7 +13,7 @@ import ( "github.com/appscode/go/types" "github.com/appscode/stash/apis/stash/v1alpha1" stash_scheme "github.com/appscode/stash/client/clientset/versioned/scheme" - "github.com/appscode/stash/pkg/docker" + "github.com/appscode/stash/pkg/cmds/docker" "github.com/appscode/stash/pkg/restic" "github.com/appscode/stash/pkg/util" "github.com/spf13/cobra" @@ -30,14 +32,6 @@ const ( unlockJobSecretVolume = "secret-volume" ) -var ( - image = docker.Docker{ - Registry: docker.ACRegistry, - Image: docker.ImageStash, - Tag: "latest", // TODO: update default release tag - } -) - func NewUnlockRepositoryCmd() *cobra.Command { var ( kubeConfig string @@ -63,7 +57,6 @@ func NewUnlockRepositoryCmd() *cobra.Command { if err != nil { return err } - // unlock local backend if repository.Spec.Backend.Local != nil { if err = unlockLocalRepo(c, repository); err != nil { @@ -71,41 +64,33 @@ func NewUnlockRepositoryCmd() *cobra.Command { } return nil } - // get source repository secret secret, err := c.kubeClient.CoreV1().Secrets(namespace).Get(repository.Spec.Backend.StorageSecretName, metav1.GetOptions{}) if err != nil { return err } - // cleanup whole scratch/secret dir at the end - defer os.RemoveAll(cliScratchDir) - defer os.RemoveAll(cliSecretDir) - - // write repository secrets in a temp dir - if err := os.MkdirAll(cliSecretDir, 0755); err != nil { - return err - } - for key, value := range secret.Data { - if err := ioutil.WriteFile(filepath.Join(cliSecretDir, key), value, 0755); err != nil { - return err - } - } - + // configure restic wrapper extraOpt := util.ExtraOptions{ - SecretDir: cliSecretDir, + SecretDir: docker.SecretDir, EnableCache: false, - ScratchDir: cliScratchDir, + ScratchDir: docker.ScratchDir, } setupOpt, err := util.SetupOptionsForRepository(*repository, extraOpt) if err != nil { - return fmt.Errorf("setup option for repository fail") + return fmt.Errorf("setup option for repository failed") } - resticWrapper, err := restic.NewResticWrapper(setupOpt) - if err != nil { + + // write secret and config + // cleanup whole config/secret dir at the end + defer os.RemoveAll(cliSecretDir) + defer os.RemoveAll(cliConfigDir) + if err = prepareDockerVolumeForUnlock(*secret, setupOpt); err != nil { return err } - if err = resticWrapper.UnlockRepository(); err != nil { + + // run unlock inside docker + if err = runUnlockViaDocker(); err != nil { return err } log.Infof("Repository %s/%s unlocked", namespace, repositoryName) @@ -123,6 +108,42 @@ func NewUnlockRepositoryCmd() *cobra.Command { return cmd } +func prepareDockerVolumeForUnlock(secret core.Secret, setupOpt restic.SetupOptions) error { + // write repository secrets + if err := os.MkdirAll(cliSecretDir, 0755); err != nil { + return err + } + for key, value := range secret.Data { + if err := ioutil.WriteFile(filepath.Join(cliSecretDir, key), value, 0755); err != nil { + return err + } + } + // write restic setup options + return docker.WriteSetupOptionToFile(&setupOpt, filepath.Join(cliConfigDir, docker.SetupOptionsFile)) +} + +func runUnlockViaDocker() error { + // get current user + currentUser, err := user.Current() + if err != nil { + return err + } + args := []string{ + "run", + "--rm", + "-u", currentUser.Uid, + "-v", cliConfigDir + ":" + docker.ConfigDir, + "-v", cliSecretDir + ":" + docker.SecretDir, + image.ToContainerImage(), + "docker", + "unlock-repository", + } + log.Infoln("Running docker with args:", args) + out, err := exec.Command("docker", args...).CombinedOutput() + log.Infoln("Output:", string(out)) + return err +} + func unlockLocalRepo(c *stashCLIController, repo *v1alpha1.Repository) error { _, path, err := util.GetBucketAndPrefix(&repo.Spec.Backend) if err != nil { diff --git a/pkg/cmds/docker/docker.go b/pkg/cmds/docker/docker.go new file mode 100644 index 000000000..73da1f506 --- /dev/null +++ b/pkg/cmds/docker/docker.go @@ -0,0 +1,95 @@ +package docker + +import ( + "encoding/json" + "io/ioutil" + "os" + "path/filepath" + + "github.com/appscode/stash/pkg/restic" + "github.com/spf13/cobra" +) + +const ( + ScratchDir = "/tmp/scratch" + SecretDir = "/tmp/secret" + ConfigDir = "/tmp/config" + DestinationDir = "/tmp/destination" + SetupOptionsFile = "setup.json" + RestoreOptionsFile = "restore.json" +) + +func NewDockerCmd() *cobra.Command { + var cmd = &cobra.Command{ + Use: "docker", + Short: `Run restic commands inside Docker`, + Long: `Run restic commands inside Docker`, + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + return nil + }, + } + + cmd.AddCommand(NewUnlockRepositoryCmd()) + cmd.AddCommand(NewDownloadCmd()) + + return cmd +} + +func WriteSetupOptionToFile(options *restic.SetupOptions, fileName string) error { + jsonOutput, err := json.MarshalIndent(options, "", " ") + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(fileName), 0755); err != nil { + return err + } + if err := ioutil.WriteFile(fileName, jsonOutput, 0755); err != nil { + return err + } + return nil +} + +func ReadSetupOptionFromFile(filename string) (*restic.SetupOptions, error) { + data, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + + options := &restic.SetupOptions{} + err = json.Unmarshal(data, options) + if err != nil { + return nil, err + } + + return options, nil +} + +func WriteRestoreOptionToFile(options *restic.RestoreOptions, fileName string) error { + jsonOutput, err := json.MarshalIndent(options, "", " ") + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(fileName), 0755); err != nil { + return err + } + if err := ioutil.WriteFile(fileName, jsonOutput, 0755); err != nil { + return err + } + return nil +} + +func ReadRestoreOptionFromFile(filename string) (*restic.RestoreOptions, error) { + data, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + + options := &restic.RestoreOptions{} + err = json.Unmarshal(data, options) + if err != nil { + return nil, err + } + + return options, nil +} diff --git a/pkg/cmds/docker/download.go b/pkg/cmds/docker/download.go new file mode 100644 index 000000000..71ac7ecf1 --- /dev/null +++ b/pkg/cmds/docker/download.go @@ -0,0 +1,39 @@ +package docker + +import ( + "path/filepath" + + "github.com/appscode/go/log" + "github.com/appscode/stash/pkg/restic" + "github.com/spf13/cobra" +) + +func NewDownloadCmd() *cobra.Command { + var cmd = &cobra.Command{ + Use: "download-snapshots", + Short: `Download snapshots`, + Long: `Download contents of snapshots from Repository`, + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + setupOpt, err := ReadSetupOptionFromFile(filepath.Join(ConfigDir, SetupOptionsFile)) + if err != nil { + return err + } + restoreOpt, err := ReadRestoreOptionFromFile(filepath.Join(ConfigDir, RestoreOptionsFile)) + if err != nil { + return err + } + resticWrapper, err := restic.NewResticWrapper(*setupOpt) + if err != nil { + return err + } + // run restore + if _, err = resticWrapper.RunRestore(*restoreOpt); err != nil { + return err + } + log.Infof("Restore completed") + return nil + }, + } + return cmd +} diff --git a/pkg/cmds/docker/unlock_repository.go b/pkg/cmds/docker/unlock_repository.go new file mode 100644 index 000000000..b85a6a5b4 --- /dev/null +++ b/pkg/cmds/docker/unlock_repository.go @@ -0,0 +1,35 @@ +package docker + +import ( + "path/filepath" + + "github.com/appscode/go/log" + "github.com/appscode/stash/pkg/restic" + "github.com/spf13/cobra" +) + +func NewUnlockRepositoryCmd() *cobra.Command { + var cmd = &cobra.Command{ + Use: "unlock-repository", + Short: `Unlock Restic Repository`, + Long: `Unlock Restic Repository`, + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + setupOpt, err := ReadSetupOptionFromFile(filepath.Join(ConfigDir, SetupOptionsFile)) + if err != nil { + return err + } + resticWrapper, err := restic.NewResticWrapper(*setupOpt) + if err != nil { + return err + } + // run unlock + if err = resticWrapper.UnlockRepository(); err != nil { + return err + } + log.Infof("Unlock completed") + return nil + }, + } + return cmd +} diff --git a/pkg/cmds/root.go b/pkg/cmds/root.go index 5f9670bd7..ab5521c23 100644 --- a/pkg/cmds/root.go +++ b/pkg/cmds/root.go @@ -10,6 +10,7 @@ import ( "github.com/appscode/stash/apis" "github.com/appscode/stash/client/clientset/versioned/scheme" stash_cli "github.com/appscode/stash/pkg/cmds/cli" + "github.com/appscode/stash/pkg/cmds/docker" "github.com/appscode/stash/pkg/util" "github.com/spf13/cobra" genericapiserver "k8s.io/apiserver/pkg/server" @@ -72,6 +73,7 @@ func NewRootCmd() *cobra.Command { rootCmd.AddCommand(NewCmdUpdateStatus()) rootCmd.AddCommand(stash_cli.NewCLICmd()) + rootCmd.AddCommand(docker.NewDockerCmd()) return rootCmd }