Skip to content

Commit

Permalink
Implement snapshots for v1beta1 api (#749)
Browse files Browse the repository at this point in the history
  • Loading branch information
Dipta Das authored and tamalsaha committed Apr 27, 2019
1 parent 3f0ae2d commit bcf8b90
Show file tree
Hide file tree
Showing 11 changed files with 506 additions and 136 deletions.
61 changes: 59 additions & 2 deletions pkg/cmds/cli/cli.go
Original file line number Diff line number Diff line change
@@ -1,11 +1,16 @@
package cli

import (
"io/ioutil"
"os"
"path/filepath"

cs "github.com/appscode/stash/client/clientset/versioned"
"github.com/appscode/stash/pkg/cmds/docker"
docker_image "github.com/appscode/stash/pkg/docker"
"github.com/appscode/stash/pkg/restic"
"github.com/spf13/cobra"
core "k8s.io/api/core/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/util/homedir"
Expand All @@ -14,8 +19,8 @@ import (
)

const (
cliSecretDir = "/tmp/stash-cli/secret"
cliConfigDir = "/tmp/stash-cli/config"
secretDirName = "secret"
configDirName = "config"
)

type stashCLIController struct {
Expand All @@ -24,6 +29,12 @@ type stashCLIController struct {
stashClient cs.Interface
}

type cliLocalDirectories struct {
secretDir string // temp dir
configDir string // temp dir
downloadDir string // user provided or, current working dir
}

var (
image = docker_image.Docker{
Registry: docker_image.ACRegistry,
Expand All @@ -49,6 +60,7 @@ func NewCLICmd() *cobra.Command {
cmd.AddCommand(NewTriggerBackupCmd())
cmd.AddCommand(NewBackupPVCmd())
cmd.AddCommand(NewDownloadCmd())
cmd.AddCommand(NewDeleteSnapshotCmd())

return cmd
}
Expand All @@ -72,3 +84,48 @@ func newStashCLIController(kubeConfig string) (*stashCLIController, error) {
}
return controller, nil
}

func (localDirs *cliLocalDirectories) prepareSecretDir(tempDir string, secret *core.Secret) error {
// write repository secrets in a sub-dir insider tempDir
localDirs.secretDir = filepath.Join(tempDir, secretDirName)
if err := os.MkdirAll(localDirs.secretDir, 0755); err != nil {
return err
}
for key, value := range secret.Data {
if err := ioutil.WriteFile(filepath.Join(localDirs.secretDir, key), value, 0755); err != nil {
return err
}
}
return nil
}

func (localDirs *cliLocalDirectories) prepareConfigDir(tempDir string, setupOpt *restic.SetupOptions, restoreOpt *restic.RestoreOptions) error {
// write restic options in a sub-dir insider tempDir
localDirs.configDir = filepath.Join(tempDir, configDirName)
if err := os.MkdirAll(localDirs.secretDir, 0755); err != nil {
return err
}
if setupOpt != nil {
err := docker.WriteSetupOptionToFile(setupOpt, filepath.Join(localDirs.configDir, docker.SetupOptionsFile))
if err != nil {
return err
}
}
if restoreOpt != nil {
err := docker.WriteRestoreOptionToFile(restoreOpt, filepath.Join(localDirs.configDir, docker.RestoreOptionsFile))
if err != nil {
return err
}
}
return nil
}

func (localDirs *cliLocalDirectories) prepareDownloadDir() (err error) {
// if destination flag is not specified, restore in current directory
if localDirs.downloadDir == "" {
if localDirs.downloadDir, err = os.Getwd(); err != nil {
return err
}
}
return os.MkdirAll(localDirs.downloadDir, 0755)
}
127 changes: 127 additions & 0 deletions pkg/cmds/cli/delete_snapshot.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
package cli

import (
"fmt"
"io/ioutil"
"os"
"os/exec"
"os/user"

"github.com/appscode/go/log"
"github.com/appscode/stash/pkg/cmds/docker"
"github.com/appscode/stash/pkg/registry/snapshot"
"github.com/appscode/stash/pkg/util"
"github.com/spf13/cobra"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func NewDeleteSnapshotCmd() *cobra.Command {
var (
kubeConfig string
namespace string
localDirs = &cliLocalDirectories{}
)

var cmd = &cobra.Command{
Use: "delete-snapshot",
Short: `Delete a snapshot from repository backend`,
Long: `Delete a snapshot from repository backend`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
if len(args) == 0 {
return fmt.Errorf("snapshot name not provided")
}
repoName, snapshotId, err := util.GetRepoNameAndSnapshotID(args[0])
if err != nil {
return err
}

c, err := newStashCLIController(kubeConfig)
if err != nil {
return err
}

// get source repository
repository, err := c.stashClient.StashV1alpha1().Repositories(namespace).Get(repoName, metav1.GetOptions{})
if err != nil {
return err
}
// delete from local backend
if repository.Spec.Backend.Local != nil {
r := snapshot.NewREST(c.clientConfig)
return r.ForgetVersionedSnapshots(repository, []string{snapshotId}, false)
}

// get source repository secret
secret, err := c.kubeClient.CoreV1().Secrets(namespace).Get(repository.Spec.Backend.StorageSecretName, metav1.GetOptions{})
if err != nil {
return err
}

// configure restic wrapper
extraOpt := util.ExtraOptions{
SecretDir: docker.SecretDir,
EnableCache: false,
ScratchDir: docker.ScratchDir,
}
setupOpt, err := util.SetupOptionsForRepository(*repository, extraOpt)
if err != nil {
return fmt.Errorf("setup option for repository failed")
}

// write secret and config in a temp dir
// cleanup whole tempDir dir at the end
tempDir, err := ioutil.TempDir("", "stash-cli")
if err != nil {
return err
}
defer os.RemoveAll(tempDir)

// prepare local dirs
if err = localDirs.prepareSecretDir(tempDir, secret); err != nil {
return err
}
if err = localDirs.prepareConfigDir(tempDir, &setupOpt, nil); err != nil {
return err
}

// run unlock inside docker
if err = runDeleteSnapshotViaDocker(*localDirs, snapshotId); err != nil {
return err
}
log.Infof("Snapshot %s deleted from repository %s/%s", snapshotId, namespace, repoName)
return nil
},
}

cmd.Flags().StringVar(&kubeConfig, "kubeconfig", kubeConfig, "Path of the Kube config file.")
cmd.Flags().StringVar(&namespace, "namespace", "default", "Namespace of the Repository.")

cmd.Flags().StringVar(&image.Registry, "docker-registry", image.Registry, "Docker image registry")
cmd.Flags().StringVar(&image.Tag, "image-tag", image.Tag, "Stash image tag")

return cmd
}

func runDeleteSnapshotViaDocker(localDirs cliLocalDirectories, snapshotId string) error {
// get current user
currentUser, err := user.Current()
if err != nil {
return err
}
args := []string{
"run",
"--rm",
"-u", currentUser.Uid,
"-v", localDirs.configDir + ":" + docker.ConfigDir,
"-v", localDirs.secretDir + ":" + docker.SecretDir,
image.ToContainerImage(),
"docker",
"delete-snapshot",
"--snapshot", snapshotId,
}
log.Infoln("Running docker with args:", args)
out, err := exec.Command("docker", args...).CombinedOutput()
log.Infoln("Output:", string(out))
return err
}
77 changes: 29 additions & 48 deletions pkg/cmds/cli/download.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,25 +6,23 @@ import (
"os"
"os/exec"
"os/user"
"path/filepath"

"github.com/appscode/go/flags"
"github.com/appscode/go/log"
"github.com/appscode/stash/pkg/cmds/docker"
"github.com/appscode/stash/pkg/restic"
"github.com/appscode/stash/pkg/util"
"github.com/spf13/cobra"
core "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func NewDownloadCmd() *cobra.Command {
var (
kubeConfig string
repositoryName string
namespace string
localDestination string
restoreOpt = restic.RestoreOptions{
kubeConfig string
repositoryName string
namespace string
localDirs = &cliLocalDirectories{}
restoreOpt = restic.RestoreOptions{
SourceHost: restic.DefaultHost,
Destination: docker.DestinationDir,
}
Expand Down Expand Up @@ -69,16 +67,27 @@ func NewDownloadCmd() *cobra.Command {
return fmt.Errorf("setup option for repository failed")
}

// write secret and config
// cleanup whole config/secret dir at the end
defer os.RemoveAll(cliSecretDir)
defer os.RemoveAll(cliConfigDir)
if err = prepareDockerVolumeForRestore(*secret, setupOpt, restoreOpt); err != nil {
// write secret and config in a temp dir
// cleanup whole tempDir dir at the end
tempDir, err := ioutil.TempDir("", "stash-cli")
if err != nil {
return err
}
defer os.RemoveAll(tempDir)

// prepare local dirs
if err = localDirs.prepareSecretDir(tempDir, secret); err != nil {
return err
}
if err = localDirs.prepareConfigDir(tempDir, &setupOpt, &restoreOpt); err != nil {
return err
}
if err = localDirs.prepareDownloadDir(); err != nil {
return err
}

// run restore inside docker
if err = runRestoreViaDocker(localDestination); err != nil {
if err = runRestoreViaDocker(*localDirs); err != nil {
return err
}
log.Infof("Repository %s/%s restored in path %s", namespace, repositoryName, restoreOpt.Destination)
Expand All @@ -89,59 +98,31 @@ func NewDownloadCmd() *cobra.Command {
cmd.Flags().StringVar(&kubeConfig, "kubeconfig", kubeConfig, "Path of the Kube config file.")
cmd.Flags().StringVar(&repositoryName, "repository", repositoryName, "Name of the Repository.")
cmd.Flags().StringVar(&namespace, "namespace", "default", "Namespace of the Repository.")
cmd.Flags().StringVar(&localDestination, "destination", localDestination, "Destination path where snapshot will be restored.")
cmd.Flags().StringVar(&localDirs.downloadDir, "destination", localDirs.downloadDir, "Destination path where snapshot will be restored.")

cmd.Flags().StringVar(&restoreOpt.SourceHost, "host", restoreOpt.SourceHost, "Name of the source host machine")
cmd.Flags().StringSliceVar(&restoreOpt.RestoreDirs, "directories", restoreOpt.RestoreDirs, "List of directories to be restored")
cmd.Flags().StringSliceVar(&restoreOpt.Snapshots, "snapshots", restoreOpt.Snapshots, "List of snapshots to be restored")

cmd.Flags().StringVar(&image.Registry, "docker-registry", image.Registry, "Docker image registry for unlock job")
cmd.Flags().StringVar(&image.Tag, "image-tag", image.Tag, "Stash image tag for unlock job")
cmd.Flags().StringVar(&image.Registry, "docker-registry", image.Registry, "Docker image registry")
cmd.Flags().StringVar(&image.Tag, "image-tag", image.Tag, "Stash image tag")

return cmd
}

func prepareDockerVolumeForRestore(secret core.Secret, setupOpt restic.SetupOptions, restoreOpt restic.RestoreOptions) error {
// write repository secrets
if err := os.MkdirAll(cliSecretDir, 0755); err != nil {
return err
}
for key, value := range secret.Data {
if err := ioutil.WriteFile(filepath.Join(cliSecretDir, key), value, 0755); err != nil {
return err
}
}
// write restic options
err := docker.WriteSetupOptionToFile(&setupOpt, filepath.Join(cliConfigDir, docker.SetupOptionsFile))
if err != nil {
return err
}
return docker.WriteRestoreOptionToFile(&restoreOpt, filepath.Join(cliConfigDir, docker.RestoreOptionsFile))
}

func runRestoreViaDocker(localDestination string) error {
func runRestoreViaDocker(localDirs cliLocalDirectories) error {
// get current user
currentUser, err := user.Current()
if err != nil {
return err
}
// if destination flag is not specified, restore in current directory
if localDestination == "" {
if localDestination, err = os.Getwd(); err != nil {
return err
}
}
// create local destination dir
if err := os.MkdirAll(localDestination, 0755); err != nil {
return err
}
args := []string{
"run",
"--rm",
"-u", currentUser.Uid,
"-v", cliConfigDir + ":" + docker.ConfigDir,
"-v", cliSecretDir + ":" + docker.SecretDir,
"-v", localDestination + ":" + docker.DestinationDir,
"-v", localDirs.configDir + ":" + docker.ConfigDir,
"-v", localDirs.secretDir + ":" + docker.SecretDir,
"-v", localDirs.downloadDir + ":" + docker.DestinationDir,
image.ToContainerImage(),
"docker",
"download-snapshots",
Expand Down
Loading

0 comments on commit bcf8b90

Please sign in to comment.