Skip to content

Commit

Permalink
Run restic commands using docker (#754)
Browse files Browse the repository at this point in the history
  • Loading branch information
@tamalsaha
Dipta Das authored and tamalsaha committed Apr 25, 2019
1 parent 17feb41 commit 3f0ae2d
Show file tree
Hide file tree
Showing 7 changed files with 312 additions and 70 deletions.
11 changes: 10 additions & 1 deletion pkg/cmds/cli/cli.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"path/filepath"

cs "github.com/appscode/stash/client/clientset/versioned"
docker_image "github.com/appscode/stash/pkg/docker"
"github.com/spf13/cobra"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
Expand All @@ -13,8 +14,8 @@ import (
)

const (
cliScratchDir = "/tmp/stash-cli/scratch"
cliSecretDir = "/tmp/stash-cli/secret"
cliConfigDir = "/tmp/stash-cli/config"
)

type stashCLIController struct {
Expand All @@ -23,6 +24,14 @@ type stashCLIController struct {
stashClient cs.Interface
}

var (
image = docker_image.Docker{
Registry: docker_image.ACRegistry,
Image: docker_image.ImageStash,
Tag: "latest", // TODO: update default release tag
}
)

func NewCLICmd() *cobra.Command {
var cmd = &cobra.Command{
Use: "cli",
Expand Down
117 changes: 79 additions & 38 deletions pkg/cmds/cli/download.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,23 +4,29 @@ import (
"fmt"
"io/ioutil"
"os"
"os/exec"
"os/user"
"path/filepath"

"github.com/appscode/go/flags"
"github.com/appscode/go/log"
"github.com/appscode/stash/pkg/cmds/docker"
"github.com/appscode/stash/pkg/restic"
"github.com/appscode/stash/pkg/util"
"github.com/spf13/cobra"
core "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func NewDownloadCmd() *cobra.Command {
var (
kubeConfig string
repositoryName string
namespace string
restoreOpt = restic.RestoreOptions{
SourceHost: restic.DefaultHost,
kubeConfig string
repositoryName string
namespace string
localDestination string
restoreOpt = restic.RestoreOptions{
SourceHost: restic.DefaultHost,
Destination: docker.DestinationDir,
}
)

Expand All @@ -42,55 +48,37 @@ func NewDownloadCmd() *cobra.Command {
if err != nil {
return err
}

// unlock local backend
if repository.Spec.Backend.Local != nil {
return fmt.Errorf("can't restore from repository with local backend")
}

// get source repository secret
// get repository secret
secret, err := c.kubeClient.CoreV1().Secrets(namespace).Get(repository.Spec.Backend.StorageSecretName, metav1.GetOptions{})
if err != nil {
return err
}

// cleanup whole scratch/secret dir at the end
defer os.RemoveAll(cliScratchDir)
defer os.RemoveAll(cliSecretDir)

// write repository secrets in a temp dir
if err := os.MkdirAll(cliSecretDir, 0755); err != nil {
return err
}
for key, value := range secret.Data {
if err := ioutil.WriteFile(filepath.Join(cliSecretDir, key), value, 0755); err != nil {
return err
}
}

// configure restic wrapper
extraOpt := util.ExtraOptions{
SecretDir: cliSecretDir,
SecretDir: docker.SecretDir,
EnableCache: false,
ScratchDir: cliScratchDir,
ScratchDir: docker.ScratchDir,
}
setupOpt, err := util.SetupOptionsForRepository(*repository, extraOpt)
if err != nil {
return fmt.Errorf("setup option for repository fail")
return fmt.Errorf("setup option for repository failed")
}
resticWrapper, err := restic.NewResticWrapper(setupOpt)
if err != nil {

// write secret and config
// cleanup whole config/secret dir at the end
defer os.RemoveAll(cliSecretDir)
defer os.RemoveAll(cliConfigDir)
if err = prepareDockerVolumeForRestore(*secret, setupOpt, restoreOpt); err != nil {
return err
}
// if destination flag not specified, restore in current directory
if restoreOpt.Destination == "" {
restoreOpt.Destination, err = os.Getwd()
if err != nil {
return err
}
}
// run restore
if _, err = resticWrapper.RunRestore(restoreOpt); err != nil {

// run restore inside docker
if err = runRestoreViaDocker(localDestination); err != nil {
return err
}
log.Infof("Repository %s/%s restored in path %s", namespace, repositoryName, restoreOpt.Destination)
Expand All @@ -101,12 +89,65 @@ func NewDownloadCmd() *cobra.Command {
cmd.Flags().StringVar(&kubeConfig, "kubeconfig", kubeConfig, "Path of the Kube config file.")
cmd.Flags().StringVar(&repositoryName, "repository", repositoryName, "Name of the Repository.")
cmd.Flags().StringVar(&namespace, "namespace", "default", "Namespace of the Repository.")
cmd.Flags().StringVar(&localDestination, "destination", localDestination, "Destination path where snapshot will be restored.")

cmd.Flags().StringVar(&restoreOpt.Destination, "destination", restoreOpt.Destination, "Destination path where snapshot will be restored.")
cmd.Flags().StringVar(&restoreOpt.SourceHost, "host", restoreOpt.SourceHost, "Name of the source host machine")
cmd.Flags().StringSliceVar(&restoreOpt.RestoreDirs, "directories", restoreOpt.RestoreDirs, "List of directories to be restored")
// TODO: only allow a single snapshot ?
cmd.Flags().StringSliceVar(&restoreOpt.Snapshots, "snapshots", restoreOpt.Snapshots, "List of snapshots to be restored")

cmd.Flags().StringVar(&image.Registry, "docker-registry", image.Registry, "Docker image registry for unlock job")
cmd.Flags().StringVar(&image.Tag, "image-tag", image.Tag, "Stash image tag for unlock job")

return cmd
}

func prepareDockerVolumeForRestore(secret core.Secret, setupOpt restic.SetupOptions, restoreOpt restic.RestoreOptions) error {
// write repository secrets
if err := os.MkdirAll(cliSecretDir, 0755); err != nil {
return err
}
for key, value := range secret.Data {
if err := ioutil.WriteFile(filepath.Join(cliSecretDir, key), value, 0755); err != nil {
return err
}
}
// write restic options
err := docker.WriteSetupOptionToFile(&setupOpt, filepath.Join(cliConfigDir, docker.SetupOptionsFile))
if err != nil {
return err
}
return docker.WriteRestoreOptionToFile(&restoreOpt, filepath.Join(cliConfigDir, docker.RestoreOptionsFile))
}

func runRestoreViaDocker(localDestination string) error {
// get current user
currentUser, err := user.Current()
if err != nil {
return err
}
// if destination flag is not specified, restore in current directory
if localDestination == "" {
if localDestination, err = os.Getwd(); err != nil {
return err
}
}
// create local destination dir
if err := os.MkdirAll(localDestination, 0755); err != nil {
return err
}
args := []string{
"run",
"--rm",
"-u", currentUser.Uid,
"-v", cliConfigDir + ":" + docker.ConfigDir,
"-v", cliSecretDir + ":" + docker.SecretDir,
"-v", localDestination + ":" + docker.DestinationDir,
image.ToContainerImage(),
"docker",
"download-snapshots",
}
log.Infoln("Running docker with args:", args)
out, err := exec.Command("docker", args...).CombinedOutput()
log.Infoln("Output:", string(out))
return err
}
83 changes: 52 additions & 31 deletions pkg/cmds/cli/unlock_repository.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,16 @@ import (
"fmt"
"io/ioutil"
"os"
"os/exec"
"os/user"
"path/filepath"

"github.com/appscode/go/flags"
"github.com/appscode/go/log"
"github.com/appscode/go/types"
"github.com/appscode/stash/apis/stash/v1alpha1"
stash_scheme "github.com/appscode/stash/client/clientset/versioned/scheme"
"github.com/appscode/stash/pkg/docker"
"github.com/appscode/stash/pkg/cmds/docker"
"github.com/appscode/stash/pkg/restic"
"github.com/appscode/stash/pkg/util"
"github.com/spf13/cobra"
Expand All @@ -30,14 +32,6 @@ const (
unlockJobSecretVolume = "secret-volume"
)

var (
image = docker.Docker{
Registry: docker.ACRegistry,
Image: docker.ImageStash,
Tag: "latest", // TODO: update default release tag
}
)

func NewUnlockRepositoryCmd() *cobra.Command {
var (
kubeConfig string
Expand All @@ -63,49 +57,40 @@ func NewUnlockRepositoryCmd() *cobra.Command {
if err != nil {
return err
}

// unlock local backend
if repository.Spec.Backend.Local != nil {
if err = unlockLocalRepo(c, repository); err != nil {
return fmt.Errorf("can't unlock repository for local backend, reason: %s", err)
}
return nil
}

// get source repository secret
secret, err := c.kubeClient.CoreV1().Secrets(namespace).Get(repository.Spec.Backend.StorageSecretName, metav1.GetOptions{})
if err != nil {
return err
}

// cleanup whole scratch/secret dir at the end
defer os.RemoveAll(cliScratchDir)
defer os.RemoveAll(cliSecretDir)

// write repository secrets in a temp dir
if err := os.MkdirAll(cliSecretDir, 0755); err != nil {
return err
}
for key, value := range secret.Data {
if err := ioutil.WriteFile(filepath.Join(cliSecretDir, key), value, 0755); err != nil {
return err
}
}

// configure restic wrapper
extraOpt := util.ExtraOptions{
SecretDir: cliSecretDir,
SecretDir: docker.SecretDir,
EnableCache: false,
ScratchDir: cliScratchDir,
ScratchDir: docker.ScratchDir,
}
setupOpt, err := util.SetupOptionsForRepository(*repository, extraOpt)
if err != nil {
return fmt.Errorf("setup option for repository fail")
return fmt.Errorf("setup option for repository failed")
}
resticWrapper, err := restic.NewResticWrapper(setupOpt)
if err != nil {

// write secret and config
// cleanup whole config/secret dir at the end
defer os.RemoveAll(cliSecretDir)
defer os.RemoveAll(cliConfigDir)
if err = prepareDockerVolumeForUnlock(*secret, setupOpt); err != nil {
return err
}
if err = resticWrapper.UnlockRepository(); err != nil {

// run unlock inside docker
if err = runUnlockViaDocker(); err != nil {
return err
}
log.Infof("Repository %s/%s unlocked", namespace, repositoryName)
Expand All @@ -123,6 +108,42 @@ func NewUnlockRepositoryCmd() *cobra.Command {
return cmd
}

func prepareDockerVolumeForUnlock(secret core.Secret, setupOpt restic.SetupOptions) error {
// write repository secrets
if err := os.MkdirAll(cliSecretDir, 0755); err != nil {
return err
}
for key, value := range secret.Data {
if err := ioutil.WriteFile(filepath.Join(cliSecretDir, key), value, 0755); err != nil {
return err
}
}
// write restic setup options
return docker.WriteSetupOptionToFile(&setupOpt, filepath.Join(cliConfigDir, docker.SetupOptionsFile))
}

func runUnlockViaDocker() error {
// get current user
currentUser, err := user.Current()
if err != nil {
return err
}
args := []string{
"run",
"--rm",
"-u", currentUser.Uid,
"-v", cliConfigDir + ":" + docker.ConfigDir,
"-v", cliSecretDir + ":" + docker.SecretDir,
image.ToContainerImage(),
"docker",
"unlock-repository",
}
log.Infoln("Running docker with args:", args)
out, err := exec.Command("docker", args...).CombinedOutput()
log.Infoln("Output:", string(out))
return err
}

func unlockLocalRepo(c *stashCLIController, repo *v1alpha1.Repository) error {
_, path, err := util.GetBucketAndPrefix(&repo.Spec.Backend)
if err != nil {
Expand Down
Loading

0 comments on commit 3f0ae2d

Please sign in to comment.