From 13fb68bda7e45695db3fb36eda15738a263fa00d Mon Sep 17 00:00:00 2001 From: Dipta Das Date: Thu, 21 Mar 2019 13:17:03 +0600 Subject: [PATCH] Post backup/restore status update (#691) * Post backup/restore status update * Add status package * Handle backup/restore error * Fail pod for back/restore error --- apis/constants.go | 7 + docs/examples/pvc/backup-error.yaml | 35 +++ docs/examples/pvc/backup.yaml | 3 + docs/examples/pvc/functions.yaml | 60 +++-- docs/examples/pvc/rbac.yaml | 41 ++++ docs/examples/pvc/restore-error.yaml | 24 ++ docs/examples/pvc/restore.yaml | 3 + docs/examples/pvc/tasks.yaml | 22 +- docs/guides/backup_restore_pvc.md | 77 ++++-- glide.lock | 10 +- hack/dev/run.sh | 1 + pkg/cmds/backup_pvc.go | 27 ++- pkg/cmds/restore_pvc.go | 16 +- pkg/cmds/root.go | 1 + pkg/cmds/update_status.go | 64 +++++ pkg/controller/backup_session.go | 15 +- pkg/controller/inputs.go | 3 + pkg/controller/restore_session.go | 15 +- pkg/eventer/recorder.go | 52 ++-- pkg/resolve/task_test.go | 4 +- pkg/restic/output.go | 6 +- pkg/status/status.go | 177 ++++++++++++++ vendor/github.com/gomodules/envsubst/LICENSE | 21 -- vendor/github.com/gomodules/envsubst/eval.go | 57 ----- vendor/github.com/gomodules/envsubst/funcs.go | 228 ------------------ .../github.com/gomodules/envsubst/template.go | 180 -------------- vendor/gomodules.xyz/envsubst/eval.go | 2 +- vendor/gomodules.xyz/envsubst/parse/parse.go | 6 + 28 files changed, 574 insertions(+), 583 deletions(-) create mode 100644 docs/examples/pvc/backup-error.yaml create mode 100644 docs/examples/pvc/rbac.yaml create mode 100644 docs/examples/pvc/restore-error.yaml create mode 100644 pkg/cmds/update_status.go create mode 100644 pkg/status/status.go delete mode 100644 vendor/github.com/gomodules/envsubst/LICENSE delete mode 100644 vendor/github.com/gomodules/envsubst/eval.go delete mode 100644 vendor/github.com/gomodules/envsubst/funcs.go delete mode 100644 vendor/github.com/gomodules/envsubst/template.go diff --git a/apis/constants.go b/apis/constants.go index d23c00ea3..efc72fa68 100644 --- a/apis/constants.go +++ b/apis/constants.go @@ -1,6 +1,11 @@ package apis const ( + Namespace = "NAMESPACE" + BackupSession = "BACKUP_SESSION" + RestoreSession = "RESTORE_SESSION" + + RepositoryName = "REPOSITORY_NAME" RepositoryProvider = "REPOSITORY_PROVIDER" RepositorySecretName = "REPOSITORY_SECRET_NAME" RepositoryBucket = "REPOSITORY_BUCKET" @@ -25,4 +30,6 @@ const ( RetentionKeepTags = "RETENTION_KEEP_TAGS" RetentionPrune = "RETENTION_PRUNE" RetentionDryRun = "RETENTION_DRY_RUN" + + StatusSubresourceEnabled = "ENABLE_STATUS_SUBRESOURCE" ) diff --git a/docs/examples/pvc/backup-error.yaml b/docs/examples/pvc/backup-error.yaml new file mode 100644 index 000000000..ae6524a0f --- /dev/null +++ b/docs/examples/pvc/backup-error.yaml @@ -0,0 +1,35 @@ +apiVersion: stash.appscode.com/v1beta1 +kind: BackupConfiguration +metadata: + name: pvc-backup-config + namespace: demo +spec: + runtimeSettings: + pod: + serviceAccountName: pvc-backup-restore + task: + name: pvc-backup-task + repository: + name: hello-repo + schedule: "0 * * * *" + target: + ref: + apiVersion: v1 + kind: PersistentVolumeClaim + name: test-pvc-source + mountPath: /etc/target + directories: + - /unknown-path/target/dir-01 + - /unknown-path/target/dir-02 + retentionPolicy: + keepLast: 5 + prune: true +--- +apiVersion: stash.appscode.com/v1beta1 +kind: BackupSession +metadata: + name: pvc-backup-02 + namespace: demo +spec: + backupConfiguration: + name: pvc-backup-config \ No newline at end of file diff --git a/docs/examples/pvc/backup.yaml b/docs/examples/pvc/backup.yaml index 493e5d665..6b0f437d0 100644 --- a/docs/examples/pvc/backup.yaml +++ b/docs/examples/pvc/backup.yaml @@ -4,6 +4,9 @@ metadata: name: pvc-backup-config namespace: demo spec: + runtimeSettings: + pod: + serviceAccountName: pvc-backup-restore task: name: pvc-backup-task repository: diff --git a/docs/examples/pvc/functions.yaml b/docs/examples/pvc/functions.yaml index 789098e27..89447704e 100644 --- a/docs/examples/pvc/functions.yaml +++ b/docs/examples/pvc/functions.yaml @@ -1,29 +1,46 @@ apiVersion: stash.appscode.com/v1beta1 kind: Function +metadata: + name: update-status +spec: + image: diptadas/stash:status + args: + - update-status + - --namespace=${NAMESPACE="default"} + - --repository=${REPOSITORY_NAME:=} + - --backup-session=${BACKUP_SESSION:=} + - --restore-session=${RESTORE_SESSION:=} + - --output-dir=${outputDir} + - --enable-status-subresource=${ENABLE_STATUS_SUBRESOURCE:=false} + volumeMounts: + - name: ${outputVolume} + mountPath: ${outputDir} +--- +apiVersion: stash.appscode.com/v1beta1 +kind: Function metadata: name: pvc-backup spec: - image: diptadas/stash:pvc + image: diptadas/stash:status args: - backup-pvc - --provider=${REPOSITORY_PROVIDER} - - --bucket=${REPOSITORY_BUCKET=""} - - --endpoint=${REPOSITORY_ENDPOINT=""} - - --path=${REPOSITORY_PREFIX=""} + - --bucket=${REPOSITORY_BUCKET:=} + - --endpoint=${REPOSITORY_ENDPOINT:=} + - --path=${REPOSITORY_PREFIX:=} - --secret-dir=/etc/repository/secret # specified - --scratch-dir=/tmp/restic/scratch # specified - - --enable-cache=false # specified - --hostname=${HOSTNAME} - --backup-dirs=${TARGET_DIRECTORIES} - - --retention-keep-last=${RETENTION_KEEP_LAST=0} - - --retention-keep-hourly=${RETENTION_KEEP_HOURLY=0} - - --retention-keep-daily=${RETENTION_KEEP_DAILY=0} - - --retention-keep-weekly=${RETENTION_KEEP_WEEKLY=0} - - --retention-keep-monthly=${RETENTION_KEEP_MONTHLY=0} - - --retention-keep-yearly=${RETENTION_KEEP_YEARLY=0} - - --retention-keep-tags=${RETENTION_KEEP_TAGS=""} - - --retention-prune=${RETENTION_PRUNE=false} - - --retention-dry-run=${RETENTION_DRY_RUN=false} + - --retention-keep-last=${RETENTION_KEEP_LAST:=0} + - --retention-keep-hourly=${RETENTION_KEEP_HOURLY:=0} + - --retention-keep-daily=${RETENTION_KEEP_DAILY:=0} + - --retention-keep-weekly=${RETENTION_KEEP_WEEKLY:=0} + - --retention-keep-monthly=${RETENTION_KEEP_MONTHLY:=0} + - --retention-keep-yearly=${RETENTION_KEEP_YEARLY:=0} + - --retention-keep-tags=${RETENTION_KEEP_TAGS:=} + - --retention-prune=${RETENTION_PRUNE:=false} + - --retention-dry-run=${RETENTION_DRY_RUN:=false} - --output-dir=${outputDir} #- --metrics-enabled #- --metrics-pushgateway-url @@ -36,25 +53,26 @@ spec: mountPath: ${TARGET_MOUNT_PATH} - name: ${secretVolume} mountPath: /etc/repository/secret + - name: ${outputVolume} + mountPath: ${outputDir} --- apiVersion: stash.appscode.com/v1beta1 kind: Function metadata: name: pvc-restore spec: - image: diptadas/stash:pvc + image: diptadas/stash:status args: - restore-pvc - --provider=${REPOSITORY_PROVIDER} - - --bucket=${REPOSITORY_BUCKET=""} - - --endpoint=${REPOSITORY_ENDPOINT=""} - - --path=${REPOSITORY_PREFIX=""} + - --bucket=${REPOSITORY_BUCKET:=} + - --endpoint=${REPOSITORY_ENDPOINT:=} + - --path=${REPOSITORY_PREFIX:=} - --secret-dir=/etc/repository/secret # specified - --scratch-dir=/tmp/restic/scratch # specified - - --enable-cache=false # specified - --hostname=${HOSTNAME} - --restore-dirs=${RESTORE_DIRECTORIES} - - --snapshots=${RESTORE_SNAPSHOTS=""} + - --snapshots=${RESTORE_SNAPSHOTS:=} - --output-dir=${outputDir} #- --metrics-enabled #- --metrics-pushgateway-url @@ -67,3 +85,5 @@ spec: mountPath: ${TARGET_MOUNT_PATH} - name: ${secretVolume} mountPath: /etc/repository/secret + - name: ${outputVolume} + mountPath: ${outputDir} diff --git a/docs/examples/pvc/rbac.yaml b/docs/examples/pvc/rbac.yaml new file mode 100644 index 000000000..4f582be15 --- /dev/null +++ b/docs/examples/pvc/rbac.yaml @@ -0,0 +1,41 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: update-status-roles + labels: + app: stash +rules: +- apiGroups: + - stash.appscode.com + resources: + - "*" + verbs: ["*"] +- apiGroups: + - "" + resources: + - "events" + verbs: ["create"] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: pvc-backup-restore + namespace: demo + labels: + app: stash +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: update-status-binding + namespace: demo + labels: + app: stash +roleRef: + kind: ClusterRole + apiGroup: rbac.authorization.k8s.io + name: update-status-roles +subjects: +- kind: ServiceAccount + name: pvc-backup-restore + namespace: demo \ No newline at end of file diff --git a/docs/examples/pvc/restore-error.yaml b/docs/examples/pvc/restore-error.yaml new file mode 100644 index 000000000..acd11cf82 --- /dev/null +++ b/docs/examples/pvc/restore-error.yaml @@ -0,0 +1,24 @@ +apiVersion: stash.appscode.com/v1beta1 +kind: RestoreSession +metadata: + name: pvc-restore-02 + namespace: demo +spec: + runtimeSettings: + pod: + serviceAccountName: pvc-backup-restore + task: + name: pvc-restore-task + repository: + name: hello-repo + target: + ref: + apiVersion: v1 + kind: PersistentVolumeClaim + name: test-pvc-dest + mountPath: /etc/target + rules: + - hosts: + - demo-host + paths: + - /etc/target/unknown-path diff --git a/docs/examples/pvc/restore.yaml b/docs/examples/pvc/restore.yaml index 903aeb538..bc0827ee4 100644 --- a/docs/examples/pvc/restore.yaml +++ b/docs/examples/pvc/restore.yaml @@ -4,6 +4,9 @@ metadata: name: pvc-restore-01 namespace: demo spec: + runtimeSettings: + pod: + serviceAccountName: pvc-backup-restore task: name: pvc-restore-task repository: diff --git a/docs/examples/pvc/tasks.yaml b/docs/examples/pvc/tasks.yaml index 854d086e5..9823ee0be 100644 --- a/docs/examples/pvc/tasks.yaml +++ b/docs/examples/pvc/tasks.yaml @@ -14,6 +14,14 @@ spec: value: target-volume # specified - name: secretVolume value: secret-volume # specified + - name: outputVolume + value: output-volume # specified + - name: update-status + params: + - name: outputDir + value: /etc/backup # specified + - name: outputVolume + value: output-volume # specified volumes: - name: target-volume persistentVolumeClaim: @@ -23,6 +31,8 @@ spec: secretName: ${REPOSITORY_SECRET_NAME} - name: scratch-volume emptyDir: {} + - name: output-volume + emptyDir: {} --- apiVersion: stash.appscode.com/v1beta1 kind: Task @@ -40,6 +50,14 @@ spec: value: target-volume # specified - name: secretVolume value: secret-volume # specified + - name: outputVolume + value: output-volume # specified + - name: update-status + params: + - name: outputDir + value: /etc/backup # specified + - name: outputVolume + value: output-volume # specified volumes: - name: target-volume persistentVolumeClaim: @@ -48,4 +66,6 @@ spec: secret: secretName: ${REPOSITORY_SECRET_NAME} - name: scratch-volume - emptyDir: {} \ No newline at end of file + emptyDir: {} + - name: output-volume + emptyDir: {} diff --git a/docs/guides/backup_restore_pvc.md b/docs/guides/backup_restore_pvc.md index db9451d7c..d38d2c65a 100644 --- a/docs/guides/backup_restore_pvc.md +++ b/docs/guides/backup_restore_pvc.md @@ -29,16 +29,19 @@ $ kubectl create ns demo namespace/demo created ``` ->Note: YAML files used in this tutorial are stored in [/docs/examples/pvc](/docs/examples/pvc) directory of [appscode/stash](https://github.com/appscode/stash) repository. +>Note: YAML files used in this tutorial are stored in [./docs/examples/pvc](./docs/examples/pvc) directory of [appscode/stash](https://github.com/appscode/stash) repository. + +## Create Service Account + +```bash +$ kubectl apply -f ./docs/examples/pvc/rbac.yaml +``` ## Create Functions and Tasks ```bash -$ kubectl apply -f /docs/examples/pvc/functions.yaml; kubectl apply -f /docs/examples/pvc/tasks.yaml -function.stash.appscode.com/pvc-backup created -task.stash.appscode.com/pvc-backup-task created -function.stash.appscode.com/pvc-restore created -task.stash.appscode.com/pvc-restore-task created +$ kubectl apply -f ./docs/examples/pvc/functions.yaml +$ kubectl apply -f ./docs/examples/pvc/tasks.yaml ``` ## Create Repository @@ -49,7 +52,7 @@ secret/gcs-secret created ``` ```bash -$ kubectl apply -f /docs/examples/pvc/repo.yaml +$ kubectl apply -f ./docs/examples/pvc/repo.yaml repository.stash.appscode.com/hello-repo created ``` @@ -58,7 +61,7 @@ repository.stash.appscode.com/hello-repo created Create Persistent Volume and two separate PVC for source and destination. ```bash -$ kubectl apply -f /docs/examples/pvc/pvc.yaml +$ kubectl apply -f ./docs/examples/pvc/pvc.yaml persistentvolume/test-pv-volume created persistentvolumeclaim/test-pvc-source created persistentvolumeclaim/test-pvc-dest created @@ -67,34 +70,80 @@ persistentvolumeclaim/test-pvc-dest created ## Write to Source PVC ```bash -$ kubectl apply -f /docs/examples/pvc/write.yaml +$ kubectl apply -f ./docs/examples/pvc/write.yaml pod/test-write-soruce created ``` ## Backup Source PVC ```bash -$ kubectl apply -f /docs/examples/pvc/backup.yaml +$ kubectl apply -f ./docs/examples/pvc/backup.yaml backupconfiguration.stash.appscode.com/pvc-backup-config created backupsession.stash.appscode.com/pvc-backup-01 created ``` +## Check Backup Session Status + +```yaml + status: + phase: Succeeded + stats: + - directory: /etc/target/dir-01 + fileStats: + modifiedFiles: 0 + newFiles: 0 + totalFiles: 0 + unmodifiedFiles: 0 + processingTime: 0m7s + size: 0 B + snapshot: f5268c39 + uploaded: 336 B + - directory: /etc/target/dir-02 + fileStats: + modifiedFiles: 0 + newFiles: 0 + totalFiles: 0 + unmodifiedFiles: 0 + processingTime: 0m7s + size: 0 B + snapshot: e6277c72 + uploaded: 336 B +``` + +## Check Repository Status + +```yaml + status: + integrity: true + size: 5.913 KiB + snapshotCount: 10 + snapshotRemovedOnLastCleanup: 2 +``` + ## Restore to Destination PVC ```bash -$ kubectl apply -f /docs/examples/pvc/restore.yaml +$ kubectl apply -f ./docs/examples/pvc/restore.yaml restoresession.stash.appscode.com/pvc-restore-01 created ``` +## Check Backup Session Status + +```yaml + status: + duration: 31.229339267s + phase: Succeeded +``` + ## Check Destination PVC ```bash -$ kubectl apply -f /docs/examples/pvc/check.yaml +$ kubectl apply -f ./docs/examples/pvc/check.yaml pod/test-check-dest created ``` ```bash -$ kc logs -n demo -f test-check-dest +$ kubectl logs -n demo -f test-check-dest + ls /etc/target dir-01 dir-02 @@ -105,6 +154,6 @@ dir-02 To cleanup the Kubernetes resources created by this tutorial, run: ```console -$ kubectl delete -f /docs/examples/pvc +$ kubectl delete -f ./docs/examples/pvc $ kubectl delete namespace demo ``` diff --git a/glide.lock b/glide.lock index ba7826aa5..991891b34 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 134c933c03f7084d90961edc2ccd2c421bbab0f0f72e6ec0003f1d087e75d169 -updated: 2019-03-20T10:55:11.971962007+06:00 +hash: 45e3485dc7cb339382b2522acc137ed71e6c1cf1d4808448cca3e97bce9ca68f +updated: 2019-03-21T12:02:31.07046905+06:00 imports: - name: bitbucket.org/ww/goautoneg version: 75cd24fc2f2c2a2088577d12123ddee5f54e0675 @@ -495,7 +495,7 @@ imports: subpackages: - rate - name: gomodules.xyz/envsubst - version: 0c160502a3a0fe54f555c149265e031f4fa2beab + version: c745d52104afee8575a85f1ceb419e1abe913657 subpackages: - parse - path @@ -1096,6 +1096,4 @@ imports: - runtime/serializer/versioning - name: sigs.k8s.io/yaml version: fd68e9863619f6ec2fdd8625fe1f02e7c877e480 -testImports: -- name: github.com/gomodules/envsubst - version: 0c160502a3a0fe54f555c149265e031f4fa2beab +testImports: [] diff --git a/hack/dev/run.sh b/hack/dev/run.sh index 86c6fd312..c595ea905 100755 --- a/hack/dev/run.sh +++ b/hack/dev/run.sh @@ -126,5 +126,6 @@ if [ "$STASH_E2E_TEST" = false ]; then # don't run operator while run this scrip --docker-registry="$STASH_DOCKER_REGISTRY" \ --image-tag="$STASH_IMAGE_TAG" \ --rbac=true + --enable-status-subresource=true fi popd diff --git a/pkg/cmds/backup_pvc.go b/pkg/cmds/backup_pvc.go index b2b06e082..522a67f3d 100644 --- a/pkg/cmds/backup_pvc.go +++ b/pkg/cmds/backup_pvc.go @@ -4,6 +4,7 @@ import ( "path/filepath" "github.com/appscode/go/flags" + "github.com/appscode/go/log" "github.com/appscode/stash/pkg/restic" "github.com/spf13/cobra" "k8s.io/apimachinery/pkg/util/errors" @@ -36,7 +37,7 @@ func NewCmdBackupPVC() *cobra.Command { // init restic wrapper resticWrapper, err := restic.NewResticWrapper(setupOpt) if err != nil { - return err + return handleResticError(outputDir, restic.DefaultOutputFileName, err) } // Run backup backupOutput, backupErr := resticWrapper.RunBackup(&backupOpt) @@ -44,17 +45,17 @@ func NewCmdBackupPVC() *cobra.Command { if metrics.Enabled { err := backupOutput.HandleMetrics(&metrics, backupErr) if err != nil { - return errors.NewAggregate([]error{backupErr, err}) + return handleResticError(outputDir, restic.DefaultOutputFileName, errors.NewAggregate([]error{backupErr, err})) } } + if backupErr != nil { + return handleResticError(outputDir, restic.DefaultOutputFileName, backupErr) + } // If output directory specified, then write the output in "output.json" file in the specified directory - if backupErr == nil && outputDir != "" { - err := backupOutput.WriteOutput(filepath.Join(outputDir, restic.DefaultOutputFileName)) - if err != nil { - return err - } + if outputDir != "" { + return backupOutput.WriteOutput(filepath.Join(outputDir, restic.DefaultOutputFileName)) } - return backupErr + return nil }, } @@ -88,3 +89,13 @@ func NewCmdBackupPVC() *cobra.Command { return cmd } + +// works for both backup and restore output +func handleResticError(outputDir, fileName string, backupErr error) error { + if outputDir == "" || fileName == "" { + return backupErr + } + log.Infoln("Writing restic error to output file, error:", backupErr.Error()) + backupOut := restic.BackupOutput{Error: backupErr.Error()} + return backupOut.WriteOutput(filepath.Join(outputDir, fileName)) +} diff --git a/pkg/cmds/restore_pvc.go b/pkg/cmds/restore_pvc.go index c3dcc1d13..f4f1b87e8 100644 --- a/pkg/cmds/restore_pvc.go +++ b/pkg/cmds/restore_pvc.go @@ -35,7 +35,7 @@ func NewCmdRestorePVC() *cobra.Command { // init restic wrapper resticWrapper, err := restic.NewResticWrapper(setupOpt) if err != nil { - return err + return handleResticError(outputDir, restic.DefaultOutputFileName, err) } // Run restore restoreOutput, restoreErr := resticWrapper.RunRestore(restoreOpt) @@ -43,17 +43,17 @@ func NewCmdRestorePVC() *cobra.Command { if metrics.Enabled { err := restoreOutput.HandleMetrics(&metrics, restoreErr) if err != nil { - return errors.NewAggregate([]error{restoreErr, err}) + return handleResticError(outputDir, restic.DefaultOutputFileName, errors.NewAggregate([]error{restoreErr, err})) } } + if restoreErr != nil { + return handleResticError(outputDir, restic.DefaultOutputFileName, restoreErr) + } // If output directory specified, then write the output in "output.json" file in the specified directory - if restoreErr == nil && outputDir != "" { - err := restoreOutput.WriteOutput(filepath.Join(outputDir, restic.DefaultOutputFileName)) - if err != nil { - return err - } + if outputDir != "" { + return restoreOutput.WriteOutput(filepath.Join(outputDir, restic.DefaultOutputFileName)) } - return restoreErr + return nil }, } diff --git a/pkg/cmds/root.go b/pkg/cmds/root.go index 64c2c64d8..4bc771d41 100644 --- a/pkg/cmds/root.go +++ b/pkg/cmds/root.go @@ -45,6 +45,7 @@ func NewRootCmd() *cobra.Command { rootCmd.AddCommand(NewCmdBackup()) rootCmd.AddCommand(NewCmdBackupPVC()) rootCmd.AddCommand(NewCmdRestorePVC()) + rootCmd.AddCommand(NewCmdUpdateStatus()) rootCmd.AddCommand(NewCmdRecover()) rootCmd.AddCommand(NewCmdCheck()) rootCmd.AddCommand(NewCmdScaleDown()) diff --git a/pkg/cmds/update_status.go b/pkg/cmds/update_status.go new file mode 100644 index 000000000..16ee4c2a9 --- /dev/null +++ b/pkg/cmds/update_status.go @@ -0,0 +1,64 @@ +package cmds + +import ( + "fmt" + + "github.com/appscode/go/flags" + cs "github.com/appscode/stash/client/clientset/versioned" + "github.com/appscode/stash/pkg/restic" + "github.com/appscode/stash/pkg/status" + "github.com/spf13/cobra" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +func NewCmdUpdateStatus() *cobra.Command { + var ( + masterURL string + kubeconfigPath string + opt = status.UpdateStatusOptions{ + OutputFileName: restic.DefaultOutputFileName, + } + ) + + cmd := &cobra.Command{ + Use: "update-status", + Short: "Update status of Repository, Backup/Restore Session", + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + flags.EnsureRequiredFlags(cmd, "namespace", "output-dir") + + config, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfigPath) + if err != nil { + return err + } + opt.KubeClient, err = kubernetes.NewForConfig(config) + if err != nil { + return err + } + opt.StashClient, err = cs.NewForConfig(config) + if err != nil { + return err + } + + if opt.BackupSession != "" { + return opt.UpdateBackupStatusFromFile() + } + if opt.RestoreSession != "" { + return opt.UpdateRestoreStatusFromFile() + } + return fmt.Errorf("backup-session or, restore-session not specified") + }, + } + + cmd.Flags().StringVar(&masterURL, "master", masterURL, "The address of the Kubernetes API server (overrides any value in kubeconfig)") + cmd.Flags().StringVar(&kubeconfigPath, "kubeconfig", kubeconfigPath, "Path to kubeconfig file with authorization information (the master location is set by the master flag).") + + cmd.Flags().StringVar(&opt.Namespace, "namespace", "default", "Namespace of Backup/Restore Session") + cmd.Flags().StringVar(&opt.Repository, "repository", opt.Repository, "Name of the Repository") + cmd.Flags().StringVar(&opt.BackupSession, "backup-session", opt.BackupSession, "Name of the Backup Session") + cmd.Flags().StringVar(&opt.RestoreSession, "restore-session", opt.RestoreSession, "Name of the Restore Session") + cmd.Flags().StringVar(&opt.OutputDir, "output-dir", opt.OutputDir, "Directory where output.json file will be written (keep empty if you don't need to write output in file)") + + return cmd +} diff --git a/pkg/controller/backup_session.go b/pkg/controller/backup_session.go index a0817c6d1..498e10575 100644 --- a/pkg/controller/backup_session.go +++ b/pkg/controller/backup_session.go @@ -28,11 +28,7 @@ import ( ) const ( - BackupJobPrefix = "stash-backup-" - BackupSessionEventComponent = "stash-backup-session" - EventReasonInvalidBackupSession = "InvalidBackupSession" - EventReasonBackupSessionFailed = "BackupSessionFailedToExecute" - EventReasonBackupSessionJobCreated = "BackupSessionJobCreated" + BackupJobPrefix = "stash-backup-" ) func (c *StashController) NewBackupSessionWebhook() hooks.AdmissionHook { @@ -69,7 +65,7 @@ func (c *StashController) initBackupSessionWatcher() { AddFunc: func(obj interface{}) { if r, ok := obj.(*api.BackupSession); ok { if err := r.IsValid(); err != nil { - eventer.CreateEvent(c.kubeClient, BackupSessionEventComponent, r, core.EventTypeWarning, EventReasonInvalidBackupSession, err.Error()) + eventer.CreateEvent(c.kubeClient, eventer.BackupSessionEventComponent, r, core.EventTypeWarning, eventer.EventReasonInvalidBackupSession, err.Error()) return } queue.Enqueue(c.backupSessionQueue.GetQueue(), obj) @@ -96,7 +92,7 @@ func (c *StashController) runBackupSessionInjector(key string) error { job, err := c.executeBackupSession(backupSession) if err != nil { log.Errorln(err) - eventer.CreateEvent(c.kubeClient, BackupSessionEventComponent, backupSession, core.EventTypeWarning, EventReasonBackupSessionFailed, err.Error()) + eventer.CreateEvent(c.kubeClient, eventer.BackupSessionEventComponent, backupSession, core.EventTypeWarning, eventer.EventReasonBackupSessionFailed, err.Error()) stash_util.UpdateBackupSessionStatus(c.stashClient.StashV1beta1(), backupSession, func(in *api.BackupSessionStatus) *api.BackupSessionStatus { in.Phase = api.BackupSessionFailed return in @@ -104,7 +100,7 @@ func (c *StashController) runBackupSessionInjector(key string) error { return err } if job != nil { // job successfully created - eventer.CreateEvent(c.kubeClient, BackupSessionEventComponent, backupSession, core.EventTypeNormal, EventReasonBackupSessionJobCreated, fmt.Sprintf("backup job %s created", job.Name)) + eventer.CreateEvent(c.kubeClient, eventer.BackupSessionEventComponent, backupSession, core.EventTypeNormal, eventer.EventReasonBackupSessionJobCreated, fmt.Sprintf("backup job %s created", job.Name)) stash_util.UpdateBackupSessionStatus(c.stashClient.StashV1beta1(), backupSession, func(in *api.BackupSessionStatus) *api.BackupSessionStatus { in.Phase = api.BackupSessionRunning return in @@ -141,6 +137,9 @@ func (c *StashController) executeBackupSession(backupSession *api.BackupSession) if err != nil { return nil, fmt.Errorf("cannot resolve implicit inputs for BackupConfiguration %s/%s, reason: %s", backupConfig.Namespace, backupConfig.Name, err) } + implicitInputs[apis.Namespace] = backupSession.Namespace + implicitInputs[apis.BackupSession] = backupSession.Name + implicitInputs[apis.StatusSubresourceEnabled] = fmt.Sprint(apis.EnableStatusSubresource) taskResolver := resolve.TaskResolver{ StashClient: c.stashClient, diff --git a/pkg/controller/inputs.go b/pkg/controller/inputs.go index 47ca30dd3..bc6250a4a 100644 --- a/pkg/controller/inputs.go +++ b/pkg/controller/inputs.go @@ -77,6 +77,9 @@ func (c *StashController) inputsForRepository(repository *apiAlpha.Repository) ( if repository == nil { return } + if repository.Name != "" { + inputs[apis.RepositoryName] = repository.Name + } if inputs[apis.RepositoryProvider], err = util.GetProvider(repository.Spec.Backend); err != nil { return } diff --git a/pkg/controller/restore_session.go b/pkg/controller/restore_session.go index 646753dd4..19c6cbc5a 100644 --- a/pkg/controller/restore_session.go +++ b/pkg/controller/restore_session.go @@ -28,11 +28,7 @@ import ( ) const ( - RestoreJobPrefix = "stash-restore-" - RestoreSessionEventComponent = "stash-restore-session" - EventReasonInvalidRestoreSession = "InvalidRestoreSession" - EventReasonRestoreSessionFailed = "RestoreSessionFailedToExecute" - EventReasonRestoreSessionJobCreated = "RestoreSessionJobCreated" + RestoreJobPrefix = "stash-restore-" ) func (c *StashController) NewRestoreSessionWebhook() hooks.AdmissionHook { @@ -69,7 +65,7 @@ func (c *StashController) initRestoreSessionWatcher() { AddFunc: func(obj interface{}) { if r, ok := obj.(*api.RestoreSession); ok { if err := r.IsValid(); err != nil { - eventer.CreateEvent(c.kubeClient, RestoreSessionEventComponent, r, core.EventTypeWarning, EventReasonInvalidRestoreSession, err.Error()) + eventer.CreateEvent(c.kubeClient, eventer.RestoreSessionEventComponent, r, core.EventTypeWarning, eventer.EventReasonInvalidRestoreSession, err.Error()) return } queue.Enqueue(c.restoreSessionQueue.GetQueue(), obj) @@ -96,7 +92,7 @@ func (c *StashController) runRestoreSessionInjector(key string) error { job, err := c.executeRestoreSession(restoreSession) if err != nil { log.Errorln(err) - eventer.CreateEvent(c.kubeClient, RestoreSessionEventComponent, restoreSession, core.EventTypeWarning, EventReasonRestoreSessionFailed, err.Error()) + eventer.CreateEvent(c.kubeClient, eventer.RestoreSessionEventComponent, restoreSession, core.EventTypeWarning, eventer.EventReasonRestoreSessionFailed, err.Error()) stash_util.UpdateRestoreSessionStatus(c.stashClient.StashV1beta1(), restoreSession, func(in *api.RestoreSessionStatus) *api.RestoreSessionStatus { in.Phase = api.RestoreFailed return in @@ -104,7 +100,7 @@ func (c *StashController) runRestoreSessionInjector(key string) error { return err } if job != nil { // job successfully created - eventer.CreateEvent(c.kubeClient, RestoreSessionEventComponent, restoreSession, core.EventTypeNormal, EventReasonRestoreSessionJobCreated, fmt.Sprintf("restore job %s created", job.Name)) + eventer.CreateEvent(c.kubeClient, eventer.RestoreSessionEventComponent, restoreSession, core.EventTypeNormal, eventer.EventReasonRestoreSessionJobCreated, fmt.Sprintf("restore job %s created", job.Name)) stash_util.UpdateRestoreSessionStatus(c.stashClient.StashV1beta1(), restoreSession, func(in *api.RestoreSessionStatus) *api.RestoreSessionStatus { in.Phase = api.RestoreRunning return in @@ -133,6 +129,9 @@ func (c *StashController) executeRestoreSession(restoreSession *api.RestoreSessi if err != nil { return nil, fmt.Errorf("cannot resolve implicit inputs for RestoreSession %s/%s, reason: %s", restoreSession.Namespace, restoreSession.Name, err) } + implicitInputs[apis.Namespace] = restoreSession.Namespace + implicitInputs[apis.RestoreSession] = restoreSession.Name + implicitInputs[apis.StatusSubresourceEnabled] = fmt.Sprint(apis.EnableStatusSubresource) taskResolver := resolve.TaskResolver{ StashClient: c.stashClient, diff --git a/pkg/eventer/recorder.go b/pkg/eventer/recorder.go index 94ffb1801..de2b5a112 100644 --- a/pkg/eventer/recorder.go +++ b/pkg/eventer/recorder.go @@ -15,26 +15,38 @@ import ( ) const ( - EventReasonInvalidRestic = "InvalidRestic" - EventReasonInvalidRecovery = "InvalidRecovery" - EventReasonInvalidCronExpression = "InvalidCronExpression" - EventReasonSuccessfulCronExpressionReset = "SuccessfulCronExpressionReset" - EventReasonSuccessfulBackup = "SuccessfulBackup" - EventReasonFailedToBackup = "FailedBackup" - EventReasonSuccessfulRecovery = "SuccessfulRecovery" - EventReasonFailedToRecover = "FailedRecovery" - EventReasonSuccessfulCheck = "SuccessfulCheck" - EventReasonFailedToCheck = "FailedCheck" - EventReasonFailedToRetention = "FailedRetention" - EventReasonFailedToUpdate = "FailedUpdateBackup" - EventReasonFailedCronJob = "FailedCronJob" - EventReasonFailedToDelete = "FailedDelete" - EventReasonJobCreated = "RecoveryJobCreated" - EventReasonJobFailedToCreate = "RecoveryJobFailedToCreate" - EventReasonCheckJobCreated = "CheckJobCreated" - EventReasonFailedSetup = "SetupFailed" - EventReasonAdmissionWebhookNotActivated string = "AdmissionWebhookNotActivated" - EventReasonInvalidBackupConfiguration = "InvalidBackupConfiguration" + EventReasonInvalidRestic = "InvalidRestic" + EventReasonInvalidRecovery = "InvalidRecovery" + EventReasonInvalidCronExpression = "InvalidCronExpression" + EventReasonSuccessfulCronExpressionReset = "SuccessfulCronExpressionReset" + EventReasonSuccessfulBackup = "SuccessfulBackup" + EventReasonFailedToBackup = "FailedBackup" + EventReasonSuccessfulRecovery = "SuccessfulRecovery" + EventReasonFailedToRecover = "FailedRecovery" + EventReasonSuccessfulCheck = "SuccessfulCheck" + EventReasonFailedToCheck = "FailedCheck" + EventReasonFailedToRetention = "FailedRetention" + EventReasonFailedToUpdate = "FailedUpdateBackup" + EventReasonFailedCronJob = "FailedCronJob" + EventReasonFailedToDelete = "FailedDelete" + EventReasonJobCreated = "RecoveryJobCreated" + EventReasonJobFailedToCreate = "RecoveryJobFailedToCreate" + EventReasonCheckJobCreated = "CheckJobCreated" + EventReasonFailedSetup = "SetupFailed" + EventReasonAdmissionWebhookNotActivated = "AdmissionWebhookNotActivated" + EventReasonInvalidBackupConfiguration = "InvalidBackupConfiguration" + + BackupSessionEventComponent = "stash-backup-session" + EventReasonInvalidBackupSession = "InvalidBackupSession" + EventReasonBackupSessionSucceeded = "BackupSessionSucceeded" + EventReasonBackupSessionFailed = "BackupSessionFailedToExecute" + EventReasonBackupSessionJobCreated = "BackupSessionJobCreated" + + RestoreSessionEventComponent = "stash-restore-session" + EventReasonInvalidRestoreSession = "InvalidRestoreSession" + EventReasonRestoreSessionSucceeded = "RestoreSessionSucceeded" + EventReasonRestoreSessionFailed = "RestoreSessionFailedToExecute" + EventReasonRestoreSessionJobCreated = "RestoreSessionJobCreated" ) func NewEventRecorder(client kubernetes.Interface, component string) record.EventRecorder { diff --git a/pkg/resolve/task_test.go b/pkg/resolve/task_test.go index e2934c5f0..b5eed3ca5 100644 --- a/pkg/resolve/task_test.go +++ b/pkg/resolve/task_test.go @@ -13,7 +13,8 @@ func TestResolveWithInputs(t *testing.T) { Args: []string{ "arg", "--p1=${p1}", - "--p2=${p2=d2}", + "--p2=${p2:=d2}", + "--p3=${p3:=}", }, }, } @@ -24,6 +25,7 @@ func TestResolveWithInputs(t *testing.T) { if err != nil { t.Error(err) } + t.Log(function) function = v1beta1.Function{ Spec: v1beta1.FunctionSpec{ diff --git a/pkg/restic/output.go b/pkg/restic/output.go index 8e61980d9..491fb651a 100644 --- a/pkg/restic/output.go +++ b/pkg/restic/output.go @@ -22,6 +22,8 @@ type BackupOutput struct { SessionDuration string `json:"sessionDuration,omitempty"` // RepositoryStats shows statistics of repository after last backup RepositoryStats RepositoryStats `json:"repository,omitempty"` + // string value of backup error + Error string `json:"error,omitempty"` } type RepositoryStats struct { @@ -38,6 +40,8 @@ type RepositoryStats struct { type RestoreOutput struct { // SessionDuration show total time taken to complete the restore session SessionDuration string `json:"sessionDuration,omitempty"` + // string value of restore error + Error string `json:"error,omitempty"` } // WriteOutput write output of backup process into output.json file in the directory @@ -71,7 +75,6 @@ func (out *RestoreOutput) WriteOutput(fileName string) error { } func ReadBackupOutput(filename string) (*BackupOutput, error) { - data, err := ioutil.ReadFile(filename) if err != nil { return nil, err @@ -87,7 +90,6 @@ func ReadBackupOutput(filename string) (*BackupOutput, error) { } func ReadRestoreOutput(filename string) (*RestoreOutput, error) { - data, err := ioutil.ReadFile(filename) if err != nil { return nil, err diff --git a/pkg/status/status.go b/pkg/status/status.go new file mode 100644 index 000000000..df1f8b347 --- /dev/null +++ b/pkg/status/status.go @@ -0,0 +1,177 @@ +package status + +import ( + "fmt" + "path/filepath" + + "github.com/appscode/stash/apis" + api "github.com/appscode/stash/apis/stash/v1alpha1" + api_v1beta1 "github.com/appscode/stash/apis/stash/v1beta1" + cs "github.com/appscode/stash/client/clientset/versioned" + stash_util "github.com/appscode/stash/client/clientset/versioned/typed/stash/v1alpha1/util" + stash_util_v1beta1 "github.com/appscode/stash/client/clientset/versioned/typed/stash/v1beta1/util" + "github.com/appscode/stash/pkg/eventer" + "github.com/appscode/stash/pkg/restic" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/pkg/apis/core" +) + +type UpdateStatusOptions struct { + KubeClient kubernetes.Interface + StashClient *cs.Clientset + + Namespace string + Repository string + BackupSession string + RestoreSession string + OutputDir string + OutputFileName string +} + +func (o UpdateStatusOptions) UpdateBackupStatusFromFile() error { + // read backup output from file + backupOutput, err := restic.ReadBackupOutput(filepath.Join(o.OutputDir, o.OutputFileName)) + if err != nil { + return err + } + var backupErr error + if backupOutput.Error != "" { + backupErr = fmt.Errorf(backupOutput.Error) + } + updateStatusErr := o.UpdatePostBackupStatus(backupOutput) + return errors.NewAggregate([]error{backupErr, updateStatusErr}) +} + +func (o UpdateStatusOptions) UpdateRestoreStatusFromFile() error { + // read restore output from file + restoreOutput, err := restic.ReadRestoreOutput(filepath.Join(o.OutputDir, o.OutputFileName)) + if err != nil { + return err + } + var restoreErr error + if restoreOutput.Error != "" { + restoreErr = fmt.Errorf(restoreOutput.Error) + } + updateStatusErr := o.UpdatePostRestoreStatus(restoreOutput) + return errors.NewAggregate([]error{restoreErr, updateStatusErr}) +} + +func (o UpdateStatusOptions) UpdatePostBackupStatus(backupOutput *restic.BackupOutput) error { + // get backup session, update status and create event + backupSession, err := o.StashClient.StashV1beta1().BackupSessions(o.Namespace).Get(o.BackupSession, metav1.GetOptions{}) + if err != nil { + return err + } + _, err = stash_util_v1beta1.UpdateBackupSessionStatus( + o.StashClient.StashV1beta1(), + backupSession, + func(in *api_v1beta1.BackupSessionStatus) *api_v1beta1.BackupSessionStatus { + if backupOutput.Error != "" { + in.Phase = api_v1beta1.BackupSessionFailed + } else { + in.Phase = api_v1beta1.BackupSessionSucceeded + in.Stats = backupOutput.BackupStats + } + return in + }, + apis.EnableStatusSubresource, + ) + if err != nil { + return err + } + + // create event for backup session + var eventType, eventReason, eventMessage string + if backupOutput.Error != "" { + eventType = core.EventTypeWarning + eventReason = eventer.EventReasonBackupSessionFailed + eventMessage = fmt.Sprintf("backup session failed, reason: %s", backupOutput.Error) + } else { + eventType = core.EventTypeNormal + eventReason = eventer.EventReasonBackupSessionSucceeded + eventMessage = "backup session succeeded" + } + _, err = eventer.CreateEvent( + o.KubeClient, + eventer.BackupSessionEventComponent, + backupSession, + eventType, + eventReason, + eventMessage, + ) + if err != nil { + return err + } + + // no need to update repository status for failed backup + if backupOutput.Error != "" { + return nil + } + // get repository and update status + repository, err := o.StashClient.StashV1alpha1().Repositories(o.Namespace).Get(o.Repository, metav1.GetOptions{}) + if err != nil { + return err + } + _, err = stash_util.UpdateRepositoryStatus( + o.StashClient.StashV1alpha1(), + repository, + func(in *api.RepositoryStatus) *api.RepositoryStatus { + // TODO: fix Restic Wrapper + in.Integrity = backupOutput.RepositoryStats.Integrity + in.Size = backupOutput.RepositoryStats.Size + in.SnapshotCount = backupOutput.RepositoryStats.SnapshotCount + in.SnapshotRemovedOnLastCleanup = backupOutput.RepositoryStats.SnapshotRemovedOnLastCleanup + return in + }, + apis.EnableStatusSubresource, + ) + return err +} + +func (o UpdateStatusOptions) UpdatePostRestoreStatus(restoreOutput *restic.RestoreOutput) error { + // get restore session, update status and create event + restoreSession, err := o.StashClient.StashV1beta1().RestoreSessions(o.Namespace).Get(o.RestoreSession, metav1.GetOptions{}) + if err != nil { + return err + } + _, err = stash_util_v1beta1.UpdateRestoreSessionStatus( + o.StashClient.StashV1beta1(), + restoreSession, + func(in *api_v1beta1.RestoreSessionStatus) *api_v1beta1.RestoreSessionStatus { + if restoreOutput.Error != "" { + in.Phase = api_v1beta1.RestoreFailed + } else { + in.Phase = api_v1beta1.RestoreSucceeded + } + in.Duration = restoreOutput.SessionDuration + return in + }, + apis.EnableStatusSubresource, + ) + if err != nil { + return err + } + + // create event for restore session + var eventType, eventReason, eventMessage string + if restoreOutput.Error != "" { + eventType = core.EventTypeWarning + eventReason = eventer.EventReasonRestoreSessionFailed + eventMessage = fmt.Sprintf("restore session failed, reason: %s", restoreOutput.Error) + } else { + eventType = core.EventTypeNormal + eventReason = eventer.EventReasonRestoreSessionSucceeded + eventMessage = "restore session succeeded" + } + _, err = eventer.CreateEvent( + o.KubeClient, + eventer.RestoreSessionEventComponent, + restoreSession, + eventType, + eventReason, + eventMessage, + ) + return err +} diff --git a/vendor/github.com/gomodules/envsubst/LICENSE b/vendor/github.com/gomodules/envsubst/LICENSE deleted file mode 100644 index 1de55b7f4..000000000 --- a/vendor/github.com/gomodules/envsubst/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2017 drone.io - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/vendor/github.com/gomodules/envsubst/eval.go b/vendor/github.com/gomodules/envsubst/eval.go deleted file mode 100644 index 19bccdd10..000000000 --- a/vendor/github.com/gomodules/envsubst/eval.go +++ /dev/null @@ -1,57 +0,0 @@ -package envsubst - -import "os" - -// Eval replaces ${var} in the string based on the mapping function. -func Eval(s string, mapping func(string) string) (string, error) { - t, err := Parse(s) - if err != nil { - return s, err - } - // convert mapping to match new mapper function - mapper := func(node string, key string, args []string) (string, []string, error) { - return mapping(key), args, nil - } - return t.Execute(mapper) -} - -// EvalEnv replaces ${var} in the string according to the values of the -// current environment variables. References to undefined variables are -// replaced by the empty string. -func EvalEnv(s string) (string, error) { - return Eval(s, os.Getenv) -} - -func EvalMap(s string, values map[string]string) (string, error) { - if values == nil { - values = make(map[string]string) - } - mapper := func(node string, key string, args []string) (string, []string, error) { - v, ok := values[key] - // return error if key not found and default not specified - if !ok && (!isDefault(node) || len(args) == 0) { - return "", nil, &valueNotFoundError{key} - } - // if key found, remove args for default - // so that empty value will not be replaced by default value - if ok && isDefault(node) { - return v, nil, nil - } - return v, args, nil - } - - t, err := Parse(s) - if err != nil { - return s, err - } - return t.Execute(mapper) -} - -func isDefault(name string) bool { - switch name { - case "=", ":=", ":-": - return true - default: - return false - } -} diff --git a/vendor/github.com/gomodules/envsubst/funcs.go b/vendor/github.com/gomodules/envsubst/funcs.go deleted file mode 100644 index 07cad8116..000000000 --- a/vendor/github.com/gomodules/envsubst/funcs.go +++ /dev/null @@ -1,228 +0,0 @@ -package envsubst - -import ( - "strconv" - "strings" - "unicode" - "unicode/utf8" - - "gomodules.xyz/envsubst/path" -) - -// defines a parameter substitution function. -type substituteFunc func(string, ...string) string - -// toLen returns the length of string s. -func toLen(s string, args ...string) string { - return strconv.Itoa(len(s)) -} - -// toLower returns a copy of the string s with all characters -// mapped to their lower case. -func toLower(s string, args ...string) string { - return strings.ToLower(s) -} - -// toUpper returns a copy of the string s with all characters -// mapped to their upper case. -func toUpper(s string, args ...string) string { - return strings.ToUpper(s) -} - -// toLowerFirst returns a copy of the string s with the first -// character mapped to its lower case. -func toLowerFirst(s string, args ...string) string { - if s == "" { - return s - } - r, n := utf8.DecodeRuneInString(s) - return string(unicode.ToLower(r)) + s[n:] -} - -// toUpperFirst returns a copy of the string s with the first -// character mapped to its upper case. -func toUpperFirst(s string, args ...string) string { - if s == "" { - return s - } - r, n := utf8.DecodeRuneInString(s) - return string(unicode.ToUpper(r)) + s[n:] -} - -// toDefault returns a copy of the string s if not empty, else -// returns a copy of the first string arugment. -func toDefault(s string, args ...string) string { - if len(s) == 0 && len(args) == 1 { - s = args[0] - } - return s -} - -// toSubstr returns a slice of the string s at the specified -// length and position. -func toSubstr(s string, args ...string) string { - if len(args) == 0 { - return s // should never happen - } - - pos, err := strconv.Atoi(args[0]) - if err != nil { - // bash returns the string if the position - // cannot be parsed. - return s - } - - if len(args) == 1 { - if pos < len(s) { - return s[pos:] - } - // if the position exceeds the length of the - // string an empty string is returned - return "" - } - - length, err := strconv.Atoi(args[1]) - if err != nil { - // bash returns the string if the length - // cannot be parsed. - return s - } - - if pos+length >= len(s) { - // if the position exceeds the length of the - // string just return the rest of it like bash - return s[pos:] - } - - return s[pos : pos+length] -} - -// replaceAll returns a copy of the string s with all instances -// of the substring replaced with the replacement string. -func replaceAll(s string, args ...string) string { - switch len(args) { - case 0: - return s - case 1: - return strings.Replace(s, args[0], "", -1) - default: - return strings.Replace(s, args[0], args[1], -1) - } -} - -// replaceFirst returns a copy of the string s with the first -// instance of the substring replaced with the replacement string. -func replaceFirst(s string, args ...string) string { - switch len(args) { - case 0: - return s - case 1: - return strings.Replace(s, args[0], "", 1) - default: - return strings.Replace(s, args[0], args[1], 1) - } -} - -// replacePrefix returns a copy of the string s with the matching -// prefix replaced with the replacement string. -func replacePrefix(s string, args ...string) string { - if len(args) != 2 { - return s - } - if strings.HasPrefix(s, args[0]) { - return strings.Replace(s, args[0], args[1], 1) - } - return s -} - -// replaceSuffix returns a copy of the string s with the matching -// suffix replaced with the replacement string. -func replaceSuffix(s string, args ...string) string { - if len(args) != 2 { - return s - } - if strings.HasSuffix(s, args[0]) { - s = strings.TrimSuffix(s, args[0]) - s = s + args[1] - } - return s -} - -// TODO - -func trimShortestPrefix(s string, args ...string) string { - if len(args) != 0 { - s = trimShortest(s, args[0]) - } - return s -} - -func trimShortestSuffix(s string, args ...string) string { - if len(args) != 0 { - r := reverse(s) - rarg := reverse(args[0]) - s = reverse(trimShortest(r, rarg)) - } - return s -} - -func trimLongestPrefix(s string, args ...string) string { - if len(args) != 0 { - s = trimLongest(s, args[0]) - } - return s -} - -func trimLongestSuffix(s string, args ...string) string { - if len(args) != 0 { - r := reverse(s) - rarg := reverse(args[0]) - s = reverse(trimLongest(r, rarg)) - } - return s -} - -func trimShortest(s, arg string) string { - var shortestMatch string - for i := 0; i < len(s); i++ { - match, err := path.Match(arg, s[0:len(s)-i]) - - if err != nil { - return s - } - - if match { - shortestMatch = s[0 : len(s)-i] - } - } - - if shortestMatch != "" { - return strings.TrimPrefix(s, shortestMatch) - } - - return s -} - -func trimLongest(s, arg string) string { - for i := 0; i < len(s); i++ { - match, err := path.Match(arg, s[0:len(s)-i]) - - if err != nil { - return s - } - - if match { - return strings.TrimPrefix(s, s[0:len(s)-i]) - } - } - - return s -} - -func reverse(s string) string { - r := []rune(s) - for i, j := 0, len(r)-1; i < len(r)/2; i, j = i+1, j-1 { - r[i], r[j] = r[j], r[i] - } - return string(r) -} diff --git a/vendor/github.com/gomodules/envsubst/template.go b/vendor/github.com/gomodules/envsubst/template.go deleted file mode 100644 index 0a26ce3f8..000000000 --- a/vendor/github.com/gomodules/envsubst/template.go +++ /dev/null @@ -1,180 +0,0 @@ -package envsubst - -import ( - "bytes" - "fmt" - "io" - "io/ioutil" - - "gomodules.xyz/envsubst/parse" -) - -type valueNotFoundError struct { - key string -} - -var _ error = &valueNotFoundError{} - -func (e *valueNotFoundError) Error() string { - return fmt.Sprintf("input/default value not found for key %s", e.key) -} - -func IsValueNotFoundError(v interface{}) bool { - switch v.(type) { - case *valueNotFoundError: - return true - } - return false -} - -// state represents the state of template execution. It is not part of the -// template so that multiple executions can run in parallel. -type state struct { - template *Template - writer io.Writer - node parse.Node // current node - - // maps variable names to values with additional behaviours - // returns value, args and error - mapper func(node string, key string, args []string) (string, []string, error) -} - -// Template is the representation of a parsed shell format string. -type Template struct { - tree *parse.Tree -} - -// Parse creates a new shell format template and parses the template -// definition from string s. -func Parse(s string) (t *Template, err error) { - t = new(Template) - t.tree, err = parse.Parse(s) - if err != nil { - return nil, err - } - return t, nil -} - -// ParseFile creates a new shell format template and parses the template -// definition from the named file. -func ParseFile(path string) (*Template, error) { - b, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - return Parse(string(b)) -} - -// Execute applies a parsed template to the specified data mapping. -func (t *Template) Execute(mapping func(node string, key string, args []string) (string, []string, error)) (str string, err error) { - b := new(bytes.Buffer) - s := new(state) - s.node = t.tree.Root - s.mapper = mapping - s.writer = b - err = t.eval(s) - if err != nil { - return - } - return b.String(), nil -} - -func (t *Template) eval(s *state) (err error) { - switch node := s.node.(type) { - case *parse.TextNode: - err = t.evalText(s, node) - case *parse.FuncNode: - err = t.evalFunc(s, node) - case *parse.ListNode: - err = t.evalList(s, node) - } - return err -} - -func (t *Template) evalText(s *state, node *parse.TextNode) error { - _, err := io.WriteString(s.writer, node.Value) - return err -} - -func (t *Template) evalList(s *state, node *parse.ListNode) (err error) { - for _, n := range node.Nodes { - s.node = n - err = t.eval(s) - if err != nil { - return err - } - } - return nil -} - -func (t *Template) evalFunc(s *state, node *parse.FuncNode) error { - var w = s.writer - var buf bytes.Buffer - var args []string - for _, n := range node.Args { - buf.Reset() - s.writer = &buf - s.node = n - err := t.eval(s) - if err != nil { - return err - } - args = append(args, buf.String()) - } - - // restore the origin writer - s.writer = w - s.node = node - - v, args, err := s.mapper(node.Name, node.Param, args) - if err != nil { - return err - } - - fn := lookupFunc(node.Name, len(args)) - - _, err = io.WriteString(s.writer, fn(v, args...)) - return err -} - -// lookupFunc returns the parameters substitution function by name. If the -// named function does not exists, a default function is returned. -func lookupFunc(name string, args int) substituteFunc { - switch name { - case ",": - return toLowerFirst - case ",,": - return toLower - case "^": - return toUpperFirst - case "^^": - return toUpper - case "#": - if args == 0 { - return toLen - } - return trimShortestPrefix - case "##": - return trimLongestPrefix - case "%": - return trimShortestSuffix - case "%%": - return trimLongestSuffix - case ":": - return toSubstr - case "/#": - return replacePrefix - case "/%": - return replaceSuffix - case "/": - return replaceFirst - case "//": - return replaceAll - case "=", ":=", ":-": - return toDefault - case ":?", ":+", "-", "+": - return toDefault - default: - return toDefault - } -} diff --git a/vendor/gomodules.xyz/envsubst/eval.go b/vendor/gomodules.xyz/envsubst/eval.go index 19bccdd10..6bc8e79dc 100644 --- a/vendor/gomodules.xyz/envsubst/eval.go +++ b/vendor/gomodules.xyz/envsubst/eval.go @@ -29,7 +29,7 @@ func EvalMap(s string, values map[string]string) (string, error) { mapper := func(node string, key string, args []string) (string, []string, error) { v, ok := values[key] // return error if key not found and default not specified - if !ok && (!isDefault(node) || len(args) == 0) { + if !ok && !isDefault(node) { return "", nil, &valueNotFoundError{key} } // if key found, remove args for default diff --git a/vendor/gomodules.xyz/envsubst/parse/parse.go b/vendor/gomodules.xyz/envsubst/parse/parse.go index 683b44d54..5dbecff93 100644 --- a/vendor/gomodules.xyz/envsubst/parse/parse.go +++ b/vendor/gomodules.xyz/envsubst/parse/parse.go @@ -293,6 +293,12 @@ func (t *Tree) parseDefaultFunc(name string) (Node, error) { return nil, ErrBadSubstitution } + // check for blank string + switch t.scanner.peek() { + case '}': + return node, t.consumeRbrack() + } + // scan arg[1] { param, err := t.parseParam(acceptNotClosing, scanIdent)