From 9dbfd9f2b8c5a0f9a4c764feb4775cc807973dcf Mon Sep 17 00:00:00 2001 From: Emruz Hossain Date: Wed, 13 Apr 2022 14:58:15 +0600 Subject: [PATCH] Finish implementing the plugin Signed-off-by: Emruz Hossain --- .github/.kodiak.toml | 18 ++ .github/workflows/cherry-pick.yml | 34 +++ .github/workflows/ci.yml | 42 +++ .github/workflows/release-tracker.yml | 36 +++ .github/workflows/release.yml | 50 ++++ Makefile | 2 +- README.md | 4 +- cmd/manifest-backup/main.go | 2 +- go.mod | 5 +- go.sum | 15 +- pkg/backup.go | 19 +- pkg/manager/application.go | 24 +- pkg/manager/generic_resources.go | 16 ++ pkg/manager/manager.go | 17 +- pkg/manager/manager_test.go | 26 +- pkg/manager/resources_processor.go | 16 ++ pkg/root.go | 4 +- pkg/sanitizers/metadata.go | 16 ++ pkg/sanitizers/pod.go | 16 ++ pkg/sanitizers/sanitizer.go | 16 ++ pkg/sanitizers/workload.go | 16 ++ pkg/utils.go | 11 +- .../client-go/api/v1/timeofday.go | 12 +- .../kmodules.xyz/client-go/meta/incluster.go | 2 +- vendor/modules.txt | 9 +- .../apis/stash/v1beta1/openapi_generated.go | 6 + .../apimachinery/apis/stash/v1beta1/types.go | 2 + .../stash.appscode.com_backupbatches.yaml | 4 + ...ash.appscode.com_backupconfigurations.yaml | 2 + .../stash.appscode.com_backupsessions.yaml | 2 + .../stash.appscode.com_restorebatches.yaml | 4 + .../stash.appscode.com_restoresessions.yaml | 2 + .../pkg/conditions/backup_invoker.go | 21 +- .../pkg/conditions/restore_invoker.go | 13 +- .../pkg/invoker/backup_invoker.go | 5 +- .../apimachinery/pkg/invoker/backupbatch.go | 14 +- .../pkg/invoker/backupconfiguration.go | 11 +- .../apimachinery/pkg/invoker/restorebatch.go | 9 +- .../pkg/invoker/restoresession.go | 2 +- .../apimachinery/pkg/util/addon.go | 4 +- .../elasticsearch/LICENSE.md | 5 - .../elasticsearch/pkg/backup.go | 243 ------------------ .../elasticsearch/pkg/restore.go | 220 ---------------- .../elasticsearch/pkg/root.go | 48 ---- .../elasticsearch/pkg/utils.go | 165 ------------ 45 files changed, 437 insertions(+), 773 deletions(-) create mode 100644 .github/.kodiak.toml create mode 100644 .github/workflows/cherry-pick.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/release-tracker.yml create mode 100644 .github/workflows/release.yml delete mode 100644 vendor/stash.appscode.dev/elasticsearch/LICENSE.md delete mode 100644 vendor/stash.appscode.dev/elasticsearch/pkg/backup.go delete mode 100644 vendor/stash.appscode.dev/elasticsearch/pkg/restore.go delete mode 100644 vendor/stash.appscode.dev/elasticsearch/pkg/root.go delete mode 100644 vendor/stash.appscode.dev/elasticsearch/pkg/utils.go diff --git a/.github/.kodiak.toml b/.github/.kodiak.toml new file mode 100644 index 00000000..ded81e43 --- /dev/null +++ b/.github/.kodiak.toml @@ -0,0 +1,18 @@ +version = 1 + +[merge] +method = "squash" # default: "merge" +delete_branch_on_merge = true # default: false +optimistic_updates = true # default: true +prioritize_ready_to_merge = true # default: false + +[merge.message] +title = "pull_request_title" # default: "github_default" +body = "github_default" # default: "github_default" +strip_html_comments = true # default: false + +[update] +always = true # default: false + +[approve] +auto_approve_usernames = ["1gtm", "tamalsaha"] diff --git a/.github/workflows/cherry-pick.yml b/.github/workflows/cherry-pick.yml new file mode 100644 index 00000000..d254244f --- /dev/null +++ b/.github/workflows/cherry-pick.yml @@ -0,0 +1,34 @@ +name: cherry-pick + +on: + push: + branches: + - master + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v1 + + - name: Prepare git + env: + GITHUB_USER: 1gtm + GITHUB_TOKEN: ${{ secrets.LGTM_GITHUB_TOKEN }} + run: | + git config --global user.name "${GITHUB_USER}" + git config --global user.email "${GITHUB_USER}@appscode.com" + git remote set-url origin https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git + + - name: Install GitHub CLI + run: | + curl -fsSL https://github.com/github/hub/raw/master/script/get | bash -s 2.14.1 + sudo mv bin/hub /usr/local/bin + + - name: Update release branches + env: + GITHUB_USER: 1gtm + GITHUB_TOKEN: ${{ secrets.LGTM_GITHUB_TOKEN }} + run: | + ./hack/scripts/cherry-pick.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..30758a6b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,42 @@ +name: CI + +on: + pull_request: + branches: + - "*" + push: + branches: + - master + +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - name: Set up Go 1.17 + uses: actions/setup-go@v1 + with: + go-version: 1.17 + id: go + + - uses: actions/checkout@v1 + + - name: Set up QEMU + id: qemu + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Run checks + run: | + make ci + + - name: Build + env: + REGISTRY: appscodeci + DOCKER_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + USERNAME: 1gtm + run: | + docker login --username ${USERNAME} --password ${DOCKER_TOKEN} + make push diff --git a/.github/workflows/release-tracker.yml b/.github/workflows/release-tracker.yml new file mode 100644 index 00000000..95308eaf --- /dev/null +++ b/.github/workflows/release-tracker.yml @@ -0,0 +1,36 @@ +name: release-tracker + +on: + pull_request: + types: [closed] + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v1 + + - name: Prepare git + env: + GITHUB_USER: 1gtm + GITHUB_TOKEN: ${{ secrets.LGTM_GITHUB_TOKEN }} + run: | + git config --global user.name "${GITHUB_USER}" + git config --global user.email "${GITHUB_USER}@appscode.com" + git remote set-url origin https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git + + - name: Install GitHub CLI + run: | + curl -fsSL https://github.com/github/hub/raw/master/script/get | bash -s 2.14.1 + sudo mv bin/hub /usr/local/bin + + - name: Update release tracker + if: | + github.event.action == 'closed' && + github.event.pull_request.merged == true + env: + GITHUB_USER: 1gtm + GITHUB_TOKEN: ${{ secrets.LGTM_GITHUB_TOKEN }} + run: | + ./hack/scripts/update-release-tracker.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..84370dbe --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,50 @@ +name: Release + +on: + push: + tags: + - "*.*" + +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v1 + + - name: Install GitHub CLI + run: | + curl -fsSL https://github.com/github/hub/raw/master/script/get | bash -s 2.14.1 + sudo mv bin/hub /usr/local/bin + + - name: Print version info + id: semver + run: | + make version + + - name: Set up QEMU + id: qemu + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Publish to GitHub Container Registry + env: + REGISTRY: ghcr.io/stashed + DOCKER_TOKEN: ${{ secrets.LGTM_GITHUB_TOKEN }} + USERNAME: 1gtm + APPSCODE_ENV: prod + run: | + docker login ghcr.io --username ${USERNAME} --password ${DOCKER_TOKEN} + make release + + - name: Publish to Docker Registry + env: + DOCKER_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + USERNAME: 1gtm + APPSCODE_ENV: prod + run: | + docker login --username ${USERNAME} --password ${DOCKER_TOKEN} + make release diff --git a/Makefile b/Makefile index c3bb92e0..2b93bbef 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ else endif endif -RESTIC_VER := 0.13.0 +RESTIC_VER := 0.13.1 ### ### These variables should not need tweaking. diff --git a/README.md b/README.md index 057572d8..b2f33c3e 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -[![Go Report Card](https://goreportcard.com/badge/stash.appscode.dev/manifest-backup)](https://goreportcard.com/report/stash.appscode.dev/elasticsearch) -![CI](https://github.com/stashed/elasticsearch/workflows/CI/badge.svg) +[![Go Report Card](https://goreportcard.com/badge/stash.appscode.dev/manifest-backup)](https://goreportcard.com/report/stash.appscode.dev/manifest-backup) +![CI](https://github.com/stashed/manifest-backup/workflows/CI/badge.svg) [![Docker Pulls](https://img.shields.io/docker/pulls/stashed/manifest-backup.svg)](https://hub.docker.com/r/stashed/manifest-backup/) [![Slack](https://shields.io/badge/Join_Slack-salck?color=4A154B&logo=slack)](https://slack.appscode.com) [![Twitter](https://img.shields.io/twitter/follow/kubestash.svg?style=social&logo=twitter&label=Follow)](https://twitter.com/intent/follow?screen_name=KubeStash) diff --git a/cmd/manifest-backup/main.go b/cmd/manifest-backup/main.go index f7963aea..79df2f04 100644 --- a/cmd/manifest-backup/main.go +++ b/cmd/manifest-backup/main.go @@ -21,7 +21,7 @@ import ( "runtime" _ "stash.appscode.dev/apimachinery/client/clientset/versioned/fake" - "stash.appscode.dev/elasticsearch/pkg" + "stash.appscode.dev/manifest-backup/pkg" "gomodules.xyz/logs" _ "k8s.io/client-go/kubernetes/fake" diff --git a/go.mod b/go.mod index 18d7b96a..a8b272b5 100644 --- a/go.mod +++ b/go.mod @@ -13,12 +13,11 @@ require ( k8s.io/apimachinery v0.21.1 k8s.io/client-go v0.21.1 k8s.io/klog/v2 v2.9.0 - kmodules.xyz/client-go v0.0.0-20220317213815-2a6d5a5784f2 + kmodules.xyz/client-go v0.0.0-20220404224906-af7b092cfac5 kmodules.xyz/custom-resources v0.0.0-20220317220154-7beb809b1f5e kmodules.xyz/offshoot-api v0.0.0-20220329041708-c076b2bcb0f8 sigs.k8s.io/yaml v1.3.0 - stash.appscode.dev/apimachinery v0.19.1-0.20220407092220-1fb00845d1f7 - stash.appscode.dev/elasticsearch v0.0.0-20220329183444-6b505bc9eb9f + stash.appscode.dev/apimachinery v0.19.1-0.20220413043008-5fa3647ef8f8 ) replace bitbucket.org/ww/goautoneg => gomodules.xyz/goautoneg v0.0.0-20120707110453-a547fc61f48d diff --git a/go.sum b/go.sum index a757e05e..e3cdab12 100644 --- a/go.sum +++ b/go.sum @@ -165,6 +165,7 @@ github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNE github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1-0.20220316001817-d5090ed65664/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= @@ -961,8 +962,9 @@ gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= gomodules.xyz/logs v0.0.6 h1:8+9Wkud5yBPtIvkVszubyTeFxNII30lWODom0+GZD8U= gomodules.xyz/logs v0.0.6/go.mod h1:Q+fFtZFLEB5q86KmDehXCGuMP72Rv+Rwz0KuVxK+Gi4= -gomodules.xyz/mergo v0.3.13-0.20210702100041-9d62ff8ece4d h1:i6r7bk2jF965W3xmORgz+KYhlH608oRvrAPpcNHorlo= gomodules.xyz/mergo v0.3.13-0.20210702100041-9d62ff8ece4d/go.mod h1:i2WNHvGpzLKI+/qWRhscddeashtzrtxCAucS2H7hrtM= +gomodules.xyz/mergo v0.3.13-0.20220214162359-48efe39fd402 h1:l67pCtMNx51TcCkBhGV1qSVnMrJ/eZPflxm02G6nhZQ= +gomodules.xyz/mergo v0.3.13-0.20220214162359-48efe39fd402/go.mod h1:i2WNHvGpzLKI+/qWRhscddeashtzrtxCAucS2H7hrtM= gomodules.xyz/password-generator v0.2.6/go.mod h1:TvwYYTx9+P1pPwKQKfZgB/wr2Id9MqAQ3B5auY7reNg= gomodules.xyz/password-generator v0.2.7/go.mod h1:TvwYYTx9+P1pPwKQKfZgB/wr2Id9MqAQ3B5auY7reNg= gomodules.xyz/pointer v0.0.0-20201105040656-991dd254b680/go.mod h1:sPLsC0+yLTRecUiC5yVlyvXhZ6LAGojNCRWNNqoplvo= @@ -1096,8 +1098,8 @@ k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ kmodules.xyz/client-go v0.0.0-20210617233340-13d22e91512b/go.mod h1:A6GAK6xP5zBuWK6A/vUkkjKzcuywkms7fIxRf5wblO4= kmodules.xyz/client-go v0.0.0-20211107190155-5bb4090d2728/go.mod h1:ENUu8pPK19xzBkVpAJHoGCI2QRvb1SqffWRt0K2sV5I= kmodules.xyz/client-go v0.0.0-20220215012708-9963581d69a7/go.mod h1:sOq5P3AkZdv6D/skvUPwEG15NDYy5olwBllH/JXfhGI= -kmodules.xyz/client-go v0.0.0-20220317213815-2a6d5a5784f2 h1:s6MK8BVV6GHJhS6V4+5IJeV6W1GHcC4fqILencMdRkw= -kmodules.xyz/client-go v0.0.0-20220317213815-2a6d5a5784f2/go.mod h1:7pExIHGzUdu8ZGveYvAaXEhS4GdczoOy8z+hq6x6K9A= +kmodules.xyz/client-go v0.0.0-20220404224906-af7b092cfac5 h1:UkXLQ5SMS98RwnJffKAU0ANXVq9AHSLabQ9NKUhkCgA= +kmodules.xyz/client-go v0.0.0-20220404224906-af7b092cfac5/go.mod h1:7pExIHGzUdu8ZGveYvAaXEhS4GdczoOy8z+hq6x6K9A= kmodules.xyz/constants v0.0.0-20210218100002-2c304bfda278/go.mod h1:DbiFk1bJ1KEO94t1SlAn7tzc+Zz95rSXgyUKa2nzPmY= kmodules.xyz/crd-schema-fuzz v0.0.0-20210618002152-fae23aef5fb4/go.mod h1:IIkUctlfoptoci0BOrsUf8ya+MOG5uaeh1PE4uzaIbA= kmodules.xyz/crd-schema-fuzz v0.0.0-20211025154117-6edb24ef11bc/go.mod h1:yLOBJKasPhnCodKSZGFZ6OGFFrp0tq3ALS9rDnYFjkg= @@ -1137,8 +1139,5 @@ sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= -stash.appscode.dev/apimachinery v0.19.0/go.mod h1:vgzwLa2KFxfbcrTS3gzhJLYmx4WdZpHfW9n8a7Hzhzc= -stash.appscode.dev/apimachinery v0.19.1-0.20220407092220-1fb00845d1f7 h1:aoFOTVQujG44AnJeaR0wKUaS0mzzFX0ktb5vilDoUgY= -stash.appscode.dev/apimachinery v0.19.1-0.20220407092220-1fb00845d1f7/go.mod h1:vgzwLa2KFxfbcrTS3gzhJLYmx4WdZpHfW9n8a7Hzhzc= -stash.appscode.dev/elasticsearch v0.0.0-20220329183444-6b505bc9eb9f h1:Z5I7YV5Z9t4R5a/zPpzyaFos4OdeC50fKKRnvJxq/CY= -stash.appscode.dev/elasticsearch v0.0.0-20220329183444-6b505bc9eb9f/go.mod h1:RePOPlnW1mvK3RJmI6Vmv28mAe/Efl6Hsi79mt9Af/8= +stash.appscode.dev/apimachinery v0.19.1-0.20220413043008-5fa3647ef8f8 h1:fI1VWIsVCa0VVpFBBCICSRiMi+d3FmLixIpyxmHm7rU= +stash.appscode.dev/apimachinery v0.19.1-0.20220413043008-5fa3647ef8f8/go.mod h1:Wvh/C9ZGQXWex8d6LKU8TodOriuPcirMuTnzgBQooLk= diff --git a/pkg/backup.go b/pkg/backup.go index 99f43c1b..81c16a62 100644 --- a/pkg/backup.go +++ b/pkg/backup.go @@ -44,7 +44,6 @@ func NewCmdBackup() *cobra.Command { masterURL string kubeconfigPath string opt = options{ - waitTimeout: 300, setupOptions: restic.SetupOptions{ ScratchDir: restic.DefaultScratchDir, EnableCache: false, @@ -57,10 +56,10 @@ func NewCmdBackup() *cobra.Command { cmd := &cobra.Command{ Use: "backup-manifest", - Short: "Takes a backup of Elasticsearch DB", + Short: "Takes a backup of Kubernetes manifests", DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - flags.EnsureRequiredFlags(cmd, "appbinding", "provider", "storage-secret-name", "storage-secret-namespace") + flags.EnsureRequiredFlags(cmd, "provider", "storage-secret-name", "storage-secret-namespace") time.Sleep(time.Second * 5) // prepare client @@ -119,9 +118,6 @@ func NewCmdBackup() *cobra.Command { return nil }, } - - cmd.Flags().Int32Var(&opt.waitTimeout, "wait-timeout", opt.waitTimeout, "Number of seconds to wait for the database to be ready") - cmd.Flags().StringVar(&masterURL, "master", masterURL, "The address of the Kubernetes API server (overrides any value in kubeconfig)") cmd.Flags().StringVar(&kubeconfigPath, "kubeconfig", kubeconfigPath, "Path to kubeconfig file with authorization information (the master location is set by the master flag).") cmd.Flags().StringVar(&opt.namespace, "namespace", "default", "Namespace of Backup/Restore Session") @@ -139,7 +135,6 @@ func NewCmdBackup() *cobra.Command { cmd.Flags().Int64Var(&opt.setupOptions.MaxConnections, "max-connections", opt.setupOptions.MaxConnections, "Specify maximum concurrent connections for GCS, Azure and B2 backend") cmd.Flags().StringVar(&opt.backupOptions.Host, "hostname", opt.backupOptions.Host, "Name of the host machine") - cmd.Flags().StringVar(&opt.interimDataDir, "interim-data-dir", opt.interimDataDir, "Directory where the targeted data will be stored temporarily before uploading to the backend.") cmd.Flags().StringVar(&opt.invokerKind, "invoker-kind", opt.invokerKind, "Kind of the backup invoker") cmd.Flags().StringVar(&opt.invokerName, "invoker-name", opt.invokerName, "Name of the respective backup invoker") cmd.Flags().StringVar(&opt.targetKind, "target-kind", opt.targetKind, "Kind of the Target") @@ -156,6 +151,7 @@ func NewCmdBackup() *cobra.Command { cmd.Flags().BoolVar(&opt.backupOptions.RetentionPolicy.DryRun, "retention-dry-run", opt.backupOptions.RetentionPolicy.DryRun, "Specify whether to test retention policy without deleting actual data") cmd.Flags().StringVar(&opt.outputDir, "output-dir", opt.outputDir, "Directory where output.json file will be written (keep empty if you don't need to write output in file)") + cmd.Flags().BoolVar(&opt.sanitize, "output-dir", true, "Specify whether to remove the decorators from the manifest (default is true)") return cmd } @@ -194,15 +190,16 @@ func (opt *options) backupManifests(targetRef v1beta1.TargetRef) (*restic.Backup return nil, err } - klog.Infoln("Cleaning up directory: ", opt.interimDataDir) - if err := clearDir(opt.interimDataDir); err != nil { + klog.Infoln("Cleaning up directory: ", opt.dataDir) + opt.dataDir = filepath.Join(opt.setupOptions.ScratchDir, "manifests") + if err := clearDir(opt.dataDir); err != nil { return nil, err } mgOpts := manager.BackupOptions{ Config: opt.config, Sanitize: opt.sanitize, - DataDir: opt.interimDataDir, + DataDir: opt.dataDir, Target: targetRef, Storage: manager.NewFileWriter(), } @@ -212,7 +209,7 @@ func (opt *options) backupManifests(targetRef v1beta1.TargetRef) (*restic.Backup } // dumped data has been stored in the interim data dir. Now, we will backup this directory using Stash. - opt.backupOptions.BackupPaths = []string{opt.interimDataDir} + opt.backupOptions.BackupPaths = []string{opt.dataDir} // init restic wrapper resticWrapper, err := restic.NewResticWrapper(opt.setupOptions) diff --git a/pkg/manager/application.go b/pkg/manager/application.go index 7c65677f..9014a75b 100644 --- a/pkg/manager/application.go +++ b/pkg/manager/application.go @@ -1,3 +1,19 @@ +/* +Copyright AppsCode Inc. and Contributors + +Licensed under the AppsCode Free Trial License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Free-Trial-1.0.0.md + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package manager import ( @@ -19,7 +35,6 @@ import ( type applicationBackupManager struct { di dynamic.Interface - namespace string storage Writer config *rest.Config sanitize bool @@ -36,7 +51,6 @@ func newApplicationBackupManager(opt BackupOptions) BackupManager { sanitize: opt.Sanitize, dataDir: opt.DataDir, selector: opt.Selector, - namespace: opt.Namespace, includeDependants: opt.IncludeDependants, target: opt.Target, } @@ -104,14 +118,14 @@ func (opt *applicationBackupManager) getRootObject(gvr schema.GroupVersionResour if err != nil { return nil, err } - ri := opt.di.Resource(gvr).Namespace(opt.namespace) + ri := opt.di.Resource(gvr).Namespace(opt.target.Namespace) return ri.Get(context.TODO(), opt.target.Name, metav1.GetOptions{}) } func (opt *applicationBackupManager) generateDependencyTree(tb *treeBuilder) error { rp := resourceProcessor{ config: opt.config, - namespace: opt.namespace, + namespace: opt.target.Namespace, selector: opt.selector, itemProcessor: tb, } @@ -196,7 +210,7 @@ func (opt *applicationBackupManager) dumpItem(r resourceRef, prefix string) (typ func (opt *applicationBackupManager) getFileName(r *unstructured.Unstructured, prefix string) string { if opt.target.Kind == r.GetKind() && opt.target.Name == r.GetName() && - opt.namespace == r.GetNamespace() { + opt.target.Namespace == r.GetNamespace() { return filepath.Join(prefix, r.GetName()) + ".yaml" } return filepath.Join(prefix, r.GetKind(), r.GetName(), r.GetName()) + ".yaml" diff --git a/pkg/manager/generic_resources.go b/pkg/manager/generic_resources.go index 4541180b..9b23d48d 100644 --- a/pkg/manager/generic_resources.go +++ b/pkg/manager/generic_resources.go @@ -1,3 +1,19 @@ +/* +Copyright AppsCode Inc. and Contributors + +Licensed under the AppsCode Free Trial License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Free-Trial-1.0.0.md + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package manager import ( diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 934b8487..1341ced8 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -1,3 +1,19 @@ +/* +Copyright AppsCode Inc. and Contributors + +Licensed under the AppsCode Free Trial License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Free-Trial-1.0.0.md + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package manager import ( @@ -23,7 +39,6 @@ type BackupOptions struct { Target v1beta1.TargetRef IncludeDependants bool Storage Writer - Namespace string } func NewBackupManager(opt BackupOptions) BackupManager { diff --git a/pkg/manager/manager_test.go b/pkg/manager/manager_test.go index 3691d811..a92813c9 100644 --- a/pkg/manager/manager_test.go +++ b/pkg/manager/manager_test.go @@ -1,3 +1,19 @@ +/* +Copyright AppsCode Inc. and Contributors + +Licensed under the AppsCode Free Trial License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Free-Trial-1.0.0.md + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package manager_test import ( @@ -66,9 +82,9 @@ func Test_Dump(t *testing.T) { APIVersion: "apps/v1", Kind: apis.KindDeployment, Name: "coredns", + Namespace: "kube-system", }, - Sanitize: true, - Namespace: "kube-system", + Sanitize: true, }, }, { @@ -79,9 +95,9 @@ func Test_Dump(t *testing.T) { APIVersion: "apps/v1", Kind: apis.KindDeployment, Name: "coredns", + Namespace: "kube-system", }, Sanitize: true, - Namespace: "kube-system", IncludeDependants: true, }, }, @@ -93,9 +109,9 @@ func Test_Dump(t *testing.T) { APIVersion: "v1", Kind: "ConfigMap", Name: "coredns", + Namespace: "kube-system", }, - Sanitize: true, - Namespace: "kube-system", + Sanitize: true, }, }, } diff --git a/pkg/manager/resources_processor.go b/pkg/manager/resources_processor.go index 60860a16..9f7f2d4a 100644 --- a/pkg/manager/resources_processor.go +++ b/pkg/manager/resources_processor.go @@ -1,3 +1,19 @@ +/* +Copyright AppsCode Inc. and Contributors + +Licensed under the AppsCode Free Trial License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Free-Trial-1.0.0.md + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package manager import ( diff --git a/pkg/root.go b/pkg/root.go index 370b0e99..491d360f 100644 --- a/pkg/root.go +++ b/pkg/root.go @@ -31,8 +31,8 @@ var licenseApiService string func NewRootCmd() *cobra.Command { rootCmd := &cobra.Command{ Use: "manifest-backup", - Short: `Elasticsearch backup & restore plugin for Stash by AppsCode`, - Long: `Elasticsearch backup & restore plugin for Stash by AppsCode. For more information, visit here: https://appscode.com/products/stash`, + Short: `Kubernetes manifest backup plugin for Stash by AppsCode`, + Long: `Kubernetes manifest backup plugin for Stash by AppsCode. For more information, visit here: https://appscode.com/products/stash`, DisableAutoGenTag: true, PersistentPreRunE: func(c *cobra.Command, args []string) error { return scheme.AddToScheme(clientsetscheme.Scheme) diff --git a/pkg/sanitizers/metadata.go b/pkg/sanitizers/metadata.go index 4d86c07f..fd7427a1 100644 --- a/pkg/sanitizers/metadata.go +++ b/pkg/sanitizers/metadata.go @@ -1,3 +1,19 @@ +/* +Copyright AppsCode Inc. and Contributors + +Licensed under the AppsCode Free Trial License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Free-Trial-1.0.0.md + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package sanitizers type metadataSanitizer struct{} diff --git a/pkg/sanitizers/pod.go b/pkg/sanitizers/pod.go index 56ddfca0..90c71a15 100644 --- a/pkg/sanitizers/pod.go +++ b/pkg/sanitizers/pod.go @@ -1,3 +1,19 @@ +/* +Copyright AppsCode Inc. and Contributors + +Licensed under the AppsCode Free Trial License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Free-Trial-1.0.0.md + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package sanitizers import ( diff --git a/pkg/sanitizers/sanitizer.go b/pkg/sanitizers/sanitizer.go index 22b14b91..0f34d959 100644 --- a/pkg/sanitizers/sanitizer.go +++ b/pkg/sanitizers/sanitizer.go @@ -1,3 +1,19 @@ +/* +Copyright AppsCode Inc. and Contributors + +Licensed under the AppsCode Free Trial License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Free-Trial-1.0.0.md + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package sanitizers type Sanitizer interface { diff --git a/pkg/sanitizers/workload.go b/pkg/sanitizers/workload.go index ab5e3fe1..d4f648e9 100644 --- a/pkg/sanitizers/workload.go +++ b/pkg/sanitizers/workload.go @@ -1,3 +1,19 @@ +/* +Copyright AppsCode Inc. and Contributors + +Licensed under the AppsCode Free Trial License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Free-Trial-1.0.0.md + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package sanitizers import "fmt" diff --git a/pkg/utils.go b/pkg/utils.go index a52fe85e..3594aab8 100644 --- a/pkg/utils.go +++ b/pkg/utils.go @@ -29,14 +29,6 @@ import ( appcatalog_cs "kmodules.xyz/custom-resources/client/clientset/versioned" ) -const ( - ESUser = "ADMIN_USERNAME" - ESPassword = "ADMIN_PASSWORD" - MultiElasticDumpCMD = "multielasticdump" - ESCACertFile = "root.pem" - ESAuthFile = "auth.txt" -) - type options struct { kubeClient kubernetes.Interface stashClient stash.Interface @@ -44,13 +36,12 @@ type options struct { namespace string backupSessionName string - interimDataDir string outputDir string storageSecret kmapi.ObjectReference - waitTimeout int32 sanitize bool config *rest.Config + dataDir string invokerKind string invokerName string diff --git a/vendor/kmodules.xyz/client-go/api/v1/timeofday.go b/vendor/kmodules.xyz/client-go/api/v1/timeofday.go index b071abee..88ea1a8b 100644 --- a/vendor/kmodules.xyz/client-go/api/v1/timeofday.go +++ b/vendor/kmodules.xyz/client-go/api/v1/timeofday.go @@ -43,8 +43,16 @@ func (t *TimeOfDay) DeepCopyInto(out *TimeOfDay) { // NewTime returns a wrapped instance of the provided time func NewTime(t time.Time) TimeOfDay { - utc := t.UTC() - return TimeOfDay{time.Date(0, 0, 0, utc.Hour(), utc.Minute(), utc.Second(), 0, time.UTC)} + return TimeOfDay{time.Date(0, 0, 0, t.Hour(), t.Minute(), t.Second(), 0, time.UTC)} +} + +// NewTimeInLocation returns a wrapped instance of the provided time according to location +func NewTimeInLocation(t time.Time, loc *time.Location) TimeOfDay { + if loc == nil { + loc = time.UTC + } + t = t.In(loc) + return TimeOfDay{time.Date(0, 0, 0, t.Hour(), t.Minute(), t.Second(), 0, loc)} } // Date returns the TimeOfDay corresponding to the supplied parameters diff --git a/vendor/kmodules.xyz/client-go/meta/incluster.go b/vendor/kmodules.xyz/client-go/meta/incluster.go index 3bdfc4ff..f5df04dd 100644 --- a/vendor/kmodules.xyz/client-go/meta/incluster.go +++ b/vendor/kmodules.xyz/client-go/meta/incluster.go @@ -36,7 +36,7 @@ func Namespace() string { return ns } - if ns := os.Getenv("MY_POD_NAMESPACE"); ns != "" { + if ns := os.Getenv("POD_NAMESPACE"); ns != "" { return ns } diff --git a/vendor/modules.txt b/vendor/modules.txt index b8d96fd6..4c685e31 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -155,7 +155,7 @@ gomodules.xyz/jsonpatch/v2 # gomodules.xyz/logs v0.0.6 ## explicit gomodules.xyz/logs -# gomodules.xyz/mergo v0.3.13-0.20210702100041-9d62ff8ece4d +# gomodules.xyz/mergo v0.3.13-0.20220214162359-48efe39fd402 gomodules.xyz/mergo # gomodules.xyz/pointer v0.1.0 gomodules.xyz/pointer @@ -509,7 +509,7 @@ k8s.io/utils/buffer k8s.io/utils/integer k8s.io/utils/pointer k8s.io/utils/trace -# kmodules.xyz/client-go v0.0.0-20220317213815-2a6d5a5784f2 +# kmodules.xyz/client-go v0.0.0-20220404224906-af7b092cfac5 ## explicit kmodules.xyz/client-go kmodules.xyz/client-go/api/v1 @@ -554,7 +554,7 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.3.0 ## explicit sigs.k8s.io/yaml -# stash.appscode.dev/apimachinery v0.19.1-0.20220407092220-1fb00845d1f7 +# stash.appscode.dev/apimachinery v0.19.1-0.20220413043008-5fa3647ef8f8 ## explicit stash.appscode.dev/apimachinery/apis stash.appscode.dev/apimachinery/apis/repositories @@ -581,9 +581,6 @@ stash.appscode.dev/apimachinery/pkg/conditions stash.appscode.dev/apimachinery/pkg/invoker stash.appscode.dev/apimachinery/pkg/restic stash.appscode.dev/apimachinery/pkg/util -# stash.appscode.dev/elasticsearch v0.0.0-20220329183444-6b505bc9eb9f -## explicit -stash.appscode.dev/elasticsearch/pkg # bitbucket.org/ww/goautoneg => gomodules.xyz/goautoneg v0.0.0-20120707110453-a547fc61f48d # cloud.google.com/go => cloud.google.com/go v0.54.0 # cloud.google.com/go/bigquery => cloud.google.com/go/bigquery v1.4.0 diff --git a/vendor/stash.appscode.dev/apimachinery/apis/stash/v1beta1/openapi_generated.go b/vendor/stash.appscode.dev/apimachinery/apis/stash/v1beta1/openapi_generated.go index 82a43c22..13ae2b30 100644 --- a/vendor/stash.appscode.dev/apimachinery/apis/stash/v1beta1/openapi_generated.go +++ b/vendor/stash.appscode.dev/apimachinery/apis/stash/v1beta1/openapi_generated.go @@ -21808,6 +21808,12 @@ func schema_apimachinery_apis_stash_v1beta1_TargetRef(ref common.ReferenceCallba Format: "", }, }, + "namespace": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, }, }, }, diff --git a/vendor/stash.appscode.dev/apimachinery/apis/stash/v1beta1/types.go b/vendor/stash.appscode.dev/apimachinery/apis/stash/v1beta1/types.go index 9cd85728..321036a7 100644 --- a/vendor/stash.appscode.dev/apimachinery/apis/stash/v1beta1/types.go +++ b/vendor/stash.appscode.dev/apimachinery/apis/stash/v1beta1/types.go @@ -140,6 +140,8 @@ type TargetRef struct { APIVersion string `json:"apiVersion,omitempty"` Kind string `json:"kind,omitempty"` Name string `json:"name,omitempty"` + // +optional + Namespace string `json:"namespace,omitempty"` } type ExecutionOrder string diff --git a/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_backupbatches.yaml b/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_backupbatches.yaml index 4f839d30..2c20413c 100644 --- a/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_backupbatches.yaml +++ b/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_backupbatches.yaml @@ -3218,6 +3218,8 @@ spec: type: string name: type: string + namespace: + type: string type: object replicas: description: replicas are the desired number of replicas @@ -5536,6 +5538,8 @@ spec: type: string name: type: string + namespace: + type: string type: object required: - target diff --git a/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_backupconfigurations.yaml b/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_backupconfigurations.yaml index d59da36d..60b79fcf 100644 --- a/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_backupconfigurations.yaml +++ b/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_backupconfigurations.yaml @@ -2808,6 +2808,8 @@ spec: type: string name: type: string + namespace: + type: string type: object replicas: description: replicas are the desired number of replicas whose diff --git a/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_backupsessions.yaml b/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_backupsessions.yaml index 1491612b..a5de60d3 100644 --- a/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_backupsessions.yaml +++ b/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_backupsessions.yaml @@ -219,6 +219,8 @@ spec: type: string name: type: string + namespace: + type: string type: object stats: description: Stats shows statistics of individual hosts for diff --git a/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_restorebatches.yaml b/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_restorebatches.yaml index b3c16489..9363b64b 100644 --- a/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_restorebatches.yaml +++ b/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_restorebatches.yaml @@ -3196,6 +3196,8 @@ spec: type: string name: type: string + namespace: + type: string type: object replicas: description: replicas is the desired number of replicas @@ -3826,6 +3828,8 @@ spec: type: string name: type: string + namespace: + type: string type: object stats: description: Stats shows restore statistics of individual hosts diff --git a/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_restoresessions.yaml b/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_restoresessions.yaml index 642d6918..b3ba979d 100644 --- a/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_restoresessions.yaml +++ b/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_restoresessions.yaml @@ -2792,6 +2792,8 @@ spec: type: string name: type: string + namespace: + type: string type: object replicas: description: replicas is the desired number of replicas of the diff --git a/vendor/stash.appscode.dev/apimachinery/pkg/conditions/backup_invoker.go b/vendor/stash.appscode.dev/apimachinery/pkg/conditions/backup_invoker.go index 04b945c6..1ae9d1f6 100644 --- a/vendor/stash.appscode.dev/apimachinery/pkg/conditions/backup_invoker.go +++ b/vendor/stash.appscode.dev/apimachinery/pkg/conditions/backup_invoker.go @@ -18,7 +18,6 @@ package conditions import ( "fmt" - "strings" "stash.appscode.dev/apimachinery/apis/stash/v1beta1" "stash.appscode.dev/apimachinery/pkg/invoker" @@ -34,8 +33,8 @@ func SetBackupTargetFoundConditionToUnknown(invoker invoker.BackupInvoker, tref Status: core.ConditionUnknown, Reason: v1beta1.UnableToCheckTargetAvailability, Message: fmt.Sprintf("Failed to check whether backup target %s %s/%s exist or not. Reason: %v", - tref.APIVersion, - strings.ToLower(tref.Kind), + tref.Kind, + tref.Namespace, tref.Name, err.Error(), ), @@ -50,8 +49,8 @@ func SetBackupTargetFoundConditionToFalse(invoker invoker.BackupInvoker, tref v1 Status: core.ConditionFalse, Reason: v1beta1.TargetNotAvailable, Message: fmt.Sprintf("Backup target %s %s/%s does not exist.", - tref.APIVersion, - strings.ToLower(tref.Kind), + tref.Kind, + tref.Namespace, tref.Name, ), LastTransitionTime: metav1.Now(), @@ -64,8 +63,8 @@ func SetBackupTargetFoundConditionToTrue(invoker invoker.BackupInvoker, tref v1b Status: core.ConditionTrue, Reason: v1beta1.TargetAvailable, Message: fmt.Sprintf("Backup target %s %s/%s found.", - tref.APIVersion, - strings.ToLower(tref.Kind), + tref.Kind, + tref.Namespace, tref.Name, ), LastTransitionTime: metav1.Now(), @@ -98,8 +97,8 @@ func SetSidecarInjectedConditionToTrue(invoker invoker.BackupInvoker, tref v1bet Status: core.ConditionTrue, Reason: v1beta1.SidecarInjectionSucceeded, Message: fmt.Sprintf("Successfully injected stash sidecar into %s %s/%s", - tref.APIVersion, - strings.ToLower(tref.Kind), + tref.Kind, + tref.Namespace, tref.Name, ), LastTransitionTime: metav1.Now(), @@ -112,8 +111,8 @@ func SetSidecarInjectedConditionToFalse(invoker invoker.BackupInvoker, tref v1be Status: core.ConditionFalse, Reason: v1beta1.SidecarInjectionFailed, Message: fmt.Sprintf("Failed to inject stash sidecar into %s %s/%s. Reason: %v", - tref.APIVersion, - strings.ToLower(tref.Kind), + tref.Kind, + tref.Namespace, tref.Name, err.Error(), ), diff --git a/vendor/stash.appscode.dev/apimachinery/pkg/conditions/restore_invoker.go b/vendor/stash.appscode.dev/apimachinery/pkg/conditions/restore_invoker.go index a6939f51..f22212e9 100644 --- a/vendor/stash.appscode.dev/apimachinery/pkg/conditions/restore_invoker.go +++ b/vendor/stash.appscode.dev/apimachinery/pkg/conditions/restore_invoker.go @@ -18,7 +18,6 @@ package conditions import ( "fmt" - "strings" "stash.appscode.dev/apimachinery/apis/stash/v1beta1" "stash.appscode.dev/apimachinery/pkg/invoker" @@ -35,8 +34,8 @@ func SetRestoreTargetFoundConditionToTrue(inv invoker.RestoreInvoker, index int) Status: core.ConditionTrue, Reason: v1beta1.TargetAvailable, Message: fmt.Sprintf("Restore target %s %s/%s found.", - target.Ref.APIVersion, - strings.ToLower(target.Ref.Kind), + target.Ref.Kind, + target.Ref.Namespace, target.Ref.Name, ), LastTransitionTime: metav1.Now(), @@ -50,8 +49,8 @@ func SetRestoreTargetFoundConditionToFalse(inv invoker.RestoreInvoker, index int Status: core.ConditionFalse, Reason: v1beta1.TargetNotAvailable, Message: fmt.Sprintf("Restore target %s %s/%s does not exist.", - target.Ref.APIVersion, - strings.ToLower(target.Ref.Kind), + target.Ref.Kind, + target.Ref.Namespace, target.Ref.Name, ), LastTransitionTime: metav1.Now(), @@ -65,8 +64,8 @@ func SetRestoreTargetFoundConditionToUnknown(inv invoker.RestoreInvoker, index i Status: core.ConditionUnknown, Reason: v1beta1.UnableToCheckTargetAvailability, Message: fmt.Sprintf("Failed to check whether restore target %s %s/%s exist or not. Reason: %v", - target.Ref.APIVersion, - strings.ToLower(target.Ref.Kind), + target.Ref.Kind, + target.Ref.Namespace, target.Ref.Name, err, ), diff --git a/vendor/stash.appscode.dev/apimachinery/pkg/invoker/backup_invoker.go b/vendor/stash.appscode.dev/apimachinery/pkg/invoker/backup_invoker.go index be52b33c..dc2000b0 100644 --- a/vendor/stash.appscode.dev/apimachinery/pkg/invoker/backup_invoker.go +++ b/vendor/stash.appscode.dev/apimachinery/pkg/invoker/backup_invoker.go @@ -112,7 +112,10 @@ func getMemberCondition(conditions []v1beta1.MemberConditions, target v1beta1.Ta } func TargetMatched(t1, t2 v1beta1.TargetRef) bool { - return t1.APIVersion == t2.APIVersion && t1.Kind == t2.Kind && t1.Name == t2.Name + return t1.APIVersion == t2.APIVersion && + t1.Kind == t2.Kind && + t1.Namespace == t2.Namespace && + t1.Name == t2.Name } func setMemberCondition(conditions []v1beta1.MemberConditions, target v1beta1.TargetRef, newCondition kmapi.Condition) []v1beta1.MemberConditions { diff --git a/vendor/stash.appscode.dev/apimachinery/pkg/invoker/backupbatch.go b/vendor/stash.appscode.dev/apimachinery/pkg/invoker/backupbatch.go index 0c996a22..3d7ad1f8 100644 --- a/vendor/stash.appscode.dev/apimachinery/pkg/invoker/backupbatch.go +++ b/vendor/stash.appscode.dev/apimachinery/pkg/invoker/backupbatch.go @@ -153,7 +153,7 @@ func (inv *BackupBatchInvoker) GetTargetInfo() []BackupTargetInfo { for _, member := range inv.backupBatch.Spec.Members { targetInfo = append(targetInfo, BackupTargetInfo{ Task: member.Task, - Target: withDefaultTarget(member.Target), + Target: getBackupTarget(member.Target, inv.backupBatch.Namespace), RuntimeSettings: member.RuntimeSettings, TempDir: member.TempDir, InterimVolumeTemplate: member.InterimVolumeTemplate, @@ -163,6 +163,18 @@ func (inv *BackupBatchInvoker) GetTargetInfo() []BackupTargetInfo { return targetInfo } +func getBackupTarget(target *v1beta1.BackupTarget, invNamespace string) *v1beta1.BackupTarget { + if target == nil { + return &v1beta1.BackupTarget{ + Ref: v1beta1.EmptyTargetRef(), + } + } + if target.Ref.Namespace == "" { + target.Ref.Namespace = invNamespace + } + return target +} + func (inv *BackupBatchInvoker) GetDriver() v1beta1.Snapshotter { driver := inv.backupBatch.Spec.Driver if driver == "" { diff --git a/vendor/stash.appscode.dev/apimachinery/pkg/invoker/backupconfiguration.go b/vendor/stash.appscode.dev/apimachinery/pkg/invoker/backupconfiguration.go index 3f1d6b3f..8d7736b0 100644 --- a/vendor/stash.appscode.dev/apimachinery/pkg/invoker/backupconfiguration.go +++ b/vendor/stash.appscode.dev/apimachinery/pkg/invoker/backupconfiguration.go @@ -137,7 +137,7 @@ func (inv *BackupConfigurationInvoker) GetTargetInfo() []BackupTargetInfo { return []BackupTargetInfo{ { Task: inv.backupConfig.Spec.Task, - Target: withDefaultTarget(inv.backupConfig.Spec.Target), + Target: getBackupTarget(inv.backupConfig.Spec.Target, inv.backupConfig.Namespace), RuntimeSettings: inv.backupConfig.Spec.RuntimeSettings, TempDir: inv.backupConfig.Spec.TempDir, InterimVolumeTemplate: inv.backupConfig.Spec.InterimVolumeTemplate, @@ -237,12 +237,3 @@ func (inv *BackupConfigurationInvoker) GetSummary(target v1beta1.TargetRef, sess } return summary } - -func withDefaultTarget(target *v1beta1.BackupTarget) *v1beta1.BackupTarget { - if target != nil { - return target - } - return &v1beta1.BackupTarget{ - Ref: v1beta1.EmptyTargetRef(), - } -} diff --git a/vendor/stash.appscode.dev/apimachinery/pkg/invoker/restorebatch.go b/vendor/stash.appscode.dev/apimachinery/pkg/invoker/restorebatch.go index abf9131c..10d030b0 100644 --- a/vendor/stash.appscode.dev/apimachinery/pkg/invoker/restorebatch.go +++ b/vendor/stash.appscode.dev/apimachinery/pkg/invoker/restorebatch.go @@ -154,7 +154,7 @@ func (inv *RestoreBatchInvoker) GetTargetInfo() []RestoreTargetInfo { for _, member := range inv.restoreBatch.Spec.Members { targetInfo = append(targetInfo, RestoreTargetInfo{ Task: member.Task, - Target: member.Target, + Target: getRestoreTarget(member.Target, inv.restoreBatch.Namespace), RuntimeSettings: member.RuntimeSettings, TempDir: member.TempDir, InterimVolumeTemplate: member.InterimVolumeTemplate, @@ -164,6 +164,13 @@ func (inv *RestoreBatchInvoker) GetTargetInfo() []RestoreTargetInfo { return targetInfo } +func getRestoreTarget(target *v1beta1.RestoreTarget, invNamespace string) *v1beta1.RestoreTarget { + if target != nil && target.Ref.Namespace == "" { + target.Ref.Namespace = invNamespace + } + return target +} + func (inv *RestoreBatchInvoker) GetDriver() v1beta1.Snapshotter { driver := inv.restoreBatch.Spec.Driver if driver == "" { diff --git a/vendor/stash.appscode.dev/apimachinery/pkg/invoker/restoresession.go b/vendor/stash.appscode.dev/apimachinery/pkg/invoker/restoresession.go index 5479f773..5cfbab07 100644 --- a/vendor/stash.appscode.dev/apimachinery/pkg/invoker/restoresession.go +++ b/vendor/stash.appscode.dev/apimachinery/pkg/invoker/restoresession.go @@ -139,7 +139,7 @@ func (inv *RestoreSessionInvoker) GetTargetInfo() []RestoreTargetInfo { return []RestoreTargetInfo{ { Task: inv.restoreSession.Spec.Task, - Target: inv.restoreSession.Spec.Target, + Target: getRestoreTarget(inv.restoreSession.Spec.Target, inv.restoreSession.Namespace), RuntimeSettings: inv.restoreSession.Spec.RuntimeSettings, TempDir: inv.restoreSession.Spec.TempDir, InterimVolumeTemplate: inv.restoreSession.Spec.InterimVolumeTemplate, diff --git a/vendor/stash.appscode.dev/apimachinery/pkg/util/addon.go b/vendor/stash.appscode.dev/apimachinery/pkg/util/addon.go index 3e4da0f8..cff35cbf 100644 --- a/vendor/stash.appscode.dev/apimachinery/pkg/util/addon.go +++ b/vendor/stash.appscode.dev/apimachinery/pkg/util/addon.go @@ -28,13 +28,13 @@ import ( appcatalog_cs "kmodules.xyz/custom-resources/client/clientset/versioned" ) -func ExtractAddonInfo(appClient appcatalog_cs.Interface, task v1beta1.TaskRef, targetRef v1beta1.TargetRef, namespace string) (*appcat.StashTaskSpec, error) { +func ExtractAddonInfo(appClient appcatalog_cs.Interface, task v1beta1.TaskRef, targetRef v1beta1.TargetRef) (*appcat.StashTaskSpec, error) { var params appcat.StashAddon // If the target is AppBinding and it has addon information set in the parameters section, then extract the addon info. if invoker.TargetOfGroupKind(targetRef, appcat.SchemeGroupVersion.Group, appcat.ResourceKindApp) { // get the AppBinding - appBinding, err := appClient.AppcatalogV1alpha1().AppBindings(namespace).Get(context.TODO(), targetRef.Name, metav1.GetOptions{}) + appBinding, err := appClient.AppcatalogV1alpha1().AppBindings(targetRef.Namespace).Get(context.TODO(), targetRef.Name, metav1.GetOptions{}) if err != nil { return nil, err } diff --git a/vendor/stash.appscode.dev/elasticsearch/LICENSE.md b/vendor/stash.appscode.dev/elasticsearch/LICENSE.md deleted file mode 100644 index ed87e53b..00000000 --- a/vendor/stash.appscode.dev/elasticsearch/LICENSE.md +++ /dev/null @@ -1,5 +0,0 @@ -## License - -Source code in this repository, Binaries, Docker images and Charts produced by the build process are licensed under the AppsCode Free Trial License 1.0.0. You may obtain a copy of the License at - - - [AppsCode-Trial-1.0.0](https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Free-Trial-1.0.0.md) diff --git a/vendor/stash.appscode.dev/elasticsearch/pkg/backup.go b/vendor/stash.appscode.dev/elasticsearch/pkg/backup.go deleted file mode 100644 index dd39809f..00000000 --- a/vendor/stash.appscode.dev/elasticsearch/pkg/backup.go +++ /dev/null @@ -1,243 +0,0 @@ -/* -Copyright AppsCode Inc. and Contributors - -Licensed under the AppsCode Free Trial License 1.0.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Free-Trial-1.0.0.md - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package pkg - -import ( - "context" - "fmt" - "path/filepath" - "time" - - api_v1beta1 "stash.appscode.dev/apimachinery/apis/stash/v1beta1" - stash "stash.appscode.dev/apimachinery/client/clientset/versioned" - "stash.appscode.dev/apimachinery/pkg/restic" - api_util "stash.appscode.dev/apimachinery/pkg/util" - - "github.com/spf13/cobra" - license "go.bytebuilders.dev/license-verifier/kubernetes" - "gomodules.xyz/flags" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/klog/v2" - appcatalog "kmodules.xyz/custom-resources/apis/appcatalog/v1alpha1" - appcatalog_cs "kmodules.xyz/custom-resources/client/clientset/versioned" - v1 "kmodules.xyz/offshoot-api/api/v1" -) - -func NewCmdBackup() *cobra.Command { - var ( - masterURL string - kubeconfigPath string - opt = esOptions{ - waitTimeout: 300, - setupOptions: restic.SetupOptions{ - ScratchDir: restic.DefaultScratchDir, - EnableCache: false, - }, - backupOptions: restic.BackupOptions{ - Host: restic.DefaultHost, - }, - } - ) - - cmd := &cobra.Command{ - Use: "backup-es", - Short: "Takes a backup of Elasticsearch DB", - DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { - flags.EnsureRequiredFlags(cmd, "appbinding", "provider", "storage-secret-name", "storage-secret-namespace") - time.Sleep(time.Second * 5) - - // prepare client - config, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfigPath) - if err != nil { - return err - } - err = license.CheckLicenseEndpoint(config, licenseApiService, SupportedProducts) - if err != nil { - return err - } - opt.kubeClient, err = kubernetes.NewForConfig(config) - if err != nil { - return err - } - opt.stashClient, err = stash.NewForConfig(config) - if err != nil { - return err - } - opt.catalogClient, err = appcatalog_cs.NewForConfig(config) - if err != nil { - return err - } - - targetRef := api_v1beta1.TargetRef{ - APIVersion: appcatalog.SchemeGroupVersion.String(), - Kind: appcatalog.ResourceKindApp, - Name: opt.appBindingName, - } - var backupOutput *restic.BackupOutput - backupOutput, err = opt.backupElasticsearch(targetRef) - if err != nil { - backupOutput = &restic.BackupOutput{ - BackupTargetStatus: api_v1beta1.BackupTargetStatus{ - Ref: targetRef, - Stats: []api_v1beta1.HostBackupStats{ - { - Hostname: opt.backupOptions.Host, - Phase: api_v1beta1.HostBackupFailed, - Error: err.Error(), - }, - }, - }, - } - } - // If output directory specified, then write the output in "output.json" file in the specified directory - if opt.outputDir != "" { - return backupOutput.WriteOutput(filepath.Join(opt.outputDir, restic.DefaultOutputFileName)) - } - return nil - }, - } - - cmd.Flags().StringVar(&opt.esArgs, "es-args", opt.esArgs, "Additional arguments") - cmd.Flags().Int32Var(&opt.waitTimeout, "wait-timeout", opt.waitTimeout, "Number of seconds to wait for the database to be ready") - - cmd.Flags().StringVar(&masterURL, "master", masterURL, "The address of the Kubernetes API server (overrides any value in kubeconfig)") - cmd.Flags().StringVar(&kubeconfigPath, "kubeconfig", kubeconfigPath, "Path to kubeconfig file with authorization information (the master location is set by the master flag).") - cmd.Flags().StringVar(&opt.namespace, "namespace", "default", "Namespace of Backup/Restore Session") - cmd.Flags().StringVar(&opt.backupSessionName, "backupsession", opt.backupSessionName, "Name of the Backup Session") - cmd.Flags().StringVar(&opt.appBindingName, "appbinding", opt.appBindingName, "Name of the app binding") - cmd.Flags().StringVar(&opt.storageSecret.Name, "storage-secret-name", opt.storageSecret.Name, "Name of the storage secret") - cmd.Flags().StringVar(&opt.storageSecret.Namespace, "storage-secret-namespace", opt.storageSecret.Namespace, "Namespace of the storage secret") - - cmd.Flags().StringVar(&opt.setupOptions.Provider, "provider", opt.setupOptions.Provider, "Backend provider (i.e. gcs, s3, azure etc)") - cmd.Flags().StringVar(&opt.setupOptions.Bucket, "bucket", opt.setupOptions.Bucket, "Name of the cloud bucket/container (keep empty for local backend)") - cmd.Flags().StringVar(&opt.setupOptions.Endpoint, "endpoint", opt.setupOptions.Endpoint, "Endpoint for s3/s3 compatible backend or REST server URL") - cmd.Flags().StringVar(&opt.setupOptions.Region, "region", opt.setupOptions.Region, "Region for s3/s3 compatible backend") - cmd.Flags().StringVar(&opt.setupOptions.Path, "path", opt.setupOptions.Path, "Directory inside the bucket where backup will be stored") - cmd.Flags().StringVar(&opt.setupOptions.ScratchDir, "scratch-dir", opt.setupOptions.ScratchDir, "Temporary directory") - cmd.Flags().BoolVar(&opt.setupOptions.EnableCache, "enable-cache", opt.setupOptions.EnableCache, "Specify whether to enable caching for restic") - cmd.Flags().Int64Var(&opt.setupOptions.MaxConnections, "max-connections", opt.setupOptions.MaxConnections, "Specify maximum concurrent connections for GCS, Azure and B2 backend") - - cmd.Flags().StringVar(&opt.backupOptions.Host, "hostname", opt.backupOptions.Host, "Name of the host machine") - cmd.Flags().StringVar(&opt.interimDataDir, "interim-data-dir", opt.interimDataDir, "Directory where the targeted data will be stored temporarily before uploading to the backend.") - - cmd.Flags().Int64Var(&opt.backupOptions.RetentionPolicy.KeepLast, "retention-keep-last", opt.backupOptions.RetentionPolicy.KeepLast, "Specify value for retention strategy") - cmd.Flags().Int64Var(&opt.backupOptions.RetentionPolicy.KeepHourly, "retention-keep-hourly", opt.backupOptions.RetentionPolicy.KeepHourly, "Specify value for retention strategy") - cmd.Flags().Int64Var(&opt.backupOptions.RetentionPolicy.KeepDaily, "retention-keep-daily", opt.backupOptions.RetentionPolicy.KeepDaily, "Specify value for retention strategy") - cmd.Flags().Int64Var(&opt.backupOptions.RetentionPolicy.KeepWeekly, "retention-keep-weekly", opt.backupOptions.RetentionPolicy.KeepWeekly, "Specify value for retention strategy") - cmd.Flags().Int64Var(&opt.backupOptions.RetentionPolicy.KeepMonthly, "retention-keep-monthly", opt.backupOptions.RetentionPolicy.KeepMonthly, "Specify value for retention strategy") - cmd.Flags().Int64Var(&opt.backupOptions.RetentionPolicy.KeepYearly, "retention-keep-yearly", opt.backupOptions.RetentionPolicy.KeepYearly, "Specify value for retention strategy") - cmd.Flags().StringSliceVar(&opt.backupOptions.RetentionPolicy.KeepTags, "retention-keep-tags", opt.backupOptions.RetentionPolicy.KeepTags, "Specify value for retention strategy") - cmd.Flags().BoolVar(&opt.backupOptions.RetentionPolicy.Prune, "retention-prune", opt.backupOptions.RetentionPolicy.Prune, "Specify whether to prune old snapshot data") - cmd.Flags().BoolVar(&opt.backupOptions.RetentionPolicy.DryRun, "retention-dry-run", opt.backupOptions.RetentionPolicy.DryRun, "Specify whether to test retention policy without deleting actual data") - - cmd.Flags().StringVar(&opt.outputDir, "output-dir", opt.outputDir, "Directory where output.json file will be written (keep empty if you don't need to write output in file)") - - return cmd -} - -func (opt *esOptions) backupElasticsearch(targetRef api_v1beta1.TargetRef) (*restic.BackupOutput, error) { - var err error - opt.setupOptions.StorageSecret, err = opt.kubeClient.CoreV1().Secrets(opt.storageSecret.Namespace).Get(context.TODO(), opt.storageSecret.Name, metav1.GetOptions{}) - if err != nil { - return nil, err - } - - // if any pre-backup actions has been assigned to it, execute them - actionOptions := api_util.ActionOptions{ - StashClient: opt.stashClient, - TargetRef: targetRef, - SetupOptions: opt.setupOptions, - BackupSessionName: opt.backupSessionName, - Namespace: opt.namespace, - } - err = api_util.ExecutePreBackupActions(actionOptions) - if err != nil { - return nil, err - } - // wait until the backend repository has been initialized. - err = api_util.WaitForBackendRepository(actionOptions) - if err != nil { - return nil, err - } - // apply nice, ionice settings from env - opt.setupOptions.Nice, err = v1.NiceSettingsFromEnv() - if err != nil { - return nil, err - } - opt.setupOptions.IONice, err = v1.IONiceSettingsFromEnv() - if err != nil { - return nil, err - } - - appBinding, err := opt.catalogClient.AppcatalogV1alpha1().AppBindings(opt.namespace).Get(context.TODO(), opt.appBindingName, metav1.GetOptions{}) - if err != nil { - return nil, err - } - - // clear directory before running multielasticdump - session := opt.newSessionWrapper(MultiElasticDumpCMD) - - err = opt.setDatabaseCredentials(appBinding, session.cmd) - if err != nil { - return nil, err - } - klog.Infoln("Cleaning up directory: ", opt.interimDataDir) - if err := clearDir(opt.interimDataDir); err != nil { - return nil, err - } - - url, err := appBinding.URL() - if err != nil { - return nil, err - } - session.cmd.Args = append(session.cmd.Args, []interface{}{ - fmt.Sprintf(`--input=%v`, url), - fmt.Sprintf(`--output=%v`, opt.interimDataDir), - }...) - - err = session.setTLSParameters(appBinding, opt.setupOptions.ScratchDir) - if err != nil { - return nil, err - } - - err = opt.waitForDBReady(appBinding) - if err != nil { - return nil, err - } - - session.sh.ShowCMD = false - session.setUserArgs(opt.esArgs) - session.sh.Command(session.cmd.Name, session.cmd.Args...) // xref: multielasticdump: https://github.com/taskrabbit/elasticsearch-dump#multielasticdump - - if err := session.sh.Run(); err != nil { - return nil, err - } - - // dumped data has been stored in the interim data dir. Now, we will backup this directory using Stash. - opt.backupOptions.BackupPaths = []string{opt.interimDataDir} - - // init restic wrapper - resticWrapper, err := restic.NewResticWrapper(opt.setupOptions) - if err != nil { - return nil, err - } - - return resticWrapper.RunBackup(opt.backupOptions, targetRef) -} diff --git a/vendor/stash.appscode.dev/elasticsearch/pkg/restore.go b/vendor/stash.appscode.dev/elasticsearch/pkg/restore.go deleted file mode 100644 index 9e5cf651..00000000 --- a/vendor/stash.appscode.dev/elasticsearch/pkg/restore.go +++ /dev/null @@ -1,220 +0,0 @@ -/* -Copyright AppsCode Inc. and Contributors - -Licensed under the AppsCode Free Trial License 1.0.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Free-Trial-1.0.0.md - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package pkg - -import ( - "context" - "fmt" - "path/filepath" - - api_v1beta1 "stash.appscode.dev/apimachinery/apis/stash/v1beta1" - "stash.appscode.dev/apimachinery/pkg/restic" - - "github.com/spf13/cobra" - license "go.bytebuilders.dev/license-verifier/kubernetes" - "gomodules.xyz/flags" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/klog/v2" - appcatalog "kmodules.xyz/custom-resources/apis/appcatalog/v1alpha1" - appcatalog_cs "kmodules.xyz/custom-resources/client/clientset/versioned" - v1 "kmodules.xyz/offshoot-api/api/v1" -) - -func NewCmdRestore() *cobra.Command { - var ( - masterURL string - kubeconfigPath string - opt = esOptions{ - waitTimeout: 300, - setupOptions: restic.SetupOptions{ - ScratchDir: restic.DefaultScratchDir, - EnableCache: false, - }, - restoreOptions: restic.RestoreOptions{ - Host: restic.DefaultHost, - }, - } - ) - - cmd := &cobra.Command{ - Use: "restore-es", - Short: "Restores Elasticsearch DB Backup", - DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { - flags.EnsureRequiredFlags(cmd, "appbinding", "provider", "storage-secret-name", "storage-secret-namespace") - - // prepare client - config, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfigPath) - if err != nil { - return err - } - err = license.CheckLicenseEndpoint(config, licenseApiService, SupportedProducts) - if err != nil { - return err - } - opt.kubeClient, err = kubernetes.NewForConfig(config) - if err != nil { - return err - } - opt.catalogClient, err = appcatalog_cs.NewForConfig(config) - if err != nil { - return err - } - - targetRef := api_v1beta1.TargetRef{ - APIVersion: appcatalog.SchemeGroupVersion.String(), - Kind: appcatalog.ResourceKindApp, - Name: opt.appBindingName, - } - var restoreOutput *restic.RestoreOutput - restoreOutput, err = opt.restoreElasticsearch(targetRef) - if err != nil { - restoreOutput = &restic.RestoreOutput{ - RestoreTargetStatus: api_v1beta1.RestoreMemberStatus{ - Ref: targetRef, - Stats: []api_v1beta1.HostRestoreStats{ - { - Hostname: opt.restoreOptions.Host, - Phase: api_v1beta1.HostRestoreFailed, - Error: err.Error(), - }, - }, - }, - } - } - // If output directory specified, then write the output in "output.json" file in the specified directory - if opt.outputDir != "" { - return restoreOutput.WriteOutput(filepath.Join(opt.outputDir, restic.DefaultOutputFileName)) - } - return nil - }, - } - - cmd.Flags().StringVar(&opt.esArgs, "es-args", opt.esArgs, "Additional arguments") - cmd.Flags().Int32Var(&opt.waitTimeout, "wait-timeout", opt.waitTimeout, "Number of seconds to wait for the database to be ready") - - cmd.Flags().StringVar(&masterURL, "master", masterURL, "The address of the Kubernetes API server (overrides any value in kubeconfig)") - cmd.Flags().StringVar(&kubeconfigPath, "kubeconfig", kubeconfigPath, "Path to kubeconfig file with authorization information (the master location is set by the master flag).") - cmd.Flags().StringVar(&opt.namespace, "namespace", "default", "Namespace of Backup/Restore Session") - cmd.Flags().StringVar(&opt.appBindingName, "appbinding", opt.appBindingName, "Name of the app binding") - cmd.Flags().StringVar(&opt.storageSecret.Name, "storage-secret-name", opt.storageSecret.Name, "Name of the storage secret") - cmd.Flags().StringVar(&opt.storageSecret.Namespace, "storage-secret-namespace", opt.storageSecret.Namespace, "Namespace of the storage secret") - - cmd.Flags().StringVar(&opt.setupOptions.Provider, "provider", opt.setupOptions.Provider, "Backend provider (i.e. gcs, s3, azure etc)") - cmd.Flags().StringVar(&opt.setupOptions.Bucket, "bucket", opt.setupOptions.Bucket, "Name of the cloud bucket/container (keep empty for local backend)") - cmd.Flags().StringVar(&opt.setupOptions.Endpoint, "endpoint", opt.setupOptions.Endpoint, "Endpoint for s3/s3 compatible backend or REST server URL") - cmd.Flags().StringVar(&opt.setupOptions.Region, "region", opt.setupOptions.Region, "Region for s3/s3 compatible backend") - cmd.Flags().StringVar(&opt.setupOptions.Path, "path", opt.setupOptions.Path, "Directory inside the bucket where backup will be stored") - cmd.Flags().StringVar(&opt.setupOptions.ScratchDir, "scratch-dir", opt.setupOptions.ScratchDir, "Temporary directory") - cmd.Flags().BoolVar(&opt.setupOptions.EnableCache, "enable-cache", opt.setupOptions.EnableCache, "Specify whether to enable caching for restic") - cmd.Flags().Int64Var(&opt.setupOptions.MaxConnections, "max-connections", opt.setupOptions.MaxConnections, "Specify maximum concurrent connections for GCS, Azure and B2 backend") - - cmd.Flags().StringVar(&opt.restoreOptions.Host, "hostname", opt.restoreOptions.Host, "Name of the host machine") - cmd.Flags().StringVar(&opt.restoreOptions.SourceHost, "source-hostname", opt.restoreOptions.SourceHost, "Name of the host whose data will be restored") - cmd.Flags().StringSliceVar(&opt.restoreOptions.Snapshots, "snapshot", opt.restoreOptions.Snapshots, "Snapshots to restore") - cmd.Flags().StringVar(&opt.interimDataDir, "interim-data-dir", opt.interimDataDir, "Directory where the restored data will be stored temporarily before injecting into the desired database.") - - cmd.Flags().StringVar(&opt.outputDir, "output-dir", opt.outputDir, "Directory where output.json file will be written (keep empty if you don't need to write output in file)") - - return cmd -} - -func (opt *esOptions) restoreElasticsearch(targetRef api_v1beta1.TargetRef) (*restic.RestoreOutput, error) { - var err error - opt.setupOptions.StorageSecret, err = opt.kubeClient.CoreV1().Secrets(opt.storageSecret.Namespace).Get(context.TODO(), opt.storageSecret.Name, metav1.GetOptions{}) - if err != nil { - return nil, err - } - - // apply nice, ionice settings from env - opt.setupOptions.Nice, err = v1.NiceSettingsFromEnv() - if err != nil { - return nil, err - } - opt.setupOptions.IONice, err = v1.IONiceSettingsFromEnv() - if err != nil { - return nil, err - } - - appBinding, err := opt.catalogClient.AppcatalogV1alpha1().AppBindings(opt.namespace).Get(context.TODO(), opt.appBindingName, metav1.GetOptions{}) - if err != nil { - return nil, err - } - - // clear directory before running multielasticdump - session := opt.newSessionWrapper(MultiElasticDumpCMD) - - err = opt.setDatabaseCredentials(appBinding, session.cmd) - if err != nil { - return nil, err - } - - // clear directory before running multielasticdump - klog.Infoln("Cleaning up directory: ", opt.interimDataDir) - if err := clearDir(opt.interimDataDir); err != nil { - return nil, err - } - - url, err := appBinding.URL() - if err != nil { - return nil, err - } - - session.cmd.Args = append(session.cmd.Args, []interface{}{ - "--direction=load", - fmt.Sprintf(`--input=%v`, opt.interimDataDir), - fmt.Sprintf(`--output=%v`, url), - }...) - - err = session.setTLSParameters(appBinding, opt.setupOptions.ScratchDir) - if err != nil { - return nil, err - } - - err = opt.waitForDBReady(appBinding) - if err != nil { - return nil, err - } - - // we will restore the desired data into interim data dir before injecting into the desired database - opt.restoreOptions.RestorePaths = []string{opt.interimDataDir} - - // init restic wrapper - resticWrapper, err := restic.NewResticWrapper(opt.setupOptions) - if err != nil { - return nil, err - } - - restoreOutput, err := resticWrapper.RunRestore(opt.restoreOptions, targetRef) - if err != nil { - return nil, err - } - - // run separate shell to restore indices - // klog.Infoln("Performing multielasticdump on", hostname) - - session.sh.ShowCMD = false - session.setUserArgs(opt.esArgs) - session.sh.Command(session.cmd.Name, session.cmd.Args...) // xref: multielasticdump: https://github.com/taskrabbit/elasticsearch-dump#multielasticdump - - if err := session.sh.Run(); err != nil { - return nil, err - } - return restoreOutput, nil -} diff --git a/vendor/stash.appscode.dev/elasticsearch/pkg/root.go b/vendor/stash.appscode.dev/elasticsearch/pkg/root.go deleted file mode 100644 index afcb86a4..00000000 --- a/vendor/stash.appscode.dev/elasticsearch/pkg/root.go +++ /dev/null @@ -1,48 +0,0 @@ -/* -Copyright AppsCode Inc. and Contributors - -Licensed under the AppsCode Free Trial License 1.0.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Free-Trial-1.0.0.md - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package pkg - -import ( - "stash.appscode.dev/apimachinery/client/clientset/versioned/scheme" - - "github.com/spf13/cobra" - v "gomodules.xyz/x/version" - clientsetscheme "k8s.io/client-go/kubernetes/scheme" -) - -var SupportedProducts = []string{"stash-enterprise", "kubedb-ext-stash"} - -var licenseApiService string - -func NewRootCmd() *cobra.Command { - rootCmd := &cobra.Command{ - Use: "stash-elasticsearch", - Short: `Elasticsearch backup & restore plugin for Stash by AppsCode`, - Long: `Elasticsearch backup & restore plugin for Stash by AppsCode. For more information, visit here: https://appscode.com/products/stash`, - DisableAutoGenTag: true, - PersistentPreRunE: func(c *cobra.Command, args []string) error { - return scheme.AddToScheme(clientsetscheme.Scheme) - }, - } - rootCmd.PersistentFlags().StringVar(&licenseApiService, "license-apiservice", "", "Name of ApiService used to expose License endpoint") - - rootCmd.AddCommand(v.NewCmdVersion()) - rootCmd.AddCommand(NewCmdBackup()) - rootCmd.AddCommand(NewCmdRestore()) - - return rootCmd -} diff --git a/vendor/stash.appscode.dev/elasticsearch/pkg/utils.go b/vendor/stash.appscode.dev/elasticsearch/pkg/utils.go deleted file mode 100644 index 35fdd49a..00000000 --- a/vendor/stash.appscode.dev/elasticsearch/pkg/utils.go +++ /dev/null @@ -1,165 +0,0 @@ -/* -Copyright AppsCode Inc. and Contributors - -Licensed under the AppsCode Free Trial License 1.0.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Free-Trial-1.0.0.md - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package pkg - -import ( - "context" - "fmt" - "io/ioutil" - "os" - "os/exec" - "path/filepath" - "strings" - "time" - - stash "stash.appscode.dev/apimachinery/client/clientset/versioned" - "stash.appscode.dev/apimachinery/pkg/restic" - - shell "gomodules.xyz/go-sh" - core "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" - kmapi "kmodules.xyz/client-go/api/v1" - meta_util "kmodules.xyz/client-go/meta" - appcatalog "kmodules.xyz/custom-resources/apis/appcatalog/v1alpha1" - appcatalog_cs "kmodules.xyz/custom-resources/client/clientset/versioned" -) - -const ( - ESUser = "ADMIN_USERNAME" - ESPassword = "ADMIN_PASSWORD" - MultiElasticDumpCMD = "multielasticdump" - ESCACertFile = "root.pem" - ESAuthFile = "auth.txt" -) - -type esOptions struct { - kubeClient kubernetes.Interface - stashClient stash.Interface - catalogClient appcatalog_cs.Interface - - namespace string - backupSessionName string - appBindingName string - esArgs string - interimDataDir string - outputDir string - storageSecret kmapi.ObjectReference - waitTimeout int32 - - setupOptions restic.SetupOptions - backupOptions restic.BackupOptions - restoreOptions restic.RestoreOptions -} -type sessionWrapper struct { - sh *shell.Session - cmd *restic.Command -} - -func (opt *esOptions) newSessionWrapper(cmd string) *sessionWrapper { - return &sessionWrapper{ - sh: shell.NewSession(), - cmd: &restic.Command{ - Name: cmd, - }, - } -} - -func (opt *esOptions) setDatabaseCredentials(appBinding *appcatalog.AppBinding, cmd *restic.Command) error { - appBindingSecret, err := opt.kubeClient.CoreV1().Secrets(appBinding.Namespace).Get(context.TODO(), appBinding.Spec.Secret.Name, metav1.GetOptions{}) - if err != nil { - return err - } - - err = appBinding.TransformSecret(opt.kubeClient, appBindingSecret.Data) - if err != nil { - return err - } - // write the credential ifo into a file - // TODO: support backup without authentication - httpAuthFilePath := filepath.Join(opt.setupOptions.ScratchDir, ESAuthFile) - err = writeAuthFile(httpAuthFilePath, appBindingSecret) - if err != nil { - return err - } - cmd.Args = append(cmd.Args, fmt.Sprintf("--httpAuthFile=%s", httpAuthFilePath)) - return nil -} - -func (session *sessionWrapper) setUserArgs(args string) { - for _, arg := range strings.Fields(args) { - session.cmd.Args = append(session.cmd.Args, arg) - } -} - -func (session *sessionWrapper) setTLSParameters(appBinding *appcatalog.AppBinding, scratchDir string) error { - session.sh.SetEnv("NODE_TLS_REJECT_UNAUTHORIZED", "0") // xref: https://github.com/taskrabbit/elasticsearch-dump#bypassing-self-sign-certificate-errors - if appBinding.Spec.ClientConfig.CABundle != nil { - if err := ioutil.WriteFile(filepath.Join(scratchDir, ESCACertFile), appBinding.Spec.ClientConfig.CABundle, os.ModePerm); err != nil { - return err - } - session.cmd.Args = append(session.cmd.Args, fmt.Sprintf("--ca-input=%v", filepath.Join(scratchDir, ESCACertFile))) - } - return nil -} - -func (opt esOptions) waitForDBReady(appBinding *appcatalog.AppBinding) error { - hostname, err := appBinding.Hostname() - if err != nil { - return err - } - - port, err := appBinding.Port() - if err != nil { - return err - } - - klog.Infoln("Checking database connection") - cmd := fmt.Sprintf(`nc "%s" "%d" -w %d`, hostname, port, opt.waitTimeout) - for { - if err := exec.Command(cmd).Run(); err != nil { - break - } - klog.Infoln("Waiting... database is not ready yet") - time.Sleep(5 * time.Second) - } - klog.Infoln("Performing multielasticdump on", hostname) - return nil -} - -func clearDir(dir string) error { - if err := os.RemoveAll(dir); err != nil { - return fmt.Errorf("unable to clean datadir: %v. Reason: %v", dir, err) - } - return os.MkdirAll(dir, os.ModePerm) -} - -func must(v []byte, err error) string { - if err != nil { - panic(err) - } - return string(v) -} - -func writeAuthFile(filename string, cred *core.Secret) error { - authKeys := fmt.Sprintf("user=%s\npassword=%q", - must(meta_util.GetBytesForKeys(cred.Data, core.BasicAuthUsernameKey, ESUser)), - must(meta_util.GetBytesForKeys(cred.Data, core.BasicAuthPasswordKey, ESPassword)), - ) - return ioutil.WriteFile(filename, []byte(authKeys), 0o400) // only redable to owner -}