Skip to content

Commit

Permalink
Finish implementing the plugin
Browse files Browse the repository at this point in the history
Signed-off-by: Emruz Hossain <[email protected]>
  • Loading branch information
Emruz Hossain committed Apr 13, 2022
1 parent 103434d commit 9dbfd9f
Show file tree
Hide file tree
Showing 45 changed files with 437 additions and 773 deletions.
18 changes: 18 additions & 0 deletions .github/.kodiak.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
version = 1

[merge]
method = "squash" # default: "merge"
delete_branch_on_merge = true # default: false
optimistic_updates = true # default: true
prioritize_ready_to_merge = true # default: false

[merge.message]
title = "pull_request_title" # default: "github_default"
body = "github_default" # default: "github_default"
strip_html_comments = true # default: false

[update]
always = true # default: false

[approve]
auto_approve_usernames = ["1gtm", "tamalsaha"]
34 changes: 34 additions & 0 deletions .github/workflows/cherry-pick.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
name: cherry-pick

on:
push:
branches:
- master

jobs:
build:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v1

- name: Prepare git
env:
GITHUB_USER: 1gtm
GITHUB_TOKEN: ${{ secrets.LGTM_GITHUB_TOKEN }}
run: |
git config --global user.name "${GITHUB_USER}"
git config --global user.email "${GITHUB_USER}@appscode.com"
git remote set-url origin https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git
- name: Install GitHub CLI
run: |
curl -fsSL https://github.com/github/hub/raw/master/script/get | bash -s 2.14.1
sudo mv bin/hub /usr/local/bin
- name: Update release branches
env:
GITHUB_USER: 1gtm
GITHUB_TOKEN: ${{ secrets.LGTM_GITHUB_TOKEN }}
run: |
./hack/scripts/cherry-pick.sh
42 changes: 42 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
name: CI

on:
pull_request:
branches:
- "*"
push:
branches:
- master

jobs:
build:
name: Build
runs-on: ubuntu-latest
steps:
- name: Set up Go 1.17
uses: actions/setup-go@v1
with:
go-version: 1.17
id: go

- uses: actions/checkout@v1

- name: Set up QEMU
id: qemu
uses: docker/setup-qemu-action@v1

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1

- name: Run checks
run: |
make ci
- name: Build
env:
REGISTRY: appscodeci
DOCKER_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
USERNAME: 1gtm
run: |
docker login --username ${USERNAME} --password ${DOCKER_TOKEN}
make push
36 changes: 36 additions & 0 deletions .github/workflows/release-tracker.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
name: release-tracker

on:
pull_request:
types: [closed]

jobs:
build:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v1

- name: Prepare git
env:
GITHUB_USER: 1gtm
GITHUB_TOKEN: ${{ secrets.LGTM_GITHUB_TOKEN }}
run: |
git config --global user.name "${GITHUB_USER}"
git config --global user.email "${GITHUB_USER}@appscode.com"
git remote set-url origin https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git
- name: Install GitHub CLI
run: |
curl -fsSL https://github.com/github/hub/raw/master/script/get | bash -s 2.14.1
sudo mv bin/hub /usr/local/bin
- name: Update release tracker
if: |
github.event.action == 'closed' &&
github.event.pull_request.merged == true
env:
GITHUB_USER: 1gtm
GITHUB_TOKEN: ${{ secrets.LGTM_GITHUB_TOKEN }}
run: |
./hack/scripts/update-release-tracker.sh
50 changes: 50 additions & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
name: Release

on:
push:
tags:
- "*.*"

jobs:
build:
name: Build
runs-on: ubuntu-latest
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v1

- name: Install GitHub CLI
run: |
curl -fsSL https://github.com/github/hub/raw/master/script/get | bash -s 2.14.1
sudo mv bin/hub /usr/local/bin
- name: Print version info
id: semver
run: |
make version
- name: Set up QEMU
id: qemu
uses: docker/setup-qemu-action@v1

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1

- name: Publish to GitHub Container Registry
env:
REGISTRY: ghcr.io/stashed
DOCKER_TOKEN: ${{ secrets.LGTM_GITHUB_TOKEN }}
USERNAME: 1gtm
APPSCODE_ENV: prod
run: |
docker login ghcr.io --username ${USERNAME} --password ${DOCKER_TOKEN}
make release
- name: Publish to Docker Registry
env:
DOCKER_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
USERNAME: 1gtm
APPSCODE_ENV: prod
run: |
docker login --username ${USERNAME} --password ${DOCKER_TOKEN}
make release
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ else
endif
endif

RESTIC_VER := 0.13.0
RESTIC_VER := 0.13.1

###
### These variables should not need tweaking.
Expand Down
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[![Go Report Card](https://goreportcard.com/badge/stash.appscode.dev/manifest-backup)](https://goreportcard.com/report/stash.appscode.dev/elasticsearch)
![CI](https://github.com/stashed/elasticsearch/workflows/CI/badge.svg)
[![Go Report Card](https://goreportcard.com/badge/stash.appscode.dev/manifest-backup)](https://goreportcard.com/report/stash.appscode.dev/manifest-backup)
![CI](https://github.com/stashed/manifest-backup/workflows/CI/badge.svg)
[![Docker Pulls](https://img.shields.io/docker/pulls/stashed/manifest-backup.svg)](https://hub.docker.com/r/stashed/manifest-backup/)
[![Slack](https://shields.io/badge/Join_Slack-salck?color=4A154B&logo=slack)](https://slack.appscode.com)
[![Twitter](https://img.shields.io/twitter/follow/kubestash.svg?style=social&logo=twitter&label=Follow)](https://twitter.com/intent/follow?screen_name=KubeStash)
Expand Down
2 changes: 1 addition & 1 deletion cmd/manifest-backup/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import (
"runtime"

_ "stash.appscode.dev/apimachinery/client/clientset/versioned/fake"
"stash.appscode.dev/elasticsearch/pkg"
"stash.appscode.dev/manifest-backup/pkg"

"gomodules.xyz/logs"
_ "k8s.io/client-go/kubernetes/fake"
Expand Down
5 changes: 2 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,11 @@ require (
k8s.io/apimachinery v0.21.1
k8s.io/client-go v0.21.1
k8s.io/klog/v2 v2.9.0
kmodules.xyz/client-go v0.0.0-20220317213815-2a6d5a5784f2
kmodules.xyz/client-go v0.0.0-20220404224906-af7b092cfac5
kmodules.xyz/custom-resources v0.0.0-20220317220154-7beb809b1f5e
kmodules.xyz/offshoot-api v0.0.0-20220329041708-c076b2bcb0f8
sigs.k8s.io/yaml v1.3.0
stash.appscode.dev/apimachinery v0.19.1-0.20220407092220-1fb00845d1f7
stash.appscode.dev/elasticsearch v0.0.0-20220329183444-6b505bc9eb9f
stash.appscode.dev/apimachinery v0.19.1-0.20220413043008-5fa3647ef8f8
)

replace bitbucket.org/ww/goautoneg => gomodules.xyz/goautoneg v0.0.0-20120707110453-a547fc61f48d
Expand Down
15 changes: 7 additions & 8 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,7 @@ github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNE
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dustin/go-humanize v1.0.1-0.20220316001817-d5090ed65664/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
Expand Down Expand Up @@ -961,8 +962,9 @@ gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY
gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY=
gomodules.xyz/logs v0.0.6 h1:8+9Wkud5yBPtIvkVszubyTeFxNII30lWODom0+GZD8U=
gomodules.xyz/logs v0.0.6/go.mod h1:Q+fFtZFLEB5q86KmDehXCGuMP72Rv+Rwz0KuVxK+Gi4=
gomodules.xyz/mergo v0.3.13-0.20210702100041-9d62ff8ece4d h1:i6r7bk2jF965W3xmORgz+KYhlH608oRvrAPpcNHorlo=
gomodules.xyz/mergo v0.3.13-0.20210702100041-9d62ff8ece4d/go.mod h1:i2WNHvGpzLKI+/qWRhscddeashtzrtxCAucS2H7hrtM=
gomodules.xyz/mergo v0.3.13-0.20220214162359-48efe39fd402 h1:l67pCtMNx51TcCkBhGV1qSVnMrJ/eZPflxm02G6nhZQ=
gomodules.xyz/mergo v0.3.13-0.20220214162359-48efe39fd402/go.mod h1:i2WNHvGpzLKI+/qWRhscddeashtzrtxCAucS2H7hrtM=
gomodules.xyz/password-generator v0.2.6/go.mod h1:TvwYYTx9+P1pPwKQKfZgB/wr2Id9MqAQ3B5auY7reNg=
gomodules.xyz/password-generator v0.2.7/go.mod h1:TvwYYTx9+P1pPwKQKfZgB/wr2Id9MqAQ3B5auY7reNg=
gomodules.xyz/pointer v0.0.0-20201105040656-991dd254b680/go.mod h1:sPLsC0+yLTRecUiC5yVlyvXhZ6LAGojNCRWNNqoplvo=
Expand Down Expand Up @@ -1096,8 +1098,8 @@ k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/
kmodules.xyz/client-go v0.0.0-20210617233340-13d22e91512b/go.mod h1:A6GAK6xP5zBuWK6A/vUkkjKzcuywkms7fIxRf5wblO4=
kmodules.xyz/client-go v0.0.0-20211107190155-5bb4090d2728/go.mod h1:ENUu8pPK19xzBkVpAJHoGCI2QRvb1SqffWRt0K2sV5I=
kmodules.xyz/client-go v0.0.0-20220215012708-9963581d69a7/go.mod h1:sOq5P3AkZdv6D/skvUPwEG15NDYy5olwBllH/JXfhGI=
kmodules.xyz/client-go v0.0.0-20220317213815-2a6d5a5784f2 h1:s6MK8BVV6GHJhS6V4+5IJeV6W1GHcC4fqILencMdRkw=
kmodules.xyz/client-go v0.0.0-20220317213815-2a6d5a5784f2/go.mod h1:7pExIHGzUdu8ZGveYvAaXEhS4GdczoOy8z+hq6x6K9A=
kmodules.xyz/client-go v0.0.0-20220404224906-af7b092cfac5 h1:UkXLQ5SMS98RwnJffKAU0ANXVq9AHSLabQ9NKUhkCgA=
kmodules.xyz/client-go v0.0.0-20220404224906-af7b092cfac5/go.mod h1:7pExIHGzUdu8ZGveYvAaXEhS4GdczoOy8z+hq6x6K9A=
kmodules.xyz/constants v0.0.0-20210218100002-2c304bfda278/go.mod h1:DbiFk1bJ1KEO94t1SlAn7tzc+Zz95rSXgyUKa2nzPmY=
kmodules.xyz/crd-schema-fuzz v0.0.0-20210618002152-fae23aef5fb4/go.mod h1:IIkUctlfoptoci0BOrsUf8ya+MOG5uaeh1PE4uzaIbA=
kmodules.xyz/crd-schema-fuzz v0.0.0-20211025154117-6edb24ef11bc/go.mod h1:yLOBJKasPhnCodKSZGFZ6OGFFrp0tq3ALS9rDnYFjkg=
Expand Down Expand Up @@ -1137,8 +1139,5 @@ sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU=
stash.appscode.dev/apimachinery v0.19.0/go.mod h1:vgzwLa2KFxfbcrTS3gzhJLYmx4WdZpHfW9n8a7Hzhzc=
stash.appscode.dev/apimachinery v0.19.1-0.20220407092220-1fb00845d1f7 h1:aoFOTVQujG44AnJeaR0wKUaS0mzzFX0ktb5vilDoUgY=
stash.appscode.dev/apimachinery v0.19.1-0.20220407092220-1fb00845d1f7/go.mod h1:vgzwLa2KFxfbcrTS3gzhJLYmx4WdZpHfW9n8a7Hzhzc=
stash.appscode.dev/elasticsearch v0.0.0-20220329183444-6b505bc9eb9f h1:Z5I7YV5Z9t4R5a/zPpzyaFos4OdeC50fKKRnvJxq/CY=
stash.appscode.dev/elasticsearch v0.0.0-20220329183444-6b505bc9eb9f/go.mod h1:RePOPlnW1mvK3RJmI6Vmv28mAe/Efl6Hsi79mt9Af/8=
stash.appscode.dev/apimachinery v0.19.1-0.20220413043008-5fa3647ef8f8 h1:fI1VWIsVCa0VVpFBBCICSRiMi+d3FmLixIpyxmHm7rU=
stash.appscode.dev/apimachinery v0.19.1-0.20220413043008-5fa3647ef8f8/go.mod h1:Wvh/C9ZGQXWex8d6LKU8TodOriuPcirMuTnzgBQooLk=
19 changes: 8 additions & 11 deletions pkg/backup.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ func NewCmdBackup() *cobra.Command {
masterURL string
kubeconfigPath string
opt = options{
waitTimeout: 300,
setupOptions: restic.SetupOptions{
ScratchDir: restic.DefaultScratchDir,
EnableCache: false,
Expand All @@ -57,10 +56,10 @@ func NewCmdBackup() *cobra.Command {

cmd := &cobra.Command{
Use: "backup-manifest",
Short: "Takes a backup of Elasticsearch DB",
Short: "Takes a backup of Kubernetes manifests",
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
flags.EnsureRequiredFlags(cmd, "appbinding", "provider", "storage-secret-name", "storage-secret-namespace")
flags.EnsureRequiredFlags(cmd, "provider", "storage-secret-name", "storage-secret-namespace")
time.Sleep(time.Second * 5)

// prepare client
Expand Down Expand Up @@ -119,9 +118,6 @@ func NewCmdBackup() *cobra.Command {
return nil
},
}

cmd.Flags().Int32Var(&opt.waitTimeout, "wait-timeout", opt.waitTimeout, "Number of seconds to wait for the database to be ready")

cmd.Flags().StringVar(&masterURL, "master", masterURL, "The address of the Kubernetes API server (overrides any value in kubeconfig)")
cmd.Flags().StringVar(&kubeconfigPath, "kubeconfig", kubeconfigPath, "Path to kubeconfig file with authorization information (the master location is set by the master flag).")
cmd.Flags().StringVar(&opt.namespace, "namespace", "default", "Namespace of Backup/Restore Session")
Expand All @@ -139,7 +135,6 @@ func NewCmdBackup() *cobra.Command {
cmd.Flags().Int64Var(&opt.setupOptions.MaxConnections, "max-connections", opt.setupOptions.MaxConnections, "Specify maximum concurrent connections for GCS, Azure and B2 backend")

cmd.Flags().StringVar(&opt.backupOptions.Host, "hostname", opt.backupOptions.Host, "Name of the host machine")
cmd.Flags().StringVar(&opt.interimDataDir, "interim-data-dir", opt.interimDataDir, "Directory where the targeted data will be stored temporarily before uploading to the backend.")
cmd.Flags().StringVar(&opt.invokerKind, "invoker-kind", opt.invokerKind, "Kind of the backup invoker")
cmd.Flags().StringVar(&opt.invokerName, "invoker-name", opt.invokerName, "Name of the respective backup invoker")
cmd.Flags().StringVar(&opt.targetKind, "target-kind", opt.targetKind, "Kind of the Target")
Expand All @@ -156,6 +151,7 @@ func NewCmdBackup() *cobra.Command {
cmd.Flags().BoolVar(&opt.backupOptions.RetentionPolicy.DryRun, "retention-dry-run", opt.backupOptions.RetentionPolicy.DryRun, "Specify whether to test retention policy without deleting actual data")

cmd.Flags().StringVar(&opt.outputDir, "output-dir", opt.outputDir, "Directory where output.json file will be written (keep empty if you don't need to write output in file)")
cmd.Flags().BoolVar(&opt.sanitize, "output-dir", true, "Specify whether to remove the decorators from the manifest (default is true)")

return cmd
}
Expand Down Expand Up @@ -194,15 +190,16 @@ func (opt *options) backupManifests(targetRef v1beta1.TargetRef) (*restic.Backup
return nil, err
}

klog.Infoln("Cleaning up directory: ", opt.interimDataDir)
if err := clearDir(opt.interimDataDir); err != nil {
klog.Infoln("Cleaning up directory: ", opt.dataDir)
opt.dataDir = filepath.Join(opt.setupOptions.ScratchDir, "manifests")
if err := clearDir(opt.dataDir); err != nil {
return nil, err
}

mgOpts := manager.BackupOptions{
Config: opt.config,
Sanitize: opt.sanitize,
DataDir: opt.interimDataDir,
DataDir: opt.dataDir,
Target: targetRef,
Storage: manager.NewFileWriter(),
}
Expand All @@ -212,7 +209,7 @@ func (opt *options) backupManifests(targetRef v1beta1.TargetRef) (*restic.Backup
}

// dumped data has been stored in the interim data dir. Now, we will backup this directory using Stash.
opt.backupOptions.BackupPaths = []string{opt.interimDataDir}
opt.backupOptions.BackupPaths = []string{opt.dataDir}

// init restic wrapper
resticWrapper, err := restic.NewResticWrapper(opt.setupOptions)
Expand Down
Loading

0 comments on commit 9dbfd9f

Please sign in to comment.