Add max-connections for GCS, Azure, B2 backend (#730)

- [x] Merge and revendor kmodules/objectstore-api#8 Fixes #545
stashed · Apr 5, 2019 · a8aef59 · a8aef59
1 parent dd3f459
commit a8aef59
Show file tree

Hide file tree

Showing 71 changed files with 3,117 additions and 1,570 deletions.
diff --git a/apis/constants.go b/apis/constants.go
@@ -37,7 +37,8 @@ const (
 
 	// default true
 	// false when TmpDir.DisableCaching is true in backupConfig/restoreSession
-	EnableCache = "ENABLE_CACHE"
+	EnableCache    = "ENABLE_CACHE"
+	MaxConnections = "MAX_CONNECTIONS"
 
 	StatusSubresourceEnabled = "ENABLE_STATUS_SUBRESOURCE"
 )
diff --git a/docs/examples/functions/pg_backup.yaml b/docs/examples/functions/pg_backup.yaml
@@ -23,6 +23,8 @@ spec:
   - --retention-prune=${RETENTION_PRUNE:=}
   - --retention-dry-run=${RETENTION_DRY_RUN:=}
   - --output-dir=${outputDir:=}
+  - --enable-cache=${ENABLE_CACHE:=true}
+  - --max-connections=${MAX_CONNECTIONS:=0}
   #- --metrics-enabled
   #- --metrics-pushgateway-url
   #- --metrics-dir

diff --git a/docs/examples/functions/pvc_backup.yaml b/docs/examples/functions/pvc_backup.yaml
@@ -24,6 +24,8 @@ spec:
   - --retention-prune=${RETENTION_PRUNE:=false}
   - --retention-dry-run=${RETENTION_DRY_RUN:=false}
   - --output-dir=${outputDir}
+  - --enable-cache=${ENABLE_CACHE:=true}
+  - --max-connections=${MAX_CONNECTIONS:=0}
   #- --metrics-enabled
   #- --metrics-pushgateway-url
   #- --metrics-dir

diff --git a/docs/examples/functions/pvc_restore.yaml b/docs/examples/functions/pvc_restore.yaml
@@ -16,6 +16,8 @@ spec:
   - --restore-dirs=${RESTORE_DIRECTORIES}
   - --snapshots=${RESTORE_SNAPSHOTS:=}
   - --output-dir=${outputDir}
+  - --enable-cache=${ENABLE_CACHE:=true}
+  - --max-connections=${MAX_CONNECTIONS:=0}
   #- --metrics-enabled
   #- --metrics-pushgateway-url
   #- --metrics-dir

diff --git a/docs/examples/mongo/functions.yaml b/docs/examples/mongo/functions.yaml
@@ -40,6 +40,8 @@ spec:
   - --retention-prune=${RETENTION_PRUNE:=}
   - --retention-dry-run=${RETENTION_DRY_RUN:=}
   - --output-dir=${outputDir:=}
+  - --enable-cache=${ENABLE_CACHE:=true}
+  - --max-connections=${MAX_CONNECTIONS:=0}
   #- --metrics-enabled
   #- --metrics-pushgateway-url
   #- --metrics-dir
@@ -74,6 +76,8 @@ spec:
   - --app-binding=${TARGET_NAME:=}
   - --snapshot=${RESTORE_SNAPSHOTS:=} # TODO: sliceVar
   - --output-dir=${outputDir:=}
+  - --enable-cache=${ENABLE_CACHE:=true}
+  - --max-connections=${MAX_CONNECTIONS:=0}
   #- --metrics-enabled
   #- --metrics-pushgateway-url
   #- --metrics-dir

diff --git a/docs/examples/mysql/functions.yaml b/docs/examples/mysql/functions.yaml
@@ -40,6 +40,8 @@ spec:
   - --retention-prune=${RETENTION_PRUNE:=}
   - --retention-dry-run=${RETENTION_DRY_RUN:=}
   - --output-dir=${outputDir:=}
+  - --enable-cache=${ENABLE_CACHE:=true}
+  - --max-connections=${MAX_CONNECTIONS:=0}
   #- --metrics-enabled
   #- --metrics-pushgateway-url
   #- --metrics-dir
@@ -74,6 +76,8 @@ spec:
   - --app-binding=${TARGET_NAME:=}
   - --snapshot=${RESTORE_SNAPSHOTS:=} # TODO: sliceVar
   - --output-dir=${outputDir:=}
+  - --enable-cache=${ENABLE_CACHE:=true}
+  - --max-connections=${MAX_CONNECTIONS:=0}
   #- --metrics-enabled
   #- --metrics-pushgateway-url
   #- --metrics-dir

diff --git a/docs/examples/pg/functions.yaml b/docs/examples/pg/functions.yaml
@@ -38,6 +38,8 @@ spec:
   - --retention-prune=${RETENTION_PRUNE:=false}
   - --retention-dry-run=${RETENTION_DRY_RUN:=false}
   - --output-dir=${outputDir:=}
+  - --enable-cache=${ENABLE_CACHE:=true}
+  - --max-connections=${MAX_CONNECTIONS:=0}
   #- --metrics-enabled
   #- --metrics-pushgateway-url
   #- --metrics-dir
@@ -70,6 +72,8 @@ spec:
   - --app-binding=${TARGET_NAME:=}
   - --snapshot=${RESTORE_SNAPSHOTS:=} # TODO: sliceVar
   - --output-dir=${outputDir:=}
+  - --enable-cache=${ENABLE_CACHE:=true}
+  - --max-connections=${MAX_CONNECTIONS:=0}
   #- --metrics-enabled
   #- --metrics-pushgateway-url
   #- --metrics-dir

diff --git a/docs/examples/pvc/functions.yaml b/docs/examples/pvc/functions.yaml
@@ -43,6 +43,7 @@ spec:
   - --retention-dry-run=${RETENTION_DRY_RUN:=false}
   - --output-dir=${outputDir}
   - --enable-cache=${ENABLE_CACHE:=true}
+  - --max-connections=${MAX_CONNECTIONS:=0}
   #- --metrics-enabled
   #- --metrics-pushgateway-url
   #- --metrics-dir
@@ -74,6 +75,7 @@ spec:
   - --snapshots=${RESTORE_SNAPSHOTS:=}
   - --output-dir=${outputDir}
   - --enable-cache=${ENABLE_CACHE:=true}
+  - --max-connections=${MAX_CONNECTIONS:=0}
   #- --metrics-enabled
   #- --metrics-pushgateway-url
   #- --metrics-dir

diff --git a/docs/examples/rest/functions.yaml b/docs/examples/rest/functions.yaml
@@ -43,6 +43,8 @@ spec:
   - --retention-prune=${RETENTION_PRUNE:=false}
   - --retention-dry-run=${RETENTION_DRY_RUN:=false}
   - --output-dir=${outputDir}
+  - --enable-cache=${ENABLE_CACHE:=true}
+  - --max-connections=${MAX_CONNECTIONS:=0}
   #- --metrics-enabled
   #- --metrics-pushgateway-url
   #- --metrics-dir
@@ -76,6 +78,8 @@ spec:
   - --restore-dirs=${RESTORE_DIRECTORIES}
   - --snapshots=${RESTORE_SNAPSHOTS:=}
   - --output-dir=${outputDir}
+  - --enable-cache=${ENABLE_CACHE:=true}
+  - --max-connections=${MAX_CONNECTIONS:=0}
   #- --metrics-enabled
   #- --metrics-pushgateway-url
   #- --metrics-dir

diff --git a/glide.lock b/glide.lock
diff --git a/glide.yaml b/glide.yaml
@@ -7,19 +7,23 @@ import:
   version: v1.14.12
 - package: github.com/cenkalti/backoff
   version: v2.1.1
+- package: github.com/codeskyblue/go-sh
+  version: master
 - package: github.com/cpuguy83/go-md2man
   version: v1.0.8
 - package: github.com/davecgh/go-spew
   version: v1.1.1
 - package: github.com/evanphx/json-patch
-  version: v4.1.0
+  version: 5858425f75500d40c52783dce87d085a483ce135
 - package: github.com/go-ini/ini
   version: v1.40.0
 - package: github.com/go-openapi/spec
   version: v0.18.0
 - package: github.com/golang/glog
 - package: github.com/golang/protobuf
   version: v1.1.0
+- package: github.com/google/go-cmp
+  version: 6f77996f0c42f7b84e5a2b252227263f93432e9b
 - package: github.com/gorilla/websocket
   version: v1.4.0
 - package: github.com/graymeta/stow
@@ -80,27 +84,29 @@ import:
   repo: https://github.com/kmodules/apiserver.git
   vcs: git
   version: ac-1.13.0
+- package: k8s.io/cli-runtime
+  version: kubernetes-1.13.0
 - package: k8s.io/client-go
   version: v10.0.0
 - package: k8s.io/kube-aggregator
   version: kubernetes-1.13.0
-- package: k8s.io/cli-runtime
-  version: kubernetes-1.13.0
 - package: k8s.io/kube-openapi
   version: c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d
 - package: k8s.io/kubernetes
   version: v1.13.0
+- package: k8s.io/utils
+  version: 66066c83e385e385ccc3c964b44fd7dcd413d0ed
 - package: kmodules.xyz/client-go
   version: release-10.0
+- package: kmodules.xyz/custom-resources
+  version: release-10.0
 - package: kmodules.xyz/objectstore-api
   version: release-10.0
 - package: kmodules.xyz/offshoot-api
   version: release-10.0
-- package: kmodules.xyz/webhook-runtime
-  version: release-10.0
-- package: github.com/codeskyblue/go-sh
-  version: master
-- package: kmodules.xyz/custom-resources
-  version: release-10.0
 - package: kmodules.xyz/openshift
   version: release-10.0
+- package: kmodules.xyz/webhook-runtime
+  version: release-10.0
+- package: sigs.k8s.io/yaml
+  version: v1.1.0
diff --git a/pkg/cmds/backup_mongo.go b/pkg/cmds/backup_mongo.go
@@ -136,6 +136,7 @@ func NewCmdBackupMongo() *cobra.Command {
 	cmd.Flags().StringVar(&setupOpt.SecretDir, "secret-dir", setupOpt.SecretDir, "Directory where storage secret has been mounted")
 	cmd.Flags().StringVar(&setupOpt.ScratchDir, "scratch-dir", setupOpt.ScratchDir, "Temporary directory")
 	cmd.Flags().BoolVar(&setupOpt.EnableCache, "enable-cache", setupOpt.EnableCache, "Specify weather to enable caching for restic")
+	cmd.Flags().IntVar(&setupOpt.MaxConnections, "max-connections", setupOpt.MaxConnections, "Specify maximum concurrent connections for GCS, Azure and B2 backend")
 
 	cmd.Flags().StringVar(&backupOpt.Host, "hostname", backupOpt.Host, "Name of the host machine")
 

diff --git a/pkg/cmds/backup_mysql.go b/pkg/cmds/backup_mysql.go
@@ -133,6 +133,7 @@ func NewCmdBackupMySql() *cobra.Command {
 	cmd.Flags().StringVar(&setupOpt.SecretDir, "secret-dir", setupOpt.SecretDir, "Directory where storage secret has been mounted")
 	cmd.Flags().StringVar(&setupOpt.ScratchDir, "scratch-dir", setupOpt.ScratchDir, "Temporary directory")
 	cmd.Flags().BoolVar(&setupOpt.EnableCache, "enable-cache", setupOpt.EnableCache, "Specify weather to enable caching for restic")
+	cmd.Flags().IntVar(&setupOpt.MaxConnections, "max-connections", setupOpt.MaxConnections, "Specify maximum concurrent connections for GCS, Azure and B2 backend")
 
 	cmd.Flags().StringVar(&backupOpt.Host, "hostname", backupOpt.Host, "Name of the host machine")
 

diff --git a/pkg/cmds/backup_pg.go b/pkg/cmds/backup_pg.go
@@ -137,6 +137,7 @@ func NewCmdBackupPG() *cobra.Command {
 	cmd.Flags().StringVar(&setupOpt.SecretDir, "secret-dir", setupOpt.SecretDir, "Directory where storage secret has been mounted")
 	cmd.Flags().StringVar(&setupOpt.ScratchDir, "scratch-dir", setupOpt.ScratchDir, "Temporary directory")
 	cmd.Flags().BoolVar(&setupOpt.EnableCache, "enable-cache", setupOpt.EnableCache, "Specify weather to enable caching for restic")
+	cmd.Flags().IntVar(&setupOpt.MaxConnections, "max-connections", setupOpt.MaxConnections, "Specify maximum concurrent connections for GCS, Azure and B2 backend")
 
 	cmd.Flags().StringVar(&backupOpt.Host, "hostname", backupOpt.Host, "Name of the host machine")
 

diff --git a/pkg/cmds/backup_pvc.go b/pkg/cmds/backup_pvc.go
@@ -68,6 +68,7 @@ func NewCmdBackupPVC() *cobra.Command {
 	cmd.Flags().StringVar(&setupOpt.SecretDir, "secret-dir", setupOpt.SecretDir, "Directory where storage secret has been mounted")
 	cmd.Flags().StringVar(&setupOpt.ScratchDir, "scratch-dir", setupOpt.ScratchDir, "Temporary directory")
 	cmd.Flags().BoolVar(&setupOpt.EnableCache, "enable-cache", setupOpt.EnableCache, "Specify weather to enable caching for restic")
+	cmd.Flags().IntVar(&setupOpt.MaxConnections, "max-connections", setupOpt.MaxConnections, "Specify maximum concurrent connections for GCS, Azure and B2 backend")
 
 	cmd.Flags().StringVar(&backupOpt.Host, "hostname", backupOpt.Host, "Name of the host machine")
 	cmd.Flags().StringSliceVar(&backupOpt.BackupDirs, "backup-dirs", backupOpt.BackupDirs, "List of directories to be backed up")

diff --git a/pkg/cmds/restore.go b/pkg/cmds/restore.go
@@ -57,6 +57,7 @@ func NewCmdRestore() *cobra.Command {
 	cmd.Flags().StringVar(&opt.RestoreSessionName, "restore-session", opt.RestoreSessionName, "Name of the RestoreSession CRD.")
 	cmd.Flags().DurationVar(&opt.BackoffMaxWait, "backoff-max-wait", 0, "Maximum wait for initial response from kube apiserver; 0 disables the timeout")
 	cmd.Flags().BoolVar(&opt.SetupOpt.EnableCache, "enable-cache", opt.SetupOpt.EnableCache, "Specify weather to enable caching for restic")
+	cmd.Flags().IntVar(&opt.SetupOpt.MaxConnections, "max-connections", opt.SetupOpt.MaxConnections, "Specify maximum concurrent connections for GCS, Azure and B2 backend")
 	cmd.Flags().StringVar(&opt.SetupOpt.SecretDir, "secret-dir", opt.SetupOpt.SecretDir, "Directory where storage secret has been mounted")
 	cmd.Flags().BoolVar(&opt.SetupOpt.EnableCache, "enable-cache", opt.SetupOpt.EnableCache, "Specify weather to enable caching for restic")
 

diff --git a/pkg/cmds/restore_mongo.go b/pkg/cmds/restore_mongo.go
@@ -127,6 +127,7 @@ func NewCmdRestoreMongo() *cobra.Command {
 	cmd.Flags().StringVar(&setupOpt.SecretDir, "secret-dir", setupOpt.SecretDir, "Directory where storage secret has been mounted")
 	cmd.Flags().StringVar(&setupOpt.ScratchDir, "scratch-dir", setupOpt.ScratchDir, "Temporary directory")
 	cmd.Flags().BoolVar(&setupOpt.EnableCache, "enable-cache", setupOpt.EnableCache, "Specify weather to enable caching for restic")
+	cmd.Flags().IntVar(&setupOpt.MaxConnections, "max-connections", setupOpt.MaxConnections, "Specify maximum concurrent connections for GCS, Azure and B2 backend")
 
 	cmd.Flags().StringVar(&dumpOpt.Host, "hostname", dumpOpt.Host, "Name of the host machine")
 	// TODO: sliceVar

diff --git a/pkg/cmds/restore_mysql.go b/pkg/cmds/restore_mysql.go
@@ -123,6 +123,7 @@ func NewCmdRestoreMySql() *cobra.Command {
 	cmd.Flags().StringVar(&setupOpt.SecretDir, "secret-dir", setupOpt.SecretDir, "Directory where storage secret has been mounted")
 	cmd.Flags().StringVar(&setupOpt.ScratchDir, "scratch-dir", setupOpt.ScratchDir, "Temporary directory")
 	cmd.Flags().BoolVar(&setupOpt.EnableCache, "enable-cache", setupOpt.EnableCache, "Specify weather to enable caching for restic")
+	cmd.Flags().IntVar(&setupOpt.MaxConnections, "max-connections", setupOpt.MaxConnections, "Specify maximum concurrent connections for GCS, Azure and B2 backend")
 
 	cmd.Flags().StringVar(&dumpOpt.Host, "hostname", dumpOpt.Host, "Name of the host machine")
 	// TODO: sliceVar

diff --git a/pkg/cmds/restore_pg.go b/pkg/cmds/restore_pg.go
@@ -123,6 +123,7 @@ func NewCmdRestorePG() *cobra.Command {
 	cmd.Flags().StringVar(&setupOpt.SecretDir, "secret-dir", setupOpt.SecretDir, "Directory where storage secret has been mounted")
 	cmd.Flags().StringVar(&setupOpt.ScratchDir, "scratch-dir", setupOpt.ScratchDir, "Temporary directory")
 	cmd.Flags().BoolVar(&setupOpt.EnableCache, "enable-cache", setupOpt.EnableCache, "Specify weather to enable caching for restic")
+	cmd.Flags().IntVar(&setupOpt.MaxConnections, "max-connections", setupOpt.MaxConnections, "Specify maximum concurrent connections for GCS, Azure and B2 backend")
 
 	cmd.Flags().StringVar(&dumpOpt.Host, "hostname", dumpOpt.Host, "Name of the host machine")
 	// TODO: sliceVar

diff --git a/pkg/cmds/restore_pvc.go b/pkg/cmds/restore_pvc.go
@@ -65,6 +65,7 @@ func NewCmdRestorePVC() *cobra.Command {
 	cmd.Flags().StringVar(&setupOpt.SecretDir, "secret-dir", setupOpt.SecretDir, "Directory where storage secret has been mounted")
 	cmd.Flags().StringVar(&setupOpt.ScratchDir, "scratch-dir", setupOpt.ScratchDir, "Temporary directory")
 	cmd.Flags().BoolVar(&setupOpt.EnableCache, "enable-cache", setupOpt.EnableCache, "Specify weather to enable caching for restic")
+	cmd.Flags().IntVar(&setupOpt.MaxConnections, "max-connections", setupOpt.MaxConnections, "Specify maximum concurrent connections for GCS, Azure and B2 backend")
 
 	cmd.Flags().StringVar(&restoreOpt.SourceHost, "hostname", restoreOpt.SourceHost, "Name of the host machine")
 	cmd.Flags().StringSliceVar(&restoreOpt.RestoreDirs, "restore-dirs", restoreOpt.RestoreDirs, "List of directories to be restored")

diff --git a/pkg/cmds/run_backup.go b/pkg/cmds/run_backup.go
@@ -64,6 +64,7 @@ func NewCmdRunBackup() *cobra.Command {
 	cmd.Flags().StringVar(&con.BackupConfigurationName, "backup-configuration", con.BackupConfigurationName, "Set BackupConfiguration Name")
 	cmd.Flags().StringVar(&con.SetupOpt.SecretDir, "secret-dir", con.SetupOpt.SecretDir, "Directory where storage secret has been mounted")
 	cmd.Flags().BoolVar(&con.SetupOpt.EnableCache, "enable-cache", con.SetupOpt.EnableCache, "Specify weather to enable caching for restic")
+	cmd.Flags().IntVar(&con.SetupOpt.MaxConnections, "max-connections", con.SetupOpt.MaxConnections, "Specify maximum concurrent connections for GCS, Azure and B2 backend")
 	cmd.Flags().BoolVar(&con.Metrics.Enabled, "metrics-enabled", con.Metrics.Enabled, "Specify weather to export Prometheus metrics")
 	cmd.Flags().BoolVar(&con.EnableRBAC, "enable-rbac", con.EnableRBAC, "Enable RBAC")
 	cmd.Flags().StringVar(&con.Metrics.PushgatewayURL, "pushgateway-url", con.Metrics.PushgatewayURL, "URL of Prometheus pushgateway used to cache backup metrics")

diff --git a/pkg/controller/inputs.go b/pkg/controller/inputs.go
@@ -101,6 +101,7 @@ func (c *StashController) inputsForRepository(repository *apiAlpha.Repository) (
 	if repository.Spec.Backend.Rest != nil && repository.Spec.Backend.Rest.URL != "" {
 		inputs[apis.RepositoryURL] = repository.Spec.Backend.Rest.URL
 	}
+	inputs[apis.MaxConnections] = strconv.Itoa(util.GetMaxConnections(repository.Spec.Backend))
 	return
 }
-Original file line number
+Diff line change
@@ Expand Up @@
     	if repository.Spec.Backend.Rest != nil && repository.Spec.Backend.Rest.URL != "" {
     		inputs[apis.RepositoryURL] = repository.Spec.Backend.Rest.URL
     	}
+    	inputs[apis.MaxConnections] = strconv.Itoa(util.GetMaxConnections(repository.Spec.Backend))
     	return
     }
@@ Expand Down @@