Add max-connections for GCS, Azure, B2 backend (#730)

- [x] Merge and revendor kmodules/objectstore-api#8

Fixes #545

apis/constants.go

@@ -37,7 +37,8 @@ const (
 
 	// default true
 	// false when TmpDir.DisableCaching is true in backupConfig/restoreSession
-	EnableCache = "ENABLE_CACHE"
+	EnableCache    = "ENABLE_CACHE"
+	MaxConnections = "MAX_CONNECTIONS"
 
 	StatusSubresourceEnabled = "ENABLE_STATUS_SUBRESOURCE"
 )

docs/examples/functions/pg_backup.yaml

@@ -23,6 +23,8 @@ spec:
   - --retention-prune=${RETENTION_PRUNE:=}
   - --retention-dry-run=${RETENTION_DRY_RUN:=}
   - --output-dir=${outputDir:=}
+  - --enable-cache=${ENABLE_CACHE:=true}
+  - --max-connections=${MAX_CONNECTIONS:=0}
   #- --metrics-enabled
   #- --metrics-pushgateway-url
   #- --metrics-dir

docs/examples/functions/pvc_backup.yaml

@@ -24,6 +24,8 @@ spec:
   - --retention-prune=${RETENTION_PRUNE:=false}
   - --retention-dry-run=${RETENTION_DRY_RUN:=false}
   - --output-dir=${outputDir}
+  - --enable-cache=${ENABLE_CACHE:=true}
+  - --max-connections=${MAX_CONNECTIONS:=0}
   #- --metrics-enabled
   #- --metrics-pushgateway-url
   #- --metrics-dir

docs/examples/functions/pvc_restore.yaml

@@ -16,6 +16,8 @@ spec:
   - --restore-dirs=${RESTORE_DIRECTORIES}
   - --snapshots=${RESTORE_SNAPSHOTS:=}
   - --output-dir=${outputDir}
+  - --enable-cache=${ENABLE_CACHE:=true}
+  - --max-connections=${MAX_CONNECTIONS:=0}
   #- --metrics-enabled
   #- --metrics-pushgateway-url
   #- --metrics-dir

docs/examples/mongo/functions.yaml

@@ -40,6 +40,8 @@ spec:
   - --retention-prune=${RETENTION_PRUNE:=}
   - --retention-dry-run=${RETENTION_DRY_RUN:=}
   - --output-dir=${outputDir:=}
+  - --enable-cache=${ENABLE_CACHE:=true}
+  - --max-connections=${MAX_CONNECTIONS:=0}
   #- --metrics-enabled
   #- --metrics-pushgateway-url
   #- --metrics-dir
@@ -74,6 +76,8 @@ spec:
   - --app-binding=${TARGET_NAME:=}
   - --snapshot=${RESTORE_SNAPSHOTS:=} # TODO: sliceVar
   - --output-dir=${outputDir:=}
+  - --enable-cache=${ENABLE_CACHE:=true}
+  - --max-connections=${MAX_CONNECTIONS:=0}
   #- --metrics-enabled
   #- --metrics-pushgateway-url
   #- --metrics-dir

docs/examples/mysql/functions.yaml

@@ -40,6 +40,8 @@ spec:
   - --retention-prune=${RETENTION_PRUNE:=}
   - --retention-dry-run=${RETENTION_DRY_RUN:=}
   - --output-dir=${outputDir:=}
+  - --enable-cache=${ENABLE_CACHE:=true}
+  - --max-connections=${MAX_CONNECTIONS:=0}
   #- --metrics-enabled
   #- --metrics-pushgateway-url
   #- --metrics-dir
@@ -74,6 +76,8 @@ spec:
   - --app-binding=${TARGET_NAME:=}
   - --snapshot=${RESTORE_SNAPSHOTS:=} # TODO: sliceVar
   - --output-dir=${outputDir:=}
+  - --enable-cache=${ENABLE_CACHE:=true}
+  - --max-connections=${MAX_CONNECTIONS:=0}
   #- --metrics-enabled
   #- --metrics-pushgateway-url
   #- --metrics-dir

docs/examples/pg/functions.yaml

@@ -38,6 +38,8 @@ spec:
   - --retention-prune=${RETENTION_PRUNE:=false}
   - --retention-dry-run=${RETENTION_DRY_RUN:=false}
   - --output-dir=${outputDir:=}
+  - --enable-cache=${ENABLE_CACHE:=true}
+  - --max-connections=${MAX_CONNECTIONS:=0}
   #- --metrics-enabled
   #- --metrics-pushgateway-url
   #- --metrics-dir
@@ -70,6 +72,8 @@ spec:
   - --app-binding=${TARGET_NAME:=}
   - --snapshot=${RESTORE_SNAPSHOTS:=} # TODO: sliceVar
   - --output-dir=${outputDir:=}
+  - --enable-cache=${ENABLE_CACHE:=true}
+  - --max-connections=${MAX_CONNECTIONS:=0}
   #- --metrics-enabled
   #- --metrics-pushgateway-url
   #- --metrics-dir

docs/examples/pvc/functions.yaml

@@ -43,6 +43,7 @@ spec:
   - --retention-dry-run=${RETENTION_DRY_RUN:=false}
   - --output-dir=${outputDir}
   - --enable-cache=${ENABLE_CACHE:=true}
+  - --max-connections=${MAX_CONNECTIONS:=0}
   #- --metrics-enabled
   #- --metrics-pushgateway-url
   #- --metrics-dir
@@ -74,6 +75,7 @@ spec:
   - --snapshots=${RESTORE_SNAPSHOTS:=}
   - --output-dir=${outputDir}
   - --enable-cache=${ENABLE_CACHE:=true}
+  - --max-connections=${MAX_CONNECTIONS:=0}
   #- --metrics-enabled
   #- --metrics-pushgateway-url
   #- --metrics-dir

docs/examples/rest/functions.yaml

@@ -43,6 +43,8 @@ spec:
   - --retention-prune=${RETENTION_PRUNE:=false}
   - --retention-dry-run=${RETENTION_DRY_RUN:=false}
   - --output-dir=${outputDir}
+  - --enable-cache=${ENABLE_CACHE:=true}
+  - --max-connections=${MAX_CONNECTIONS:=0}
   #- --metrics-enabled
   #- --metrics-pushgateway-url
   #- --metrics-dir
@@ -76,6 +78,8 @@ spec:
   - --restore-dirs=${RESTORE_DIRECTORIES}
   - --snapshots=${RESTORE_SNAPSHOTS:=}
   - --output-dir=${outputDir}
+  - --enable-cache=${ENABLE_CACHE:=true}
+  - --max-connections=${MAX_CONNECTIONS:=0}
   #- --metrics-enabled
   #- --metrics-pushgateway-url
   #- --metrics-dir

glide.yaml

@@ -7,19 +7,23 @@ import:
   version: v1.14.12
 - package: github.com/cenkalti/backoff
   version: v2.1.1
+- package: github.com/codeskyblue/go-sh
+  version: master
 - package: github.com/cpuguy83/go-md2man
   version: v1.0.8
 - package: github.com/davecgh/go-spew
   version: v1.1.1
 - package: github.com/evanphx/json-patch
-  version: v4.1.0
+  version: 5858425f75500d40c52783dce87d085a483ce135
 - package: github.com/go-ini/ini
   version: v1.40.0
 - package: github.com/go-openapi/spec
   version: v0.18.0
 - package: github.com/golang/glog
 - package: github.com/golang/protobuf
   version: v1.1.0
+- package: github.com/google/go-cmp
+  version: 6f77996f0c42f7b84e5a2b252227263f93432e9b
 - package: github.com/gorilla/websocket
   version: v1.4.0
 - package: github.com/graymeta/stow
@@ -80,27 +84,29 @@ import:
   repo: https://github.com/kmodules/apiserver.git
   vcs: git
   version: ac-1.13.0
+- package: k8s.io/cli-runtime
+  version: kubernetes-1.13.0
 - package: k8s.io/client-go
   version: v10.0.0
 - package: k8s.io/kube-aggregator
   version: kubernetes-1.13.0
-- package: k8s.io/cli-runtime
-  version: kubernetes-1.13.0
 - package: k8s.io/kube-openapi
   version: c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d
 - package: k8s.io/kubernetes
   version: v1.13.0
+- package: k8s.io/utils
+  version: 66066c83e385e385ccc3c964b44fd7dcd413d0ed
 - package: kmodules.xyz/client-go
   version: release-10.0
+- package: kmodules.xyz/custom-resources
+  version: release-10.0
 - package: kmodules.xyz/objectstore-api
   version: release-10.0
 - package: kmodules.xyz/offshoot-api
   version: release-10.0
-- package: kmodules.xyz/webhook-runtime
-  version: release-10.0
-- package: github.com/codeskyblue/go-sh
-  version: master
-- package: kmodules.xyz/custom-resources
-  version: release-10.0
 - package: kmodules.xyz/openshift
   version: release-10.0
+- package: kmodules.xyz/webhook-runtime
+  version: release-10.0
+- package: sigs.k8s.io/yaml
+  version: v1.1.0

pkg/restic/commands.go

@@ -37,6 +37,7 @@ func (w *ResticWrapper) listSnapshots(snapshotIDs []string) ([]Snapshot, error)
 	result := make([]Snapshot, 0)
 	args := w.appendCacheDirFlag([]interface{}{"snapshots", "--json", "--quiet", "--no-lock"})
 	args = w.appendCaCertFlag(args)
+	args = w.appendMaxConnectionsFlag(args)
 	for _, id := range snapshotIDs {
 		args = append(args, id)
 	}
@@ -51,6 +52,7 @@ func (w *ResticWrapper) listSnapshots(snapshotIDs []string) ([]Snapshot, error)
 func (w *ResticWrapper) deleteSnapshots(snapshotIDs []string) ([]byte, error) {
 	args := w.appendCacheDirFlag([]interface{}{"forget", "--quiet", "--prune"})
 	args = w.appendCaCertFlag(args)
+	args = w.appendMaxConnectionsFlag(args)
 	for _, id := range snapshotIDs {
 		args = append(args, id)
 	}
@@ -62,9 +64,11 @@ func (w *ResticWrapper) initRepositoryIfAbsent() ([]byte, error) {
 	log.Infoln("Ensuring restic repository in the backend")
 	args := w.appendCacheDirFlag([]interface{}{"snapshots", "--json"})
 	args = w.appendCaCertFlag(args)
+	args = w.appendMaxConnectionsFlag(args)
 	if _, err := w.run(Command{Name: ResticCMD, Args: args}); err != nil {
 		args = w.appendCacheDirFlag([]interface{}{"init"})
 		args = w.appendCaCertFlag(args)
+		args = w.appendMaxConnectionsFlag(args)
 
 		return w.run(Command{Name: ResticCMD, Args: args})
 	}
@@ -86,6 +90,7 @@ func (w *ResticWrapper) backup(path, host string, tags []string) ([]byte, error)
 	args = w.appendCacheDirFlag(args)
 	args = w.appendCleanupCacheFlag(args)
 	args = w.appendCaCertFlag(args)
+	args = w.appendMaxConnectionsFlag(args)
 
 	return w.run(Command{Name: ResticCMD, Args: args})
 }
@@ -111,6 +116,7 @@ func (w *ResticWrapper) backupFromStdin(options BackupOptions) ([]byte, error) {
 	args = w.appendCacheDirFlag(args)
 	args = w.appendCleanupCacheFlag(args)
 	args = w.appendCaCertFlag(args)
+	args = w.appendMaxConnectionsFlag(args)
 
 	commands = append(commands, Command{Name: ResticCMD, Args: args})
 	return w.run(commands...)
@@ -159,6 +165,7 @@ func (w *ResticWrapper) cleanup(retentionPolicy v1alpha1.RetentionPolicy) ([]byt
 	if len(args) > 1 {
 		args = w.appendCacheDirFlag(args)
 		args = w.appendCaCertFlag(args)
+		args = w.appendMaxConnectionsFlag(args)
 
 		return w.run(Command{Name: ResticCMD, Args: args})
 	}
@@ -186,6 +193,7 @@ func (w *ResticWrapper) restore(path, host, snapshotID string) ([]byte, error) {
 	args = append(args, "--target", "/") // restore in absolute path
 	args = w.appendCacheDirFlag(args)
 	args = w.appendCaCertFlag(args)
+	args = w.appendMaxConnectionsFlag(args)
 
 	return w.run(Command{Name: ResticCMD, Args: args})
 }
@@ -215,6 +223,7 @@ func (w *ResticWrapper) dump(dumpOptions DumpOptions) ([]byte, error) {
 
 	args = w.appendCacheDirFlag(args)
 	args = w.appendCaCertFlag(args)
+	args = w.appendMaxConnectionsFlag(args)
 
 	// first add restic command, then add StdoutPipeCommand
 	commands := []Command{
@@ -230,13 +239,15 @@ func (w *ResticWrapper) check() ([]byte, error) {
 	log.Infoln("Checking integrity of repository")
 	args := w.appendCacheDirFlag([]interface{}{"check"})
 	args = w.appendCaCertFlag(args)
+	args = w.appendMaxConnectionsFlag(args)
 
 	return w.run(Command{Name: ResticCMD, Args: args})
 }
 
 func (w *ResticWrapper) stats() ([]byte, error) {
 	log.Infoln("Reading repository status")
 	args := w.appendCacheDirFlag([]interface{}{"stats"})
+	args = w.appendMaxConnectionsFlag(args)
 	args = append(args, "--mode=raw-data", "--quiet")
 	args = w.appendCaCertFlag(args)
 
@@ -251,6 +262,24 @@ func (w *ResticWrapper) appendCacheDirFlag(args []interface{}) []interface{} {
 	return append(args, "--no-cache")
 }
 
+func (w *ResticWrapper) appendMaxConnectionsFlag(args []interface{}) []interface{} {
+	var maxConOption string
+	if w.config.MaxConnections > 0 {
+		switch w.config.Provider {
+		case ProviderGCS:
+			maxConOption = fmt.Sprintf("gs.connections=%d", w.config.MaxConnections)
+		case ProviderAzure:
+			maxConOption = fmt.Sprintf("azure.connections=%d", w.config.MaxConnections)
+		case ProviderB2:
+			maxConOption = fmt.Sprintf("b2.connections=%d", w.config.MaxConnections)
+		}
+	}
+	if maxConOption != "" {
+		return append(args, "--option", maxConOption)
+	}
+	return args
+}
+
 func (w *ResticWrapper) appendCleanupCacheFlag(args []interface{}) []interface{} {
 	if w.config.EnableCache {
 		return append(args, "--cleanup-cache")

pkg/restic/config.go

@@ -46,17 +46,18 @@ type DumpOptions struct {
 }
 
 type SetupOptions struct {
-	Provider    string
-	Bucket      string
-	Endpoint    string
-	Path        string
-	URL         string
-	SecretDir   string
-	CacertFile  string
-	ScratchDir  string
-	EnableCache bool
-	Nice        *ofst.NiceSettings
-	IONice      *ofst.IONiceSettings
+	Provider       string
+	Bucket         string
+	Endpoint       string
+	Path           string
+	URL            string
+	SecretDir      string
+	CacertFile     string
+	ScratchDir     string
+	EnableCache    bool
+	MaxConnections int
+	Nice           *ofst.NiceSettings
+	IONice         *ofst.IONiceSettings
 }
 
 type MetricsOptions struct {