Upgrade the go.etcd.io/etcd dependency to v3.4.34 (#795)

* Upgrade `go.etcd.io/etcd` to `v3.4.34`.

* Remove all `replace` directives which locked versions of dependencies.

* Use the `Restore` API provided by the `clientv3/snapshot` package,
  and remove copied code for performing restoration of the etcd database
  from the base snapshot.

* The temporary directory to persist full and delta snapshots in created
  at the start of the `Restore()` call. This was created only before
  deltas were fetched previously.

* Base full snapshot is also now stored in the `TempSnapshotDir`.

* It is now assumed that the etcd data directory, which is typically
  `default.etcd` itself is not present before restoration, and not just
  the `member` directory.

* Unit tests are adapted accordingly such that the etcd data directory
  is removed before restoration.

* Compactor does not create the database directory anymore, since the Restore API does it.

* Restoration through the `Restore` API will fail if a directory for the database is
  already present. Due to this, explicitly creating a directory before restoration is
  triggered is not performed anymore. The `Restore` call creates the directory.
  Cleanup of this directory is still performed, as a deferred call.

* Fix broken unit tests in `pkg/compactor`.

* Upgrade the etcd version to `v3.4.34` in the integration tests.

* Replace references to `etcd-custom-image` with `etcd-wrapper`.

* References to `etcd-custom-image` with version `v3.4.13-bootstrap-1` are now
  replaced with references to `etcd-wrapper` with version `v0.2.0`.

---------

Co-authored-by: Seshachalam Yerasala Venkata <[email protected]>

.ci/integration_test

@@ -70,7 +70,7 @@ function setup_ginkgo() {
 
 function setup_etcd(){
   echo "Downloading and installing etcd..."
-  export ETCD_VER=v3.4.13
+  export ETCD_VER=v3.4.34
   if [[ $(uname) == 'Darwin' ]]; then
     curl -L https://storage.googleapis.com/etcd/${ETCD_VER}/etcd-${ETCD_VER}-darwin-amd64.zip -o etcd-${ETCD_VER}-darwin-amd64.zip
     unzip etcd-${ETCD_VER}-darwin-amd64.zip > /dev/null
@@ -307,11 +307,12 @@ function run_test_on_cluster() {
     setup-aws-infrastructure
   fi
 
-  export ETCD_VERSION=${ETCD_VERSION:-"v3.4.13-bootstrap-1"}
-  echo "Etcd version: ${ETCD_VERSION}"
+  # TODO: change the etcd wrapper version to a newer version which spawns etcd v3.4.34
+  export ETCD_WRAPPER_VERSION=${ETCD_WRAPPER_VERSION:-"v0.2.0"}
+  echo "etcd-wrapper version: ${ETCD_WRAPPER_VERSION}"
 
   export ETCDBR_VERSION=${ETCDBR_VERSION:-${ETCDBR_VER:-"v0.12.1"}}
-  echo "Etcd-backup-restore version: ${ETCDBR_VERSION}"
+  echo "etcd-backup-restore version: ${ETCDBR_VERSION}"
 
   echo "Starting integration tests on k8s cluster."
 

.ci/performance_regression_test

@@ -31,9 +31,10 @@ if [ "$PERF_TEST_KUBECONFIG" == "" ]; then
   PERF_TEST_KUBECONFIG=$TM_KUBECONFIG_PATH/shoot.config
 fi
 
-ETCD_VERSION=${ETCD_VERSION:-"v3.4.13-bootstrap-1"}
-if [ "$ETCD_VERSION" != "" ]; then
-  ETCD_IMAGE=${ETCD_IMAGE:-"europe-docker.pkg.dev/gardener-project/public/gardener/etcd:$ETCD_VERSION"}
+# TODO: change the etcd wrapper version to a newer version which spawns etcd v3.4.34
+ETCD_WRAPPER_VERSION=${ETCD_WRAPPER_VERSION:-"v0.2.0"}
+if [ "$ETCD_WRAPPER_VERSION" != "" ]; then
+  ETCD_IMAGE=${ETCD_IMAGE:-"europe-docker.pkg.dev/gardener-project/public/gardener/etcd-wrapper:$ETCD_WRAPPER_VERSION"}
 fi
 
 if [ "$ETCDBR_VERSION" != "" ]; then

chart/etcd-backup-restore/values.yaml

@@ -1,8 +1,9 @@
 images:
   # etcd image to use
   etcd:
-    repository: europe-docker.pkg.dev/gardener-project/public/gardener/etcd
-    tag: v3.4.13-bootstrap-1
+    # TODO: change the etcd wrapper version to a newer version which spawns etcd v3.4.34
+    repository: europe-docker.pkg.dev/gardener-project/public/gardener/etcd-wrapper
+    tag: v0.2.0
     pullPolicy: IfNotPresent
   # etcd-backup-restore image to use
   etcdBackupRestore:

docs/development/testing_and_dependencies.md

@@ -42,7 +42,8 @@ make integration-test-cluster
 :warning: Prerequisite for this command is to set the following environment variables:
 
 - INTEGRATION_TEST_KUBECONFIG: kubeconfig to the cluster on which you wish to run the test
-- ETCD_VERSION: optional, defaults to `v3.4.13-bootstrap-1`
+<!-- TODO: change the etcd wrapper version to a newer version which spawns etcd v3.4.34 -->
+- ETCD_WRAPPER_VERSION: optional, defaults to `v0.2.0`
 - ETCDBR_VERSION: optional, defaults to `v0.12.1`
 - ACCESS_KEY_ID: S3 credentials
 - SECRET_ACCESS_KEY: S3 credentials

go.mod

@@ -3,35 +3,28 @@ module github.com/gardener/etcd-backup-restore
 go 1.23.2
 
 require (
-	cloud.google.com/go v0.97.0
-	cloud.google.com/go/storage v1.10.0
+	cloud.google.com/go/iam v1.1.1
+	cloud.google.com/go/storage v1.30.1
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2
 	github.com/aliyun/aliyun-oss-go-sdk v2.1.8+incompatible
 	github.com/aws/aws-sdk-go v1.54.20
-	github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect
-	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect
-	github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484 // indirect
 	github.com/ghodss/yaml v1.0.0
-	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
 	github.com/golang/mock v1.6.0
 	github.com/gophercloud/gophercloud v0.17.0
 	github.com/gophercloud/utils v0.0.0-20200204043447-9864b6f1f12f
-	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
-	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
-	github.com/huandu/xstrings v1.3.2 // indirect
-	github.com/mitchellh/reflectwalk v1.0.1 // indirect
 	github.com/onsi/ginkgo/v2 v2.11.0
 	github.com/onsi/gomega v1.27.8
 	github.com/prometheus/client_golang v1.14.0
 	github.com/robfig/cron/v3 v3.0.1
-	github.com/sirupsen/logrus v1.8.1
+	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.6.0
 	github.com/spf13/pflag v1.0.5
-	go.etcd.io/bbolt v1.3.7
-	go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489
-	go.uber.org/atomic v1.7.0 // indirect
+	go.etcd.io/bbolt v1.3.11
+	// Etcd 3.4.x vendoring issue Ref: https://github.com/etcd-io/etcd/issues/11154#issuecomment-677940701
+	go.etcd.io/etcd v0.0.0-20240911181550-c123b3ea3db3 // etcd v3.4.34 pseudoversion
 	go.uber.org/zap v1.24.0
-	google.golang.org/api v0.57.0
-	google.golang.org/grpc v1.49.0 // indirect
+	google.golang.org/api v0.126.0
 	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/api v0.26.1
 	k8s.io/apimachinery v0.26.1
@@ -42,24 +35,24 @@ require (
 )
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2
-)
-
-require (
+	cloud.google.com/go v0.110.4 // indirect
+	cloud.google.com/go/compute v1.21.0 // indirect
+	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
 	github.com/BurntSushi/toml v0.3.1 // indirect
 	github.com/Masterminds/goutils v1.1.0 // indirect
 	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/Masterminds/sprig v2.22.0+incompatible // indirect
+	github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/coreos/go-semver v0.3.0 // indirect
+	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect
 	github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect
 	github.com/cyphar/filepath-securejoin v0.2.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
+	github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484 // indirect
 	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
@@ -70,17 +63,23 @@ require (
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/btree v1.0.1 // indirect
 	github.com/google/gnostic v0.5.7-v3refs // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gofuzz v1.1.0 // indirect
 	github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
+	github.com/google/s2a-go v0.1.4 // indirect
 	github.com/google/uuid v1.6.0 // indirect
-	github.com/googleapis/gax-go/v2 v2.1.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
+	github.com/googleapis/gax-go/v2 v2.11.0 // indirect
 	github.com/gorilla/websocket v1.4.2 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
+	github.com/huandu/xstrings v1.5.0 // indirect
 	github.com/imdario/mergo v0.3.11 // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
@@ -90,6 +89,7 @@ require (
 	github.com/mailru/easyjson v0.7.6 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect
 	github.com/mitchellh/copystructure v1.0.0 // indirect
+	github.com/mitchellh/reflectwalk v1.0.0 // indirect
 	github.com/moby/spdystream v0.2.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
@@ -102,20 +102,25 @@ require (
 	github.com/soheilhy/cmux v0.1.5 // indirect
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect
 	github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
-	go.opencensus.io v0.23.0 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	golang.org/x/crypto v0.21.0 // indirect
-	golang.org/x/net v0.22.0 // indirect
-	golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect
+	golang.org/x/net v0.23.0 // indirect
+	golang.org/x/oauth2 v0.10.0 // indirect
+	golang.org/x/sync v0.5.0 // indirect
 	golang.org/x/sys v0.18.0 // indirect
 	golang.org/x/term v0.18.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.9.3 // indirect
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
+	google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect
+	google.golang.org/grpc v1.58.3 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/klog/v2 v2.80.1 // indirect
@@ -124,21 +129,3 @@ require (
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
-
-replace (
-	// Ref: https://github.com/Azure/go-autorest/issues/414
-	github.com/Azure/go-autorest => github.com/Azure/go-autorest v13.3.2+incompatible
-	github.com/containerd/containerd => github.com/containerd/containerd v1.4.11
-	// Etcd issue Ref: https://github.com/etcd-io/etcd/issues/12068
-	// Etcd 3.4.x vendoring issue Ref: https://github.com/etcd-io/etcd/issues/11154#issuecomment-677940701
-	github.com/coreos/etcd => go.etcd.io/etcd v0.5.0-alpha.5.0.20200824191128-ae9734ed278b // ae9734ed278b is the SHA for git tag v3.4.13
-	// RefL https://github.com/golang-jwt/jwt/blob/v3.2.1/MIGRATION_GUIDE.md https://github.com/dgrijalva/jwt-go/issues/462
-	github.com/dgrijalva/jwt-go => github.com/golang-jwt/jwt v3.2.1+incompatible
-	github.com/docker/docker => github.com/moby/moby v0.7.3-0.20190826074503-38ab9da00309
-	// Ref: https://github.com/etcd-io/etcd/issues/11992
-	github.com/golang/protobuf => github.com/golang/protobuf v1.4.0
-	github.com/opencontainers/runc => github.com/opencontainers/runc v1.0.0
-	// Ref: https://github.com/etcd-io/etcd/issues/11707
-	google.golang.org/grpc => google.golang.org/grpc v1.29.0
-	k8s.io/client-go => k8s.io/client-go v0.26.1
-)