Update Go client libraries for etcd

changelog/11980.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+physical/etcd: Upgrade etcd3 client to v3.5.0 and etcd2 to v2.305.0.
+```

go.mod

@@ -6,6 +6,8 @@ replace github.com/hashicorp/vault/api => ./api
 
 replace github.com/hashicorp/vault/sdk => ./sdk
 
+replace go.etcd.io/etcd/client/pkg/v3 v3.5.0 => go.etcd.io/etcd/client/pkg/v3 v3.0.0-20210928084031-3df272774672
+
 require (
 	cloud.google.com/go v0.56.0
 	cloud.google.com/go/spanner v1.5.1
@@ -33,7 +35,7 @@ require (
 	github.com/client9/misspell v0.3.4
 	github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c
 	github.com/containerd/containerd v1.4.3 // indirect
-	github.com/coreos/go-semver v0.2.0
+	github.com/coreos/go-semver v0.3.0
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
 	github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc
 	github.com/docker/distribution v2.7.1+incompatible // indirect
@@ -51,11 +53,11 @@ require (
 	github.com/go-sql-driver/mysql v1.5.0
 	github.com/go-test/deep v1.0.7
 	github.com/gocql/gocql v0.0.0-20210401103645-80ab1e13e309
-	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.2
 	github.com/google/go-cmp v0.5.5
 	github.com/google/go-github v17.0.0+incompatible
 	github.com/google/go-metrics-stackdriver v0.2.0
+	github.com/gorilla/mux v1.7.3 // indirect
 	github.com/hashicorp/consul-template v0.27.0
 	github.com/hashicorp/consul/api v1.4.0
 	github.com/hashicorp/errwrap v1.1.0
@@ -122,6 +124,7 @@ require (
 	github.com/jefferai/isbadcipher v0.0.0-20190226160619-51d2077c035f
 	github.com/jefferai/jsonx v1.0.0
 	github.com/joyent/triton-go v1.7.1-0.20200416154420-6801d15b779f
+	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/keybase/go-crypto v0.0.0-20190403132359-d65b6b94177f
 	github.com/kr/pretty v0.2.1
 	github.com/kr/text v0.2.0
@@ -153,15 +156,17 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/posener/complete v1.2.3
 	github.com/pquerna/otp v1.2.1-0.20191009055518-468c2dd2b58d
-	github.com/prometheus/client_golang v1.7.1
-	github.com/prometheus/common v0.11.1
+	github.com/prometheus/client_golang v1.11.0
+	github.com/prometheus/common v0.26.0
 	github.com/rboyer/safeio v0.2.1
 	github.com/ryanuber/columnize v2.1.0+incompatible
 	github.com/ryanuber/go-glob v1.0.0
 	github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da
 	github.com/sasha-s/go-deadlock v0.2.0
 	github.com/sethvargo/go-limiter v0.7.1
 	github.com/shirou/gopsutil v3.21.5+incompatible
+	github.com/smartystreets/goconvey v1.6.4 // indirect
+	github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271 // indirect
 	github.com/stretchr/testify v1.7.0
 	github.com/tencentcloud/tencentcloud-sdk-go v3.0.171+incompatible // indirect
 	github.com/tidwall/pretty v1.0.1 // indirect
@@ -171,22 +176,27 @@ require (
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
 	github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da // indirect
 	go.etcd.io/bbolt v1.3.5
-	go.etcd.io/etcd v0.5.0-alpha.5.0.20200425165423-262c93980547
+	go.etcd.io/etcd/client/pkg/v3 v3.5.0
+	go.etcd.io/etcd/client/v2 v2.305.0
+	go.etcd.io/etcd/client/v3 v3.5.0
 	go.mongodb.org/mongo-driver v1.4.6
 	go.opentelemetry.io/otel v0.20.0
 	go.opentelemetry.io/otel/sdk v0.20.0
 	go.opentelemetry.io/otel/trace v0.20.0
-	go.uber.org/atomic v1.6.0
-	go.uber.org/goleak v1.1.10
+	go.uber.org/atomic v1.9.0
+	go.uber.org/goleak v1.1.11-0.20210813005559-691160354723
+	go.uber.org/multierr v1.7.0 // indirect
+	go.uber.org/zap v1.19.1 // indirect
 	golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97
-	golang.org/x/net v0.0.0-20210614182718-04defd469f4e
+	golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
-	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c
+	golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6
 	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1
-	golang.org/x/tools v0.0.0-20210106214847-113979e3529a
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/tools v0.1.5
 	google.golang.org/api v0.29.0
-	google.golang.org/grpc v1.29.1
+	google.golang.org/genproto v0.0.0-20210928142010-c7af6a1a74c9 // indirect
+	google.golang.org/grpc v1.41.0
 	google.golang.org/protobuf v1.27.1
 	gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce
 	gopkg.in/ory-am/dockertest.v3 v3.3.4

helper/testhelpers/docker/testhelpers.go

@@ -146,7 +146,7 @@ func (d *Runner) StartService(ctx context.Context, connect ServiceAdapter) (*Ser
 
 	bo := backoff.NewExponentialBackOff()
 	bo.MaxInterval = time.Second * 5
-	bo.MaxElapsedTime = time.Minute
+	bo.MaxElapsedTime = 2 * time.Minute
 
 	pieces := strings.Split(hostIPs[0], ":")
 	portInt, err := strconv.Atoi(pieces[1])

helper/testhelpers/etcd/etcdhelper.go

@@ -0,0 +1,86 @@
+package etcd
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/hashicorp/vault/helper/testhelpers/docker"
+	clientv3 "go.etcd.io/etcd/client/v3"
+)
+
+type Config struct {
+	docker.ServiceURL
+}
+
+// PrepareTestContainer creates etcd docker container.  If environment variabe
+// ETCD_ADDR is set, the tests are executed against specified address and etcd
+// container is not launched.
+func PrepareTestContainer(t *testing.T, version string) (func(), *Config) {
+	if addr := os.Getenv("ETCD_ADDR"); addr != "" {
+		url, err := docker.NewServiceURLParse(addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return func() {}, &Config{ServiceURL: *url}
+	}
+
+	// Check https://github.com/etcd-io/etcd/releases for latest releases.
+	runner, err := docker.NewServiceRunner(docker.RunOptions{
+		ContainerName: "etcd",
+		ImageRepo:     "gcr.io/etcd-development/etcd",
+		ImageTag:      version,
+		Cmd: []string{"/usr/local/bin/etcd",
+			"--name", "s1",
+			"--listen-client-urls", "http://0.0.0.0:2379",
+			"--advertise-client-urls", "http://0.0.0.0:2379",
+			"--listen-peer-urls", "http://0.0.0.0:2380",
+			"--initial-advertise-peer-urls", "http://0.0.0.0:2380",
+			"--initial-cluster", "s1=http://0.0.0.0:2380",
+			"--initial-cluster-token", "tkn",
+			"--initial-cluster-state", "new",
+			"--log-level", "info",
+			"--logger", "zap",
+			"--log-outputs", "stderr",
+		},
+		Ports: []string{"2379/tcp"},
+	})
+	if err != nil {
+		t.Fatalf("Could not start docker etcd container: %s", err)
+	}
+
+	svc, err := runner.StartService(context.Background(), func(ctx context.Context, host string, port int) (docker.ServiceConfig, error) {
+		address := fmt.Sprintf("%s:%d", host, port)
+		s := docker.NewServiceURL(url.URL{
+			Scheme: "http",
+			Host:   address,
+		})
+
+		client, err := clientv3.New(clientv3.Config{
+			Endpoints:   []string{address},
+			DialTimeout: 2 * time.Minute,
+		})
+		if err != nil {
+			return nil, fmt.Errorf("could not connect to etcd container: %w", err)
+		}
+
+		// Enable authentication for the tests.
+		client.RoleAdd(ctx, "root")
+		client.UserAdd(ctx, "root", "insecure")
+		client.UserGrantRole(ctx, "root", "root")
+		client.AuthEnable(ctx)
+		client.Close()
+
+		return &Config{
+			ServiceURL: *s,
+		}, nil
+	})
+	if err != nil {
+		t.Fatalf("Could not start docker etcd container: %s", err)
+	}
+
+	return svc.Cleanup, svc.Config.(*Config)
+}

physical/etcd/etcd.go

@@ -11,7 +11,7 @@ import (
 	"github.com/coreos/go-semver/semver"
 	log "github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/vault/sdk/physical"
-	"go.etcd.io/etcd/client"
+	"go.etcd.io/etcd/client/v2"
 )
 
 var (

physical/etcd/etcd2.go

@@ -15,8 +15,8 @@ import (
 	log "github.com/hashicorp/go-hclog"
 	multierror "github.com/hashicorp/go-multierror"
 	"github.com/hashicorp/vault/sdk/physical"
-	"go.etcd.io/etcd/client"
-	"go.etcd.io/etcd/pkg/transport"
+	"go.etcd.io/etcd/client/pkg/v3/transport"
+	"go.etcd.io/etcd/client/v2"
 )
 
 const (

physical/etcd/etcd3.go

@@ -16,9 +16,9 @@ import (
 	"github.com/hashicorp/go-secure-stdlib/parseutil"
 	"github.com/hashicorp/go-secure-stdlib/strutil"
 	"github.com/hashicorp/vault/sdk/physical"
-	"go.etcd.io/etcd/clientv3"
-	"go.etcd.io/etcd/clientv3/concurrency"
-	"go.etcd.io/etcd/pkg/transport"
+	"go.etcd.io/etcd/client/pkg/v3/transport"
+	clientv3 "go.etcd.io/etcd/client/v3"
+	"go.etcd.io/etcd/client/v3/concurrency"
 )
 
 // EtcdBackend is a physical backend that stores data at specific

physical/etcd/etcd3_test.go

@@ -2,33 +2,37 @@ package etcd
 
 import (
 	"fmt"
-	"os"
 	"testing"
 	"time"
 
 	log "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/vault/helper/testhelpers/etcd"
 	"github.com/hashicorp/vault/sdk/helper/logging"
 	"github.com/hashicorp/vault/sdk/physical"
 )
 
 func TestEtcd3Backend(t *testing.T) {
-	addr := os.Getenv("ETCD_ADDR")
-	if addr == "" {
-		t.Skipf("Skipped. No etcd3 server found")
-	}
+	cleanup, config := etcd.PrepareTestContainer(t, "v3.5.0")
+	defer cleanup()
 
 	logger := logging.NewVaultLogger(log.Debug)
-	config := map[string]string{
+	configMap := map[string]string{
+		"address":  config.URL().String(),
 		"path":     fmt.Sprintf("/vault-%d", time.Now().Unix()),
 		"etcd_api": "3",
+		"username": "root",
+		"password": "insecure",
+
+		// Syncing adverticed client urls should be disabled since docker port mapping confuses the client.
+		"sync": "false",
 	}
 
-	b, err := NewEtcdBackend(config, logger)
+	b, err := NewEtcdBackend(configMap, logger)
 	if err != nil {
 		t.Fatalf("err: %s", err)
 	}
 
-	b2, err := NewEtcdBackend(config, logger)
+	b2, err := NewEtcdBackend(configMap, logger)
 	if err != nil {
 		t.Fatalf("err: %s", err)
 	}

vault/cluster/inmem_layer_test.go

@@ -256,6 +256,6 @@ func TestInmemCluster_ConnectCluster(t *testing.T) {
 	wg.Wait()
 
 	if accepted.Load() != 18 {
-		t.Fatalf("expected 18 connections to be accepted, got %d", accepted)
+		t.Fatalf("expected 18 connections to be accepted, got %d", accepted.Load())
 	}
 }