From 9cfd562e8792346169b26e881667f4904c1cec02 Mon Sep 17 00:00:00 2001 From: Dhia Ayachi Date: Wed, 15 Mar 2023 19:39:41 +0000 Subject: [PATCH 1/8] backport of commit a8525f831580580df029003d79c35a37d2e6fc15 --- .../libs/cluster/container.go | 10 ++- .../test/snapshot/snapshot_restore_test.go | 82 +++++++++++++++++++ 2 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 test/integration/consul-container/test/snapshot/snapshot_restore_test.go diff --git a/test/integration/consul-container/libs/cluster/container.go b/test/integration/consul-container/libs/cluster/container.go index 3ad1b2d35169..4e3e1c39901c 100644 --- a/test/integration/consul-container/libs/cluster/container.go +++ b/test/integration/consul-container/libs/cluster/container.go @@ -154,11 +154,17 @@ func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster, po info AgentInfo ) if httpPort > 0 { - uri, err := podContainer.PortEndpoint(ctx, "8500", "http") + for i := 0; i < 10; i++ { + uri, err := podContainer.PortEndpoint(ctx, "8500", "http") + if err != nil { + time.Sleep(500 * time.Millisecond) + continue + } + clientAddr = uri + } if err != nil { return nil, err } - clientAddr = uri } else if httpsPort > 0 { uri, err := podContainer.PortEndpoint(ctx, "8501", "https") diff --git a/test/integration/consul-container/test/snapshot/snapshot_restore_test.go b/test/integration/consul-container/test/snapshot/snapshot_restore_test.go new file mode 100644 index 000000000000..7681ce97a502 --- /dev/null +++ b/test/integration/consul-container/test/snapshot/snapshot_restore_test.go @@ -0,0 +1,82 @@ +package snapshot + +import ( + "fmt" + "github.com/hashicorp/consul/api" + "github.com/hashicorp/consul/sdk/testutil" + libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster" + libtopology "github.com/hashicorp/consul/test/integration/consul-container/libs/topology" + "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" + "github.com/stretchr/testify/require" + "io" + "testing" +) + +func TestSnapshotRestore(t *testing.T) { + const ( + numServers = 3 + ) + + // Create initial cluster + cluster, _, _ := libtopology.NewCluster(t, &libtopology.ClusterConfig{ + NumServers: numServers, + NumClients: 0, + BuildOpts: &libcluster.BuildOptions{ + Datacenter: "dc1", + ConsulImageName: utils.LatestImageName, + ConsulVersion: utils.LatestVersion, + }, + ApplyDefaultProxySettings: true, + }) + + client := cluster.APIClient(0) + libcluster.WaitForLeader(t, cluster, client) + libcluster.WaitForMembers(t, client, 3) + + testutil.RunStep(t, "Create Data", func(t *testing.T) { + for i := 0; i < 100; i++ { + _, err := client.KV().Put(&api.KVPair{Key: fmt.Sprintf("key-%d", i), Value: []byte(fmt.Sprintf("value-%d", i))}, nil) + require.NoError(t, err) + } + }) + + var snapshot io.ReadCloser + testutil.RunStep(t, "Save snapshot", func(t *testing.T) { + var err error + snapshot, _, err = client.Snapshot().Save(nil) + require.NoError(t, err) + }) + + err := cluster.Terminate() + require.NoError(t, err) + // Create a fresh cluster from scratch + cluster2, _, _ := libtopology.NewCluster(t, &libtopology.ClusterConfig{ + NumServers: numServers, + NumClients: 0, + BuildOpts: &libcluster.BuildOptions{ + Datacenter: "dc1", + ConsulImageName: utils.LatestImageName, + ConsulVersion: utils.LatestVersion, + }, + ApplyDefaultProxySettings: true, + }) + client2 := cluster2.APIClient(0) + + testutil.RunStep(t, "Restore saved snapshot", func(t *testing.T) { + libcluster.WaitForLeader(t, cluster2, client2) + libcluster.WaitForMembers(t, client2, 3) + + // Restore the saved snapshot + require.NoError(t, client2.Snapshot().Restore(nil, snapshot)) + + libcluster.WaitForLeader(t, cluster2, client2) + + }) + + for i := 0; i < 100; i++ { + kv, _, err := client2.KV().Get(fmt.Sprintf("key-%d", i), nil) + require.NoError(t, err) + require.Equal(t, kv.Key, fmt.Sprintf("key-%d", i)) + require.Equal(t, kv.Value, []byte(fmt.Sprintf("value-%d", i))) + } +} From 1f7023267abd4ce17249477860367f298720ef00 Mon Sep 17 00:00:00 2001 From: Dhia Ayachi Date: Wed, 15 Mar 2023 20:18:44 +0000 Subject: [PATCH 2/8] backport of commit a2d560517ca76e0848ce5cd16c73d54c0896cec1 --- .../consul-container/libs/cluster/builder.go | 17 +++++++ .../libs/topology/peering_topology.go | 1 + .../test/snapshot/snapshot_restore_test.go | 51 +++++++++++-------- 3 files changed, 47 insertions(+), 22 deletions(-) diff --git a/test/integration/consul-container/libs/cluster/builder.go b/test/integration/consul-container/libs/cluster/builder.go index 9ce5bf5c8c34..9d30e6aea49f 100644 --- a/test/integration/consul-container/libs/cluster/builder.go +++ b/test/integration/consul-container/libs/cluster/builder.go @@ -20,6 +20,13 @@ const ( ConsulCACertKey = "consul-agent-ca-key.pem" ) +type LogStore string + +const ( + LogStore_WAL LogStore = "wal" + LogStore_BoltDB LogStore = "boltdb" +) + // BuildContext provides a reusable object meant to share common configuration settings // between agent configuration builders. type BuildContext struct { @@ -41,6 +48,7 @@ type BuildContext struct { tlsCertIndex int // keeps track of the certificates issued for naming purposes aclEnabled bool + logStore LogStore } func (c *BuildContext) DockerImage() string { @@ -89,6 +97,9 @@ type BuildOptions struct { // ACLEnabled configures acl in agent configuration ACLEnabled bool + + //StoreLog define which LogStore to use + LogStore LogStore } func NewBuildContext(t *testing.T, opts BuildOptions) *BuildContext { @@ -103,6 +114,7 @@ func NewBuildContext(t *testing.T, opts BuildOptions) *BuildContext { useAPIWithTLS: opts.UseAPIWithTLS, useGRPCWithTLS: opts.UseGRPCWithTLS, aclEnabled: opts.ACLEnabled, + logStore: opts.LogStore, } if ctx.consulImageName == "" { @@ -201,6 +213,11 @@ func NewConfigBuilder(ctx *BuildContext) *Builder { b.conf.Set("acl.default_policy", "deny") b.conf.Set("acl.enable_token_persistence", true) } + ls := string(ctx.logStore) + if ls != string(LogStore_WAL) && ls != string(LogStore_BoltDB) { + ls = string(LogStore_BoltDB) + } + b.conf.Set("raft_logstore.backend", ls) return b } diff --git a/test/integration/consul-container/libs/topology/peering_topology.go b/test/integration/consul-container/libs/topology/peering_topology.go index 0e853f7adda3..3e7ac8d067b0 100644 --- a/test/integration/consul-container/libs/topology/peering_topology.go +++ b/test/integration/consul-container/libs/topology/peering_topology.go @@ -198,6 +198,7 @@ func NewCluster( AllowHTTPAnyway: true, ConsulVersion: config.BuildOpts.ConsulVersion, ACLEnabled: config.BuildOpts.ACLEnabled, + LogStore: config.BuildOpts.LogStore, } ctx := libcluster.NewBuildContext(t, opts) diff --git a/test/integration/consul-container/test/snapshot/snapshot_restore_test.go b/test/integration/consul-container/test/snapshot/snapshot_restore_test.go index 7681ce97a502..31280fec0a7f 100644 --- a/test/integration/consul-container/test/snapshot/snapshot_restore_test.go +++ b/test/integration/consul-container/test/snapshot/snapshot_restore_test.go @@ -3,7 +3,6 @@ package snapshot import ( "fmt" "github.com/hashicorp/consul/api" - "github.com/hashicorp/consul/sdk/testutil" libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster" libtopology "github.com/hashicorp/consul/test/integration/consul-container/libs/topology" "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" @@ -13,6 +12,18 @@ import ( ) func TestSnapshotRestore(t *testing.T) { + + cases := []libcluster.LogStore{libcluster.LogStore_WAL, libcluster.LogStore_BoltDB} + + for _, c := range cases { + t.Run(fmt.Sprintf("test log store: %s", c), func(t *testing.T) { + testSnapShotRestoreForLogStore(t, c) + }) + } +} + +func testSnapShotRestoreForLogStore(t *testing.T, logStore libcluster.LogStore) { + const ( numServers = 3 ) @@ -25,6 +36,7 @@ func TestSnapshotRestore(t *testing.T) { Datacenter: "dc1", ConsulImageName: utils.LatestImageName, ConsulVersion: utils.LatestVersion, + LogStore: logStore, }, ApplyDefaultProxySettings: true, }) @@ -33,21 +45,17 @@ func TestSnapshotRestore(t *testing.T) { libcluster.WaitForLeader(t, cluster, client) libcluster.WaitForMembers(t, client, 3) - testutil.RunStep(t, "Create Data", func(t *testing.T) { - for i := 0; i < 100; i++ { - _, err := client.KV().Put(&api.KVPair{Key: fmt.Sprintf("key-%d", i), Value: []byte(fmt.Sprintf("value-%d", i))}, nil) - require.NoError(t, err) - } - }) + for i := 0; i < 100; i++ { + _, err := client.KV().Put(&api.KVPair{Key: fmt.Sprintf("key-%d", i), Value: []byte(fmt.Sprintf("value-%d", i))}, nil) + require.NoError(t, err) + } var snapshot io.ReadCloser - testutil.RunStep(t, "Save snapshot", func(t *testing.T) { - var err error - snapshot, _, err = client.Snapshot().Save(nil) - require.NoError(t, err) - }) + var err error + snapshot, _, err = client.Snapshot().Save(nil) + require.NoError(t, err) - err := cluster.Terminate() + err = cluster.Terminate() require.NoError(t, err) // Create a fresh cluster from scratch cluster2, _, _ := libtopology.NewCluster(t, &libtopology.ClusterConfig{ @@ -57,26 +65,25 @@ func TestSnapshotRestore(t *testing.T) { Datacenter: "dc1", ConsulImageName: utils.LatestImageName, ConsulVersion: utils.LatestVersion, + LogStore: logStore, }, ApplyDefaultProxySettings: true, }) client2 := cluster2.APIClient(0) - testutil.RunStep(t, "Restore saved snapshot", func(t *testing.T) { - libcluster.WaitForLeader(t, cluster2, client2) - libcluster.WaitForMembers(t, client2, 3) - - // Restore the saved snapshot - require.NoError(t, client2.Snapshot().Restore(nil, snapshot)) + libcluster.WaitForLeader(t, cluster2, client2) + libcluster.WaitForMembers(t, client2, 3) - libcluster.WaitForLeader(t, cluster2, client2) + // Restore the saved snapshot + require.NoError(t, client2.Snapshot().Restore(nil, snapshot)) - }) + libcluster.WaitForLeader(t, cluster2, client2) for i := 0; i < 100; i++ { - kv, _, err := client2.KV().Get(fmt.Sprintf("key-%d", i), nil) + kv, _, err := client2.KV().Get(fmt.Sprintf("key-%d", i), &api.QueryOptions{AllowStale: true}) require.NoError(t, err) require.Equal(t, kv.Key, fmt.Sprintf("key-%d", i)) require.Equal(t, kv.Value, []byte(fmt.Sprintf("value-%d", i))) } + } From bf97c64fbd900b2016eeb32d21637f2f42976213 Mon Sep 17 00:00:00 2001 From: Paul Banks Date: Thu, 16 Mar 2023 13:00:09 +0000 Subject: [PATCH 3/8] backport of commit 958c279dbcfac502d5dc9f9799a528aa5a41728a --- .../test/snapshot/snapshot_restore_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/integration/consul-container/test/snapshot/snapshot_restore_test.go b/test/integration/consul-container/test/snapshot/snapshot_restore_test.go index 31280fec0a7f..56cad07d4732 100644 --- a/test/integration/consul-container/test/snapshot/snapshot_restore_test.go +++ b/test/integration/consul-container/test/snapshot/snapshot_restore_test.go @@ -34,8 +34,8 @@ func testSnapShotRestoreForLogStore(t *testing.T, logStore libcluster.LogStore) NumClients: 0, BuildOpts: &libcluster.BuildOptions{ Datacenter: "dc1", - ConsulImageName: utils.LatestImageName, - ConsulVersion: utils.LatestVersion, + ConsulImageName: utils.TargetImageName, + ConsulVersion: utils.TargetVersion, LogStore: logStore, }, ApplyDefaultProxySettings: true, @@ -63,8 +63,8 @@ func testSnapShotRestoreForLogStore(t *testing.T, logStore libcluster.LogStore) NumClients: 0, BuildOpts: &libcluster.BuildOptions{ Datacenter: "dc1", - ConsulImageName: utils.LatestImageName, - ConsulVersion: utils.LatestVersion, + ConsulImageName: utils.TargetImageName, + ConsulVersion: utils.TargetVersion, LogStore: logStore, }, ApplyDefaultProxySettings: true, From 57fc837fdb9354f19192358086bf680c08723786 Mon Sep 17 00:00:00 2001 From: Dhia Ayachi Date: Thu, 16 Mar 2023 13:37:55 +0000 Subject: [PATCH 4/8] backport of commit 1c1b88bc6a0060712a2f5942a2a474b400fd827b --- .../test/snapshot/snapshot_restore_test.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/integration/consul-container/test/snapshot/snapshot_restore_test.go b/test/integration/consul-container/test/snapshot/snapshot_restore_test.go index 56cad07d4732..1d82b1cfb10c 100644 --- a/test/integration/consul-container/test/snapshot/snapshot_restore_test.go +++ b/test/integration/consul-container/test/snapshot/snapshot_restore_test.go @@ -79,8 +79,16 @@ func testSnapShotRestoreForLogStore(t *testing.T, logStore libcluster.LogStore) libcluster.WaitForLeader(t, cluster2, client2) + followers, err := cluster2.Followers() + require.NoError(t, err) + require.Len(t, followers, 2) + + // use a follower api client and set `AllowStale` to true + // to test the follower snapshot install code path as well. + fc := followers[0].GetClient() + for i := 0; i < 100; i++ { - kv, _, err := client2.KV().Get(fmt.Sprintf("key-%d", i), &api.QueryOptions{AllowStale: true}) + kv, _, err := fc.KV().Get(fmt.Sprintf("key-%d", i), &api.QueryOptions{AllowStale: true}) require.NoError(t, err) require.Equal(t, kv.Key, fmt.Sprintf("key-%d", i)) require.Equal(t, kv.Value, []byte(fmt.Sprintf("value-%d", i))) From 6af1ba9def0c80a565aaabfef3b37e9055ebc464 Mon Sep 17 00:00:00 2001 From: John Murret Date: Fri, 17 Mar 2023 21:30:39 +0000 Subject: [PATCH 5/8] backport of commit cd25976169f80a39a8dfc995b1d51fff213bdd3c --- go.mod | 10 +++++----- go.sum | 21 ++++++++++++--------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 2143ed05bb9e..0a525cbc8e50 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ exclude ( require ( github.com/NYTimes/gziphandler v1.0.1 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e - github.com/armon/go-metrics v0.3.10 + github.com/armon/go-metrics v0.4.1 github.com/armon/go-radix v1.0.0 github.com/aws/aws-sdk-go v1.42.34 github.com/coredns/coredns v1.6.6 @@ -62,10 +62,10 @@ require ( github.com/hashicorp/hcp-sdk-go v0.23.1-0.20220921131124-49168300a7dc github.com/hashicorp/hil v0.0.0-20200423225030-a18a1cd20038 github.com/hashicorp/memberlist v0.5.0 - github.com/hashicorp/raft v1.3.11 + github.com/hashicorp/raft v1.4.0 github.com/hashicorp/raft-autopilot v0.1.6 github.com/hashicorp/raft-boltdb/v2 v2.2.2 - github.com/hashicorp/raft-wal v0.2.4 + github.com/hashicorp/raft-wal v0.3.0 github.com/hashicorp/serf v0.10.1 github.com/hashicorp/vault/api v1.8.2 github.com/hashicorp/vault/api/auth/gcp v0.3.0 @@ -89,7 +89,7 @@ require ( github.com/rboyer/safeio v0.2.1 github.com/ryanuber/columnize v2.1.2+incompatible github.com/shirou/gopsutil/v3 v3.22.8 - github.com/stretchr/testify v1.8.0 + github.com/stretchr/testify v1.8.2 go.etcd.io/bbolt v1.3.6 go.uber.org/goleak v1.1.10 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d @@ -213,7 +213,7 @@ require ( github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect github.com/softlayer/softlayer-go v0.0.0-20180806151055-260589d94c7d // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/objx v0.4.0 // indirect + github.com/stretchr/objx v0.5.0 // indirect github.com/tencentcloud/tencentcloud-sdk-go v1.0.162 // indirect github.com/tklauser/go-sysconf v0.3.10 // indirect github.com/tklauser/numcpus v0.4.0 // indirect diff --git a/go.sum b/go.sum index 7234a8e0275a..461cd0136ff0 100644 --- a/go.sum +++ b/go.sum @@ -142,8 +142,8 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= github.com/armon/go-metrics v0.3.9/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= -github.com/armon/go-metrics v0.3.10 h1:FR+drcQStOe+32sYyJYyZ7FIdgoGGBnwLl+flodp8Uo= -github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= +github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= +github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -607,8 +607,9 @@ github.com/hashicorp/net-rpc-msgpackrpc/v2 v2.0.0 h1:kBpVVl1sl3MaSrs97e0+pDQhSrq github.com/hashicorp/net-rpc-msgpackrpc/v2 v2.0.0/go.mod h1:6pdNz0vo0mF0GvhwDG56O3N18qBrAz/XRIcfINfTbwo= github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= github.com/hashicorp/raft v1.2.0/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8= -github.com/hashicorp/raft v1.3.11 h1:p3v6gf6l3S797NnK5av3HcczOC1T5CLoaRvg0g9ys4A= github.com/hashicorp/raft v1.3.11/go.mod h1:J8naEwc6XaaCfts7+28whSeRvCqTd6e20BlCU3LtEO4= +github.com/hashicorp/raft v1.4.0 h1:tn28S/AWv0BtRQgwZv/1NELu8sCvI0FixqL8C8MYKeY= +github.com/hashicorp/raft v1.4.0/go.mod h1:nz64BIjXphDLATfKGG5RzHtNUPioLeKFsXEm88yTVew= github.com/hashicorp/raft-autopilot v0.1.6 h1:C1q3RNF2FfXNZfHWbvVAu0QixaQK8K5pX4O5lh+9z4I= github.com/hashicorp/raft-autopilot v0.1.6/go.mod h1:Af4jZBwaNOI+tXfIqIdbcAnh/UyyqIMj/pOISIfhArw= github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea/go.mod h1:pNv7Wc3ycL6F5oOWn+tPGo2gWD4a5X+yp/ntwdKLjRk= @@ -616,8 +617,8 @@ github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea/go.mod h1:qR github.com/hashicorp/raft-boltdb v0.0.0-20220329195025-15018e9b97e0 h1:CO8dBMLH6dvE1jTn/30ZZw3iuPsNfajshWoJTnVc5cc= github.com/hashicorp/raft-boltdb/v2 v2.2.2 h1:rlkPtOllgIcKLxVT4nutqlTH2NRFn+tO1wwZk/4Dxqw= github.com/hashicorp/raft-boltdb/v2 v2.2.2/go.mod h1:N8YgaZgNJLpZC+h+by7vDu5rzsRgONThTEeUS3zWbfY= -github.com/hashicorp/raft-wal v0.2.4 h1:Ke0ytMj8XyOVKQqFDmmgs/6hqkTJg0b/GO2a2XQBZ6A= -github.com/hashicorp/raft-wal v0.2.4/go.mod h1:JQ/4RbnKFi5Q/4rA73CekaYtHCJhU7qM7AQ4X5Y6q4M= +github.com/hashicorp/raft-wal v0.3.0 h1:Mi6RPoRbsxIIYZryI+bSTXHD97Ua6rIYO51ibYV9bkY= +github.com/hashicorp/raft-wal v0.3.0/go.mod h1:A6vP5o8hGOs1LHfC1Okh9xPwWDcmb6Vvuz/QyqUXlOE= github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY= github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4= github.com/hashicorp/vault/api v1.8.0/go.mod h1:uJrw6D3y9Rv7hhmS17JQC50jbPDAZdjZoTtrCCxxs7E= @@ -697,8 +698,8 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -921,8 +922,8 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rs/zerolog v1.4.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/russross/blackfriday v0.0.0-20170610170232-067529f716f4/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= @@ -977,8 +978,9 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -987,8 +989,9 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/tencentcloud/tencentcloud-sdk-go v1.0.162 h1:8fDzz4GuVg4skjY2B0nMN7h6uN61EDVkuLyI2+qGHhI= github.com/tencentcloud/tencentcloud-sdk-go v1.0.162/go.mod h1:asUz5BPXxgoPGaRgZaVm1iGcUAuHyYUo1nXqKa83cvI= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= From 2691801ca34f6c64d570b1ecccead3969c2e9569 Mon Sep 17 00:00:00 2001 From: John Murret Date: Fri, 17 Mar 2023 21:38:49 +0000 Subject: [PATCH 6/8] backport of commit d95619f85f0f05f45a20607b6dc613de354ac0dc --- .changelog/16647.txt | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .changelog/16647.txt diff --git a/.changelog/16647.txt b/.changelog/16647.txt new file mode 100644 index 000000000000..7083a6192d8c --- /dev/null +++ b/.changelog/16647.txt @@ -0,0 +1,4 @@ +integration test to cover the use case of taking and restoring a snapshot on a consul cluster +```release-note:bug +raft_logstore: Fixes a bug where restoring a snapshot with a WAL backend causes panic because raf-wal expects a monotonically increasing log index and the restore inserts a gap where no files exist. +``` From b0a7424cc310a0f501468eaab57ba7f80d190708 Mon Sep 17 00:00:00 2001 From: John Murret Date: Sat, 18 Mar 2023 16:29:14 +0000 Subject: [PATCH 7/8] backport of commit 15a4645e7fd7e6d220a25b80eec16a13025db5f5 --- .changelog/16647.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.changelog/16647.txt b/.changelog/16647.txt index 7083a6192d8c..cbe38b3ed280 100644 --- a/.changelog/16647.txt +++ b/.changelog/16647.txt @@ -1,4 +1,3 @@ -integration test to cover the use case of taking and restoring a snapshot on a consul cluster ```release-note:bug -raft_logstore: Fixes a bug where restoring a snapshot with a WAL backend causes panic because raf-wal expects a monotonically increasing log index and the restore inserts a gap where no files exist. +raft_logstore: Fixes a bug where restoring a snapshot when using the experimental WAL storage backend causes a panic. ``` From 4efaa2f4e812d2bdc99ec3cf2eef27b3f25c53dc Mon Sep 17 00:00:00 2001 From: John Murret Date: Sat, 18 Mar 2023 20:16:36 +0000 Subject: [PATCH 8/8] backport of commit abb2203af5c88889373fa3bac9ce0fda2fe2e9e5 --- .../consul-container/libs/cluster/builder.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/test/integration/consul-container/libs/cluster/builder.go b/test/integration/consul-container/libs/cluster/builder.go index 9d30e6aea49f..2807985e17e5 100644 --- a/test/integration/consul-container/libs/cluster/builder.go +++ b/test/integration/consul-container/libs/cluster/builder.go @@ -213,11 +213,18 @@ func NewConfigBuilder(ctx *BuildContext) *Builder { b.conf.Set("acl.default_policy", "deny") b.conf.Set("acl.enable_token_persistence", true) } + ls := string(ctx.logStore) - if ls != string(LogStore_WAL) && ls != string(LogStore_BoltDB) { - ls = string(LogStore_BoltDB) + if ls != "" && (ctx.consulVersion == "local" || + semver.Compare("v"+ctx.consulVersion, "v1.15.0") >= 0) { + // Enable logstore backend for version after v1.15.0 + if ls != string(LogStore_WAL) && ls != string(LogStore_BoltDB) { + ls = string(LogStore_BoltDB) + } + b.conf.Set("raft_logstore.backend", ls) + } else { + b.conf.Unset("raft_logstore.backend") } - b.conf.Set("raft_logstore.backend", ls) return b }