auto-apply memberlist ring config when join_members provided (#4400)

* auto-apply memberlist ring config when join_members provided Signed-off-by: Trevor Whitney <[email protected]> * move "memberlist" to a const Signed-off-by: Trevor Whitney <[email protected]> * add comment for memberlist config auto-apply, update changelog Signed-off-by: Trevor Whitney <[email protected]> * add docs Signed-off-by: Trevor Whitney <[email protected]>
grafana · Oct 7, 2021 · b07920c · b07920c
1 parent 1921c3d
commit b07920c
Show file tree

Hide file tree

Showing 5 changed files with 128 additions and 0 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,4 +1,5 @@
 ## Main
+* [4400](https://github.com/grafana/loki/pull/4400) **trevorwhitney**: Config: automatically apply memberlist config too all rings when provided
 
 # 2.3.0 (2021/08/06)
 

diff --git a/docs/sources/configuration/_index.md b/docs/sources/configuration/_index.md
@@ -1046,6 +1046,10 @@ The `memberlist_config` block configures the gossip ring to discover and connect
 between distributors, ingesters and queriers. The configuration is unique for all
 three components to ensure a single shared ring.
 
+When a `memberlist_config` with least 1 `join_members` is defined, a `kvstore` of type `memberlist` is
+automatically configured for the `distributor`, `ingester`, and `ruler` rings unless otherwise specified in
+those components specific configuration sections.
+
 ```yaml
 # Name of the node in memberlist cluster. Defaults to hostname.
 # CLI flag: -memberlist.nodename

diff --git a/docs/sources/upgrading/_index.md b/docs/sources/upgrading/_index.md
@@ -17,8 +17,53 @@ If possible try to stay current and do sequential updates. If you want to skip v
 
 ## Master / Unreleased
 
+### Loki
+
+#### Memberlist config now automatically applies to all non-configured rings
+PR [4400](https://github.com/grafana/loki/pull/4400) **trevorwhitney**: Config: automatically apply memberlist config too all rings when provided
+
+This change affects the behavior of the ingester, distributor, and ruler rings. Previously, if you wanted to use memberlist for all of these rings, you
+had to provide a `memberlist` configuration as well as specify `store: memberlist` for the `kvstore` of each of the rings you wanted to use memberlist.
+For example, your configuration might look something like this:
+
+```yaml
+memberlist:
+  join_members:
+    - loki.namespace.svc.cluster.local
+distributor:
+  ring:
+    kvstore:
+      store: memberlist
+ingester:
+    lifecycler:
+      ring:
+        kvstore:
+          store: memberlist
+ruler:
+  ring:
+    kvstore:
+      store: memberlist
+```
+
+Now, if your provide a `memberlist` configuration with at least one `join_members`, loki will default all rings to use a `kvstore` of type `memberlist`.
+You can change this behavior by overriding specific configurations. For example, if you wanted to use `consul` for you `ruler` rings, but `memberlist`
+for the `ingester` and `distributor`, you could do so with the following config:
+
+```yaml
+memberlist:
+  join_members:
+    - loki.namespace.svc.cluster.local
+ruler:
+  ring:
+    kvstore:
+      store: consul
+      consul:
+        host: consul.namespace.svc.cluster.local:8500
+```
+
 -_add changes here which are unreleased_
 
+
 ## 2.3.0
 
 ### Loki

diff --git a/pkg/loki/config_wrapper.go b/pkg/loki/config_wrapper.go
@@ -43,6 +43,8 @@ func (c *ConfigWrapper) Clone() flagext.Registerer {
 	}(*c)
 }
 
+const memberlistStr = "memberlist"
+
 // ApplyDynamicConfig satisfies WithCommonCloneable interface, and applies all rules for setting Loki
 // config values from the common section of the Loki config file.
 // This method's purpose is to simplify Loki's config in an opinionated way so that Loki can be run
@@ -69,6 +71,21 @@ func (c *ConfigWrapper) ApplyDynamicConfig() cfg.Source {
 			}
 		}
 
+		applyMemberlistConfig(r)
+
 		return nil
 	}
 }
+
+// applyMemberlistConfig will change the default ingester, distributor, and ruler ring configurations to use memberlist
+// if the -memberlist.join_members config is provided. The idea here is that if a user explicitly configured the
+// memberlist configuration section, they probably want to be using memberlist for all their ring configurations.
+// Since a user can still explicitly override a specific ring configuration (for example, use consul for the distributor),
+// it seems harmless to take a guess at better defaults here.
+func applyMemberlistConfig(r *ConfigWrapper) {
+	if len(r.MemberlistKV.JoinMembers) > 0 {
+		r.Ingester.LifecyclerConfig.RingConfig.KVStore.Store = memberlistStr
+		r.Distributor.DistributorRing.KVStore.Store = memberlistStr
+		r.Ruler.Ring.KVStore.Store = memberlistStr
+	}
+}
diff --git a/pkg/loki/config_wrapper_test.go b/pkg/loki/config_wrapper_test.go
@@ -84,4 +84,65 @@ common:
 			assert.EqualValues(t, "/opt/loki/wal", config.Ingester.WAL.Dir)
 		})
 	})
+
+	t.Run("common memberlist config", func(t *testing.T) {
+		// components with rings
+		// * ingester
+		// * distributor
+		// * ruler
+
+		t.Run("does not automatically configure memberlist when no top-level memberlist config is provided", func(t *testing.T) {
+			configFileString := `---`
+			config, defaults := testContext(configFileString, nil)
+
+			assert.EqualValues(t, defaults.Ingester.LifecyclerConfig.RingConfig.KVStore.Store, config.Ingester.LifecyclerConfig.RingConfig.KVStore.Store)
+			assert.EqualValues(t, defaults.Distributor.DistributorRing.KVStore.Store, config.Distributor.DistributorRing.KVStore.Store)
+			assert.EqualValues(t, defaults.Ruler.Ring.KVStore.Store, config.Ruler.Ring.KVStore.Store)
+		})
+
+		t.Run("when top-level memberlist join_members are provided, all applicable rings are defaulted to use memberlist", func(t *testing.T) {
+			configFileString := `---
+memberlist:
+  join_members:
+    - foo.bar.example.com`
+
+			config, _ := testContext(configFileString, nil)
+
+			assert.EqualValues(t, memberlistStr, config.Ingester.LifecyclerConfig.RingConfig.KVStore.Store)
+			assert.EqualValues(t, memberlistStr, config.Distributor.DistributorRing.KVStore.Store)
+			assert.EqualValues(t, memberlistStr, config.Ruler.Ring.KVStore.Store)
+		})
+
+		t.Run("explicit ring configs provided via config file are preserved", func(t *testing.T) {
+			configFileString := `---
+memberlist:
+  join_members:
+    - foo.bar.example.com
+distributor:
+  ring:
+    kvstore:
+      store: etcd`
+
+			config, _ := testContext(configFileString, nil)
+
+			assert.EqualValues(t, "etcd", config.Distributor.DistributorRing.KVStore.Store)
+
+			assert.EqualValues(t, memberlistStr, config.Ingester.LifecyclerConfig.RingConfig.KVStore.Store)
+			assert.EqualValues(t, memberlistStr, config.Ruler.Ring.KVStore.Store)
+		})
+
+		t.Run("explicit ring configs provided via command line are preserved", func(t *testing.T) {
+			configFileString := `---
+memberlist:
+  join_members:
+    - foo.bar.example.com`
+
+			config, _ := testContext(configFileString, []string{"-ruler.ring.store", "inmemory"})
+
+			assert.EqualValues(t, "inmemory", config.Ruler.Ring.KVStore.Store)
+
+			assert.EqualValues(t, memberlistStr, config.Ingester.LifecyclerConfig.RingConfig.KVStore.Store)
+			assert.EqualValues(t, memberlistStr, config.Distributor.DistributorRing.KVStore.Store)
+		})
+	})
 }