Add memcached support to index cache (#1881)

* Moved index cache key struct outside of the file containing the in-memory cache backend because generic Signed-off-by: Marco Pracucci <[email protected]> * Added cacheKey.string() to make it memcached friendly Signed-off-by: Marco Pracucci <[email protected]> * Added MemcachedIndexCache support Signed-off-by: Marco Pracucci <[email protected]> * Export Prometheus metrics from MemcachedIndexCache Signed-off-by: Marco Pracucci <[email protected]> * Fixed linter issues Signed-off-by: Marco Pracucci <[email protected]> * Simplified workers wait group in memcachedClient Signed-off-by: Marco Pracucci <[email protected]> * Fixed memcached client GetMulti() results batches channel buffer Signed-off-by: Marco Pracucci <[email protected]> * Wait for addrs resolution gorouting to complete on memcachedClient.Stop() Signed-off-by: Marco Pracucci <[email protected]> * Return struct from NewMemcachedClient() Signed-off-by: Marco Pracucci <[email protected]> * Update pkg/cacheutil/memcached_client.go Co-Authored-By: Bartlomiej Plotka <[email protected]> Signed-off-by: Marco Pracucci <[email protected]> * Simplified memcachedClient tests Signed-off-by: Marco Pracucci <[email protected]> * Cleaned up code based on feedback Signed-off-by: Marco Pracucci <[email protected]> * Removed error from GetMulti() return and introduced metrics and tracing in MemcachedClient Signed-off-by: Marco Pracucci <[email protected]> * Fixed compilation errors in store E2E tests Signed-off-by: Marco Pracucci <[email protected]> * Added leaktest check to all tests Signed-off-by: Marco Pracucci <[email protected]> * Introduced --index.cache-config support Signed-off-by: Marco Pracucci <[email protected]> * Fixed compilation errors in store E2E tests Signed-off-by: Marco Pracucci <[email protected]> * Updated store flags doc Signed-off-by: Marco Pracucci <[email protected]> * Updated index cache doc Signed-off-by: Marco Pracucci <[email protected]> * Updated changelog Signed-off-by: Marco Pracucci <[email protected]> * Increased default memcached client timeout from 100ms to 500ms Signed-off-by: Marco Pracucci <[email protected]> * Disabled memcached client max batch size by default Signed-off-by: Marco Pracucci <[email protected]> * Fixed index cache key max length Signed-off-by: Marco Pracucci <[email protected]> * Removed TODO from memcached client since looks the general consensus is to have a global limit on the max concurrent batches Signed-off-by: Marco Pracucci <[email protected]> * Allow to configure in-memory index cache using byte units Signed-off-by: Marco Pracucci <[email protected]> * Refactored index cache config file doc Signed-off-by: Marco Pracucci <[email protected]> * Fixed nits in comments Signed-off-by: Marco Pracucci <[email protected]> * Replaced hardcoded 16 with ULID calculated length based on review comment Signed-off-by: Marco Pracucci <[email protected]> * Do not expose jumpHash func Signed-off-by: Marco Pracucci <[email protected]> * Updated changelog Signed-off-by: Marco Pracucci <[email protected]> * Switched MemcachedClient GetMulti() concurrency limit to a gate Signed-off-by: Marco Pracucci <[email protected]> * Fixed flaky tests Signed-off-by: Marco Pracucci <[email protected]> * Fixed typos in comments Signed-off-by: Marco Pracucci <[email protected]> * Renamed memcached config option addrs to addresses Signed-off-by: Marco Pracucci <[email protected]> Co-authored-by: Bartlomiej Plotka <[email protected]>
thanos-io · Jan 3, 2020 · bb346c0 · bb346c0
1 parent a37ac09
commit bb346c0
Show file tree

Hide file tree

Showing 23 changed files with 2,104 additions and 133 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -22,6 +22,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel
 - [#1854](https://github.com/thanos-io/thanos/pull/1854) Update Rule UI to support alerts count displaying and filtering.
 - [#1838](https://github.com/thanos-io/thanos/pull/1838) Ruler: Add TLS and authentication support for Alertmanager with the `--alertmanagers.config` and `--alertmanagers.config-file` CLI flags. See [documentation](docs/components/rule.md/#configuration) for further information.
 - [#1838](https://github.com/thanos-io/thanos/pull/1838) Ruler: Add a new `--alertmanagers.sd-dns-interval` CLI option to specify the interval between DNS resolutions of Alertmanager hosts.
+- [#1881](https://github.com/thanos-io/thanos/pull/1881) Store Gateway: memcached support for index cache. See [documentation](docs/components/store.md/#index-cache) for further information.
 
 ## [v0.9.0](https://github.com/thanos-io/thanos/releases/tag/v0.9.0) - 2019.12.03
 

diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go
@@ -36,9 +36,13 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application) {
 	dataDir := cmd.Flag("data-dir", "Data directory in which to cache remote blocks.").
 		Default("./data").String()
 
-	indexCacheSize := cmd.Flag("index-cache-size", "Maximum size of items held in the index cache.").
+	indexCacheSize := cmd.Flag("index-cache-size", "Maximum size of items held in the in-memory index cache. Ignored if --index-cache.config or --index-cache.config-file option is specified.").
 		Default("250MB").Bytes()
 
+	indexCacheConfig := extflag.RegisterPathOrContent(cmd, "index-cache.config",
+		"YAML file that contains index cache configuration. See format details: https://thanos.io/components/store.md/#index-cache",
+		false)
+
 	chunkPoolSize := cmd.Flag("chunk-pool-size", "Maximum size of concurrently allocatable bytes for chunks.").
 		Default("2GB").Bytes()
 
@@ -77,6 +81,7 @@ func registerStore(m map[string]setupFunc, app *kingpin.Application) {
 			logger,
 			reg,
 			tracer,
+			indexCacheConfig,
 			objStoreConfig,
 			*dataDir,
 			*grpcBindAddr,
@@ -110,6 +115,7 @@ func runStore(
 	logger log.Logger,
 	reg *prometheus.Registry,
 	tracer opentracing.Tracer,
+	indexCacheConfig *extflag.PathOrContent,
 	objStoreConfig *extflag.PathOrContent,
 	dataDir string,
 	grpcBindAddr string,
@@ -169,20 +175,29 @@ func runStore(
 		return err
 	}
 
+	indexCacheContentYaml, err := indexCacheConfig.Content()
+	if err != nil {
+		return errors.Wrap(err, "get content of index cache configuration")
+	}
+
 	// Ensure we close up everything properly.
 	defer func() {
 		if err != nil {
 			runutil.CloseWithLogOnErr(logger, bkt, "bucket client")
 		}
 	}()
 
-	// TODO(bwplotka): Add as a flag?
-	maxItemSizeBytes := indexCacheSizeBytes / 2
-
-	indexCache, err := storecache.NewInMemoryIndexCache(logger, reg, storecache.Opts{
-		MaxSizeBytes:     indexCacheSizeBytes,
-		MaxItemSizeBytes: maxItemSizeBytes,
-	})
+	// Create the index cache loading its config from config file, while keeping
+	// backward compatibility with the pre-config file era.
+	var indexCache storecache.IndexCache
+	if len(indexCacheContentYaml) > 0 {
+		indexCache, err = storecache.NewIndexCache(logger, indexCacheContentYaml, reg)
+	} else {
+		indexCache, err = storecache.NewInMemoryIndexCacheWithConfig(logger, reg, storecache.InMemoryIndexCacheConfig{
+			MaxSize:     storecache.Bytes(indexCacheSizeBytes),
+			MaxItemSize: storecache.DefaultInMemoryIndexCacheConfig.MaxItemSize,
+		})
+	}
 	if err != nil {
 		return errors.Wrap(err, "create index cache")
 	}

diff --git a/docs/components/store.md b/docs/components/store.md
@@ -68,7 +68,19 @@ Flags:
                                  CA is specified, there is no client
                                  verification on server side. (tls.NoClientCert)
       --data-dir="./data"        Data directory in which to cache remote blocks.
-      --index-cache-size=250MB   Maximum size of items held in the index cache.
+      --index-cache-size=250MB   Maximum size of items held in the in-memory
+                                 index cache. Ignored if --index-cache.config or
+                                 --index-cache.config-file option is specified.
+      --index-cache.config-file=<file-path>
+                                 Path to YAML file that contains index cache
+                                 configuration. See format details:
+                                 https://thanos.io/components/store.md/#index-cache
+      --index-cache.config=<content>
+                                 Alternative to 'index-cache.config-file' flag
+                                 (lower priority). Content of YAML file that
+                                 contains index cache configuration. See format
+                                 details:
+                                 https://thanos.io/components/store.md/#index-cache
       --chunk-pool-size=2GB      Maximum size of concurrently allocatable bytes
                                  for chunks.
       --store.grpc.series-sample-limit=0
@@ -151,3 +163,61 @@ Filtering is done on a Chunk level, so Thanos Store might still return Samples w
   - `/-/ready` starts after all the bootstrapping completed (e.g initial index building) and ready to serve traffic.
 
 > NOTE: Metric endpoint starts immediately so, make sure you set up readiness probe on designated HTTP `/-/ready` path.
+
+## Index cache
+
+Thanos Store Gateway supports an index cache to speed up postings and series lookups from TSDB blocks indexes. Two types of caches are supported:
+
+- `in-memory` (_default_)
+- `memcached`
+
+### In-memory index cache
+
+The `in-memory` index cache is enabled by default and its max size can be configured through the flag `--index-cache-size`.
+
+Alternatively, the `in-memory` index cache can also by configured using `--index-cache.config-file` to reference to the configuration file or `--index-cache.config` to put yaml config directly:
+
+[embedmd]:# (../flags/config_index_cache_in_memory.txt yaml)
+```yaml
+type: IN-MEMORY
+config:
+  max_size: 0
+  max_item_size: 0
+```
+
+All the settings are **optional**:
+
+- `max_size`: overall maximum number of bytes cache can contain. The value should be specified with a bytes unit (ie. `250MB`).
+- `max_item_size`: maximum size of single item, in bytes. The value should be specified with a bytes unit (ie. `125MB`).
+
+### Memcached index cache
+
+The `memcached` index cache allows to use [Memcached](https://memcached.org) as cache backend. This cache type is configured using `--index-cache.config-file` to reference to the configuration file or `--index-cache.config` to put yaml config directly:
+
+[embedmd]:# (../flags/config_index_cache_memcached.txt yaml)
+```yaml
+type: MEMCACHED
+config:
+  addresses: []
+  timeout: 0s
+  max_idle_connections: 0
+  max_async_concurrency: 0
+  max_async_buffer_size: 0
+  max_get_multi_concurrency: 0
+  max_get_multi_batch_size: 0
+  dns_provider_update_interval: 0s
+```
+
+The **required** settings are:
+
+- `addresses`: list of memcached addresses, that will get resolved with the [DNS service discovery](../service-discovery.md/#dns-service-discovery) provider.
+
+While the remaining settings are **optional**:
+
+- `timeout`: the socket read/write timeout.
+- `max_idle_connections`: maximum number of idle connections that will be maintained per address.
+- `max_async_concurrency`: maximum number of concurrent asynchronous operations can occur.
+- `max_async_buffer_size`: maximum number of enqueued asynchronous operations allowed.
+- `max_get_multi_concurrency`: maximum number of concurrent connections when fetching keys. If set to `0`, the concurrency is unlimited.
+- `max_get_multi_batch_size`: maximum number of keys a single underlying operation should fetch. If more keys are specified, internally keys are splitted into multiple batches and fetched concurrently, honoring `max_get_multi_concurrency`. If set to `0`, the batch size is unlimited.
+- `dns_provider_update_interval`: the DNS discovery update interval.
diff --git a/go.mod b/go.mod
@@ -13,17 +13,19 @@ require (
 	github.com/Azure/go-autorest/autorest/validation v0.2.1-0.20191028180845-3492b2aff503 // indirect
 	github.com/NYTimes/gziphandler v1.1.1
 	github.com/OneOfOne/xxhash v1.2.6 // indirect
-	github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect
+	github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d
 	github.com/aliyun/aliyun-oss-go-sdk v2.0.4+incompatible
 	github.com/armon/go-metrics v0.3.0
 	github.com/aws/aws-sdk-go v1.25.35 // indirect
 	github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect
+	github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b
 	github.com/cespare/xxhash v1.1.0
 	github.com/cespare/xxhash/v2 v2.1.1 // indirect
 	github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
 	github.com/elastic/go-sysinfo v1.1.1 // indirect
 	github.com/elastic/go-windows v1.0.1 // indirect
 	github.com/evanphx/json-patch v4.5.0+incompatible // indirect
+	github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb
 	github.com/fatih/structtag v1.1.0
 	github.com/fortytw2/leaktest v1.3.0
 	github.com/fsnotify/fsnotify v1.4.7
@@ -83,7 +85,7 @@ require (
 	go.opencensus.io v0.22.2 // indirect
 	go.uber.org/atomic v1.5.0 // indirect
 	go.uber.org/automaxprocs v1.2.0
-	golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708 // indirect
+	golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
 	golang.org/x/sys v0.0.0-20191113165036-4c7a9d0fe056 // indirect

diff --git a/go.sum b/go.sum
@@ -93,6 +93,8 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b h1:L/QXpzIa3pOvUGt1D1lA5KjYhPBAN/3iWdP7xeFS9F0=
+github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA=
 github.com/cenkalti/backoff v0.0.0-20181003080854-62661b46c409/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v0.0.0-20181017004759-096ff4a8a059/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
@@ -134,6 +136,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
 github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M=
 github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb h1:IT4JYU7k4ikYg1SCxNI1/Tieq/NFvh6dzLdgi7eu0tM=
+github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb/go.mod h1:bH6Xx7IW64qjjJq8M2u4dxNaBiDfKK+z/3eGDpXEQhc=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/structtag v1.1.0 h1:6j4mUV/ES2duvnAzKMFkN6/A5mCaNYPD3xfbAkLLOF8=
 github.com/fatih/structtag v1.1.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94=

diff --git a/pkg/cacheutil/jump_hash.go b/pkg/cacheutil/jump_hash.go
@@ -0,0 +1,18 @@
+package cacheutil
+
+// jumpHash consistently chooses a hash bucket number in the range
+// [0, numBuckets) for the given key. numBuckets must be >= 1.
+//
+// Copied from github.com/dgryski/go-jump/blob/master/jump.go (MIT license).
+func jumpHash(key uint64, numBuckets int) int32 {
+	var b int64 = -1
+	var j int64
+
+	for j < int64(numBuckets) {
+		b = j
+		key = key*2862933555777941757 + 1
+		j = int64(float64(b+1) * (float64(int64(1)<<31) / float64((key>>33)+1)))
+	}
+
+	return int32(b)
+}