Improve metric queries by computing samples at the edges.

docs/operations/upgrade.md

@@ -6,6 +6,11 @@ Unfortunately Loki is software and software is hard and sometimes things are not
 
 On this page we will document any upgrade issues/gotchas/considerations we are aware of.
 
+## 1.6.0
+
+A new ingester GRPC API has been added allowing to speed up metric queries, to ensure a rollout without query errors make sure you upgrade all ingesters first.
+Once this is done you can then proceed with the rest of the deployment, this is to ensure that queriers won't look for an API not yet available.
+
 ## 1.5.0
 
 Note: The required upgrade path outlined for version 1.4.0 below is still true for moving to 1.5.0 from any release older than 1.4.0 (e.g. 1.3.0->1.5.0 needs to also look at the 1.4.0 upgrade requirements).
@@ -102,8 +107,8 @@ docker run -d --name=loki --mount source=loki-data,target=/loki -p 3100:3100 gra
 
 Notice the change in the `target=/loki` for 1.5.0 to the new data directory location specified in the [included Loki config file](../../cmd/loki/loki-docker-config.yaml).
 
-The intermediate step of using an ubuntu image to change the ownership of the Loki files to the new user might not be necessary if you can easily access these files to run the `chown` command directly.  
-That is if you have access to `/var/lib/docker/volumes` or if you mounted to a different local filesystem directory, you can change the ownership directly without using a container. 
+The intermediate step of using an ubuntu image to change the ownership of the Loki files to the new user might not be necessary if you can easily access these files to run the `chown` command directly.
+That is if you have access to `/var/lib/docker/volumes` or if you mounted to a different local filesystem directory, you can change the ownership directly without using a container.
 
 
 ### Loki Duration Configs
@@ -146,7 +151,7 @@ The new values are:
 ```yaml
 min_period:
 max_period:
-max_retries: 
+max_retries:
 ```
 
 ## 1.4.0
@@ -157,9 +162,9 @@ One such config change which will affect Loki users:
 
 In the [cache_config](../configuration/README.md#cache_config):
 
-`defaul_validity` has changed to `default_validity` 
- 
-Also in the unlikely case you were configuring your schema via arguments and not a config file, this is no longer supported.  This is not something we had ever provided as an option via docs and is unlikely anyone is doing, but worth mentioning.  
+`defaul_validity` has changed to `default_validity`
+
+Also in the unlikely case you were configuring your schema via arguments and not a config file, this is no longer supported.  This is not something we had ever provided as an option via docs and is unlikely anyone is doing, but worth mentioning.
 
 The other config changes should not be relevant to Loki.
 
@@ -184,7 +189,7 @@ There are two options for upgrade if you are not on version 1.3.0 and are using
 
 * Upgrade first to v1.3.0 **BEFORE** upgrading to v1.4.0
 
-OR 
+OR
 
 **Note:** If you are running a single binary you only need to add this flag to your single binary command.
 

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/blang/semver v3.5.1+incompatible // indirect
 	github.com/bmatcuk/doublestar v1.2.2
 	github.com/c2h5oh/datasize v0.0.0-20200112174442-28bbd4740fee
+	github.com/cespare/xxhash/v2 v2.1.1
 	github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448 // indirect
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
 	github.com/cortexproject/cortex v1.2.1-0.20200702073552-0ea5a8b50b19
@@ -44,6 +45,7 @@ require (
 	github.com/prometheus/client_model v0.2.0
 	github.com/prometheus/common v0.10.0
 	github.com/prometheus/prometheus v1.8.2-0.20200626180636-d17d88935c8d
+	github.com/segmentio/fasthash v1.0.2
 	github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749
 	github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd
 	github.com/stretchr/testify v1.5.1
@@ -53,6 +55,7 @@ require (
 	github.com/weaveworks/common v0.0.0-20200512154658-384f10054ec5
 	go.etcd.io/bbolt v1.3.5-0.20200615073812-232d8fc87f50
 	golang.org/x/net v0.0.0-20200602114024-627f9648deb9
+	golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae // indirect
 	google.golang.org/grpc v1.29.1
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/fsnotify.v1 v1.4.7

go.sum

@@ -1074,6 +1074,8 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUt
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/segmentio/fasthash v0.0.0-20180216231524-a72b379d632e h1:uO75wNGioszjmIzcY/tvdDYKRLVvzggtAmmJkn9j4GQ=
 github.com/segmentio/fasthash v0.0.0-20180216231524-a72b379d632e/go.mod h1:tm/wZFQ8e24NYaBGIlnO2WGCAi67re4HHuOm0sftE/M=
+github.com/segmentio/fasthash v1.0.2 h1:86fGDl2hB+iSHYlccB/FP9qRGvLNuH/fhEEFn6gnQUs=
+github.com/segmentio/fasthash v1.0.2/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY=
 github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo=
 github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo=
 github.com/sercand/kuberesolver v2.1.0+incompatible h1:iJ1oCzPQ/aacsbCWLfJW1hPKkHMvCEgNSA9kvWcb9MY=
@@ -1429,6 +1431,8 @@ golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80=
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

pkg/chunkenc/dumb_chunk.go

@@ -93,6 +93,10 @@ func (c *dumbChunk) Iterator(_ context.Context, from, through time.Time, directi
 	}, nil
 }
 
+func (c *dumbChunk) SampleIterator(_ context.Context, from, through time.Time, _ logql.LineFilter, _ logql.SampleExtractor) iter.SampleIterator {
+	return nil
+}
+
 func (c *dumbChunk) Bytes() ([]byte, error) {
 	return nil, nil
 }

pkg/chunkenc/hash_test.go

@@ -0,0 +1,73 @@
+package chunkenc
+
+import (
+	"hash/fnv"
+	"hash/maphash"
+	"testing"
+
+	"github.com/cespare/xxhash/v2"
+	"github.com/segmentio/fasthash/fnv1a"
+	"github.com/stretchr/testify/require"
+
+	"github.com/grafana/loki/pkg/chunkenc/testdata"
+)
+
+var res uint64
+
+func Benchmark_fnv64a(b *testing.B) {
+	for n := 0; n < b.N; n++ {
+		for i := 0; i < len(testdata.LogsBytes); i++ {
+			h := fnv.New64a()
+			_, _ = h.Write(testdata.LogsBytes[i])
+			res = h.Sum64()
+		}
+	}
+}
+
+func Benchmark_fnv64a_third_party(b *testing.B) {
+	for n := 0; n < b.N; n++ {
+		for i := 0; i < len(testdata.LogsBytes); i++ {
+			res = fnv1a.HashBytes64(testdata.LogsBytes[i])
+		}
+	}
+}
+
+func Benchmark_xxhash(b *testing.B) {
+	for n := 0; n < b.N; n++ {
+		for i := 0; i < len(testdata.LogsBytes); i++ {
+			res = xxhash.Sum64(testdata.LogsBytes[i])
+		}
+	}
+}
+
+func Benchmark_hashmap(b *testing.B) {
+	// I discarded hashmap/map as it will compute different value on different binary for the same entry
+	var h maphash.Hash
+	for n := 0; n < b.N; n++ {
+		for i := 0; i < len(testdata.LogsBytes); i++ {
+			h.SetSeed(maphash.MakeSeed())
+			_, _ = h.Write(testdata.LogsBytes[i])
+			res = h.Sum64()
+		}
+	}
+}
+
+func Test_xxhash_integrity(t *testing.T) {
+	data := []uint64{}
+
+	for i := 0; i < len(testdata.LogsBytes); i++ {
+		data = append(data, xxhash.Sum64(testdata.LogsBytes[i]))
+	}
+
+	for i := 0; i < len(testdata.LogsBytes); i++ {
+		require.Equal(t, data[i], xxhash.Sum64(testdata.LogsBytes[i]))
+	}
+
+	unique := map[uint64]struct{}{}
+	for i := 0; i < len(testdata.LogsBytes); i++ {
+		_, ok := unique[xxhash.Sum64(testdata.LogsBytes[i])]
+		require.False(t, ok, string(testdata.LogsBytes[i])) // all lines have been made unique
+		unique[xxhash.Sum64(testdata.LogsBytes[i])] = struct{}{}
+	}
+
+}

pkg/chunkenc/interface.go

@@ -98,6 +98,7 @@ type Chunk interface {
 	SpaceFor(*logproto.Entry) bool
 	Append(*logproto.Entry) error
 	Iterator(ctx context.Context, from, through time.Time, direction logproto.Direction, filter logql.LineFilter) (iter.EntryIterator, error)
+	SampleIterator(ctx context.Context, from, through time.Time, filter logql.LineFilter, extractor logql.SampleExtractor) iter.SampleIterator
 	// Returns the list of blocks in the chunks.
 	Blocks(mintT, maxtT time.Time) []Block
 	Size() int
@@ -121,4 +122,6 @@ type Block interface {
 	Entries() int
 	// Iterator returns an entry iterator for the block.
 	Iterator(context.Context, logql.LineFilter) iter.EntryIterator
+	// SampleIterator returns a sample iterator for the block.
+	SampleIterator(context.Context, logql.LineFilter, logql.SampleExtractor) iter.SampleIterator
 }