store: Filter blocks before loading it. Sort advertise labels; Added sharding e2e test.

CHANGELOG.md

@@ -18,6 +18,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel
 ### Fixed
 
 - [#1656](https://github.com/thanos-io/thanos/pull/1656) Thanos Store now starts metric and status probe HTTP server earlier in its start-up sequence. `/-/healthy` endpoint now starts to respond with success earlier. `/metrics` endpoint starts serving metrics earlier as well. Make sure to point your readiness probes to the `/-/ready` endpoint rather than `/metrics`.
+- [#1669](https://github.com/thanos-io/thanos/pull/1669) Fixed store sharding. Now it does not load excluded meta.jsons and load/fetch index-cache.json files.
 
 ## [v0.8.1](https://github.com/thanos-io/thanos/releases/tag/v0.8.1) - 2019.10.14
 

pkg/objstore/azure/azure.go

@@ -291,7 +291,7 @@ func NewTestBucket(t testing.TB, component string) (objstore.Bucket, func(), err
 	conf := &Config{
 		StorageAccountName: os.Getenv("AZURE_STORAGE_ACCOUNT"),
 		StorageAccountKey:  os.Getenv("AZURE_STORAGE_ACCESS_KEY"),
-		ContainerName:      "thanos-e2e-test",
+		ContainerName:      objstore.CreateTemporaryTestBucketName(t),
 	}
 
 	bc, err := yaml.Marshal(conf)

pkg/objstore/cos/cos.go

@@ -4,19 +4,17 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"math/rand"
 	"net/http"
 	"os"
 	"strings"
 	"testing"
-	"time"
 
 	"github.com/go-kit/kit/log"
-	cos "github.com/mozillazg/go-cos"
+	"github.com/mozillazg/go-cos"
 	"github.com/pkg/errors"
 	"github.com/thanos-io/thanos/pkg/objstore"
 	"github.com/thanos-io/thanos/pkg/runutil"
-	yaml "gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v2"
 )
 
 // DirDelim is the delimiter used to model a directory structure in an object store bucket.
@@ -311,14 +309,7 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) {
 		t.Log("WARNING. Reusing", c.Bucket, "COS bucket for COS tests. Manual cleanup afterwards is required")
 		return b, func() {}, nil
 	}
-
-	src := rand.NewSource(time.Now().UnixNano())
-
-	tmpBucketName := strings.Replace(fmt.Sprintf("test_%x", src.Int63()), "_", "-", -1)
-	if len(tmpBucketName) >= 31 {
-		tmpBucketName = tmpBucketName[:31]
-	}
-	c.Bucket = tmpBucketName
+	c.Bucket = objstore.CreateTemporaryTestBucketName(t)
 
 	bc, err := yaml.Marshal(c)
 	if err != nil {
@@ -333,12 +324,12 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) {
 	if _, err := b.client.Bucket.Put(context.Background(), nil); err != nil {
 		return nil, nil, err
 	}
-	t.Log("created temporary COS bucket for COS tests with name", tmpBucketName)
+	t.Log("created temporary COS bucket for COS tests with name", c.Bucket)
 
 	return b, func() {
 		objstore.EmptyBucket(t, context.Background(), b)
 		if _, err := b.client.Bucket.Delete(context.Background()); err != nil {
-			t.Logf("deleting bucket %s failed: %s", tmpBucketName, err)
+			t.Logf("deleting bucket %s failed: %s", c.Bucket, err)
 		}
 	}, nil
 }

pkg/objstore/gcs/gcs.go

@@ -5,11 +5,9 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"math/rand"
 	"runtime"
 	"strings"
 	"testing"
-	"time"
 
 	"cloud.google.com/go/storage"
 	"github.com/go-kit/kit/log"
@@ -19,7 +17,7 @@ import (
 	"golang.org/x/oauth2/google"
 	"google.golang.org/api/iterator"
 	"google.golang.org/api/option"
-	yaml "gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v2"
 )
 
 // DirDelim is the delimiter used to model a directory structure in an object store bucket.
@@ -169,9 +167,8 @@ func (b *Bucket) Close() error {
 func NewTestBucket(t testing.TB, project string) (objstore.Bucket, func(), error) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	src := rand.NewSource(time.Now().UnixNano())
 	gTestConfig := Config{
-		Bucket: fmt.Sprintf("test_%s_%x", strings.ToLower(t.Name()), src.Int63()),
+		Bucket: objstore.CreateTemporaryTestBucketName(t),
 	}
 
 	bc, err := yaml.Marshal(gTestConfig)

pkg/objstore/objstore.go

@@ -2,10 +2,13 @@ package objstore
 
 import (
 	"context"
+	"fmt"
 	"io"
+	"math/rand"
 	"os"
 	"path/filepath"
 	"strings"
+	"testing"
 	"time"
 
 	"github.com/go-kit/kit/log"
@@ -109,7 +112,7 @@ func DownloadFile(ctx context.Context, logger log.Logger, bkt BucketReader, src,
 
 	rc, err := bkt.Get(ctx, src)
 	if err != nil {
-		return errors.Wrap(err, "get file")
+		return errors.Wrapf(err, "get file %s", src)
 	}
 	defer runutil.CloseWithLogOnErr(logger, rc, "download block's file reader")
 
@@ -372,3 +375,14 @@ func (rc *timingReadCloser) Read(b []byte) (n int, err error) {
 	}
 	return n, err
 }
+
+func CreateTemporaryTestBucketName(t testing.TB) string {
+	src := rand.NewSource(time.Now().UnixNano())
+
+	// Bucket name need to conform: https://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-s3-bucket-naming-requirements.html.
+	name := strings.Replace(strings.Replace(fmt.Sprintf("test_%x_%s", src.Int63(), strings.ToLower(t.Name())), "_", "-", -1), "/", "-", -1)
+	if len(name) >= 63 {
+		name = name[:63]
+	}
+	return name
+}

pkg/objstore/objtesting/foreach.go

@@ -3,9 +3,7 @@ package objtesting
 import (
 	"os"
 	"testing"
-	"time"
 
-	"github.com/fortytw2/leaktest"
 	"github.com/thanos-io/thanos/pkg/objstore"
 	"github.com/thanos-io/thanos/pkg/objstore/azure"
 	"github.com/thanos-io/thanos/pkg/objstore/cos"
@@ -21,98 +19,98 @@ import (
 // that deletes it after test was run.
 // Use THANOS_SKIP_<objstorename>_TESTS to skip explicitly certain tests.
 func ForeachStore(t *testing.T, testFn func(t testing.TB, bkt objstore.Bucket)) {
+	t.Parallel()
+
 	// Mandatory Inmem.
 	if ok := t.Run("inmem", func(t *testing.T) {
-		defer leaktest.CheckTimeout(t, 10*time.Second)()
-
+		t.Parallel()
 		testFn(t, inmem.NewBucket())
-
 	}); !ok {
 		return
 	}
 
 	// Optional GCS.
 	if _, ok := os.LookupEnv("THANOS_SKIP_GCS_TESTS"); !ok {
-		bkt, closeFn, err := gcs.NewTestBucket(t, os.Getenv("GCP_PROJECT"))
-		testutil.Ok(t, err)
+		t.Run("gcs", func(t *testing.T) {
+			bkt, closeFn, err := gcs.NewTestBucket(t, os.Getenv("GCP_PROJECT"))
+			testutil.Ok(t, err)
+
+			t.Parallel()
+			defer closeFn()
 
-		ok := t.Run("gcs", func(t *testing.T) {
 			// TODO(bwplotka): Add leaktest when https://github.com/GoogleCloudPlatform/google-cloud-go/issues/1025 is resolved.
 			testFn(t, bkt)
 		})
-		closeFn()
-		if !ok {
-			return
-		}
+
 	} else {
 		t.Log("THANOS_SKIP_GCS_TESTS envvar present. Skipping test against GCS.")
 	}
 
 	// Optional S3.
 	if _, ok := os.LookupEnv("THANOS_SKIP_S3_AWS_TESTS"); !ok {
-		// TODO(bwplotka): Allow taking location from envvar.
-		bkt, closeFn, err := s3.NewTestBucket(t, "us-west-2")
-		testutil.Ok(t, err)
+		t.Run("aws s3", func(t *testing.T) {
+			// TODO(bwplotka): Allow taking location from envvar.
+			bkt, closeFn, err := s3.NewTestBucket(t, "us-west-2")
+			testutil.Ok(t, err)
+
+			t.Parallel()
+			defer closeFn()
 
-		ok := t.Run("aws s3", func(t *testing.T) {
 			// TODO(bwplotka): Add leaktest when we fix potential leak in minio library.
 			// We cannot use leaktest for detecting our own potential leaks, when leaktest detects leaks in minio itself.
 			// This needs to be investigated more.
 
 			testFn(t, bkt)
 		})
-		closeFn()
-		if !ok {
-			return
-		}
+
 	} else {
 		t.Log("THANOS_SKIP_S3_AWS_TESTS envvar present. Skipping test against S3 AWS.")
 	}
 
 	// Optional Azure.
 	if _, ok := os.LookupEnv("THANOS_SKIP_AZURE_TESTS"); !ok {
-		bkt, closeFn, err := azure.NewTestBucket(t, "e2e-tests")
-		testutil.Ok(t, err)
+		t.Run("azure", func(t *testing.T) {
+			bkt, closeFn, err := azure.NewTestBucket(t, "e2e-tests")
+			testutil.Ok(t, err)
+
+			t.Parallel()
+			defer closeFn()
 
-		ok := t.Run("azure", func(t *testing.T) {
 			testFn(t, bkt)
 		})
-		closeFn()
-		if !ok {
-			return
-		}
+
 	} else {
 		t.Log("THANOS_SKIP_AZURE_TESTS envvar present. Skipping test against Azure.")
 	}
 
 	// Optional SWIFT.
 	if _, ok := os.LookupEnv("THANOS_SKIP_SWIFT_TESTS"); !ok {
-		container, closeFn, err := swift.NewTestContainer(t)
-		testutil.Ok(t, err)
+		t.Run("swift", func(t *testing.T) {
+			container, closeFn, err := swift.NewTestContainer(t)
+			testutil.Ok(t, err)
+
+			t.Parallel()
+			defer closeFn()
 
-		ok := t.Run("swift", func(t *testing.T) {
 			testFn(t, container)
 		})
-		closeFn()
-		if !ok {
-			return
-		}
+
 	} else {
 		t.Log("THANOS_SKIP_SWIFT_TESTS envvar present. Skipping test against swift.")
 	}
 
 	// Optional COS.
 	if _, ok := os.LookupEnv("THANOS_SKIP_TENCENT_COS_TESTS"); !ok {
-		bkt, closeFn, err := cos.NewTestBucket(t)
-		testutil.Ok(t, err)
+		t.Run("Tencent cos", func(t *testing.T) {
+			bkt, closeFn, err := cos.NewTestBucket(t)
+			testutil.Ok(t, err)
+
+			t.Parallel()
+			defer closeFn()
 
-		ok := t.Run("Tencent cos", func(t *testing.T) {
 			testFn(t, bkt)
 		})
-		closeFn()
-		if !ok {
-			return
-		}
+
 	} else {
 		t.Log("THANOS_SKIP_TENCENT_COS_TESTS envvar present. Skipping test against Tencent COS.")
 	}

pkg/objstore/s3/s3.go

@@ -6,7 +6,6 @@ import (
 	"crypto/tls"
 	"fmt"
 	"io"
-	"math/rand"
 	"net"
 	"net/http"
 	"os"
@@ -18,15 +17,15 @@ import (
 
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
-	minio "github.com/minio/minio-go/v6"
+	"github.com/minio/minio-go/v6"
 	"github.com/minio/minio-go/v6/pkg/credentials"
 	"github.com/minio/minio-go/v6/pkg/encrypt"
 	"github.com/pkg/errors"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/version"
 	"github.com/thanos-io/thanos/pkg/objstore"
 	"github.com/thanos-io/thanos/pkg/runutil"
-	yaml "gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v2"
 )
 
 // DirDelim is the delimiter used to model a directory structure in an object store bucket.
@@ -403,13 +402,7 @@ func NewTestBucketFromConfig(t testing.TB, location string, c Config, reuseBucke
 	}
 
 	if c.Bucket == "" {
-		src := rand.NewSource(time.Now().UnixNano())
-
-		// Bucket name need to conform: https://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-s3-bucket-naming-requirements.html.
-		bktToCreate = strings.Replace(fmt.Sprintf("test_%s_%x", strings.ToLower(t.Name()), src.Int63()), "_", "-", -1)
-		if len(bktToCreate) >= 63 {
-			bktToCreate = bktToCreate[:63]
-		}
+		bktToCreate = objstore.CreateTemporaryTestBucketName(t)
 	}
 
 	if err := b.client.MakeBucket(bktToCreate, location); err != nil {

pkg/objstore/swift/swift.go

@@ -5,11 +5,9 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"math/rand"
 	"os"
 	"strings"
 	"testing"
-	"time"
 
 	"github.com/go-kit/kit/log"
 	"github.com/gophercloud/gophercloud"
@@ -19,7 +17,7 @@ import (
 	"github.com/gophercloud/gophercloud/pagination"
 	"github.com/pkg/errors"
 	"github.com/thanos-io/thanos/pkg/objstore"
-	yaml "gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v2"
 )
 
 // DirDelim is the delimiter used to model a directory structure in an object store bucket.
@@ -291,12 +289,7 @@ func NewTestContainer(t testing.TB) (objstore.Bucket, func(), error) {
 		return c, func() {}, nil
 	}
 
-	src := rand.NewSource(time.Now().UnixNano())
-
-	tmpContainerName := fmt.Sprintf("test_%s_%x", strings.ToLower(t.Name()), src.Int63())
-	if len(tmpContainerName) >= 63 {
-		tmpContainerName = tmpContainerName[:63]
-	}
+	tmpContainerName := objstore.CreateTemporaryTestBucketName(t)
 
 	if err := c.createContainer(tmpContainerName); err != nil {
 		return nil, nil, err