Fix Receiver panic when querying uninitialized TSDBs (thanos-io#6067)

* Fix Receiver panic when querying uninitialized TSDBs Receivers currently panic when retrieving labels for a TSDB that is being initialized. The reason for this is that the tenant is added to the tenants map and the TSDB is started in the background. When retrieving labels, the MultiTSDB creates Clients for each TSDB, even if a TSDB is not yet ready. Signed-off-by: Filip Petkovski <[email protected]> * Add CHANGELOG Signed-off-by: Filip Petkovski <[email protected]> * Gracefully terminate test Signed-off-by: Filip Petkovski <[email protected]> Signed-off-by: Filip Petkovski <[email protected]>
ngraham20 · Apr 17, 2023 · 96a4dcd · 96a4dcd
1 parent 4f34de7
commit 96a4dcd
Show file tree

Hide file tree

Showing 3 changed files with 42 additions and 1 deletion.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -24,6 +24,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#5995](https://github.com/thanos-io/thanos/pull/5995) Sidecar: Loads the TLS certificate during startup.
 - [#6044](https://github.com/thanos-io/thanos/pull/6044) Receive: mark ouf of window errors as conflict, if out-of-window samples ingestion is activated
 - [#6066](https://github.com/thanos-io/thanos/pull/6066) Tracing: fixed panic because of nil sampler
+- [#6067](https://github.com/thanos-io/thanos/pull/6067) Receive: fixed panic when querying uninitialized TSDBs.
 
 ### Changed
 

diff --git a/pkg/receive/multitsdb.go b/pkg/receive/multitsdb.go
@@ -164,6 +164,9 @@ func (t *tenant) client() store.Client {
 	defer t.mtx.RUnlock()
 
 	store := t.store()
+	if store == nil {
+		return nil
+	}
 	client := storepb.ServerAsClient(store, 0)
 	return newLocalClient(client, store.LabelSet, store.TimeRange)
 }
@@ -429,7 +432,10 @@ func (t *MultiTSDB) TSDBLocalClients() []store.Client {
 
 	res := make([]store.Client, 0, len(t.tenants))
 	for _, tenant := range t.tenants {
-		res = append(res, tenant.client())
+		client := tenant.client()
+		if client != nil {
+			res = append(res, client)
+		}
 	}
 
 	return res

diff --git a/pkg/receive/multitsdb_test.go b/pkg/receive/multitsdb_test.go
@@ -22,6 +22,7 @@ import (
 	"golang.org/x/sync/errgroup"
 
 	"github.com/efficientgo/core/testutil"
+
 	"github.com/thanos-io/thanos/pkg/block/metadata"
 	"github.com/thanos-io/thanos/pkg/exemplars/exemplarspb"
 	"github.com/thanos-io/thanos/pkg/runutil"
@@ -521,6 +522,39 @@ func TestMultiTSDBStats(t *testing.T) {
 	}
 }
 
+// Regression test for https://github.com/thanos-io/thanos/issues/6047.
+func TestMultiTSDBWithNilStore(t *testing.T) {
+	dir := t.TempDir()
+
+	m := NewMultiTSDB(dir, log.NewNopLogger(), prometheus.NewRegistry(),
+		&tsdb.Options{
+			MinBlockDuration:  (2 * time.Hour).Milliseconds(),
+			MaxBlockDuration:  (2 * time.Hour).Milliseconds(),
+			RetentionDuration: (6 * time.Hour).Milliseconds(),
+		},
+		labels.FromStrings("replica", "test"),
+		"tenant_id",
+		nil,
+		false,
+		metadata.NoneFunc,
+	)
+	defer func() { testutil.Ok(t, m.Close()) }()
+
+	const tenantID = "test-tenant"
+	_, err := m.TenantAppendable(tenantID)
+	testutil.Ok(t, err)
+
+	// Get LabelSets of newly created TSDB.
+	clients := m.TSDBLocalClients()
+	for _, client := range clients {
+		testutil.Ok(t, testutil.FaultOrPanicToErr(func() { client.LabelSets() }))
+	}
+
+	// Wait for tenant to become ready before terminating the test.
+	// This allows the tear down procedure to cleanup properly.
+	testutil.Ok(t, appendSample(m, tenantID, time.Now()))
+}
+
 func appendSample(m *MultiTSDB, tenant string, timestamp time.Time) error {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()