From 7ed50400d11b186a987dfcdd248bcf4f927d4086 Mon Sep 17 00:00:00 2001 From: Michael Hoffmann Date: Fri, 1 Sep 2023 17:49:22 +0200 Subject: [PATCH] Store: move acceptance tests to acceptance_test.go Signed-off-by: Michael Hoffmann --- pkg/store/acceptance_test.go | 134 +++++++++++++++++++++++++++++++++++ pkg/store/bucket_test.go | 77 -------------------- pkg/store/prometheus_test.go | 27 ------- pkg/store/tsdb_test.go | 15 ---- 4 files changed, 134 insertions(+), 119 deletions(-) diff --git a/pkg/store/acceptance_test.go b/pkg/store/acceptance_test.go index 1469297a217..a444866920a 100644 --- a/pkg/store/acceptance_test.go +++ b/pkg/store/acceptance_test.go @@ -5,17 +5,32 @@ package store import ( "context" + "fmt" + "net/url" + "os" + "path/filepath" "testing" "time" + "github.com/go-kit/log" "github.com/pkg/errors" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/timestamp" "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/tsdb" "golang.org/x/exp/slices" "github.com/efficientgo/core/testutil" + "github.com/thanos-io/objstore" + "github.com/thanos-io/objstore/providers/filesystem" + "github.com/thanos-io/thanos/pkg/block" + "github.com/thanos-io/thanos/pkg/block/metadata" + "github.com/thanos-io/thanos/pkg/component" + "github.com/thanos-io/thanos/pkg/promclient" "github.com/thanos-io/thanos/pkg/store/storepb" + "github.com/thanos-io/thanos/pkg/stringset" + "github.com/thanos-io/thanos/pkg/testutil/custom" + "github.com/thanos-io/thanos/pkg/testutil/e2eutil" ) type labelNameCallCase struct { @@ -466,3 +481,122 @@ func testStoreAPIsAcceptance(t *testing.T, startStore func(t *testing.T, extLset }) } } + +func TestBucketStore_Acceptance(t *testing.T) { + t.Cleanup(func() { custom.TolerantVerifyLeak(t) }) + + testStoreAPIsAcceptance(t, func(tt *testing.T, extLset labels.Labels, appendFn func(app storage.Appender)) storepb.StoreServer { + tmpDir := tt.TempDir() + bktDir := filepath.Join(tmpDir, "bkt") + auxDir := filepath.Join(tmpDir, "aux") + metaDir := filepath.Join(tmpDir, "meta") + + testutil.Ok(tt, os.MkdirAll(metaDir, os.ModePerm)) + testutil.Ok(tt, os.MkdirAll(auxDir, os.ModePerm)) + + bkt, err := filesystem.NewBucket(bktDir) + testutil.Ok(tt, err) + tt.Cleanup(func() { testutil.Ok(tt, bkt.Close()) }) + + headOpts := tsdb.DefaultHeadOptions() + headOpts.ChunkDirRoot = tmpDir + headOpts.ChunkRange = 1000 + h, err := tsdb.NewHead(nil, nil, nil, nil, headOpts, nil) + testutil.Ok(tt, err) + tt.Cleanup(func() { testutil.Ok(tt, h.Close()) }) + logger := log.NewNopLogger() + + appendFn(h.Appender(context.Background())) + + if h.NumSeries() == 0 { + tt.Skip("Bucket Store cannot handle empty HEAD") + } + + id := createBlockFromHead(tt, auxDir, h) + + auxBlockDir := filepath.Join(auxDir, id.String()) + _, err = metadata.InjectThanos(log.NewNopLogger(), auxBlockDir, metadata.Thanos{ + Labels: extLset.Map(), + Downsample: metadata.ThanosDownsample{Resolution: 0}, + Source: metadata.TestSource, + }, nil) + testutil.Ok(tt, err) + + testutil.Ok(tt, block.Upload(context.Background(), logger, bkt, auxBlockDir, metadata.NoneFunc)) + testutil.Ok(tt, block.Upload(context.Background(), logger, bkt, auxBlockDir, metadata.NoneFunc)) + + chunkPool, err := NewDefaultChunkBytesPool(2e5) + testutil.Ok(tt, err) + + metaFetcher, err := block.NewMetaFetcher(logger, 20, objstore.WithNoopInstr(bkt), metaDir, nil, []block.MetadataFilter{ + block.NewTimePartitionMetaFilter(allowAllFilterConf.MinTime, allowAllFilterConf.MaxTime), + }) + testutil.Ok(tt, err) + + bucketStore, err := NewBucketStore( + objstore.WithNoopInstr(bkt), + metaFetcher, + "", + NewChunksLimiterFactory(10e6), + NewSeriesLimiterFactory(10e6), + NewBytesLimiterFactory(10e6), + NewGapBasedPartitioner(PartitionerMaxGapSize), + 20, + true, + DefaultPostingOffsetInMemorySampling, + false, + false, + 1*time.Minute, + WithChunkPool(chunkPool), + WithFilterConfig(allowAllFilterConf), + ) + testutil.Ok(tt, err) + tt.Cleanup(func() { testutil.Ok(tt, bucketStore.Close()) }) + + testutil.Ok(tt, bucketStore.SyncBlocks(context.Background())) + + return bucketStore + }) +} + +func TestPrometheusStore_Acceptance(t *testing.T) { + t.Cleanup(func() { custom.TolerantVerifyLeak(t) }) + + testStoreAPIsAcceptance(t, func(tt *testing.T, extLset labels.Labels, appendFn func(app storage.Appender)) storepb.StoreServer { + p, err := e2eutil.NewPrometheus() + testutil.Ok(tt, err) + tt.Cleanup(func() { testutil.Ok(tt, p.Stop()) }) + + appendFn(p.Appender()) + + testutil.Ok(tt, p.Start()) + u, err := url.Parse(fmt.Sprintf("http://%s", p.Addr())) + testutil.Ok(tt, err) + + version, err := promclient.NewDefaultClient().BuildVersion(context.Background(), u) + testutil.Ok(tt, err) + + promStore, err := NewPrometheusStore(nil, nil, promclient.NewDefaultClient(), u, component.Sidecar, + func() labels.Labels { return extLset }, + func() (int64, int64) { return timestamp.FromTime(minTime), timestamp.FromTime(maxTime) }, + func() stringset.Set { return stringset.AllStrings() }, + func() string { return version }) + testutil.Ok(tt, err) + + return promStore + }) +} + +func TestTSDBStore_Acceptance(t *testing.T) { + t.Cleanup(func() { custom.TolerantVerifyLeak(t) }) + testStoreAPIsAcceptance(t, func(tt *testing.T, extLset labels.Labels, appendFn func(app storage.Appender)) storepb.StoreServer { + db, err := e2eutil.NewTSDB() + testutil.Ok(tt, err) + tt.Cleanup(func() { testutil.Ok(tt, db.Close()) }) + + tsdbStore := NewTSDBStore(nil, db, component.Rule, extLset) + + appendFn(db.Appender(context.Background())) + return tsdbStore + }) +} diff --git a/pkg/store/bucket_test.go b/pkg/store/bucket_test.go index fbcaf39fedb..54dd9a09e48 100644 --- a/pkg/store/bucket_test.go +++ b/pkg/store/bucket_test.go @@ -3341,80 +3341,3 @@ func TestBucketIndexReader_decodeCachedPostingsErrors(t *testing.T) { testutil.NotOk(t, err) }) } - -func TestBucketStore_Acceptance(t *testing.T) { - t.Cleanup(func() { custom.TolerantVerifyLeak(t) }) - - testStoreAPIsAcceptance(t, func(tt *testing.T, extLset labels.Labels, appendFn func(app storage.Appender)) storepb.StoreServer { - tmpDir := tt.TempDir() - bktDir := filepath.Join(tmpDir, "bkt") - auxDir := filepath.Join(tmpDir, "aux") - metaDir := filepath.Join(tmpDir, "meta") - - testutil.Ok(tt, os.MkdirAll(metaDir, os.ModePerm)) - testutil.Ok(tt, os.MkdirAll(auxDir, os.ModePerm)) - - bkt, err := filesystem.NewBucket(bktDir) - testutil.Ok(tt, err) - tt.Cleanup(func() { testutil.Ok(tt, bkt.Close()) }) - - headOpts := tsdb.DefaultHeadOptions() - headOpts.ChunkDirRoot = tmpDir - headOpts.ChunkRange = 1000 - h, err := tsdb.NewHead(nil, nil, nil, nil, headOpts, nil) - testutil.Ok(tt, err) - tt.Cleanup(func() { testutil.Ok(tt, h.Close()) }) - logger := log.NewNopLogger() - - appendFn(h.Appender(context.Background())) - - if h.NumSeries() == 0 { - tt.Skip("Bucket Store cannot handle empty HEAD") - } - - id := createBlockFromHead(tt, auxDir, h) - - auxBlockDir := filepath.Join(auxDir, id.String()) - _, err = metadata.InjectThanos(log.NewNopLogger(), auxBlockDir, metadata.Thanos{ - Labels: extLset.Map(), - Downsample: metadata.ThanosDownsample{Resolution: 0}, - Source: metadata.TestSource, - }, nil) - testutil.Ok(tt, err) - - testutil.Ok(tt, block.Upload(context.Background(), logger, bkt, auxBlockDir, metadata.NoneFunc)) - testutil.Ok(tt, block.Upload(context.Background(), logger, bkt, auxBlockDir, metadata.NoneFunc)) - - chunkPool, err := NewDefaultChunkBytesPool(2e5) - testutil.Ok(tt, err) - - metaFetcher, err := block.NewMetaFetcher(logger, 20, objstore.WithNoopInstr(bkt), metaDir, nil, []block.MetadataFilter{ - block.NewTimePartitionMetaFilter(allowAllFilterConf.MinTime, allowAllFilterConf.MaxTime), - }) - testutil.Ok(tt, err) - - bucketStore, err := NewBucketStore( - objstore.WithNoopInstr(bkt), - metaFetcher, - "", - NewChunksLimiterFactory(10e6), - NewSeriesLimiterFactory(10e6), - NewBytesLimiterFactory(10e6), - NewGapBasedPartitioner(PartitionerMaxGapSize), - 20, - true, - DefaultPostingOffsetInMemorySampling, - false, - false, - 1*time.Minute, - WithChunkPool(chunkPool), - WithFilterConfig(allowAllFilterConf), - ) - testutil.Ok(tt, err) - tt.Cleanup(func() { testutil.Ok(tt, bucketStore.Close()) }) - - testutil.Ok(tt, bucketStore.SyncBlocks(context.Background())) - - return bucketStore - }) -} diff --git a/pkg/store/prometheus_test.go b/pkg/store/prometheus_test.go index 5925645f24e..82965672c72 100644 --- a/pkg/store/prometheus_test.go +++ b/pkg/store/prometheus_test.go @@ -388,33 +388,6 @@ func TestPrometheusStore_SeriesLabels_e2e(t *testing.T) { } } -func TestPrometheusStore_Acceptance(t *testing.T) { - t.Cleanup(func() { custom.TolerantVerifyLeak(t) }) - testStoreAPIsAcceptance(t, func(tt *testing.T, extLset labels.Labels, appendFn func(app storage.Appender)) storepb.StoreServer { - p, err := e2eutil.NewPrometheus() - testutil.Ok(tt, err) - tt.Cleanup(func() { testutil.Ok(tt, p.Stop()) }) - - appendFn(p.Appender()) - - testutil.Ok(tt, p.Start()) - u, err := url.Parse(fmt.Sprintf("http://%s", p.Addr())) - testutil.Ok(tt, err) - - version, err := promclient.NewDefaultClient().BuildVersion(context.Background(), u) - testutil.Ok(tt, err) - - promStore, err := NewPrometheusStore(nil, nil, promclient.NewDefaultClient(), u, component.Sidecar, - func() labels.Labels { return extLset }, - func() (int64, int64) { return timestamp.FromTime(minTime), timestamp.FromTime(maxTime) }, - func() stringset.Set { return stringset.AllStrings() }, - func() string { return version }) - testutil.Ok(tt, err) - - return promStore - }) -} - func TestPrometheusStore_Series_MatchExternalLabel(t *testing.T) { defer custom.TolerantVerifyLeak(t) diff --git a/pkg/store/tsdb_test.go b/pkg/store/tsdb_test.go index b0e346263de..6dcc033c1c7 100644 --- a/pkg/store/tsdb_test.go +++ b/pkg/store/tsdb_test.go @@ -15,7 +15,6 @@ import ( "github.com/cespare/xxhash" "github.com/go-kit/log" "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb" "github.com/efficientgo/core/testutil" @@ -229,20 +228,6 @@ func TestTSDBStore_Series(t *testing.T) { } } -func TestTSDBStore_Acceptance(t *testing.T) { - t.Cleanup(func() { custom.TolerantVerifyLeak(t) }) - testStoreAPIsAcceptance(t, func(tt *testing.T, extLset labels.Labels, appendFn func(app storage.Appender)) storepb.StoreServer { - db, err := e2eutil.NewTSDB() - testutil.Ok(tt, err) - tt.Cleanup(func() { testutil.Ok(tt, db.Close()) }) - - tsdbStore := NewTSDBStore(nil, db, component.Rule, extLset) - - appendFn(db.Appender(context.Background())) - return tsdbStore - }) -} - // Regression test for https://github.com/thanos-io/thanos/issues/1038. func TestTSDBStore_Series_SplitSamplesIntoChunksWithMaxSizeOf120(t *testing.T) { defer custom.TolerantVerifyLeak(t)