Pad compaction planner size check (#3773)

Pad the index size check in the compaction planner by 15% to avoid situations where the sum of index bytes bloats to be larger than the sum of the original index sizes. * #3724 * #3750 Signed-off-by: Ben Kochie <[email protected]>
thanos-io · Feb 5, 2021 · 42888ab · 42888ab
1 parent 82545cd
commit 42888ab
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 5 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -17,6 +17,10 @@ We use _breaking :warning:_ to mark changes that are not backward compatible (re
 - [#3700](https://github.com/thanos-io/thanos/pull/3700) ui: make old bucket viewer UI work with vanilla Prometheus blocks
 - [#2641](https://github.com/thanos-io/thanos/issues/2641) Query Frontend: Added `--query-range.request-downsampled` flag enabling additional queries for downsampled data in case of empty or incomplete response to range request.
 
+### Fixed
+
+- [#3773](https://github.com/thanos-io/thanos/pull/3773) Compact: Pad compaction planner size check
+
 ### Changed
 
 - [#3705](https://github.com/thanos-io/thanos/pull/3705) Store: Fix race condition leading to failing queries or possibly incorrect query results.

diff --git a/pkg/compact/planner.go b/pkg/compact/planner.go
@@ -234,7 +234,7 @@ type largeTotalIndexSizeFilter struct {
 var _ Planner = &largeTotalIndexSizeFilter{}
 
 // WithLargeTotalIndexSizeFilter wraps Planner with largeTotalIndexSizeFilter that checks the given plans and estimates total index size.
-// When found, it marks block for no compaction by placing no-compact.json and updating cache.
+// When found, it marks block for no compaction by placing no-compact-mark.json and updating cache.
 // NOTE: The estimation is very rough as it assumes extreme cases of indexes sharing no bytes, thus summing all source index sizes.
 // Adjust limit accordingly reducing to some % of actual limit you want to give.
 // TODO(bwplotka): This is short term fix for https://github.com/thanos-io/thanos/issues/1424, replace with vertical block sharding https://github.com/thanos-io/thanos/pull/3390.
@@ -278,7 +278,8 @@ PlanLoop:
 				biggestIndex = i
 			}
 			totalIndexBytes += indexSize
-			if totalIndexBytes >= t.totalMaxIndexSizeBytes {
+			// Leave 15% headroom for index compaction bloat.
+			if totalIndexBytes >= int64(float64(t.totalMaxIndexSizeBytes)*0.85) {
 				// Marking blocks for no compact to limit size.
 				// TODO(bwplotka): Make sure to reset cache once this is done: https://github.com/thanos-io/thanos/issues/3408
 				if err := block.MarkForNoCompact(

diff --git a/pkg/compact/planner_test.go b/pkg/compact/planner_test.go
@@ -710,11 +710,11 @@ func TestLargeTotalIndexSizeFilter_Plan(t *testing.T) {
 		{
 			name: "Blocks to fill the entire parent, but with last size exceeded (should not matter and not even marked).",
 			metas: []*metadata.Meta{
-				{Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 30}}},
+				{Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 10}}},
 					BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(1, nil), MinTime: 0, MaxTime: 20}},
-				{Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 30}}},
+				{Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 10}}},
 					BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(2, nil), MinTime: 20, MaxTime: 40}},
-				{Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 30}}},
+				{Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 10}}},
 					BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(3, nil), MinTime: 40, MaxTime: 60}},
 				{Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 90}}},
 					BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(4, nil), MinTime: 60, MaxTime: 80}},