From bccfdcc14278adffeb0befc7bab86d0d1bef0407 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Sat, 16 Feb 2019 00:35:10 +0200 Subject: [PATCH] compact: relax index label checking It is possible that Prometheus will produce time series with duplicate labels. E.g.: a{b="2",b="2"} 1 Where such metric is pushed to Prometheus either wittingly or unwittingly, the compact process will stall because the check will not pass. For example: {"caller":"main.go:181","err":"error executing compaction: compaction failed: compaction: gather index issues for block /data/compact/0@{monitor="monitor",replica="repl"}/01D34EDQMSQ29RHAC47XGKHGC7: out-of-order label set {**name**="foo",exported_job="vv",host="172_16_226_56",host="172_16_226_56",region="lt",subtask_index="5",task_attempt_id="32e4b047bb768583ff57c709be3b1046",task_attempt_num="8",task_id="688c028a219ff3372f3eecb0ee5811f9",task_name="Source:_foo",tenant="abc",tier="cooltier",tm_id="53b2ed987b08f427dec4ee1465df91fa"} for series 2594231","level":"error","msg":"running command failed","ts":"2019-02-11T13:30:33.901722306Z"} This commit fixes the mentioned issue. Amend the e2e test to test the case when a series has identical labels. --- pkg/block/index.go | 2 +- pkg/compact/compact_e2e_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/block/index.go b/pkg/block/index.go index eb0a3689c0..232bf40d37 100644 --- a/pkg/block/index.go +++ b/pkg/block/index.go @@ -347,7 +347,7 @@ func GatherIndexIssueStats(logger log.Logger, fn string, minTime int64, maxTime } l0 := lset[0] for _, l := range lset[1:] { - if l.Name <= l0.Name { + if l.Name < l0.Name { return stats, errors.Errorf("out-of-order label set %s for series %d", lset, id) } l0 = l diff --git a/pkg/compact/compact_e2e_test.go b/pkg/compact/compact_e2e_test.go index 11213dc047..dc6c7c1fd3 100644 --- a/pkg/compact/compact_e2e_test.go +++ b/pkg/compact/compact_e2e_test.go @@ -180,7 +180,7 @@ func TestGroup_Compact_e2e(t *testing.T) { extLset := labels.Labels{{Name: "e1", Value: "1"}} b1, err := testutil.CreateBlock(prepareDir, []labels.Labels{ {{Name: "a", Value: "1"}}, - {{Name: "a", Value: "2"}}, + {{Name: "a", Value: "2"}, {Name: "a", Value: "2"}}, {{Name: "a", Value: "3"}}, {{Name: "a", Value: "4"}}, }, 100, 0, 1000, extLset, 124)