From e6cfec75973cd8fd28863f54e5b6acbe5d920189 Mon Sep 17 00:00:00 2001 From: Kama Huang Date: Wed, 11 Jan 2023 00:18:33 -0800 Subject: [PATCH 1/3] support other type of matchers in replicate Signed-off-by: Kama Huang --- cmd/thanos/tools_bucket.go | 4 ++-- pkg/replicate/replicator.go | 35 +++++++++++------------------------ 2 files changed, 13 insertions(+), 26 deletions(-) diff --git a/cmd/thanos/tools_bucket.go b/cmd/thanos/tools_bucket.go index 66939853b9..a89f71a739 100644 --- a/cmd/thanos/tools_bucket.go +++ b/cmd/thanos/tools_bucket.go @@ -123,7 +123,7 @@ type bucketWebConfig struct { type bucketReplicateConfig struct { resolutions []time.Duration compactions []int - matcherStrs []string + matcherStrs string singleRun bool } @@ -207,7 +207,7 @@ func (tbc *bucketReplicateConfig) registerBucketReplicateFlag(cmd extkingpin.Fla cmd.Flag("compaction", "Only blocks with these compaction levels will be replicated. Repeated flag.").Default("1", "2", "3", "4").IntsVar(&tbc.compactions) - cmd.Flag("matcher", "Only blocks whose external labels exactly match this matcher will be replicated.").PlaceHolder("key=\"value\"").StringsVar(&tbc.matcherStrs) + cmd.Flag("matcher", "Only blocks whose external labels exactly match this matcher will be replicated.").PlaceHolder("key=\"value\"").StringVar(&tbc.matcherStrs) cmd.Flag("single-run", "Run replication only one time, then exit.").Default("false").BoolVar(&tbc.singleRun) diff --git a/pkg/replicate/replicator.go b/pkg/replicate/replicator.go index 4116bdb57c..1c42c14c8a 100644 --- a/pkg/replicate/replicator.go +++ b/pkg/replicate/replicator.go @@ -6,8 +6,6 @@ package replicate import ( "context" "math/rand" - "strconv" - "strings" "time" extflag "github.com/efficientgo/tools/extkingpin" @@ -17,6 +15,7 @@ import ( "github.com/oklog/ulid" "github.com/opentracing/opentracing-go" "github.com/pkg/errors" + amlabels "github.com/prometheus/alertmanager/pkg/labels" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/common/model" @@ -42,29 +41,17 @@ const ( ) // ParseFlagMatchers parse flag into matchers. -func ParseFlagMatchers(s []string) ([]*labels.Matcher, error) { - matchers := make([]*labels.Matcher, 0, len(s)) - - for _, l := range s { - parts := strings.SplitN(l, "=", 2) - if len(parts) != 2 { - return nil, errors.Errorf("unrecognized label %q", l) - } - - labelName := parts[0] - if !model.LabelName.IsValid(model.LabelName(labelName)) { - return nil, errors.Errorf("unsupported format for label %s", l) - } - - labelValue, err := strconv.Unquote(parts[1]) - if err != nil { - return nil, errors.Wrap(err, "unquote label value") - } - newEqualMatcher, err := labels.NewMatcher(labels.MatchEqual, labelName, labelValue) - if err != nil { - return nil, errors.Wrap(err, "new equal matcher") +func ParseFlagMatchers(s string) ([]*labels.Matcher, error) { + amMatchers, err := amlabels.ParseMatchers(s) + if err != nil { + return nil, err + } + matchers := make([]*labels.Matcher, 0, len(amMatchers)) + for _, a := range amMatchers { + if !model.LabelName.IsValid(model.LabelName(a.Name)) { + return nil, errors.Errorf("unsupported format for label %s", a.Name) } - matchers = append(matchers, newEqualMatcher) + matchers = append(matchers, labels.MustNewMatcher(labels.MatchType(a.Type), a.Name, a.Value)) } return matchers, nil From bd511ae25f6bf6c38b72570be188cd12d4a93d95 Mon Sep 17 00:00:00 2001 From: Kama Huang Date: Wed, 11 Jan 2023 23:34:39 -0800 Subject: [PATCH 2/3] fix the documentation error Signed-off-by: Kama Huang --- cmd/thanos/tools_bucket.go | 2 +- docs/components/tools.md | 128 ++++++++++++++++++------------------- 2 files changed, 64 insertions(+), 66 deletions(-) diff --git a/cmd/thanos/tools_bucket.go b/cmd/thanos/tools_bucket.go index a89f71a739..c1cf236420 100644 --- a/cmd/thanos/tools_bucket.go +++ b/cmd/thanos/tools_bucket.go @@ -207,7 +207,7 @@ func (tbc *bucketReplicateConfig) registerBucketReplicateFlag(cmd extkingpin.Fla cmd.Flag("compaction", "Only blocks with these compaction levels will be replicated. Repeated flag.").Default("1", "2", "3", "4").IntsVar(&tbc.compactions) - cmd.Flag("matcher", "Only blocks whose external labels exactly match this matcher will be replicated.").PlaceHolder("key=\"value\"").StringVar(&tbc.matcherStrs) + cmd.Flag("matcher", "blocks whose external labels match this matcher will be replicated. All Prometheus matchers are supported, including =, !=, =~ and !~.").StringVar(&tbc.matcherStrs) cmd.Flag("single-run", "Run replication only one time, then exit.").Default("false").BoolVar(&tbc.singleRun) diff --git a/docs/components/tools.md b/docs/components/tools.md index bf78655bdd..1033d0813e 100644 --- a/docs/components/tools.md +++ b/docs/components/tools.md @@ -507,81 +507,79 @@ Replicate data from one object storage to another. NOTE: Currently it works only with Thanos blocks (meta.json has to have Thanos metadata). Flags: - --compaction=1... ... Only blocks with these compaction levels will - be replicated. Repeated flag. - -h, --help Show context-sensitive help (also try - --help-long and --help-man). + --compaction=1... ... Only blocks with these compaction levels will be + replicated. Repeated flag. + -h, --help Show context-sensitive help (also try --help-long + and --help-man). --http-address="0.0.0.0:10902" - Listen host:port for HTTP endpoints. - --http-grace-period=2m Time to wait after an interrupt received for - HTTP Server. - --http.config="" [EXPERIMENTAL] Path to the configuration file - that can enable TLS or authentication for all - HTTP endpoints. - --id=ID ... Block to be replicated to the destination - bucket. IDs will be used to match blocks and - other matchers will be ignored. When specified, - this command will be run only once after - successful replication. Repeated field + Listen host:port for HTTP endpoints. + --http-grace-period=2m Time to wait after an interrupt received for HTTP + Server. + --http.config="" [EXPERIMENTAL] Path to the configuration file + that can enable TLS or authentication for all HTTP + endpoints. + --id=ID ... Block to be replicated to the destination bucket. + IDs will be used to match blocks and other + matchers will be ignored. When specified, this + command will be run only once after successful + replication. Repeated field --ignore-marked-for-deletion - Do not replicate blocks that have deletion - mark. - --log.format=logfmt Log format to use. Possible options: logfmt or - json. - --log.level=info Log filtering level. - --matcher=key="value" ... Only blocks whose external labels exactly match - this matcher will be replicated. + Do not replicate blocks that have deletion mark. + --log.format=logfmt Log format to use. Possible options: logfmt or + json. + --log.level=info Log filtering level. + --matcher=MATCHER blocks whose external labels match this matcher + will be replicated. All Prometheus matchers are + supported, including =, !=, =~ and !~. --max-time=9999-12-31T23:59:59Z - End of time range limit to replicate. - Thanos Replicate will replicate only metrics, - which happened earlier than this value. - Option can be a constant time in RFC3339 format - or time duration relative to current time, such - as -1d or 2h45m. Valid duration units are ms, - s, m, h, d, w, y. + End of time range limit to replicate. Thanos + Replicate will replicate only metrics, which + happened earlier than this value. Option can be a + constant time in RFC3339 format or time duration + relative to current time, such as -1d or 2h45m. + Valid duration units are ms, s, m, h, d, w, y. --min-time=0000-01-01T00:00:00Z - Start of time range limit to replicate. - Thanos Replicate will replicate only metrics, - which happened later than this value. Option - can be a constant time in RFC3339 format or - time duration relative to current time, such as - -1d or 2h45m. Valid duration units are ms, s, - m, h, d, w, y. + Start of time range limit to replicate. Thanos + Replicate will replicate only metrics, which + happened later than this value. Option can be a + constant time in RFC3339 format or time duration + relative to current time, such as -1d or 2h45m. + Valid duration units are ms, s, m, h, d, w, y. --objstore-to.config= - Alternative to 'objstore-to.config-file' - flag (mutually exclusive). Content of - YAML file that contains object store-to - configuration. See format details: - https://thanos.io/tip/thanos/storage.md/#configuration - The object storage which replicate data to. + Alternative to 'objstore-to.config-file' + flag (mutually exclusive). Content of + YAML file that contains object store-to + configuration. See format details: + https://thanos.io/tip/thanos/storage.md/#configuration + The object storage which replicate data to. --objstore-to.config-file= - Path to YAML file that contains object - store-to configuration. See format details: - https://thanos.io/tip/thanos/storage.md/#configuration - The object storage which replicate data to. + Path to YAML file that contains object + store-to configuration. See format details: + https://thanos.io/tip/thanos/storage.md/#configuration + The object storage which replicate data to. --objstore.config= - Alternative to 'objstore.config-file' - flag (mutually exclusive). Content of - YAML file that contains object store - configuration. See format details: - https://thanos.io/tip/thanos/storage.md/#configuration + Alternative to 'objstore.config-file' + flag (mutually exclusive). Content of + YAML file that contains object store + configuration. See format details: + https://thanos.io/tip/thanos/storage.md/#configuration --objstore.config-file= - Path to YAML file that contains object - store configuration. See format details: - https://thanos.io/tip/thanos/storage.md/#configuration - --resolution=0s... ... Only blocks with these resolutions will be - replicated. Repeated flag. - --single-run Run replication only one time, then exit. + Path to YAML file that contains object + store configuration. See format details: + https://thanos.io/tip/thanos/storage.md/#configuration + --resolution=0s... ... Only blocks with these resolutions will be + replicated. Repeated flag. + --single-run Run replication only one time, then exit. --tracing.config= - Alternative to 'tracing.config-file' flag - (mutually exclusive). Content of YAML file - with tracing configuration. See format details: - https://thanos.io/tip/thanos/tracing.md/#configuration + Alternative to 'tracing.config-file' flag + (mutually exclusive). Content of YAML file + with tracing configuration. See format details: + https://thanos.io/tip/thanos/tracing.md/#configuration --tracing.config-file= - Path to YAML file with tracing - configuration. See format details: - https://thanos.io/tip/thanos/tracing.md/#configuration - --version Show application version. + Path to YAML file with tracing + configuration. See format details: + https://thanos.io/tip/thanos/tracing.md/#configuration + --version Show application version. ``` From 448794946b34c454e5a5ca1ea8f16ffe7665ca86 Mon Sep 17 00:00:00 2001 From: Kama Huang Date: Wed, 11 Jan 2023 23:41:47 -0800 Subject: [PATCH 3/3] add change log Signed-off-by: Kama Huang --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10897a2683..5367674583 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,10 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#5990](https://github.com/thanos-io/thanos/pull/5990) Cache/Redis: add support for Redis Sentinel via new option `master_name`. - [#6008](https://github.com/thanos-io/thanos/pull/6008) *: Add counter metric `gate_queries_total` to gate. +### Changed + +- [#6035](https://github.com/thanos-io/thanos/pull/6035) Replicate: Support all types of matchers to match blocks for replication. Change matcher parameter from string slice to a single string. + ### Fixed - [#5995] (https://github.com/thanos-io/thanos/pull/5993) Sidecar: Loads the TLS certificate during startup.