Skip to content

Commit

Permalink
Merge pull request #6035 from kama910/replicate-change
Browse files Browse the repository at this point in the history 
support other type of matchers in replicate
  • Loading branch information
@fpetkovski
fpetkovski authored Jan 13, 2023
2 parents fb344a1 + 4487949 commit 02cda32
Show file tree
Hide file tree
Showing 4 changed files with 80 additions and 91 deletions.
4 changes: 4 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,10 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
- [#5990](https://github.com/thanos-io/thanos/pull/5990) Cache/Redis: add support for Redis Sentinel via new option `master_name`.
- [#6008](https://github.com/thanos-io/thanos/pull/6008) *: Add counter metric `gate_queries_total` to gate.

### Changed

- [#6035](https://github.com/thanos-io/thanos/pull/6035) Replicate: Support all types of matchers to match blocks for replication. Change matcher parameter from string slice to a single string.

### Fixed
- [#5995] (https://github.com/thanos-io/thanos/pull/5993) Sidecar: Loads the TLS certificate during startup.

Expand Down
4 changes: 2 additions & 2 deletions cmd/thanos/tools_bucket.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ type bucketWebConfig struct {
type bucketReplicateConfig struct {
resolutions []time.Duration
compactions []int
matcherStrs []string
matcherStrs string
singleRun bool
}

Expand Down Expand Up @@ -207,7 +207,7 @@ func (tbc *bucketReplicateConfig) registerBucketReplicateFlag(cmd extkingpin.Fla

cmd.Flag("compaction", "Only blocks with these compaction levels will be replicated. Repeated flag.").Default("1", "2", "3", "4").IntsVar(&tbc.compactions)

cmd.Flag("matcher", "Only blocks whose external labels exactly match this matcher will be replicated.").PlaceHolder("key=\"value\"").StringsVar(&tbc.matcherStrs)
cmd.Flag("matcher", "blocks whose external labels match this matcher will be replicated. All Prometheus matchers are supported, including =, !=, =~ and !~.").StringVar(&tbc.matcherStrs)

cmd.Flag("single-run", "Run replication only one time, then exit.").Default("false").BoolVar(&tbc.singleRun)

Expand Down
128 changes: 63 additions & 65 deletions docs/components/tools.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -507,81 +507,79 @@ Replicate data from one object storage to another. NOTE: Currently it works only
with Thanos blocks (meta.json has to have Thanos metadata).
Flags:
--compaction=1... ... Only blocks with these compaction levels will
be replicated. Repeated flag.
-h, --help Show context-sensitive help (also try
--help-long and --help-man).
--compaction=1... ... Only blocks with these compaction levels will be
replicated. Repeated flag.
-h, --help Show context-sensitive help (also try --help-long
and --help-man).
--http-address="0.0.0.0:10902"
Listen host:port for HTTP endpoints.
--http-grace-period=2m Time to wait after an interrupt received for
HTTP Server.
--http.config="" [EXPERIMENTAL] Path to the configuration file
that can enable TLS or authentication for all
HTTP endpoints.
--id=ID ... Block to be replicated to the destination
bucket. IDs will be used to match blocks and
other matchers will be ignored. When specified,
this command will be run only once after
successful replication. Repeated field
Listen host:port for HTTP endpoints.
--http-grace-period=2m Time to wait after an interrupt received for HTTP
Server.
--http.config="" [EXPERIMENTAL] Path to the configuration file
that can enable TLS or authentication for all HTTP
endpoints.
--id=ID ... Block to be replicated to the destination bucket.
IDs will be used to match blocks and other
matchers will be ignored. When specified, this
command will be run only once after successful
replication. Repeated field
--ignore-marked-for-deletion
Do not replicate blocks that have deletion
mark.
--log.format=logfmt Log format to use. Possible options: logfmt or
json.
--log.level=info Log filtering level.
--matcher=key="value" ... Only blocks whose external labels exactly match
this matcher will be replicated.
Do not replicate blocks that have deletion mark.
--log.format=logfmt Log format to use. Possible options: logfmt or
json.
--log.level=info Log filtering level.
--matcher=MATCHER blocks whose external labels match this matcher
will be replicated. All Prometheus matchers are
supported, including =, !=, =~ and !~.
--max-time=9999-12-31T23:59:59Z
End of time range limit to replicate.
Thanos Replicate will replicate only metrics,
which happened earlier than this value.
Option can be a constant time in RFC3339 format
or time duration relative to current time, such
as -1d or 2h45m. Valid duration units are ms,
s, m, h, d, w, y.
End of time range limit to replicate. Thanos
Replicate will replicate only metrics, which
happened earlier than this value. Option can be a
constant time in RFC3339 format or time duration
relative to current time, such as -1d or 2h45m.
Valid duration units are ms, s, m, h, d, w, y.
--min-time=0000-01-01T00:00:00Z
Start of time range limit to replicate.
Thanos Replicate will replicate only metrics,
which happened later than this value. Option
can be a constant time in RFC3339 format or
time duration relative to current time, such as
-1d or 2h45m. Valid duration units are ms, s,
m, h, d, w, y.
Start of time range limit to replicate. Thanos
Replicate will replicate only metrics, which
happened later than this value. Option can be a
constant time in RFC3339 format or time duration
relative to current time, such as -1d or 2h45m.
Valid duration units are ms, s, m, h, d, w, y.
--objstore-to.config=<content>
Alternative to 'objstore-to.config-file'
flag (mutually exclusive). Content of
YAML file that contains object store-to
configuration. See format details:
https://thanos.io/tip/thanos/storage.md/#configuration
The object storage which replicate data to.
Alternative to 'objstore-to.config-file'
flag (mutually exclusive). Content of
YAML file that contains object store-to
configuration. See format details:
https://thanos.io/tip/thanos/storage.md/#configuration
The object storage which replicate data to.
--objstore-to.config-file=<file-path>
Path to YAML file that contains object
store-to configuration. See format details:
https://thanos.io/tip/thanos/storage.md/#configuration
The object storage which replicate data to.
Path to YAML file that contains object
store-to configuration. See format details:
https://thanos.io/tip/thanos/storage.md/#configuration
The object storage which replicate data to.
--objstore.config=<content>
Alternative to 'objstore.config-file'
flag (mutually exclusive). Content of
YAML file that contains object store
configuration. See format details:
https://thanos.io/tip/thanos/storage.md/#configuration
Alternative to 'objstore.config-file'
flag (mutually exclusive). Content of
YAML file that contains object store
configuration. See format details:
https://thanos.io/tip/thanos/storage.md/#configuration
--objstore.config-file=<file-path>
Path to YAML file that contains object
store configuration. See format details:
https://thanos.io/tip/thanos/storage.md/#configuration
--resolution=0s... ... Only blocks with these resolutions will be
replicated. Repeated flag.
--single-run Run replication only one time, then exit.
Path to YAML file that contains object
store configuration. See format details:
https://thanos.io/tip/thanos/storage.md/#configuration
--resolution=0s... ... Only blocks with these resolutions will be
replicated. Repeated flag.
--single-run Run replication only one time, then exit.
--tracing.config=<content>
Alternative to 'tracing.config-file' flag
(mutually exclusive). Content of YAML file
with tracing configuration. See format details:
https://thanos.io/tip/thanos/tracing.md/#configuration
Alternative to 'tracing.config-file' flag
(mutually exclusive). Content of YAML file
with tracing configuration. See format details:
https://thanos.io/tip/thanos/tracing.md/#configuration
--tracing.config-file=<file-path>
Path to YAML file with tracing
configuration. See format details:
https://thanos.io/tip/thanos/tracing.md/#configuration
--version Show application version.
Path to YAML file with tracing
configuration. See format details:
https://thanos.io/tip/thanos/tracing.md/#configuration
--version Show application version.
```

Expand Down
35 changes: 11 additions & 24 deletions pkg/replicate/replicator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@ package replicate
import (
"context"
"math/rand"
"strconv"
"strings"
"time"

extflag "github.com/efficientgo/tools/extkingpin"
Expand All @@ -17,6 +15,7 @@ import (
"github.com/oklog/ulid"
"github.com/opentracing/opentracing-go"
"github.com/pkg/errors"
amlabels "github.com/prometheus/alertmanager/pkg/labels"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/prometheus/common/model"
Expand All @@ -42,29 +41,17 @@ const (
)

// ParseFlagMatchers parse flag into matchers.
func ParseFlagMatchers(s []string) ([]*labels.Matcher, error) {
matchers := make([]*labels.Matcher, 0, len(s))

for _, l := range s {
parts := strings.SplitN(l, "=", 2)
if len(parts) != 2 {
return nil, errors.Errorf("unrecognized label %q", l)
}

labelName := parts[0]
if !model.LabelName.IsValid(model.LabelName(labelName)) {
return nil, errors.Errorf("unsupported format for label %s", l)
}

labelValue, err := strconv.Unquote(parts[1])
if err != nil {
return nil, errors.Wrap(err, "unquote label value")
}
newEqualMatcher, err := labels.NewMatcher(labels.MatchEqual, labelName, labelValue)
if err != nil {
return nil, errors.Wrap(err, "new equal matcher")
func ParseFlagMatchers(s string) ([]*labels.Matcher, error) {
amMatchers, err := amlabels.ParseMatchers(s)
if err != nil {
return nil, err
}
matchers := make([]*labels.Matcher, 0, len(amMatchers))
for _, a := range amMatchers {
if !model.LabelName.IsValid(model.LabelName(a.Name)) {
return nil, errors.Errorf("unsupported format for label %s", a.Name)
}
matchers = append(matchers, newEqualMatcher)
matchers = append(matchers, labels.MustNewMatcher(labels.MatchType(a.Type), a.Name, a.Value))
}

return matchers, nil
Expand Down

0 comments on commit 02cda32

Please sign in to comment.