Store: Add tenant label to exported metrics (thanos-io#6690)

* Store: Add tenant label to exported metrics With this commit we add a tenant label to relevant metrics exported by the store gateway. Signed-off-by: Jacob Baungard Hansen <[email protected]> * Query: Don't hide tenant related cmd args As we now have some value of these args, with store metrics being enhanced with tenant information, we no longer hide these tenant flags. Signed-off-by: Jacob Baungard Hansen <[email protected]> * Query: Make default-tenant flag match receive Ensure that the commandline flag matches what we currently have on receive. Signed-off-by: Jacob Baungard Hansen <[email protected]> * Promclient: Use http.header type for headers Instead of using `map[string]string` for adding additional headers to requests in `req2xx`. Signed-off-by: Jacob Baungard Hansen <[email protected]> * Store: Add warning about tenant label to changelog Adds a more clear warning to the Changelog regarding that the added tenant label could potentially cause issues for custom dashboards. Signed-off-by: Jacob Baungard Hansen <[email protected]> --------- Signed-off-by: Jacob Baungard Hansen <[email protected]>
coleenquadros · Oct 6, 2023 · dfe056b · dfe056b
1 parent 07b98a7
commit dfe056b
Show file tree

Hide file tree

Showing 15 changed files with 368 additions and 317 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -18,6 +18,8 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#6308](https://github.com/thanos-io/thanos/pull/6308) Ruler: Support configuration flag that allows customizing template for alert message.
 - [#6760](https://github.com/thanos-io/thanos/pull/6760) Query Frontend: Added TLS support in `--query-frontend.downstream-tripper-config` and `--query-frontend.downstream-tripper-config-file`
 - [#6749](https://github.com/thanos-io/thanos/pull/6308) Store Gateway: Added `thanos_store_index_cache_fetch_duration_seconds` histogram for tracking latency of fetching data from index cache.
+- [#6690](https://github.com/thanos-io/thanos/pull/6690) Store: *breaking :warning:* Add tenant label to relevant exported metrics. Note that this change may cause some pre-existing dashboard queries to be incorrect due to the added label.
+- [#6530](https://github.com/thanos-io/thanos/pull/6530) / [#6690](https://github.com/thanos-io/thanos/pull/6690) Query: Add command line arguments for configuring tenants and forward tenant information to Store Gateway.
 
 ### Changed
 

diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go
@@ -217,9 +217,9 @@ func registerQuery(app *extkingpin.App) {
 	queryTelemetrySamplesQuantiles := cmd.Flag("query.telemetry.request-samples-quantiles", "The quantiles for exporting metrics about the samples count quantiles.").Default("100", "1000", "10000", "100000", "1000000").Float64List()
 	queryTelemetrySeriesQuantiles := cmd.Flag("query.telemetry.request-series-seconds-quantiles", "The quantiles for exporting metrics about the series count quantiles.").Default("10", "100", "1000", "10000", "100000").Float64List()
 
-	tenantHeader := cmd.Flag("query.tenant-header", "HTTP header to determine tenant.").Default(tenancy.DefaultTenantHeader).Hidden().String()
-	defaultTenant := cmd.Flag("query.default-tenant", "Name of the default tenant.").Default(tenancy.DefaultTenant).Hidden().String()
-	tenantCertField := cmd.Flag("query.tenant-certificate-field", "Use TLS client's certificate field to determine tenant for write requests. Must be one of "+tenancy.CertificateFieldOrganization+", "+tenancy.CertificateFieldOrganizationalUnit+" or "+tenancy.CertificateFieldCommonName+". This setting will cause the query.tenant-header flag value to be ignored.").Default("").Hidden().Enum("", tenancy.CertificateFieldOrganization, tenancy.CertificateFieldOrganizationalUnit, tenancy.CertificateFieldCommonName)
+	tenantHeader := cmd.Flag("query.tenant-header", "HTTP header to determine tenant.").Default(tenancy.DefaultTenantHeader).String()
+	defaultTenant := cmd.Flag("query.default-tenant-id", "Default tenant ID to use if tenant header is not present").Default(tenancy.DefaultTenant).String()
+	tenantCertField := cmd.Flag("query.tenant-certificate-field", "Use TLS client's certificate field to determine tenant for write requests. Must be one of "+tenancy.CertificateFieldOrganization+", "+tenancy.CertificateFieldOrganizationalUnit+" or "+tenancy.CertificateFieldCommonName+". This setting will cause the query.tenant-header flag value to be ignored.").Default("").Enum("", tenancy.CertificateFieldOrganization, tenancy.CertificateFieldOrganizationalUnit, tenancy.CertificateFieldCommonName)
 
 	var storeRateLimits store.SeriesSelectLimits
 	storeRateLimits.RegisterFlags(cmd)

diff --git a/docs/components/query.md b/docs/components/query.md
@@ -360,6 +360,9 @@ Flags:
                                  = max(rangeSeconds / 250, defaultStep)).
                                  This will not work from Grafana, but Grafana
                                  has __step variable which can be used.
+      --query.default-tenant-id="default-tenant"
+                                 Default tenant ID to use if tenant header is
+                                 not present
       --query.lookback-delta=QUERY.LOOKBACK-DELTA
                                  The maximum lookback duration for retrieving
                                  metrics during expression evaluations.
@@ -404,6 +407,14 @@ Flags:
       --query.telemetry.request-series-seconds-quantiles=10... ...
                                  The quantiles for exporting metrics about the
                                  series count quantiles.
+      --query.tenant-certificate-field=
+                                 Use TLS client's certificate field to determine
+                                 tenant for write requests. Must be one of
+                                 organization, organizationalUnit or commonName.
+                                 This setting will cause the query.tenant-header
+                                 flag value to be ignored.
+      --query.tenant-header="THANOS-TENANT"
+                                 HTTP header to determine tenant.
       --query.timeout=2m         Maximum time to process query by query node.
       --request.logging-config=<content>
                                  Alternative to 'request.logging-config-file'

diff --git a/pkg/promclient/promclient.go b/pkg/promclient/promclient.go
@@ -105,7 +105,7 @@ func NewWithTracingClient(logger log.Logger, httpClient *http.Client, userAgent
 
 // req2xx sends a request to the given url.URL. If method is http.MethodPost then
 // the raw query is encoded in the body and the appropriate Content-Type is set.
-func (c *Client) req2xx(ctx context.Context, u *url.URL, method string) (_ []byte, _ int, err error) {
+func (c *Client) req2xx(ctx context.Context, u *url.URL, method string, headers http.Header) (_ []byte, _ int, err error) {
 	var b io.Reader
 	if method == http.MethodPost {
 		rq := u.RawQuery
@@ -117,6 +117,10 @@ func (c *Client) req2xx(ctx context.Context, u *url.URL, method string) (_ []byt
 	if err != nil {
 		return nil, 0, errors.Wrapf(err, "create %s request", method)
 	}
+	if headers != nil {
+		req.Header = headers
+	}
+
 	if c.userAgent != "" {
 		req.Header.Set("User-Agent", c.userAgent)
 	}
@@ -166,7 +170,7 @@ func (c *Client) ExternalLabels(ctx context.Context, base *url.URL) (labels.Labe
 	span, ctx := tracing.StartSpan(ctx, "/prom_config HTTP[client]")
 	defer span.Finish()
 
-	body, _, err := c.req2xx(ctx, &u, http.MethodGet)
+	body, _, err := c.req2xx(ctx, &u, http.MethodGet, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -366,6 +370,7 @@ type QueryOptions struct {
 	MaxSourceResolution     string
 	Engine                  string
 	Explain                 bool
+	HTTPHeaders             http.Header
 }
 
 func (p *QueryOptions) AddTo(values url.Values) error {
@@ -426,7 +431,7 @@ func (c *Client) QueryInstant(ctx context.Context, base *url.URL, query string,
 		method = http.MethodGet
 	}
 
-	body, _, err := c.req2xx(ctx, &u, method)
+	body, _, err := c.req2xx(ctx, &u, method, opts.HTTPHeaders)
 	if err != nil {
 		return nil, nil, nil, errors.Wrap(err, "read query instant response")
 	}
@@ -532,7 +537,7 @@ func (c *Client) QueryRange(ctx context.Context, base *url.URL, query string, st
 	span, ctx := tracing.StartSpan(ctx, "/prom_query_range HTTP[client]")
 	defer span.Finish()
 
-	body, _, err := c.req2xx(ctx, &u, http.MethodGet)
+	body, _, err := c.req2xx(ctx, &u, http.MethodGet, opts.HTTPHeaders)
 	if err != nil {
 		return nil, nil, nil, errors.Wrap(err, "read query range response")
 	}
@@ -615,7 +620,7 @@ func (c *Client) AlertmanagerAlerts(ctx context.Context, base *url.URL) ([]*mode
 	span, ctx := tracing.StartSpan(ctx, "/alertmanager_alerts HTTP[client]")
 	defer span.Finish()
 
-	body, _, err := c.req2xx(ctx, &u, http.MethodGet)
+	body, _, err := c.req2xx(ctx, &u, http.MethodGet, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -646,7 +651,7 @@ func (c *Client) BuildVersion(ctx context.Context, base *url.URL) (string, error
 	defer span.Finish()
 
 	// We get status code 404 or 405 for prometheus versions lower than 2.14.0
-	body, code, err := c.req2xx(ctx, &u, http.MethodGet)
+	body, code, err := c.req2xx(ctx, &u, http.MethodGet, nil)
 	if err != nil {
 		if code == http.StatusNotFound {
 			return "0", nil
@@ -678,7 +683,7 @@ func (c *Client) get2xxResultWithGRPCErrors(ctx context.Context, spanName string
 	span, ctx := tracing.StartSpan(ctx, spanName)
 	defer span.Finish()
 
-	body, code, err := c.req2xx(ctx, u, http.MethodGet)
+	body, code, err := c.req2xx(ctx, u, http.MethodGet, nil)
 	if err != nil {
 		if code, exists := statusToCode[code]; exists && code != 0 {
 			return status.Error(code, err.Error())