diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go index 969e2baa1d9..ea25ccfd820 100644 --- a/cmd/thanos/query.go +++ b/cmd/thanos/query.go @@ -220,6 +220,8 @@ func registerQuery(app *extkingpin.App) { tenantHeader := cmd.Flag("query.tenant-header", "HTTP header to determine tenant.").Default(tenancy.DefaultTenantHeader).Hidden().String() defaultTenant := cmd.Flag("query.default-tenant", "Name of the default tenant.").Default(tenancy.DefaultTenant).Hidden().String() tenantCertField := cmd.Flag("query.tenant-certificate-field", "Use TLS client's certificate field to determine tenant for write requests. Must be one of "+tenancy.CertificateFieldOrganization+", "+tenancy.CertificateFieldOrganizationalUnit+" or "+tenancy.CertificateFieldCommonName+". This setting will cause the query.tenant-header flag value to be ignored.").Default("").Hidden().Enum("", tenancy.CertificateFieldOrganization, tenancy.CertificateFieldOrganizationalUnit, tenancy.CertificateFieldCommonName) + enforceTenancy := cmd.Flag("query.enable-tenancy", "Enable tenancy. Only responses where the value of the configured tenant-label-name and value of the tenant header matches are returned.").Default("false").Bool() + tenantLabel := cmd.Flag("query.tenant-label-name", "Label name to use when enforce tenancy when -querier.tenancy is enabled").Default(tenancy.DefaultTenantLabel).String() var storeRateLimits store.SeriesSelectLimits storeRateLimits.RegisterFlags(cmd) @@ -343,6 +345,8 @@ func registerQuery(app *extkingpin.App) { *tenantHeader, *defaultTenant, *tenantCertField, + *enforceTenancy, + *tenantLabel, ) }) } @@ -422,6 +426,8 @@ func runQuery( tenantHeader string, defaultTenant string, tenantCertField string, + enforceTenancy bool, + tenantLabel string, ) error { if alertQueryURL == "" { lastColon := strings.LastIndex(httpBindAddr, ":") @@ -759,6 +765,8 @@ func runQuery( tenantHeader, defaultTenant, tenantCertField, + enforceTenancy, + tenantLabel, ) api.Register(router.WithPrefix("/api/v1"), tracer, logger, ins, logMiddleware) diff --git a/docs/components/query.md b/docs/components/query.md index 237eb837942..1314bbf9be3 100644 --- a/docs/components/query.md +++ b/docs/components/query.md @@ -360,6 +360,9 @@ Flags: = max(rangeSeconds / 250, defaultStep)). This will not work from Grafana, but Grafana has __step variable which can be used. + --query.enable-tenancy Enable tenancy. Only responses where the value + of the configured tenant-label-name and value + of the tenant header matches are returned. --query.lookback-delta=QUERY.LOOKBACK-DELTA The maximum lookback duration for retrieving metrics during expression evaluations. @@ -404,6 +407,9 @@ Flags: --query.telemetry.request-series-seconds-quantiles=10... ... The quantiles for exporting metrics about the series count quantiles. + --query.tenant-label-name="tenant_id" + Label name to use when enforce tenancy when + -querier.tenancy is enabled --query.timeout=2m Maximum time to process query by query node. --request.logging-config= Alternative to 'request.logging-config-file' diff --git a/go.mod b/go.mod index fab037080d9..16074c24519 100644 --- a/go.mod +++ b/go.mod @@ -118,15 +118,18 @@ require ( require ( github.com/onsi/gomega v1.27.10 + github.com/prometheus-community/prom-label-proxy v0.7.0 go.opentelemetry.io/contrib/propagators/autoprop v0.38.0 go4.org/intern v0.0.0-20230525184215-6c62f75575cb golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 ) require ( + github.com/go-openapi/runtime v0.26.0 // indirect github.com/golang-jwt/jwt/v5 v5.0.0 // indirect github.com/google/s2a-go v0.1.4 // indirect github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible // indirect + github.com/metalmatze/signal v0.0.0-20210307161603-1c9aa721a97a // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/onsi/ginkgo v1.16.5 // indirect go.opentelemetry.io/collector/pdata v1.0.0-rcv0014 // indirect diff --git a/go.sum b/go.sum index 05a679f7666..08770f58ac0 100644 --- a/go.sum +++ b/go.sum @@ -87,6 +87,7 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 h1:WpB/QDNLpMw github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DATA-DOG/go-sqlmock v1.4.1/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.8.3 h1:i84ZOPT35YCJROyuf97VP/VEdYhQce/8NTLOWq5tqJw= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.8.3/go.mod h1:3+qm+VCJbVmQ9uscVz+8h1rRkJEy9ZNFGgpT1XB9mPg= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.32.3 h1:FhsH8qgWFkkPlPXBZ68uuT/FH/R+DLTtVPxjLEBs1v4= @@ -321,6 +322,8 @@ github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En github.com/go-openapi/loads v0.21.1/go.mod h1:/DtAMXXneXFjbQMGEtbamCZb+4x7eGwkvZCvBmwUG+g= github.com/go-openapi/loads v0.21.2 h1:r2a/xFIYeZ4Qd2TnGpWDIQNcP80dIaZgf704za8enro= github.com/go-openapi/loads v0.21.2/go.mod h1:Jq58Os6SSGz0rzh62ptiu8Z31I+OTHqmULx5e/gJbNw= +github.com/go-openapi/runtime v0.26.0 h1:HYOFtG00FM1UvqrcxbEJg/SwvDRvYLQKGhw2zaQjTcc= +github.com/go-openapi/runtime v0.26.0/go.mod h1:QgRGeZwrUcSHdeh4Ka9Glvo0ug1LC5WyE+EV88plZrQ= github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= github.com/go-openapi/spec v0.20.6/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA= github.com/go-openapi/spec v0.20.9 h1:xnlYNQAwKd2VQRRfwTEI0DcK+2cbuvI/0c7jx3gA8/8= @@ -595,6 +598,7 @@ github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -663,6 +667,8 @@ github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/metalmatze/signal v0.0.0-20210307161603-1c9aa721a97a h1:0usWxe5SGXKQovz3p+BiQ81Jy845xSMu2CWKuXsXuUM= +github.com/metalmatze/signal v0.0.0-20210307161603-1c9aa721a97a/go.mod h1:3OETvrxfELvGsU2RoGGWercfeZ4bCL3+SOwzIWtJH/Q= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.55 h1:GoQ4hpsj0nFLYe+bWiCToyrBEJXkQfOOIvFGFy0lEgo= @@ -779,12 +785,15 @@ github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndr github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= +github.com/prometheus-community/prom-label-proxy v0.7.0 h1:1iNHXF7V8z2iOCinEyxKDUHu2jppPAAd6PmBCi3naok= +github.com/prometheus-community/prom-label-proxy v0.7.0/go.mod h1:wR9C/Mwp5aBbiqM6gQ+FZdFRwL8pCzzhsje8lTAx/aA= github.com/prometheus/alertmanager v0.25.1 h1:LGBNMspOfv8h7brb+LWj2wnwBCg2ZuuKWTh6CAVw2/Y= github.com/prometheus/alertmanager v0.25.1/go.mod h1:MEZ3rFVHqKZsw7IcNS/m4AWZeXThmJhumpiWR4eHU/w= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= +github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= @@ -802,6 +811,7 @@ github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJ github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= diff --git a/pkg/api/query/v1.go b/pkg/api/query/v1.go index ff8422086b7..196e768c5b0 100644 --- a/pkg/api/query/v1.go +++ b/pkg/api/query/v1.go @@ -167,6 +167,8 @@ type QueryAPI struct { tenantHeader string defaultTenant string tenantCertField string + enforceTenancy bool + tenantLabel string } // NewQueryAPI returns an initialized QueryAPI type. @@ -200,6 +202,8 @@ func NewQueryAPI( tenantHeader string, defaultTenant string, tenantCertField string, + enforceTenancy bool, + tenantLabel string, ) *QueryAPI { if statsAggregatorFactory == nil { statsAggregatorFactory = &store.NoopSeriesStatsAggregatorFactory{} @@ -233,6 +237,8 @@ func NewQueryAPI( tenantHeader: tenantHeader, defaultTenant: defaultTenant, tenantCertField: tenantCertField, + enforceTenancy: enforceTenancy, + tenantLabel: tenantLabel, queryRangeHist: promauto.With(reg).NewHistogram(prometheus.HistogramOpts{ Name: "thanos_query_range_requested_timespan_duration_seconds", @@ -519,6 +525,15 @@ func (qapi *QueryAPI) query(r *http.Request) (interface{}, []error, *api.ApiErro } ctx = context.WithValue(ctx, tenancy.TenantKey, tenant) + queryStr := r.FormValue("query") + + if qapi.enforceTenancy { + queryStr, err = tenancy.EnforceQueryTenancy(qapi.tenantLabel, tenant, queryStr) + if err != nil { + return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: err}, func() {} + } + } + // We are starting promQL tracing span here, because we have no control over promQL code. span, ctx := tracing.StartSpan(ctx, "promql_instant_query") defer span.Finish() @@ -538,7 +553,7 @@ func (qapi *QueryAPI) query(r *http.Request) (interface{}, []error, *api.ApiErro query.NewAggregateStatsReporter(&seriesStats), ), promql.NewPrometheusQueryOpts(false, lookbackDelta), - r.FormValue("query"), + queryStr, ts, ) @@ -691,6 +706,15 @@ func (qapi *QueryAPI) queryRange(r *http.Request) (interface{}, []error, *api.Ap // Record the query range requested. qapi.queryRangeHist.Observe(end.Sub(start).Seconds()) + queryStr := r.FormValue("query") + + if qapi.enforceTenancy { + queryStr, err = tenancy.EnforceQueryTenancy(qapi.tenantLabel, tenant, queryStr) + if err != nil { + return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: err}, func() {} + } + } + // We are starting promQL tracing span here, because we have no control over promQL code. span, ctx := tracing.StartSpan(ctx, "promql_range_query") defer span.Finish() @@ -710,7 +734,7 @@ func (qapi *QueryAPI) queryRange(r *http.Request) (interface{}, []error, *api.Ap query.NewAggregateStatsReporter(&seriesStats), ), promql.NewPrometheusQueryOpts(false, lookbackDelta), - r.FormValue("query"), + queryStr, start, end, step, @@ -785,15 +809,6 @@ func (qapi *QueryAPI) labelValues(r *http.Request) (interface{}, []error, *api.A return nil, nil, apiErr, func() {} } - var matcherSets [][]*labels.Matcher - for _, s := range r.Form[MatcherParam] { - matchers, err := parser.ParseMetricSelector(s) - if err != nil { - return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: err}, func() {} - } - matcherSets = append(matcherSets, matchers) - } - tenant, err := tenancy.GetTenantFromHTTP(r, qapi.tenantHeader, qapi.defaultTenant, qapi.tenantCertField) if err != nil { apiErr = &api.ApiError{Typ: api.ErrorBadData, Err: err} @@ -801,6 +816,11 @@ func (qapi *QueryAPI) labelValues(r *http.Request) (interface{}, []error, *api.A } ctx = context.WithValue(ctx, tenancy.TenantKey, tenant) + matcherSets, apiErr := qapi.getLabelMatchers(r.Form[MatcherParam], tenant) + if apiErr != nil { + return nil, nil, apiErr, func() {} + } + q, err := qapi.queryableCreate( true, nil, @@ -868,13 +888,16 @@ func (qapi *QueryAPI) series(r *http.Request) (interface{}, []error, *api.ApiErr return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: err}, func() {} } - var matcherSets [][]*labels.Matcher - for _, s := range r.Form[MatcherParam] { - matchers, err := parser.ParseMetricSelector(s) - if err != nil { - return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: err}, func() {} - } - matcherSets = append(matcherSets, matchers) + tenant, err := tenancy.GetTenantFromHTTP(r, qapi.tenantHeader, qapi.defaultTenant, "") + if err != nil { + apiErr := &api.ApiError{Typ: api.ErrorBadData, Err: err} + return nil, nil, apiErr, func() {} + } + ctx := context.WithValue(r.Context(), tenancy.TenantKey, tenant) + + matcherSets, apiErr := qapi.getLabelMatchers(r.Form[MatcherParam], tenant) + if apiErr != nil { + return nil, nil, apiErr, func() {} } enableDedup, apiErr := qapi.parseEnableDedupParam(r) @@ -897,13 +920,6 @@ func (qapi *QueryAPI) series(r *http.Request) (interface{}, []error, *api.ApiErr return nil, nil, apiErr, func() {} } - tenant, err := tenancy.GetTenantFromHTTP(r, qapi.tenantHeader, qapi.defaultTenant, "") - if err != nil { - apiErr = &api.ApiError{Typ: api.ErrorBadData, Err: err} - return nil, nil, apiErr, func() {} - } - ctx := context.WithValue(r.Context(), tenancy.TenantKey, tenant) - q, err := qapi.queryableCreate( enableDedup, replicaLabels, @@ -955,15 +971,6 @@ func (qapi *QueryAPI) labelNames(r *http.Request) (interface{}, []error, *api.Ap return nil, nil, apiErr, func() {} } - var matcherSets [][]*labels.Matcher - for _, s := range r.Form[MatcherParam] { - matchers, err := parser.ParseMetricSelector(s) - if err != nil { - return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: err}, func() {} - } - matcherSets = append(matcherSets, matchers) - } - tenant, err := tenancy.GetTenantFromHTTP(r, qapi.tenantHeader, qapi.defaultTenant, "") if err != nil { apiErr = &api.ApiError{Typ: api.ErrorBadData, Err: err} @@ -971,6 +978,11 @@ func (qapi *QueryAPI) labelNames(r *http.Request) (interface{}, []error, *api.Ap } ctx := context.WithValue(r.Context(), tenancy.TenantKey, tenant) + matcherSets, apiErr := qapi.getLabelMatchers(r.Form[MatcherParam], tenant) + if apiErr != nil { + return nil, nil, apiErr, func() {} + } + q, err := qapi.queryableCreate( true, nil, @@ -1037,6 +1049,49 @@ func (qapi *QueryAPI) stores(_ *http.Request) (interface{}, []error, *api.ApiErr return statuses, nil, nil, func() {} } +func (qapi *QueryAPI) getLabelMatchers(matchers []string, tenant string) ([][]*labels.Matcher, *api.ApiError) { + tenantLabelMatcher := &labels.Matcher{ + Name: qapi.tenantLabel, + Type: labels.MatchEqual, + Value: tenant, + } + + var matcherSets [][]*labels.Matcher + + // If tenancy is enforced, but there are no matchers at all, add the tenant matcher + if len(matchers) == 0 && qapi.enforceTenancy { + var matcher []*labels.Matcher + matcher = append(matcher, tenantLabelMatcher) + matcherSets = append(matcherSets, matcher) + return matcherSets, nil + } + + for _, s := range matchers { + matchers, err := parser.ParseMetricSelector(s) + if err != nil { + return nil, &api.ApiError{Typ: api.ErrorBadData, Err: err} + } + if qapi.enforceTenancy { + // first check if there's a tenant matcher already, in which case we overwrite it + // if there are multiple tenant matchers, we overwrite all of them + found := false + for idx, matchValue := range matchers { + if matchValue.Name == qapi.tenantLabel { + matchers[idx] = tenantLabelMatcher + found = true + } + } + // if there are no pre-existing tenant matchers, add it. + if !found { + matchers = append(matchers, tenantLabelMatcher) + } + } + matcherSets = append(matcherSets, matchers) + } + + return matcherSets, nil +} + // NewTargetsHandler created handler compatible with HTTP /api/v1/targets https://prometheus.io/docs/prometheus/latest/querying/api/#targets // which uses gRPC Unary Targets API. func NewTargetsHandler(client targets.UnaryClient, enablePartialResponse bool) func(*http.Request) (interface{}, []error, *api.ApiError, func()) { diff --git a/pkg/tenancy/tenancy.go b/pkg/tenancy/tenancy.go index 13775cf6e1f..7ac6e5cf06c 100644 --- a/pkg/tenancy/tenancy.go +++ b/pkg/tenancy/tenancy.go @@ -8,9 +8,11 @@ import ( "net/http" "path" - "google.golang.org/grpc/metadata" - "github.com/pkg/errors" + "github.com/prometheus-community/prom-label-proxy/injectproxy" + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/promql/parser" + "google.golang.org/grpc/metadata" ) type contextKey int @@ -109,3 +111,30 @@ func GetTenantFromGRPCMetadata(ctx context.Context) (string, bool) { } return md.Get(DefaultTenantHeader)[0], true } + +func EnforceQueryTenancy(tenantLabel string, tenant string, query string) (string, error) { + labelMatcher := &labels.Matcher{ + Name: tenantLabel, + Type: labels.MatchEqual, + Value: tenant, + } + + e := injectproxy.NewEnforcer(false, labelMatcher) + + expr, err := parser.ParseExpr(query) + if err != nil { + return "", errors.Wrap(err, "error parsing query string, when enforcing tenenacy") + } + + if err := e.EnforceNode(expr); err != nil { + var illegalLabelMatcherError *injectproxy.IllegalLabelMatcherError + if errors.As(err, *illegalLabelMatcherError) { + return "", illegalLabelMatcherError + } + return "", errors.Wrap(err, "error enforcing label") + } + + queryStr := expr.String() + + return queryStr, nil +} diff --git a/test/e2e/e2ethanos/services.go b/test/e2e/e2ethanos/services.go index 65d20a53ab9..3db4a33feb1 100644 --- a/test/e2e/e2ethanos/services.go +++ b/test/e2e/e2ethanos/services.go @@ -265,6 +265,8 @@ type QuerierBuilder struct { telemetrySamplesQuantiles []float64 telemetrySeriesQuantiles []float64 + enforceTenancy bool + e2e.Linkable f e2e.FutureRunnable } @@ -386,6 +388,11 @@ func (q *QuerierBuilder) WithTelemetryQuantiles(duration []float64, samples []fl return q } +func (q *QuerierBuilder) WithTenancy(enforceTenancy bool) *QuerierBuilder { + q.enforceTenancy = enforceTenancy + return q +} + func (q *QuerierBuilder) Init() *e2emon.InstrumentedRunnable { args, err := q.collectArgs() if err != nil { @@ -486,6 +493,9 @@ func (q *QuerierBuilder) collectArgs() ([]string, error) { for _, bucket := range q.telemetrySeriesQuantiles { args = append(args, "--query.telemetry.request-series-seconds-quantiles="+strconv.FormatFloat(bucket, 'f', -1, 64)) } + if q.enforceTenancy { + args = append(args, "--query.enable-tenancy") + } return args, nil } diff --git a/test/e2e/query_test.go b/test/e2e/query_test.go index 042e053f65d..259f98712ab 100644 --- a/test/e2e/query_test.go +++ b/test/e2e/query_test.go @@ -56,6 +56,7 @@ import ( "github.com/thanos-io/thanos/pkg/store/labelpb" prompb_copy "github.com/thanos-io/thanos/pkg/store/storepb/prompb" "github.com/thanos-io/thanos/pkg/targets/targetspb" + "github.com/thanos-io/thanos/pkg/tenancy" "github.com/thanos-io/thanos/pkg/testutil/e2eutil" "github.com/thanos-io/thanos/test/e2e/e2ethanos" ) @@ -2303,3 +2304,237 @@ func TestSidecarPrefersExtLabels(t *testing.T) { Timestamp: model.TimeFromUnixNano(now.Add(time.Hour).UnixNano()), }}, retv) } + +func TestQueryTenancyEnforcement(t *testing.T) { + t.Parallel() + + // Build up. + e, err := e2e.New(e2e.WithName("tenancyEnforce")) + testutil.Ok(t, err) + t.Cleanup(e2ethanos.CleanScenario(t, e)) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + t.Cleanup(cancel) + + bucket := "store-gw-test" + minio := e2edb.NewMinio(e, "thanos-minio", bucket, e2edb.WithMinioTLS()) + testutil.Ok(t, e2e.StartAndWaitReady(minio)) + + l := log.NewLogfmtLogger(os.Stdout) + bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test") + testutil.Ok(t, err) + + // Add series from different tenants + now := time.Now() + tenantLabel01 := labels.FromStrings(tenancy.DefaultTenantLabel, "tenant-01") + tenantLabel02 := labels.FromStrings(tenancy.DefaultTenantLabel, "tenant-02") + tenantLabel03 := labels.FromStrings(tenancy.DefaultTenantLabel, "default-tenant") + dir := filepath.Join(e.SharedDir(), "tmp") + testutil.Ok(t, os.MkdirAll(filepath.Join(e.SharedDir(), dir), os.ModePerm)) + + series1 := []labels.Labels{labels.FromStrings("a", "1")} + series2 := []labels.Labels{labels.FromStrings("b", "2")} + series3 := []labels.Labels{labels.FromStrings("c", "3")} + + blockID1, err := e2eutil.CreateBlockWithBlockDelay(ctx, + dir, + series1, + 10, + timestamp.FromTime(now), + timestamp.FromTime(now.Add(2*time.Hour)), + 30*time.Minute, + tenantLabel01, + 0, + metadata.NoneFunc, + ) + testutil.Ok(t, err) + + blockID2, err := e2eutil.CreateBlockWithBlockDelay(ctx, + dir, + series2, + 10, + timestamp.FromTime(now), + timestamp.FromTime(now.Add(2*time.Hour)), + 30*time.Minute, + tenantLabel02, + 0, + metadata.NoneFunc, + ) + testutil.Ok(t, err) + + blockID3, err := e2eutil.CreateBlockWithBlockDelay(ctx, + dir, + series3, + 10, + timestamp.FromTime(now), + timestamp.FromTime(now.Add(2*time.Hour)), + 30*time.Minute, + tenantLabel03, + 0, + metadata.NoneFunc, + ) + testutil.Ok(t, err) + + testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, blockID1.String()), blockID1.String())) + testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, blockID2.String()), blockID2.String())) + testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, blockID3.String()), blockID3.String())) + + storeGW := e2ethanos.NewStoreGW( + e, + "s1", + client.BucketConfig{ + Type: client.S3, + Config: e2ethanos.NewS3Config(bucket, minio.InternalEndpoint("http"), minio.InternalDir()), + }, + "", + "", + nil, + ) + + querierEnforce := e2ethanos.NewQuerierBuilder(e, "1", storeGW.InternalEndpoint("grpc")).WithTenancy(true).Init() + querierNoEnforce := e2ethanos.NewQuerierBuilder(e, "2", storeGW.InternalEndpoint("grpc")).Init() + testutil.Ok(t, e2e.StartAndWaitReady(storeGW, querierEnforce, querierNoEnforce)) + testutil.Ok(t, storeGW.WaitSumMetrics(e2emon.Equals(3), "thanos_blocks_meta_synced")) + + // default-tenant should only see part of the results + queryAndAssertSeries(t, ctx, querierEnforce.Endpoint("http"), func() string { return "{c=\"3\"}" }, + time.Now, promclient.QueryOptions{ + Deduplicate: false, + }, + []model.Metric{ + { + "c": "3", + "tenant_id": "default-tenant", + }, + }, + ) + + // With no enforcement enabled, default tenant can see everything + queryAndAssertSeries(t, ctx, querierNoEnforce.Endpoint("http"), func() string { return "{a=\"1\"}" }, + time.Now, promclient.QueryOptions{ + Deduplicate: false, + }, + []model.Metric{ + { + "a": "1", + "tenant_id": "tenant-01", + }, + }, + ) + + // Default tenant don't see "a" when tenancy is enforced + queryAndAssertSeries(t, ctx, querierEnforce.Endpoint("http"), func() string { return "{a=\"1\"}" }, + time.Now, promclient.QueryOptions{ + Deduplicate: false, + }, + nil, + ) + + // default-tenant cannot attempt to view other tenants data, by setting the tenant id + queryAndAssertSeries(t, ctx, querierEnforce.Endpoint("http"), func() string { return "{tenant_id=\"tenant-01\"}" }, + time.Now, promclient.QueryOptions{ + Deduplicate: false, + }, + nil, + ) + + rangeQuery(t, ctx, querierEnforce.Endpoint("http"), func() string { return "{a=\"1\"}" }, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), 3600, + promclient.QueryOptions{ + Deduplicate: true, + }, func(res model.Matrix) error { + if res.Len() == 0 { + return nil + } else { + return errors.New("default-tenant shouldn't be able to see results with label a") + } + }) + + rangeQuery(t, ctx, querierNoEnforce.Endpoint("http"), func() string { return "{a=\"1\"}" }, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), 3600, + promclient.QueryOptions{ + Deduplicate: true, + }, func(res model.Matrix) error { + if res[0].Metric["a"] == "1" { + return nil + } else { + return errors.New("default-tenant should be able to see results with label a when enforcement is off") + } + }) + + rangeQuery(t, ctx, querierEnforce.Endpoint("http"), func() string { return "{c=\"3\"}" }, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), 3600, + promclient.QueryOptions{ + Deduplicate: true, + }, func(res model.Matrix) error { + if res[0].Metric["c"] == "3" { + return nil + } else { + return errors.New("default-tenant should be able to see its own data when enforcement is enabled") + } + }) + + // default-tenant should only see two labels when enforcing is on (c,tenant_id) + labelNames(t, ctx, querierEnforce.Endpoint("http"), nil, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + return len(res) == 2 + }) + + // default-tenant should only see all labels when enforcing is not on (a,b,c,tenant_id) + labelNames(t, ctx, querierNoEnforce.Endpoint("http"), nil, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + return len(res) == 4 + }) + + // default tenant can just the value of the C label + labelValues(t, ctx, querierEnforce.Endpoint("http"), "c", nil, + timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + return len(res) == 1 + }, + ) + labelValues(t, ctx, querierEnforce.Endpoint("http"), "a", nil, + timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + return len(res) == 0 + }, + ) + + // Series endpoint tests + var matcherSetC []*labels.Matcher + labelMatcherC := &labels.Matcher{ + Name: "c", + Type: labels.MatchEqual, + Value: "3", + } + matcherSetC = append(matcherSetC, labelMatcherC) + + var matcherSetB []*labels.Matcher + labelMatcher := &labels.Matcher{ + Name: "b", + Type: labels.MatchEqual, + Value: "2", + } + matcherSetB = append(matcherSetB, labelMatcher) + + // default-tenant can see series with matcher C + series(t, ctx, querierEnforce.Endpoint("http"), matcherSetC, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []map[string]string) bool { + var expected = []map[string]string{ + { + "c": "3", + "tenant_id": "default-tenant", + }, + } + return reflect.DeepEqual(res, expected) + }) + + // default-tenant cannot see series with matcher B when tenancy is enabled + series(t, ctx, querierEnforce.Endpoint("http"), matcherSetB, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []map[string]string) bool { + return len(res) == 0 + }) + + // default-tenant can see series with matcher B when tenancy is not enabled + series(t, ctx, querierNoEnforce.Endpoint("http"), matcherSetB, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []map[string]string) bool { + var expected = []map[string]string{ + { + "b": "2", + "tenant_id": "tenant-02", + }, + } + return reflect.DeepEqual(res, expected) + }) + +}