From 4fd07bbe80c66095c88f2beef74add3984a82297 Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Tue, 13 Oct 2020 11:59:51 -0400 Subject: [PATCH 01/55] Fix go with timeout goroutine launching --- src/x/sync/pooled_worker_pool.go | 14 +----- src/x/sync/pooled_worker_pool_test.go | 65 +++++++++++++++------------ 2 files changed, 37 insertions(+), 42 deletions(-) diff --git a/src/x/sync/pooled_worker_pool.go b/src/x/sync/pooled_worker_pool.go index 93af3f8c6d..e1eb1220e3 100644 --- a/src/x/sync/pooled_worker_pool.go +++ b/src/x/sync/pooled_worker_pool.go @@ -35,10 +35,6 @@ const ( numGoroutinesGaugeSampleRate = 1000 ) -var ( - pooledWorkerPoolGoroutinesCapacity *int -) - type pooledWorkerPool struct { sync.Mutex numRoutinesAtomic int64 @@ -65,15 +61,7 @@ func NewPooledWorkerPool(size int, opts PooledWorkerPoolOptions) (PooledWorkerPo workChs := make([]chan Work, numShards) for i := range workChs { - if c := pooledWorkerPoolGoroutinesCapacity; c != nil { - if *c == 0 { - workChs[i] = make(chan Work) - } else { - workChs[i] = make(chan Work, *c) - } - } else { - workChs[i] = make(chan Work, int64(size)/numShards) - } + workChs[i] = make(chan Work, int64(size)/numShards) } return &pooledWorkerPool{ diff --git a/src/x/sync/pooled_worker_pool_test.go b/src/x/sync/pooled_worker_pool_test.go index 6c297e8337..e546115f9d 100644 --- a/src/x/sync/pooled_worker_pool_test.go +++ b/src/x/sync/pooled_worker_pool_test.go @@ -26,13 +26,10 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -var ( - disableParallelExecutionLock sync.Mutex -) - func TestPooledWorkerPoolGo(t *testing.T) { var count uint32 @@ -55,42 +52,52 @@ func TestPooledWorkerPoolGo(t *testing.T) { func TestPooledWorkerPoolGoWithTimeout(t *testing.T) { var ( - workers = 2 - channelCapacity = 1 + workers = 2 + opts = NewPooledWorkerPoolOptions().SetNumShards(int64(workers)) ) - // So we can control how empty the worker pool chanel is we - // set capacity to be same as num workers. - disableParallelExecutionLock.Lock() - pooledWorkerPoolGoroutinesCapacity = &channelCapacity - defer func() { - pooledWorkerPoolGoroutinesCapacity = nil - disableParallelExecutionLock.Unlock() - }() - - p, err := NewPooledWorkerPool(workers, NewPooledWorkerPoolOptions()) + p, err := NewPooledWorkerPool(workers, opts) require.NoError(t, err) p.Init() - var ( - resultsTrue = 0 - resultsFalse = 0 - wg sync.WaitGroup - ) + pooledWorkerPool, ok := p.(*pooledWorkerPool) + require.True(t, ok) + + // First fill up all the queues without blocking. + var wg sync.WaitGroup wg.Add(1) - for i := 0; i < workers*2; i++ { + + // Enqueue workers * 2 since buffered channel will allow workers / shards + // (which is 1, since 2 / 2 = 1) which means we need to enqueue two times + // the workers. + totalEnqueue := workers * 2 + for i := 0; i < totalEnqueue; i++ { + // Set now in such a way that independent shards are selected. + now := time.Now(). + Truncate(time.Duration(totalEnqueue) * time.Nanosecond). + Add(time.Duration(i) * time.Nanosecond) + pooledWorkerPool.nowFn = func() time.Time { + return now + } + result := p.GoWithTimeout(func() { wg.Wait() }, 100*time.Millisecond) - if result { - resultsTrue++ - } else { - resultsFalse++ - } + assert.True(t, result) } - wg.Done() + // Restore the now fn. + pooledWorkerPool.nowFn = time.Now + + // Now ensure all further enqueues time out. + for i := 0; i < workers; i++ { + result := p.GoWithTimeout(func() { + wg.Wait() + }, 100*time.Millisecond) + assert.False(t, result) + } - require.True(t, resultsFalse > 0) + // Release goroutines. + wg.Done() } func TestPooledWorkerPoolGrowOnDemand(t *testing.T) { From 3792ac359a1f75804b6918183281fb0631810fee Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Tue, 13 Oct 2020 13:00:59 -0400 Subject: [PATCH 02/55] Extract now from inner loop --- src/x/sync/pooled_worker_pool_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/x/sync/pooled_worker_pool_test.go b/src/x/sync/pooled_worker_pool_test.go index e546115f9d..6aa884861d 100644 --- a/src/x/sync/pooled_worker_pool_test.go +++ b/src/x/sync/pooled_worker_pool_test.go @@ -70,13 +70,14 @@ func TestPooledWorkerPoolGoWithTimeout(t *testing.T) { // (which is 1, since 2 / 2 = 1) which means we need to enqueue two times // the workers. totalEnqueue := workers * 2 + now := time.Now() for i := 0; i < totalEnqueue; i++ { // Set now in such a way that independent shards are selected. - now := time.Now(). + shardNowSelect := now. Truncate(time.Duration(totalEnqueue) * time.Nanosecond). Add(time.Duration(i) * time.Nanosecond) pooledWorkerPool.nowFn = func() time.Time { - return now + return shardNowSelect } result := p.GoWithTimeout(func() { From c996f46c53f5cd9d960db4240b98d93c8bb811e8 Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Tue, 13 Oct 2020 17:22:50 -0400 Subject: [PATCH 03/55] WIP First pass removing all deprecated config fields --- src/cmd/services/m3dbnode/config/bootstrap.go | 4 - src/cmd/services/m3dbnode/config/config.go | 4 - src/cmd/services/m3query/config/config.go | 92 +----------------- src/query/api/v1/handler/namespace/add.go | 4 - src/query/api/v1/handler/namespace/delete.go | 4 - src/query/api/v1/handler/namespace/get.go | 4 - src/query/api/v1/handler/placement/add.go | 4 - src/query/api/v1/handler/placement/common.go | 17 ---- src/query/api/v1/handler/placement/delete.go | 4 - .../api/v1/handler/placement/delete_all.go | 4 - src/query/api/v1/handler/placement/get.go | 4 - src/query/api/v1/handler/placement/init.go | 4 - src/query/models/params.go | 12 --- src/query/models/types.go | 10 -- src/query/storage/m3/config.go | 11 +-- src/x/config/config.go | 7 +- src/x/config/listenaddress/listenaddress.go | 97 ++----------------- 17 files changed, 18 insertions(+), 268 deletions(-) diff --git a/src/cmd/services/m3dbnode/config/bootstrap.go b/src/cmd/services/m3dbnode/config/bootstrap.go index f39ef73fdd..3753f93ee3 100644 --- a/src/cmd/services/m3dbnode/config/bootstrap.go +++ b/src/cmd/services/m3dbnode/config/bootstrap.go @@ -49,10 +49,6 @@ var ( // BootstrapConfiguration specifies the config for bootstrappers. type BootstrapConfiguration struct { - // Bootstrappers is the list of bootstrappers, ordered by precedence in - // descending order. - Bootstrappers []string `yaml:"bootstrappers" validate:"nonzero"` - // Filesystem bootstrapper configuration. Filesystem *BootstrapFilesystemConfiguration `yaml:"fs"` diff --git a/src/cmd/services/m3dbnode/config/config.go b/src/cmd/services/m3dbnode/config/config.go index 8933aa516c..9f7e0b312c 100644 --- a/src/cmd/services/m3dbnode/config/config.go +++ b/src/cmd/services/m3dbnode/config/config.go @@ -292,10 +292,6 @@ type CommitLogPolicy struct { // works in most cases because the default size of the QueueChannel should be large // enough for almost all workloads assuming a reasonable batch size is used. QueueChannel *CommitLogQueuePolicy `yaml:"queueChannel"` - - // Deprecated. Left in struct to keep old YAMLs parseable. - // TODO(V1): remove - DeprecatedBlockSize *time.Duration `yaml:"blockSize"` } // CalculationType is a type of configuration parameter. diff --git a/src/cmd/services/m3query/config/config.go b/src/cmd/services/m3query/config/config.go index 658d34230c..769ba6f28d 100644 --- a/src/cmd/services/m3query/config/config.go +++ b/src/cmd/services/m3query/config/config.go @@ -39,7 +39,6 @@ import ( "github.com/m3db/m3/src/query/storage/m3/storagemetadata" xconfig "github.com/m3db/m3/src/x/config" "github.com/m3db/m3/src/x/config/listenaddress" - "github.com/m3db/m3/src/x/cost" xdocs "github.com/m3db/m3/src/x/docs" "github.com/m3db/m3/src/x/instrument" xlog "github.com/m3db/m3/src/x/log" @@ -151,15 +150,6 @@ type Configuration struct { // Experimental is the configuration for the experimental API group. Experimental ExperimentalAPIConfiguration `yaml:"experimental"` - // Cache configurations. - // - // Deprecated: cache configurations are no longer supported. Remove from file - // when we can make breaking changes. - // (If/when removed it will make existing configurations with the cache - // stanza not able to startup the binary since we parse YAML in strict mode - // by default). - DeprecatedCache CacheConfiguration `yaml:"cache"` - // MultiProcess is the multi-process configuration. MultiProcess MultiProcessConfiguration `yaml:"multiProcess"` } @@ -190,17 +180,6 @@ type FilterConfiguration struct { CompleteTags Filter `yaml:"completeTags"` } -// CacheConfiguration contains the cache configurations. -type CacheConfiguration struct { - // Deprecated: remove from config. - DeprecatedQueryConversion *DeprecatedQueryConversionCacheConfiguration `yaml:"queryConversion"` -} - -// DeprecatedQueryConversionCacheConfiguration is deprecated: remove from config. -type DeprecatedQueryConversionCacheConfiguration struct { - Size *int `yaml:"size"` -} - // ResultOptions are the result options for query. type ResultOptions struct { // KeepNans keeps NaNs before returning query results. @@ -297,41 +276,6 @@ func (c PrometheusQueryConfiguration) MaxSamplesPerQueryOrDefault() int { type LimitsConfiguration struct { // PerQuery configures limits which apply to each query individually. PerQuery PerQueryLimitsConfiguration `yaml:"perQuery"` - - // Global configures limits which apply across all queries running on this - // instance. - Global GlobalLimitsConfiguration `yaml:"global"` - - // deprecated: use PerQuery.MaxComputedDatapoints instead. - DeprecatedMaxComputedDatapoints int `yaml:"maxComputedDatapoints"` -} - -// MaxComputedDatapoints is a getter providing backwards compatibility between -// LimitsConfiguration.DeprecatedMaxComputedDatapoints and -// LimitsConfiguration.PerQuery.PrivateMaxComputedDatapoints. See -// LimitsConfiguration.PerQuery.PrivateMaxComputedDatapoints for a comment on -// the semantics. -func (lc LimitsConfiguration) MaxComputedDatapoints() int { - if lc.PerQuery.PrivateMaxComputedDatapoints != 0 { - return lc.PerQuery.PrivateMaxComputedDatapoints - } - - return lc.DeprecatedMaxComputedDatapoints -} - -// GlobalLimitsConfiguration represents limits on resource usage across a query -// instance. Zero or negative values imply no limit. -type GlobalLimitsConfiguration struct { - // MaxFetchedDatapoints limits the max number of datapoints allowed to be - // used by all queries at any point in time, this is applied at the query - // service after the result has been returned by a storage node. - MaxFetchedDatapoints int `yaml:"maxFetchedDatapoints"` -} - -// AsLimitManagerOptions converts this configuration to -// cost.LimitManagerOptions for MaxFetchedDatapoints. -func (l *GlobalLimitsConfiguration) AsLimitManagerOptions() cost.LimitManagerOptions { - return toLimitManagerOptions(l.MaxFetchedDatapoints) } // PerQueryLimitsConfiguration represents limits on resource usage within a @@ -349,26 +293,6 @@ type PerQueryLimitsConfiguration struct { // RequireExhaustive results in an error if the query exceeds any limit. RequireExhaustive bool `yaml:"requireExhaustive"` - - // MaxFetchedDatapoints limits the max number of datapoints allowed to be - // used by a given query, this is applied at the query service after the - // result has been returned by a storage node. - MaxFetchedDatapoints int `yaml:"maxFetchedDatapoints"` - - // PrivateMaxComputedDatapoints limits the number of datapoints that can be - // returned by a query. It's determined purely - // from the size of the time range and the step size (end - start / step). - // - // N.B.: the hacky "Private" prefix is to indicate that callers should use - // LimitsConfiguration.MaxComputedDatapoints() instead of accessing - // this field directly. - PrivateMaxComputedDatapoints int `yaml:"maxComputedDatapoints"` -} - -// AsLimitManagerOptions converts this configuration to -// cost.LimitManagerOptions for MaxFetchedDatapoints. -func (l *PerQueryLimitsConfiguration) AsLimitManagerOptions() cost.LimitManagerOptions { - return toLimitManagerOptions(l.MaxFetchedDatapoints) } // AsFetchOptionsBuilderLimitsOptions converts this configuration to @@ -391,13 +315,6 @@ func (l *PerQueryLimitsConfiguration) AsFetchOptionsBuilderLimitsOptions() handl } } -func toLimitManagerOptions(limit int) cost.LimitManagerOptions { - return cost.NewLimitManagerOptions().SetDefaultLimit(cost.Limit{ - Threshold: cost.Cost(limit), - Enabled: limit > 0, - }) -} - // IngestConfiguration is the configuration for ingestion server. type IngestConfiguration struct { // Ingester is the configuration for storage based ingester. @@ -441,12 +358,9 @@ type CarbonConfiguration struct { // CarbonIngesterConfiguration is the configuration struct for carbon ingestion. type CarbonIngesterConfiguration struct { - // Deprecated: simply use the logger debug level, this has been deprecated - // in favor of setting the log level to debug. - DeprecatedDebug bool `yaml:"debug"` - ListenAddress string `yaml:"listenAddress"` - MaxConcurrency int `yaml:"maxConcurrency"` - Rules []CarbonIngesterRuleConfiguration `yaml:"rules"` + ListenAddress string `yaml:"listenAddress"` + MaxConcurrency int `yaml:"maxConcurrency"` + Rules []CarbonIngesterRuleConfiguration `yaml:"rules"` } // LookbackDurationOrDefault validates the LookbackDuration diff --git a/src/query/api/v1/handler/namespace/add.go b/src/query/api/v1/handler/namespace/add.go index d0925885b8..274ec356b8 100644 --- a/src/query/api/v1/handler/namespace/add.go +++ b/src/query/api/v1/handler/namespace/add.go @@ -42,10 +42,6 @@ import ( ) var ( - // DeprecatedM3DBAddURL is the old url for the namespace add handler, maintained - // for backwards compatibility. - DeprecatedM3DBAddURL = path.Join(handler.RoutePrefixV1, NamespacePathName) - // M3DBAddURL is the url for the M3DB namespace add handler. M3DBAddURL = path.Join(handler.RoutePrefixV1, M3DBServiceNamespacePathName) diff --git a/src/query/api/v1/handler/namespace/delete.go b/src/query/api/v1/handler/namespace/delete.go index 575ca03e14..0d0f786298 100644 --- a/src/query/api/v1/handler/namespace/delete.go +++ b/src/query/api/v1/handler/namespace/delete.go @@ -48,10 +48,6 @@ const ( ) var ( - // DeprecatedM3DBDeleteURL is the deprecated url for the M3DB namespace delete handler. - // Maintained for backwards compatibility. - DeprecatedM3DBDeleteURL = fmt.Sprintf("%s/namespace/{%s}", handler.RoutePrefixV1, namespaceIDVar) - // M3DBDeleteURL is the url for the M3DB namespace delete handler. M3DBDeleteURL = path.Join( handler.RoutePrefixV1, diff --git a/src/query/api/v1/handler/namespace/get.go b/src/query/api/v1/handler/namespace/get.go index 071c3c701d..ad783a9963 100644 --- a/src/query/api/v1/handler/namespace/get.go +++ b/src/query/api/v1/handler/namespace/get.go @@ -43,10 +43,6 @@ import ( ) var ( - // DeprecatedM3DBGetURL is the deprecated url for the namespace get handler (with the GET method). - // Maintained for backwards compatibility. - DeprecatedM3DBGetURL = path.Join(handler.RoutePrefixV1, NamespacePathName) - // M3DBGetURL is the url for the namespace get handler (with the GET method). M3DBGetURL = path.Join(handler.RoutePrefixV1, M3DBServiceNamespacePathName) diff --git a/src/query/api/v1/handler/placement/add.go b/src/query/api/v1/handler/placement/add.go index a7903b9205..1031861a36 100644 --- a/src/query/api/v1/handler/placement/add.go +++ b/src/query/api/v1/handler/placement/add.go @@ -42,10 +42,6 @@ const ( ) var ( - // DeprecatedM3DBAddURL is the old url for the placement add handler, maintained for - // backwards compatibility. - DeprecatedM3DBAddURL = path.Join(handler.RoutePrefixV1, PlacementPathName) - // M3DBAddURL is the url for the placement add handler (with the POST method) // for the M3DB service. M3DBAddURL = path.Join(handler.RoutePrefixV1, M3DBServicePlacementPathName) diff --git a/src/query/api/v1/handler/placement/common.go b/src/query/api/v1/handler/placement/common.go index 62740ea455..d53aaeb65b 100644 --- a/src/query/api/v1/handler/placement/common.go +++ b/src/query/api/v1/handler/placement/common.go @@ -411,23 +411,6 @@ func applyMiddleware( ).ServeHTTP } -func applyDeprecatedMiddleware( - f func(svc handleroptions.ServiceNameAndDefaults, w http.ResponseWriter, r *http.Request), - defaults []handleroptions.ServiceOptionsDefault, - instrumentOpts instrument.Options, -) func(w http.ResponseWriter, r *http.Request) { - return logging.WithResponseTimeAndPanicErrorLoggingFunc( - func(w http.ResponseWriter, r *http.Request) { - svc := handleroptions.ServiceNameAndDefaults{ - ServiceName: handleroptions.M3DBServiceName, - Defaults: defaults, - } - f(svc, w, r) - }, - instrumentOpts, - ).ServeHTTP -} - func parseServiceMiddleware( next func(svc handleroptions.ServiceNameAndDefaults, w http.ResponseWriter, r *http.Request), defaults []handleroptions.ServiceOptionsDefault, diff --git a/src/query/api/v1/handler/placement/delete.go b/src/query/api/v1/handler/placement/delete.go index 620f01f7c7..f868b60bcf 100644 --- a/src/query/api/v1/handler/placement/delete.go +++ b/src/query/api/v1/handler/placement/delete.go @@ -49,10 +49,6 @@ const ( var ( placementIDPath = fmt.Sprintf("{%s}", placementIDVar) - // DeprecatedM3DBDeleteURL is the old url for the placement delete handler, maintained - // for backwards compatibility. - DeprecatedM3DBDeleteURL = path.Join(handler.RoutePrefixV1, PlacementPathName, placementIDPath) - // M3DBDeleteURL is the url for the placement delete handler for the M3DB service. M3DBDeleteURL = path.Join(handler.RoutePrefixV1, M3DBServicePlacementPathName, placementIDPath) diff --git a/src/query/api/v1/handler/placement/delete_all.go b/src/query/api/v1/handler/placement/delete_all.go index 6b38ed3c11..1c3f9d62bc 100644 --- a/src/query/api/v1/handler/placement/delete_all.go +++ b/src/query/api/v1/handler/placement/delete_all.go @@ -41,10 +41,6 @@ const ( ) var ( - // DeprecatedM3DBDeleteAllURL is the old url for the handler to delete all placements, maintained - // for backwards compatibility. - DeprecatedM3DBDeleteAllURL = path.Join(handler.RoutePrefixV1, PlacementPathName) - // M3DBDeleteAllURL is the url for the handler to delete all placements (with the DELETE method) // for the M3DB service. M3DBDeleteAllURL = path.Join(handler.RoutePrefixV1, M3DBServicePlacementPathName) diff --git a/src/query/api/v1/handler/placement/get.go b/src/query/api/v1/handler/placement/get.go index 77121c902d..10111c74fa 100644 --- a/src/query/api/v1/handler/placement/get.go +++ b/src/query/api/v1/handler/placement/get.go @@ -44,10 +44,6 @@ const ( ) var ( - // DeprecatedM3DBGetURL is the old url for the placement get handler, maintained for - // backwards compatibility. - DeprecatedM3DBGetURL = path.Join(handler.RoutePrefixV1, PlacementPathName) - // M3DBGetURL is the url for the placement get handler (with the GET method) // for the M3DB service. M3DBGetURL = path.Join(handler.RoutePrefixV1, M3DBServicePlacementPathName) diff --git a/src/query/api/v1/handler/placement/init.go b/src/query/api/v1/handler/placement/init.go index aa0c0c6fe5..dce3b2ee42 100644 --- a/src/query/api/v1/handler/placement/init.go +++ b/src/query/api/v1/handler/placement/init.go @@ -42,10 +42,6 @@ const ( ) var ( - // DeprecatedM3DBInitURL is the old url for the placement init handler, maintained for backwards - // compatibility. (with the POST method). - DeprecatedM3DBInitURL = path.Join(handler.RoutePrefixV1, PlacementPathName, initPathName) - // M3DBInitURL is the url for the placement init handler, (with the POST method). M3DBInitURL = path.Join(handler.RoutePrefixV1, M3DBServicePlacementPathName, initPathName) diff --git a/src/query/models/params.go b/src/query/models/params.go index 2ea97bb1ff..3ddbe9adf7 100644 --- a/src/query/models/params.go +++ b/src/query/models/params.go @@ -31,18 +31,12 @@ type FormatType int const ( // FormatPromQL returns results in Prom format FormatPromQL FormatType = iota - // FormatM3QL returns results in M3QL format - FormatM3QL infoMsg = "if this is causing issues for your use case, please file an " + "issue on https://github.com/m3db/m3" ) var ( - // ErrDecodedBlockDeprecated indicates decoded blocks are deprecated. - ErrDecodedBlockDeprecated = fmt.Errorf("decoded block has been deprecated %s", - infoMsg) - // ErrMultiBlockDisabled indicates multi blocks are temporarily disabled. ErrMultiBlockDisabled = fmt.Errorf("multiblock is temporarily disabled %s", infoMsg) @@ -61,12 +55,6 @@ const ( // // NB: Currently disabled. TypeMultiBlock - // TypeDecodedBlock represents a single block which contains all fetched series - // which get decoded. - // - // NB: this is a legacy block type, will be deprecated once there is - // sufficient confidence that other block types are performing correctly. - TypeDecodedBlock ) // RequestParams represents the params from the request. diff --git a/src/query/models/types.go b/src/query/models/types.go index ece31d6a55..66ce809fd4 100644 --- a/src/query/models/types.go +++ b/src/query/models/types.go @@ -42,16 +42,6 @@ const ( // TypeDefault is an invalid scheme that indicates that the default scheme // for the tag options version option should be used. TypeDefault IDSchemeType = iota - // TypeLegacy describes a scheme where IDs are generated by appending - // tag name/value pairs with = and , separators. Note that an additional , is - // added to the end of the ID. - // - // NB: this should not be used, and exists here as a deprecated legacy - // ID generation scheme, as it may cause collisions in situations where - // incoming tags contain the following characters: << =," >>, for example: - // {t1:v1},{t2:v2} -> t1=v1,t2=v2, - // {t1:v1,t2:v2} -> t1=v1,t2=v2, - TypeLegacy // TypeQuoted describes a scheme where IDs are generated by appending // tag names with explicitly quoted and escaped tag values. Tag names are // also escaped if they contain invalid characters. This is equivalent to diff --git a/src/query/storage/m3/config.go b/src/query/storage/m3/config.go index 4631d984b2..742169e1b8 100644 --- a/src/query/storage/m3/config.go +++ b/src/query/storage/m3/config.go @@ -35,9 +35,8 @@ import ( ) var ( - errNotAggregatedClusterNamespace = goerrors.New("not an aggregated cluster namespace") - errBothNamespaceTypeNewAndDeprecatedFieldsSet = goerrors.New("cannot specify both deprecated and non-deprecated fields for namespace type") - errNoNamespaceInitializerSet = goerrors.New("no namespace initializer set") + errNotAggregatedClusterNamespace = goerrors.New("not an aggregated cluster namespace") + errNoNamespaceInitializerSet = goerrors.New("no namespace initializer set") ) // ClustersStaticConfiguration is a set of static cluster configurations. @@ -87,12 +86,6 @@ type ClusterStaticNamespaceConfiguration struct { // Downsample is the configuration for downsampling options to use with // the namespace. Downsample *DownsampleClusterStaticNamespaceConfiguration `yaml:"downsample"` - - // StorageMetricsType is the namespace type. - // - // Deprecated: Use "Type" field when specifying config instead, it is - // invalid to use both. - StorageMetricsType storagemetadata.MetricsType `yaml:"storageMetricsType"` } func (c ClusterStaticNamespaceConfiguration) metricsType() (storagemetadata.MetricsType, error) { diff --git a/src/x/config/config.go b/src/x/config/config.go index 6cbfe2ee77..b9dc360f1b 100644 --- a/src/x/config/config.go +++ b/src/x/config/config.go @@ -117,7 +117,12 @@ func deprecationCheck(cfg interface{}, df []string) []string { df = deprecationCheck(v.Interface(), df) } name := reflect.TypeOf(cfg).Field(i).Name - if strings.HasPrefix(name, deprecatedPrefix) { + if strings.HasPrefix(name, deprecatedPrefix) && !v.IsZero() { + // TODO: Add test to only warn on deprecation field + // if the value is actually set. + // Before it was appearing even if the value wasn't set + // so any struct with a deprecated field at all + // was causing deprecated fields to show up. df = append(df, name) } } diff --git a/src/x/config/listenaddress/listenaddress.go b/src/x/config/listenaddress/listenaddress.go index 1f0d6354f5..7f058ea10c 100644 --- a/src/x/config/listenaddress/listenaddress.go +++ b/src/x/config/listenaddress/listenaddress.go @@ -20,29 +20,8 @@ // Package listenaddress provides a configuration struct for resolving // a listen address from YAML. -// -// Deprecation notice: -// The environment resolution behavior of this -// class has largely been superseded -// by config environment interpolation in go.uber.org/config (see config/README.md for -// details). Instead of: -// -// listenAddress: -// type: environment -// envVarListenPort: MY_PORT_VAR -// -// one can do: -// -// listenAddress: -// value: 0.0.0.0:${MY_PORT_VAR} package listenaddress -import ( - "fmt" - "os" - "strconv" -) - const ( defaultHostname = "0.0.0.0" ) @@ -50,77 +29,15 @@ const ( // Resolver is a type of port resolver type Resolver string -const ( - // ConfigResolver resolves port using a value provided in config - ConfigResolver Resolver = "config" - // EnvironmentResolver resolves port using an environment variable - // of which the name is provided in config - EnvironmentResolver Resolver = "environment" -) - // Configuration is the configuration for resolving a listen address. type Configuration struct { - // DeprecatedListenAddressType is the port type for the port - // DEPRECATED: use config interpolation with `value` (config/README.md) - DeprecatedListenAddressType Resolver `yaml:"type"` - // Value is the config specified listen address if using config port type. - Value *string `yaml:"value"` - - // EnvVarListenPort specifies the environment variable name for the listen address port. - // DEPRECATED: use config interpolation with `value` (config/README.md) - DeprecatedEnvVarListenPort *string `yaml:"envVarListenPort"` - - // DeprecatedEnvVarListenHost specifies the environment variable name for the listen address hostname. - // DEPRECATED: use config interpolation with `value` (config/README.md) - DeprecatedEnvVarListenHost *string `yaml:"envVarListenHost"` + Value string `yaml:"value"` } -// Resolve returns the resolved listen address given the configuration. -func (c Configuration) Resolve() (string, error) { - listenAddrType := c.DeprecatedListenAddressType - - if listenAddrType == "" { - // Default to ConfigResolver - listenAddrType = ConfigResolver - } - - var listenAddress string - switch listenAddrType { - case ConfigResolver: - if c.Value == nil { - err := fmt.Errorf("missing listen address value using: resolver=%s", - string(listenAddrType)) - return "", err - } - listenAddress = *c.Value - - case EnvironmentResolver: - // environment variable for port is required - if c.DeprecatedEnvVarListenPort == nil { - err := fmt.Errorf("missing port env var name using: resolver=%s", - string(listenAddrType)) - return "", err - } - portStr := os.Getenv(*c.DeprecatedEnvVarListenPort) - port, err := strconv.Atoi(portStr) - if err != nil { - err := fmt.Errorf("invalid port env var value using: resolver=%s, name=%s", - string(listenAddrType), *c.DeprecatedEnvVarListenPort) - return "", err - } - // if environment variable for hostname is not set, use the default - if c.DeprecatedEnvVarListenHost == nil { - listenAddress = fmt.Sprintf("%s:%d", defaultHostname, port) - } else { - envHost := os.Getenv(*c.DeprecatedEnvVarListenHost) - listenAddress = fmt.Sprintf("%s:%d", envHost, port) - } - - default: - return "", fmt.Errorf("unknown listen address type: resolver=%s", - string(listenAddrType)) - } - - return listenAddress, nil -} +// TODO: in fact, maybe just get rid of this entire package and then +// add local structs with the listen address string directly inline. +// e.g. +// type FooConfiguration struct { +// ListenAddress string `yaml:"listenAddress"` +// } From 7d5f0f8978662bec456ff5de437a242f5dfaf5e6 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Tue, 20 Oct 2020 12:11:33 -0400 Subject: [PATCH 04/55] deprecated warning test --- src/x/config/config.go | 7 ++----- src/x/config/config_test.go | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/src/x/config/config.go b/src/x/config/config.go index b9dc360f1b..5f938c8384 100644 --- a/src/x/config/config.go +++ b/src/x/config/config.go @@ -118,11 +118,8 @@ func deprecationCheck(cfg interface{}, df []string) []string { } name := reflect.TypeOf(cfg).Field(i).Name if strings.HasPrefix(name, deprecatedPrefix) && !v.IsZero() { - // TODO: Add test to only warn on deprecation field - // if the value is actually set. - // Before it was appearing even if the value wasn't set - // so any struct with a deprecated field at all - // was causing deprecated fields to show up. + // Exclude unset deprecated config values from + // raising warnings via checking IsZero. df = append(df, name) } } diff --git a/src/x/config/config_test.go b/src/x/config/config_test.go index 195ea505de..582aace5f8 100644 --- a/src/x/config/config_test.go +++ b/src/x/config/config_test.go @@ -492,6 +492,43 @@ baz: null fmt.Sprintf("expect %#v should be equal actual %#v", expect, actual)) }) + t.Run("DeprecatedNilValue", func(t *testing.T) { + // OK + var cfg configuration + fname := writeFile(t, goodConfig) + defer func() { + require.NoError(t, os.Remove(fname)) + }() + + err := LoadFile(&cfg, fname, Options{}) + require.NoError(t, err) + + df := []string{} + ss := deprecationCheck(cfg, df) + require.Equal(t, 0, len(ss)) + + // Deprecated nil/unset value should be ok and not printed + validConfig := ` +listen_address: localhost:4385 +buffer_space: 1024 +servers: + - server1:8090 + - server2:8010 +` + var cfg2 configurationDeprecated + fname2 := writeFile(t, validConfig) + defer func() { + require.NoError(t, os.Remove(fname2)) + }() + + err = LoadFile(&cfg2, fname2, Options{}) + require.NoError(t, err) + + actual := deprecationCheck(cfg2, df) + require.Equal(t, 0, len(actual), + fmt.Sprintf("expect %#v should be equal actual %#v", 0, actual)) + }) + t.Run("NestedConfig", func(t *testing.T) { // Single Deprecation var cfg nestedConfigurationDeprecated From a7bfc035f876260e4ebc31c24e028b984f9407ec Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Tue, 20 Oct 2020 12:13:19 -0400 Subject: [PATCH 05/55] remove listenaddress struct --- src/cmd/services/m3collector/config/config.go | 3 +- src/cmd/services/m3query/config/config.go | 3 +- src/x/config/listenaddress/listenaddress.go | 43 ----- .../listenaddress/listenaddress_test.go | 149 ------------------ 4 files changed, 2 insertions(+), 196 deletions(-) delete mode 100644 src/x/config/listenaddress/listenaddress.go delete mode 100644 src/x/config/listenaddress/listenaddress_test.go diff --git a/src/cmd/services/m3collector/config/config.go b/src/cmd/services/m3collector/config/config.go index 722024e508..2d607524ad 100644 --- a/src/cmd/services/m3collector/config/config.go +++ b/src/cmd/services/m3collector/config/config.go @@ -26,7 +26,6 @@ import ( "github.com/m3db/m3/src/metrics/matcher" "github.com/m3db/m3/src/metrics/matcher/cache" "github.com/m3db/m3/src/x/clock" - "github.com/m3db/m3/src/x/config/listenaddress" "github.com/m3db/m3/src/x/instrument" "github.com/m3db/m3/src/x/pool" @@ -37,7 +36,7 @@ import ( type Configuration struct { Metrics instrument.MetricsConfiguration `yaml:"metrics"` Logging zap.Config `yaml:"logging"` - ListenAddress listenaddress.Configuration `yaml:"listenAddress" validate:"nonzero"` + ListenAddress string `yaml:"listenAddress" validate:"nonzero"` Etcd etcdclient.Configuration `yaml:"etcd"` Reporter ReporterConfiguration `yaml:"reporter"` } diff --git a/src/cmd/services/m3query/config/config.go b/src/cmd/services/m3query/config/config.go index aca07d6674..03a9442576 100644 --- a/src/cmd/services/m3query/config/config.go +++ b/src/cmd/services/m3query/config/config.go @@ -38,7 +38,6 @@ import ( "github.com/m3db/m3/src/query/storage/m3/consolidators" "github.com/m3db/m3/src/query/storage/m3/storagemetadata" xconfig "github.com/m3db/m3/src/x/config" - "github.com/m3db/m3/src/x/config/listenaddress" "github.com/m3db/m3/src/x/debug/config" xdocs "github.com/m3db/m3/src/x/docs" "github.com/m3db/m3/src/x/instrument" @@ -104,7 +103,7 @@ type Configuration struct { ClusterManagement *ClusterManagementConfiguration `yaml:"clusterManagement"` // ListenAddress is the server listen address. - ListenAddress *listenaddress.Configuration `yaml:"listenAddress" validate:"nonzero"` + ListenAddress string `yaml:"listenAddress" validate:"nonzero"` // Filter is the read/write/complete tags filter configuration. Filter FilterConfiguration `yaml:"filter"` diff --git a/src/x/config/listenaddress/listenaddress.go b/src/x/config/listenaddress/listenaddress.go deleted file mode 100644 index 7f058ea10c..0000000000 --- a/src/x/config/listenaddress/listenaddress.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -// Package listenaddress provides a configuration struct for resolving -// a listen address from YAML. -package listenaddress - -const ( - defaultHostname = "0.0.0.0" -) - -// Resolver is a type of port resolver -type Resolver string - -// Configuration is the configuration for resolving a listen address. -type Configuration struct { - // Value is the config specified listen address if using config port type. - Value string `yaml:"value"` -} - -// TODO: in fact, maybe just get rid of this entire package and then -// add local structs with the listen address string directly inline. -// e.g. -// type FooConfiguration struct { -// ListenAddress string `yaml:"listenAddress"` -// } diff --git a/src/x/config/listenaddress/listenaddress_test.go b/src/x/config/listenaddress/listenaddress_test.go deleted file mode 100644 index 7d6ede6b9b..0000000000 --- a/src/x/config/listenaddress/listenaddress_test.go +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -// Package listenaddress provides a configuration struct for resolving -// a listen address from YAML. -package listenaddress - -import ( - "os" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -var ( - envListenPort = "ENV_LISTEN_PORT" - envListenHost = "ENV_LISTEN_HOST" - defaultListen = "0.0.0.0:9000" -) - -func TestListenAddressResolver(t *testing.T) { - cfg := Configuration{ - DeprecatedListenAddressType: ConfigResolver, - Value: &defaultListen, - } - - value, err := cfg.Resolve() - require.NoError(t, err) - - assert.Equal(t, defaultListen, value) -} - -func TestListenAddressResolverDefaultsToConfigResolver(t *testing.T) { - cfg := Configuration{ - Value: &defaultListen, - } - - value, err := cfg.Resolve() - require.NoError(t, err) - - assert.Equal(t, defaultListen, value) -} - -func TestConfigResolverErrorWhenMissing(t *testing.T) { - cfg := Configuration{ - DeprecatedListenAddressType: ConfigResolver, - } - - _, err := cfg.Resolve() - require.Error(t, err) -} - -func TestEnvironmentVariableResolverWithDefault(t *testing.T) { - envPort := "9000" - - require.NoError(t, os.Setenv(envListenPort, envPort)) - - cfg := Configuration{ - DeprecatedListenAddressType: EnvironmentResolver, - DeprecatedEnvVarListenPort: &envListenPort, - } - - value, err := cfg.Resolve() - require.NoError(t, err) - - assert.Equal(t, defaultListen, value) -} - -func TestEnvironmentVariableResolver(t *testing.T) { - envHost := "127.0.0.1" - envPort := "9000" - - require.NoError(t, os.Setenv(envListenPort, envPort)) - require.NoError(t, os.Setenv(envListenHost, envHost)) - - cfg := Configuration{ - DeprecatedListenAddressType: EnvironmentResolver, - DeprecatedEnvVarListenPort: &envListenPort, - DeprecatedEnvVarListenHost: &envListenHost, - } - - value, err := cfg.Resolve() - require.NoError(t, err) - - assert.Equal(t, "127.0.0.1:9000", value) -} - -func TestInvalidEnvironmentVariableResolver(t *testing.T) { - varName := "BAD_LISTEN_ENV_PORT" - expected := "foo" - - require.NoError(t, os.Setenv(varName, expected)) - - cfg := Configuration{ - DeprecatedListenAddressType: EnvironmentResolver, - DeprecatedEnvVarListenPort: &varName, - } - - _, err := cfg.Resolve() - require.Error(t, err) -} - -func TestEnvironmentResolverErrorWhenNameMissing(t *testing.T) { - cfg := Configuration{ - DeprecatedListenAddressType: EnvironmentResolver, - } - - _, err := cfg.Resolve() - require.Error(t, err) -} - -func TestEnvironmentResolverErrorWhenValueMissing(t *testing.T) { - varName := "OTHER_LISTEN_ENV_PORT" - - cfg := Configuration{ - DeprecatedListenAddressType: EnvironmentResolver, - DeprecatedEnvVarListenPort: &varName, - } - - _, err := cfg.Resolve() - require.Error(t, err) -} - -func TestUnknownResolverError(t *testing.T) { - cfg := Configuration{ - DeprecatedListenAddressType: "some-unknown-resolver", - } - - _, err := cfg.Resolve() - require.Error(t, err) -} From be9f380693cbea5136071fd84f7b68a4801ae5d8 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Tue, 20 Oct 2020 13:36:32 -0400 Subject: [PATCH 06/55] remove typelegacy scheme --- src/cmd/services/m3query/config/config.go | 2 +- src/cmd/services/m3query/config/config_test.go | 9 +++++---- src/query/models/block_type.go | 2 -- src/query/models/config.go | 5 +---- src/query/models/config_test.go | 3 --- src/query/models/options.go | 2 +- src/query/models/options_test.go | 2 +- src/query/models/tags_id_schemes.go | 16 ---------------- src/query/models/tags_test.go | 18 ------------------ 9 files changed, 9 insertions(+), 50 deletions(-) diff --git a/src/cmd/services/m3query/config/config.go b/src/cmd/services/m3query/config/config.go index 03a9442576..683390763d 100644 --- a/src/cmd/services/m3query/config/config.go +++ b/src/cmd/services/m3query/config/config.go @@ -568,7 +568,7 @@ type TagOptionsConfiguration struct { // If not provided, defaults to `le`. BucketName string `yaml:"bucketName"` - // Scheme determines the default ID generation scheme. Defaults to TypeLegacy. + // Scheme determines the default ID generation scheme. Defaults to TypeQuoted. Scheme models.IDSchemeType `yaml:"idScheme"` // Filters are optional tag filters, removing all series with tags diff --git a/src/cmd/services/m3query/config/config_test.go b/src/cmd/services/m3query/config/config_test.go index 0143abf2fc..a4f9c12dfd 100644 --- a/src/cmd/services/m3query/config/config_test.go +++ b/src/cmd/services/m3query/config/config_test.go @@ -45,8 +45,10 @@ func TestTagOptionsFromEmptyConfigErrors(t *testing.T) { } func TestTagOptionsFromConfigWithIDGenerationScheme(t *testing.T) { - schemes := []models.IDSchemeType{models.TypeLegacy, - models.TypePrependMeta, models.TypeQuoted} + schemes := []models.IDSchemeType{ + models.TypePrependMeta, + models.TypeQuoted, + } for _, scheme := range schemes { cfg := TagOptionsConfiguration{ Scheme: scheme, @@ -64,7 +66,7 @@ func TestTagOptionsFromConfig(t *testing.T) { name := "foobar" cfg := TagOptionsConfiguration{ MetricName: name, - Scheme: models.TypeLegacy, + Scheme: models.TypeQuoted, Filters: []TagFilter{ {Name: "foo", Values: []string{".", "abc"}}, {Name: "bar", Values: []string{".*"}}, @@ -246,7 +248,6 @@ func TestTagOptionsConfigWithTagGenerationScheme(t *testing.T) { schemeStr string scheme models.IDSchemeType }{ - {"legacy", models.TypeLegacy}, {"prepend_meta", models.TypePrependMeta}, {"quoted", models.TypeQuoted}, } diff --git a/src/query/models/block_type.go b/src/query/models/block_type.go index 1cae3fee8e..ba17c71307 100644 --- a/src/query/models/block_type.go +++ b/src/query/models/block_type.go @@ -31,8 +31,6 @@ func (t FetchedBlockType) Validate() error { return nil case TypeMultiBlock: return ErrMultiBlockDisabled - case TypeDecodedBlock: - return ErrDecodedBlockDeprecated default: return fmt.Errorf(`invalid fetched block type "%v"`, t) } diff --git a/src/query/models/config.go b/src/query/models/config.go index 618afe0359..0d0e2cc882 100644 --- a/src/query/models/config.go +++ b/src/query/models/config.go @@ -26,7 +26,6 @@ import ( ) var validIDSchemes = []IDSchemeType{ - TypeLegacy, TypeQuoted, TypePrependMeta, TypeGraphite, @@ -38,7 +37,7 @@ func (t IDSchemeType) Validate() error { return errors.New("id scheme type not set") } - if t >= TypeLegacy && t <= TypeGraphite { + if t >= TypeQuoted && t <= TypeGraphite { return nil } @@ -50,8 +49,6 @@ func (t IDSchemeType) String() string { switch t { case TypeDefault: return "" - case TypeLegacy: - return "legacy" case TypeQuoted: return "quoted" case TypePrependMeta: diff --git a/src/query/models/config_test.go b/src/query/models/config_test.go index 949680db33..7b06f1e01b 100644 --- a/src/query/models/config_test.go +++ b/src/query/models/config_test.go @@ -32,8 +32,6 @@ import ( func TestIDSchemeValidation(t *testing.T) { err := TypeDefault.Validate() assert.EqualError(t, err, "id scheme type not set") - err = TypeLegacy.Validate() - assert.NoError(t, err) err = TypePrependMeta.Validate() assert.NoError(t, err) err = TypeQuoted.Validate() @@ -51,7 +49,6 @@ func TestMetricsTypeUnmarshalYAML(t *testing.T) { } validParseSchemes := []IDSchemeType{ - TypeLegacy, TypeQuoted, TypePrependMeta, } diff --git a/src/query/models/options.go b/src/query/models/options.go index d54bb2623b..794c91d9f5 100644 --- a/src/query/models/options.go +++ b/src/query/models/options.go @@ -53,7 +53,7 @@ func NewTagOptions() TagOptions { version: 0, metricName: defaultMetricName, bucketName: defaultBucketName, - idScheme: TypeLegacy, + idScheme: TypeQuoted, allowTagNameDuplicates: defaultAllowTagNameDuplicates, allowTagValueEmpty: defaultAllowTagValueEmpty, } diff --git a/src/query/models/options_test.go b/src/query/models/options_test.go index 0014ccd93a..14e19baed0 100644 --- a/src/query/models/options_test.go +++ b/src/query/models/options_test.go @@ -30,7 +30,7 @@ func TestDefaultTagOptions(t *testing.T) { opts := NewTagOptions() assert.NoError(t, opts.Validate()) assert.Equal(t, defaultMetricName, opts.MetricName()) - assert.Equal(t, TypeLegacy, opts.IDSchemeType()) + assert.Equal(t, TypeQuoted, opts.IDSchemeType()) } func TestValidTagOptions(t *testing.T) { diff --git a/src/query/models/tags_id_schemes.go b/src/query/models/tags_id_schemes.go index 33160247bc..fc2c52334f 100644 --- a/src/query/models/tags_id_schemes.go +++ b/src/query/models/tags_id_schemes.go @@ -36,8 +36,6 @@ func id(t Tags) []byte { } switch schemeType { - case TypeLegacy: - return legacyID(t) case TypeQuoted: return quotedID(t) case TypePrependMeta: @@ -51,20 +49,6 @@ func id(t Tags) []byte { } } -func legacyID(t Tags) []byte { - // TODO: pool these bytes. - id := make([]byte, idLen(t)) - idx := -1 - for _, tag := range t.Tags { - idx += copy(id[idx+1:], tag.Name) + 1 - id[idx] = eq - idx += copy(id[idx+1:], tag.Value) + 1 - id[idx] = sep - } - - return id -} - func idLen(t Tags) int { idLen := 2 * t.Len() // account for separators for _, tag := range t.Tags { diff --git a/src/query/models/tags_test.go b/src/query/models/tags_test.go index e5a0df6cca..859b51bd5e 100644 --- a/src/query/models/tags_test.go +++ b/src/query/models/tags_test.go @@ -32,7 +32,6 @@ import ( xerrors "github.com/m3db/m3/src/x/errors" xtest "github.com/m3db/m3/src/x/test" - "github.com/cespare/xxhash/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -49,13 +48,6 @@ func testLongTagIDOutOfOrder(t *testing.T, scheme IDSchemeType) Tags { return tags } -func TestLongTagNewIDOutOfOrderLegacy(t *testing.T) { - tags := testLongTagIDOutOfOrder(t, TypeLegacy) - actual := tags.ID() - assert.Equal(t, idLen(tags), len(actual)) - assert.Equal(t, []byte("t1=v1,t2=v2,t3=v3,t4=v4,"), actual) -} - func TestLongTagNewIDOutOfOrderQuoted(t *testing.T) { tags := testLongTagIDOutOfOrder(t, TypeQuoted) actual := tags.ID() @@ -84,14 +76,6 @@ func TestLongTagNewIDOutOfOrderGraphite(t *testing.T) { assert.Equal(t, []byte("v0.v1.v2.v3.v4.v5.v6.v7.v8.v9.v10.v11.v12"), actual) } -func TestHashedID(t *testing.T) { - tags := testLongTagIDOutOfOrder(t, TypeLegacy) - actual := tags.HashedID() - - expected := xxhash.Sum64String("t1=v1,t2=v2,t3=v3,t4=v4,") - assert.Equal(t, expected, actual) -} - func TestLongTagNewIDOutOfOrderQuotedWithEscape(t *testing.T) { tags := testLongTagIDOutOfOrder(t, TypeQuoted) tags = tags.AddTag(Tag{Name: []byte(`t5""`), Value: []byte(`v"5`)}) @@ -580,7 +564,6 @@ func TestEmptyTags(t *testing.T) { idType IDSchemeType expected string }{ - {TypeLegacy, ""}, {TypePrependMeta, ""}, {TypeGraphite, ""}, {TypeQuoted, "{}"}, @@ -609,7 +592,6 @@ var tagIDSchemes = []struct { name string scheme IDSchemeType }{ - {"___legacy", TypeLegacy}, {"_graphite", TypeGraphite}, {"__prepend", TypePrependMeta}, // only simple quotable tag values. From 05d2923032d8fc5609bd5e22a2369072986635fe Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Tue, 20 Oct 2020 15:11:33 -0400 Subject: [PATCH 07/55] metric type config --- src/query/storage/m3/config.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/query/storage/m3/config.go b/src/query/storage/m3/config.go index 742169e1b8..d83ffcfc1c 100644 --- a/src/query/storage/m3/config.go +++ b/src/query/storage/m3/config.go @@ -90,21 +90,12 @@ type ClusterStaticNamespaceConfiguration struct { func (c ClusterStaticNamespaceConfiguration) metricsType() (storagemetadata.MetricsType, error) { unset := storagemetadata.MetricsType(0) - if c.Type != unset && c.StorageMetricsType != unset { - // Don't allow both to not be default - return unset, errBothNamespaceTypeNewAndDeprecatedFieldsSet - } if c.Type != unset { // New field value set return c.Type, nil } - if c.StorageMetricsType != unset { - // Deprecated field value set - return c.StorageMetricsType, nil - } - // Both are unset return storagemetadata.DefaultMetricsType, nil } From c88d677305d991c6c403881b89ba4e3f1608b767 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Tue, 20 Oct 2020 15:15:34 -0400 Subject: [PATCH 08/55] metric type config 2 --- src/query/models/block_type_test.go | 2 -- src/query/storage/m3/storage_test.go | 6 +----- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/query/models/block_type_test.go b/src/query/models/block_type_test.go index a5083d81cf..04665d5e2b 100644 --- a/src/query/models/block_type_test.go +++ b/src/query/models/block_type_test.go @@ -28,8 +28,6 @@ import ( func TestFetchedBlockType(t *testing.T) { assert.NoError(t, TypeSingleBlock.Validate()) - assert.EqualError(t, TypeDecodedBlock.Validate(), - ErrDecodedBlockDeprecated.Error()) assert.EqualError(t, TypeMultiBlock.Validate(), ErrMultiBlockDisabled.Error()) assert.EqualError(t, FetchedBlockType(3).Validate(), diff --git a/src/query/storage/m3/storage_test.go b/src/query/storage/m3/storage_test.go index c519da0152..4d2a9938eb 100644 --- a/src/query/storage/m3/storage_test.go +++ b/src/query/storage/m3/storage_test.go @@ -858,11 +858,7 @@ func TestInvalidBlockTypes(t *testing.T) { require.NoError(t, err) query := &storage.FetchQuery{} - fetchOpts := &storage.FetchOptions{BlockType: models.TypeDecodedBlock} - _, err = s.FetchBlocks(context.TODO(), query, fetchOpts) - assert.Error(t, err) - - fetchOpts.BlockType = models.TypeMultiBlock + fetchOpts := &storage.FetchOptions{BlockType: models.TypeMultiBlock} _, err = s.FetchBlocks(context.TODO(), query, fetchOpts) assert.Error(t, err) } From 8151c0f3f0c4c710cb68cad0cf857b6f3d786291 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Tue, 20 Oct 2020 15:25:21 -0400 Subject: [PATCH 09/55] fs to filesystem 1 --- kube/bundle.yaml | 2 +- kube/m3dbnode-configmap.yaml | 2 +- kube/terraform/main.tf | 2 +- scripts/development/m3_stack/m3dbnode.yml | 2 +- .../dedicated_etcd_embedded_coordinator/m3dbnode.yml | 2 +- .../multi_cluster_write/m3dbnode-cluster-a.yml | 2 +- .../multi_cluster_write/m3dbnode-cluster-b.yml | 2 +- scripts/docker-integration-tests/repair/m3dbnode.yml | 2 +- .../repair_and_replication/m3dbnode-cluster-a.yml | 2 +- .../repair_and_replication/m3dbnode-cluster-b.yml | 2 +- .../replication/m3dbnode-cluster-a.yml | 2 +- .../replication/m3dbnode-cluster-b.yml | 2 +- site/content/docs/operational_guide/fileset_migrations.md | 2 +- src/cmd/services/m3dbnode/config/bootstrap.go | 2 +- src/cmd/services/m3dbnode/config/config.go | 2 +- src/cmd/services/m3dbnode/config/config_test.go | 8 ++++---- src/cmd/services/m3dbnode/main/main_index_test.go | 2 +- src/cmd/services/m3dbnode/main/main_test.go | 2 +- .../dtest/docker/harness/resources/config/m3dbnode.yml | 2 +- src/dbnode/config/m3dbnode-all-config.yml | 2 +- src/dbnode/config/m3dbnode-cluster-template.yml | 2 +- src/dbnode/config/m3dbnode-local-etcd-proto.yml | 2 +- src/dbnode/config/m3dbnode-local-etcd.yml | 2 +- 23 files changed, 26 insertions(+), 26 deletions(-) diff --git a/kube/bundle.yaml b/kube/bundle.yaml index c1a3d30f1e..5078207c25 100644 --- a/kube/bundle.yaml +++ b/kube/bundle.yaml @@ -184,7 +184,7 @@ data: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: /var/lib/m3db config: diff --git a/kube/m3dbnode-configmap.yaml b/kube/m3dbnode-configmap.yaml index 7706004656..c9de2a86e8 100644 --- a/kube/m3dbnode-configmap.yaml +++ b/kube/m3dbnode-configmap.yaml @@ -74,7 +74,7 @@ data: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: /var/lib/m3db config: diff --git a/kube/terraform/main.tf b/kube/terraform/main.tf index 4003e3f637..96c4a0bbee 100755 --- a/kube/terraform/main.tf +++ b/kube/terraform/main.tf @@ -133,7 +133,7 @@ resource "kubernetes_config_map" "m3dbnode_config" { namespace = "m3db" } data { - m3dbnode.yml = "coordinator:\n listenAddress:\n type: \"config\"\n value: \"0.0.0.0:7201\"\n local:\n namespaces:\n - namespace: default\n type: unaggregated\n retention: 48h\n metrics:\n scope:\n prefix: \"coordinator\"\n prometheus:\n handlerPath: /metrics\n listenAddress: 0.0.0.0:7203\n sanitization: prometheus\n samplingRate: 1.0\n extended: none\n tagOptions:\n idScheme: quoted\n\ndb:\n logging:\n level: info\n\n metrics:\n prometheus:\n handlerPath: /metrics\n sanitization: prometheus\n samplingRate: 1.0\n extended: detailed\n\n listenAddress: 0.0.0.0:9000\n clusterListenAddress: 0.0.0.0:9001\n httpNodeListenAddress: 0.0.0.0:9002\n httpClusterListenAddress: 0.0.0.0:9003\n debugListenAddress: 0.0.0.0:9004\n\n hostID:\n resolver: hostname\n\n client:\n writeConsistencyLevel: majority\n readConsistencyLevel: unstrict_majority\n\n gcPercentage: 100\n\n writeNewSeriesAsync: true\n writeNewSeriesLimitPerSecond: 1048576\n writeNewSeriesBackoffDuration: 2ms\n\n bootstrap:\n bootstrappers:\n - filesystem\n - commitlog\n - peers\n - uninitialized_topology\n fs:\n numProcessorsPerCPU: 0.125\n commitlog:\n returnUnfulfilledForCorruptCommitLogFiles: false\n\n commitlog:\n flushMaxBytes: 524288\n flushEvery: 1s\n queue:\n calculationType: fixed\n size: 2097152\n\n fs:\n filePathPrefix: /var/lib/m3db\n\n config:\n service:\n env: default_env\n zone: embedded\n service: m3db\n cacheDir: /var/lib/m3kv\n etcdClusters:\n - zone: embedded\n endpoints:\n - http://etcd-0.etcd:2379\n - http://etcd-1.etcd:2379\n - http://etcd-2.etcd:2379\n" + m3dbnode.yml = "coordinator:\n listenAddress:\n type: \"config\"\n value: \"0.0.0.0:7201\"\n local:\n namespaces:\n - namespace: default\n type: unaggregated\n retention: 48h\n metrics:\n scope:\n prefix: \"coordinator\"\n prometheus:\n handlerPath: /metrics\n listenAddress: 0.0.0.0:7203\n sanitization: prometheus\n samplingRate: 1.0\n extended: none\n tagOptions:\n idScheme: quoted\n\ndb:\n logging:\n level: info\n\n metrics:\n prometheus:\n handlerPath: /metrics\n sanitization: prometheus\n samplingRate: 1.0\n extended: detailed\n\n listenAddress: 0.0.0.0:9000\n clusterListenAddress: 0.0.0.0:9001\n httpNodeListenAddress: 0.0.0.0:9002\n httpClusterListenAddress: 0.0.0.0:9003\n debugListenAddress: 0.0.0.0:9004\n\n hostID:\n resolver: hostname\n\n client:\n writeConsistencyLevel: majority\n readConsistencyLevel: unstrict_majority\n\n gcPercentage: 100\n\n writeNewSeriesAsync: true\n writeNewSeriesLimitPerSecond: 1048576\n writeNewSeriesBackoffDuration: 2ms\n\n bootstrap:\n bootstrappers:\n - filesystem\n - commitlog\n - peers\n - uninitialized_topology\n filesystem:\n numProcessorsPerCPU: 0.125\n commitlog:\n returnUnfulfilledForCorruptCommitLogFiles: false\n\n commitlog:\n flushMaxBytes: 524288\n flushEvery: 1s\n queue:\n calculationType: fixed\n size: 2097152\n\n filesystem:\n filePathPrefix: /var/lib/m3db\n\n config:\n service:\n env: default_env\n zone: embedded\n service: m3db\n cacheDir: /var/lib/m3kv\n etcdClusters:\n - zone: embedded\n endpoints:\n - http://etcd-0.etcd:2379\n - http://etcd-1.etcd:2379\n - http://etcd-2.etcd:2379\n" } } diff --git a/scripts/development/m3_stack/m3dbnode.yml b/scripts/development/m3_stack/m3dbnode.yml index 2b6ee4ac4a..42e92f51ad 100644 --- a/scripts/development/m3_stack/m3dbnode.yml +++ b/scripts/development/m3_stack/m3dbnode.yml @@ -60,7 +60,7 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: /var/lib/m3db config: diff --git a/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml b/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml index 6c793f8330..b4ca27f29f 100644 --- a/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml +++ b/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml @@ -69,7 +69,7 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: /var/lib/m3db config: diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml index f724192e96..505a5f009a 100644 --- a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml +++ b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml @@ -81,7 +81,7 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: /var/lib/m3db config: diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml index fa32e79853..c945395a70 100644 --- a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml +++ b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml @@ -60,7 +60,7 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: /var/lib/m3db config: diff --git a/scripts/docker-integration-tests/repair/m3dbnode.yml b/scripts/docker-integration-tests/repair/m3dbnode.yml index 6bdae716a7..ef658c32a3 100644 --- a/scripts/docker-integration-tests/repair/m3dbnode.yml +++ b/scripts/docker-integration-tests/repair/m3dbnode.yml @@ -60,7 +60,7 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: /var/lib/m3db config: diff --git a/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-a.yml b/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-a.yml index 002f7dd1cf..f1958a70b3 100644 --- a/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-a.yml +++ b/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-a.yml @@ -60,7 +60,7 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: /var/lib/m3db config: diff --git a/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-b.yml b/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-b.yml index 59dcc55b6b..ec809604f0 100644 --- a/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-b.yml +++ b/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-b.yml @@ -60,7 +60,7 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: /var/lib/m3db config: diff --git a/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml b/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml index aca38c3af9..f750badb74 100644 --- a/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml +++ b/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml @@ -60,7 +60,7 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: /var/lib/m3db config: diff --git a/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml b/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml index 558e511304..7d478eb280 100644 --- a/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml +++ b/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml @@ -60,7 +60,7 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: /var/lib/m3db config: diff --git a/site/content/docs/operational_guide/fileset_migrations.md b/site/content/docs/operational_guide/fileset_migrations.md index b099725aff..95c40c7ba2 100644 --- a/site/content/docs/operational_guide/fileset_migrations.md +++ b/site/content/docs/operational_guide/fileset_migrations.md @@ -16,7 +16,7 @@ Migrations are enabled by setting the following fields in the M3 configuration ( ```yaml db: bootstrap: - fs: + filesystem: migration: targetMigrationVersion: "1.1" # Optional. Defaults to the number of available CPUs / 2. diff --git a/src/cmd/services/m3dbnode/config/bootstrap.go b/src/cmd/services/m3dbnode/config/bootstrap.go index f1a1545616..1d7957ba9f 100644 --- a/src/cmd/services/m3dbnode/config/bootstrap.go +++ b/src/cmd/services/m3dbnode/config/bootstrap.go @@ -52,7 +52,7 @@ type BootstrapConfiguration struct { Bootstrappers []string `yaml:"bootstrappers" validate:"nonzero"` // Filesystem bootstrapper configuration. - Filesystem *BootstrapFilesystemConfiguration `yaml:"fs"` + Filesystem *BootstrapFilesystemConfiguration `yaml:"filesystem"` // Commitlog bootstrapper configuration. Commitlog *BootstrapCommitlogConfiguration `yaml:"commitlog"` diff --git a/src/cmd/services/m3dbnode/config/config.go b/src/cmd/services/m3dbnode/config/config.go index fb3f98862e..94f8223104 100644 --- a/src/cmd/services/m3dbnode/config/config.go +++ b/src/cmd/services/m3dbnode/config/config.go @@ -126,7 +126,7 @@ type DBConfiguration struct { Cache CacheConfigurations `yaml:"cache"` // The filesystem configuration for the node. - Filesystem FilesystemConfiguration `yaml:"fs"` + Filesystem FilesystemConfiguration `yaml:"filesystem"` // The commit log policy for the node. CommitLog CommitLogPolicy `yaml:"commitlog"` diff --git a/src/cmd/services/m3dbnode/config/config_test.go b/src/cmd/services/m3dbnode/config/config_test.go index 37c967129e..2426ec5263 100644 --- a/src/cmd/services/m3dbnode/config/config_test.go +++ b/src/cmd/services/m3dbnode/config/config_test.go @@ -103,7 +103,7 @@ db: - filesystem - peers - noop-all - fs: + filesystem: numProcessorsPerCPU: 0.42 commitlog: returnUnfulfilledForCorruptCommitLogFiles: false @@ -115,7 +115,7 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: /var/lib/m3db writeBufferSize: 65536 dataReadBufferSize: 65536 @@ -421,7 +421,7 @@ func TestConfiguration(t *testing.T) { - filesystem - peers - noop-all - fs: + filesystem: numProcessorsPerCPU: 0.42 migration: null commitlog: @@ -436,7 +436,7 @@ func TestConfiguration(t *testing.T) { size: 100 cacheRegexp: false cacheTerms: false - fs: + filesystem: filePathPrefix: /var/lib/m3db writeBufferSize: 65536 dataReadBufferSize: 65536 diff --git a/src/cmd/services/m3dbnode/main/main_index_test.go b/src/cmd/services/m3dbnode/main/main_index_test.go index b5022112af..3e4a665f92 100644 --- a/src/cmd/services/m3dbnode/main/main_index_test.go +++ b/src/cmd/services/m3dbnode/main/main_index_test.go @@ -364,7 +364,7 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: {{.DataDir}} writeBufferSize: 65536 dataReadBufferSize: 65536 diff --git a/src/cmd/services/m3dbnode/main/main_test.go b/src/cmd/services/m3dbnode/main/main_test.go index 690f639d24..f1ab0f2801 100644 --- a/src/cmd/services/m3dbnode/main/main_test.go +++ b/src/cmd/services/m3dbnode/main/main_test.go @@ -520,7 +520,7 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: {{.DataDir}} writeBufferSize: 65536 dataReadBufferSize: 65536 diff --git a/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml b/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml index 6970916644..6686c9df13 100644 --- a/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml +++ b/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml @@ -81,7 +81,7 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: /var/lib/m3db config: diff --git a/src/dbnode/config/m3dbnode-all-config.yml b/src/dbnode/config/m3dbnode-all-config.yml index 4b62a4d565..6f4e4be33b 100644 --- a/src/dbnode/config/m3dbnode-all-config.yml +++ b/src/dbnode/config/m3dbnode-all-config.yml @@ -135,7 +135,7 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: # Directory to store M3DB data in. filePathPrefix: /var/lib/m3db # Various fixed-sized buffers used for M3DB I/O. diff --git a/src/dbnode/config/m3dbnode-cluster-template.yml b/src/dbnode/config/m3dbnode-cluster-template.yml index 23b5c9b223..c77c9ba04d 100644 --- a/src/dbnode/config/m3dbnode-cluster-template.yml +++ b/src/dbnode/config/m3dbnode-cluster-template.yml @@ -99,5 +99,5 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: /var/lib/m3db diff --git a/src/dbnode/config/m3dbnode-local-etcd-proto.yml b/src/dbnode/config/m3dbnode-local-etcd-proto.yml index 7cae28ed10..c52ab991fd 100644 --- a/src/dbnode/config/m3dbnode-local-etcd-proto.yml +++ b/src/dbnode/config/m3dbnode-local-etcd-proto.yml @@ -81,7 +81,7 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: /var/lib/m3db config: diff --git a/src/dbnode/config/m3dbnode-local-etcd.yml b/src/dbnode/config/m3dbnode-local-etcd.yml index 6970916644..6686c9df13 100644 --- a/src/dbnode/config/m3dbnode-local-etcd.yml +++ b/src/dbnode/config/m3dbnode-local-etcd.yml @@ -81,7 +81,7 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: /var/lib/m3db config: From 437291a5e3bd2f768931cbb6612b450eaa4969a2 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Tue, 20 Oct 2020 16:06:12 -0400 Subject: [PATCH 10/55] bootstrap order --- src/cmd/services/m3dbnode/config/bootstrap.go | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/cmd/services/m3dbnode/config/bootstrap.go b/src/cmd/services/m3dbnode/config/bootstrap.go index ea8e3563a9..192578c56e 100644 --- a/src/cmd/services/m3dbnode/config/bootstrap.go +++ b/src/cmd/services/m3dbnode/config/bootstrap.go @@ -43,6 +43,15 @@ import ( var ( // defaultNumProcessorsPerCPU is the default number of processors per CPU. defaultNumProcessorsPerCPU = 0.125 + + // order in which bootstrappers are run + // (run in ascending order of precedence) + orderedBootstrappers = []string{ + uninitialized.UninitializedTopologyBootstrapperName, + peers.PeersBootstrapperName, + commitlog.CommitLogBootstrapperName, + bfs.FileSystemBootstrapperName, + } ) // BootstrapConfiguration specifies the config for bootstrappers. @@ -157,7 +166,6 @@ func newDefaultBootstrapPeersConfiguration() BootstrapPeersConfiguration { // Useful for tests and perhaps verifying same options set across multiple // bootstrappers. type BootstrapConfigurationValidator interface { - ValidateBootstrappersOrder(names []string) error ValidateFilesystemBootstrapperOptions(opts bfs.Options) error ValidateCommitLogBootstrapperOptions(opts commitlog.Options) error ValidatePeersBootstrapperOptions(opts peers.Options) error @@ -173,10 +181,6 @@ func (bsc BootstrapConfiguration) New( origin topology.Host, adminClient client.AdminClient, ) (bootstrap.ProcessProvider, error) { - if err := validator.ValidateBootstrappersOrder(bsc.Bootstrappers); err != nil { - return nil, err - } - idxOpts := opts.IndexOptions() compactor, err := compaction.NewCompactor(idxOpts.DocumentArrayPool(), index.DocumentArrayPoolCapacity, @@ -198,9 +202,8 @@ func (bsc BootstrapConfiguration) New( bs bootstrap.BootstrapperProvider fsOpts = opts.CommitLogOptions().FilesystemOptions() ) - // Start from the end of the list because the bootstrappers are ordered by precedence in descending order. - for i := len(bsc.Bootstrappers) - 1; i >= 0; i-- { - switch bsc.Bootstrappers[i] { + for _, b := range orderedBootstrappers { + switch b { case bootstrapper.NoOpAllBootstrapperName: bs = bootstrapper.NewNoOpAllBootstrapperProvider() case bootstrapper.NoOpNoneBootstrapperName: @@ -279,7 +282,7 @@ func (bsc BootstrapConfiguration) New( } bs = uninitialized.NewUninitializedTopologyBootstrapperProvider(uOpts, bs) default: - return nil, fmt.Errorf("unknown bootstrapper: %s", bsc.Bootstrappers[i]) + return nil, fmt.Errorf("unknown bootstrapper: %s", b) } } From 2e3963f30b6dc742fe2a91b6afc9d8729201dfab Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Tue, 20 Oct 2020 16:16:54 -0400 Subject: [PATCH 11/55] more build fixes 1 --- .../api/v1/handler/prometheus/native/read.go | 9 ------- .../handler/prometheus/native/read_common.go | 25 ------------------- 2 files changed, 34 deletions(-) diff --git a/src/query/api/v1/handler/prometheus/native/read.go b/src/query/api/v1/handler/prometheus/native/read.go index c5f8a958ba..735acc3488 100644 --- a/src/query/api/v1/handler/prometheus/native/read.go +++ b/src/query/api/v1/handler/prometheus/native/read.go @@ -27,7 +27,6 @@ import ( "github.com/m3db/m3/src/query/api/v1/handler" "github.com/m3db/m3/src/query/api/v1/handler/prometheus/handleroptions" "github.com/m3db/m3/src/query/api/v1/options" - "github.com/m3db/m3/src/query/models" "github.com/m3db/m3/src/query/util/logging" xhttp "github.com/m3db/m3/src/x/net/http" xopentracing "github.com/m3db/m3/src/x/opentracing" @@ -93,9 +92,6 @@ func newHandler(opts options.HandlerOptions, instant bool) http.Handler { opts: opts, instant: instant, } - - maxDatapoints := opts.Config().Limits.MaxComputedDatapoints() - h.promReadMetrics.maxDatapoints.Update(float64(maxDatapoints)) return h } @@ -140,11 +136,6 @@ func (h *promReadHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - if parsedOptions.Params.FormatType == models.FormatM3QL { - renderM3QLResultsJSON(w, result.Series, parsedOptions.Params) - return - } - err = RenderResultsJSON(w, result, RenderResultsOptions{ Start: parsedOptions.Params.Start, End: parsedOptions.Params.End, diff --git a/src/query/api/v1/handler/prometheus/native/read_common.go b/src/query/api/v1/handler/prometheus/native/read_common.go index 434329d464..6445be8db3 100644 --- a/src/query/api/v1/handler/prometheus/native/read_common.go +++ b/src/query/api/v1/handler/prometheus/native/read_common.go @@ -22,7 +22,6 @@ package native import ( "context" - "fmt" "math" "net/http" @@ -47,7 +46,6 @@ type promReadMetrics struct { fetchErrorsServer tally.Counter fetchErrorsClient tally.Counter fetchTimerSuccess tally.Timer - maxDatapoints tally.Gauge } func newPromReadMetrics(scope tally.Scope) promReadMetrics { @@ -116,11 +114,6 @@ func ParseRequest( return ParsedOptions{}, rErr } - maxPoints := opts.Config().Limits.MaxComputedDatapoints() - if err := validateRequest(params, maxPoints); err != nil { - return ParsedOptions{}, xhttp.NewParseError(err, http.StatusBadRequest) - } - return ParsedOptions{ QueryOpts: queryOpts, FetchOpts: fetchOpts, @@ -128,24 +121,6 @@ func ParseRequest( }, nil } -func validateRequest(params models.RequestParams, maxPoints int) error { - // Impose a rough limit on the number of returned time series. - // This is intended to prevent things like querying from the beginning of - // time with a 1s step size. - numSteps := int(params.End.Sub(params.Start) / params.Step) - if maxPoints > 0 && numSteps > maxPoints { - return fmt.Errorf( - "querying from %v to %v with step size %v would result in too many "+ - "datapoints (end - start / step > %d). Either decrease the query "+ - "resolution (?step=XX), decrease the time window, or increase "+ - "the limit (`limits.maxComputedDatapoints`)", - params.Start, params.End, params.Step, maxPoints, - ) - } - - return nil -} - // ParsedOptions are parsed options for the query. type ParsedOptions struct { QueryOpts *executor.QueryOptions From 7dd523380bf238e7ee6552522c00ccc895fc02c6 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Tue, 20 Oct 2020 16:17:57 -0400 Subject: [PATCH 12/55] more build fixes 2 --- src/query/api/v1/handler/placement/common.go | 30 +++++++------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/src/query/api/v1/handler/placement/common.go b/src/query/api/v1/handler/placement/common.go index d53aaeb65b..0344f9a7b8 100644 --- a/src/query/api/v1/handler/placement/common.go +++ b/src/query/api/v1/handler/placement/common.go @@ -228,55 +228,45 @@ func RegisterRoutes( ) { // Init var ( - initHandler = NewInitHandler(opts) - deprecatedInitFn = applyDeprecatedMiddleware(initHandler.ServeHTTP, defaults, opts.instrumentOptions) - initFn = applyMiddleware(initHandler.ServeHTTP, defaults, opts.instrumentOptions) + initHandler = NewInitHandler(opts) + initFn = applyMiddleware(initHandler.ServeHTTP, defaults, opts.instrumentOptions) ) - r.HandleFunc(DeprecatedM3DBInitURL, deprecatedInitFn).Methods(InitHTTPMethod) r.HandleFunc(M3DBInitURL, initFn).Methods(InitHTTPMethod) r.HandleFunc(M3AggInitURL, initFn).Methods(InitHTTPMethod) r.HandleFunc(M3CoordinatorInitURL, initFn).Methods(InitHTTPMethod) // Get var ( - getHandler = NewGetHandler(opts) - deprecatedGetFn = applyDeprecatedMiddleware(getHandler.ServeHTTP, defaults, opts.instrumentOptions) - getFn = applyMiddleware(getHandler.ServeHTTP, defaults, opts.instrumentOptions) + getHandler = NewGetHandler(opts) + getFn = applyMiddleware(getHandler.ServeHTTP, defaults, opts.instrumentOptions) ) - r.HandleFunc(DeprecatedM3DBGetURL, deprecatedGetFn).Methods(GetHTTPMethod) r.HandleFunc(M3DBGetURL, getFn).Methods(GetHTTPMethod) r.HandleFunc(M3AggGetURL, getFn).Methods(GetHTTPMethod) r.HandleFunc(M3CoordinatorGetURL, getFn).Methods(GetHTTPMethod) // Delete all var ( - deleteAllHandler = NewDeleteAllHandler(opts) - deprecatedDeleteAllFn = applyDeprecatedMiddleware(deleteAllHandler.ServeHTTP, defaults, opts.instrumentOptions) - deleteAllFn = applyMiddleware(deleteAllHandler.ServeHTTP, defaults, opts.instrumentOptions) + deleteAllHandler = NewDeleteAllHandler(opts) + deleteAllFn = applyMiddleware(deleteAllHandler.ServeHTTP, defaults, opts.instrumentOptions) ) - r.HandleFunc(DeprecatedM3DBDeleteAllURL, deprecatedDeleteAllFn).Methods(DeleteAllHTTPMethod) r.HandleFunc(M3DBDeleteAllURL, deleteAllFn).Methods(DeleteAllHTTPMethod) r.HandleFunc(M3AggDeleteAllURL, deleteAllFn).Methods(DeleteAllHTTPMethod) r.HandleFunc(M3CoordinatorDeleteAllURL, deleteAllFn).Methods(DeleteAllHTTPMethod) // Add var ( - addHandler = NewAddHandler(opts) - deprecatedAddFn = applyDeprecatedMiddleware(addHandler.ServeHTTP, defaults, opts.instrumentOptions) - addFn = applyMiddleware(addHandler.ServeHTTP, defaults, opts.instrumentOptions) + addHandler = NewAddHandler(opts) + addFn = applyMiddleware(addHandler.ServeHTTP, defaults, opts.instrumentOptions) ) - r.HandleFunc(DeprecatedM3DBAddURL, deprecatedAddFn).Methods(AddHTTPMethod) r.HandleFunc(M3DBAddURL, addFn).Methods(AddHTTPMethod) r.HandleFunc(M3AggAddURL, addFn).Methods(AddHTTPMethod) r.HandleFunc(M3CoordinatorAddURL, addFn).Methods(AddHTTPMethod) // Delete var ( - deleteHandler = NewDeleteHandler(opts) - deprecatedDeleteFn = applyDeprecatedMiddleware(deleteHandler.ServeHTTP, defaults, opts.instrumentOptions) - deleteFn = applyMiddleware(deleteHandler.ServeHTTP, defaults, opts.instrumentOptions) + deleteHandler = NewDeleteHandler(opts) + deleteFn = applyMiddleware(deleteHandler.ServeHTTP, defaults, opts.instrumentOptions) ) - r.HandleFunc(DeprecatedM3DBDeleteURL, deprecatedDeleteFn).Methods(DeleteHTTPMethod) r.HandleFunc(M3DBDeleteURL, deleteFn).Methods(DeleteHTTPMethod) r.HandleFunc(M3AggDeleteURL, deleteFn).Methods(DeleteHTTPMethod) r.HandleFunc(M3CoordinatorDeleteURL, deleteFn).Methods(DeleteHTTPMethod) From 4b2cdc170a0c448c9a63b7ff6b5a8a24ec79ef07 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Tue, 20 Oct 2020 16:18:30 -0400 Subject: [PATCH 13/55] more build fixes 3 --- src/query/api/v1/handler/namespace/common.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/query/api/v1/handler/namespace/common.go b/src/query/api/v1/handler/namespace/common.go index 9032cf529d..eb21169fc8 100644 --- a/src/query/api/v1/handler/namespace/common.go +++ b/src/query/api/v1/handler/namespace/common.go @@ -129,13 +129,11 @@ func RegisterRoutes( // Get M3DB namespaces. getHandler := wrapped( applyMiddleware(NewGetHandler(client, instrumentOpts).ServeHTTP, defaults)) - r.HandleFunc(DeprecatedM3DBGetURL, getHandler.ServeHTTP).Methods(GetHTTPMethod) r.HandleFunc(M3DBGetURL, getHandler.ServeHTTP).Methods(GetHTTPMethod) // Add M3DB namespaces. addHandler := wrapped( applyMiddleware(NewAddHandler(client, instrumentOpts).ServeHTTP, defaults)) - r.HandleFunc(DeprecatedM3DBAddURL, addHandler.ServeHTTP).Methods(AddHTTPMethod) r.HandleFunc(M3DBAddURL, addHandler.ServeHTTP).Methods(AddHTTPMethod) // Update M3DB namespaces. @@ -146,7 +144,6 @@ func RegisterRoutes( // Delete M3DB namespaces. deleteHandler := wrapped( applyMiddleware(NewDeleteHandler(client, instrumentOpts).ServeHTTP, defaults)) - r.HandleFunc(DeprecatedM3DBDeleteURL, deleteHandler.ServeHTTP).Methods(DeleteHTTPMethod) r.HandleFunc(M3DBDeleteURL, deleteHandler.ServeHTTP).Methods(DeleteHTTPMethod) // Deploy M3DB schemas. From cd7cf2ad6d3cb99dc84018ea381fcebda2756f91 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Tue, 20 Oct 2020 16:52:08 -0400 Subject: [PATCH 14/55] bootstrap misc config changes 1 --- src/cmd/services/m3dbnode/config/bootstrap.go | 113 +--------------- .../m3dbnode/config/bootstrap_test.go | 81 ----------- .../services/m3dbnode/config/config_test.go | 9 +- src/dbnode/server/server.go | 93 +------------ .../database/config_bootstrappers_get.go | 92 ------------- .../database/config_bootstrappers_get_test.go | 105 --------------- .../database/config_bootstrappers_set.go | 112 --------------- .../database/config_bootstrappers_set_test.go | 127 ------------------ .../v1/handler/prometheus/native/common.go | 6 - .../handler/prometheus/native/read_common.go | 1 - 10 files changed, 16 insertions(+), 723 deletions(-) delete mode 100644 src/cmd/services/m3dbnode/config/bootstrap_test.go delete mode 100644 src/query/api/v1/handler/database/config_bootstrappers_get.go delete mode 100644 src/query/api/v1/handler/database/config_bootstrappers_get_test.go delete mode 100644 src/query/api/v1/handler/database/config_bootstrappers_set.go delete mode 100644 src/query/api/v1/handler/database/config_bootstrappers_set_test.go diff --git a/src/cmd/services/m3dbnode/config/bootstrap.go b/src/cmd/services/m3dbnode/config/bootstrap.go index 192578c56e..77e0ba0eff 100644 --- a/src/cmd/services/m3dbnode/config/bootstrap.go +++ b/src/cmd/services/m3dbnode/config/bootstrap.go @@ -48,8 +48,10 @@ var ( // (run in ascending order of precedence) orderedBootstrappers = []string{ uninitialized.UninitializedTopologyBootstrapperName, + // Peers and commitlog must come after uninitialized topology bootrapping. peers.PeersBootstrapperName, commitlog.CommitLogBootstrapperName, + // Filesystem bootstrapping must be last. bfs.FileSystemBootstrapperName, } ) @@ -161,20 +163,8 @@ func newDefaultBootstrapPeersConfiguration() BootstrapPeersConfiguration { } } -// BootstrapConfigurationValidator can be used to validate the option sets -// that the bootstrap configuration builds. -// Useful for tests and perhaps verifying same options set across multiple -// bootstrappers. -type BootstrapConfigurationValidator interface { - ValidateFilesystemBootstrapperOptions(opts bfs.Options) error - ValidateCommitLogBootstrapperOptions(opts commitlog.Options) error - ValidatePeersBootstrapperOptions(opts peers.Options) error - ValidateUninitializedBootstrapperOptions(opts uninitialized.Options) error -} - // New creates a bootstrap process based on the bootstrap configuration. func (bsc BootstrapConfiguration) New( - validator BootstrapConfigurationValidator, rsOpts result.Options, opts storage.Options, topoMapProvider topology.MapProvider, @@ -224,7 +214,7 @@ func (bsc BootstrapConfiguration) New( if v := bsc.IndexSegmentConcurrency; v != nil { fsbOpts = fsbOpts.SetIndexSegmentConcurrency(*v) } - if err := validator.ValidateFilesystemBootstrapperOptions(fsbOpts); err != nil { + if err := fsbOpts.Validate(); err != nil { return nil, err } bs, err = bfs.NewFileSystemBootstrapperProvider(fsbOpts, bs) @@ -238,7 +228,7 @@ func (bsc BootstrapConfiguration) New( SetCommitLogOptions(opts.CommitLogOptions()). SetRuntimeOptionsManager(opts.RuntimeOptionsManager()). SetReturnUnfulfilledForCorruptCommitLogFiles(cCfg.ReturnUnfulfilledForCorruptCommitLogFiles) - if err := validator.ValidateCommitLogBootstrapperOptions(cOpts); err != nil { + if err := cOpts.Validate(); err != nil { return nil, err } inspection, err := fs.InspectFilesystem(fsOpts) @@ -266,7 +256,7 @@ func (bsc BootstrapConfiguration) New( if v := bsc.IndexSegmentConcurrency; v != nil { pOpts = pOpts.SetIndexSegmentConcurrency(*v) } - if err := validator.ValidatePeersBootstrapperOptions(pOpts); err != nil { + if err := pOpts.Validate(); err != nil { return nil, err } bs, err = peers.NewPeersBootstrapperProvider(pOpts, bs) @@ -277,7 +267,7 @@ func (bsc BootstrapConfiguration) New( uOpts := uninitialized.NewOptions(). SetResultOptions(rsOpts). SetInstrumentOptions(opts.InstrumentOptions()) - if err := validator.ValidateUninitializedBootstrapperOptions(uOpts); err != nil { + if err := uOpts.Validate(); err != nil { return nil, err } bs = uninitialized.NewUninitializedTopologyBootstrapperProvider(uOpts, bs) @@ -315,94 +305,3 @@ func (bsc BootstrapConfiguration) peersConfig() BootstrapPeersConfiguration { } return newDefaultBootstrapPeersConfiguration() } - -type bootstrapConfigurationValidator struct { -} - -// NewBootstrapConfigurationValidator returns a new bootstrap configuration -// validator that validates certain options configured by the bootstrap -// configuration. -func NewBootstrapConfigurationValidator() BootstrapConfigurationValidator { - return bootstrapConfigurationValidator{} -} - -func (v bootstrapConfigurationValidator) ValidateBootstrappersOrder(names []string) error { - dataFetchingBootstrappers := []string{ - bfs.FileSystemBootstrapperName, - peers.PeersBootstrapperName, - commitlog.CommitLogBootstrapperName, - } - - precedingBootstrappersAllowedByBootstrapper := map[string][]string{ - bootstrapper.NoOpAllBootstrapperName: dataFetchingBootstrappers, - bootstrapper.NoOpNoneBootstrapperName: dataFetchingBootstrappers, - bfs.FileSystemBootstrapperName: []string{ - // Filesystem bootstrapper must always appear first - }, - peers.PeersBootstrapperName: []string{ - // Peers must always appear after filesystem - bfs.FileSystemBootstrapperName, - // Peers may appear before OR after commitlog - commitlog.CommitLogBootstrapperName, - }, - commitlog.CommitLogBootstrapperName: []string{ - // Commit log bootstrapper may appear after filesystem or peers - bfs.FileSystemBootstrapperName, - peers.PeersBootstrapperName, - }, - uninitialized.UninitializedTopologyBootstrapperName: []string{ - // Unintialized bootstrapper may appear after filesystem or peers or commitlog - bfs.FileSystemBootstrapperName, - commitlog.CommitLogBootstrapperName, - peers.PeersBootstrapperName, - }, - } - - validated := make(map[string]struct{}) - for _, name := range names { - precedingAllowed, ok := precedingBootstrappersAllowedByBootstrapper[name] - if !ok { - return fmt.Errorf("unknown bootstrapper: %v", name) - } - - allowed := make(map[string]struct{}) - for _, elem := range precedingAllowed { - allowed[elem] = struct{}{} - } - - for existing := range validated { - if _, ok := allowed[existing]; !ok { - return fmt.Errorf("bootstrapper %s cannot appear after %s: ", - name, existing) - } - } - - validated[name] = struct{}{} - } - - return nil -} - -func (v bootstrapConfigurationValidator) ValidateFilesystemBootstrapperOptions( - opts bfs.Options, -) error { - return opts.Validate() -} - -func (v bootstrapConfigurationValidator) ValidateCommitLogBootstrapperOptions( - opts commitlog.Options, -) error { - return opts.Validate() -} - -func (v bootstrapConfigurationValidator) ValidatePeersBootstrapperOptions( - opts peers.Options, -) error { - return opts.Validate() -} - -func (v bootstrapConfigurationValidator) ValidateUninitializedBootstrapperOptions( - opts uninitialized.Options, -) error { - return opts.Validate() -} diff --git a/src/cmd/services/m3dbnode/config/bootstrap_test.go b/src/cmd/services/m3dbnode/config/bootstrap_test.go deleted file mode 100644 index d9d332cd1f..0000000000 --- a/src/cmd/services/m3dbnode/config/bootstrap_test.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package config - -import ( - "fmt" - "testing" - - "github.com/m3db/m3/src/dbnode/storage/bootstrap/bootstrapper" - "github.com/m3db/m3/src/dbnode/storage/bootstrap/bootstrapper/commitlog" - bfs "github.com/m3db/m3/src/dbnode/storage/bootstrap/bootstrapper/fs" - "github.com/m3db/m3/src/dbnode/storage/bootstrap/bootstrapper/peers" - - "github.com/stretchr/testify/require" -) - -var ( - peersBs = peers.PeersBootstrapperName - fsBs = bfs.FileSystemBootstrapperName - commitLogBs = commitlog.CommitLogBootstrapperName - noOpAllBs = bootstrapper.NoOpAllBootstrapperName - noOpNoneBs = bootstrapper.NoOpNoneBootstrapperName -) - -func TestValidatorValidateBootstrappersOrder(t *testing.T) { - tests := []struct { - valid bool - bootstrappers []string - }{ - {true, []string{fsBs, peersBs, commitLogBs, noOpNoneBs}}, - {true, []string{fsBs, commitLogBs, noOpNoneBs}}, - {true, []string{commitLogBs, noOpNoneBs}}, - {true, []string{fsBs, noOpNoneBs}}, - {true, []string{fsBs}}, - {true, []string{fsBs, peersBs}}, - {true, []string{peersBs}}, - {true, []string{peersBs, commitLogBs}}, - {true, []string{fsBs, commitLogBs}}, - {true, []string{noOpNoneBs}}, - {true, []string{noOpAllBs}}, - // Do not allow peers to appear before FS - {false, []string{peersBs, fsBs, commitLogBs, noOpNoneBs}}, - // Do not allow a non-data fetching bootstrapper twice - {false, []string{commitLogBs, noOpAllBs, noOpNoneBs}}, - // Do not allow multiple bootstrappers to appear - {false, []string{commitLogBs, commitLogBs, noOpNoneBs}}, - // Do not allow unknown bootstrappers - {false, []string{"foo"}}, - } - - for _, tt := range tests { - name := fmt.Sprintf("%v", tt.bootstrappers) - t.Run(name, func(t *testing.T) { - validator := NewBootstrapConfigurationValidator() - err := validator.ValidateBootstrappersOrder(tt.bootstrappers) - if tt.valid { - require.NoError(t, err) - } else { - require.Error(t, err) - } - }) - } -} diff --git a/src/cmd/services/m3dbnode/config/config_test.go b/src/cmd/services/m3dbnode/config/config_test.go index 37c967129e..b67692fa08 100644 --- a/src/cmd/services/m3dbnode/config/config_test.go +++ b/src/cmd/services/m3dbnode/config/config_test.go @@ -1057,7 +1057,12 @@ db: origin := topology.NewMockHost(ctrl) adminClient := client.NewMockAdminClient(ctrl) - _, err = cfg.DB.Bootstrap.New(validator, - result.NewOptions(), storage.DefaultTestOptions(), mapProvider, origin, adminClient) + _, err = cfg.DB.Bootstrap.New( + result.NewOptions(), + storage.DefaultTestOptions(), + mapProvider, + origin, + adminClient, + ) require.NoError(t, err) } diff --git a/src/dbnode/server/server.go b/src/dbnode/server/server.go index 5d4f6b8684..d98b85f2ec 100644 --- a/src/dbnode/server/server.go +++ b/src/dbnode/server/server.go @@ -39,7 +39,6 @@ import ( "github.com/m3db/m3/src/cluster/client/etcd" "github.com/m3db/m3/src/cluster/generated/proto/commonpb" "github.com/m3db/m3/src/cluster/kv" - "github.com/m3db/m3/src/cluster/kv/util" "github.com/m3db/m3/src/cmd/services/m3dbnode/config" queryconfig "github.com/m3db/m3/src/cmd/services/m3query/config" "github.com/m3db/m3/src/dbnode/client" @@ -848,52 +847,13 @@ func Run(runOpts RunOptions) { // recent as the one that triggered the bootstrap, if not newer. // See GitHub issue #1013 for more details. topoMapProvider := newTopoMapProvider(topo) - bs, err := cfg.Bootstrap.New(config.NewBootstrapConfigurationValidator(), - rsOpts, opts, topoMapProvider, origin, m3dbClient) + bs, err := cfg.Bootstrap.New( + rsOpts, opts, topoMapProvider, origin, m3dbClient, + ) if err != nil { logger.Fatal("could not create bootstrap process", zap.Error(err)) } - opts = opts.SetBootstrapProcessProvider(bs) - timeout := bootstrapConfigInitTimeout - - bsGauge := instrument.NewStringListEmitter(scope, "bootstrappers") - if err := bsGauge.Start(cfg.Bootstrap.Bootstrappers); err != nil { - logger.Error("unable to start emitting bootstrap gauge", - zap.Strings("bootstrappers", cfg.Bootstrap.Bootstrappers), - zap.Error(err), - ) - } - defer func() { - if err := bsGauge.Close(); err != nil { - logger.Error("stop emitting bootstrap gauge failed", zap.Error(err)) - } - }() - - kvWatchBootstrappers(syncCfg.KVStore, logger, timeout, cfg.Bootstrap.Bootstrappers, - func(bootstrappers []string) { - if len(bootstrappers) == 0 { - logger.Error("updated bootstrapper list is empty") - return - } - - cfg.Bootstrap.Bootstrappers = bootstrappers - updated, err := cfg.Bootstrap.New(config.NewBootstrapConfigurationValidator(), - rsOpts, opts, topoMapProvider, origin, m3dbClient) - if err != nil { - logger.Error("updated bootstrapper list failed", zap.Error(err)) - return - } - - bs.SetBootstrapperProvider(updated.BootstrapperProvider()) - - if err := bsGauge.UpdateStringList(bootstrappers); err != nil { - logger.Error("unable to update bootstrap gauge with new bootstrappers", - zap.Strings("bootstrappers", bootstrappers), - zap.Error(err), - ) - } - }) // Start the cluster services now that the M3DB client is available. tchannelthriftClusterClose, err := ttcluster.NewServer(m3dbClient, @@ -1308,53 +1268,6 @@ func setEncodersPerBlockLimitOnChange( return runtimeOptsMgr.Update(newRuntimeOpts) } -// this function will block for at most waitTimeout to try to get an initial value -// before we kick off the bootstrap -func kvWatchBootstrappers( - kv kv.Store, - logger *zap.Logger, - waitTimeout time.Duration, - defaultBootstrappers []string, - onUpdate func(bootstrappers []string), -) { - vw, err := kv.Watch(kvconfig.BootstrapperKey) - if err != nil { - logger.Fatal("could not watch value for key with KV", - zap.String("key", kvconfig.BootstrapperKey)) - } - - initializedCh := make(chan struct{}) - - var initialized bool - go func() { - opts := util.NewOptions().SetLogger(logger) - - for range vw.C() { - v, err := util.StringArrayFromValue(vw.Get(), - kvconfig.BootstrapperKey, defaultBootstrappers, opts) - if err != nil { - logger.Error("error converting KV update to string array", - zap.String("key", kvconfig.BootstrapperKey), - zap.Error(err), - ) - continue - } - - onUpdate(v) - - if !initialized { - initialized = true - close(initializedCh) - } - } - }() - - select { - case <-time.After(waitTimeout): - case <-initializedCh: - } -} - func withEncodingAndPoolingOptions( cfg config.DBConfiguration, logger *zap.Logger, diff --git a/src/query/api/v1/handler/database/config_bootstrappers_get.go b/src/query/api/v1/handler/database/config_bootstrappers_get.go deleted file mode 100644 index afccbb4086..0000000000 --- a/src/query/api/v1/handler/database/config_bootstrappers_get.go +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package database - -import ( - "net/http" - - clusterclient "github.com/m3db/m3/src/cluster/client" - "github.com/m3db/m3/src/cluster/generated/proto/commonpb" - "github.com/m3db/m3/src/cluster/kv" - "github.com/m3db/m3/src/dbnode/kvconfig" - "github.com/m3db/m3/src/query/api/v1/handler" - "github.com/m3db/m3/src/query/util/logging" - "github.com/m3db/m3/src/x/instrument" - xhttp "github.com/m3db/m3/src/x/net/http" - - "go.uber.org/zap" -) - -const ( - // ConfigGetBootstrappersURL is the url for the database create handler. - ConfigGetBootstrappersURL = handler.RoutePrefixV1 + "/database/config/bootstrappers" - - // ConfigGetBootstrappersHTTPMethod is the HTTP method used with this resource. - ConfigGetBootstrappersHTTPMethod = http.MethodGet -) - -type configGetBootstrappersHandler struct { - client clusterclient.Client - instrumentOpts instrument.Options -} - -// NewConfigGetBootstrappersHandler returns a new instance of a database create handler. -func NewConfigGetBootstrappersHandler( - client clusterclient.Client, - instrumentOpts instrument.Options, -) http.Handler { - return &configGetBootstrappersHandler{ - client: client, - instrumentOpts: instrumentOpts, - } -} - -func (h *configGetBootstrappersHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - logger := logging.WithContext(ctx, h.instrumentOpts) - - store, err := h.client.KV() - if err != nil { - logger.Error("unable to get kv store", zap.Error(err)) - xhttp.Error(w, err, http.StatusInternalServerError) - return - } - - value, err := store.Get(kvconfig.BootstrapperKey) - if err != nil && err == kv.ErrNotFound { - xhttp.Error(w, err, http.StatusNotFound) - return - } - if err != nil { - logger.Error("unable to get kv key", zap.Error(err)) - xhttp.Error(w, err, http.StatusInternalServerError) - return - } - - array := new(commonpb.StringArrayProto) - if err := value.Unmarshal(array); err != nil { - logger.Error("unable to unmarshal kv key", zap.Error(err)) - xhttp.Error(w, err, http.StatusInternalServerError) - return - } - - xhttp.WriteProtoMsgJSONResponse(w, array, logger) -} diff --git a/src/query/api/v1/handler/database/config_bootstrappers_get_test.go b/src/query/api/v1/handler/database/config_bootstrappers_get_test.go deleted file mode 100644 index df46f40f98..0000000000 --- a/src/query/api/v1/handler/database/config_bootstrappers_get_test.go +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package database - -import ( - "io/ioutil" - "net/http" - "net/http/httptest" - "testing" - - "github.com/m3db/m3/src/cluster/generated/proto/commonpb" - "github.com/m3db/m3/src/cluster/kv" - "github.com/m3db/m3/src/dbnode/kvconfig" - "github.com/m3db/m3/src/x/instrument" - xjson "github.com/m3db/m3/src/x/json" - xtest "github.com/m3db/m3/src/x/test" - - "github.com/gogo/protobuf/proto" - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestConfigGetBootstrappersHandler(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockClient, mockStore, _ := SetupDatabaseTest(t, ctrl) - handler := NewConfigGetBootstrappersHandler(mockClient, - instrument.NewOptions()) - w := httptest.NewRecorder() - - value := kv.NewMockValue(ctrl) - value.EXPECT(). - Unmarshal(gomock.Any()). - Return(nil). - Do(func(v proto.Message) { - array, ok := v.(*commonpb.StringArrayProto) - require.True(t, ok) - array.Values = []string{"filesystem", "commitlog", "peers", "uninitialized_topology"} - }) - - mockStore.EXPECT(). - Get(kvconfig.BootstrapperKey). - Return(value, nil) - - req := httptest.NewRequest("GET", "/database/config/bootstrappers", nil) - require.NotNil(t, req) - - handler.ServeHTTP(w, req) - - resp := w.Result() - body, err := ioutil.ReadAll(resp.Body) - assert.NoError(t, err) - assert.Equal(t, http.StatusOK, resp.StatusCode) - - expectedResp := xjson.Map{ - "values": xjson.Array{"filesystem", "commitlog", "peers", "uninitialized_topology"}, - } - - expected := xtest.MustPrettyJSONMap(t, expectedResp) - actual := xtest.MustPrettyJSONString(t, string(body)) - - assert.Equal(t, expected, actual, xtest.Diff(expected, actual)) -} - -func TestConfigGetBootstrappersHandlerNotFound(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockClient, mockStore, _ := SetupDatabaseTest(t, ctrl) - handler := NewConfigGetBootstrappersHandler(mockClient, - instrument.NewOptions()) - w := httptest.NewRecorder() - - mockStore.EXPECT(). - Get(kvconfig.BootstrapperKey). - Return(nil, kv.ErrNotFound) - - req := httptest.NewRequest("GET", "/database/config/bootstrappers", nil) - require.NotNil(t, req) - - handler.ServeHTTP(w, req) - - resp := w.Result() - assert.Equal(t, http.StatusNotFound, resp.StatusCode) -} diff --git a/src/query/api/v1/handler/database/config_bootstrappers_set.go b/src/query/api/v1/handler/database/config_bootstrappers_set.go deleted file mode 100644 index 98363a06ac..0000000000 --- a/src/query/api/v1/handler/database/config_bootstrappers_set.go +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package database - -import ( - "fmt" - "net/http" - - clusterclient "github.com/m3db/m3/src/cluster/client" - "github.com/m3db/m3/src/cluster/generated/proto/commonpb" - dbconfig "github.com/m3db/m3/src/cmd/services/m3dbnode/config" - "github.com/m3db/m3/src/dbnode/kvconfig" - "github.com/m3db/m3/src/query/api/v1/handler" - "github.com/m3db/m3/src/query/util/logging" - "github.com/m3db/m3/src/x/instrument" - xhttp "github.com/m3db/m3/src/x/net/http" - - "github.com/gogo/protobuf/jsonpb" - "go.uber.org/zap" -) - -const ( - // ConfigSetBootstrappersURL is the url for the database create handler. - ConfigSetBootstrappersURL = handler.RoutePrefixV1 + "/database/config/bootstrappers" - - // ConfigSetBootstrappersHTTPMethod is the HTTP method used with this resource. - ConfigSetBootstrappersHTTPMethod = http.MethodPost -) - -type configSetBootstrappersHandler struct { - client clusterclient.Client - instrumentOpts instrument.Options -} - -// NewConfigSetBootstrappersHandler returns a new instance of a database create handler. -func NewConfigSetBootstrappersHandler( - client clusterclient.Client, - instrumentOpts instrument.Options, -) http.Handler { - return &configSetBootstrappersHandler{ - client: client, - instrumentOpts: instrumentOpts, - } -} - -func (h *configSetBootstrappersHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - logger := logging.WithContext(ctx, h.instrumentOpts) - - value, rErr := h.parseRequest(r) - if rErr != nil { - logger.Error("unable to parse request", zap.Error(rErr)) - xhttp.Error(w, rErr.Inner(), rErr.Code()) - return - } - - store, err := h.client.KV() - if err != nil { - logger.Error("unable to get kv store", zap.Error(err)) - xhttp.Error(w, err, http.StatusInternalServerError) - return - } - - if _, err := store.Set(kvconfig.BootstrapperKey, value); err != nil { - logger.Error("unable to set kv key", zap.Error(err)) - xhttp.Error(w, err, http.StatusInternalServerError) - return - } - - xhttp.WriteProtoMsgJSONResponse(w, value, logger) -} - -func (h *configSetBootstrappersHandler) parseRequest( - r *http.Request, -) (*commonpb.StringArrayProto, *xhttp.ParseError) { - array := new(commonpb.StringArrayProto) - - defer r.Body.Close() - - if err := jsonpb.Unmarshal(r.Body, array); err != nil { - return nil, xhttp.NewParseError(err, http.StatusBadRequest) - } - - if len(array.Values) == 0 { - return nil, xhttp.NewParseError(fmt.Errorf("no values"), http.StatusBadRequest) - } - - validator := dbconfig.NewBootstrapConfigurationValidator() - if err := validator.ValidateBootstrappersOrder(array.Values); err != nil { - return nil, xhttp.NewParseError(err, http.StatusBadRequest) - } - - return array, nil -} diff --git a/src/query/api/v1/handler/database/config_bootstrappers_set_test.go b/src/query/api/v1/handler/database/config_bootstrappers_set_test.go deleted file mode 100644 index 8ae3a3fe8e..0000000000 --- a/src/query/api/v1/handler/database/config_bootstrappers_set_test.go +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package database - -import ( - "io/ioutil" - "net/http" - "net/http/httptest" - "testing" - - "github.com/m3db/m3/src/cluster/generated/proto/commonpb" - "github.com/m3db/m3/src/dbnode/kvconfig" - "github.com/m3db/m3/src/x/instrument" - xjson "github.com/m3db/m3/src/x/json" - xtest "github.com/m3db/m3/src/x/test" - - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestConfigSetBootstrappersHandler(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockClient, mockStore, _ := SetupDatabaseTest(t, ctrl) - handler := NewConfigSetBootstrappersHandler(mockClient, - instrument.NewOptions()) - w := httptest.NewRecorder() - - jsonInput := xjson.Map{ - "values": xjson.Array{"filesystem", "commitlog", "peers", "uninitialized_topology"}, - } - - mockStore.EXPECT(). - Set(kvconfig.BootstrapperKey, gomock.Any()). - Return(int(1), nil). - Do(func(key string, value *commonpb.StringArrayProto) { - assert.Equal(t, []string{ - "filesystem", "commitlog", "peers", "uninitialized_topology", - }, value.Values) - }) - - req := httptest.NewRequest("POST", "/database/config/bootstrappers", - xjson.MustNewTestReader(t, jsonInput)) - require.NotNil(t, req) - - handler.ServeHTTP(w, req) - - resp := w.Result() - body, err := ioutil.ReadAll(resp.Body) - assert.NoError(t, err) - assert.Equal(t, http.StatusOK, resp.StatusCode) - - expectedResp := xjson.Map{ - "values": xjson.Array{"filesystem", "commitlog", "peers", "uninitialized_topology"}, - } - - expected := xtest.MustPrettyJSONMap(t, expectedResp) - actual := xtest.MustPrettyJSONString(t, string(body)) - - assert.Equal(t, expected, actual, xtest.Diff(expected, actual)) -} - -func TestConfigSetBootstrappersHandlerNoValues(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockClient, _, _ := SetupDatabaseTest(t, ctrl) - handler := NewConfigSetBootstrappersHandler(mockClient, - instrument.NewOptions()) - w := httptest.NewRecorder() - - jsonInput := xjson.Map{ - "values": xjson.Array{}, - } - - req := httptest.NewRequest("POST", "/database/config/bootstrappers", - xjson.MustNewTestReader(t, jsonInput)) - require.NotNil(t, req) - - handler.ServeHTTP(w, req) - - resp := w.Result() - assert.Equal(t, http.StatusBadRequest, resp.StatusCode) -} - -func TestConfigSetBootstrappersHandlerInvalidValue(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockClient, _, _ := SetupDatabaseTest(t, ctrl) - handler := NewConfigSetBootstrappersHandler(mockClient, - instrument.NewOptions()) - w := httptest.NewRecorder() - - jsonInput := xjson.Map{ - "values": xjson.Array{"filesystem", "foo"}, - } - - req := httptest.NewRequest("POST", "/database/config/bootstrappers", - xjson.MustNewTestReader(t, jsonInput)) - require.NotNil(t, req) - - handler.ServeHTTP(w, req) - - resp := w.Result() - assert.Equal(t, http.StatusBadRequest, resp.StatusCode) -} diff --git a/src/query/api/v1/handler/prometheus/native/common.go b/src/query/api/v1/handler/prometheus/native/common.go index bae194745c..3f5955096a 100644 --- a/src/query/api/v1/handler/prometheus/native/common.go +++ b/src/query/api/v1/handler/prometheus/native/common.go @@ -26,7 +26,6 @@ import ( "math" "net/http" "strconv" - "strings" "time" "github.com/m3db/m3/src/query/api/v1/handler/prometheus" @@ -41,7 +40,6 @@ import ( "github.com/m3db/m3/src/query/util" "github.com/m3db/m3/src/query/util/json" "github.com/m3db/m3/src/query/util/logging" - "github.com/m3db/m3/src/x/headers" "github.com/m3db/m3/src/x/instrument" xhttp "github.com/m3db/m3/src/x/net/http" @@ -153,10 +151,6 @@ func parseParams( params.IncludeEnd = !excludeEnd } - if strings.ToLower(r.Header.Get(headers.RenderFormat)) == "m3ql" { - params.FormatType = models.FormatM3QL - } - params.LookbackDuration = engineOpts.LookbackDuration() if v := fetchOpts.LookbackDuration; v != nil { params.LookbackDuration = *v diff --git a/src/query/api/v1/handler/prometheus/native/read_common.go b/src/query/api/v1/handler/prometheus/native/read_common.go index 6445be8db3..33d4ca7090 100644 --- a/src/query/api/v1/handler/prometheus/native/read_common.go +++ b/src/query/api/v1/handler/prometheus/native/read_common.go @@ -56,7 +56,6 @@ func newPromReadMetrics(scope tally.Scope) promReadMetrics { fetchErrorsClient: scope.Tagged(map[string]string{"code": "4XX"}). Counter("fetch.errors"), fetchTimerSuccess: scope.Timer("fetch.success.latency"), - maxDatapoints: scope.Gauge("max_datapoints"), } } From 6bf8f67589d16d2fcf0a8001d53366d90f0eb26e Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Tue, 20 Oct 2020 16:53:55 -0400 Subject: [PATCH 15/55] bootstrap misc config changes 2 --- src/query/api/v1/handler/database/common.go | 14 -------------- src/query/server/query.go | 13 ++++--------- 2 files changed, 4 insertions(+), 23 deletions(-) diff --git a/src/query/api/v1/handler/database/common.go b/src/query/api/v1/handler/database/common.go index ee2d654fdb..f7960909b9 100644 --- a/src/query/api/v1/handler/database/common.go +++ b/src/query/api/v1/handler/database/common.go @@ -64,24 +64,10 @@ func RegisterRoutes( wrapped(createHandler).ServeHTTP). Methods(CreateHTTPMethod) - r.HandleFunc(ConfigGetBootstrappersURL, wrapped( - NewConfigGetBootstrappersHandler(client, instrumentOpts)).ServeHTTP). - Methods(ConfigGetBootstrappersHTTPMethod) - r.HandleFunc(ConfigSetBootstrappersURL, wrapped( - NewConfigSetBootstrappersHandler(client, instrumentOpts)).ServeHTTP). - Methods(ConfigSetBootstrappersHTTPMethod) - // Register the same handler under two different endpoints. This just makes explaining things in // our documentation easier so we can separate out concepts, but share the underlying code. r.HandleFunc(CreateURL, wrapped(createHandler).ServeHTTP).Methods(CreateHTTPMethod) r.HandleFunc(CreateNamespaceURL, wrapped(createHandler).ServeHTTP).Methods(CreateNamespaceHTTPMethod) - r.HandleFunc(ConfigGetBootstrappersURL, wrapped( - NewConfigGetBootstrappersHandler(client, instrumentOpts)).ServeHTTP). - Methods(ConfigGetBootstrappersHTTPMethod) - r.HandleFunc(ConfigSetBootstrappersURL, wrapped( - NewConfigSetBootstrappersHandler(client, instrumentOpts)).ServeHTTP). - Methods(ConfigSetBootstrappersHTTPMethod) - return nil } diff --git a/src/query/server/query.go b/src/query/server/query.go index 8352ecf0e5..b94366737c 100644 --- a/src/query/server/query.go +++ b/src/query/server/query.go @@ -510,12 +510,7 @@ func Run(runOpts RunOptions) { logger.Fatal("unable to register routes", zap.Error(err)) } - listenAddress, err := cfg.ListenAddress.Resolve() - if err != nil { - logger.Fatal("unable to get listen address", zap.Error(err)) - } - - srv := &http.Server{Addr: listenAddress, Handler: handler.Router()} + srv := &http.Server{Addr: cfg.ListenAddress, Handler: handler.Router()} defer func() { logger.Info("closing server") if err := srv.Shutdown(context.Background()); err != nil { @@ -523,10 +518,10 @@ func Run(runOpts RunOptions) { } }() - listener, err := listenerOpts.Listen("tcp", listenAddress) + listener, err := listenerOpts.Listen("tcp", cfg.ListenAddress) if err != nil { logger.Fatal("unable to listen on listen address", - zap.String("address", listenAddress), + zap.String("address", cfg.ListenAddress), zap.Error(err)) } if runOpts.ListenerCh != nil { @@ -536,7 +531,7 @@ func Run(runOpts RunOptions) { logger.Info("starting API server", zap.Stringer("address", listener.Addr())) if err := srv.Serve(listener); err != nil && err != http.ErrServerClosed { logger.Fatal("server serve error", - zap.String("address", listenAddress), + zap.String("address", cfg.ListenAddress), zap.Error(err)) } }() From 5516cb03d4202175b3b07cb7ea607f46f71a8223 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Tue, 20 Oct 2020 17:10:57 -0400 Subject: [PATCH 16/55] reporters 1 --- src/query/server/cost_reporters.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/query/server/cost_reporters.go b/src/query/server/cost_reporters.go index 1f6e2d0b5a..19a892dacb 100644 --- a/src/query/server/cost_reporters.go +++ b/src/query/server/cost_reporters.go @@ -79,8 +79,10 @@ func newConfiguredChainedEnforcer( globalReporterScope := globalScope.SubScope(reporterScopeName) globalLimitMgr := cost.NewStaticLimitManager( - cfg.Limits.Global.AsLimitManagerOptions(). - SetInstrumentOptions(instrumentOptions.SetMetricsScope(globalLimitManagerScope))) + cost.NewLimitManagerOptions().SetDefaultLimit(cost.Limit{ + Threshold: cost.Cost(0), + Enabled: false, + }).SetInstrumentOptions(instrumentOptions.SetMetricsScope(globalLimitManagerScope))) globalTracker := cost.NewTracker() @@ -97,8 +99,10 @@ func newConfiguredChainedEnforcer( queryReporterScope := queryScope.SubScope(reporterScopeName) queryLimitMgr := cost.NewStaticLimitManager( - cfg.Limits.PerQuery.AsLimitManagerOptions(). - SetInstrumentOptions(instrumentOptions.SetMetricsScope(queryLimitManagerScope))) + cost.NewLimitManagerOptions().SetDefaultLimit(cost.Limit{ + Threshold: cost.Cost(0), + Enabled: false, + }).SetInstrumentOptions(instrumentOptions.SetMetricsScope(queryLimitManagerScope))) queryTracker := cost.NewTracker() From 5f28d5dc1d9a711c64aee7e980a6c215eb6d688a Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Tue, 20 Oct 2020 17:11:46 -0400 Subject: [PATCH 17/55] listen addr --- src/collector/server/server.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/collector/server/server.go b/src/collector/server/server.go index e28578a424..c8b1d53189 100644 --- a/src/collector/server/server.go +++ b/src/collector/server/server.go @@ -136,12 +136,7 @@ func Run(runOpts RunOptions) { logger.Fatal("unable to register routes", zap.Error(err)) } - listenAddress, err := cfg.ListenAddress.Resolve() - if err != nil { - logger.Fatal("unable to resolve listen address", zap.Error(err)) - } - - srv := &http.Server{Addr: listenAddress, Handler: handler.Router()} + srv := &http.Server{Addr: cfg.ListenAddress, Handler: handler.Router()} defer func() { logger.Info("closing server") if err := srv.Shutdown(ctx); err != nil { @@ -150,10 +145,10 @@ func Run(runOpts RunOptions) { }() go func() { - logger.Info("starting server", zap.String("address", listenAddress)) + logger.Info("starting server", zap.String("address", cfg.ListenAddress)) if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { logger.Fatal("server error while listening", - zap.String("address", listenAddress), zap.Error(err)) + zap.String("address", cfg.ListenAddress), zap.Error(err)) } }() From 69efb04d072dd197b7647d327c2d71bca1b5e547 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Tue, 20 Oct 2020 17:27:54 -0400 Subject: [PATCH 18/55] reporters 2 --- src/query/server/cost_reporters.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/query/server/cost_reporters.go b/src/query/server/cost_reporters.go index 19a892dacb..0ff5a724f6 100644 --- a/src/query/server/cost_reporters.go +++ b/src/query/server/cost_reporters.go @@ -26,7 +26,6 @@ import ( "fmt" "sync" - "github.com/m3db/m3/src/cmd/services/m3query/config" qcost "github.com/m3db/m3/src/query/cost" "github.com/m3db/m3/src/x/close" "github.com/m3db/m3/src/x/cost" @@ -62,7 +61,6 @@ const ( // cost_reporter_max_datapoints_hist{limiter=~"(global|per_query)"}: histogram; // > represents the distribution of the maximum datapoints used at any point in each query. func newConfiguredChainedEnforcer( - cfg *config.Configuration, instrumentOptions instrument.Options, ) (qcost.ChainedEnforcer, close.SimpleCloser, error) { scope := instrumentOptions.MetricsScope().SubScope(costScopeName) From 8d057a1623e20b4f7888bf5c2c88c8d3f225ed7e Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Wed, 21 Oct 2020 10:42:13 -0400 Subject: [PATCH 19/55] bootstrap config tests --- .../services/m3dbnode/config/config_mock.go | 129 ------------------ .../services/m3dbnode/config/config_test.go | 32 ----- src/dbnode/generated/mocks/generate.go | 1 - 3 files changed, 162 deletions(-) delete mode 100644 src/cmd/services/m3dbnode/config/config_mock.go diff --git a/src/cmd/services/m3dbnode/config/config_mock.go b/src/cmd/services/m3dbnode/config/config_mock.go deleted file mode 100644 index b0e94f8b4d..0000000000 --- a/src/cmd/services/m3dbnode/config/config_mock.go +++ /dev/null @@ -1,129 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/m3db/m3/src/cmd/services/m3dbnode/config (interfaces: BootstrapConfigurationValidator) - -// Copyright (c) 2020 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -// Package config is a generated GoMock package. -package config - -import ( - "reflect" - - "github.com/m3db/m3/src/dbnode/storage/bootstrap/bootstrapper/commitlog" - "github.com/m3db/m3/src/dbnode/storage/bootstrap/bootstrapper/fs" - "github.com/m3db/m3/src/dbnode/storage/bootstrap/bootstrapper/peers" - "github.com/m3db/m3/src/dbnode/storage/bootstrap/bootstrapper/uninitialized" - - "github.com/golang/mock/gomock" -) - -// MockBootstrapConfigurationValidator is a mock of BootstrapConfigurationValidator interface -type MockBootstrapConfigurationValidator struct { - ctrl *gomock.Controller - recorder *MockBootstrapConfigurationValidatorMockRecorder -} - -// MockBootstrapConfigurationValidatorMockRecorder is the mock recorder for MockBootstrapConfigurationValidator -type MockBootstrapConfigurationValidatorMockRecorder struct { - mock *MockBootstrapConfigurationValidator -} - -// NewMockBootstrapConfigurationValidator creates a new mock instance -func NewMockBootstrapConfigurationValidator(ctrl *gomock.Controller) *MockBootstrapConfigurationValidator { - mock := &MockBootstrapConfigurationValidator{ctrl: ctrl} - mock.recorder = &MockBootstrapConfigurationValidatorMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockBootstrapConfigurationValidator) EXPECT() *MockBootstrapConfigurationValidatorMockRecorder { - return m.recorder -} - -// ValidateBootstrappersOrder mocks base method -func (m *MockBootstrapConfigurationValidator) ValidateBootstrappersOrder(arg0 []string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ValidateBootstrappersOrder", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// ValidateBootstrappersOrder indicates an expected call of ValidateBootstrappersOrder -func (mr *MockBootstrapConfigurationValidatorMockRecorder) ValidateBootstrappersOrder(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ValidateBootstrappersOrder", reflect.TypeOf((*MockBootstrapConfigurationValidator)(nil).ValidateBootstrappersOrder), arg0) -} - -// ValidateCommitLogBootstrapperOptions mocks base method -func (m *MockBootstrapConfigurationValidator) ValidateCommitLogBootstrapperOptions(arg0 commitlog.Options) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ValidateCommitLogBootstrapperOptions", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// ValidateCommitLogBootstrapperOptions indicates an expected call of ValidateCommitLogBootstrapperOptions -func (mr *MockBootstrapConfigurationValidatorMockRecorder) ValidateCommitLogBootstrapperOptions(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ValidateCommitLogBootstrapperOptions", reflect.TypeOf((*MockBootstrapConfigurationValidator)(nil).ValidateCommitLogBootstrapperOptions), arg0) -} - -// ValidateFilesystemBootstrapperOptions mocks base method -func (m *MockBootstrapConfigurationValidator) ValidateFilesystemBootstrapperOptions(arg0 fs.Options) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ValidateFilesystemBootstrapperOptions", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// ValidateFilesystemBootstrapperOptions indicates an expected call of ValidateFilesystemBootstrapperOptions -func (mr *MockBootstrapConfigurationValidatorMockRecorder) ValidateFilesystemBootstrapperOptions(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ValidateFilesystemBootstrapperOptions", reflect.TypeOf((*MockBootstrapConfigurationValidator)(nil).ValidateFilesystemBootstrapperOptions), arg0) -} - -// ValidatePeersBootstrapperOptions mocks base method -func (m *MockBootstrapConfigurationValidator) ValidatePeersBootstrapperOptions(arg0 peers.Options) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ValidatePeersBootstrapperOptions", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// ValidatePeersBootstrapperOptions indicates an expected call of ValidatePeersBootstrapperOptions -func (mr *MockBootstrapConfigurationValidatorMockRecorder) ValidatePeersBootstrapperOptions(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ValidatePeersBootstrapperOptions", reflect.TypeOf((*MockBootstrapConfigurationValidator)(nil).ValidatePeersBootstrapperOptions), arg0) -} - -// ValidateUninitializedBootstrapperOptions mocks base method -func (m *MockBootstrapConfigurationValidator) ValidateUninitializedBootstrapperOptions(arg0 uninitialized.Options) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ValidateUninitializedBootstrapperOptions", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// ValidateUninitializedBootstrapperOptions indicates an expected call of ValidateUninitializedBootstrapperOptions -func (mr *MockBootstrapConfigurationValidatorMockRecorder) ValidateUninitializedBootstrapperOptions(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ValidateUninitializedBootstrapperOptions", reflect.TypeOf((*MockBootstrapConfigurationValidator)(nil).ValidateUninitializedBootstrapperOptions), arg0) -} diff --git a/src/cmd/services/m3dbnode/config/config_test.go b/src/cmd/services/m3dbnode/config/config_test.go index b67692fa08..95b37b666f 100644 --- a/src/cmd/services/m3dbnode/config/config_test.go +++ b/src/cmd/services/m3dbnode/config/config_test.go @@ -99,10 +99,6 @@ db: writeNewSeriesBackoffDuration: 2ms bootstrap: - bootstrappers: - - filesystem - - peers - - noop-all fs: numProcessorsPerCPU: 0.42 commitlog: @@ -417,10 +413,6 @@ func TestConfiguration(t *testing.T) { writeNewSeriesBackoffDuration: 2ms tick: null bootstrap: - bootstrappers: - - filesystem - - peers - - noop-all fs: numProcessorsPerCPU: 0.42 migration: null @@ -457,7 +449,6 @@ func TestConfiguration(t *testing.T) { calculationType: fixed size: 2097152 queueChannel: null - blockSize: null repair: enabled: false throttle: 2m0s @@ -941,10 +932,6 @@ db: httpNodeListenAddress: 0.0.0.0:9002 httpClusterListenAddress: 0.0.0.0:9003 - bootstrap: - bootstrappers: - - noop-all - commitlog: flushMaxBytes: 524288 flushEvery: 1s @@ -1009,11 +996,6 @@ db: httpClusterListenAddress: 0.0.0.0:9003 bootstrap: - bootstrappers: - - filesystem - - commitlog - - peers - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: ` + notDefaultStr + ` @@ -1039,20 +1021,6 @@ db: require.NoError(t, err) require.NotNil(t, cfg.DB) - validator := NewMockBootstrapConfigurationValidator(ctrl) - validator.EXPECT().ValidateBootstrappersOrder(gomock.Any()).Return(nil).AnyTimes() - validator.EXPECT().ValidateFilesystemBootstrapperOptions(gomock.Any()).Return(nil) - validator.EXPECT().ValidatePeersBootstrapperOptions(gomock.Any()).Return(nil) - validator.EXPECT().ValidateUninitializedBootstrapperOptions(gomock.Any()).Return(nil) - validator.EXPECT(). - ValidateCommitLogBootstrapperOptions(gomock.Any()). - DoAndReturn(func(opts commitlog.Options) error { - actual := opts.ReturnUnfulfilledForCorruptCommitLogFiles() - expected := notDefault - require.Equal(t, expected, actual) - return nil - }) - mapProvider := topology.NewMockMapProvider(ctrl) origin := topology.NewMockHost(ctrl) adminClient := client.NewMockAdminClient(ctrl) diff --git a/src/dbnode/generated/mocks/generate.go b/src/dbnode/generated/mocks/generate.go index 1313816ff6..8da07ecdc5 100644 --- a/src/dbnode/generated/mocks/generate.go +++ b/src/dbnode/generated/mocks/generate.go @@ -26,7 +26,6 @@ //go:generate sh -c "mockgen -package=digest -destination=$GOPATH/src/$PACKAGE/src/dbnode/digest/digest_mock.go $PACKAGE/src/dbnode/digest ReaderWithDigest" //go:generate sh -c "mockgen -package=series $PACKAGE/src/dbnode/storage/series DatabaseSeries,QueryableBlockRetriever | genclean -pkg $PACKAGE/src/dbnode/storage/series -out $GOPATH/src/$PACKAGE/src/dbnode/storage/series/series_mock.go" //go:generate sh -c "mockgen -package=lookup $PACKAGE/src/dbnode/storage/series/lookup OnReleaseReadWriteRef,IndexWriter | genclean -pkg $PACKAGE/src/dbnode/storage/series/lookup -out $GOPATH/src/$PACKAGE/src/dbnode/storage/series/lookup/lookup_mock.go" -//go:generate sh -c "mockgen -package=config $PACKAGE/src/cmd/services/m3dbnode/config BootstrapConfigurationValidator | genclean -pkg $PACKAGE/src/cmd/services/m3dbnode/config -out $GOPATH/src/$PACKAGE/src/cmd/services/m3dbnode/config/config_mock.go" // mockgen rules for generating mocks for unexported interfaces (file mode) //go:generate sh -c "mockgen -package=encoding -destination=$GOPATH/src/$PACKAGE/src/dbnode/encoding/encoding_mock.go -source=$GOPATH/src/$PACKAGE/src/dbnode/encoding/types.go" From 0c6f74ab169fb1e78c448f0d886cd2ee20d362b0 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Wed, 21 Oct 2020 10:56:18 -0400 Subject: [PATCH 20/55] test fix --- src/query/models/config_test.go | 2 +- src/query/models/options_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/query/models/config_test.go b/src/query/models/config_test.go index 7b06f1e01b..e5a4a09548 100644 --- a/src/query/models/config_test.go +++ b/src/query/models/config_test.go @@ -40,7 +40,7 @@ func TestIDSchemeValidation(t *testing.T) { assert.NoError(t, err) err = IDSchemeType(5).Validate() assert.EqualError(t, err, "invalid config id schema type 'unknown':"+ - " should be one of [legacy quoted prepend_meta graphite]") + " should be one of [quoted prepend_meta graphite]") } func TestMetricsTypeUnmarshalYAML(t *testing.T) { diff --git a/src/query/models/options_test.go b/src/query/models/options_test.go index 14e19baed0..ccc65641b3 100644 --- a/src/query/models/options_test.go +++ b/src/query/models/options_test.go @@ -69,7 +69,7 @@ func TestBadBucketTagOptions(t *testing.T) { func TestBadSchemeTagOptions(t *testing.T) { msg := "invalid config id schema type 'unknown': should be one of" + - " [legacy quoted prepend_meta graphite]" + " [quoted prepend_meta graphite]" opts := NewTagOptions(). SetIDSchemeType(IDSchemeType(6)) assert.EqualError(t, opts.Validate(), msg) From 20fb0b82f741becdcfa6b7d9338fc61e978dad98 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Wed, 21 Oct 2020 11:01:18 -0400 Subject: [PATCH 21/55] test fix 2 --- src/query/server/cost_reporters_test.go | 26 +++---------------------- src/query/server/query.go | 5 +++-- src/query/server/query_test.go | 13 +------------ 3 files changed, 7 insertions(+), 37 deletions(-) diff --git a/src/query/server/cost_reporters_test.go b/src/query/server/cost_reporters_test.go index a301123bb8..b6b7c70086 100644 --- a/src/query/server/cost_reporters_test.go +++ b/src/query/server/cost_reporters_test.go @@ -25,7 +25,6 @@ import ( "math" "testing" - "github.com/m3db/m3/src/cmd/services/m3query/config" "github.com/m3db/m3/src/query/cost" "github.com/m3db/m3/src/x/close" "github.com/m3db/m3/src/x/cost/test" @@ -51,16 +50,7 @@ func TestNewConfiguredChainedEnforcer(t *testing.T) { s := tally.NewTestScope("", nil) iopts := instrument.NewOptions().SetMetricsScope(s) - globalEnforcer, closer, err := newConfiguredChainedEnforcer(&config.Configuration{ - Limits: config.LimitsConfiguration{ - PerQuery: config.PerQueryLimitsConfiguration{ - MaxFetchedDatapoints: perQueryLimit, - }, - Global: config.GlobalLimitsConfiguration{ - MaxFetchedDatapoints: globalLimit, - }, - }, - }, iopts) + globalEnforcer, closer, err := newConfiguredChainedEnforcer(iopts) require.NoError(t, err) @@ -132,23 +122,13 @@ func TestNewConfiguredChainedEnforcer(t *testing.T) { qe1, qe2 := tctx.GlobalEnforcer.Child(cost.QueryLevel), tctx.GlobalEnforcer.Child(cost.QueryLevel) r := qe1.Add(6) - test.AssertLimitErrorWithMsg( - t, - r.Error, - "exceeded query limit: exceeded limits.perQuery.maxFetchedDatapoints", - 6, - 6) + assert.NoError(t, r.Error) r = qe2.Add(3) require.NoError(t, r.Error) r = qe2.Add(2) - test.AssertLimitErrorWithMsg( - t, - r.Error, - "exceeded global limit: exceeded limits.global.maxFetchedDatapoints", - 11, - 10) + require.NoError(t, r.Error) test.AssertCurrentCost(t, 11, tctx.GlobalEnforcer) diff --git a/src/query/server/query.go b/src/query/server/query.go index b94366737c..2920536a4d 100644 --- a/src/query/server/query.go +++ b/src/query/server/query.go @@ -424,8 +424,9 @@ func Run(runOpts RunOptions) { logger.Fatal("unrecognized backend", zap.String("backend", string(cfg.Backend))) } - chainedEnforcer, chainedEnforceCloser, err := newConfiguredChainedEnforcer(&cfg, - instrumentOptions) + chainedEnforcer, chainedEnforceCloser, err := newConfiguredChainedEnforcer( + instrumentOptions, + ) if err != nil { logger.Fatal("unable to setup chained enforcer", zap.Error(err)) } diff --git a/src/query/server/query_test.go b/src/query/server/query_test.go index 9563f98f58..fed6ce0e3f 100644 --- a/src/query/server/query_test.go +++ b/src/query/server/query_test.go @@ -520,18 +520,7 @@ func TestNewPerQueryEnforcer(t *testing.T) { SetMetricsScope(scope) setup := func(t *testing.T, globalLimit, queryLimit int) testContext { - cfg := &config.Configuration{ - Limits: config.LimitsConfiguration{ - Global: config.GlobalLimitsConfiguration{ - MaxFetchedDatapoints: 100, - }, - PerQuery: config.PerQueryLimitsConfiguration{ - MaxFetchedDatapoints: 10, - }, - }, - } - - global, closer, err := newConfiguredChainedEnforcer(cfg, instrumentOpts) + global, closer, err := newConfiguredChainedEnforcer(instrumentOpts) require.NoError(t, err) queryLvl := global.Child(cost.QueryLevel) From cd8e97474622db79d4e17cd913630795278504e7 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Wed, 21 Oct 2020 11:31:07 -0400 Subject: [PATCH 22/55] test fix 3 --- kube/bundle.yaml | 4 +--- kube/m3dbnode-configmap.yaml | 4 +--- scripts/comparator/m3query.yml | 4 +--- scripts/development/m3_stack/m3collector.yml | 4 +--- .../m3_stack/m3coordinator-aggregator.yml | 3 +-- .../m3_stack/m3coordinator-standard.yml | 3 +-- .../aggregator/m3coordinator.yml | 3 +-- .../aggregator_legacy/m3coordinator.yml | 3 +-- .../carbon/m3coordinator.yml | 3 +-- .../cold_writes_simple/m3coordinator.yml | 3 +-- .../m3coordinator.yml | 4 ++-- .../coordinator_noop/m3coordinator.yml | 3 +-- .../m3dbnode.yml | 3 +-- .../m3coordinator-cluster-a.yml | 3 +-- .../m3coordinator-cluster-b.yml | 3 +-- .../prometheus/m3coordinator.yml | 3 +-- .../m3coordinator01.yml | 3 +-- .../m3coordinator02.yml | 3 +-- .../query_fanout/m3coordinator-cluster-a.yml | 3 +-- .../query_fanout/m3coordinator-cluster-b.yml | 3 +-- .../query_fanout/m3coordinator-cluster-c.yml | 3 +-- .../repair/m3coordinator.yml | 3 +-- .../m3coordinator-cluster-a.yml | 3 +-- .../m3coordinator-cluster-b.yml | 3 +-- .../replication/m3coordinator-cluster-a.yml | 3 +-- .../replication/m3coordinator-cluster-b.yml | 3 +-- .../simple_v2_batch_apis/m3coordinator.yml | 3 +-- .../m3query/config/testdata/config.yml | 3 +-- .../m3query/config/testdata/config_test.yml | 3 +-- .../resources/config/m3coordinator.yml | 3 +-- .../config/m3coordinator-cluster-template.yml | 3 +-- src/query/config/m3coordinator-local-etcd.yml | 3 +-- src/query/config/m3query-dev-etcd.yml | 3 +-- src/query/config/m3query-local-etcd.yml | 3 +-- src/query/server/query_test.go | 20 ++++++++----------- 35 files changed, 43 insertions(+), 84 deletions(-) diff --git a/kube/bundle.yaml b/kube/bundle.yaml index c1a3d30f1e..4fdf832eae 100644 --- a/kube/bundle.yaml +++ b/kube/bundle.yaml @@ -116,9 +116,7 @@ metadata: data: m3dbnode.yml: |+ coordinator: - listenAddress: - type: "config" - value: "0.0.0.0:7201" + listenAddress: 0.0.0.0:7201 local: namespaces: - namespace: default diff --git a/kube/m3dbnode-configmap.yaml b/kube/m3dbnode-configmap.yaml index 7706004656..46ab73bdf2 100644 --- a/kube/m3dbnode-configmap.yaml +++ b/kube/m3dbnode-configmap.yaml @@ -6,9 +6,7 @@ metadata: data: m3dbnode.yml: |+ coordinator: - listenAddress: - type: "config" - value: "0.0.0.0:7201" + listenAddress: 0.0.0.0:7201 local: namespaces: - namespace: default diff --git a/scripts/comparator/m3query.yml b/scripts/comparator/m3query.yml index 2bd5f8a204..5e67225baa 100644 --- a/scripts/comparator/m3query.yml +++ b/scripts/comparator/m3query.yml @@ -1,6 +1,4 @@ -listenAddress: - type: "config" - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 backend: grpc diff --git a/scripts/development/m3_stack/m3collector.yml b/scripts/development/m3_stack/m3collector.yml index ce940de31d..f3d4c15eca 100644 --- a/scripts/development/m3_stack/m3collector.yml +++ b/scripts/development/m3_stack/m3collector.yml @@ -1,6 +1,4 @@ -listenAddress: - type: config - value: 0.0.0.0:7206 +listenAddress: 0.0.0.0:7206 metrics: scope: diff --git a/scripts/development/m3_stack/m3coordinator-aggregator.yml b/scripts/development/m3_stack/m3coordinator-aggregator.yml index a9a4d40b32..b622668a4a 100644 --- a/scripts/development/m3_stack/m3coordinator-aggregator.yml +++ b/scripts/development/m3_stack/m3coordinator-aggregator.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/development/m3_stack/m3coordinator-standard.yml b/scripts/development/m3_stack/m3coordinator-standard.yml index bf72b395a9..dbe275d3a1 100644 --- a/scripts/development/m3_stack/m3coordinator-standard.yml +++ b/scripts/development/m3_stack/m3coordinator-standard.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/aggregator/m3coordinator.yml b/scripts/docker-integration-tests/aggregator/m3coordinator.yml index 8c730ecfc0..35a42c248f 100644 --- a/scripts/docker-integration-tests/aggregator/m3coordinator.yml +++ b/scripts/docker-integration-tests/aggregator/m3coordinator.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7202" +listenAddress: 0.0.0.0:7202 logging: level: info diff --git a/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml b/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml index 475afe191f..db8feefcd7 100644 --- a/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml +++ b/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7202" +listenAddress: 0.0.0.0:7202 logging: level: info diff --git a/scripts/docker-integration-tests/carbon/m3coordinator.yml b/scripts/docker-integration-tests/carbon/m3coordinator.yml index ff075d84bc..ba69263bc6 100644 --- a/scripts/docker-integration-tests/carbon/m3coordinator.yml +++ b/scripts/docker-integration-tests/carbon/m3coordinator.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml b/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml index 6c6ebd622c..64ae45cdca 100644 --- a/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml +++ b/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml b/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml index 5cf1fa9db2..5e983f62b9 100644 --- a/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml +++ b/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml @@ -1,5 +1,5 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 + value: "" logging: level: info diff --git a/scripts/docker-integration-tests/coordinator_noop/m3coordinator.yml b/scripts/docker-integration-tests/coordinator_noop/m3coordinator.yml index e35b204202..c3c08c0104 100644 --- a/scripts/docker-integration-tests/coordinator_noop/m3coordinator.yml +++ b/scripts/docker-integration-tests/coordinator_noop/m3coordinator.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml b/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml index 6c793f8330..a27219015c 100644 --- a/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml +++ b/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml @@ -1,6 +1,5 @@ coordinator: - listenAddress: - value: "0.0.0.0:7201" + listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-a.yml b/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-a.yml index e31c37acdc..1a165ab6b9 100644 --- a/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-a.yml +++ b/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-a.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-b.yml b/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-b.yml index b13af81125..0f434a0865 100644 --- a/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-b.yml +++ b/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-b.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/prometheus/m3coordinator.yml b/scripts/docker-integration-tests/prometheus/m3coordinator.yml index 6dbb0970d5..f9d46f31b3 100644 --- a/scripts/docker-integration-tests/prometheus/m3coordinator.yml +++ b/scripts/docker-integration-tests/prometheus/m3coordinator.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/prometheus_replication/m3coordinator01.yml b/scripts/docker-integration-tests/prometheus_replication/m3coordinator01.yml index 2a559916c2..83d61c1b74 100644 --- a/scripts/docker-integration-tests/prometheus_replication/m3coordinator01.yml +++ b/scripts/docker-integration-tests/prometheus_replication/m3coordinator01.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/prometheus_replication/m3coordinator02.yml b/scripts/docker-integration-tests/prometheus_replication/m3coordinator02.yml index 9a2d14308f..8d15b050d8 100644 --- a/scripts/docker-integration-tests/prometheus_replication/m3coordinator02.yml +++ b/scripts/docker-integration-tests/prometheus_replication/m3coordinator02.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-a.yml b/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-a.yml index 39879fc86d..a3ef780416 100644 --- a/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-a.yml +++ b/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-a.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-b.yml b/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-b.yml index 4d1dbcfce0..2464d98685 100644 --- a/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-b.yml +++ b/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-b.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-c.yml b/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-c.yml index adbf3ca63d..034036b6ef 100644 --- a/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-c.yml +++ b/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-c.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/repair/m3coordinator.yml b/scripts/docker-integration-tests/repair/m3coordinator.yml index 6c6ebd622c..64ae45cdca 100644 --- a/scripts/docker-integration-tests/repair/m3coordinator.yml +++ b/scripts/docker-integration-tests/repair/m3coordinator.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/repair_and_replication/m3coordinator-cluster-a.yml b/scripts/docker-integration-tests/repair_and_replication/m3coordinator-cluster-a.yml index e31c37acdc..1a165ab6b9 100644 --- a/scripts/docker-integration-tests/repair_and_replication/m3coordinator-cluster-a.yml +++ b/scripts/docker-integration-tests/repair_and_replication/m3coordinator-cluster-a.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/repair_and_replication/m3coordinator-cluster-b.yml b/scripts/docker-integration-tests/repair_and_replication/m3coordinator-cluster-b.yml index b13af81125..0f434a0865 100644 --- a/scripts/docker-integration-tests/repair_and_replication/m3coordinator-cluster-b.yml +++ b/scripts/docker-integration-tests/repair_and_replication/m3coordinator-cluster-b.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/replication/m3coordinator-cluster-a.yml b/scripts/docker-integration-tests/replication/m3coordinator-cluster-a.yml index e31c37acdc..1a165ab6b9 100644 --- a/scripts/docker-integration-tests/replication/m3coordinator-cluster-a.yml +++ b/scripts/docker-integration-tests/replication/m3coordinator-cluster-a.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/replication/m3coordinator-cluster-b.yml b/scripts/docker-integration-tests/replication/m3coordinator-cluster-b.yml index b13af81125..0f434a0865 100644 --- a/scripts/docker-integration-tests/replication/m3coordinator-cluster-b.yml +++ b/scripts/docker-integration-tests/replication/m3coordinator-cluster-b.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml b/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml index 1f9333bdec..68f407d984 100644 --- a/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml +++ b/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/src/cmd/services/m3query/config/testdata/config.yml b/src/cmd/services/m3query/config/testdata/config.yml index 0e27dfe751..916af89329 100644 --- a/src/cmd/services/m3query/config/testdata/config.yml +++ b/src/cmd/services/m3query/config/testdata/config.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/src/cmd/services/m3query/config/testdata/config_test.yml b/src/cmd/services/m3query/config/testdata/config_test.yml index f8e982ed1a..4ff212c097 100644 --- a/src/cmd/services/m3query/config/testdata/config_test.yml +++ b/src/cmd/services/m3query/config/testdata/config_test.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/src/cmd/tools/dtest/docker/harness/resources/config/m3coordinator.yml b/src/cmd/tools/dtest/docker/harness/resources/config/m3coordinator.yml index 25da9de16f..53764669f0 100644 --- a/src/cmd/tools/dtest/docker/harness/resources/config/m3coordinator.yml +++ b/src/cmd/tools/dtest/docker/harness/resources/config/m3coordinator.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/src/query/config/m3coordinator-cluster-template.yml b/src/query/config/m3coordinator-cluster-template.yml index dd6167eb34..79887d3388 100644 --- a/src/query/config/m3coordinator-cluster-template.yml +++ b/src/query/config/m3coordinator-cluster-template.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/src/query/config/m3coordinator-local-etcd.yml b/src/query/config/m3coordinator-local-etcd.yml index 3c4afc7137..0dfefbd45f 100644 --- a/src/query/config/m3coordinator-local-etcd.yml +++ b/src/query/config/m3coordinator-local-etcd.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/src/query/config/m3query-dev-etcd.yml b/src/query/config/m3query-dev-etcd.yml index 09c1278e6b..cdaf363ce9 100644 --- a/src/query/config/m3query-dev-etcd.yml +++ b/src/query/config/m3query-dev-etcd.yml @@ -1,8 +1,7 @@ # m3query configuration for local development setup. Mostly the same as m3query-local-etcd.yml, but using fewer # resources (threads primarily). -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/src/query/config/m3query-local-etcd.yml b/src/query/config/m3query-local-etcd.yml index bc97a8000d..f63b801a41 100644 --- a/src/query/config/m3query-local-etcd.yml +++ b/src/query/config/m3query-local-etcd.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/src/query/server/query_test.go b/src/query/server/query_test.go index fed6ce0e3f..a5ed325584 100644 --- a/src/query/server/query_test.go +++ b/src/query/server/query_test.go @@ -63,9 +63,7 @@ import ( ) var configYAML = ` -listenAddress: - type: "config" - value: "127.0.0.1:0" +listenAddress: 127.0.0.1:0 logging: level: info @@ -411,9 +409,7 @@ func TestGRPCBackend(t *testing.T) { require.NoError(t, err) grpcAddr := lis.Addr().String() var grpcConfigYAML = fmt.Sprintf(` -listenAddress: - type: "config" - value: "127.0.0.1:0" +listenAddress: 127.0.0.1:0 logging: level: info @@ -539,7 +535,7 @@ func TestNewPerQueryEnforcer(t *testing.T) { // Spot check that limits are setup properly for each level. r := tctx.Query.Add(11) - require.Error(t, r.Error) + require.NoError(t, r.Error) floatsEqual := func(f1, f2 float64) { assert.InDelta(t, f1, f2, 0.0000001) @@ -573,14 +569,14 @@ func TestNewPerQueryEnforcer(t *testing.T) { "cost.reporter.over_datapoints_limit+enabled=true,limiter=global": 0, "cost.reporter.datapoints_counter+limiter=global": 11, "cost.reporter.over_datapoints_limit+enabled=false,limiter=query": 0, - "cost.reporter.over_datapoints_limit+enabled=true,limiter=query": 1, + "cost.reporter.over_datapoints_limit+enabled=true,limiter=query": 0, } expectGaugeValues := map[string]float64{ - "cost.limits.threshold+limiter=global": 100, - "cost.limits.enabled+limiter=global": 1, + "cost.limits.threshold+limiter=global": 0, + "cost.limits.enabled+limiter=global": 0, "cost.reporter.datapoints+limiter=global": 11, - "cost.limits.threshold+limiter=query": 10, - "cost.limits.enabled+limiter=query": 1, + "cost.limits.threshold+limiter=query": 0, + "cost.limits.enabled+limiter=query": 0, } snapshot := scope.Snapshot() From 6396cd811a135fc11768038e63e439033498b75c Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Wed, 21 Oct 2020 11:56:25 -0400 Subject: [PATCH 23/55] test fix 4 --- src/query/functions/temporal/base_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/query/functions/temporal/base_test.go b/src/query/functions/temporal/base_test.go index 489bd9211d..b2fa8d1c3c 100644 --- a/src/query/functions/temporal/base_test.go +++ b/src/query/functions/temporal/base_test.go @@ -136,7 +136,7 @@ func testTemporalFunc(t *testing.T, opGen opGenerator, tests []testCase) { test.EqualsWithNansWithDelta(t, tt.expected, sink.Values, 0.0001) metaOne := block.SeriesMeta{ - Name: []byte("t1=v1,"), + Name: []byte("{t1=\"v1\"}"), Tags: models.EmptyTags().AddTags([]models.Tag{{ Name: []byte("t1"), Value: []byte("v1"), @@ -144,7 +144,7 @@ func testTemporalFunc(t *testing.T, opGen opGenerator, tests []testCase) { } metaTwo := block.SeriesMeta{ - Name: []byte("t1=v2,"), + Name: []byte("{t1=\"v2\"}"), Tags: models.EmptyTags().AddTags([]models.Tag{{ Name: []byte("t1"), Value: []byte("v2"), @@ -338,7 +338,7 @@ func testParallelProcess(t *testing.T, warning bool) { for i, m := range sink.Metas { expected := fmt.Sprint(expected[i]) - expectedName := fmt.Sprintf("tag=%s,", expected) + expectedName := fmt.Sprintf("{tag=\"%s\"}", expected) assert.Equal(t, expectedName, string(m.Name)) require.Equal(t, 1, m.Tags.Len()) tag, found := m.Tags.Get([]byte(tagName)) From 17cd9a076339891bf93986ce686f01c78a7cb267 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Wed, 21 Oct 2020 12:29:36 -0400 Subject: [PATCH 24/55] test fix 5 --- .../v1/handler/prometheus/native/read_test.go | 128 ------------------ 1 file changed, 128 deletions(-) diff --git a/src/query/api/v1/handler/prometheus/native/read_test.go b/src/query/api/v1/handler/prometheus/native/read_test.go index 10cb2e5e56..aa6d10ced4 100644 --- a/src/query/api/v1/handler/prometheus/native/read_test.go +++ b/src/query/api/v1/handler/prometheus/native/read_test.go @@ -23,7 +23,6 @@ package native import ( "context" "encoding/json" - "io/ioutil" "net/http" "net/http/httptest" "net/url" @@ -301,133 +300,6 @@ func newTestSetup( } } -func TestPromReadHandlerServeHTTPMaxComputedDatapoints(t *testing.T) { - setup := newTestSetup(timeoutOpts, nil) - opts := setup.Handlers.read.opts - setup.Handlers.read.opts = opts.SetConfig(config.Configuration{ - Limits: config.LimitsConfiguration{ - PerQuery: config.PerQueryLimitsConfiguration{ - PrivateMaxComputedDatapoints: 3599, - }, - }, - }) - - params := defaultParams() - params.Set(startParam, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC). - Format(time.RFC3339Nano)) - params.Set(endParam, time.Date(2018, 1, 1, 1, 0, 0, 0, time.UTC). - Format(time.RFC3339Nano)) - params.Set(handleroptions.StepParam, (time.Second).String()) - req := newReadRequest(t, params) - - recorder := httptest.NewRecorder() - setup.Handlers.read.ServeHTTP(recorder, req) - resp := recorder.Result() - - assert.Equal(t, http.StatusBadRequest, resp.StatusCode) - - d, err := ioutil.ReadAll(resp.Body) - require.NoError(t, err) - - // not a public struct in xhttp, but it's small. - var errResp struct { - Error string `json:"error"` - } - require.NoError(t, json.Unmarshal(d, &errResp)) - - expected := "querying from 2018-01-01 00:00:00 +0000 UTC to 2018-01-01 01:00:00 +0000 UTC with step size 1s " + - "would result in too many datapoints (end - start / step > 3599). Either decrease the query resolution " + - "(?step=XX), decrease the time window, or increase the limit (`limits.maxComputedDatapoints`)" - assert.Equal(t, expected, errResp.Error) -} - -func TestPromReadHandler_validateRequest(t *testing.T) { - dt := func(year int, month time.Month, day, hour int) time.Time { - return time.Date(year, month, day, hour, 0, 0, 0, time.UTC) - } - - cases := []struct { - name string - params models.RequestParams - max int - errorExpected bool - }{{ - name: "under limit", - params: models.RequestParams{ - Step: time.Second, - Start: dt(2018, 1, 1, 0), - End: dt(2018, 1, 1, 1), - }, - max: 3601, - errorExpected: false, - }, { - name: "at limit", - params: models.RequestParams{ - Step: time.Second, - Start: dt(2018, 1, 1, 0), - End: dt(2018, 1, 1, 1), - }, - max: 3600, - errorExpected: false, - }, { - name: "over limit", - params: models.RequestParams{ - Step: time.Second, - Start: dt(2018, 1, 1, 0), - End: dt(2018, 1, 1, 1), - }, - max: 3599, - errorExpected: true, - }, { - name: "large query, limit disabled (0)", - params: models.RequestParams{ - Step: time.Second, - Start: dt(2018, 1, 1, 0), - End: dt(2018, 1, 1, 1), - }, - max: 0, - errorExpected: false, - }, { - name: "large query, limit disabled (negative)", - params: models.RequestParams{ - Step: time.Second, - Start: dt(2018, 1, 1, 0), - End: dt(2018, 1, 1, 1), - }, - max: -50, - errorExpected: false, - }, { - name: "uneven step over limit", - params: models.RequestParams{ - Step: 34 * time.Minute, - Start: dt(2018, 1, 1, 0), - End: dt(2018, 1, 1, 11), - }, - max: 1, - errorExpected: true, - }, { - name: "uneven step under limit", - params: models.RequestParams{ - Step: 34 * time.Minute, - Start: dt(2018, 1, 1, 0), - End: dt(2018, 1, 1, 1), - }, - max: 2, - errorExpected: false}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - err := validateRequest(tc.params, tc.max) - if tc.errorExpected { - require.Error(t, err) - } else { - require.NoError(t, err) - } - }) - } -} - type cancelWatcher struct { delay time.Duration } From fe3eca78e689ee279a7720152a3a33445363a3c2 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Wed, 21 Oct 2020 12:37:04 -0400 Subject: [PATCH 25/55] test fix 6 --- src/query/config/m3query-dev-remote.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/query/config/m3query-dev-remote.yml b/src/query/config/m3query-dev-remote.yml index 49972a8340..560830e9fd 100644 --- a/src/query/config/m3query-dev-remote.yml +++ b/src/query/config/m3query-dev-remote.yml @@ -1,6 +1,4 @@ -listenAddress: - type: "config" - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 backend: grpc From 1d79f40d8d003021d1582c1b5882b2be071eb6a1 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Wed, 21 Oct 2020 12:43:02 -0400 Subject: [PATCH 26/55] test fix 7 --- .../services/m3query/config/config_test.go | 97 +------------------ .../m3query/config/testdata/config.yml | 8 +- 2 files changed, 7 insertions(+), 98 deletions(-) diff --git a/src/cmd/services/m3query/config/config_test.go b/src/cmd/services/m3query/config/config_test.go index a4f9c12dfd..8af9c7de77 100644 --- a/src/cmd/services/m3query/config/config_test.go +++ b/src/cmd/services/m3query/config/config_test.go @@ -26,7 +26,6 @@ import ( "github.com/m3db/m3/src/query/models" xconfig "github.com/m3db/m3/src/x/config" - "github.com/m3db/m3/src/x/cost" xdocs "github.com/m3db/m3/src/x/docs" "github.com/stretchr/testify/assert" @@ -88,103 +87,15 @@ func TestTagOptionsFromConfig(t *testing.T) { } } -func TestLimitsConfigurationAsLimitManagerOptions(t *testing.T) { - cases := []struct { - input interface { - AsLimitManagerOptions() cost.LimitManagerOptions - } - expectedDefault int - }{{ - input: &PerQueryLimitsConfiguration{ - MaxFetchedDatapoints: 5, - }, - expectedDefault: 5, - }, { - input: &GlobalLimitsConfiguration{ - MaxFetchedDatapoints: 6, - }, - expectedDefault: 6, - }} - - for _, tc := range cases { - t.Run(fmt.Sprintf("type_%T", tc.input), func(t *testing.T) { - res := tc.input.AsLimitManagerOptions() - assert.Equal(t, cost.Limit{ - Threshold: cost.Cost(tc.expectedDefault), - Enabled: true, - }, res.DefaultLimit()) - }) - } -} - -func TestLimitsConfigurationMaxComputedDatapoints(t *testing.T) { - t.Run("uses PerQuery value if provided", func(t *testing.T) { - lc := &LimitsConfiguration{ - DeprecatedMaxComputedDatapoints: 6, - PerQuery: PerQueryLimitsConfiguration{ - PrivateMaxComputedDatapoints: 5, - }, - } - - assert.Equal(t, 5, lc.MaxComputedDatapoints()) - }) - - t.Run("uses deprecated value if PerQuery not provided", func(t *testing.T) { - lc := &LimitsConfiguration{ - DeprecatedMaxComputedDatapoints: 6, - } - - assert.Equal(t, 6, lc.MaxComputedDatapoints()) - }) -} - -func TestToLimitManagerOptions(t *testing.T) { - cases := []struct { - name string - input int - expectedLimit cost.Limit - }{{ - name: "negative is disabled", - input: -5, - expectedLimit: cost.Limit{ - Threshold: cost.Cost(-5), - Enabled: false, - }, - }, { - name: "zero is disabled", - input: 0, - expectedLimit: cost.Limit{ - Threshold: cost.Cost(0), - Enabled: false, - }, - }, { - name: "positive is enabled", - input: 5, - expectedLimit: cost.Limit{ - Threshold: cost.Cost(5), - Enabled: true, - }, - }} - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - assert.Equal(t, tc.expectedLimit, toLimitManagerOptions(tc.input).DefaultLimit()) - }) - } -} - func TestConfigLoading(t *testing.T) { var cfg Configuration require.NoError(t, xconfig.LoadFile(&cfg, testConfigFile, xconfig.Options{})) assert.Equal(t, &LimitsConfiguration{ - DeprecatedMaxComputedDatapoints: 10555, PerQuery: PerQueryLimitsConfiguration{ - PrivateMaxComputedDatapoints: 12000, - MaxFetchedDatapoints: 11000, - }, - Global: GlobalLimitsConfiguration{ - MaxFetchedDatapoints: 13000, + MaxFetchedSeries: 12000, + MaxFetchedDocs: 11000, + RequireExhaustive: true, }, }, &cfg.Limits) // TODO: assert on more fields here. @@ -219,7 +130,7 @@ func TestConfigValidation(t *testing.T) { cfg := baseCfg(t) cfg.Limits = LimitsConfiguration{ PerQuery: PerQueryLimitsConfiguration{ - PrivateMaxComputedDatapoints: tc.limit, + MaxFetchedSeries: tc.limit, }} assert.NoError(t, validator.Validate(cfg)) diff --git a/src/cmd/services/m3query/config/testdata/config.yml b/src/cmd/services/m3query/config/testdata/config.yml index 916af89329..5ee0e2ff7b 100644 --- a/src/cmd/services/m3query/config/testdata/config.yml +++ b/src/cmd/services/m3query/config/testdata/config.yml @@ -52,9 +52,7 @@ clusters: backgroundHealthCheckFailThrottleFactor: 0.5 limits: - maxComputedDatapoints: 10555 perQuery: - maxComputedDatapoints: 12000 - maxFetchedDatapoints: 11000 - global: - maxFetchedDatapoints: 13000 + maxFetchedSeries: 12000 + maxFetchedDocs: 11000 + requireExhaustive: true From 9909bb5683acedd45025c5c28719efabb155c694 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Wed, 21 Oct 2020 12:54:19 -0400 Subject: [PATCH 27/55] bootstrap remove from config 1 --- kube/bundle.yaml | 5 ----- kube/m3dbnode-configmap.yaml | 5 ----- scripts/development/m3_stack/m3dbnode.yml | 5 ----- .../dedicated_etcd_embedded_coordinator/m3dbnode.yml | 5 ----- .../multi_cluster_write/m3dbnode-cluster-a.yml | 5 ----- .../multi_cluster_write/m3dbnode-cluster-b.yml | 5 ----- scripts/docker-integration-tests/repair/m3dbnode.yml | 5 ----- .../repair_and_replication/m3dbnode-cluster-a.yml | 5 ----- .../repair_and_replication/m3dbnode-cluster-b.yml | 5 ----- .../replication/m3dbnode-cluster-a.yml | 5 ----- .../replication/m3dbnode-cluster-b.yml | 5 ----- src/cmd/services/m3dbnode/main/main_index_test.go | 11 ++--------- src/cmd/services/m3dbnode/main/main_test.go | 7 ------- .../docker/harness/resources/config/m3dbnode.yml | 5 ----- src/dbnode/config/m3dbnode-all-config.yml | 8 -------- src/dbnode/config/m3dbnode-cluster-template.yml | 5 ----- src/dbnode/config/m3dbnode-local-etcd-proto.yml | 5 ----- src/dbnode/config/m3dbnode-local-etcd.yml | 5 ----- 18 files changed, 2 insertions(+), 99 deletions(-) diff --git a/kube/bundle.yaml b/kube/bundle.yaml index 4fdf832eae..304c20cd6d 100644 --- a/kube/bundle.yaml +++ b/kube/bundle.yaml @@ -167,11 +167,6 @@ data: writeNewSeriesBackoffDuration: 2ms bootstrap: - bootstrappers: - - filesystem - - commitlog - - peers - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: false diff --git a/kube/m3dbnode-configmap.yaml b/kube/m3dbnode-configmap.yaml index 46ab73bdf2..ef1ec932dd 100644 --- a/kube/m3dbnode-configmap.yaml +++ b/kube/m3dbnode-configmap.yaml @@ -57,11 +57,6 @@ data: writeNewSeriesBackoffDuration: 2ms bootstrap: - bootstrappers: - - filesystem - - commitlog - - peers - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: false diff --git a/scripts/development/m3_stack/m3dbnode.yml b/scripts/development/m3_stack/m3dbnode.yml index 2b6ee4ac4a..77a48f3cf9 100644 --- a/scripts/development/m3_stack/m3dbnode.yml +++ b/scripts/development/m3_stack/m3dbnode.yml @@ -39,11 +39,6 @@ db: writeNewSeriesBackoffDuration: 2ms bootstrap: - bootstrappers: - - filesystem - - peers - - commitlog - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: false diff --git a/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml b/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml index a27219015c..c4e5d7b5b7 100644 --- a/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml +++ b/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml @@ -49,11 +49,6 @@ db: writeNewSeriesBackoffDuration: 2ms bootstrap: - # Intentionally disable peers bootstrapper to ensure it doesn't interfere with test. - bootstrappers: - - filesystem - - commitlog - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: false diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml index f724192e96..845a989afc 100644 --- a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml +++ b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml @@ -60,11 +60,6 @@ db: writeNewSeriesBackoffDuration: 2ms bootstrap: - # Intentionally disable peers bootstrapper to ensure it doesn't interfere with test. - bootstrappers: - - filesystem - - commitlog - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: false diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml index fa32e79853..18c5efd475 100644 --- a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml +++ b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml @@ -39,11 +39,6 @@ db: writeNewSeriesBackoffDuration: 2ms bootstrap: - # Intentionally disable peers bootstrapper to ensure it doesn't interfere with test. - bootstrappers: - - filesystem - - commitlog - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: false diff --git a/scripts/docker-integration-tests/repair/m3dbnode.yml b/scripts/docker-integration-tests/repair/m3dbnode.yml index 6bdae716a7..7cb50aac0c 100644 --- a/scripts/docker-integration-tests/repair/m3dbnode.yml +++ b/scripts/docker-integration-tests/repair/m3dbnode.yml @@ -39,11 +39,6 @@ db: writeNewSeriesBackoffDuration: 2ms bootstrap: - # Intentionally disable peers bootstrapper to ensure it doesn't interfere with test. - bootstrappers: - - filesystem - - commitlog - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: false diff --git a/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-a.yml b/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-a.yml index 002f7dd1cf..288e19431a 100644 --- a/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-a.yml +++ b/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-a.yml @@ -39,11 +39,6 @@ db: writeNewSeriesBackoffDuration: 2ms bootstrap: - # Intentionally disable peers bootstrapper to ensure it doesn't interfere with test. - bootstrappers: - - filesystem - - commitlog - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: false diff --git a/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-b.yml b/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-b.yml index 59dcc55b6b..27b7814dee 100644 --- a/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-b.yml +++ b/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-b.yml @@ -39,11 +39,6 @@ db: writeNewSeriesBackoffDuration: 2ms bootstrap: - # Intentionally disable peers bootstrapper to ensure it doesn't interfere with test. - bootstrappers: - - filesystem - - commitlog - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: false diff --git a/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml b/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml index aca38c3af9..6d5492d226 100644 --- a/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml +++ b/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml @@ -39,11 +39,6 @@ db: writeNewSeriesBackoffDuration: 2ms bootstrap: - # Intentionally disable peers bootstrapper to ensure it doesn't interfere with test. - bootstrappers: - - filesystem - - commitlog - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: false diff --git a/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml b/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml index 558e511304..b45bcadf30 100644 --- a/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml +++ b/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml @@ -39,11 +39,6 @@ db: writeNewSeriesBackoffDuration: 2ms bootstrap: - # Intentionally disable peers bootstrapper to ensure it doesn't interfere with test. - bootstrappers: - - filesystem - - commitlog - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: false diff --git a/src/cmd/services/m3dbnode/main/main_index_test.go b/src/cmd/services/m3dbnode/main/main_index_test.go index b5022112af..88222fee41 100644 --- a/src/cmd/services/m3dbnode/main/main_index_test.go +++ b/src/cmd/services/m3dbnode/main/main_index_test.go @@ -348,15 +348,8 @@ db: writeNewSeriesAsync: false writeNewSeriesLimitPerSecond: 1048576 - writeNewSeriesBackoffDuration: 2ms - - bootstrap: - bootstrappers: - - filesystem - - commitlog - - peers - - uninitialized_topology - + writeNewSeriesBackoffDuration: 2ms + commitlog: flushMaxBytes: 524288 flushEvery: 1s diff --git a/src/cmd/services/m3dbnode/main/main_test.go b/src/cmd/services/m3dbnode/main/main_test.go index 690f639d24..f24a644d31 100644 --- a/src/cmd/services/m3dbnode/main/main_test.go +++ b/src/cmd/services/m3dbnode/main/main_test.go @@ -506,13 +506,6 @@ db: writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms - bootstrap: - bootstrappers: - - filesystem - - commitlog - - peers - - uninitialized_topology - commitlog: flushMaxBytes: 524288 flushEvery: 1s diff --git a/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml b/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml index 6970916644..108cc53907 100644 --- a/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml +++ b/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml @@ -60,11 +60,6 @@ db: writeNewSeriesBackoffDuration: 2ms bootstrap: - bootstrappers: - - filesystem - - commitlog - - peers - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: false diff --git a/src/dbnode/config/m3dbnode-all-config.yml b/src/dbnode/config/m3dbnode-all-config.yml index 4b62a4d565..63541b2061 100644 --- a/src/dbnode/config/m3dbnode-all-config.yml +++ b/src/dbnode/config/m3dbnode-all-config.yml @@ -108,14 +108,6 @@ db: writeNewSeriesBackoffDuration: 2ms bootstrap: - # Order in which to run the bootstrappers. Don't change these values unless - # you know what you're doing as non-standard configurations can cause data - # loss or make recovery from disaster scenarios difficult. - bootstrappers: - - filesystem - - commitlog - - peers - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: false diff --git a/src/dbnode/config/m3dbnode-cluster-template.yml b/src/dbnode/config/m3dbnode-cluster-template.yml index 23b5c9b223..79e2097c1a 100644 --- a/src/dbnode/config/m3dbnode-cluster-template.yml +++ b/src/dbnode/config/m3dbnode-cluster-template.yml @@ -78,11 +78,6 @@ db: writeNewSeriesBackoffDuration: 2ms bootstrap: - bootstrappers: - - filesystem - - commitlog - - peers - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: false diff --git a/src/dbnode/config/m3dbnode-local-etcd-proto.yml b/src/dbnode/config/m3dbnode-local-etcd-proto.yml index 7cae28ed10..0b7dba5a43 100644 --- a/src/dbnode/config/m3dbnode-local-etcd-proto.yml +++ b/src/dbnode/config/m3dbnode-local-etcd-proto.yml @@ -60,11 +60,6 @@ db: writeNewSeriesBackoffDuration: 2ms bootstrap: - bootstrappers: - - filesystem - - commitlog - - peers - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: false diff --git a/src/dbnode/config/m3dbnode-local-etcd.yml b/src/dbnode/config/m3dbnode-local-etcd.yml index 6970916644..108cc53907 100644 --- a/src/dbnode/config/m3dbnode-local-etcd.yml +++ b/src/dbnode/config/m3dbnode-local-etcd.yml @@ -60,11 +60,6 @@ db: writeNewSeriesBackoffDuration: 2ms bootstrap: - bootstrappers: - - filesystem - - commitlog - - peers - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: false From 3ed1002aa3ed14e756b7828415b11475a20e989d Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Wed, 21 Oct 2020 13:50:14 -0400 Subject: [PATCH 28/55] for config changes 1 --- config/m3db/local-etcd/generated.yaml | 2 - config/m3db/local-etcd/m3dbnode.libsonnet | 3 - .../m3query/config/testdata/config_test.yml | 6 +- .../harness/resources/config/m3dbnode.yml | 3 - src/dbnode/config/m3dbnode-all-config.yml | 3 - .../config/m3dbnode-local-etcd-proto.yml | 3 - src/dbnode/config/m3dbnode-local-etcd.yml | 3 - .../v1/handler/prometheus/native/read_test.go | 60 ------------------- src/query/generated/assets/openapi/spec.yml | 54 ----------------- 9 files changed, 2 insertions(+), 135 deletions(-) diff --git a/config/m3db/local-etcd/generated.yaml b/config/m3db/local-etcd/generated.yaml index d82bacfc16..cbacb9e2ce 100644 --- a/config/m3db/local-etcd/generated.yaml +++ b/config/m3db/local-etcd/generated.yaml @@ -1,6 +1,4 @@ "coordinator": - "limits": - "maxComputedDatapoints": 10000 "listenAddress": "type": "config" "value": "0.0.0.0:7201" diff --git a/config/m3db/local-etcd/m3dbnode.libsonnet b/config/m3db/local-etcd/m3dbnode.libsonnet index c99ced7d9f..19b20a9009 100644 --- a/config/m3db/local-etcd/m3dbnode.libsonnet +++ b/config/m3db/local-etcd/m3dbnode.libsonnet @@ -28,9 +28,6 @@ function(coordinator={}, db={}) { "samplingRate": 1.0, "extended": "none" }, - "limits": { - "maxComputedDatapoints": 10000 - }, "tagOptions": { // Configuration setting for generating metric IDs from tags. "idScheme": "quoted" diff --git a/src/cmd/services/m3query/config/testdata/config_test.yml b/src/cmd/services/m3query/config/testdata/config_test.yml index 4ff212c097..68250c3f0d 100644 --- a/src/cmd/services/m3query/config/testdata/config_test.yml +++ b/src/cmd/services/m3query/config/testdata/config_test.yml @@ -53,7 +53,5 @@ clusters: limits: perQuery: - maxComputedDatapoints: 12000 - maxFetchedDatapoints: 11000 - global: - maxFetchedDatapoints: 13000 + maxFetchedSeries: 12000 + maxFetchedDocs: 11000 diff --git a/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml b/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml index 108cc53907..472f1741da 100644 --- a/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml +++ b/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml @@ -21,9 +21,6 @@ coordinator: samplingRate: 1.0 extended: none - limits: - maxComputedDatapoints: 10000 - tagOptions: # Configuration setting for generating metric IDs from tags. idScheme: quoted diff --git a/src/dbnode/config/m3dbnode-all-config.yml b/src/dbnode/config/m3dbnode-all-config.yml index 63541b2061..ca69283310 100644 --- a/src/dbnode/config/m3dbnode-all-config.yml +++ b/src/dbnode/config/m3dbnode-all-config.yml @@ -30,9 +30,6 @@ coordinator: samplingRate: 1.0 extended: none - limits: - maxComputedDatapoints: 10000 - tagOptions: # Configuration setting for generating metric IDs from tags. idScheme: quoted diff --git a/src/dbnode/config/m3dbnode-local-etcd-proto.yml b/src/dbnode/config/m3dbnode-local-etcd-proto.yml index 0b7dba5a43..87a0788315 100644 --- a/src/dbnode/config/m3dbnode-local-etcd-proto.yml +++ b/src/dbnode/config/m3dbnode-local-etcd-proto.yml @@ -21,9 +21,6 @@ coordinator: samplingRate: 1.0 extended: none - limits: - maxComputedDatapoints: 10000 - tagOptions: # Configuration setting for generating metric IDs from tags. idScheme: quoted diff --git a/src/dbnode/config/m3dbnode-local-etcd.yml b/src/dbnode/config/m3dbnode-local-etcd.yml index 108cc53907..472f1741da 100644 --- a/src/dbnode/config/m3dbnode-local-etcd.yml +++ b/src/dbnode/config/m3dbnode-local-etcd.yml @@ -21,9 +21,6 @@ coordinator: samplingRate: 1.0 extended: none - limits: - maxComputedDatapoints: 10000 - tagOptions: # Configuration setting for generating metric IDs from tags. idScheme: quoted diff --git a/src/query/api/v1/handler/prometheus/native/read_test.go b/src/query/api/v1/handler/prometheus/native/read_test.go index aa6d10ced4..9bd936afe7 100644 --- a/src/query/api/v1/handler/prometheus/native/read_test.go +++ b/src/query/api/v1/handler/prometheus/native/read_test.go @@ -22,9 +22,7 @@ package native import ( "context" - "encoding/json" "net/http" - "net/http/httptest" "net/url" "testing" "time" @@ -170,64 +168,6 @@ func testPromReadHandlerRead( } } -type M3QLResp []struct { - Target string `json:"target"` - Tags map[string]string `json:"tags"` - Datapoints [][]float64 `json:"datapoints"` - StepSizeMs int `json:"step_size_ms"` -} - -func TestM3PromReadHandlerRead(t *testing.T) { - testM3PromReadHandlerRead(t, block.NewResultMetadata(), "") - testM3PromReadHandlerRead(t, buildWarningMeta("foo", "bar"), "foo_bar") - testM3PromReadHandlerRead(t, block.ResultMetadata{Exhaustive: false}, - headers.LimitHeaderSeriesLimitApplied) -} - -func testM3PromReadHandlerRead( - t *testing.T, - resultMeta block.ResultMetadata, - ex string, -) { - values, bounds := test.GenerateValuesAndBounds(nil, nil) - - setup := newTestSetup(timeoutOpts, nil) - promRead := setup.Handlers.read - - seriesMeta := test.NewSeriesMeta("dummy", len(values)) - meta := block.Metadata{ - Bounds: bounds, - Tags: models.NewTags(0, models.NewTagOptions()), - ResultMetadata: resultMeta, - } - - b := test.NewBlockFromValuesWithMetaAndSeriesMeta(meta, seriesMeta, values) - setup.Storage.SetFetchBlocksResult(block.Result{Blocks: []block.Block{b}}, nil) - - req, _ := http.NewRequest("GET", PromReadURL, nil) - req.Header.Add(headers.RenderFormat, "m3ql") - req.URL.RawQuery = defaultParams().Encode() - - recorder := httptest.NewRecorder() - promRead.ServeHTTP(recorder, req) - - header := recorder.Header().Get(headers.LimitHeader) - assert.Equal(t, ex, header) - - var m3qlResp M3QLResp - require.NoError(t, json.Unmarshal(recorder.Body.Bytes(), &m3qlResp)) - - assert.Len(t, m3qlResp, 2) - assert.Equal(t, "dummy0", m3qlResp[0].Target) - assert.Equal(t, map[string]string{"__name__": "dummy0", "dummy0": "dummy0"}, - m3qlResp[0].Tags) - assert.Equal(t, 10000, m3qlResp[0].StepSizeMs) - assert.Equal(t, "dummy1", m3qlResp[1].Target) - assert.Equal(t, map[string]string{"__name__": "dummy1", "dummy1": "dummy1"}, - m3qlResp[1].Tags) - assert.Equal(t, 10000, m3qlResp[1].StepSizeMs) -} - func newReadRequest(t *testing.T, params url.Values) *http.Request { req, err := http.NewRequest("GET", PromReadURL, nil) require.NoError(t, err) diff --git a/src/query/generated/assets/openapi/spec.yml b/src/query/generated/assets/openapi/spec.yml index 2efbc9eec9..2fbc4bf634 100644 --- a/src/query/generated/assets/openapi/spec.yml +++ b/src/query/generated/assets/openapi/spec.yml @@ -617,60 +617,6 @@ paths: description: "" schema: $ref: "#/definitions/GenericError" - /services/m3db/database/config/bootstrappers: - post: - tags: - - "M3DB Database" - summary: "Dynamically override the bootstrappers configured in M3DBs node-level configuration" - operationId: "databaseConfigSetBootstrappers" - consumes: - - "application/json" - produces: - - "application/json" - parameters: - - name: "body" - in: "body" - schema: - $ref: "#/definitions/DatabaseConfigBootstrappers" - responses: - 200: - description: "" - schema: - $ref: "#/definitions/DatabaseConfigBootstrappers" - 400: - description: "" - schema: - $ref: "#/definitions/GenericError" - 500: - description: "" - schema: - $ref: "#/definitions/GenericError" - get: - tags: - - "M3DB Database" - summary: "Retrieve the dynamically configured bootstrappers override, if any" - operationId: "databaseConfigGetBootstrappers" - consumes: - - "application/json" - produces: - - "application/json" - responses: - 200: - description: "" - schema: - $ref: "#/definitions/DatabaseConfigBootstrappers" - 400: - description: "" - schema: - $ref: "#/definitions/GenericError" - 404: - description: "not found if not set" - schema: - $ref: "#/definitions/GenericError" - 500: - description: "" - schema: - $ref: "#/definitions/GenericError" definitions: NamespaceAddRequest: type: "object" From f15e43eb4f3fcda1b6e95915c80ff3d5e49cf3a8 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Wed, 21 Oct 2020 14:03:00 -0400 Subject: [PATCH 29/55] for config changes 2 --- scripts/vagrant/provision/manifests/m3coordinator-two.yaml | 4 +--- site/content/docs/how_to/prometheus.md | 4 +--- site/content/docs/integrations/prometheus.md | 4 +--- site/content/docs/m3query/config/annotated_config.yaml | 4 +--- .../tools/dtest/docker/harness/resources/config/m3dbnode.yml | 3 +-- src/collector/config/m3collector.yml | 4 +--- src/dbnode/config/m3dbnode-all-config.yml | 3 +-- src/dbnode/config/m3dbnode-cluster-template.yml | 3 +-- src/dbnode/config/m3dbnode-local-etcd-proto.yml | 3 +-- src/dbnode/config/m3dbnode-local-etcd.yml | 3 +-- 10 files changed, 10 insertions(+), 25 deletions(-) diff --git a/scripts/vagrant/provision/manifests/m3coordinator-two.yaml b/scripts/vagrant/provision/manifests/m3coordinator-two.yaml index c837a2ab43..d12e18489e 100644 --- a/scripts/vagrant/provision/manifests/m3coordinator-two.yaml +++ b/scripts/vagrant/provision/manifests/m3coordinator-two.yaml @@ -7,9 +7,7 @@ metadata: name: m3coordinator-dedicated-test-cluster data: coordinator.yaml: |+ - listenAddress: - type: "config" - value: "0.0.0.0:7201" + listenAddress: 0.0.0.0:7201 metrics: scope: prefix: "coordinator" diff --git a/site/content/docs/how_to/prometheus.md b/site/content/docs/how_to/prometheus.md index 6e4853f96c..a40f850d36 100644 --- a/site/content/docs/how_to/prometheus.md +++ b/site/content/docs/how_to/prometheus.md @@ -29,9 +29,7 @@ Start by downloading the config template. Update the namespaces and the client s You'll need to specify the static IPs or hostnames of your M3DB seed nodes, and the name and retention values of the namespace you set up. You can leave the namespace storage metrics type as unaggregated since it's required by default to have a cluster that receives all Prometheus metrics unaggregated. In the future you might also want to aggregate and downsample metrics for longer retention, and you can come back and update the config once you've setup those clusters. You can read more about our aggregation functionality here. It should look something like: -listenAddress: - type: "config" - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/site/content/docs/integrations/prometheus.md b/site/content/docs/integrations/prometheus.md index eb74b5dcee..512a3e9ae5 100644 --- a/site/content/docs/integrations/prometheus.md +++ b/site/content/docs/integrations/prometheus.md @@ -16,9 +16,7 @@ You'll need to specify the static IPs or hostnames of your M3DB seed nodes, and It should look something like: ```yaml -listenAddress: - type: "config" - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/site/content/docs/m3query/config/annotated_config.yaml b/site/content/docs/m3query/config/annotated_config.yaml index 4c7080847e..3b5da983d2 100644 --- a/site/content/docs/m3query/config/annotated_config.yaml +++ b/site/content/docs/m3query/config/annotated_config.yaml @@ -1,6 +1,4 @@ -listenAddress: - type: "config" - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml b/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml index 472f1741da..68a9e17844 100644 --- a/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml +++ b/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml @@ -1,6 +1,5 @@ coordinator: - listenAddress: - value: "0.0.0.0:7201" + listenAddress: 0.0.0.0:7201 local: namespaces: diff --git a/src/collector/config/m3collector.yml b/src/collector/config/m3collector.yml index 15bf496756..b555eeccd7 100644 --- a/src/collector/config/m3collector.yml +++ b/src/collector/config/m3collector.yml @@ -1,6 +1,4 @@ -listenAddress: - type: config - value: 0.0.0.0:7206 +listenAddress: 0.0.0.0:7206 metrics: scope: diff --git a/src/dbnode/config/m3dbnode-all-config.yml b/src/dbnode/config/m3dbnode-all-config.yml index ca69283310..4157d0885b 100644 --- a/src/dbnode/config/m3dbnode-all-config.yml +++ b/src/dbnode/config/m3dbnode-all-config.yml @@ -1,8 +1,7 @@ # Include this field if you want to enable an embedded M3Coordinator instance. coordinator: # Address for M3Coordinator to listen for traffic. - listenAddress: - value: "0.0.0.0:7201" + listenAddress: 0.0.0.0:7201 # All configured M3DB namespaces must be listed in this config if running an # embedded M3Coordinator instance. diff --git a/src/dbnode/config/m3dbnode-cluster-template.yml b/src/dbnode/config/m3dbnode-cluster-template.yml index 79e2097c1a..94bc808052 100644 --- a/src/dbnode/config/m3dbnode-cluster-template.yml +++ b/src/dbnode/config/m3dbnode-cluster-template.yml @@ -1,6 +1,5 @@ coordinator: - listenAddress: - value: "0.0.0.0:7201" + listenAddress: 0.0.0.0:7201 local: namespaces: diff --git a/src/dbnode/config/m3dbnode-local-etcd-proto.yml b/src/dbnode/config/m3dbnode-local-etcd-proto.yml index 87a0788315..5226f907c0 100644 --- a/src/dbnode/config/m3dbnode-local-etcd-proto.yml +++ b/src/dbnode/config/m3dbnode-local-etcd-proto.yml @@ -1,6 +1,5 @@ coordinator: - listenAddress: - value: "0.0.0.0:7201" + listenAddress: 0.0.0.0:7201 local: namespaces: diff --git a/src/dbnode/config/m3dbnode-local-etcd.yml b/src/dbnode/config/m3dbnode-local-etcd.yml index 472f1741da..68a9e17844 100644 --- a/src/dbnode/config/m3dbnode-local-etcd.yml +++ b/src/dbnode/config/m3dbnode-local-etcd.yml @@ -1,6 +1,5 @@ coordinator: - listenAddress: - value: "0.0.0.0:7201" + listenAddress: 0.0.0.0:7201 local: namespaces: From ebeef926f35fc314a4b2523d2ac26c4fa25e2bd0 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Wed, 21 Oct 2020 15:16:08 -0400 Subject: [PATCH 30/55] for config changes 3 --- config/m3db/clustered-etcd/generated.yaml | 5 ----- config/m3db/clustered-etcd/m3dbnode.libsonnet | 6 ------ config/m3db/local-etcd/generated.yaml | 5 ----- config/m3db/local-etcd/m3dbnode.libsonnet | 6 ------ kube/terraform/main.tf | 2 +- src/query/generated/assets/openapi/spec.yml | 7 ------- 6 files changed, 1 insertion(+), 30 deletions(-) diff --git a/config/m3db/clustered-etcd/generated.yaml b/config/m3db/clustered-etcd/generated.yaml index ee47a4f7cd..701d2379a4 100644 --- a/config/m3db/clustered-etcd/generated.yaml +++ b/config/m3db/clustered-etcd/generated.yaml @@ -22,11 +22,6 @@ "idScheme": "quoted" "db": "bootstrap": - "bootstrappers": - - "filesystem" - - "commitlog" - - "peers" - - "uninitialized_topology" "commitlog": "returnUnfulfilledForCorruptCommitLogFiles": false "cache": diff --git a/config/m3db/clustered-etcd/m3dbnode.libsonnet b/config/m3db/clustered-etcd/m3dbnode.libsonnet index 4e1833c33f..a3439439b5 100644 --- a/config/m3db/clustered-etcd/m3dbnode.libsonnet +++ b/config/m3db/clustered-etcd/m3dbnode.libsonnet @@ -104,12 +104,6 @@ function(cluster, coordinator={}, db={}) { "writeNewSeriesLimitPerSecond": 1048576, "writeNewSeriesBackoffDuration": "2ms", "bootstrap": { - "bootstrappers": [ - "filesystem", - "commitlog", - "peers", - "uninitialized_topology" - ], "commitlog": { "returnUnfulfilledForCorruptCommitLogFiles": false } diff --git a/config/m3db/local-etcd/generated.yaml b/config/m3db/local-etcd/generated.yaml index cbacb9e2ce..a223c1e69b 100644 --- a/config/m3db/local-etcd/generated.yaml +++ b/config/m3db/local-etcd/generated.yaml @@ -22,11 +22,6 @@ "idScheme": "quoted" "db": "bootstrap": - "bootstrappers": - - "filesystem" - - "commitlog" - - "peers" - - "uninitialized_topology" "commitlog": "returnUnfulfilledForCorruptCommitLogFiles": false "cache": diff --git a/config/m3db/local-etcd/m3dbnode.libsonnet b/config/m3db/local-etcd/m3dbnode.libsonnet index 19b20a9009..a04ab6fb3a 100644 --- a/config/m3db/local-etcd/m3dbnode.libsonnet +++ b/config/m3db/local-etcd/m3dbnode.libsonnet @@ -63,12 +63,6 @@ function(coordinator={}, db={}) { "writeNewSeriesLimitPerSecond": 1048576, "writeNewSeriesBackoffDuration": "2ms", "bootstrap": { - "bootstrappers": [ - "filesystem", - "commitlog", - "peers", - "uninitialized_topology" - ], "commitlog": { "returnUnfulfilledForCorruptCommitLogFiles": false } diff --git a/kube/terraform/main.tf b/kube/terraform/main.tf index 4003e3f637..9a4ea68bce 100755 --- a/kube/terraform/main.tf +++ b/kube/terraform/main.tf @@ -133,7 +133,7 @@ resource "kubernetes_config_map" "m3dbnode_config" { namespace = "m3db" } data { - m3dbnode.yml = "coordinator:\n listenAddress:\n type: \"config\"\n value: \"0.0.0.0:7201\"\n local:\n namespaces:\n - namespace: default\n type: unaggregated\n retention: 48h\n metrics:\n scope:\n prefix: \"coordinator\"\n prometheus:\n handlerPath: /metrics\n listenAddress: 0.0.0.0:7203\n sanitization: prometheus\n samplingRate: 1.0\n extended: none\n tagOptions:\n idScheme: quoted\n\ndb:\n logging:\n level: info\n\n metrics:\n prometheus:\n handlerPath: /metrics\n sanitization: prometheus\n samplingRate: 1.0\n extended: detailed\n\n listenAddress: 0.0.0.0:9000\n clusterListenAddress: 0.0.0.0:9001\n httpNodeListenAddress: 0.0.0.0:9002\n httpClusterListenAddress: 0.0.0.0:9003\n debugListenAddress: 0.0.0.0:9004\n\n hostID:\n resolver: hostname\n\n client:\n writeConsistencyLevel: majority\n readConsistencyLevel: unstrict_majority\n\n gcPercentage: 100\n\n writeNewSeriesAsync: true\n writeNewSeriesLimitPerSecond: 1048576\n writeNewSeriesBackoffDuration: 2ms\n\n bootstrap:\n bootstrappers:\n - filesystem\n - commitlog\n - peers\n - uninitialized_topology\n fs:\n numProcessorsPerCPU: 0.125\n commitlog:\n returnUnfulfilledForCorruptCommitLogFiles: false\n\n commitlog:\n flushMaxBytes: 524288\n flushEvery: 1s\n queue:\n calculationType: fixed\n size: 2097152\n\n fs:\n filePathPrefix: /var/lib/m3db\n\n config:\n service:\n env: default_env\n zone: embedded\n service: m3db\n cacheDir: /var/lib/m3kv\n etcdClusters:\n - zone: embedded\n endpoints:\n - http://etcd-0.etcd:2379\n - http://etcd-1.etcd:2379\n - http://etcd-2.etcd:2379\n" + m3dbnode.yml = "coordinator:\n listenAddress:\n type: \"config\"\n value: \"0.0.0.0:7201\"\n local:\n namespaces:\n - namespace: default\n type: unaggregated\n retention: 48h\n metrics:\n scope:\n prefix: \"coordinator\"\n prometheus:\n handlerPath: /metrics\n listenAddress: 0.0.0.0:7203\n sanitization: prometheus\n samplingRate: 1.0\n extended: none\n tagOptions:\n idScheme: quoted\n\ndb:\n logging:\n level: info\n\n metrics:\n prometheus:\n handlerPath: /metrics\n sanitization: prometheus\n samplingRate: 1.0\n extended: detailed\n\n listenAddress: 0.0.0.0:9000\n clusterListenAddress: 0.0.0.0:9001\n httpNodeListenAddress: 0.0.0.0:9002\n httpClusterListenAddress: 0.0.0.0:9003\n debugListenAddress: 0.0.0.0:9004\n\n hostID:\n resolver: hostname\n\n client:\n writeConsistencyLevel: majority\n readConsistencyLevel: unstrict_majority\n\n gcPercentage: 100\n\n writeNewSeriesAsync: true\n writeNewSeriesLimitPerSecond: 1048576\n writeNewSeriesBackoffDuration: 2ms\n\n bootstrap:\n fs:\n numProcessorsPerCPU: 0.125\n commitlog:\n returnUnfulfilledForCorruptCommitLogFiles: false\n\n commitlog:\n flushMaxBytes: 524288\n flushEvery: 1s\n queue:\n calculationType: fixed\n size: 2097152\n\n fs:\n filePathPrefix: /var/lib/m3db\n\n config:\n service:\n env: default_env\n zone: embedded\n service: m3db\n cacheDir: /var/lib/m3kv\n etcdClusters:\n - zone: embedded\n endpoints:\n - http://etcd-0.etcd:2379\n - http://etcd-1.etcd:2379\n - http://etcd-2.etcd:2379\n" } } diff --git a/src/query/generated/assets/openapi/spec.yml b/src/query/generated/assets/openapi/spec.yml index 2fbc4bf634..20730c6a67 100644 --- a/src/query/generated/assets/openapi/spec.yml +++ b/src/query/generated/assets/openapi/spec.yml @@ -915,10 +915,3 @@ definitions: $ref: "#/definitions/NamespaceGetResponse" placement: $ref: "#/definitions/PlacementGetResponse" - DatabaseConfigBootstrappers: - type: "object" - properties: - values: - type: "array" - items: - type: "string" From cb04ded5cd93676dd61575ecd5da984f6909f8f4 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Wed, 21 Oct 2020 15:38:26 -0400 Subject: [PATCH 31/55] integration test fixes --- CHANGELOG.md | 2 +- scripts/development/m3_stack/start_m3.sh | 16 +++++++-------- scripts/docker-integration-tests/common.sh | 20 +++++++++---------- .../docker-integration-tests/simple/test.sh | 18 ++++++++--------- scripts/m3dbnode_bootstrapped.sh | 4 ++-- site/content/docs/how_to/kubernetes.md | 6 +++--- site/content/docs/how_to/single_node.md | 2 +- .../namespace_configuration.md | 8 ++++---- .../placement_configuration.md | 2 +- src/cmd/tools/m3ctl/namespaces/types.go | 2 +- 10 files changed, 40 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d66198616..53969f97d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -485,7 +485,7 @@ If you run into any issues with the upgrade or need to downgrade to a previous v ## Features - **M3Query**: Add multi-zone and multi-region configuration for coordinator ([#1687](https://github.com/m3db/m3/pull/1687)) -- **M3Query**: Add debug param to `GET` `/api/v1/namespace` endpoint for better readability ([#1698](https://github.com/m3db/m3/pull/1698)) +- **M3Query**: Add debug param to `GET` `/api/v1/services/m3db/namespace` endpoint for better readability ([#1698](https://github.com/m3db/m3/pull/1698)) - **M3Coordinator**: Add "ingest_latency" histogram metric and return datapoint too old/new errors with offending timestamps ([#1716](https://github.com/m3db/m3/pull/1716)) ## Performance diff --git a/scripts/development/m3_stack/start_m3.sh b/scripts/development/m3_stack/start_m3.sh index 43b7c52deb..29d3fa9e97 100755 --- a/scripts/development/m3_stack/start_m3.sh +++ b/scripts/development/m3_stack/start_m3.sh @@ -172,7 +172,7 @@ else fi echo "Initializing namespaces" -curl -vvvsSf -X POST localhost:7201/api/v1/namespace -d '{ +curl -vvvsSf -X POST localhost:7201/api/v1/services/m3db/namespace -d '{ "name": "metrics_0_30m", "options": { "bootstrapEnabled": true, @@ -202,7 +202,7 @@ curl -vvvsSf -X POST localhost:7201/api/v1/namespace -d '{ } } }' -curl -vvvsSf -X POST localhost:7201/api/v1/namespace -d '{ +curl -vvvsSf -X POST localhost:7201/api/v1/services/m3db/namespace -d '{ "name": "metrics_30s_24h", "options": { "bootstrapEnabled": true, @@ -239,13 +239,13 @@ curl -vvvsSf -X POST localhost:7201/api/v1/namespace -d '{ echo "Done initializing namespaces" echo "Validating namespace" -[ "$(curl -sSf localhost:7201/api/v1/namespace | jq .registry.namespaces.metrics_0_30m.indexOptions.enabled)" == true ] -[ "$(curl -sSf localhost:7201/api/v1/namespace | jq .registry.namespaces.metrics_30s_24h.indexOptions.enabled)" == true ] +[ "$(curl -sSf localhost:7201/api/v1/services/m3db/namespace | jq .registry.namespaces.metrics_0_30m.indexOptions.enabled)" == true ] +[ "$(curl -sSf localhost:7201/api/v1/services/m3db/namespace | jq .registry.namespaces.metrics_30s_24h.indexOptions.enabled)" == true ] echo "Done validating namespace" echo "Initializing topology" if [[ "$USE_MULTI_DB_NODES" = true ]] ; then - curl -vvvsSf -X POST localhost:7201/api/v1/placement/init -d '{ + curl -vvvsSf -X POST localhost:7201/api/v1/services/m3db/placement/init -d '{ "num_shards": 64, "replication_factor": 3, "instances": [ @@ -279,7 +279,7 @@ if [[ "$USE_MULTI_DB_NODES" = true ]] ; then ] }' else - curl -vvvsSf -X POST localhost:7201/api/v1/placement/init -d '{ + curl -vvvsSf -X POST localhost:7201/api/v1/services/m3db/placement/init -d '{ "num_shards": 64, "replication_factor": 1, "instances": [ @@ -297,12 +297,12 @@ else fi echo "Validating topology" -[ "$(curl -sSf localhost:7201/api/v1/placement | jq .placement.instances.m3db_seed.id)" == '"m3db_seed"' ] +[ "$(curl -sSf localhost:7201/api/v1/services/m3db/placement | jq .placement.instances.m3db_seed.id)" == '"m3db_seed"' ] echo "Done validating topology" echo "Waiting until shards are marked as available" ATTEMPTS=100 TIMEOUT=2 retry_with_backoff \ - '[ "$(curl -sSf 0.0.0.0:7201/api/v1/placement | grep -c INITIALIZING)" -eq 0 ]' + '[ "$(curl -sSf 0.0.0.0:7201/api/v1/services/m3db/placement | grep -c INITIALIZING)" -eq 0 ]' if [[ "$USE_AGGREGATOR" = true ]]; then echo "Initializing M3Coordinator topology" diff --git a/scripts/docker-integration-tests/common.sh b/scripts/docker-integration-tests/common.sh index df27a44042..ab54fd4c3a 100644 --- a/scripts/docker-integration-tests/common.sh +++ b/scripts/docker-integration-tests/common.sh @@ -53,7 +53,7 @@ function setup_single_m3db_node_long_namespaces { echo "Wait for API to be available" ATTEMPTS=100 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \ - '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/namespace | jq ".namespaces | length")" == "0" ]' + '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace | jq ".namespaces | length")" == "0" ]' echo "Adding placement and agg namespace" curl -vvvsSf -X POST 0.0.0.0:${coordinator_port}/api/v1/database/create -d '{ @@ -76,7 +76,7 @@ function setup_single_m3db_node_long_namespaces { echo "Wait until placement is init'd" ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \ - '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/placement | jq .placement.instances.'${dbnode_id}'.id)" == \"'${dbnode_id}'\" ]' + '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/placement | jq .placement.instances.'${dbnode_id}'.id)" == \"'${dbnode_id}'\" ]' wait_for_namespaces @@ -88,7 +88,7 @@ function setup_single_m3db_node_long_namespaces { echo "Wait until agg2d namespace is init'd" ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \ - '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/namespace | jq .registry.namespaces.agg2d.indexOptions.enabled)" == true ]' + '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace | jq .registry.namespaces.agg2d.indexOptions.enabled)" == true ]' echo "Wait until bootstrapped" @@ -106,7 +106,7 @@ function setup_single_m3db_node { echo "Wait for API to be available" ATTEMPTS=100 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \ - '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/namespace | jq ".namespaces | length")" == "0" ]' + '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace | jq ".namespaces | length")" == "0" ]' echo "Adding placement and agg namespace" curl -vvvsSf -X POST 0.0.0.0:${coordinator_port}/api/v1/database/create -d '{ @@ -129,7 +129,7 @@ function setup_single_m3db_node { echo "Wait until placement is init'd" ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \ - '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/placement | jq .placement.instances.'${dbnode_id}'.id)" == \"'${dbnode_id}'\" ]' + '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/placement | jq .placement.instances.'${dbnode_id}'.id)" == \"'${dbnode_id}'\" ]' wait_for_namespaces @@ -150,7 +150,7 @@ function setup_two_m3db_nodes { echo "Wait for API to be available" ATTEMPTS=100 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \ - '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/namespace | jq ".namespaces | length")" == "0" ]' + '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace | jq ".namespaces | length")" == "0" ]' echo "Adding placement and agg namespace" curl -vvvsSf -X POST 0.0.0.0:${coordinator_port}/api/v1/database/create -d '{ @@ -181,7 +181,7 @@ function setup_two_m3db_nodes { echo "Wait until placement is init'd" ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \ - '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/placement | jq .placement.instances.'"${dbnode_id_1}"'.id)" == \"'"${dbnode_id_1}"'\" ]' + '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/placement | jq .placement.instances.'"${dbnode_id_1}"'.id)" == \"'"${dbnode_id_1}"'\" ]' wait_for_namespaces @@ -197,7 +197,7 @@ function wait_for_namespaces { echo "Wait until agg namespace is init'd" ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \ - '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/namespace | jq .registry.namespaces.agg.indexOptions.enabled)" == true ]' + '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace | jq .registry.namespaces.agg.indexOptions.enabled)" == true ]' echo "Adding unagg namespace" curl -vvvsSf -X POST 0.0.0.0:${coordinator_port}/api/v1/database/namespace/create -d '{ @@ -207,7 +207,7 @@ function wait_for_namespaces { echo "Wait until unagg namespace is init'd" ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \ - '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/namespace | jq .registry.namespaces.unagg.indexOptions.enabled)" == true ]' + '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace | jq .registry.namespaces.unagg.indexOptions.enabled)" == true ]' echo "Adding coldWritesRepairAndNoIndex namespace" curl -vvvsSf -X POST 0.0.0.0:${coordinator_port}/api/v1/services/m3db/namespace -d '{ @@ -233,6 +233,6 @@ function wait_for_namespaces { echo "Wait until coldWritesRepairAndNoIndex namespace is init'd" ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \ - '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/namespace | jq .registry.namespaces.coldWritesRepairAndNoIndex.coldWritesEnabled)" == true ]' + '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace | jq .registry.namespaces.coldWritesRepairAndNoIndex.coldWritesEnabled)" == true ]' } diff --git a/scripts/docker-integration-tests/simple/test.sh b/scripts/docker-integration-tests/simple/test.sh index 118eab927e..d85be4fa92 100755 --- a/scripts/docker-integration-tests/simple/test.sh +++ b/scripts/docker-integration-tests/simple/test.sh @@ -37,7 +37,7 @@ ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \ 'curl -vvvsSf 0.0.0.0:7201/health' echo "Adding namespace" -curl -vvvsSf -X POST 0.0.0.0:7201/api/v1/namespace -d '{ +curl -vvvsSf -X POST 0.0.0.0:7201/api/v1/services/m3db/namespace -d '{ "name": "agg", "options": { "bootstrapEnabled": true, @@ -63,9 +63,9 @@ curl -vvvsSf -X POST 0.0.0.0:7201/api/v1/namespace -d '{ echo "Sleep until namespace is init'd" ATTEMPTS=4 TIMEOUT=1 retry_with_backoff \ - '[ "$(curl -sSf 0.0.0.0:7201/api/v1/namespace | jq .registry.namespaces.agg.indexOptions.enabled)" == true ]' + '[ "$(curl -sSf 0.0.0.0:7201/api/v1/services/m3db/namespace | jq .registry.namespaces.agg.indexOptions.enabled)" == true ]' -curl -vvvsSf -X POST 0.0.0.0:7201/api/v1/namespace -d '{ +curl -vvvsSf -X POST 0.0.0.0:7201/api/v1/services/m3db/namespace -d '{ "name": "unagg", "options": { "bootstrapEnabled": true, @@ -91,10 +91,10 @@ curl -vvvsSf -X POST 0.0.0.0:7201/api/v1/namespace -d '{ echo "Sleep until namespace is init'd" ATTEMPTS=4 TIMEOUT=1 retry_with_backoff \ - '[ "$(curl -sSf 0.0.0.0:7201/api/v1/namespace | jq .registry.namespaces.unagg.indexOptions.enabled)" == true ]' + '[ "$(curl -sSf 0.0.0.0:7201/api/v1/services/m3db/namespace | jq .registry.namespaces.unagg.indexOptions.enabled)" == true ]' echo "Placement initialization" -curl -vvvsSf -X POST 0.0.0.0:7201/api/v1/placement/init -d '{ +curl -vvvsSf -X POST 0.0.0.0:7201/api/v1/services/m3db/placement/init -d '{ "num_shards": 4, "replication_factor": 1, "instances": [ @@ -112,7 +112,7 @@ curl -vvvsSf -X POST 0.0.0.0:7201/api/v1/placement/init -d '{ echo "Sleep until placement is init'd" ATTEMPTS=4 TIMEOUT=1 retry_with_backoff \ - '[ "$(curl -sSf 0.0.0.0:7201/api/v1/placement | jq .placement.instances.m3db_local.id)" == \"m3db_local\" ]' + '[ "$(curl -sSf 0.0.0.0:7201/api/v1/services/m3db/placement | jq .placement.instances.m3db_local.id)" == \"m3db_local\" ]' echo "Sleep until bootstrapped" ATTEMPTS=7 TIMEOUT=2 retry_with_backoff \ @@ -120,7 +120,7 @@ ATTEMPTS=7 TIMEOUT=2 retry_with_backoff \ echo "Waiting until shards are marked as available" ATTEMPTS=10 TIMEOUT=1 retry_with_backoff \ - '[ "$(curl -sSf 0.0.0.0:7201/api/v1/placement | grep -c INITIALIZING)" -eq 0 ]' + '[ "$(curl -sSf 0.0.0.0:7201/api/v1/services/m3db/placement | grep -c INITIALIZING)" -eq 0 ]' echo "Write data" curl -vvvsS -X POST 0.0.0.0:9003/writetagged -d '{ @@ -163,7 +163,7 @@ else fi echo "Deleting placement" -curl -vvvsSf -X DELETE 0.0.0.0:7201/api/v1/placement +curl -vvvsSf -X DELETE 0.0.0.0:7201/api/v1/services/m3db/placement echo "Deleting namespace" -curl -vvvsSf -X DELETE 0.0.0.0:7201/api/v1/namespace/unagg +curl -vvvsSf -X DELETE 0.0.0.0:7201/api/v1/services/m3db/namespace/unagg diff --git a/scripts/m3dbnode_bootstrapped.sh b/scripts/m3dbnode_bootstrapped.sh index 85647ebde8..8a5e22657d 100755 --- a/scripts/m3dbnode_bootstrapped.sh +++ b/scripts/m3dbnode_bootstrapped.sh @@ -12,8 +12,8 @@ set -o pipefail COORDINATOR_PORT=${COORDINATOR_PORT:-7201} DBNODE_PORT=${DBNODE_PORT:-9002} -COORD_PLACEMENT_ENDPOINT="http://localhost:${COORDINATOR_PORT}/api/v1/placement" -COORD_NAMESPACE_ENDPOINT="http://localhost:${COORDINATOR_PORT}/api/v1/namespace" +COORD_PLACEMENT_ENDPOINT="http://localhost:${COORDINATOR_PORT}/api/v1/services/m3db/placement" +COORD_NAMESPACE_ENDPOINT="http://localhost:${COORDINATOR_PORT}/api/v1/services/m3db/namespace" DBNODE_ENDPOINT="http://localhost:${DBNODE_PORT}/health" HOSTNAME=${HOSTNAME:-$(hostname)} diff --git a/site/content/docs/how_to/kubernetes.md b/site/content/docs/how_to/kubernetes.md index 863192a398..ed98cfe66f 100644 --- a/site/content/docs/how_to/kubernetes.md +++ b/site/content/docs/how_to/kubernetes.md @@ -95,7 +95,7 @@ Forwarding from [::1]:7201 -> 7201 ```shell # Create an initial cluster topology -curl -sSf -X POST localhost:7201/api/v1/placement/init -d '{ +curl -sSf -X POST localhost:7201/api/v1/services/m3db/placement/init -d '{ "num_shards": 1024, "replication_factor": 3, "instances": [ @@ -132,7 +132,7 @@ curl -sSf -X POST localhost:7201/api/v1/placement/init -d '{ ```shell # Create a namespace to hold your metrics -curl -X POST localhost:7201/api/v1/namespace -d '{ +curl -X POST localhost:7201/api/v1/services/m3db/namespace -d '{ "name": "default", "options": { "bootstrapEnabled": true, @@ -262,7 +262,7 @@ Forwarding from [::1]:7201 -> 7201 ``` ```shell -curl -sSf -X POST localhost:7201/api/v1/placement -d '{ +curl -sSf -X POST localhost:7201/api/v1/services/m3db/placement -d '{ "instances": [ { "id": "m3dbnode-3", diff --git a/site/content/docs/how_to/single_node.md b/site/content/docs/how_to/single_node.md index de4e39c97b..d88aa26a26 100644 --- a/site/content/docs/how_to/single_node.md +++ b/site/content/docs/how_to/single_node.md @@ -40,7 +40,7 @@ curl -X POST http://localhost:7201/api/v1/database/create -d '{ Placement initialization may take a minute or two and you can check on the status of this by running the following: ```shell -curl http://localhost:7201/api/v1/placement | jq . +curl http://localhost:7201/api/v1/services/m3db/placement | jq . ``` Once all of the shards become `AVAILABLE`, you should see your node complete bootstrapping! Don't worry if you see warnings or errors related to a local cache file, such as `[W] could not load cache from file diff --git a/site/content/docs/operational_guide/namespace_configuration.md b/site/content/docs/operational_guide/namespace_configuration.md index 9f9f41202e..ac711d1f4f 100644 --- a/site/content/docs/operational_guide/namespace_configuration.md +++ b/site/content/docs/operational_guide/namespace_configuration.md @@ -42,7 +42,7 @@ The "advanced" API allows you to configure every aspect of the namespace that yo Adding a namespace is a simple as using the `POST` `api/v1/namespace` API on an M3Coordinator instance. ```shell -curl -X POST :/api/v1/namespace -d '{ +curl -X POST :/api/v1/services/m3db/namespace -d '{ "name": "default_unaggregated", "options": { "bootstrapEnabled": true, @@ -71,9 +71,9 @@ Adding a namespace does not require restarting M3DB, but will require modifying ### Deleting a Namespace -Deleting a namespace is a simple as using the `DELETE` `/api/v1/namespace` API on an M3Coordinator instance. +Deleting a namespace is a simple as using the `DELETE` `/api/v1/services/m3db/namespace` API on an M3Coordinator instance. -`curl -X DELETE :/api/v1/namespace/` +`curl -X DELETE :/api/v1/services/m3db/namespace/` Note that deleting a namespace will not have any effect on the M3DB nodes until they are all restarted. In addition, the namespace will need to be removed from the M3Coordinator configuration and then the M3Coordinator node will need to be restarted. @@ -85,7 +85,7 @@ Also, be very careful not to restart the M3DB nodes after deleting the namespace ### Viewing a Namespace -In order to view a namespace and its attributes, use the `GET` `/api/v1/namespace` API on a M3Coordinator instance. +In order to view a namespace and its attributes, use the `GET` `/api/v1/services/m3db/namespace` API on a M3Coordinator instance. Additionally, for readability/debugging purposes, you can add the `debug=true` parameter to the URL to view block sizes, buffer sizes, etc. in duration format as opposed to nanoseconds (default). diff --git a/site/content/docs/operational_guide/placement_configuration.md b/site/content/docs/operational_guide/placement_configuration.md index b56e829739..079f615dc0 100644 --- a/site/content/docs/operational_guide/placement_configuration.md +++ b/site/content/docs/operational_guide/placement_configuration.md @@ -265,7 +265,7 @@ If the placement for `M3DB` needs to be recreated, the `/api/v1/services/m3db/pl Please note, a placement already needs to exist to use this endpoint. If no placement exists, use the `Placement Initialization` endpoint described above. Also, as mentioned above, this endpoint creates an entirely new placement therefore complete placement information needs to be passed into the body of the request. The recommended way to this -is to get the existing placement using `/api/v1/placement` and modify that (as the `placement` field) along +is to get the existing placement using `/api/v1/services/m3db/placement` and modify that (as the `placement` field) along with two additional fields -- `version` and `confirm`. Please see below for a full example: ```shell diff --git a/src/cmd/tools/m3ctl/namespaces/types.go b/src/cmd/tools/m3ctl/namespaces/types.go index 8046270391..a4f44f6c5d 100644 --- a/src/cmd/tools/m3ctl/namespaces/types.go +++ b/src/cmd/tools/m3ctl/namespaces/types.go @@ -22,7 +22,7 @@ package namespaces const ( // DefaultPath is the default url path for the namespace api calls - DefaultPath = "/api/v1/namespace" + DefaultPath = "/api/v1/services/m3db/namespace" // DebugQS this is the query string to activate debug output in api responses DebugQS = "debug=true" ) From 8490406107b97c7820d21ccd99bd9bf094407e93 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Wed, 21 Oct 2020 15:58:45 -0400 Subject: [PATCH 32/55] more test fixing 1 --- src/cmd/services/m3dbnode/main/common_test.go | 2 +- src/cmd/services/m3dbnode/main/main_index_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cmd/services/m3dbnode/main/common_test.go b/src/cmd/services/m3dbnode/main/common_test.go index 9766515e7f..c1f744f9e7 100644 --- a/src/cmd/services/m3dbnode/main/common_test.go +++ b/src/cmd/services/m3dbnode/main/common_test.go @@ -35,8 +35,8 @@ import ( "github.com/m3db/m3/src/cluster/shard" "github.com/m3db/m3/src/dbnode/client" - "github.com/m3db/m3/src/dbnode/retention" "github.com/m3db/m3/src/dbnode/namespace" + "github.com/m3db/m3/src/dbnode/retention" "github.com/m3db/m3/src/x/ident" "github.com/gogo/protobuf/proto" diff --git a/src/cmd/services/m3dbnode/main/main_index_test.go b/src/cmd/services/m3dbnode/main/main_index_test.go index 88222fee41..f727f254c0 100644 --- a/src/cmd/services/m3dbnode/main/main_index_test.go +++ b/src/cmd/services/m3dbnode/main/main_index_test.go @@ -348,8 +348,8 @@ db: writeNewSeriesAsync: false writeNewSeriesLimitPerSecond: 1048576 - writeNewSeriesBackoffDuration: 2ms - + writeNewSeriesBackoffDuration: 2ms + commitlog: flushMaxBytes: 524288 flushEvery: 1s From 8be56c3a94611d4ebf445eb87193e63fba69b9ba Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Wed, 21 Oct 2020 15:59:43 -0400 Subject: [PATCH 33/55] more test fixing 1 --- src/query/generated/assets/openapi/assets.go | 77 ++++++++++---------- 1 file changed, 37 insertions(+), 40 deletions(-) diff --git a/src/query/generated/assets/openapi/assets.go b/src/query/generated/assets/openapi/assets.go index 73f79c953d..3ea4391484 100644 --- a/src/query/generated/assets/openapi/assets.go +++ b/src/query/generated/assets/openapi/assets.go @@ -239,48 +239,45 @@ d8dUBmQZxiF/+uI1I7E8TMF6pCyWxDpY7WPA8pmKZsl6ouawOaOS+Sj+BQAA//8by2IcfAIAAA== "/spec.yml": { local: "openapi/spec.yml", - size: 24401, + size: 22713, modtime: 12345, compressed: ` -H4sIAAAAAAAC/+xcW2/jNhZ+969gNfuwBTZRZjLbBfzmxGnGQCZjJGmBbbFAafJIZiuRGl6S8RT73wtK -sixZsi6242Rc5SWJdHh4Lt+5kJT0Bt1+ergaojvD0W8h/gMQVgr0iQ/85LMBufgNMQ8thEHJTb5AZI65 -DwppgfScKeSxAL4bqCfs+yCHyHl3euYMGPfEcICQZjqAIXI+no8vnAFCFBSRLNJM8CFyRogypSWbGQ0U -aRYCUiAZKESxxjOsABnFuI8+nj/c/4K8QGD9w3tERBhJUIoJfor+KwwimCOPcYqE0SgUEhCe2T/trAhr -9Otc60gNXZcKok7Dczo7ZcL93z8rL3+PhESCo1+vmf5gZisqn+m5mZ0SEbqW1g3Pvz+1Oj2CVIk+b0/P -rPIIEcE1JtpaACGOw8QEF2N0LYQfALqWwkROfNfIYIicbA57Q536MVk8lSekCd033yW/7cR2XMAIcAWF -CUYRJnNAN8kt9C4RpTRDSQt3FoiZG2KlQbo3k8ur2/srZzAXStthQumY/3/enb11BtYnU6znQ+S4OGLu -41tnoLGvhoOTlZ7jC3SLQ1ARJlB2+qXgHvONTPw6vojHxbTKWeMyDTCBELhuwSVa0ha5jHxfgo+1kO25 -5cZs4Dq+QOMUoWVmhdsnT4wC8gwn9q5yBorMIYTYYLFPnEGE9VxZT7oK5CMjoBLPZHZJvOxDiieEEouj -9Oekyub2R5kwxHIxRM416LKtEyIRgcRWtgkdIie7fw16SUEEVyYWOZsPR1HASDzM/V0JviSNpKCGtCKV -oCLBFeQUeXd2tvpn3apO7k5sQ5ynRegfErwhct64FDzGWWxt9zanzl064YrR+7P3e57vGjhIRq6kFHLF -4N9716s8T2TDtQIeLcAxohRhvoaPBniMKH1eeERY4hA0yBxxGn4zQRcrqzFeulQ2Yz04RpTewWcDSr8q -cJ4dDzjNttj8KaJYQ2d4JsOOB6GJPj1In2+eTbXX/TP7czL+f8KYQgAatkT0OB7cGdHJsBdDdM4Ia8C2 -3cvqkoTPhkmgQ6SlgeyyXkSWi+31uX9Q/CZ2i/s7GcYqHx69x9NnrEVJ1h3XdqgnVd18uT/Vc1jr5KtD -Irt9HC3qNKdOOcG+htaxxm9p68i40pgTSPYEoLUHj6GLnOaUeYkCXY+fY+oim8tuDVLTsts5ySTjRkFw -BKmmrhYetnYQISRlHGshuxSRy9Wwsq/XKeqqTJ5RX25eUbnZ0cN9Perr0WupRztCuVCwtslXfeV6jsqF -s7OBLoVr0ylEBUFd2Rr5fpP7Q0vU16xD1qydnJttvlvfdqxbRV/3xasvXnsrXjthulC62uasaV+3DrVb -59oRw133fiZ2Whywr1usqu3Y40lWVps+W70AjiXEf+8M5buET3Yok9VhxjutH1M+x4PsVKEe3C+4LdY2 -W++43iyl823WnEeb1yuN18fBQeOgfbbfMRQK5aCCss/+PeoPtLXUNvnvtF4rpf7Oa7a+n+9Bv0fQt8/0 -O+G+kOfLhHWA73N9D/u9LWP/XK43Ozxh2PyoQ2kx60kRdlrOvvAzhyurfFuPHPYd+9bA3s+Z6Xrf3odA -HwIv1Ml0joB9HLyUzxPb4j7WYtqDvwd/935m+cquSyRg3XJTPv8WZRHLy1cyQSVtzBPTcxQKpZGItVCI -godNoIFWY3kpz2UszjffqY8L6rxEn74uwd8Q2TEm3ZkQWmmJoyhz+PY4Hy84DhnBQbBA4hGkZDTZhinM -gsgyHChiSWOvEBcUTgJ4hCC7XThj3hAPMek96Iv8BMcTH7F6lbodNko2y3FcT5I0vPWzGfl3oCWDxwTt -NBcGOawXg2AZH/9CzEOYL9oA/fqgQP/7Qaz25TYuNPKE4dQ6zP6jVg8kfgPwzt2xYyteGE9Ypr2mmP0O -JNUvkhaUmq2QEOe7+vY07WxWVPWv5H6K0m9K5EQrvCn8zNJVMK5mnkSGBm4ZfFpnU8NqM7sCyylIJug4 -LX/rZBvUiTkYrlkIlSLV2/6uMLLgggK3VsbPstwVx7MAaMnEMyECwFmW8QKj5i1pnyTToB7EpQhDpm+E -3zSA2H9MW1EkRJjJ1sSbMFBp7Ls18qyEcBypudAtZ2Wcwpd2M05ypJnQGzDSFR93lcq3wkcj0CsBPgsE -+eOefYW29MbzQP5otJHdhkyx0l1ksvXr6kvE5KLJdWvkI0+DvBV6RAgo1doYkxIAWlkd2sGri5k3wKOD -UHHoxwox7k9BXk5/uhScGCmBk7I9uQlnkKuYnpAh1rZJEmYWQCFJbOaKtmBb9eGITpD3mdJ5hDTEW0pf -tHKBSes6mHwgqbHOYUpjKXAwrSxRrYt3+SnhDgIn+4m1KK3akOoww9p7Ls37XksTLT+Mti4a4xr8Svgw -rs/fFUTuIOdyv/CZPDdJ2ecqn10C/IiJFrJJR27C+zmWVDURMhXTNacdYrRdDj2wir6t1sA/vF/N9ZHZ -Rrd5shB/icW6Bz2hddMtjdTFbbShZjAlgjgq4q/mNRB/Fbypj30C5s91k9GA00gwrhuYqWqvYilxfltD -Q9iMsNjEBcaN9rY/2Sf66iWNhNRtXNd97fBte/B5zFc4gNynLVuY57CKx6jdTcM1xymNNTQUmyQd6dzO -vhJGEpg02S/Nm7eYC7Vt4rQ8PG9rFivZC2bLywnchMnNE+RMbicPk9HN5JfJ7bWzvDj6eTS5GV3cXGVX -bq5GP6cUFe/R7aWQbpXW1iIjW8QKSaBV25J7Hu/VadG6sFd1Obk2wk7SrpWobZeKj3F1sFYA+JFxf5Kd -uXY3W3W4YU4Zxfo1gmnrHP3syMpvQ3ZZsa7oKx1SeYa4zeLotrlsxBcHDTtwqzSX5rBAEBw4+SskMEpv -0Ug/a7wVNmUqW/ANWzENVe1iSZev0XuC2QeRYOuiKEsrx+tmFeFLBEQDvY8/4G2RFvcgagrygzBymzr5 -Qey3DcWUSlBqx37nBRva/Zu4+kx/m5TQdqOm4qGYzhsMazxqzsM6aPKIAwM7V72/AgAA//8jiOAHUV8A -AA== +H4sIAAAAAAAC/+xcX2/jNhJ/96dgtfdwfUicTfZ6gN+cOM0ayGaNJC1wLQ4oTY5kthKpJYfJZov77gdK +tixZsiXZzp815Jc44nDImflxfkNK8jty8/n+ckBurSR/RPQvINQYwKMA5NEXC/rpDyJ88qQsSRvlE2Ez +KgMwBBXBmTDEFyH80DOPNAhAD4h3enzi9YT01aBHCAoMYUC8T2ejc69HCAfDtIhRKDkg3pBwYVCLqUXg +BEUExIAWYAinSKfUALFGyIB8Oru/+434oaL40wfCVBRrMEYoeUz+oyxhVBJfSE6URRIpDYRO3Vc3KqFI +fp8hxmbQ73PFzHF0xqfHQvX/+8/Kyz8SpYmS5PcrgR/tdCkVCJzZ6TFTUd/J9qOzH4+dTQ+gTWrP++MT +ZzwhTEmkDJ0HCJE0Sl1wPiJXSgUhkCutbOwlrVaHA+JlY7gGcxwkYslQvtI26r/7If3rBnb9QsFAGigM +MIwpmwG5TpvIaTqV0gglK/rTUE37ETUIun89vri8ubv0ejNl0HVTBhP9/z49ee/1XEwmFGcD4vVpLPoP +770e0sAMekdLO0fn5IZGYGLKoBz0CyV9EVidxnV0nvRLZI23omUSUgYRSGygJV7IFrUMg0BDQFHp5tpy +fdZoHZ2T0RyhZWWF5qNHwYH4VjLXaryeYTOIIHFYEhOvF1OcGRfJvgH9IBiYNDKZX9IoBzDHEyGpx8n8 +c1Tlc/cxNoqofhoQ7wqw7OtUSMWgqZvbmA+Il7VfAS4kmJLGJlPOxqNxHAqWdOv/aZRciMZaccsaiWow +sZIGcoacnpws/1n1qpdrSXxI87KE/EODPyDeuz4HX0iReLt/kzPndj7gUtGHkw97Hu8KJGjBLrVWeqng +X3u3qzxO7JZrBTwagGPIOaFyBR818Bhy/rzwiKmmESDonPB8+U0Vf1p6TcjSpbIbN4NjyPktfLFg8E2B +8+RwwGm3xeYvMacIreGZdjschKb2dCB9vnHWcW//7+zrePS/VDGHEBC2RPQo6dwa0Wm3V0N0zgkrwHbV +y/KShi9WaOADgtpCdhmfYqfF1foyeFH8pn5L6jsdJSa/PHoPp85YWSVZdbyxQj2qqubL9SnOYKWSr14S +WfNhlKiTnDnlBPsWSscNcZuXjkIapJJBeiYAjSN4CFXkJGfMaxD0ZvwcUhVZT7sbkDqn3dZJJu03DMMD +SDWbuPBluYMppbmQFJVuQyIXy27lWK9KbGKZvKKObt4Q3ewY4Y6POj56K3y0I5QLhLVNvuqY6zmYi2b3 +BtoQ17q7EBUCm2hrGAR14Y+cUMdZL8lZOwU3O3x3sW3JW8VYd+TVkdfeyGsnTBeoq2nOmnS89VKndX3X +Y7Dr2c/YDUtD8W2LXbXrezjJylnTZatXwLGG5PvOUL5N9WQ3ZTIeFrLV/nGu53CQPTeoA/crHos1zdY7 +7jdL6XybPefB5vVK53Xr4EXXQfNsv+NSKNBBhWSX/TvUv9DRUtPkv9N+rZT6W+/Zunq+A/0eQd880++E ++0KeLwtuAnyX6zvY720b+/div9niCcP6Rx1Km1lfq6jVdvaVnzlceuX7euSwq9i3BvZ+7pmu1u3dEuiW +wCtVMq1XwD5uvJTvJzbFfWLFpAN/B/729czild0+00Cx4aF8/i3KIpYXr2SCScuYR4EzEimDRCVWGMLB +pzZE4NVYXsznIpnOd1+pjwrmvEadvjqDQ0V2rsX1rXhNMFU5zzBq+ieweSRi7TCIYhmIBAWbk9Icz0up +zS9ifY7nbxLnplZ4P+yZZ1ehuFp5CkwE6RR8XlWzQdV6dQWVE9BC8ZFN1/yq2BpzEg1WooigckqbfX9b +6FkIQUFbI+dPlUKDmsaXkk5D4CUXT5UKgWZJxg+tmTWUfdQCwdyrCxVFAq9VUNeBuX9s06loiKnQjYXX +YaDS2bcr4hk1SBqbmcKGowrJ4WuzEcc50WzSazDSFh+3lcY3wkct0CsBPg0V++tOfIOm8tb3Qf9s0ep2 +XSbUYJs5Ofq4/BoL/VQXuhXxoY+gbxQOGQNjGjtjXAJAI69DM3i1cfMaeLSYVLL0E4OEDCagLya/XCjJ +rNYgWdmf0kZTyDGmr3RE0dVEyk5DKCSJ9VrJFmqrXhduBflAGMwjpGa9zeWLXi4oacyD6c9i1PIc5TyZ +BQ0nlRTVmLzLz4a1mHC6i9yI0qptSIsRVp5urt/tLFy0+Dmc1akJiRBUwkdIPDstTLnFPBe7xGeK3Hiu +Psd8bgfwM2WodJ2N0kZ3M6q5qRMUJpGrTzvMonoAfS8q6raNDv7pw3KsT8IVuvWDRfRrMq07wDHfNNzC +SW3Cxms4QxgVJqsi+a2kGuFvStbVsY8gghnWOQ0kj5WQWKPMVEeVak3zmz2EqB5hiYsLimv97T7ZDzNt +nmmsNDYJXfu9w/cdwedxX+HYeZ++bOCelzU8Qe1uFq4EziBFqCGbNB1h7jzHKKsZjOv8N8+bN1Qqs23i +dDp8f2sVy7kX3JafJ0gbpY1HxBvfjO/Hw+vxb+ObK29xcfjrcHw9PL++zK5cXw5/nUtUvD2xFyLdKq2t +rIxsE6s0g0ZlS+4pjDdnRWNir6pycmWEG6RZKbGxXCrevG/hrRDog5DBODtpb++26uVGJRec4lsE09Y5 ++tmRlT+GbLNjXcpXBqTy5HibzdFNPW0kF3s1J3DLNDfPYaFiNPTyV1hoDW5RSD/reiscylSW4GuOYmpY +7Xwhl+foPcHso0qxdV6cS6PAY72J8DUGhsDvkp9tdUhLahAzAf1RWb0NT35U+y1DKecajNmx3nnFgnb/ +Lq6+k7NNSmh6UFNxK7T1AUNBx/8DAAD//y+7cdK5WAAA `, }, From 21ca72f6a65a116b8ec7160563bb4be5b2f73249 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Wed, 21 Oct 2020 16:58:38 -0400 Subject: [PATCH 34/55] config fix 1 --- .../coordinator_config_rules/m3coordinator.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml b/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml index 5e983f62b9..1ad54297c1 100644 --- a/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml +++ b/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml @@ -1,5 +1,4 @@ listenAddress: 0.0.0.0:7201 - value: "" logging: level: info From 963da122483e289348a382afaa6bb22f4414e638 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Wed, 21 Oct 2020 17:10:07 -0400 Subject: [PATCH 35/55] config fix 2 --- config/m3db/clustered-etcd/generated.yaml | 4 +--- config/m3db/local-etcd/generated.yaml | 4 +--- kube/terraform/main.tf | 2 +- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/config/m3db/clustered-etcd/generated.yaml b/config/m3db/clustered-etcd/generated.yaml index 701d2379a4..646ee745a8 100644 --- a/config/m3db/clustered-etcd/generated.yaml +++ b/config/m3db/clustered-etcd/generated.yaml @@ -1,7 +1,5 @@ "coordinator": - "listenAddress": - "type": "config" - "value": "0.0.0.0:7201" + "listenAddress": "0.0.0.0:7201" "local": "namespaces": - "namespace": "default" diff --git a/config/m3db/local-etcd/generated.yaml b/config/m3db/local-etcd/generated.yaml index a223c1e69b..d98d832cee 100644 --- a/config/m3db/local-etcd/generated.yaml +++ b/config/m3db/local-etcd/generated.yaml @@ -1,7 +1,5 @@ "coordinator": - "listenAddress": - "type": "config" - "value": "0.0.0.0:7201" + "listenAddress": "0.0.0.0:7201" "local": "namespaces": - "namespace": "default" diff --git a/kube/terraform/main.tf b/kube/terraform/main.tf index 9a4ea68bce..fe5a23804a 100755 --- a/kube/terraform/main.tf +++ b/kube/terraform/main.tf @@ -133,7 +133,7 @@ resource "kubernetes_config_map" "m3dbnode_config" { namespace = "m3db" } data { - m3dbnode.yml = "coordinator:\n listenAddress:\n type: \"config\"\n value: \"0.0.0.0:7201\"\n local:\n namespaces:\n - namespace: default\n type: unaggregated\n retention: 48h\n metrics:\n scope:\n prefix: \"coordinator\"\n prometheus:\n handlerPath: /metrics\n listenAddress: 0.0.0.0:7203\n sanitization: prometheus\n samplingRate: 1.0\n extended: none\n tagOptions:\n idScheme: quoted\n\ndb:\n logging:\n level: info\n\n metrics:\n prometheus:\n handlerPath: /metrics\n sanitization: prometheus\n samplingRate: 1.0\n extended: detailed\n\n listenAddress: 0.0.0.0:9000\n clusterListenAddress: 0.0.0.0:9001\n httpNodeListenAddress: 0.0.0.0:9002\n httpClusterListenAddress: 0.0.0.0:9003\n debugListenAddress: 0.0.0.0:9004\n\n hostID:\n resolver: hostname\n\n client:\n writeConsistencyLevel: majority\n readConsistencyLevel: unstrict_majority\n\n gcPercentage: 100\n\n writeNewSeriesAsync: true\n writeNewSeriesLimitPerSecond: 1048576\n writeNewSeriesBackoffDuration: 2ms\n\n bootstrap:\n fs:\n numProcessorsPerCPU: 0.125\n commitlog:\n returnUnfulfilledForCorruptCommitLogFiles: false\n\n commitlog:\n flushMaxBytes: 524288\n flushEvery: 1s\n queue:\n calculationType: fixed\n size: 2097152\n\n fs:\n filePathPrefix: /var/lib/m3db\n\n config:\n service:\n env: default_env\n zone: embedded\n service: m3db\n cacheDir: /var/lib/m3kv\n etcdClusters:\n - zone: embedded\n endpoints:\n - http://etcd-0.etcd:2379\n - http://etcd-1.etcd:2379\n - http://etcd-2.etcd:2379\n" + m3dbnode.yml = "coordinator:\n listenAddress: \"0.0.0.0:7201\"\n local:\n namespaces:\n - namespace: default\n type: unaggregated\n retention: 48h\n metrics:\n scope:\n prefix: \"coordinator\"\n prometheus:\n handlerPath: /metrics\n listenAddress: 0.0.0.0:7203\n sanitization: prometheus\n samplingRate: 1.0\n extended: none\n tagOptions:\n idScheme: quoted\n\ndb:\n logging:\n level: info\n\n metrics:\n prometheus:\n handlerPath: /metrics\n sanitization: prometheus\n samplingRate: 1.0\n extended: detailed\n\n listenAddress: 0.0.0.0:9000\n clusterListenAddress: 0.0.0.0:9001\n httpNodeListenAddress: 0.0.0.0:9002\n httpClusterListenAddress: 0.0.0.0:9003\n debugListenAddress: 0.0.0.0:9004\n\n hostID:\n resolver: hostname\n\n client:\n writeConsistencyLevel: majority\n readConsistencyLevel: unstrict_majority\n\n gcPercentage: 100\n\n writeNewSeriesAsync: true\n writeNewSeriesLimitPerSecond: 1048576\n writeNewSeriesBackoffDuration: 2ms\n\n bootstrap:\n fs:\n numProcessorsPerCPU: 0.125\n commitlog:\n returnUnfulfilledForCorruptCommitLogFiles: false\n\n commitlog:\n flushMaxBytes: 524288\n flushEvery: 1s\n queue:\n calculationType: fixed\n size: 2097152\n\n fs:\n filePathPrefix: /var/lib/m3db\n\n config:\n service:\n env: default_env\n zone: embedded\n service: m3db\n cacheDir: /var/lib/m3kv\n etcdClusters:\n - zone: embedded\n endpoints:\n - http://etcd-0.etcd:2379\n - http://etcd-1.etcd:2379\n - http://etcd-2.etcd:2379\n" } } From 361f7ed214fc492840c72925c32f9658d5577d37 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Mon, 26 Oct 2020 15:45:58 -0400 Subject: [PATCH 36/55] clarifying bootstrap order --- src/cmd/services/m3dbnode/config/bootstrap.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/cmd/services/m3dbnode/config/bootstrap.go b/src/cmd/services/m3dbnode/config/bootstrap.go index 77e0ba0eff..8bef4e8a44 100644 --- a/src/cmd/services/m3dbnode/config/bootstrap.go +++ b/src/cmd/services/m3dbnode/config/bootstrap.go @@ -47,12 +47,12 @@ var ( // order in which bootstrappers are run // (run in ascending order of precedence) orderedBootstrappers = []string{ - uninitialized.UninitializedTopologyBootstrapperName, - // Peers and commitlog must come after uninitialized topology bootrapping. + // Filesystem bootstrapping must be first. + bfs.FileSystemBootstrapperName, + // Peers and commitlog must come before the uninitialized topology bootrapping. peers.PeersBootstrapperName, commitlog.CommitLogBootstrapperName, - // Filesystem bootstrapping must be last. - bfs.FileSystemBootstrapperName, + uninitialized.UninitializedTopologyBootstrapperName, } ) @@ -192,8 +192,10 @@ func (bsc BootstrapConfiguration) New( bs bootstrap.BootstrapperProvider fsOpts = opts.CommitLogOptions().FilesystemOptions() ) - for _, b := range orderedBootstrappers { - switch b { + // Start from the end of the list because the bootstrappers are ordered by precedence in descending order. + // I.e. each bootstrapper wraps the preceding bootstrapper, and so the outer-most bootstrapper is run first. + for i := len(orderedBootstrappers) - 1; i >= 0; i-- { + switch orderedBootstrappers[i] { case bootstrapper.NoOpAllBootstrapperName: bs = bootstrapper.NewNoOpAllBootstrapperProvider() case bootstrapper.NoOpNoneBootstrapperName: From 700cfd0622b6586cc877bc40fad998a9369cdcc3 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Mon, 26 Oct 2020 16:33:37 -0400 Subject: [PATCH 37/55] changelog --- CHANGELOG.md | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53969f97d8..8643c784ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,36 @@ # Changelog +# 1.0.0 (PROVISIONAL - STILL WORK IN PROGRESS) + +## Backwards Incompatible Changes + +### Configuration +- **M3DB**: `db.bootstrap.bootstrappers` removed +- **M3Coordinator**: `cluster.namespaces.storageMetricsType` removed +- **M3Coordinator**: `tagOptions.tagOptions` no longer supports `legacy` type +- **M3Query**: `limits.perQuery.maxComputedDatapoints` removed +- **M3Query**: `limits.perQuery.maxFetchedDatapoints` removed +- **M3Query**: `limits.global.maxFetchedDatapoints` removed +- **M3Query**: `cache` removed +- **M3Query**: `listenAddress` changed to always be resolved as a string from config. Format changed from +``` +listenAddress: + config: "..." + value: "..." +``` +to +``` +listenAddress: "..." +``` + +### API +- **M3DB**: `/services/m3db/database/config/bootstrappers` dynamic bootstappers endpoint removed +- **M3Coordinator**: `/api/v1/namespace` changed to `/api/v1/services/m3db/namespace` +- **M3Coordinator**: `/api/v1/namespace/init` changed to `/api/v1/services/m3db/namespace/init` +- **M3Coordinator**: `/api/v1/namespace/unagg` changed to `/api/v1/services/m3db/namespace/unagg` +- **M3Coordinator**: `/api/v1/placement` changed to `/api/v1/services/m3db/placement` +- **M3Coordinator**: `/api/v1/placement/init` changed to `/api/v1/services/m3db/placement/init` + # 0.15.17 ## Features @@ -485,7 +516,7 @@ If you run into any issues with the upgrade or need to downgrade to a previous v ## Features - **M3Query**: Add multi-zone and multi-region configuration for coordinator ([#1687](https://github.com/m3db/m3/pull/1687)) -- **M3Query**: Add debug param to `GET` `/api/v1/services/m3db/namespace` endpoint for better readability ([#1698](https://github.com/m3db/m3/pull/1698)) +- **M3Query**: Add debug param to `GET` `/api/v1/namespace` endpoint for better readability ([#1698](https://github.com/m3db/m3/pull/1698)) - **M3Coordinator**: Add "ingest_latency" histogram metric and return datapoint too old/new errors with offending timestamps ([#1716](https://github.com/m3db/m3/pull/1716)) ## Performance From 03e2235f18586b5d5c41d32c1a53f332728f64ba Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Mon, 26 Oct 2020 17:12:10 -0400 Subject: [PATCH 38/55] remove all cost reporting code 1 --- src/cmd/services/m3dbnode/config/bootstrap.go | 2 +- src/cmd/services/m3query/config/config.go | 2 +- .../services/m3query/config/config_test.go | 4 +- .../database/config_bootstrappers_get.go | 92 ------ .../database/config_bootstrappers_set.go | 113 ------- src/query/api/v1/handler/graphite/render.go | 2 +- src/query/api/v1/handler/prom/read_test.go | 1 - .../v1/handler/prometheus/native/read_test.go | 1 - .../v1/handler/prometheus/remote/read_test.go | 9 +- src/query/api/v1/httpd/handler_test.go | 13 +- src/query/api/v1/options/handler.go | 19 -- src/query/block/accounted.go | 49 ---- src/query/block/accounted_test.go | 46 --- src/query/block/column.go | 23 +- src/query/block/column_test.go | 3 +- src/query/cost/cost.go | 208 ------------- src/query/cost/cost_mock.go | 236 --------------- src/query/cost/cost_prop_test.go | 77 ----- src/query/cost/cost_test.go | 242 --------------- src/query/executor/engine.go | 9 +- src/query/executor/engine_test.go | 14 +- src/query/executor/options.go | 12 - src/query/executor/types.go | 6 - src/query/functions/fetch_test.go | 3 +- src/query/generated/mocks/generate.go | 1 - src/query/graphite/storage/m3_wrapper.go | 11 - src/query/graphite/storage/m3_wrapper_test.go | 11 +- src/query/models/query_context.go | 23 +- src/query/remote/client.go | 5 +- src/query/server/cost_reporters.go | 257 ---------------- src/query/server/cost_reporters_test.go | 275 ------------------ src/query/server/query.go | 11 +- src/query/server/query_test.go | 118 -------- src/query/storage/fetch.go | 4 +- src/query/storage/m3/storage.go | 7 - src/query/storage/prom_converter.go | 26 +- src/query/storage/prom_converter_test.go | 10 +- src/query/storage/types.go | 4 - src/x/cost/enforcer.go | 176 ----------- src/x/cost/enforcer_test.go | 248 ---------------- src/x/cost/limit_manager.go | 177 ----------- src/x/cost/limit_manager_test.go | 127 -------- src/x/cost/options.go | 167 ----------- src/x/cost/test/assert.go | 45 --- src/x/cost/tracker.go | 40 --- src/x/cost/tracker_test.go | 111 ------- src/x/cost/types.go | 85 ------ 47 files changed, 43 insertions(+), 3082 deletions(-) delete mode 100644 src/query/api/v1/handler/database/config_bootstrappers_get.go delete mode 100644 src/query/api/v1/handler/database/config_bootstrappers_set.go delete mode 100644 src/query/block/accounted.go delete mode 100644 src/query/block/accounted_test.go delete mode 100644 src/query/cost/cost.go delete mode 100644 src/query/cost/cost_mock.go delete mode 100644 src/query/cost/cost_prop_test.go delete mode 100644 src/query/cost/cost_test.go delete mode 100644 src/query/server/cost_reporters.go delete mode 100644 src/query/server/cost_reporters_test.go delete mode 100644 src/x/cost/enforcer.go delete mode 100644 src/x/cost/enforcer_test.go delete mode 100644 src/x/cost/limit_manager.go delete mode 100644 src/x/cost/limit_manager_test.go delete mode 100644 src/x/cost/options.go delete mode 100644 src/x/cost/test/assert.go delete mode 100644 src/x/cost/tracker.go delete mode 100644 src/x/cost/tracker_test.go delete mode 100644 src/x/cost/types.go diff --git a/src/cmd/services/m3dbnode/config/bootstrap.go b/src/cmd/services/m3dbnode/config/bootstrap.go index 8bef4e8a44..907ca3e54a 100644 --- a/src/cmd/services/m3dbnode/config/bootstrap.go +++ b/src/cmd/services/m3dbnode/config/bootstrap.go @@ -274,7 +274,7 @@ func (bsc BootstrapConfiguration) New( } bs = uninitialized.NewUninitializedTopologyBootstrapperProvider(uOpts, bs) default: - return nil, fmt.Errorf("unknown bootstrapper: %s", b) + return nil, fmt.Errorf("unknown bootstrapper: %s", orderedBootstrappers[i]) } } diff --git a/src/cmd/services/m3query/config/config.go b/src/cmd/services/m3query/config/config.go index 65a2cf0e9e..823a78a743 100644 --- a/src/cmd/services/m3query/config/config.go +++ b/src/cmd/services/m3query/config/config.go @@ -304,7 +304,7 @@ type PerQueryLimitsConfiguration struct { MaxFetchedDocs int `yaml:"maxFetchedDocs"` // RequireExhaustive results in an error if the query exceeds any limit. - RequireExhaustive bool `yaml:"requireExhaustive"` + RequireExhaustive *bool `yaml:"requireExhaustive"` } // AsFetchOptionsBuilderLimitsOptions converts this configuration to diff --git a/src/cmd/services/m3query/config/config_test.go b/src/cmd/services/m3query/config/config_test.go index 8af9c7de77..1246cfefcc 100644 --- a/src/cmd/services/m3query/config/config_test.go +++ b/src/cmd/services/m3query/config/config_test.go @@ -91,11 +91,13 @@ func TestConfigLoading(t *testing.T) { var cfg Configuration require.NoError(t, xconfig.LoadFile(&cfg, testConfigFile, xconfig.Options{})) + var requireExhaustive bool + requireExhaustive = true assert.Equal(t, &LimitsConfiguration{ PerQuery: PerQueryLimitsConfiguration{ MaxFetchedSeries: 12000, MaxFetchedDocs: 11000, - RequireExhaustive: true, + RequireExhaustive: &requireExhaustive, }, }, &cfg.Limits) // TODO: assert on more fields here. diff --git a/src/query/api/v1/handler/database/config_bootstrappers_get.go b/src/query/api/v1/handler/database/config_bootstrappers_get.go deleted file mode 100644 index 6cd3690660..0000000000 --- a/src/query/api/v1/handler/database/config_bootstrappers_get.go +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package database - -import ( - "net/http" - - clusterclient "github.com/m3db/m3/src/cluster/client" - "github.com/m3db/m3/src/cluster/generated/proto/commonpb" - "github.com/m3db/m3/src/cluster/kv" - "github.com/m3db/m3/src/dbnode/kvconfig" - "github.com/m3db/m3/src/query/api/v1/handler" - "github.com/m3db/m3/src/query/util/logging" - "github.com/m3db/m3/src/x/instrument" - xhttp "github.com/m3db/m3/src/x/net/http" - - "go.uber.org/zap" -) - -const ( - // ConfigGetBootstrappersURL is the url for the database create handler. - ConfigGetBootstrappersURL = handler.RoutePrefixV1 + "/database/config/bootstrappers" - - // ConfigGetBootstrappersHTTPMethod is the HTTP method used with this resource. - ConfigGetBootstrappersHTTPMethod = http.MethodGet -) - -type configGetBootstrappersHandler struct { - client clusterclient.Client - instrumentOpts instrument.Options -} - -// NewConfigGetBootstrappersHandler returns a new instance of a database create handler. -func NewConfigGetBootstrappersHandler( - client clusterclient.Client, - instrumentOpts instrument.Options, -) http.Handler { - return &configGetBootstrappersHandler{ - client: client, - instrumentOpts: instrumentOpts, - } -} - -func (h *configGetBootstrappersHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - logger := logging.WithContext(ctx, h.instrumentOpts) - - store, err := h.client.KV() - if err != nil { - logger.Error("unable to get kv store", zap.Error(err)) - xhttp.WriteError(w, err) - return - } - - value, err := store.Get(kvconfig.BootstrapperKey) - if err != nil && err == kv.ErrNotFound { - xhttp.WriteError(w, xhttp.NewError(err, http.StatusNotFound)) - return - } - if err != nil { - logger.Error("unable to get kv key", zap.Error(err)) - xhttp.WriteError(w, err) - return - } - - array := new(commonpb.StringArrayProto) - if err := value.Unmarshal(array); err != nil { - logger.Error("unable to unmarshal kv key", zap.Error(err)) - xhttp.WriteError(w, err) - return - } - - xhttp.WriteProtoMsgJSONResponse(w, array, logger) -} diff --git a/src/query/api/v1/handler/database/config_bootstrappers_set.go b/src/query/api/v1/handler/database/config_bootstrappers_set.go deleted file mode 100644 index fb98853c98..0000000000 --- a/src/query/api/v1/handler/database/config_bootstrappers_set.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package database - -import ( - "fmt" - "net/http" - - clusterclient "github.com/m3db/m3/src/cluster/client" - "github.com/m3db/m3/src/cluster/generated/proto/commonpb" - dbconfig "github.com/m3db/m3/src/cmd/services/m3dbnode/config" - "github.com/m3db/m3/src/dbnode/kvconfig" - "github.com/m3db/m3/src/query/api/v1/handler" - "github.com/m3db/m3/src/query/util/logging" - xerrors "github.com/m3db/m3/src/x/errors" - "github.com/m3db/m3/src/x/instrument" - xhttp "github.com/m3db/m3/src/x/net/http" - - "github.com/gogo/protobuf/jsonpb" - "go.uber.org/zap" -) - -const ( - // ConfigSetBootstrappersURL is the url for the database create handler. - ConfigSetBootstrappersURL = handler.RoutePrefixV1 + "/database/config/bootstrappers" - - // ConfigSetBootstrappersHTTPMethod is the HTTP method used with this resource. - ConfigSetBootstrappersHTTPMethod = http.MethodPost -) - -type configSetBootstrappersHandler struct { - client clusterclient.Client - instrumentOpts instrument.Options -} - -// NewConfigSetBootstrappersHandler returns a new instance of a database create handler. -func NewConfigSetBootstrappersHandler( - client clusterclient.Client, - instrumentOpts instrument.Options, -) http.Handler { - return &configSetBootstrappersHandler{ - client: client, - instrumentOpts: instrumentOpts, - } -} - -func (h *configSetBootstrappersHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - logger := logging.WithContext(ctx, h.instrumentOpts) - - value, rErr := h.parseRequest(r) - if rErr != nil { - logger.Error("unable to parse request", zap.Error(rErr)) - xhttp.WriteError(w, rErr) - return - } - - store, err := h.client.KV() - if err != nil { - logger.Error("unable to get kv store", zap.Error(err)) - xhttp.WriteError(w, err) - return - } - - if _, err := store.Set(kvconfig.BootstrapperKey, value); err != nil { - logger.Error("unable to set kv key", zap.Error(err)) - xhttp.WriteError(w, err) - return - } - - xhttp.WriteProtoMsgJSONResponse(w, value, logger) -} - -func (h *configSetBootstrappersHandler) parseRequest( - r *http.Request, -) (*commonpb.StringArrayProto, error) { - array := new(commonpb.StringArrayProto) - - defer r.Body.Close() - - if err := jsonpb.Unmarshal(r.Body, array); err != nil { - return nil, xerrors.NewInvalidParamsError(err) - } - - if len(array.Values) == 0 { - return nil, xerrors.NewInvalidParamsError(fmt.Errorf("no values")) - } - - validator := dbconfig.NewBootstrapConfigurationValidator() - if err := validator.ValidateBootstrappersOrder(array.Values); err != nil { - return nil, xerrors.NewInvalidParamsError(err) - } - - return array, nil -} diff --git a/src/query/api/v1/handler/graphite/render.go b/src/query/api/v1/handler/graphite/render.go index 485e4d93e8..f5117aff51 100644 --- a/src/query/api/v1/handler/graphite/render.go +++ b/src/query/api/v1/handler/graphite/render.go @@ -68,7 +68,7 @@ type respError struct { // NewRenderHandler returns a new render handler around the given storage. func NewRenderHandler(opts options.HandlerOptions) http.Handler { wrappedStore := graphite.NewM3WrappedStorage(opts.Storage(), - opts.Enforcer(), opts.M3DBOptions(), opts.InstrumentOpts(), opts.GraphiteStorageOptions()) + opts.M3DBOptions(), opts.InstrumentOpts(), opts.GraphiteStorageOptions()) return &renderHandler{ engine: native.NewEngine(wrappedStore), queryContextOpts: opts.QueryContextOptions(), diff --git a/src/query/api/v1/handler/prom/read_test.go b/src/query/api/v1/handler/prom/read_test.go index 392a664b5d..0d9b521aae 100644 --- a/src/query/api/v1/handler/prom/read_test.go +++ b/src/query/api/v1/handler/prom/read_test.go @@ -64,7 +64,6 @@ func setupTest(t *testing.T) testHandlers { instrumentOpts := instrument.NewOptions() engineOpts := executor.NewEngineOptions(). SetLookbackDuration(time.Minute). - SetGlobalEnforcer(nil). SetInstrumentOptions(instrumentOpts) engine := executor.NewEngine(engineOpts) hOpts := options.EmptyHandlerOptions(). diff --git a/src/query/api/v1/handler/prometheus/native/read_test.go b/src/query/api/v1/handler/prometheus/native/read_test.go index 9bd936afe7..5ae88efae2 100644 --- a/src/query/api/v1/handler/prometheus/native/read_test.go +++ b/src/query/api/v1/handler/prometheus/native/read_test.go @@ -199,7 +199,6 @@ func newTestSetup( engineOpts := executor.NewEngineOptions(). SetStore(mockStorage). SetLookbackDuration(time.Minute). - SetGlobalEnforcer(nil). SetInstrumentOptions(instrumentOpts) engine := executor.NewEngine(engineOpts) if mockEngine != nil { diff --git a/src/query/api/v1/handler/prometheus/remote/read_test.go b/src/query/api/v1/handler/prometheus/remote/read_test.go index aebaebcda2..c8c2504211 100644 --- a/src/query/api/v1/handler/prometheus/remote/read_test.go +++ b/src/query/api/v1/handler/prometheus/remote/read_test.go @@ -41,7 +41,6 @@ import ( "github.com/m3db/m3/src/query/api/v1/handler/prometheus/handleroptions" "github.com/m3db/m3/src/query/api/v1/options" "github.com/m3db/m3/src/query/block" - qcost "github.com/m3db/m3/src/query/cost" "github.com/m3db/m3/src/query/executor" "github.com/m3db/m3/src/query/generated/proto/prompb" "github.com/m3db/m3/src/query/models" @@ -132,13 +131,11 @@ func TestParseExpr(t *testing.T) { func newEngine( s storage.Storage, lookbackDuration time.Duration, - enforcer qcost.ChainedEnforcer, instrumentOpts instrument.Options, ) executor.Engine { engineOpts := executor.NewEngineOptions(). SetStore(s). SetLookbackDuration(lookbackDuration). - SetGlobalEnforcer(enforcer). SetInstrumentOptions(instrumentOpts) return executor.NewEngine(engineOpts) @@ -167,7 +164,7 @@ func readHandler(store storage.Storage, }, } iOpts := instrument.NewOptions() - engine := newEngine(store, defaultLookbackDuration, nil, iOpts) + engine := newEngine(store, defaultLookbackDuration, iOpts) opts := options.EmptyHandlerOptions(). SetEngine(engine). SetInstrumentOpts(iOpts). @@ -185,7 +182,7 @@ func TestPromReadParsing(t *testing.T) { SeriesLimit: 100, }, } - engine := newEngine(storage, defaultLookbackDuration, nil, + engine := newEngine(storage, defaultLookbackDuration, instrument.NewOptions()) opts := options.EmptyHandlerOptions(). @@ -297,7 +294,7 @@ func TestReadErrorMetricsCount(t *testing.T) { SeriesLimit: 100, }, } - engine := newEngine(storage, defaultLookbackDuration, nil, + engine := newEngine(storage, defaultLookbackDuration, instrument.NewOptions()) opts := options.EmptyHandlerOptions(). SetEngine(engine). diff --git a/src/query/api/v1/httpd/handler_test.go b/src/query/api/v1/httpd/handler_test.go index 0a23f88c27..f0e64b3c65 100644 --- a/src/query/api/v1/httpd/handler_test.go +++ b/src/query/api/v1/httpd/handler_test.go @@ -37,7 +37,6 @@ import ( "github.com/m3db/m3/src/query/api/v1/handler/prometheus/native" "github.com/m3db/m3/src/query/api/v1/handler/prometheus/remote" "github.com/m3db/m3/src/query/api/v1/options" - qcost "github.com/m3db/m3/src/query/cost" "github.com/m3db/m3/src/query/executor" graphite "github.com/m3db/m3/src/query/graphite/storage" "github.com/m3db/m3/src/query/models" @@ -75,13 +74,11 @@ func makeTagOptions() models.TagOptions { func newEngine( s storage.Storage, lookbackDuration time.Duration, - enforcer qcost.ChainedEnforcer, instrumentOpts instrument.Options, ) executor.Engine { engineOpts := executor.NewEngineOptions(). SetStore(s). SetLookbackDuration(lookbackDuration). - SetGlobalEnforcer(enforcer). SetInstrumentOptions(instrumentOpts) return executor.NewEngine(engineOpts) @@ -93,7 +90,7 @@ func setupHandler( ) (*Handler, error) { instrumentOpts := instrument.NewOptions() downsamplerAndWriter := ingest.NewDownsamplerAndWriter(store, nil, testWorkerPool, instrument.NewOptions()) - engine := newEngine(store, time.Minute, nil, instrumentOpts) + engine := newEngine(store, time.Minute, instrumentOpts) opts, err := options.NewHandlerOptions( downsamplerAndWriter, makeTagOptions(), @@ -103,7 +100,6 @@ func setupHandler( nil, config.Configuration{LookbackDuration: &defaultLookbackDuration}, nil, - nil, handleroptions.NewFetchOptionsBuilder(handleroptions.FetchOptionsBuilderOptions{}), models.QueryContextOptions{}, instrumentOpts, @@ -130,7 +126,7 @@ func TestHandlerFetchTimeout(t *testing.T) { fourMin := 4 * time.Minute dbconfig := &dbconfig.DBConfiguration{Client: client.Configuration{FetchTimeout: &fourMin}} - engine := newEngine(storage, time.Minute, nil, instrument.NewOptions()) + engine := newEngine(storage, time.Minute, instrument.NewOptions()) cfg := config.Configuration{LookbackDuration: &defaultLookbackDuration} opts, err := options.NewHandlerOptions( downsamplerAndWriter, @@ -141,7 +137,6 @@ func TestHandlerFetchTimeout(t *testing.T) { nil, cfg, dbconfig, - nil, handleroptions.NewFetchOptionsBuilder(handleroptions.FetchOptionsBuilderOptions{}), models.QueryContextOptions{}, instrument.NewOptions(), @@ -406,10 +401,10 @@ func TestCustomRoutes(t *testing.T) { store, _ := m3.NewStorageAndSession(t, ctrl) instrumentOpts := instrument.NewOptions() downsamplerAndWriter := ingest.NewDownsamplerAndWriter(store, nil, testWorkerPool, instrument.NewOptions()) - engine := newEngine(store, time.Minute, nil, instrumentOpts) + engine := newEngine(store, time.Minute, instrumentOpts) opts, err := options.NewHandlerOptions( downsamplerAndWriter, makeTagOptions().SetMetricName([]byte("z")), engine, nil, nil, nil, - config.Configuration{LookbackDuration: &defaultLookbackDuration}, nil, nil, + config.Configuration{LookbackDuration: &defaultLookbackDuration}, nil, handleroptions.NewFetchOptionsBuilder(handleroptions.FetchOptionsBuilderOptions{}), models.QueryContextOptions{}, instrumentOpts, defaultCPUProfileduration, defaultPlacementServices, svcDefaultOptions, NewQueryRouter(), NewQueryRouter(), diff --git a/src/query/api/v1/options/handler.go b/src/query/api/v1/options/handler.go index 4ab364b969..15eab3f99f 100644 --- a/src/query/api/v1/options/handler.go +++ b/src/query/api/v1/options/handler.go @@ -32,7 +32,6 @@ import ( "github.com/m3db/m3/src/cmd/services/m3query/config" "github.com/m3db/m3/src/query/api/v1/handler/prometheus" "github.com/m3db/m3/src/query/api/v1/handler/prometheus/handleroptions" - "github.com/m3db/m3/src/query/cost" "github.com/m3db/m3/src/query/executor" graphite "github.com/m3db/m3/src/query/graphite/storage" "github.com/m3db/m3/src/query/models" @@ -147,11 +146,6 @@ type HandlerOptions interface { // SetTimeoutOpts sets the timeout options. SetTimeoutOpts(t *prometheus.TimeoutOpts) HandlerOptions - // Enforcer returns the enforcer. - Enforcer() cost.ChainedEnforcer - // SetEnforcer sets the enforcer. - SetEnforcer(e cost.ChainedEnforcer) HandlerOptions - // FetchOptionsBuilder returns the fetch options builder. FetchOptionsBuilder() handleroptions.FetchOptionsBuilder // SetFetchOptionsBuilder sets the fetch options builder. @@ -235,7 +229,6 @@ type handlerOptions struct { createdAt time.Time tagOptions models.TagOptions timeoutOpts *prometheus.TimeoutOpts - enforcer cost.ChainedEnforcer fetchOptionsBuilder handleroptions.FetchOptionsBuilder queryContextOptions models.QueryContextOptions instrumentOpts instrument.Options @@ -269,7 +262,6 @@ func NewHandlerOptions( clusterClient clusterclient.Client, cfg config.Configuration, embeddedDbCfg *dbconfig.DBConfiguration, - enforcer cost.ChainedEnforcer, fetchOptionsBuilder handleroptions.FetchOptionsBuilder, queryContextOptions models.QueryContextOptions, instrumentOpts instrument.Options, @@ -305,7 +297,6 @@ func NewHandlerOptions( embeddedDbCfg: embeddedDbCfg, createdAt: time.Now(), tagOptions: tagOptions, - enforcer: enforcer, fetchOptionsBuilder: fetchOptionsBuilder, queryContextOptions: queryContextOptions, instrumentOpts: instrumentOpts, @@ -431,16 +422,6 @@ func (o *handlerOptions) SetTimeoutOpts(t *prometheus.TimeoutOpts) HandlerOption return &opts } -func (o *handlerOptions) Enforcer() cost.ChainedEnforcer { - return o.enforcer -} - -func (o *handlerOptions) SetEnforcer(e cost.ChainedEnforcer) HandlerOptions { - opts := *o - opts.enforcer = e - return &opts -} - func (o *handlerOptions) FetchOptionsBuilder() handleroptions.FetchOptionsBuilder { return o.fetchOptionsBuilder } diff --git a/src/query/block/accounted.go b/src/query/block/accounted.go deleted file mode 100644 index 3dc2b45785..0000000000 --- a/src/query/block/accounted.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package block - -import "github.com/m3db/m3/src/query/cost" - -// AccountedBlock is a wrapper for a block which enforces limits on the number -// of datapoints used by the block. -type AccountedBlock struct { - Block - - enforcer cost.ChainedEnforcer -} - -// NewAccountedBlock wraps the given block and enforces datapoint limits. -func NewAccountedBlock( - wrapped Block, - enforcer cost.ChainedEnforcer, -) *AccountedBlock { - return &AccountedBlock{ - Block: wrapped, - enforcer: enforcer, - } -} - -// Close closes the block, and marks the number of datapoints used -// by this block as finished. -func (ab *AccountedBlock) Close() error { - ab.enforcer.Close() - return ab.Block.Close() -} diff --git a/src/query/block/accounted_test.go b/src/query/block/accounted_test.go deleted file mode 100644 index b5858ef210..0000000000 --- a/src/query/block/accounted_test.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) 2019 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package block - -import ( - "testing" - - "github.com/m3db/m3/src/query/cost" - - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/assert" -) - -func TestAccountedBlock_Close(t *testing.T) { - ctrl := gomock.NewController(t) - - wrapped := NewMockBlock(ctrl) - wrapped.EXPECT().Close() - - mockEnforcer := cost.NewMockChainedEnforcer(ctrl) - mockEnforcer.EXPECT().Close() - - block := NewAccountedBlock(wrapped, mockEnforcer) - - wrapped.EXPECT().Info().Return(NewBlockInfo(BlockM3TSZCompressed)) - assert.Equal(t, BlockM3TSZCompressed, block.Info().Type()) - assert.NotPanics(t, func() { block.Close() }) -} diff --git a/src/query/block/column.go b/src/query/block/column.go index 86f406a566..dfcde95b91 100644 --- a/src/query/block/column.go +++ b/src/query/block/column.go @@ -25,9 +25,7 @@ import ( "fmt" "time" - "github.com/m3db/m3/src/query/cost" "github.com/m3db/m3/src/query/models" - xcost "github.com/m3db/m3/src/x/cost" "github.com/uber-go/tally" ) @@ -39,7 +37,6 @@ type column struct { // ColumnBlockBuilder builds a block optimized for column iteration. type ColumnBlockBuilder struct { block *columnBlock - enforcer cost.ChainedEnforcer blockDatapoints tally.Counter } @@ -174,7 +171,6 @@ func NewColumnBlockBuilder( meta Metadata, seriesMeta []SeriesMeta) Builder { return ColumnBlockBuilder{ - enforcer: queryCtx.Enforcer.Child(cost.BlockLevel), blockDatapoints: queryCtx.Scope.Tagged( map[string]string{"type": "generated"}).Counter("datapoints"), block: &columnBlock{ @@ -192,11 +188,6 @@ func (cb ColumnBlockBuilder) AppendValue(idx int, value float64) error { return fmt.Errorf("idx out of range for append: %d", idx) } - r := cb.enforcer.Add(1) - if r.Error != nil { - return r.Error - } - cb.blockDatapoints.Inc(1) columns[idx].Values = append(columns[idx].Values, value) @@ -210,11 +201,6 @@ func (cb ColumnBlockBuilder) AppendValues(idx int, values []float64) error { return fmt.Errorf("idx out of range for append: %d", idx) } - r := cb.enforcer.Add(xcost.Cost(len(values))) - if r.Error != nil { - return r.Error - } - cb.blockDatapoints.Inc(int64(len(values))) columns[idx].Values = append(columns[idx].Values, values...) return nil @@ -269,22 +255,17 @@ func (cb ColumnBlockBuilder) SetRow( cb.block.columns[i].Values[idx] = v } - r := cb.enforcer.Add(xcost.Cost(len(values))) - if r.Error != nil { - return r.Error - } - cb.block.seriesMeta[idx] = meta return nil } // Build builds the block. func (cb ColumnBlockBuilder) Build() Block { - return NewAccountedBlock(cb.block, cb.enforcer) + return cb.block } // BuildAsType builds the block, forcing it to the given BlockType. func (cb ColumnBlockBuilder) BuildAsType(blockType BlockType) Block { cb.block.blockType = blockType - return NewAccountedBlock(cb.block, cb.enforcer) + return cb.block } diff --git a/src/query/block/column_test.go b/src/query/block/column_test.go index ee1c9392bc..b341792c4c 100644 --- a/src/query/block/column_test.go +++ b/src/query/block/column_test.go @@ -26,7 +26,6 @@ import ( "testing" "time" - "github.com/m3db/m3/src/query/cost" "github.com/m3db/m3/src/query/models" "github.com/stretchr/testify/assert" @@ -36,7 +35,7 @@ import ( func makeTestQueryContext() *models.QueryContext { return models.NewQueryContext(context.Background(), - tally.NoopScope, cost.NoopChainedEnforcer(), + tally.NoopScope, models.QueryContextOptions{}) } diff --git a/src/query/cost/cost.go b/src/query/cost/cost.go deleted file mode 100644 index 30e7fdafcb..0000000000 --- a/src/query/cost/cost.go +++ /dev/null @@ -1,208 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package cost - -import ( - "errors" - "fmt" - - "github.com/m3db/m3/src/x/cost" -) - -const ( - // BlockLevel identifies per-block enforcers. - BlockLevel = "block" - // QueryLevel identifies per-query enforcers. - QueryLevel = "query" - // GlobalLevel identifies global enforcers. - GlobalLevel = "global" -) - -// ChainedEnforcer is a cost.Enforcer implementation which tracks resource usage implements cost.Enforcer to enforce -// limits on multiple resources at once, linked together in a tree. -type ChainedEnforcer interface { - cost.Enforcer - - // Child creates a new ChainedEnforcer which rolls up to this one. - Child(resourceName string) ChainedEnforcer - - // Close indicates that all resources have been returned for this - // ChainedEnforcer. It should inform all parent enforcers that the - // resources have been freed. - Close() -} - -type noopChainedReporter struct{} - -func (noopChainedReporter) ReportCost(_ cost.Cost) {} -func (noopChainedReporter) ReportCurrent(_ cost.Cost) {} -func (noopChainedReporter) ReportOverLimit(_ bool) {} -func (noopChainedReporter) OnChildClose(_ cost.Cost) {} -func (noopChainedReporter) OnClose(_ cost.Cost) {} - -var noopChainedReporterInstance = noopChainedReporter{} - -// ChainedReporter is a listener for chainedEnforcer methods, which listens to Close events in addition to -// events used by cost.EnforcerReporter. -type ChainedReporter interface { - cost.EnforcerReporter - - // OnChildClose is called whenever a child of this reporter's chainedEnforcer is released. - OnChildClose(currentCost cost.Cost) - - // OnClose is called whenever this reporter's chainedEnforcer is released. - OnClose(currentCost cost.Cost) -} - -// chainedEnforcer is the actual implementation of ChainedEnforcer. -type chainedEnforcer struct { - resourceName string - local cost.Enforcer - parent *chainedEnforcer - models []cost.Enforcer - reporter ChainedReporter -} - -var noopChainedEnforcer = mustNoopChainedEnforcer() - -func mustNoopChainedEnforcer() ChainedEnforcer { - rtn, err := NewChainedEnforcer("", []cost.Enforcer{cost.NoopEnforcer()}) - if err != nil { - panic(err.Error()) - } - - return rtn -} - -// NoopChainedEnforcer returns a chainedEnforcer which enforces no limits and does no reporting. -func NoopChainedEnforcer() ChainedEnforcer { - return noopChainedEnforcer -} - -// NewChainedEnforcer constructs a chainedEnforcer which creates children using the provided models. -// models[0] enforces this instance; models[1] enforces the first level of children, and so on. -func NewChainedEnforcer(rootResourceName string, models []cost.Enforcer) (ChainedEnforcer, error) { - if len(models) == 0 { - return nil, errors.New("must provide at least one Enforcer instance for a chainedEnforcer") - } - - local := models[0] - - return &chainedEnforcer{ - resourceName: rootResourceName, - parent: nil, // root has nil parent - local: local, - models: models[1:], - reporter: upcastReporterOrNoop(local.Reporter()), - }, nil -} - -func upcastReporterOrNoop(r cost.EnforcerReporter) ChainedReporter { - if r, ok := r.(ChainedReporter); ok { - return r - } - - return noopChainedReporterInstance -} - -// Add adds the given cost both to this enforcer and any parents, working recursively until the root is reached. -// The most local error is preferred. -func (ce *chainedEnforcer) Add(c cost.Cost) cost.Report { - if ce.parent == nil { - return ce.wrapLocalResult(ce.local.Add(c)) - } - - localR := ce.local.Add(c) - globalR := ce.parent.Add(c) - - // check our local limit first - if localR.Error != nil { - return ce.wrapLocalResult(localR) - } - - // check the global limit - if globalR.Error != nil { - return globalR - } - - return localR -} - -func (ce *chainedEnforcer) wrapLocalResult(localR cost.Report) cost.Report { - if localR.Error != nil { - return cost.Report{ - Cost: localR.Cost, - Error: fmt.Errorf("exceeded %s limit: %s", ce.resourceName, localR.Error.Error()), - } - } - return localR -} - -// Child creates a new chainedEnforcer whose resource consumption rolls up into this instance. -func (ce *chainedEnforcer) Child(resourceName string) ChainedEnforcer { - // no more models; just return a noop default. TODO: this could be a panic case? Technically speaking it's - // misconfiguration. - if len(ce.models) == 0 { - return NoopChainedEnforcer() - } - - newLocal := ce.models[0] - return &chainedEnforcer{ - resourceName: resourceName, - parent: ce, - // make sure to clone the local enforcer, so that we're using an - // independent instance with the same configuration. - local: newLocal.Clone(), - models: ce.models[1:], - reporter: upcastReporterOrNoop(newLocal.Reporter()), - } -} - -// Clone on a chainedEnforcer is a noop--TODO: implement? -func (ce *chainedEnforcer) Clone() cost.Enforcer { - return ce -} - -// State returns the local state of this enforcer (ignoring anything further up the chain). -func (ce *chainedEnforcer) State() (cost.Report, cost.Limit) { - return ce.local.State() -} - -// Close releases all resources tracked by this enforcer back to the global enforcer -func (ce *chainedEnforcer) Close() { - r, _ := ce.local.State() - ce.reporter.OnClose(r.Cost) - - if ce.parent != nil { - parentR, _ := ce.parent.State() - ce.parent.reporter.OnChildClose(parentR.Cost) - } - - ce.Add(-r.Cost) -} - -func (ce *chainedEnforcer) Limit() cost.Limit { - return ce.local.Limit() -} - -func (ce *chainedEnforcer) Reporter() cost.EnforcerReporter { - return ce.local.Reporter() -} diff --git a/src/query/cost/cost_mock.go b/src/query/cost/cost_mock.go deleted file mode 100644 index 85abd1379b..0000000000 --- a/src/query/cost/cost_mock.go +++ /dev/null @@ -1,236 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/m3db/m3/src/query/cost (interfaces: ChainedEnforcer,ChainedReporter) - -// Copyright (c) 2020 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -// Package cost is a generated GoMock package. -package cost - -import ( - "reflect" - - cost0 "github.com/m3db/m3/src/x/cost" - - "github.com/golang/mock/gomock" -) - -// MockChainedEnforcer is a mock of ChainedEnforcer interface -type MockChainedEnforcer struct { - ctrl *gomock.Controller - recorder *MockChainedEnforcerMockRecorder -} - -// MockChainedEnforcerMockRecorder is the mock recorder for MockChainedEnforcer -type MockChainedEnforcerMockRecorder struct { - mock *MockChainedEnforcer -} - -// NewMockChainedEnforcer creates a new mock instance -func NewMockChainedEnforcer(ctrl *gomock.Controller) *MockChainedEnforcer { - mock := &MockChainedEnforcer{ctrl: ctrl} - mock.recorder = &MockChainedEnforcerMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockChainedEnforcer) EXPECT() *MockChainedEnforcerMockRecorder { - return m.recorder -} - -// Add mocks base method -func (m *MockChainedEnforcer) Add(arg0 cost0.Cost) cost0.Report { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Add", arg0) - ret0, _ := ret[0].(cost0.Report) - return ret0 -} - -// Add indicates an expected call of Add -func (mr *MockChainedEnforcerMockRecorder) Add(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Add", reflect.TypeOf((*MockChainedEnforcer)(nil).Add), arg0) -} - -// Child mocks base method -func (m *MockChainedEnforcer) Child(arg0 string) ChainedEnforcer { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Child", arg0) - ret0, _ := ret[0].(ChainedEnforcer) - return ret0 -} - -// Child indicates an expected call of Child -func (mr *MockChainedEnforcerMockRecorder) Child(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Child", reflect.TypeOf((*MockChainedEnforcer)(nil).Child), arg0) -} - -// Clone mocks base method -func (m *MockChainedEnforcer) Clone() cost0.Enforcer { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Clone") - ret0, _ := ret[0].(cost0.Enforcer) - return ret0 -} - -// Clone indicates an expected call of Clone -func (mr *MockChainedEnforcerMockRecorder) Clone() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Clone", reflect.TypeOf((*MockChainedEnforcer)(nil).Clone)) -} - -// Close mocks base method -func (m *MockChainedEnforcer) Close() { - m.ctrl.T.Helper() - m.ctrl.Call(m, "Close") -} - -// Close indicates an expected call of Close -func (mr *MockChainedEnforcerMockRecorder) Close() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockChainedEnforcer)(nil).Close)) -} - -// Limit mocks base method -func (m *MockChainedEnforcer) Limit() cost0.Limit { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Limit") - ret0, _ := ret[0].(cost0.Limit) - return ret0 -} - -// Limit indicates an expected call of Limit -func (mr *MockChainedEnforcerMockRecorder) Limit() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Limit", reflect.TypeOf((*MockChainedEnforcer)(nil).Limit)) -} - -// Reporter mocks base method -func (m *MockChainedEnforcer) Reporter() cost0.EnforcerReporter { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Reporter") - ret0, _ := ret[0].(cost0.EnforcerReporter) - return ret0 -} - -// Reporter indicates an expected call of Reporter -func (mr *MockChainedEnforcerMockRecorder) Reporter() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Reporter", reflect.TypeOf((*MockChainedEnforcer)(nil).Reporter)) -} - -// State mocks base method -func (m *MockChainedEnforcer) State() (cost0.Report, cost0.Limit) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "State") - ret0, _ := ret[0].(cost0.Report) - ret1, _ := ret[1].(cost0.Limit) - return ret0, ret1 -} - -// State indicates an expected call of State -func (mr *MockChainedEnforcerMockRecorder) State() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "State", reflect.TypeOf((*MockChainedEnforcer)(nil).State)) -} - -// MockChainedReporter is a mock of ChainedReporter interface -type MockChainedReporter struct { - ctrl *gomock.Controller - recorder *MockChainedReporterMockRecorder -} - -// MockChainedReporterMockRecorder is the mock recorder for MockChainedReporter -type MockChainedReporterMockRecorder struct { - mock *MockChainedReporter -} - -// NewMockChainedReporter creates a new mock instance -func NewMockChainedReporter(ctrl *gomock.Controller) *MockChainedReporter { - mock := &MockChainedReporter{ctrl: ctrl} - mock.recorder = &MockChainedReporterMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockChainedReporter) EXPECT() *MockChainedReporterMockRecorder { - return m.recorder -} - -// OnChildClose mocks base method -func (m *MockChainedReporter) OnChildClose(arg0 cost0.Cost) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "OnChildClose", arg0) -} - -// OnChildClose indicates an expected call of OnChildClose -func (mr *MockChainedReporterMockRecorder) OnChildClose(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OnChildClose", reflect.TypeOf((*MockChainedReporter)(nil).OnChildClose), arg0) -} - -// OnClose mocks base method -func (m *MockChainedReporter) OnClose(arg0 cost0.Cost) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "OnClose", arg0) -} - -// OnClose indicates an expected call of OnClose -func (mr *MockChainedReporterMockRecorder) OnClose(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OnClose", reflect.TypeOf((*MockChainedReporter)(nil).OnClose), arg0) -} - -// ReportCost mocks base method -func (m *MockChainedReporter) ReportCost(arg0 cost0.Cost) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "ReportCost", arg0) -} - -// ReportCost indicates an expected call of ReportCost -func (mr *MockChainedReporterMockRecorder) ReportCost(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportCost", reflect.TypeOf((*MockChainedReporter)(nil).ReportCost), arg0) -} - -// ReportCurrent mocks base method -func (m *MockChainedReporter) ReportCurrent(arg0 cost0.Cost) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "ReportCurrent", arg0) -} - -// ReportCurrent indicates an expected call of ReportCurrent -func (mr *MockChainedReporterMockRecorder) ReportCurrent(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportCurrent", reflect.TypeOf((*MockChainedReporter)(nil).ReportCurrent), arg0) -} - -// ReportOverLimit mocks base method -func (m *MockChainedReporter) ReportOverLimit(arg0 bool) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "ReportOverLimit", arg0) -} - -// ReportOverLimit indicates an expected call of ReportOverLimit -func (mr *MockChainedReporterMockRecorder) ReportOverLimit(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportOverLimit", reflect.TypeOf((*MockChainedReporter)(nil).ReportOverLimit), arg0) -} diff --git a/src/query/cost/cost_prop_test.go b/src/query/cost/cost_prop_test.go deleted file mode 100644 index 5bcc726b04..0000000000 --- a/src/query/cost/cost_prop_test.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package cost - -import ( - "math" - "sync" - "testing" - - "github.com/m3db/m3/src/x/cost" - - "github.com/leanovate/gopter" - "github.com/leanovate/gopter/gen" - "github.com/leanovate/gopter/prop" - "github.com/stretchr/testify/require" -) - -func TestPropertyPerQueryEnforcerAlwaysEndsUpZero(t *testing.T) { - testParams := gopter.DefaultTestParameters() - testParams.MinSuccessfulTests = 1000 - props := gopter.NewProperties(testParams) - - globalEndsUpZero := func(costs []float64, perQueryThreshold, globalThreshold float64) bool { - pqfIFace, err := NewChainedEnforcer( - "", - []cost.Enforcer{newTestEnforcer(cost.Limit{Threshold: cost.Cost(globalThreshold), Enabled: true}), - newTestEnforcer(cost.Limit{Threshold: cost.Cost(perQueryThreshold), Enabled: true})}) - require.NoError(t, err) - pqf := pqfIFace.(*chainedEnforcer) - wg := sync.WaitGroup{} - for _, c := range costs { - wg.Add(1) - go func(c float64) { - defer wg.Done() - - perQuery := pqf.Child("query") - defer perQuery.Close() - perQuery.Add(cost.Cost(c)) - }(c) - } - - wg.Wait() - r, _ := pqf.local.State() - - // do delta comparison to deal with floating point errors. TODO: cost could potentially be an int - const tolerance = 0.000001 - return math.Abs(float64(r.Cost)) < tolerance - } - - props.Property("global enforcer >= 0", - prop.ForAll( - globalEndsUpZero, - gen.SliceOf(gen.Float64Range(0, 10000)), - gen.Float64Range(0.0, 10000), - gen.Float64Range(0.0, 10000), - )) - - props.TestingRun(t) -} diff --git a/src/query/cost/cost_test.go b/src/query/cost/cost_test.go deleted file mode 100644 index a554f45d98..0000000000 --- a/src/query/cost/cost_test.go +++ /dev/null @@ -1,242 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package cost - -import ( - "fmt" - "math" - "testing" - - "github.com/m3db/m3/src/x/cost" - "github.com/m3db/m3/src/x/cost/test" - - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestChainedEnforcer_Child(t *testing.T) { - t.Run("creates independent local enforcers with shared global enforcer", func(t *testing.T) { - globalEnforcer := newTestEnforcer(cost.Limit{Threshold: 10.0, Enabled: true}) - localEnforcer := newTestEnforcer(cost.Limit{Threshold: 5.0, Enabled: true}) - - pef, err := NewChainedEnforcer("", []cost.Enforcer{globalEnforcer, localEnforcer}) - require.NoError(t, err) - - l1, l2 := pef.Child("foo"), pef.Child("foo") - - l1.Add(2) - - test.AssertCurrentCost(t, 2.0, l1) - test.AssertCurrentCost(t, 0.0, l2) - test.AssertCurrentCost(t, 2.0, globalEnforcer) - }) - - t.Run("returns a noop enforcer once out of models", func(t *testing.T) { - globalEnforcer := newTestEnforcer(cost.Limit{Threshold: 10.0, Enabled: true}) - pef, err := NewChainedEnforcer("", []cost.Enforcer{globalEnforcer}) - require.NoError(t, err) - - child := pef.Child("foo") - assert.Equal(t, noopChainedEnforcer, child) - }) -} - -func TestChainedEnforcer_Close(t *testing.T) { - t.Run("removes local total from global", func(t *testing.T) { - parentIface, err := NewChainedEnforcer( - "", - []cost.Enforcer{newTestEnforcer(cost.Limit{Threshold: 10.0, Enabled: true}), - newTestEnforcer(cost.Limit{Threshold: 5.0, Enabled: true})}) - require.NoError(t, err) - parent := parentIface.(*chainedEnforcer) - - pqe1, pqe2 := parent.Child("query"), parent.Child("query") - - pqe1.Add(cost.Cost(5.0)) - pqe1.Add(cost.Cost(6.0)) - - pqe2.Add(cost.Cost(7.0)) - - pqe1.Close() - - test.AssertCurrentCost(t, cost.Cost(7.0), parent.local) - pqe2.Close() - test.AssertCurrentCost(t, cost.Cost(0.0), parent.local) - }) - - t.Run("calls into reporter on release", func(t *testing.T) { - ctrl := gomock.NewController(t) - makeEnforcer := func(cr ChainedReporter) cost.Enforcer { - return cost.NewEnforcer(cost.NewStaticLimitManager(cost.NewLimitManagerOptions()), cost.NewTracker(), - cost.NewEnforcerOptions().SetReporter(cr)) - } - - makeReporter := func() *MockChainedReporter { - r := NewMockChainedReporter(ctrl) - r.EXPECT().ReportCurrent(gomock.Any()).AnyTimes() - r.EXPECT().ReportOverLimit(gomock.Any()).AnyTimes() - r.EXPECT().ReportCost(gomock.Any()).AnyTimes() - return r - } - - globalReporter, localReporter := makeReporter(), makeReporter() - - ce, err := NewChainedEnforcer( - "global", - []cost.Enforcer{makeEnforcer(globalReporter), makeEnforcer(localReporter)}) - - child := ce.Child("foo") - child.Add(1.0) - - require.NoError(t, err) - - globalReporter.EXPECT().OnChildClose(floatMatcher(1.0)) - localReporter.EXPECT().OnClose(floatMatcher(1.0)) - - child.Close() - }) -} - -// floatMatcher does a janky delta comparison between floats, since rounding error makes float equality treacherous -type floatMatcher float64 - -func (f floatMatcher) Matches(x interface{}) bool { - other, ok := x.(cost.Cost) - if !ok { - return false - } - - // janky delta comparison - return math.Abs(float64(f)-float64(other)) < 0.00001 -} - -func (f floatMatcher) String() string { - return fmt.Sprintf("%f", f) -} - -func TestChainedEnforcer_Add(t *testing.T) { - assertGlobalError := func(t *testing.T, err error) { - if assert.Error(t, err) { - assert.Regexp(t, "exceeded global limit", err.Error()) - } - } - - assertLocalError := func(t *testing.T, err error) { - if assert.Error(t, err) { - assert.Regexp(t, "exceeded query limit", err.Error()) - } - } - - t.Run("errors on global error", func(t *testing.T) { - pqe := newTestChainedEnforcer(5.0, 100.0) - r := pqe.Add(cost.Cost(6.0)) - assertGlobalError(t, r.Error) - }) - - t.Run("errors on local error", func(t *testing.T) { - pqe := newTestChainedEnforcer(100.0, 5.0) - r := pqe.Add(cost.Cost(6.0)) - assertLocalError(t, r.Error) - }) - - t.Run("adds to local in case of global error", func(t *testing.T) { - pqe := newTestChainedEnforcer(5.0, 100.0) - r := pqe.Add(cost.Cost(6.0)) - assertGlobalError(t, r.Error) - - r, _ = pqe.State() - assert.Equal(t, cost.Report{ - Error: nil, - Cost: 6.0}, - r) - }) - - t.Run("adds to global in case of local error", func(t *testing.T) { - pqe := newTestChainedEnforcer(100.0, 5.0) - r := pqe.Add(cost.Cost(6.0)) - assertLocalError(t, r.Error) - - r, _ = pqe.parent.State() - assert.Equal(t, cost.Report{ - Error: nil, - Cost: 6.0}, - r) - }) - - t.Run("release after local error", func(t *testing.T) { - pqe := newTestChainedEnforcer(10.0, 5.0) - - // exceeds local - r := pqe.Add(6.0) - assertLocalError(t, r.Error) - - pqe.Close() - test.AssertCurrentCost(t, 0.0, pqe.local) - }) - - t.Run("release after global error", func(t *testing.T) { - pqe := newTestChainedEnforcer(5.0, 10.0) - // exceeds global - r := pqe.Add(6.0) - assertGlobalError(t, r.Error) - pqe.Close() - test.AssertCurrentCost(t, 0.0, pqe.local) - }) -} - -func TestChainedEnforcer_State(t *testing.T) { - pqe := newTestChainedEnforcer(10.0, 5.0) - pqe.Add(15.0) - - r, l := pqe.State() - assert.Equal(t, cost.Cost(15), r.Cost) - test.AssertLimitError(t, r.Error, 15.0, 5.0) - assert.Equal(t, cost.Limit{Threshold: 5.0, Enabled: true}, l) -} - -func TestNoopChainedEnforcer_Close(t *testing.T) { - ce := NoopChainedEnforcer() - ce.Close() - test.AssertCurrentCost(t, 0.0, ce) -} - -// utils - -func newTestEnforcer(limit cost.Limit) cost.Enforcer { - return cost.NewEnforcer( - cost.NewStaticLimitManager(cost.NewLimitManagerOptions().SetDefaultLimit(limit)), - cost.NewTracker(), - nil, - ) -} - -func newTestChainedEnforcer(globalLimit float64, localLimit float64) *chainedEnforcer { - rtn, err := NewChainedEnforcer( - "global", - []cost.Enforcer{newTestEnforcer(cost.Limit{Threshold: cost.Cost(globalLimit), Enabled: true}), - newTestEnforcer(cost.Limit{Threshold: cost.Cost(localLimit), Enabled: true})}) - if err != nil { - panic(err.Error()) - } - - return rtn.Child("query").(*chainedEnforcer) -} diff --git a/src/query/executor/engine.go b/src/query/executor/engine.go index 584d141388..c8bd17937c 100644 --- a/src/query/executor/engine.go +++ b/src/query/executor/engine.go @@ -25,7 +25,6 @@ import ( "time" "github.com/m3db/m3/src/query/block" - qcost "github.com/m3db/m3/src/query/cost" "github.com/m3db/m3/src/query/models" "github.com/m3db/m3/src/query/parser" "github.com/m3db/m3/src/query/storage" @@ -48,10 +47,6 @@ type QueryOptions struct { func NewEngine( engineOpts EngineOptions, ) Engine { - if engineOpts.GlobalEnforcer() == nil { - engineOpts = engineOpts.SetGlobalEnforcer(qcost.NoopChainedEnforcer()) - } - return &engine{ metrics: newEngineMetrics(engineOpts.InstrumentOptions().MetricsScope()), opts: engineOpts, @@ -120,8 +115,6 @@ func (e *engine) ExecuteExpr( fetchOpts *storage.FetchOptions, params models.RequestParams, ) (block.Block, error) { - perQueryEnforcer := e.opts.GlobalEnforcer().Child(qcost.QueryLevel) - defer perQueryEnforcer.Close() req := newRequest(e, params, fetchOpts, e.opts.InstrumentOptions()) nodes, edges, err := req.compile(ctx, parser) if err != nil { @@ -143,7 +136,7 @@ func (e *engine) ExecuteExpr( defer sp.Finish() scope := e.opts.InstrumentOptions().MetricsScope() - queryCtx := models.NewQueryContext(ctx, scope, perQueryEnforcer, + queryCtx := models.NewQueryContext(ctx, scope, opts.QueryContextOptions) if err := state.Execute(queryCtx); err != nil { diff --git a/src/query/executor/engine_test.go b/src/query/executor/engine_test.go index 8906c613c1..aa7b80f383 100644 --- a/src/query/executor/engine_test.go +++ b/src/query/executor/engine_test.go @@ -28,8 +28,6 @@ import ( "github.com/m3db/m3/src/dbnode/client" "github.com/m3db/m3/src/query/block" - "github.com/m3db/m3/src/query/cost" - qcost "github.com/m3db/m3/src/query/cost" "github.com/m3db/m3/src/query/models" "github.com/m3db/m3/src/query/parser/promql" "github.com/m3db/m3/src/query/storage" @@ -45,13 +43,11 @@ import ( func newEngine( s storage.Storage, lookbackDuration time.Duration, - enforcer qcost.ChainedEnforcer, instrumentOpts instrument.Options, ) Engine { engineOpts := NewEngineOptions(). SetStore(s). SetLookbackDuration(lookbackDuration). - SetGlobalEnforcer(enforcer). SetInstrumentOptions(instrumentOpts) return NewEngine(engineOpts) @@ -65,7 +61,7 @@ func TestExecute(t *testing.T) { session.EXPECT().IteratorPools().Return(nil, nil) // Results is closed by execute - engine := newEngine(store, time.Minute, nil, instrument.NewOptions()) + engine := newEngine(store, time.Minute, instrument.NewOptions()) _, err := engine.ExecuteProm(context.TODO(), &storage.FetchQuery{}, &QueryOptions{}, storage.NewFetchOptions()) assert.NotNil(t, err) @@ -75,12 +71,6 @@ func TestExecuteExpr(t *testing.T) { ctrl := xtest.NewController(t) defer ctrl.Finish() - mockEnforcer := cost.NewMockChainedEnforcer(ctrl) - mockEnforcer.EXPECT().Close().Times(1) - - mockParent := cost.NewMockChainedEnforcer(ctrl) - mockParent.EXPECT().Child(gomock.Any()).Return(mockEnforcer) - parser, err := promql.Parse("foo", time.Second, models.NewTagOptions(), promql.NewParseOptions()) require.NoError(t, err) @@ -91,7 +81,7 @@ func TestExecuteExpr(t *testing.T) { Blocks: []block.Block{block.NewMockBlock(ctrl)}, }, nil) engine := newEngine(store, defaultLookbackDuration, - mockParent, instrument.NewOptions()) + instrument.NewOptions()) _, err = engine.ExecuteExpr(context.TODO(), parser, &QueryOptions{}, storage.NewFetchOptions(), models.RequestParams{ Start: time.Now().Add(-2 * time.Second), diff --git a/src/query/executor/options.go b/src/query/executor/options.go index ea4cb88f0c..456ff7f60c 100644 --- a/src/query/executor/options.go +++ b/src/query/executor/options.go @@ -23,7 +23,6 @@ package executor import ( "time" - qcost "github.com/m3db/m3/src/query/cost" "github.com/m3db/m3/src/query/parser/promql" "github.com/m3db/m3/src/query/storage" "github.com/m3db/m3/src/x/instrument" @@ -31,7 +30,6 @@ import ( type engineOptions struct { instrumentOpts instrument.Options - globalEnforcer qcost.ChainedEnforcer store storage.Storage parseOptions promql.ParseOptions lookbackDuration time.Duration @@ -54,16 +52,6 @@ func (o *engineOptions) SetInstrumentOptions(v instrument.Options) EngineOptions return &opts } -func (o *engineOptions) GlobalEnforcer() qcost.ChainedEnforcer { - return o.globalEnforcer -} - -func (o *engineOptions) SetGlobalEnforcer(v qcost.ChainedEnforcer) EngineOptions { - opts := *o - opts.globalEnforcer = v - return &opts -} - func (o *engineOptions) Store() storage.Storage { return o.store } diff --git a/src/query/executor/types.go b/src/query/executor/types.go index c5194c2f84..83ebb7efd8 100644 --- a/src/query/executor/types.go +++ b/src/query/executor/types.go @@ -25,7 +25,6 @@ import ( "time" "github.com/m3db/m3/src/query/block" - qcost "github.com/m3db/m3/src/query/cost" "github.com/m3db/m3/src/query/models" "github.com/m3db/m3/src/query/parser" "github.com/m3db/m3/src/query/parser/promql" @@ -69,11 +68,6 @@ type EngineOptions interface { // for metrics. SetInstrumentOptions(instrument.Options) EngineOptions - // GlobalEnforcer returns the query cost enforcer. - GlobalEnforcer() qcost.ChainedEnforcer - // SetGlobalEnforcer sets the query cost enforcer. - SetGlobalEnforcer(qcost.ChainedEnforcer) EngineOptions - // Store returns the storage. Store() storage.Storage // SetStore sets the storage. diff --git a/src/query/functions/fetch_test.go b/src/query/functions/fetch_test.go index 0e6ae9a1fa..9fa3d4baae 100644 --- a/src/query/functions/fetch_test.go +++ b/src/query/functions/fetch_test.go @@ -27,7 +27,6 @@ import ( "github.com/m3db/m3/src/metrics/policy" "github.com/m3db/m3/src/query/block" - "github.com/m3db/m3/src/query/cost" "github.com/m3db/m3/src/query/executor/transform" "github.com/m3db/m3/src/query/models" "github.com/m3db/m3/src/query/parser" @@ -133,7 +132,7 @@ func TestFetchWithRestrictFetch(t *testing.T) { transformtest.Options(t, transform.OptionsParams{})) ctx := models.NewQueryContext(context.Background(), - tally.NoopScope, cost.NoopChainedEnforcer(), + tally.NoopScope, models.QueryContextOptions{ RestrictFetchType: &models.RestrictFetchTypeQueryContextOptions{ MetricsType: uint(storagemetadata.AggregatedMetricsType), diff --git a/src/query/generated/mocks/generate.go b/src/query/generated/mocks/generate.go index 95393fd6e6..199b0eac0b 100644 --- a/src/query/generated/mocks/generate.go +++ b/src/query/generated/mocks/generate.go @@ -27,7 +27,6 @@ //go:generate sh -c "mockgen -package=ingest -destination=$GOPATH/src/$PACKAGE/src/cmd/services/m3coordinator/ingest/write_mock.go $PACKAGE/src/cmd/services/m3coordinator/ingest DownsamplerAndWriter" //go:generate sh -c "mockgen -package=transform -destination=$GOPATH/src/$PACKAGE/src/query/executor/transform/types_mock.go $PACKAGE/src/query/executor/transform OpNode" //go:generate sh -c "mockgen -package=executor -destination=$GOPATH/src/$PACKAGE/src/query/executor/types_mock.go $PACKAGE/src/query/executor Engine" -//go:generate sh -c "mockgen -package=cost -destination=$GOPATH/src/github.com/m3db/m3/src/query/cost/cost_mock.go $PACKAGE/src/query/cost ChainedEnforcer,ChainedReporter" //go:generate sh -c "mockgen -package=storage -destination=$GOPATH/src/$PACKAGE/src/query/graphite/storage/storage_mock.go $PACKAGE/src/query/graphite/storage Storage" // mockgen rules for generating mocks for unexported interfaces (file mode). diff --git a/src/query/graphite/storage/m3_wrapper.go b/src/query/graphite/storage/m3_wrapper.go index d5c474ceaa..0ba21ab747 100644 --- a/src/query/graphite/storage/m3_wrapper.go +++ b/src/query/graphite/storage/m3_wrapper.go @@ -29,7 +29,6 @@ import ( "time" "github.com/m3db/m3/src/query/block" - "github.com/m3db/m3/src/query/cost" xctx "github.com/m3db/m3/src/query/graphite/context" "github.com/m3db/m3/src/query/graphite/graphite" "github.com/m3db/m3/src/query/graphite/ts" @@ -48,7 +47,6 @@ var ( type m3WrappedStore struct { m3 storage.Storage - enforcer cost.ChainedEnforcer m3dbOpts m3db.Options instrumentOpts instrument.Options opts M3WrappedStorageOptions @@ -78,18 +76,12 @@ type seriesMetadata struct { // storage instance. func NewM3WrappedStorage( m3storage storage.Storage, - enforcer cost.ChainedEnforcer, m3dbOpts m3db.Options, instrumentOpts instrument.Options, opts M3WrappedStorageOptions, ) Storage { - if enforcer == nil { - enforcer = cost.NoopChainedEnforcer() - } - return &m3WrappedStore{ m3: m3storage, - enforcer: enforcer, m3dbOpts: m3dbOpts, instrumentOpts: instrumentOpts, opts: opts, @@ -396,12 +388,9 @@ func (s *m3WrappedStore) FetchByQuery( defer cancel() fetchOptions := storage.NewFetchOptions() fetchOptions.SeriesLimit = fetchOpts.Limit - perQueryEnforcer := s.enforcer.Child(cost.QueryLevel) - defer perQueryEnforcer.Close() // NB: ensure single block return. fetchOptions.BlockType = models.TypeSingleBlock - fetchOptions.Enforcer = perQueryEnforcer fetchOptions.FanoutOptions = &storage.FanoutOptions{ FanoutUnaggregated: storage.FanoutForceDisable, FanoutAggregated: storage.FanoutDefault, diff --git a/src/query/graphite/storage/m3_wrapper_test.go b/src/query/graphite/storage/m3_wrapper_test.go index cb52d52fab..bb57ce9a1b 100644 --- a/src/query/graphite/storage/m3_wrapper_test.go +++ b/src/query/graphite/storage/m3_wrapper_test.go @@ -27,7 +27,6 @@ import ( "time" "github.com/m3db/m3/src/query/block" - "github.com/m3db/m3/src/query/cost" xctx "github.com/m3db/m3/src/query/graphite/context" "github.com/m3db/m3/src/query/graphite/graphite" "github.com/m3db/m3/src/query/models" @@ -195,13 +194,7 @@ func TestFetchByQuery(t *testing.T) { store.EXPECT().FetchBlocks(gomock.Any(), gomock.Any(), gomock.Any()). Return(res, nil) - childEnforcer := cost.NewMockChainedEnforcer(ctrl) - childEnforcer.EXPECT().Close() - - enforcer := cost.NewMockChainedEnforcer(ctrl) - enforcer.EXPECT().Child(cost.QueryLevel).Return(childEnforcer).MinTimes(1) - - wrapper := NewM3WrappedStorage(store, enforcer, testM3DBOpts, + wrapper := NewM3WrappedStorage(store, testM3DBOpts, instrument.NewOptions(), M3WrappedStorageOptions{}) ctx := xctx.New() ctx.SetRequestContext(context.TODO()) @@ -243,7 +236,7 @@ func TestFetchByInvalidQuery(t *testing.T) { query := "a." ctx := xctx.New() - wrapper := NewM3WrappedStorage(store, nil, testM3DBOpts, + wrapper := NewM3WrappedStorage(store, testM3DBOpts, instrument.NewOptions(), M3WrappedStorageOptions{}) result, err := wrapper.FetchByQuery(ctx, query, opts) assert.NoError(t, err) diff --git a/src/query/models/query_context.go b/src/query/models/query_context.go index abb87f47cf..0da1e3daf5 100644 --- a/src/query/models/query_context.go +++ b/src/query/models/query_context.go @@ -24,19 +24,15 @@ import ( "context" "github.com/m3db/m3/src/metrics/policy" - "github.com/m3db/m3/src/query/cost" "github.com/uber-go/tally" ) // QueryContext provides all external state needed to execute and track a query. -// It acts as a hook back into the execution engine for things like -// cost accounting. type QueryContext struct { - Ctx context.Context - Scope tally.Scope - Enforcer cost.ChainedEnforcer - Options QueryContextOptions + Ctx context.Context + Scope tally.Scope + Options QueryContextOptions } // QueryContextOptions contains optional configuration for the query context. @@ -58,26 +54,23 @@ type RestrictFetchTypeQueryContextOptions struct { StoragePolicy policy.StoragePolicy } -// NewQueryContext constructs a QueryContext using the given Enforcer to -// enforce per query limits. +// NewQueryContext constructs a QueryContext from the provided native context. func NewQueryContext( ctx context.Context, scope tally.Scope, - enforcer cost.ChainedEnforcer, options QueryContextOptions, ) *QueryContext { return &QueryContext{ - Ctx: ctx, - Scope: scope, - Enforcer: enforcer, - Options: options, + Ctx: ctx, + Scope: scope, + Options: options, } } // NoopQueryContext returns a query context with no active components. func NoopQueryContext() *QueryContext { return NewQueryContext(context.Background(), tally.NoopScope, - cost.NoopChainedEnforcer(), QueryContextOptions{}) + QueryContextOptions{}) } // WithContext creates a shallow copy of this QueryContext using the new context. diff --git a/src/query/remote/client.go b/src/query/remote/client.go index 4fc3d93e18..e4c6e0ba7d 100644 --- a/src/query/remote/client.go +++ b/src/query/remote/client.go @@ -243,8 +243,9 @@ func (c *grpcClient) FetchProm( } return storage.SeriesIteratorsToPromResult( - result, c.opts.ReadWorkerPool(), - options.Enforcer, c.opts.TagOptions()) + result, + c.opts.ReadWorkerPool(), + c.opts.TagOptions()) } func (c *grpcClient) fetchRaw( diff --git a/src/query/server/cost_reporters.go b/src/query/server/cost_reporters.go deleted file mode 100644 index 0ff5a724f6..0000000000 --- a/src/query/server/cost_reporters.go +++ /dev/null @@ -1,257 +0,0 @@ -// Copyright (c) 2019 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package server - -// This file contains reporters and setup for our query/cost.ChainedEnforcer -// instances. -import ( - "fmt" - "sync" - - qcost "github.com/m3db/m3/src/query/cost" - "github.com/m3db/m3/src/x/close" - "github.com/m3db/m3/src/x/cost" - "github.com/m3db/m3/src/x/instrument" - - "github.com/uber-go/tally" -) - -const ( - costScopeName = "cost" - limitManagerScopeName = "limits" - reporterScopeName = "reporter" - queriesOverLimitMetric = "over_datapoints_limit" - datapointsMetric = "datapoints" - datapointsCounterMetric = "datapoints_counter" - maxDatapointsHistMetric = "max_datapoints_hist" -) - -// newConfiguredChainedEnforcer returns a ChainedEnforcer with 3 configured -// levels: global, per-query, per-block. Global and per-query both have limits -// on them (as configured by cfg.Limits); per-block is purely for accounting -// purposes. -// Our enforcers report at least these stats: -// cost_reporter_datapoints{limit="global"}: gauge; -// > the number of datapoints currently in use by this instance. -// -// cost_reporter_datapoints_counter{limiter="global"}: counter; -// > counter representation of the number of datapoints in use by this instance. -// -// cost_reporter_over_datapoints_limit{limiter=~"(global|per_query)"}: counter; -// > how many queries are over the datapoint limit. -// -// cost_reporter_max_datapoints_hist{limiter=~"(global|per_query)"}: histogram; -// > represents the distribution of the maximum datapoints used at any point in each query. -func newConfiguredChainedEnforcer( - instrumentOptions instrument.Options, -) (qcost.ChainedEnforcer, close.SimpleCloser, error) { - scope := instrumentOptions.MetricsScope().SubScope(costScopeName) - - exceededMessage := func(exceedType, exceedLimit string) string { - return fmt.Sprintf("exceeded limits.%s.%s", exceedType, exceedLimit) - } - - // Create global limit manager and enforcer. - globalScope := scope.Tagged(map[string]string{ - "limiter": "global", - }) - globalLimitManagerScope := globalScope.SubScope(limitManagerScopeName) - globalReporterScope := globalScope.SubScope(reporterScopeName) - - globalLimitMgr := cost.NewStaticLimitManager( - cost.NewLimitManagerOptions().SetDefaultLimit(cost.Limit{ - Threshold: cost.Cost(0), - Enabled: false, - }).SetInstrumentOptions(instrumentOptions.SetMetricsScope(globalLimitManagerScope))) - - globalTracker := cost.NewTracker() - - globalEnforcer := cost.NewEnforcer(globalLimitMgr, globalTracker, - cost.NewEnforcerOptions(). - SetReporter(newGlobalReporter(globalReporterScope)). - SetCostExceededMessage(exceededMessage("global", "maxFetchedDatapoints"))) - - // Create per query limit manager and enforcer. - queryScope := scope.Tagged(map[string]string{ - "limiter": "query", - }) - queryLimitManagerScope := queryScope.SubScope(limitManagerScopeName) - queryReporterScope := queryScope.SubScope(reporterScopeName) - - queryLimitMgr := cost.NewStaticLimitManager( - cost.NewLimitManagerOptions().SetDefaultLimit(cost.Limit{ - Threshold: cost.Cost(0), - Enabled: false, - }).SetInstrumentOptions(instrumentOptions.SetMetricsScope(queryLimitManagerScope))) - - queryTracker := cost.NewTracker() - - queryEnforcer := cost.NewEnforcer(queryLimitMgr, queryTracker, - cost.NewEnforcerOptions(). - SetReporter(newPerQueryReporter(queryReporterScope)). - SetCostExceededMessage(exceededMessage("perQuery", "maxFetchedDatapoints"))) - - // Create block enforcer. - blockEnforcer := cost.NewEnforcer( - cost.NewStaticLimitManager(cost.NewLimitManagerOptions().SetDefaultLimit(cost.Limit{Enabled: false})), - cost.NewTracker(), - nil) - - // Create chained enforcer. - enforcer, err := qcost.NewChainedEnforcer(qcost.GlobalLevel, []cost.Enforcer{ - globalEnforcer, - queryEnforcer, - blockEnforcer, - }) - if err != nil { - return nil, nil, err - } - - // Start reporting stats for all limit managers. - go globalLimitMgr.Report() - go queryLimitMgr.Report() - - // Close the stats at the end. - closer := close.SimpleCloserFn(func() { - globalLimitMgr.Close() - queryLimitMgr.Close() - }) - - return enforcer, closer, nil -} - -// globalReporter records ChainedEnforcer statistics for the global enforcer. -type globalReporter struct { - datapoints tally.Gauge - datapointsCounter tally.Counter - overLimit overLimitReporter -} - -// assert we implement the interface -var _ cost.EnforcerReporter = (*globalReporter)(nil) - -func newGlobalReporter(s tally.Scope) *globalReporter { - return &globalReporter{ - datapoints: s.Gauge(datapointsMetric), - datapointsCounter: s.Counter(datapointsCounterMetric), - overLimit: newOverLimitReporter(s), - } -} - -func (gr *globalReporter) ReportCurrent(c cost.Cost) { - gr.datapoints.Update(float64(c)) -} - -// ReportCost for global reporters sends the new incoming cost to a counter. -// Since counters can only be incremented, it ignores negative values. -func (gr *globalReporter) ReportCost(c cost.Cost) { - if c > 0 { - gr.datapointsCounter.Inc(int64(c)) - } -} - -// ReportOverLimit delegates to gr.overLimit -func (gr *globalReporter) ReportOverLimit(enabled bool) { - gr.overLimit.ReportOverLimit(enabled) -} - -// perQueryReporter records ChainedEnforcer statistics on a per query level. -type perQueryReporter struct { - mu *sync.Mutex - maxDatapoints cost.Cost - queryHisto tally.Histogram - overLimit overLimitReporter -} - -// assert we implement the interface -var _ qcost.ChainedReporter = (*perQueryReporter)(nil) - -func newPerQueryReporter(scope tally.Scope) *perQueryReporter { - return &perQueryReporter{ - mu: &sync.Mutex{}, - maxDatapoints: 0, - queryHisto: scope.Histogram(maxDatapointsHistMetric, - tally.MustMakeExponentialValueBuckets(10.0, 10.0, 6)), - overLimit: newOverLimitReporter(scope), - } -} - -// ReportCost is a noop for perQueryReporter because it's noisy to report -// the current cost for every query (hard to meaningfully divide out). -// Instead, we report the max datapoints at the end of the query--see on -// release. -func (perQueryReporter) ReportCost(c cost.Cost) {} - -// ReportCurrent is a noop for perQueryReporter--see ReportCost for -// explanation. -func (perQueryReporter) ReportCurrent(c cost.Cost) {} - -// ReportOverLimit reports when a query is over its per query limit. -func (pr *perQueryReporter) ReportOverLimit(enabled bool) { - pr.overLimit.ReportOverLimit(enabled) -} - -// OnChildClose takes the max of the current cost for this query and the -// previously recorded cost. We do this OnChildRelease instead of on -// ReportCurrent to avoid locking every time we add to the Enforcer. -func (pr *perQueryReporter) OnChildClose(curCost cost.Cost) { - pr.mu.Lock() - if curCost > pr.maxDatapoints { - pr.maxDatapoints = curCost - } - pr.mu.Unlock() -} - -// OnClose records the maximum cost seen by this reporter. -func (pr *perQueryReporter) OnClose(curCost cost.Cost) { - pr.mu.Lock() - pr.queryHisto.RecordValue(float64(pr.maxDatapoints)) - pr.mu.Unlock() -} - -// overLimitReporter factors out reporting over limit cases for both global -// and per query enforcer reporters. -type overLimitReporter struct { - queriesOverLimitDisabled tally.Counter - queriesOverLimitEnabled tally.Counter -} - -func newOverLimitReporter(scope tally.Scope) overLimitReporter { - return overLimitReporter{ - queriesOverLimitDisabled: scope.Tagged(map[string]string{ - "enabled": "false", - }).Counter(queriesOverLimitMetric), - - queriesOverLimitEnabled: scope.Tagged(map[string]string{ - "enabled": "true", - }).Counter(queriesOverLimitMetric), - } -} - -// ReportOverLimit increments .over_limit, tagged by -// "enabled". -func (olr overLimitReporter) ReportOverLimit(enabled bool) { - if enabled { - olr.queriesOverLimitEnabled.Inc(1) - } else { - olr.queriesOverLimitDisabled.Inc(1) - } -} diff --git a/src/query/server/cost_reporters_test.go b/src/query/server/cost_reporters_test.go deleted file mode 100644 index b6b7c70086..0000000000 --- a/src/query/server/cost_reporters_test.go +++ /dev/null @@ -1,275 +0,0 @@ -// Copyright (c) 2019 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package server - -import ( - "fmt" - "math" - "testing" - - "github.com/m3db/m3/src/query/cost" - "github.com/m3db/m3/src/x/close" - "github.com/m3db/m3/src/x/cost/test" - "github.com/m3db/m3/src/x/instrument" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/uber-go/tally" -) - -type enforcerTestCtx struct { - Scope tally.TestScope - GlobalEnforcer cost.ChainedEnforcer - Closer close.SimpleCloser -} - -func (c enforcerTestCtx) Close() { - c.Closer.Close() -} - -func TestNewConfiguredChainedEnforcer(t *testing.T) { - setup := func(t *testing.T, perQueryLimit, globalLimit int) enforcerTestCtx { - s := tally.NewTestScope("", nil) - iopts := instrument.NewOptions().SetMetricsScope(s) - - globalEnforcer, closer, err := newConfiguredChainedEnforcer(iopts) - - require.NoError(t, err) - - return enforcerTestCtx{ - Scope: s, - GlobalEnforcer: globalEnforcer, - Closer: closer, - } - } - - t.Run("has 3 valid levels", func(t *testing.T) { - tctx := setup(t, 6, 10) - defer tctx.Close() - - assertValid := func(ce cost.ChainedEnforcer) { - assert.NotEqual(t, ce, cost.NoopChainedEnforcer()) - } - - assertValid(tctx.GlobalEnforcer) - - qe := tctx.GlobalEnforcer.Child(cost.QueryLevel) - assertValid(qe) - - block := qe.Child(cost.BlockLevel) - assertValid(block) - - badLevel := block.Child("nonExistent") - assert.Equal(t, cost.NoopChainedEnforcer(), - badLevel) - }) - - t.Run("configures reporters", func(t *testing.T) { - tctx := setup(t, 6, 10) - defer tctx.Close() - - queryEf := tctx.GlobalEnforcer.Child(cost.QueryLevel) - blockEf := queryEf.Child(cost.BlockLevel) - blockEf.Add(7) - - assertHasGauge(t, - tctx.Scope.Snapshot(), - tally.KeyForPrefixedStringMap( - fmt.Sprintf("cost.reporter.%s", datapointsMetric), map[string]string{"limiter": "global"}), - 7, - ) - - blockEf.Close() - queryEf.Close() - - assertHasHistogram(t, - tctx.Scope.Snapshot(), - tally.KeyForPrefixedStringMap( - fmt.Sprintf("cost.reporter.%s", maxDatapointsHistMetric), map[string]string{"limiter": "query"}), - map[float64]int64{10: 1}, - ) - }) - - t.Run("block level doesn't have a limit", func(t *testing.T) { - tctx := setup(t, -1, -1) - defer tctx.Close() - - block := tctx.GlobalEnforcer.Child(cost.QueryLevel).Child(cost.BlockLevel) - assert.NoError(t, block.Add(math.MaxFloat64-1).Error) - }) - - t.Run("works e2e", func(t *testing.T) { - tctx := setup(t, 6, 10) - defer tctx.Close() - - qe1, qe2 := tctx.GlobalEnforcer.Child(cost.QueryLevel), tctx.GlobalEnforcer.Child(cost.QueryLevel) - r := qe1.Add(6) - assert.NoError(t, r.Error) - - r = qe2.Add(3) - require.NoError(t, r.Error) - - r = qe2.Add(2) - require.NoError(t, r.Error) - - test.AssertCurrentCost(t, 11, tctx.GlobalEnforcer) - - qe2.Close() - test.AssertCurrentCost(t, 6, tctx.GlobalEnforcer) - - // check the block level - blockEf := qe1.Child(cost.BlockLevel) - blockEf.Add(2) - - test.AssertCurrentCost(t, 2, blockEf) - test.AssertCurrentCost(t, 8, qe1) - test.AssertCurrentCost(t, 8, tctx.GlobalEnforcer) - }) -} - -func setupGlobalReporter() (tally.TestScope, *globalReporter) { - s := tally.NewTestScope("", nil) - gr := newGlobalReporter(s) - return s, gr -} - -func TestGlobalReporter_ReportCurrent(t *testing.T) { - s, gr := setupGlobalReporter() - - gr.ReportCurrent(5.0) - assertHasGauge(t, s.Snapshot(), tally.KeyForPrefixedStringMap(datapointsMetric, nil), 5.0) -} - -func TestGlobalReporter_ReportCost(t *testing.T) { - t.Run("reports positive", func(t *testing.T) { - s, gr := setupGlobalReporter() - gr.ReportCost(5.0) - assertHasCounter(t, s.Snapshot(), tally.KeyForPrefixedStringMap(datapointsCounterMetric, nil), 5.0) - }) - - t.Run("skips negative", func(t *testing.T) { - s, gr := setupGlobalReporter() - gr.ReportCost(-5.0) - - assertHasCounter(t, s.Snapshot(), tally.KeyForPrefixedStringMap(datapointsCounterMetric, nil), 0.0) - }) -} - -func TestGlobalReporter_ReportOverLimit(t *testing.T) { - s, gr := setupGlobalReporter() - gr.ReportOverLimit(true) - assertHasCounter(t, s.Snapshot(), tally.KeyForPrefixedStringMap(queriesOverLimitMetric, map[string]string{ - "enabled": "true", - }), 1) -} - -func setupPerQueryReporter() (tally.TestScope, *perQueryReporter) { - s := tally.NewTestScope("", nil) - gr := newPerQueryReporter(s) - return s, gr -} - -func TestPerQueryReporter_ReportOverLimit(t *testing.T) { - s, pqr := setupPerQueryReporter() - pqr.ReportOverLimit(true) - assertHasCounter(t, s.Snapshot(), tally.KeyForPrefixedStringMap(queriesOverLimitMetric, map[string]string{ - "enabled": "true", - }), 1) -} - -func TestPerQueryReporter_OnClose(t *testing.T) { - s, pqr := setupPerQueryReporter() - pqr.OnChildClose(5.0) - pqr.OnChildClose(110.0) - - // ignores current cost - pqr.OnClose(100.0) - assertHasHistogram(t, s.Snapshot(), - tally.KeyForPrefixedStringMap(maxDatapointsHistMetric, nil), - map[float64]int64{ - 1000.0: 1, - }) -} - -func TestPerQueryReporter_OnChildClose(t *testing.T) { - _, pqr := setupPerQueryReporter() - pqr.OnChildClose(5.0) - pqr.OnChildClose(110.0) - - assert.InDelta(t, 110.0, float64(pqr.maxDatapoints), 0.0001) -} - -func TestOverLimitReporter_ReportOverLimit(t *testing.T) { - s := tally.NewTestScope("", nil) - orl := newOverLimitReporter(s) - - orl.ReportOverLimit(true) - assertHasCounter(t, s.Snapshot(), tally.KeyForPrefixedStringMap(queriesOverLimitMetric, map[string]string{ - "enabled": "true", - }), 1) - - orl.ReportOverLimit(false) - assertHasCounter(t, s.Snapshot(), tally.KeyForPrefixedStringMap(queriesOverLimitMetric, map[string]string{ - "enabled": "true", - }), 1) -} - -func assertHasCounter(t *testing.T, snapshot tally.Snapshot, key string, v int) { - counters := snapshot.Counters() - if !assert.Contains(t, counters, key, "No such metric: %s", key) { - return - } - - counter := counters[key] - - assert.Equal(t, int(counter.Value()), v, "Incorrect value for counter %s", key) -} - -func assertHasGauge(t *testing.T, snapshot tally.Snapshot, key string, v int) { - gauges := snapshot.Gauges() - if !assert.Contains(t, gauges, key, "No such metric: %s", key) { - return - } - - gauge := gauges[key] - - assert.Equal(t, int(gauge.Value()), v, "Incorrect value for gauge %s", key) -} - -func assertHasHistogram(t *testing.T, snapshot tally.Snapshot, key string, values map[float64]int64) { - histograms := snapshot.Histograms() - if !assert.Contains(t, histograms, key, "No such metric: %s", key) { - return - } - - hist := histograms[key] - - actualValues := hist.Values() - - // filter zero values - for k, v := range actualValues { - if v == 0 { - delete(actualValues, k) - } - } - - assert.Equal(t, values, actualValues) -} diff --git a/src/query/server/query.go b/src/query/server/query.go index 479b8ef29e..f8a60dc7b4 100644 --- a/src/query/server/query.go +++ b/src/query/server/query.go @@ -422,14 +422,6 @@ func Run(runOpts RunOptions) { logger.Fatal("unrecognized backend", zap.String("backend", string(cfg.Backend))) } - chainedEnforcer, chainedEnforceCloser, err := newConfiguredChainedEnforcer( - instrumentOptions, - ) - if err != nil { - logger.Fatal("unable to setup chained enforcer", zap.Error(err)) - } - - defer chainedEnforceCloser.Close() if fn := runOpts.BackendStorageTransform; fn != nil { backendStorage = fn(backendStorage, tsdbOpts, instrumentOptions) } @@ -437,7 +429,6 @@ func Run(runOpts RunOptions) { engineOpts := executor.NewEngineOptions(). SetStore(backendStorage). SetLookbackDuration(*cfg.LookbackDuration). - SetGlobalEnforcer(chainedEnforcer). SetInstrumentOptions(instrumentOptions. SetMetricsScope(instrumentOptions.MetricsScope().SubScope("engine"))) if fn := runOpts.CustomPromQLParseFunction; fn != nil { @@ -491,7 +482,7 @@ func Run(runOpts RunOptions) { instrumentOptions) handlerOptions, err := options.NewHandlerOptions(downsamplerAndWriter, tagOptions, engine, prometheusEngine, m3dbClusters, clusterClient, cfg, - runOpts.DBConfig, chainedEnforcer, fetchOptsBuilder, queryCtxOpts, + runOpts.DBConfig, fetchOptsBuilder, queryCtxOpts, instrumentOptions, cpuProfileDuration, []string{handleroptions.M3DBServiceName}, serviceOptionDefaults, httpd.NewQueryRouter(), httpd.NewQueryRouter(), graphiteStorageOpts, tsdbOpts) diff --git a/src/query/server/query_test.go b/src/query/server/query_test.go index a5ed325584..d1f95e0171 100644 --- a/src/query/server/query_test.go +++ b/src/query/server/query_test.go @@ -42,14 +42,11 @@ import ( m3msgproto "github.com/m3db/m3/src/msg/protocol/proto" "github.com/m3db/m3/src/query/api/v1/handler/prometheus/remote" "github.com/m3db/m3/src/query/api/v1/handler/prometheus/remote/test" - "github.com/m3db/m3/src/query/cost" rpc "github.com/m3db/m3/src/query/generated/proto/rpcpb" "github.com/m3db/m3/src/query/storage/m3" xclock "github.com/m3db/m3/src/x/clock" - "github.com/m3db/m3/src/x/close" xconfig "github.com/m3db/m3/src/x/config" "github.com/m3db/m3/src/x/ident" - "github.com/m3db/m3/src/x/instrument" "github.com/m3db/m3/src/x/serialize" xtest "github.com/m3db/m3/src/x/test" @@ -57,7 +54,6 @@ import ( "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/uber-go/tally" "go.uber.org/atomic" "google.golang.org/grpc" ) @@ -503,120 +499,6 @@ writeWorkerPoolPolicy: <-doneCh } -func TestNewPerQueryEnforcer(t *testing.T) { - type testContext struct { - Global cost.ChainedEnforcer - Query cost.ChainedEnforcer - Block cost.ChainedEnforcer - Closer close.SimpleCloser - } - - scope := tally.NewTestScope("", nil) - instrumentOpts := instrument.NewTestOptions(t). - SetMetricsScope(scope) - - setup := func(t *testing.T, globalLimit, queryLimit int) testContext { - global, closer, err := newConfiguredChainedEnforcer(instrumentOpts) - require.NoError(t, err) - - queryLvl := global.Child(cost.QueryLevel) - blockLvl := queryLvl.Child(cost.BlockLevel) - - return testContext{ - Global: global, - Query: queryLvl, - Block: blockLvl, - Closer: closer, - } - } - - tctx := setup(t, 100, 10) - defer tctx.Closer.Close() - - // Spot check that limits are setup properly for each level. - r := tctx.Query.Add(11) - require.NoError(t, r.Error) - - floatsEqual := func(f1, f2 float64) { - assert.InDelta(t, f1, f2, 0.0000001) - } - - floatsEqual(float64(r.Cost), 11) - - r, _ = tctx.Query.State() - floatsEqual(float64(r.Cost), 11) - - r, _ = tctx.Global.State() - floatsEqual(float64(r.Cost), 11) - require.NoError(t, r.Error) - - // Wait for stats reporting to start. - start := time.Now() - for time.Since(start) < 15*time.Second { - gauges := scope.Snapshot().Gauges() - globalEnabled, globalOk := gauges["cost.limits.enabled+limiter=global"] - queryEnabled, queryOk := gauges["cost.limits.enabled+limiter=query"] - if globalOk && queryOk && globalEnabled.Value() == 1 && queryEnabled.Value() == 1 { - break - } - - time.Sleep(100 * time.Millisecond) - } - - // Check stats. - expectCounterValues := map[string]int64{ - "cost.reporter.over_datapoints_limit+enabled=false,limiter=global": 0, - "cost.reporter.over_datapoints_limit+enabled=true,limiter=global": 0, - "cost.reporter.datapoints_counter+limiter=global": 11, - "cost.reporter.over_datapoints_limit+enabled=false,limiter=query": 0, - "cost.reporter.over_datapoints_limit+enabled=true,limiter=query": 0, - } - expectGaugeValues := map[string]float64{ - "cost.limits.threshold+limiter=global": 0, - "cost.limits.enabled+limiter=global": 0, - "cost.reporter.datapoints+limiter=global": 11, - "cost.limits.threshold+limiter=query": 0, - "cost.limits.enabled+limiter=query": 0, - } - - snapshot := scope.Snapshot() - actualCounterValues := make(map[string]int64) - for k, v := range snapshot.Counters() { - actualCounterValues[k] = v.Value() - - expected, ok := expectCounterValues[k] - if !ok { - continue - } - - // Check match. - assert.Equal(t, expected, v.Value(), - fmt.Sprintf("stat mismatch: stat=%s", k)) - - delete(expectCounterValues, k) - } - assert.Equal(t, 0, len(expectCounterValues), - fmt.Sprintf("missing stats: %+v", expectCounterValues)) - - actualGaugeValues := make(map[string]float64) - for k, v := range snapshot.Gauges() { - actualGaugeValues[k] = v.Value() - - expected, ok := expectGaugeValues[k] - if !ok { - continue - } - - // Check match. - assert.Equal(t, expected, v.Value(), - fmt.Sprintf("stat mismatch: stat=%s", k)) - - delete(expectGaugeValues, k) - } - assert.Equal(t, 0, len(expectGaugeValues), - fmt.Sprintf("missing stats: %+v", expectGaugeValues)) -} - var _ rpc.QueryServer = &queryServer{} type queryServer struct { diff --git a/src/query/storage/fetch.go b/src/query/storage/fetch.go index a20d729eb8..36d71482a8 100644 --- a/src/query/storage/fetch.go +++ b/src/query/storage/fetch.go @@ -23,7 +23,6 @@ package storage import ( "time" - "github.com/m3db/m3/src/query/cost" "github.com/m3db/m3/src/query/models" "github.com/m3db/m3/src/query/storage/m3/storagemetadata" @@ -41,8 +40,7 @@ func NewFetchOptions() *FetchOptions { FanoutAggregated: FanoutDefault, FanoutAggregatedOptimized: FanoutDefault, }, - Enforcer: cost.NoopChainedEnforcer(), - Scope: tally.NoopScope, + Scope: tally.NoopScope, } } diff --git a/src/query/storage/m3/storage.go b/src/query/storage/m3/storage.go index 3d2cd7954c..d91edb3926 100644 --- a/src/query/storage/m3/storage.go +++ b/src/query/storage/m3/storage.go @@ -30,7 +30,6 @@ import ( "github.com/m3db/m3/src/dbnode/client" "github.com/m3db/m3/src/dbnode/storage/index" "github.com/m3db/m3/src/query/block" - "github.com/m3db/m3/src/query/cost" "github.com/m3db/m3/src/query/errors" "github.com/m3db/m3/src/query/models" "github.com/m3db/m3/src/query/storage" @@ -117,7 +116,6 @@ func (s *m3storage) FetchProm( fetchResult, err := storage.SeriesIteratorsToPromResult( result, s.opts.ReadWorkerPool(), - options.Enforcer, s.opts.TagOptions(), ) @@ -149,11 +147,6 @@ func FetchResultToBlockResult( StepSize: query.Interval, } - enforcer := options.Enforcer - if enforcer == nil { - enforcer = cost.NoopChainedEnforcer() - } - blocks, err := m3db.ConvertM3DBSeriesIterators( result, bounds, diff --git a/src/query/storage/prom_converter.go b/src/query/storage/prom_converter.go index 4089dd7746..f0b3869773 100644 --- a/src/query/storage/prom_converter.go +++ b/src/query/storage/prom_converter.go @@ -24,11 +24,9 @@ import ( "sync" "github.com/m3db/m3/src/dbnode/encoding" - "github.com/m3db/m3/src/query/cost" "github.com/m3db/m3/src/query/generated/proto/prompb" "github.com/m3db/m3/src/query/models" "github.com/m3db/m3/src/query/storage/m3/consolidators" - xcost "github.com/m3db/m3/src/x/cost" xerrors "github.com/m3db/m3/src/x/errors" xsync "github.com/m3db/m3/src/x/sync" ) @@ -38,7 +36,6 @@ const initRawFetchAllocSize = 32 func iteratorToPromResult( iter encoding.SeriesIterator, tags models.Tags, - enforcer cost.ChainedEnforcer, tagOptions models.TagOptions, ) (*prompb.TimeSeries, error) { samples := make([]prompb.Sample, 0, initRawFetchAllocSize) @@ -54,11 +51,6 @@ func iteratorToPromResult( return nil, err } - r := enforcer.Add(xcost.Cost(len(samples))) - if r.Error != nil { - return nil, r.Error - } - return &prompb.TimeSeries{ Labels: TagsToPromLabels(tags), Samples: samples, @@ -68,7 +60,6 @@ func iteratorToPromResult( // Fall back to sequential decompression if unable to decompress concurrently. func toPromSequentially( fetchResult consolidators.SeriesFetchResult, - enforcer cost.ChainedEnforcer, tagOptions models.TagOptions, ) (PromResult, error) { count := fetchResult.Count() @@ -79,7 +70,7 @@ func toPromSequentially( return PromResult{}, err } - series, err := iteratorToPromResult(iter, tags, enforcer, tagOptions) + series, err := iteratorToPromResult(iter, tags, tagOptions) if err != nil { return PromResult{}, err } @@ -99,7 +90,6 @@ func toPromSequentially( func toPromConcurrently( fetchResult consolidators.SeriesFetchResult, readWorkerPool xsync.PooledWorkerPool, - enforcer cost.ChainedEnforcer, tagOptions models.TagOptions, ) (PromResult, error) { count := fetchResult.Count() @@ -121,7 +111,7 @@ func toPromConcurrently( wg.Add(1) readWorkerPool.Go(func() { defer wg.Done() - series, err := iteratorToPromResult(iter, tags, enforcer, tagOptions) + series, err := iteratorToPromResult(iter, tags, tagOptions) if err != nil { mu.Lock() multiErr = multiErr.Add(err) @@ -155,14 +145,13 @@ func toPromConcurrently( func seriesIteratorsToPromResult( fetchResult consolidators.SeriesFetchResult, readWorkerPool xsync.PooledWorkerPool, - enforcer cost.ChainedEnforcer, tagOptions models.TagOptions, ) (PromResult, error) { if readWorkerPool == nil { - return toPromSequentially(fetchResult, enforcer, tagOptions) + return toPromSequentially(fetchResult, tagOptions) } - return toPromConcurrently(fetchResult, readWorkerPool, enforcer, tagOptions) + return toPromConcurrently(fetchResult, readWorkerPool, tagOptions) } // SeriesIteratorsToPromResult converts raw series iterators directly to a @@ -170,7 +159,6 @@ func seriesIteratorsToPromResult( func SeriesIteratorsToPromResult( fetchResult consolidators.SeriesFetchResult, readWorkerPool xsync.PooledWorkerPool, - enforcer cost.ChainedEnforcer, tagOptions models.TagOptions, ) (PromResult, error) { defer fetchResult.Close() @@ -178,12 +166,8 @@ func SeriesIteratorsToPromResult( return PromResult{}, err } - if enforcer == nil { - enforcer = cost.NoopChainedEnforcer() - } - promResult, err := seriesIteratorsToPromResult(fetchResult, - readWorkerPool, enforcer, tagOptions) + readWorkerPool, tagOptions) promResult.Metadata = fetchResult.Metadata return promResult, err } diff --git a/src/query/storage/prom_converter_test.go b/src/query/storage/prom_converter_test.go index cb290c966d..0f74a23f11 100644 --- a/src/query/storage/prom_converter_test.go +++ b/src/query/storage/prom_converter_test.go @@ -27,14 +27,12 @@ import ( "github.com/m3db/m3/src/dbnode/encoding" dts "github.com/m3db/m3/src/dbnode/ts" "github.com/m3db/m3/src/query/block" - "github.com/m3db/m3/src/query/cost" "github.com/m3db/m3/src/query/generated/proto/prompb" "github.com/m3db/m3/src/query/models" "github.com/m3db/m3/src/query/storage/m3/consolidators" "github.com/m3db/m3/src/query/test/seriesiter" "github.com/m3db/m3/src/query/ts" "github.com/m3db/m3/src/x/checked" - xcost "github.com/m3db/m3/src/x/cost" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/pool" xsync "github.com/m3db/m3/src/x/sync" @@ -76,15 +74,13 @@ func verifyExpandPromSeries( ) { iters := seriesiter.NewMockSeriesIters(ctrl, ident.Tag{}, num, 2) fetchResult := fr(t, iters, makeTag("foo", "bar", num)...) - enforcer := cost.NewMockChainedEnforcer(ctrl) - enforcer.EXPECT().Add(xcost.Cost(2)).Times(num) fetchResult.Metadata = block.ResultMetadata{ Exhaustive: ex, LocalOnly: true, Warnings: []block.Warning{block.Warning{Name: "foo", Message: "bar"}}, } - results, err := SeriesIteratorsToPromResult(fetchResult, pools, enforcer, nil) + results, err := SeriesIteratorsToPromResult(fetchResult, pools, nil) assert.NoError(t, err) require.NotNil(t, results) @@ -306,7 +302,7 @@ func TestDecodeIteratorsWithEmptySeries(t *testing.T) { } opts := models.NewTagOptions() - res, err := SeriesIteratorsToPromResult(buildIters(), nil, nil, opts) + res, err := SeriesIteratorsToPromResult(buildIters(), nil, opts) require.NoError(t, err) verifyResult(t, res) @@ -314,7 +310,7 @@ func TestDecodeIteratorsWithEmptySeries(t *testing.T) { require.NoError(t, err) pool.Init() - res, err = SeriesIteratorsToPromResult(buildIters(), pool, nil, opts) + res, err = SeriesIteratorsToPromResult(buildIters(), pool, opts) require.NoError(t, err) verifyResult(t, res) } diff --git a/src/query/storage/types.go b/src/query/storage/types.go index 9321b65fe5..a5157155e3 100644 --- a/src/query/storage/types.go +++ b/src/query/storage/types.go @@ -28,7 +28,6 @@ import ( "github.com/m3db/m3/src/metrics/policy" "github.com/m3db/m3/src/query/block" - "github.com/m3db/m3/src/query/cost" "github.com/m3db/m3/src/query/generated/proto/prompb" "github.com/m3db/m3/src/query/models" "github.com/m3db/m3/src/query/storage/m3/consolidators" @@ -130,9 +129,6 @@ type FetchOptions struct { Step time.Duration // LookbackDuration if set overrides the default lookback duration. LookbackDuration *time.Duration - // Enforcer is used to enforce resource limits on the number of datapoints - // used by a given query. Limits are imposed at time of decompression. - Enforcer cost.ChainedEnforcer // Scope is used to report metrics about the fetch. Scope tally.Scope // Timeout is the timeout for the request. diff --git a/src/x/cost/enforcer.go b/src/x/cost/enforcer.go deleted file mode 100644 index 5667735632..0000000000 --- a/src/x/cost/enforcer.go +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package cost - -import ( - "fmt" -) - -const ( - defaultCostExceededErrorFmt = "limit reached (current = %v, limit = %v)" -) - -var ( - noopManager = NewStaticLimitManager( - NewLimitManagerOptions(). - SetDefaultLimit(Limit{ - Threshold: maxCost, - Enabled: false, - }, - ), - ) - noopEnforcer = NewEnforcer(noopManager, NewNoopTracker(), nil) -) - -// Report is a report on the cost limits of an enforcer. -type Report struct { - Cost - Error error -} - -// enforcer enforces cost limits for operations. -type enforcer struct { - LimitManager - tracker Tracker - - costMsg string - metrics EnforcerReporter -} - -// NewEnforcer returns a new enforcer for cost limits. -func NewEnforcer(m LimitManager, t Tracker, opts EnforcerOptions) Enforcer { - if opts == nil { - opts = NewEnforcerOptions() - } - - reporter := opts.Reporter() - if reporter == nil { - reporter = NoopEnforcerReporter() - } - - return &enforcer{ - LimitManager: m, - tracker: t, - costMsg: opts.CostExceededMessage(), - metrics: reporter, - } -} - -func (e *enforcer) Reporter() EnforcerReporter { - return e.metrics -} - -// Add adds the cost of an operation to the enforcer's current total. If the operation exceeds -// the enforcer's limit the enforcer will return a CostLimit error in addition to the new total. -func (e *enforcer) Add(cost Cost) Report { - e.metrics.ReportCost(cost) - current := e.tracker.Add(cost) - e.metrics.ReportCurrent(current) - - limit := e.Limit() - overLimit := e.checkLimit(current, limit) - - if overLimit != nil { - // Emit metrics on number of operations that are over the limit even when not enabled. - e.metrics.ReportOverLimit(limit.Enabled) - } - - return Report{ - Cost: current, - Error: overLimit, - } -} - -// State returns the current state of the enforcer. -func (e *enforcer) State() (Report, Limit) { - cost := e.tracker.Current() - l := e.Limit() - err := e.checkLimit(cost, l) - r := Report{ - Cost: cost, - Error: err, - } - return r, l -} - -// Clone clones the current enforcer. The new enforcer uses the same Estimator and LimitManager -// as e buts its Tracker is independent. -func (e *enforcer) Clone() Enforcer { - return &enforcer{ - LimitManager: e.LimitManager, - tracker: NewTracker(), - costMsg: e.costMsg, - metrics: e.metrics, - } -} - -func (e *enforcer) checkLimit(cost Cost, limit Limit) error { - if cost < limit.Threshold || !limit.Enabled { - return nil - } - - return NewCostExceededError(e.costMsg, cost, limit.Threshold) -} - -type costExceededError struct { - Threshold Cost - Current Cost - CustomMsg string -} - -func (ce costExceededError) Error() string { - baseMsg := fmt.Sprintf( - defaultCostExceededErrorFmt, - float64(ce.Current), - float64(ce.Threshold), - ) - if ce.CustomMsg == "" { - return baseMsg - } - - return fmt.Sprintf("%s: %s", ce.CustomMsg, baseMsg) -} - -// NewCostExceededError returns an error for going over an Enforcer's limit. -func NewCostExceededError(customMessage string, cost Cost, threshold Cost) error { - return costExceededError{ - CustomMsg: customMessage, - Current: cost, - Threshold: threshold, - } -} - -// NoopEnforcer returns a new enforcer that always returns a current cost of 0 and -// is always disabled. -func NoopEnforcer() Enforcer { - return noopEnforcer -} - -type noopEnforcerReporter struct{} - -func (noopEnforcerReporter) ReportCost(c Cost) {} -func (noopEnforcerReporter) ReportCurrent(c Cost) {} -func (noopEnforcerReporter) ReportOverLimit(enabled bool) {} - -// NoopEnforcerReporter returns an EnforcerReporter which does nothing on all events. -func NoopEnforcerReporter() EnforcerReporter { - return noopEnforcerReporter{} -} diff --git a/src/x/cost/enforcer_test.go b/src/x/cost/enforcer_test.go deleted file mode 100644 index 27b9a9aa1f..0000000000 --- a/src/x/cost/enforcer_test.go +++ /dev/null @@ -1,248 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package cost - -import ( - "fmt" - "strings" - "testing" - "time" - - "github.com/m3db/m3/src/cluster/generated/proto/commonpb" - "github.com/m3db/m3/src/cluster/kv/mem" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -const ( - testThresholdKey = "threshold" - testEnabledKey = "enabled" -) - -func TestEnforcer(t *testing.T) { - tests := []struct { - input Cost - expected Cost - exceededThreshold bool - }{ - { - input: Cost(0), - expected: 0, - exceededThreshold: false, - }, - { - input: Cost(1), - expected: 1, - exceededThreshold: false, - }, - { - input: Cost(3), - expected: 4, - exceededThreshold: false, - }, - { - input: Cost(9), - expected: 13, - exceededThreshold: true, - }, - } - - limit := Limit{ - Threshold: 10, - Enabled: true, - } - mOpts := NewLimitManagerOptions().SetDefaultLimit(limit) - store := mem.NewStore() - msg := "message which contains context on the cost limit" - - m, err := NewDynamicLimitManager(store, testThresholdKey, testEnabledKey, mOpts) - require.NoError(t, err) - - opts := NewEnforcerOptions().SetCostExceededMessage(msg) - e := NewEnforcer(m, NewTracker(), opts) - - for _, test := range tests { - t.Run(fmt.Sprintf("input %v", test.input), func(t *testing.T) { - report := e.Add(test.input) - require.Equal(t, test.expected, report.Cost) - - if test.exceededThreshold { - require.EqualError(t, report.Error, NewCostExceededError(msg, 13, 10).Error()) - } else { - require.NoError(t, report.Error) - } - }) - } - - // State should return the updated cost total. - report, limit := e.State() - require.Equal(t, Cost(13), report.Cost) - require.Equal(t, Cost(10), limit.Threshold) - require.True(t, limit.Enabled) - require.EqualError(t, report.Error, NewCostExceededError(msg, 13, 10).Error()) - - // The error message should start with the message provided in the options. - require.True(t, strings.HasPrefix(report.Error.Error(), msg)) - - // When the threshold is raised, any new operations that stay below it should be legal again. - store.Set(testThresholdKey, &commonpb.Float64Proto{Value: float64(20)}) - for { - if l := e.Limit(); l.Threshold == 20 { - break - } - time.Sleep(100 * time.Millisecond) - } - - report = e.Add(Cost(3)) - require.NoError(t, err) - require.NoError(t, err) - require.Equal(t, Cost(16), report.Cost) - - report = e.Add(Cost(5)) - require.NoError(t, err) - require.NoError(t, err) - require.EqualError(t, report.Error, NewCostExceededError(msg, 21, 20).Error()) - require.Equal(t, Cost(21), report.Cost) - - // When the enforcer is disabled any input above the threshold should become legal. - store.Set(testEnabledKey, &commonpb.BoolProto{Value: false}) - for { - if l := e.Limit(); !l.Enabled { - break - } - time.Sleep(100 * time.Millisecond) - } - - report = e.Add(Cost(2)) - require.NoError(t, err) - require.NoError(t, err) - require.Equal(t, Cost(23), report.Cost) - - // State should return the updated state. - report, limit = e.State() - require.Equal(t, Cost(23), report.Cost) - require.Equal(t, Cost(20), limit.Threshold) - require.False(t, limit.Enabled) - require.NoError(t, report.Error) -} - -func TestNewCostExceedError(t *testing.T) { - t.Run("with custom error message", func(t *testing.T) { - assert.EqualError( - t, - NewCostExceededError("custom", 1, 2), - "custom: limit reached (current = 1, limit = 2)") - }) - - t.Run("with default message", func(t *testing.T) { - assert.EqualError( - t, - NewCostExceededError("", 1, 2), - "limit reached (current = 1, limit = 2)") - }) -} - -func TestEnforcerClone(t *testing.T) { - store := mem.NewStore() - threshold := Cost(30) - limit := Limit{ - Threshold: threshold, - Enabled: true, - } - mOpts := NewLimitManagerOptions(). - SetDefaultLimit(limit) - - m, err := NewDynamicLimitManager(store, testThresholdKey, testEnabledKey, mOpts) - require.NoError(t, err) - - e := NewEnforcer(m, NewTracker(), nil) - - report := e.Add(Cost(10)) - require.Equal(t, Cost(10), report.Cost) - require.NoError(t, report.Error) - - clone := e.Clone() - - // The cloned enforcer should have no initial cost. - report, limit = clone.State() - require.Equal(t, Cost(0), report.Cost) - require.NoError(t, report.Error) - require.Equal(t, threshold, limit.Threshold) - require.True(t, limit.Enabled) - - // Subsequent calls to Add on each enforcer should be independent. - report = e.Add(Cost(10)) - require.NoError(t, err) - require.NoError(t, err) - require.Equal(t, Cost(20), report.Cost) - - report = clone.Add(Cost(5)) - require.NoError(t, err) - require.NoError(t, err) - require.Equal(t, Cost(5), report.Cost) - - // Each enforcer should see the same updates to their state. - var newThreshold Cost = 40 - store.Set(testThresholdKey, &commonpb.Float64Proto{Value: float64(newThreshold)}) - store.Set(testEnabledKey, &commonpb.BoolProto{Value: false}) - - for { - if l := e.Limit(); !l.Enabled { - break - } - time.Sleep(100 * time.Millisecond) - } - - limit = e.Limit() - require.Equal(t, false, limit.Enabled) - require.Equal(t, newThreshold, limit.Threshold) - - limit = clone.Limit() - require.Equal(t, false, limit.Enabled) - require.Equal(t, newThreshold, limit.Threshold) -} - -func TestNoopEnforcer(t *testing.T) { - tests := []struct { - input Cost - }{ - { - input: Cost(0), - }, - { - input: Cost(10), - }, - } - - e := NoopEnforcer() - limit := e.Limit() - assert.Equal(t, maxCost, limit.Threshold) - assert.False(t, limit.Enabled) - - for _, test := range tests { - t.Run(fmt.Sprintf("input %v", test.input), func(t *testing.T) { - report := e.Add(test.input) - assert.Equal(t, Cost(0), report.Cost) - assert.NoError(t, report.Error) - }) - } -} diff --git a/src/x/cost/limit_manager.go b/src/x/cost/limit_manager.go deleted file mode 100644 index 4faa188fdd..0000000000 --- a/src/x/cost/limit_manager.go +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package cost - -import ( - "fmt" - "sync" - "time" - - "github.com/m3db/m3/src/cluster/kv" - "github.com/m3db/m3/src/cluster/kv/util" - - "github.com/uber-go/tally" - "go.uber.org/atomic" -) - -type limitManager struct { - sync.RWMutex - - threshold *atomic.Float64 - enabled *atomic.Bool - - thresholdWatcher kv.ValueWatch - enabledWatcher kv.ValueWatch - closed bool - closeCh chan struct{} - reportInterval time.Duration - metrics limitManagerMetrics -} - -// NewDynamicLimitManager returns a new LimitWatcher which watches for updates to the cost limit -// of an operation in KV. -func NewDynamicLimitManager( - store kv.Store, - kvLimitKey, kvEnabledKey string, - opts LimitManagerOptions, -) (LimitManager, error) { - if opts == nil { - opts = NewLimitManagerOptions() - } - iOpts := opts.InstrumentOptions() - - var ( - limit = opts.DefaultLimit() - defaultThreshold = float64(limit.Threshold) - defaultEnabled = limit.Enabled - m = &limitManager{ - threshold: atomic.NewFloat64(defaultThreshold), - enabled: atomic.NewBool(defaultEnabled), - closeCh: make(chan struct{}), - reportInterval: iOpts.ReportInterval(), - metrics: newLimitManagerMetrics(iOpts.MetricsScope()), - } - ) - - watchOpts := util.NewOptions().SetLogger(iOpts.Logger()) - thresholdWatcher, err := util.WatchAndUpdateAtomicFloat64( - store, - kvLimitKey, - m.threshold, - defaultThreshold, - watchOpts.SetValidateFn(opts.ValidateLimitFn()), - ) - if err != nil { - return nil, fmt.Errorf("unable to watch key '%s': %v", kvLimitKey, err) - } - m.thresholdWatcher = thresholdWatcher - - enabledWatcher, err := util.WatchAndUpdateAtomicBool( - store, - kvEnabledKey, - m.enabled, - defaultEnabled, - watchOpts, - ) - if err != nil { - return nil, fmt.Errorf("unable to watch key '%s': %v", kvEnabledKey, err) - } - m.enabledWatcher = enabledWatcher - - return m, nil -} - -func (m *limitManager) Limit() Limit { - return Limit{ - Threshold: Cost(m.threshold.Load()), - Enabled: m.enabled.Load(), - } -} - -func (m *limitManager) Close() { - m.Lock() - defer m.Unlock() - if m.closed { - return - } - if m.thresholdWatcher != nil { - m.thresholdWatcher.Close() - } - if m.enabledWatcher != nil { - m.enabledWatcher.Close() - } - close(m.closeCh) - m.closed = true -} - -func (m *limitManager) Report() { - ticker := time.NewTicker(m.reportInterval) - defer ticker.Stop() - - for { - select { - case <-ticker.C: - limit := m.Limit() - m.metrics.threshold.Update(float64(limit.Threshold)) - - var v float64 - if limit.Enabled { - v = 1 - } - m.metrics.enabled.Update(v) - - case <-m.closeCh: - return - } - } -} - -// NewStaticLimitManager returns a new LimitManager which always returns the same limit. -func NewStaticLimitManager(opts LimitManagerOptions) LimitManager { - if opts == nil { - opts = NewLimitManagerOptions() - } - iOpts := opts.InstrumentOptions() - - var ( - l = opts.DefaultLimit() - m = &limitManager{ - threshold: atomic.NewFloat64(float64(l.Threshold)), - enabled: atomic.NewBool(l.Enabled), - closeCh: make(chan struct{}), - reportInterval: iOpts.ReportInterval(), - metrics: newLimitManagerMetrics(iOpts.MetricsScope()), - } - ) - return m -} - -type limitManagerMetrics struct { - threshold tally.Gauge - enabled tally.Gauge -} - -func newLimitManagerMetrics(s tally.Scope) limitManagerMetrics { - return limitManagerMetrics{ - threshold: s.Gauge("threshold"), - enabled: s.Gauge("enabled"), - } -} diff --git a/src/x/cost/limit_manager_test.go b/src/x/cost/limit_manager_test.go deleted file mode 100644 index b8c5021cf1..0000000000 --- a/src/x/cost/limit_manager_test.go +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package cost - -import ( - "errors" - "testing" - "time" - - "github.com/m3db/m3/src/cluster/generated/proto/commonpb" - "github.com/m3db/m3/src/cluster/kv/mem" - - "github.com/fortytw2/leaktest" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestDynamicLimitManager(t *testing.T) { - threshold := Cost(100) - enabled := true - store := mem.NewStore() - thresholdKey := "threshold" - enabledKey := "enabled" - - validateFn := func(data interface{}) error { - val := data.(float64) - if val < 50 { - return errors.New("limit cannot be below 50") - } - return nil - } - - limit := Limit{ - Threshold: threshold, - Enabled: enabled, - } - opts := NewLimitManagerOptions(). - SetDefaultLimit(limit). - SetValidateLimitFn(validateFn) - - m, err := NewDynamicLimitManager( - store, thresholdKey, enabledKey, opts, - ) - require.NoError(t, err) - - testLimitManager(t, m, threshold, enabled) - - // Test updates to the threshold. - threshold = 200 - store.Set(thresholdKey, &commonpb.Float64Proto{Value: float64(threshold)}) - - for { - if m.Limit().Threshold == threshold { - break - } - time.Sleep(100 * time.Millisecond) - } - - testLimitManager(t, m, threshold, enabled) - - // Test updates to enabled. - enabled = false - store.Set(enabledKey, &commonpb.BoolProto{Value: enabled}) - - for { - if !m.Limit().Enabled { - break - } - time.Sleep(100 * time.Millisecond) - } - - testLimitManager(t, m, threshold, enabled) - - // Test that invalid updates are ignored. - store.Set(thresholdKey, &commonpb.Float64Proto{Value: 25}) - time.Sleep(100 * time.Millisecond) - - testLimitManager(t, m, threshold, enabled) - - // Ensure we do not leak any goroutines. - m.Close() - leaktest.Check(t) -} - -func TestStaticLimitManager(t *testing.T) { - threshold := Cost(100) - enabled := true - limit := Limit{ - Threshold: threshold, - Enabled: enabled, - } - manager := NewStaticLimitManager(NewLimitManagerOptions().SetDefaultLimit(limit)) - - limit = manager.Limit() - assert.Equal(t, threshold, limit.Threshold) - assert.Equal(t, enabled, limit.Enabled) - - // Ensure we do not leak any goroutines. - manager.Close() - leaktest.Check(t) -} - -func testLimitManager( - t *testing.T, m LimitManager, expectedThreshold Cost, expectedEnabled bool, -) { - l := m.Limit() - assert.Equal(t, expectedThreshold, l.Threshold) - assert.Equal(t, expectedEnabled, l.Enabled) -} diff --git a/src/x/cost/options.go b/src/x/cost/options.go deleted file mode 100644 index 9378cf8ab7..0000000000 --- a/src/x/cost/options.go +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package cost - -import ( - "github.com/m3db/m3/src/cluster/kv/util" - "github.com/m3db/m3/src/x/instrument" - - "github.com/uber-go/tally" -) - -// LimitManagerOptions provides a set of options for a LimitManager. -type LimitManagerOptions interface { - // SetDefaultLimit sets the default limit for the manager. - SetDefaultLimit(val Limit) LimitManagerOptions - - // DefaultLimit returns the default limit for the manager. - DefaultLimit() Limit - - // SetValidateLimitFn sets the validation function applied to updates to the cost limit. - SetValidateLimitFn(val util.ValidateFn) LimitManagerOptions - - // ValidateLimitFn returns the validation function applied to updates to the cost limit. - ValidateLimitFn() util.ValidateFn - - // SetInstrumentOptions sets the instrument options. - SetInstrumentOptions(val instrument.Options) LimitManagerOptions - - // InstrumentOptions returns the instrument options. - InstrumentOptions() instrument.Options -} - -type limitManagerOptions struct { - defaultLimit Limit - validateLimitFn util.ValidateFn - iOpts instrument.Options -} - -// NewLimitManagerOptions returns a new set of LimitManager options. -func NewLimitManagerOptions() LimitManagerOptions { - return &limitManagerOptions{ - defaultLimit: Limit{ - Threshold: maxCost, - Enabled: false, - }, - iOpts: instrument.NewOptions(), - } -} - -func (o *limitManagerOptions) SetDefaultLimit(val Limit) LimitManagerOptions { - opts := *o - opts.defaultLimit = val - return &opts -} - -func (o *limitManagerOptions) DefaultLimit() Limit { - return o.defaultLimit -} - -func (o *limitManagerOptions) SetValidateLimitFn(val util.ValidateFn) LimitManagerOptions { - opts := *o - opts.validateLimitFn = val - return &opts -} - -func (o *limitManagerOptions) ValidateLimitFn() util.ValidateFn { - return o.validateLimitFn -} - -func (o *limitManagerOptions) SetInstrumentOptions(val instrument.Options) LimitManagerOptions { - opts := *o - opts.iOpts = val - return &opts -} - -func (o *limitManagerOptions) InstrumentOptions() instrument.Options { - return o.iOpts -} - -// EnforcerOptions provides a set of options for an enforcer. -type EnforcerOptions interface { - // Reporter is the reporter which will be used on Enforcer events. - Reporter() EnforcerReporter - - // SetReporter sets Reporter() - SetReporter(r EnforcerReporter) EnforcerOptions - - // SetCostExceededMessage sets CostExceededMessage - SetCostExceededMessage(val string) EnforcerOptions - - // CostExceededMessage returns the message appended to cost limit errors to provide - // more context on the cost limit that was exceeded. - CostExceededMessage() string -} - -type enforcerOptions struct { - msg string - buckets tally.ValueBuckets - reporter EnforcerReporter - iOpts instrument.Options -} - -// NewEnforcerOptions returns a new set of enforcer options. -func NewEnforcerOptions() EnforcerOptions { - return &enforcerOptions{ - buckets: tally.MustMakeExponentialValueBuckets(1, 2, 20), - iOpts: instrument.NewOptions(), - } -} - -func (o *enforcerOptions) Reporter() EnforcerReporter { - return o.reporter -} - -func (o *enforcerOptions) SetReporter(r EnforcerReporter) EnforcerOptions { - opts := *o - opts.reporter = r - return &opts -} - -func (o *enforcerOptions) SetCostExceededMessage(val string) EnforcerOptions { - opts := *o - opts.msg = val - return &opts -} - -func (o *enforcerOptions) CostExceededMessage() string { - return o.msg -} - -func (o *enforcerOptions) SetValueBuckets(val tally.ValueBuckets) EnforcerOptions { - opts := *o - opts.buckets = val - return &opts -} - -func (o *enforcerOptions) ValueBuckets() tally.ValueBuckets { - return o.buckets -} - -func (o *enforcerOptions) SetInstrumentOptions(val instrument.Options) EnforcerOptions { - opts := *o - opts.iOpts = val - return &opts -} - -func (o *enforcerOptions) InstrumentOptions() instrument.Options { - return o.iOpts -} diff --git a/src/x/cost/test/assert.go b/src/x/cost/test/assert.go deleted file mode 100644 index b7c4ac40e3..0000000000 --- a/src/x/cost/test/assert.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (c) 2019 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -// Package test contains testing utilities for the cost package. -package test - -import ( - "github.com/m3db/m3/src/x/cost" - - "github.com/stretchr/testify/assert" -) - -// AssertCurrentCost is a helper assertion to check that an enforcer has the -// given cost. -func AssertCurrentCost(t assert.TestingT, expectedCost cost.Cost, ef cost.Enforcer) { - actual, _ := ef.State() - assert.Equal(t, expectedCost, actual.Cost) -} - -// AssertLimitError checks that err is a limit error with the given parameters -func AssertLimitError(t assert.TestingT, err error, current, threshold cost.Cost) { - AssertLimitErrorWithMsg(t, err, "", current, threshold) -} - -// AssertLimitErrorWithMsg checks that err is a limit error with the given parameters -func AssertLimitErrorWithMsg(t assert.TestingT, err error, msg string, current, threshold cost.Cost) { - assert.EqualError(t, err, cost.NewCostExceededError(msg, current, threshold).Error()) -} diff --git a/src/x/cost/tracker.go b/src/x/cost/tracker.go deleted file mode 100644 index 9599cdc53f..0000000000 --- a/src/x/cost/tracker.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package cost - -import "go.uber.org/atomic" - -type tracker struct { - total *atomic.Float64 -} - -// NewTracker returns a new Tracker which maintains a simple running total of all the -// costs it has seen so far. -func NewTracker() Tracker { return tracker{total: atomic.NewFloat64(0)} } -func (t tracker) Add(c Cost) Cost { return Cost(t.total.Add(float64(c))) } -func (t tracker) Current() Cost { return Cost(t.total.Load()) } - -type noopTracker struct{} - -// NewNoopTracker returns a tracker which always always returns a cost of 0. -func NewNoopTracker() Tracker { return noopTracker{} } -func (t noopTracker) Add(c Cost) Cost { return 0 } -func (t noopTracker) Current() Cost { return 0 } diff --git a/src/x/cost/tracker_test.go b/src/x/cost/tracker_test.go deleted file mode 100644 index 37c0c85216..0000000000 --- a/src/x/cost/tracker_test.go +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package cost - -import ( - "fmt" - "sync" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestTracker(t *testing.T) { - tests := []struct { - input Cost - expected Cost - }{ - { - input: 10, - expected: 10, - }, - { - input: 5, - expected: 15, - }, - { - input: 0, - expected: 15, - }, - { - input: 20, - expected: 35, - }, - } - - tracker := NewTracker() - for _, test := range tests { - t.Run(fmt.Sprintf("testing input: %v", test.input), func(t *testing.T) { - require.Equal(t, test.expected, tracker.Add(test.input)) - require.Equal(t, test.expected, tracker.Current()) - }) - } -} - -func TestTrackerConcurrentUpdates(t *testing.T) { - wg := sync.WaitGroup{} - tracker := NewTracker() - numGoroutines := 10 - numIterations := 1000 - - wg.Add(10) - - for i := 0; i < numGoroutines; i++ { - go func() { - for j := 0; j < numIterations; j++ { - tracker.Add(1) - } - wg.Done() - }() - } - - wg.Wait() - - actual := tracker.Current() - require.Equal(t, Cost(numGoroutines*numIterations), actual) -} - -func TestNoopTracker(t *testing.T) { - tests := []struct { - input Cost - }{ - { - input: 0, - }, - { - input: 1, - }, - { - input: 5, - }, - { - input: 20, - }, - } - - tracker := NewNoopTracker() - for _, test := range tests { - t.Run(fmt.Sprintf("input %v", test.input), func(t *testing.T) { - require.Equal(t, Cost(0), tracker.Add(test.input)) - require.Equal(t, Cost(0), tracker.Current()) - }) - } -} diff --git a/src/x/cost/types.go b/src/x/cost/types.go deleted file mode 100644 index 0e103d4210..0000000000 --- a/src/x/cost/types.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -// Package cost is a library providing utilities for estimating the cost of operations -// and enforcing limits on them. The primary type of interest is Enforcer, which tracks the cost of operations, -// and errors when that cost goes over a particular limit. -package cost - -import ( - "math" -) - -// Cost represents the cost of an operation. -type Cost float64 - -const ( - // maxCost is the maximum cost of an operation. - maxCost = Cost(math.MaxFloat64) -) - -// Limit encapulates the configuration of a cost limit for an operation. -type Limit struct { - Threshold Cost - Enabled bool -} - -// LimitManager manages configuration of a cost limit for an operation. -type LimitManager interface { - // Limit returns the current cost limit for an operation. - Limit() Limit - - // Report reports metrics on the state of the manager. - Report() - - // Close closes the manager. - Close() -} - -// Tracker tracks the cost of operations seen so far. -type Tracker interface { - // Add adds c to the tracker's current cost total and returns the new total. - Add(c Cost) Cost - - // Current returns the tracker's current cost total. - Current() Cost -} - -// Enforcer instances enforce cost limits for operations. -type Enforcer interface { - Add(op Cost) Report - State() (Report, Limit) - Limit() Limit - Clone() Enforcer - Reporter() EnforcerReporter -} - -// An EnforcerReporter is a listener for Enforcer events. -type EnforcerReporter interface { - // ReportCost is called on every call to Enforcer#Add with the added cost - ReportCost(c Cost) - - // ReportCurrent reports the current total on every call to Enforcer#Add - ReportCurrent(c Cost) - - // ReportOverLimit is called every time an enforcer goes over its limit. enabled is true if the limit manager - // says the limit is currently enabled. - ReportOverLimit(enabled bool) -} From fad7a04b0f3b7e0ec4a6fa5ce5f07ab78abb0e2b Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Mon, 26 Oct 2020 18:04:32 -0400 Subject: [PATCH 39/55] default quoted --- site/content/docs/how_to/query.md | 4 +++- src/cmd/services/m3query/config/config.go | 7 ++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/site/content/docs/how_to/query.md b/site/content/docs/how_to/query.md index 94a491ab4b..3f66a90bb2 100644 --- a/site/content/docs/how_to/query.md +++ b/site/content/docs/how_to/query.md @@ -58,7 +58,9 @@ If you run Statsite, m3agg, or some other aggregation tier, you will want to set ## ID generation -The default generation scheme for IDs, `legacy`, is unfortunately prone to collisions, but remains the default for backwards compatibility reasons. It is suggested to set the ID generation scheme to one of either `quoted` or `prepend_meta`. `quoted` generation scheme yields the most human-readable IDs, whereas `prepend_meta` is better for more compact IDs, or if tags are expected to contain non-ASCII characters. To set the ID generation scheme, add the following to your m3coordinator configuration yaml file: +As of 1.0, the default generation scheme for IDs is `quoted`. If you are using the deprecated `legacy` scheme, please contact the M3 project contributors on the open source Slack channel. + +The `quoted` generation scheme yields the most human-readable IDs, whereas `prepend_meta` is better for more compact IDs, or if tags are expected to contain non-ASCII characters. To set the ID generation scheme, add the following to your m3coordinator configuration yaml file: ```yaml tagOptions: diff --git a/src/cmd/services/m3query/config/config.go b/src/cmd/services/m3query/config/config.go index 30244bf383..c7425cdc89 100644 --- a/src/cmd/services/m3query/config/config.go +++ b/src/cmd/services/m3query/config/config.go @@ -22,7 +22,6 @@ package config import ( "errors" - "fmt" "time" etcdclient "github.com/m3db/m3/src/cluster/client/etcd" @@ -39,7 +38,6 @@ import ( "github.com/m3db/m3/src/query/storage/m3/storagemetadata" xconfig "github.com/m3db/m3/src/x/config" "github.com/m3db/m3/src/x/debug/config" - xdocs "github.com/m3db/m3/src/x/docs" "github.com/m3db/m3/src/x/instrument" xlog "github.com/m3db/m3/src/x/log" "github.com/m3db/m3/src/x/opentracing" @@ -618,9 +616,8 @@ func TagOptionsFromConfig(cfg TagOptionsConfiguration) (models.TagOptions, error } if cfg.Scheme == models.TypeDefault { - // If no config has been set, error. - docLink := xdocs.Path("how_to/query#migration") - return nil, fmt.Errorf(errNoIDGenerationScheme, docLink) + // Default to quoted if unspecified. + cfg.Scheme = models.TypeQuoted } opts = opts.SetIDSchemeType(cfg.Scheme) From 3a27236e5de0afa079259030fd35337cd91ddaa3 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Mon, 26 Oct 2020 18:21:55 -0400 Subject: [PATCH 40/55] more feedback 1 --- src/cmd/services/m3query/config/config.go | 3 -- .../services/m3query/config/config_test.go | 14 +++--- .../v1/handler/prometheus/native/common.go | 44 ------------------- 3 files changed, 5 insertions(+), 56 deletions(-) diff --git a/src/cmd/services/m3query/config/config.go b/src/cmd/services/m3query/config/config.go index c7425cdc89..bb5070bfea 100644 --- a/src/cmd/services/m3query/config/config.go +++ b/src/cmd/services/m3query/config/config.go @@ -58,9 +58,6 @@ const ( NoopEtcdStorageType BackendStorageType = "noop-etcd" defaultCarbonIngesterListenAddress = "0.0.0.0:7204" - errNoIDGenerationScheme = "error: a recent breaking change means that an ID " + - "generation scheme is required in coordinator configuration settings. " + - "More information is available here: %s" defaultQueryTimeout = 30 * time.Second diff --git a/src/cmd/services/m3query/config/config_test.go b/src/cmd/services/m3query/config/config_test.go index 1246cfefcc..acb787da8c 100644 --- a/src/cmd/services/m3query/config/config_test.go +++ b/src/cmd/services/m3query/config/config_test.go @@ -26,7 +26,6 @@ import ( "github.com/m3db/m3/src/query/models" xconfig "github.com/m3db/m3/src/x/config" - xdocs "github.com/m3db/m3/src/x/docs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -36,11 +35,11 @@ import ( const testConfigFile = "./testdata/config.yml" -func TestTagOptionsFromEmptyConfigErrors(t *testing.T) { +func TestDefaultTagOptionsFromEmptyConfig(t *testing.T) { cfg := TagOptionsConfiguration{} opts, err := TagOptionsFromConfig(cfg) - require.Error(t, err) - require.Nil(t, opts) + require.NoError(t, err) + require.Equal(t, models.TypeQuoted, opts.IDSchemeType()) } func TestTagOptionsFromConfigWithIDGenerationScheme(t *testing.T) { @@ -144,11 +143,8 @@ func TestDefaultTagOptionsConfigErrors(t *testing.T) { var cfg TagOptionsConfiguration require.NoError(t, yaml.Unmarshal([]byte(""), &cfg)) opts, err := TagOptionsFromConfig(cfg) - - docLink := xdocs.Path("how_to/query#migration") - expectedError := fmt.Sprintf(errNoIDGenerationScheme, docLink) - require.EqualError(t, err, expectedError) - require.Nil(t, opts) + require.NoError(t, err) + require.Equal(t, models.TypeQuoted, opts.IDSchemeType()) } func TestGraphiteIDGenerationSchemeIsInvalid(t *testing.T) { diff --git a/src/query/api/v1/handler/prometheus/native/common.go b/src/query/api/v1/handler/prometheus/native/common.go index b703d002d2..59ec8630e0 100644 --- a/src/query/api/v1/handler/prometheus/native/common.go +++ b/src/query/api/v1/handler/prometheus/native/common.go @@ -430,47 +430,3 @@ func renderResultsInstantaneousJSON( jw.EndObject() jw.Close() } - -func renderM3QLResultsJSON( - w io.Writer, - series []*ts.Series, - params models.RequestParams, -) { - jw := json.NewWriter(w) - jw.BeginArray() - - for _, s := range series { - jw.BeginObject() - jw.BeginObjectField("target") - jw.WriteString(string(s.Name())) - - jw.BeginObjectField("tags") - jw.BeginObject() - - for _, tag := range s.Tags.Tags { - jw.BeginObjectField(string(tag.Name)) - jw.WriteString(string(tag.Value)) - } - - jw.EndObject() - - jw.BeginObjectField("datapoints") - jw.BeginArray() - for i := 0; i < s.Len(); i++ { - dp := s.Values().DatapointAt(i) - jw.BeginArray() - jw.WriteFloat64(dp.Value) - jw.WriteInt(int(dp.Timestamp.Unix())) - jw.EndArray() - } - jw.EndArray() - - jw.BeginObjectField("step_size_ms") - jw.WriteInt(int(params.Step.Seconds() * 1000)) - - jw.EndObject() - } - - jw.EndArray() - jw.Close() -} From 4fd1f30434480c1c7fe9c485a012fda5bce2e22a Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Mon, 26 Oct 2020 18:32:44 -0400 Subject: [PATCH 41/55] changelog 2 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8643c784ae..144fa85334 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,9 @@ listenAddress: "..." - **M3Coordinator**: `/api/v1/placement` changed to `/api/v1/services/m3db/placement` - **M3Coordinator**: `/api/v1/placement/init` changed to `/api/v1/services/m3db/placement/init` +### Misc +- **M3Query**: Concept of data point limit enforcers removed in favor of the other remaining query limits (e.g. max series). This also removed metrics `cost_reporter_datapoints`, `cost_reporter_datapoints_counter`, and `cost_reporter_over_datapoints_limit`. + # 0.15.17 ## Features From e7de3f11b85203f73b00770b13f121955c36bc45 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Tue, 27 Oct 2020 11:37:35 -0400 Subject: [PATCH 42/55] default cache change --- src/dbnode/namespace/options.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/dbnode/namespace/options.go b/src/dbnode/namespace/options.go index 29e2d15595..fd84acdeb2 100644 --- a/src/dbnode/namespace/options.go +++ b/src/dbnode/namespace/options.go @@ -48,8 +48,9 @@ const ( // Namespace with cold writes disabled by default. defaultColdWritesEnabled = false - // Namespace caches retrieved blocks by default. - defaultCacheBlocksOnRetrieve = true + // Namespace does not cache retrieved blocks by default since this is only + // useful specifically for usage patterns tending towards heavy historical reads. + defaultCacheBlocksOnRetrieve = false ) var ( From 0b42cec92f92302cd55f27f5a30c931b5df644f7 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Tue, 27 Oct 2020 15:38:34 -0400 Subject: [PATCH 43/55] test fixing --- src/dbnode/namespace/convert_test.go | 2 +- src/dbnode/persist/fs/retriever_test.go | 4 +++- src/query/api/v1/handler/database/create_test.go | 16 ++++++++-------- src/query/api/v1/handler/namespace/add_test.go | 2 +- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/dbnode/namespace/convert_test.go b/src/dbnode/namespace/convert_test.go index 6ba3e3b3d1..f1db9cad28 100644 --- a/src/dbnode/namespace/convert_test.go +++ b/src/dbnode/namespace/convert_test.go @@ -376,7 +376,7 @@ func assertEqualMetadata(t *testing.T, name string, expected nsproto.NamespaceOp require.Equal(t, name, observed.ID().String()) opts := observed.Options() - expectedCacheBlocksOnRetrieve := true + expectedCacheBlocksOnRetrieve := false if expected.CacheBlocksOnRetrieve != nil { expectedCacheBlocksOnRetrieve = expected.CacheBlocksOnRetrieve.Value } diff --git a/src/dbnode/persist/fs/retriever_test.go b/src/dbnode/persist/fs/retriever_test.go index f176e53323..ad1a2167f1 100644 --- a/src/dbnode/persist/fs/retriever_test.go +++ b/src/dbnode/persist/fs/retriever_test.go @@ -268,7 +268,9 @@ func testBlockRetrieverHighConcurrentSeeks(t *testing.T, shouldCacheShardIndices return block.LeaseState{Volume: 0}, nil }).AnyTimes() - seekerMgr.blockRetrieverOpts = seekerMgr.blockRetrieverOpts.SetBlockLeaseManager(mockBlockLeaseManager) + seekerMgr.blockRetrieverOpts = seekerMgr.blockRetrieverOpts. + SetBlockLeaseManager(mockBlockLeaseManager). + SetCacheBlocksOnRetrieve(true) // Generate data. for _, shard := range shards { diff --git a/src/query/api/v1/handler/database/create_test.go b/src/query/api/v1/handler/database/create_test.go index 512e1ae7ae..378f3d757e 100644 --- a/src/query/api/v1/handler/database/create_test.go +++ b/src/query/api/v1/handler/database/create_test.go @@ -162,7 +162,7 @@ func testLocalType(t *testing.T, providedType string, placementExists bool) { ] }, "bootstrapEnabled": true, - "cacheBlocksOnRetrieve": true, + "cacheBlocksOnRetrieve": false, "flushEnabled": true, "writesToCommitLog": true, "cleanupEnabled": true, @@ -338,7 +338,7 @@ func TestLocalTypeWithNumShards(t *testing.T) { ] }, "bootstrapEnabled": true, - "cacheBlocksOnRetrieve": true, + "cacheBlocksOnRetrieve": false, "flushEnabled": true, "writesToCommitLog": true, "cleanupEnabled": true, @@ -467,7 +467,7 @@ func TestLocalWithBlockSizeNanos(t *testing.T) { ] }, "bootstrapEnabled": true, - "cacheBlocksOnRetrieve": true, + "cacheBlocksOnRetrieve": false, "flushEnabled": true, "writesToCommitLog": true, "cleanupEnabled": true, @@ -601,7 +601,7 @@ func TestLocalWithBlockSizeExpectedSeriesDatapointsPerHour(t *testing.T) { ] }, "bootstrapEnabled": true, - "cacheBlocksOnRetrieve": true, + "cacheBlocksOnRetrieve": false, "flushEnabled": true, "writesToCommitLog": true, "cleanupEnabled": true, @@ -865,7 +865,7 @@ func testClusterTypeHosts(t *testing.T, placementExists bool) { ] }, "bootstrapEnabled": true, - "cacheBlocksOnRetrieve": true, + "cacheBlocksOnRetrieve": false, "flushEnabled": true, "writesToCommitLog": true, "cleanupEnabled": true, @@ -1022,7 +1022,7 @@ func TestClusterTypeHostsWithIsolationGroup(t *testing.T) { ] }, "bootstrapEnabled": true, - "cacheBlocksOnRetrieve": true, + "cacheBlocksOnRetrieve": false, "flushEnabled": true, "writesToCommitLog": true, "cleanupEnabled": true, @@ -1239,7 +1239,7 @@ func TestLocalTypeWithAggregatedNamespace(t *testing.T) { ] }, "bootstrapEnabled": true, - "cacheBlocksOnRetrieve": true, + "cacheBlocksOnRetrieve": false, "flushEnabled": true, "writesToCommitLog": true, "cleanupEnabled": true, @@ -1281,7 +1281,7 @@ func TestLocalTypeWithAggregatedNamespace(t *testing.T) { ] }, "bootstrapEnabled": true, - "cacheBlocksOnRetrieve": true, + "cacheBlocksOnRetrieve": false, "flushEnabled": true, "writesToCommitLog": true, "cleanupEnabled": true, diff --git a/src/query/api/v1/handler/namespace/add_test.go b/src/query/api/v1/handler/namespace/add_test.go index ea036e7ebd..d0aae7c984 100644 --- a/src/query/api/v1/handler/namespace/add_test.go +++ b/src/query/api/v1/handler/namespace/add_test.go @@ -124,7 +124,7 @@ func TestNamespaceAddHandler(t *testing.T) { "testNamespace": xjson.Map{ "aggregationOptions": nil, "bootstrapEnabled": true, - "cacheBlocksOnRetrieve": true, + "cacheBlocksOnRetrieve": false, "flushEnabled": true, "writesToCommitLog": true, "cleanupEnabled": true, From b07bc54bb9261111b8421f929e02858138b3c68c Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Tue, 27 Oct 2020 15:50:34 -0400 Subject: [PATCH 44/55] test fixing 2 --- src/dbnode/persist/fs/retriever_test.go | 3 +-- src/dbnode/persist/fs/seek_manager_test.go | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/dbnode/persist/fs/retriever_test.go b/src/dbnode/persist/fs/retriever_test.go index ad1a2167f1..824e012c96 100644 --- a/src/dbnode/persist/fs/retriever_test.go +++ b/src/dbnode/persist/fs/retriever_test.go @@ -269,8 +269,7 @@ func testBlockRetrieverHighConcurrentSeeks(t *testing.T, shouldCacheShardIndices return block.LeaseState{Volume: 0}, nil }).AnyTimes() seekerMgr.blockRetrieverOpts = seekerMgr.blockRetrieverOpts. - SetBlockLeaseManager(mockBlockLeaseManager). - SetCacheBlocksOnRetrieve(true) + SetBlockLeaseManager(mockBlockLeaseManager) // Generate data. for _, shard := range shards { diff --git a/src/dbnode/persist/fs/seek_manager_test.go b/src/dbnode/persist/fs/seek_manager_test.go index 4da0287c94..275370445a 100644 --- a/src/dbnode/persist/fs/seek_manager_test.go +++ b/src/dbnode/persist/fs/seek_manager_test.go @@ -44,6 +44,8 @@ const ( var ( defaultTestBlockRetrieverOptions = NewBlockRetrieverOptions(). SetBlockLeaseManager(&block.NoopLeaseManager{}). + // Test with caching enabled. + SetCacheBlocksOnRetrieve(true). // Default value is determined by available CPUs, but for testing // we want to have this been consistent across hardware. SetFetchConcurrency(defaultTestingFetchConcurrency) From f3474f1a2f281ebcd190d99bb492f7a3be1365d8 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Tue, 27 Oct 2020 17:17:01 -0400 Subject: [PATCH 45/55] test fix 3 --- src/dbnode/persist/fs/files_test.go | 2 ++ src/dbnode/persist/fs/retriever_test.go | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dbnode/persist/fs/files_test.go b/src/dbnode/persist/fs/files_test.go index ea3337a355..60a4b68dd0 100644 --- a/src/dbnode/persist/fs/files_test.go +++ b/src/dbnode/persist/fs/files_test.go @@ -50,6 +50,7 @@ var ( testNs2ID = ident.StringID("testNs2") testNs1Metadata = func(t *testing.T) namespace.Metadata { md, err := namespace.NewMetadata(testNs1ID, namespace.NewOptions(). + SetCacheBlocksOnRetrieve(true). SetRetentionOptions(retention.NewOptions().SetBlockSize(testBlockSize)). SetIndexOptions(namespace.NewIndexOptions().SetEnabled(true).SetBlockSize(testBlockSize))) require.NoError(t, err) @@ -57,6 +58,7 @@ var ( } testNs2Metadata = func(t *testing.T) namespace.Metadata { md, err := namespace.NewMetadata(testNs2ID, namespace.NewOptions(). + SetCacheBlocksOnRetrieve(true). SetRetentionOptions(retention.NewOptions().SetBlockSize(testBlockSize)). SetIndexOptions(namespace.NewIndexOptions().SetEnabled(true).SetBlockSize(testBlockSize))) require.NoError(t, err) diff --git a/src/dbnode/persist/fs/retriever_test.go b/src/dbnode/persist/fs/retriever_test.go index 824e012c96..455c57439c 100644 --- a/src/dbnode/persist/fs/retriever_test.go +++ b/src/dbnode/persist/fs/retriever_test.go @@ -1,5 +1,3 @@ -// +build big -// // Copyright (c) 2016 Uber Technologies, Inc. // // Permission is hereby granted, free of charge, to any person obtaining a copy From df4fcb5cd9e8919256327afc7cac1d985f153012 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Wed, 28 Oct 2020 10:52:27 -0400 Subject: [PATCH 46/55] test fix 4 --- src/dbnode/persist/fs/retriever_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/dbnode/persist/fs/retriever_test.go b/src/dbnode/persist/fs/retriever_test.go index 455c57439c..d6a316f2a8 100644 --- a/src/dbnode/persist/fs/retriever_test.go +++ b/src/dbnode/persist/fs/retriever_test.go @@ -1,3 +1,5 @@ +// +build big +// // Copyright (c) 2016 Uber Technologies, Inc. // // Permission is hereby granted, free of charge, to any person obtaining a copy @@ -276,7 +278,6 @@ func testBlockRetrieverHighConcurrentSeeks(t *testing.T, shouldCacheShardIndices for _, blockStart := range blockStarts { for _, volume := range volumes { w, closer := newOpenTestWriter(t, fsOpts, shard, blockStart, volume) - for i := 0; i < idsPerShard; i++ { idString := fmt.Sprintf("foo.%d", i) shardIDStrings[shard] = append(shardIDStrings[shard], idString) From cf7b4dafcdea9b2344ff14bc345dedea5eb89a97 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Wed, 28 Oct 2020 11:50:42 -0400 Subject: [PATCH 47/55] added bootstrap mode --- src/cmd/services/m3dbnode/config/bootstrap.go | 86 +++++++++++++++++-- 1 file changed, 81 insertions(+), 5 deletions(-) diff --git a/src/cmd/services/m3dbnode/config/bootstrap.go b/src/cmd/services/m3dbnode/config/bootstrap.go index 84c3fae1c4..3df2e0fb7f 100644 --- a/src/cmd/services/m3dbnode/config/bootstrap.go +++ b/src/cmd/services/m3dbnode/config/bootstrap.go @@ -21,7 +21,9 @@ package config import ( + "errors" "fmt" + "strings" "github.com/m3db/m3/src/dbnode/client" "github.com/m3db/m3/src/dbnode/persist/fs" @@ -44,20 +46,86 @@ var ( // defaultNumProcessorsPerCPU is the default number of processors per CPU. defaultNumProcessorsPerCPU = 0.125 - // order in which bootstrappers are run - // (run in ascending order of precedence) - orderedBootstrappers = []string{ + // default order in which bootstrappers are run + // (run in ascending order of precedence). + defaultOrderedBootstrappers = []string{ // Filesystem bootstrapping must be first. bfs.FileSystemBootstrapperName, // Peers and commitlog must come before the uninitialized topology bootrapping. + commitlog.CommitLogBootstrapperName, + peers.PeersBootstrapperName, + uninitialized.UninitializedTopologyBootstrapperName, + } + + // bootstrapper order where peers is prefered over commitlog. + preferPeersOrderedBootstrappers = []string{ + // Filesystem bootstrapping must be first. + bfs.FileSystemBootstrapperName, + // Prefer peers over commitlog. peers.PeersBootstrapperName, commitlog.CommitLogBootstrapperName, uninitialized.UninitializedTopologyBootstrapperName, } + + validBootstrapModes = []BootstrapMode{ + DefaultBootstrapMode, + PreferPeersBootstrapMode, + } + + errReadBootstrapModeInvalid = errors.New("bootstrap mode invalid") ) +// BootstrapMode defines the mode in which bootstrappers are run. +type BootstrapMode uint + +const ( + // DefaultBootstrapMode executes bootstrappers in default order. + DefaultBootstrapMode BootstrapMode = iota + // PreferPeersBootstrapMode executes peers before commitlog bootstrapper. + PreferPeersBootstrapMode +) + +// UnmarshalYAML unmarshals an BootstrapMode into a valid type from string. +func (m *BootstrapMode) UnmarshalYAML(unmarshal func(interface{}) error) error { + var str string + if err := unmarshal(&str); err != nil { + return err + } + + // If unspecified, use default mode. + if str == "" { + *m = DefaultBootstrapMode + return nil + } + + strs := make([]string, 0, len(validBootstrapModes)) + for _, valid := range validBootstrapModes { + if str == valid.String() { + *m = valid + return nil + } + strs = append(strs, "'"+valid.String()+"'") + } + return fmt.Errorf("invalid BootstrapMode '%s' valid types are: %s", + str, strings.Join(strs, ", ")) +} + +// String returns the bootstrap mode as a string +func (m BootstrapMode) String() string { + switch m { + case DefaultBootstrapMode: + return "default" + case PreferPeersBootstrapMode: + return "prefer_peers" + } + return "default" +} + // BootstrapConfiguration specifies the config for bootstrappers. type BootstrapConfiguration struct { + // BootstrapMode defines the mode in which bootstrappers are run. + BootstrapMode *BootstrapMode `yaml:"mode"` + // Filesystem bootstrapper configuration. Filesystem *BootstrapFilesystemConfiguration `yaml:"filesystem"` @@ -189,8 +257,9 @@ func (bsc BootstrapConfiguration) New( } var ( - bs bootstrap.BootstrapperProvider - fsOpts = opts.CommitLogOptions().FilesystemOptions() + bs bootstrap.BootstrapperProvider + fsOpts = opts.CommitLogOptions().FilesystemOptions() + orderedBootstrappers = bsc.orderedBootstrappers() ) // Start from the end of the list because the bootstrappers are ordered by precedence in descending order. // I.e. each bootstrapper wraps the preceding bootstrapper, and so the outer-most bootstrapper is run first. @@ -307,3 +376,10 @@ func (bsc BootstrapConfiguration) peersConfig() BootstrapPeersConfiguration { } return newDefaultBootstrapPeersConfiguration() } + +func (bsc BootstrapConfiguration) orderedBootstrappers() []string { + if bsc.BootstrapMode == nil && *bsc.BootstrapMode == PreferPeersBootstrapMode { + return preferPeersOrderedBootstrappers + } + return defaultOrderedBootstrappers +} From 20ab22dba57598abcb72cf2d13df228da63c8b05 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Wed, 28 Oct 2020 12:05:16 -0400 Subject: [PATCH 48/55] added bootstrap mode 2 --- src/cmd/services/m3dbnode/config/bootstrap.go | 2 +- src/cmd/services/m3dbnode/config/config_test.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cmd/services/m3dbnode/config/bootstrap.go b/src/cmd/services/m3dbnode/config/bootstrap.go index 3df2e0fb7f..db530dd73f 100644 --- a/src/cmd/services/m3dbnode/config/bootstrap.go +++ b/src/cmd/services/m3dbnode/config/bootstrap.go @@ -378,7 +378,7 @@ func (bsc BootstrapConfiguration) peersConfig() BootstrapPeersConfiguration { } func (bsc BootstrapConfiguration) orderedBootstrappers() []string { - if bsc.BootstrapMode == nil && *bsc.BootstrapMode == PreferPeersBootstrapMode { + if bsc.BootstrapMode != nil && *bsc.BootstrapMode == PreferPeersBootstrapMode { return preferPeersOrderedBootstrappers } return defaultOrderedBootstrappers diff --git a/src/cmd/services/m3dbnode/config/config_test.go b/src/cmd/services/m3dbnode/config/config_test.go index 42e9829b0c..6907a824a2 100644 --- a/src/cmd/services/m3dbnode/config/config_test.go +++ b/src/cmd/services/m3dbnode/config/config_test.go @@ -413,6 +413,7 @@ func TestConfiguration(t *testing.T) { writeNewSeriesBackoffDuration: 2ms tick: null bootstrap: + mode: null filesystem: numProcessorsPerCPU: 0.42 migration: null From e3f65e912b5cc0cdbb17768fd1fbc494acfec7af Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Wed, 28 Oct 2020 12:17:33 -0400 Subject: [PATCH 49/55] test fix --- src/dbnode/persist/fs/retriever_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/dbnode/persist/fs/retriever_test.go b/src/dbnode/persist/fs/retriever_test.go index d6a316f2a8..824e012c96 100644 --- a/src/dbnode/persist/fs/retriever_test.go +++ b/src/dbnode/persist/fs/retriever_test.go @@ -278,6 +278,7 @@ func testBlockRetrieverHighConcurrentSeeks(t *testing.T, shouldCacheShardIndices for _, blockStart := range blockStarts { for _, volume := range volumes { w, closer := newOpenTestWriter(t, fsOpts, shard, blockStart, volume) + for i := 0; i < idsPerShard; i++ { idString := fmt.Sprintf("foo.%d", i) shardIDStrings[shard] = append(shardIDStrings[shard], idString) From 9436afc66ded8b1ca479be7232b0fd39a39ba49b Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Wed, 28 Oct 2020 12:23:02 -0400 Subject: [PATCH 50/55] changelog --- CHANGELOG.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 144fa85334..3d5cf00b25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,11 +25,11 @@ listenAddress: "..." ### API - **M3DB**: `/services/m3db/database/config/bootstrappers` dynamic bootstappers endpoint removed -- **M3Coordinator**: `/api/v1/namespace` changed to `/api/v1/services/m3db/namespace` -- **M3Coordinator**: `/api/v1/namespace/init` changed to `/api/v1/services/m3db/namespace/init` -- **M3Coordinator**: `/api/v1/namespace/unagg` changed to `/api/v1/services/m3db/namespace/unagg` -- **M3Coordinator**: `/api/v1/placement` changed to `/api/v1/services/m3db/placement` -- **M3Coordinator**: `/api/v1/placement/init` changed to `/api/v1/services/m3db/placement/init` +- **M3Coordinator**: Removed deprecated URL `/api/v1/namespace` in favor of stable preferred URL `/api/v1/services/m3db/namespace` +- **M3Coordinator**: Removed deprecated URL `/api/v1/namespace/init` in favor of stable preferred URL `/api/v1/services/m3db/namespace/init` +- **M3Coordinator**: Removed deprecated URL `/api/v1/namespace/unagg` in favor of stable preferred URL `/api/v1/services/m3db/namespace/unagg` +- **M3Coordinator**: Removed deprecated URL `/api/v1/placement` in favor of stable preferred URL `/api/v1/services/m3db/placement` +- **M3Coordinator**: Removed deprecated URL `/api/v1/placement/init` in favor of stable preferred URL `/api/v1/services/m3db/placement/init` ### Misc - **M3Query**: Concept of data point limit enforcers removed in favor of the other remaining query limits (e.g. max series). This also removed metrics `cost_reporter_datapoints`, `cost_reporter_datapoints_counter`, and `cost_reporter_over_datapoints_limit`. From 7eee1a068584c0ec7299506b308b7eeb78db2775 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Wed, 28 Oct 2020 18:37:57 -0400 Subject: [PATCH 51/55] test fix --- src/cmd/services/m3dbnode/config/config_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd/services/m3dbnode/config/config_test.go b/src/cmd/services/m3dbnode/config/config_test.go index 52fa7df927..10c82b710b 100644 --- a/src/cmd/services/m3dbnode/config/config_test.go +++ b/src/cmd/services/m3dbnode/config/config_test.go @@ -429,8 +429,8 @@ func TestConfiguration(t *testing.T) { size: 100 cacheRegexp: false cacheTerms: false - filesystem: regexp: null + filesystem: filePathPrefix: /var/lib/m3db writeBufferSize: 65536 dataReadBufferSize: 65536 From 235615533600ce8772d2d7881d7d6631121d31b0 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Wed, 28 Oct 2020 19:52:39 -0400 Subject: [PATCH 52/55] remove single node md --- site/content/docs/how_to/single_node.md | 147 ------------------------ 1 file changed, 147 deletions(-) delete mode 100644 site/content/docs/how_to/single_node.md diff --git a/site/content/docs/how_to/single_node.md b/site/content/docs/how_to/single_node.md deleted file mode 100644 index d88aa26a26..0000000000 --- a/site/content/docs/how_to/single_node.md +++ /dev/null @@ -1,147 +0,0 @@ ---- -weight: 1 -title: M3DB Single Node Deployment ---- - -Deploying a single-node cluster is a great way to experiment with M3DB and get a feel for what it -has to offer. Our Docker image by default configures a single M3DB instance as one binary -containing: - -- An M3DB storage instance (`m3dbnode`) for timeseries storage. This includes an embedded tag-based - metrics index, as well as as an embedded etcd server for storing the above mentioned cluster - topology and runtime configuration. -- A "coordinator" instance (`m3coordinator`) for writing and querying tagged metrics, as well as - managing cluster topology and runtime configuration. - -To begin, first start up a Docker container with port `7201` (used to manage the cluster topology), port `7203` which is where Prometheus scrapes metrics produced by `M3DB` and `M3Coordinator`, and port `9003` (used to read and write metrics) exposed. We recommend you create a persistent data -directory on your host for durability: - -```shell -docker pull quay.io/m3db/m3dbnode:latest -docker run -p 7201:7201 -p 7203:7203 -p 9003:9003 --name m3db -v $(pwd)/m3db_data:/var/lib/m3db quay.io/m3db/m3dbnode:latest -``` - -**Note:** For the single node case, we use this [sample config file](https://github.com/m3db/m3/blob/master/src/dbnode/config/m3dbnode-local-etcd.yml). If you inspect the file, you'll see that all the configuration is grouped by `coordinator` or `db`. That's because this setup runs `M3DB` and `M3Coordinator` as one application. While this is convenient for testing and development, you'll want to run clustered `M3DB` with a separate `M3Coordinator` in production. You can read more about that [here.](/docs/how_to/cluster_hard_way). - -Next, create an initial namespace for your metrics in the database using the cURL below. Keep in mind that the provided `namespaceName` must match the namespace in the `local` section of the `M3Coordinator` YAML configuration, and if you choose to [add any additional namespaces](/docs/operational_guide/namespace_configuration) you'll need to add them to the `local` section of `M3Coordinator`'s YAML config as well. - - - -```shell -curl -X POST http://localhost:7201/api/v1/database/create -d '{ - "type": "local", - "namespaceName": "default", - "retentionTime": "12h" -}' -``` - -**Note**: The `api/v1/database/create` endpoint is abstraction over two concepts in M3DB called [placements](/docs/operational_guide/placement) and [namespaces](/docs/operational_guide/namespace_configuration). If a placement doesn't exist, it will create one based on the `type` argument, otherwise if the placement already exists, it just creates the specified namespace. For now it's enough to just understand that it creates M3DB namespaces (tables), but if you're going to run a clustered M3 setup in production, make sure you familiarize yourself with the links above. - -Placement initialization may take a minute or two and you can check on the status of this by running the following: - -```shell -curl http://localhost:7201/api/v1/services/m3db/placement | jq . -``` - -Once all of the shards become `AVAILABLE`, you should see your node complete bootstrapping! Don't worry if you see warnings or errors related to a local cache file, such as `[W] could not load cache from file -/var/lib/m3kv/m3db_embedded.json`. Those are expected for a local instance and in general any -warn-level errors (prefixed with `[W]`) should not block bootstrapping. - -```shell -02:28:30.008072[I] updating database namespaces [{adds [default]} {updates []} {removals []}] -02:28:30.270681[I] node tchannelthrift: listening on 0.0.0.0:9000 -02:28:30.271909[I] cluster tchannelthrift: listening on 0.0.0.0:9001 -02:28:30.519468[I] node httpjson: listening on 0.0.0.0:9002 -02:28:30.520061[I] cluster httpjson: listening on 0.0.0.0:9003 -02:28:30.520652[I] bootstrap finished [{namespace metrics} {duration 55.4µs}] -02:28:30.520909[I] bootstrapped -``` - -The node also self-hosts its OpenAPI docs, outlining available endpoints. You can access this by -going to `localhost:7201/api/v1/openapi` in your browser. - -![OpenAPI Doc](/docs/redoc_screenshot.png) - -Now you can experiment with writing tagged metrics: - -```shell -curl -sS -X POST http://localhost:9003/writetagged -d '{ - "namespace": "default", - "id": "foo", - "tags": [ - { - "name": "__name__", - "value": "user_login" - }, - { - "name": "city", - "value": "new_york" - }, - { - "name": "endpoint", - "value": "/request" - } - ], - "datapoint": { - "timestamp": '"$(date "+%s")"', - "value": 42.123456789 - } -} -' -``` - -**Note:** In the above example we include the tag `__name__`. This is because `__name__` is a -reserved tag in Prometheus and will make querying the metric much easier. For example, if you have -[M3Query](/docs/how_to/query) setup as a Prometheus datasource in Grafana, you can then query for the metric -using the following PromQL query: - -```shell -user_login{city="new_york",endpoint="/request"} -``` - -And reading the metrics you've written using the M3DB `/query` endpoint: - -```shell -curl -sS -X POST http://localhost:9003/query -d '{ - "namespace": "default", - "query": { - "regexp": { - "field": "city", - "regexp": ".*" - } - }, - "rangeStart": 0, - "rangeEnd": '"$(date "+%s")"' -}' | jq . - -{ - "results": [ - { - "id": "foo", - "tags": [ - { - "name": "__name__", - "value": "user_login" - }, - { - "name": "city", - "value": "new_york" - }, - { - "name": "endpoint", - "value": "/request" - } - ], - "datapoints": [ - { - "timestamp": 1527039389, - "value": 42.123456789 - } - ] - } - ], - "exhaustive": true -} -``` - -Now that you've got the M3 stack up and running, take a look at the rest of our documentation to see how you can integrate with [Prometheus](/docs/integrations/prometheus) and [Graphite](/docs/integrations/graphite) From 56261eb50acba7b4ce60853c28ddab234c3f5d87 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Wed, 28 Oct 2020 23:40:05 -0400 Subject: [PATCH 53/55] feedback --- site/content/docs/how_to/prometheus.md | 12 +++++++++++- src/cmd/services/m3dbnode/config/bootstrap.go | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/site/content/docs/how_to/prometheus.md b/site/content/docs/how_to/prometheus.md index a40f850d36..bc2e11cbb6 100644 --- a/site/content/docs/how_to/prometheus.md +++ b/site/content/docs/how_to/prometheus.md @@ -29,6 +29,7 @@ Start by downloading the config template. Update the namespaces and the client s You'll need to specify the static IPs or hostnames of your M3DB seed nodes, and the name and retention values of the namespace you set up. You can leave the namespace storage metrics type as unaggregated since it's required by default to have a cluster that receives all Prometheus metrics unaggregated. In the future you might also want to aggregate and downsample metrics for longer retention, and you can come back and update the config once you've setup those clusters. You can read more about our aggregation functionality here. It should look something like: +```yaml listenAddress: 0.0.0.0:7201 logging: @@ -84,32 +85,41 @@ clusters: jitter: true backgroundHealthCheckFailLimit: 4 backgroundHealthCheckFailThrottleFactor: 0.5 +``` Now start the process up: -m3coordinator -f +`m3coordinator -f ` Or, use the docker container: +``` docker pull quay.io/m3db/m3coordinator:latest docker run -p 7201:7201 --name m3coordinator -v :/etc/m3coordinator/m3coordinator.yml quay.io/m3db/m3coordinator:latest +``` ### Prometheus configuration Add to your Prometheus configuration the m3coordinator sidecar remote read/write endpoints, something like: +``` remote_read: - url: "http://localhost:7201/api/v1/prom/remote/read" # To test reading even when local Prometheus has the data read_recent: true remote_write: - url: "http://localhost:7201/api/v1/prom/remote/write" +``` Also, we recommend adding M3DB and M3Coordinator/M3Query to your list of jobs under scrape_configs so that you can monitor them using Prometheus. With this scraping setup, you can also use our pre-configured M3DB Grafana dashboard. +``` - job_name: 'm3db' static_configs: - targets: [':7203', ':7203', ':7203'] - job_name: 'm3coordinator' static_configs: - targets: [':7203'] +``` NOTE: If you are running M3DB with embedded M3Coordinator, you should only have one job. We recommend just calling this job m3. For example: +``` - job_name: 'm3' static_configs: - targets: [':7203'] +``` diff --git a/src/cmd/services/m3dbnode/config/bootstrap.go b/src/cmd/services/m3dbnode/config/bootstrap.go index e598248581..35a436f1b3 100644 --- a/src/cmd/services/m3dbnode/config/bootstrap.go +++ b/src/cmd/services/m3dbnode/config/bootstrap.go @@ -115,7 +115,7 @@ func (m BootstrapMode) String() string { case PreferPeersBootstrapMode: return "prefer_peers" } - return "default" + return "unknown" } // BootstrapConfiguration specifies the config for bootstrappers. From 88efec468e72861d412f5e4a181d831adab26827 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Wed, 28 Oct 2020 23:44:33 -0400 Subject: [PATCH 54/55] feedback 2 --- src/cmd/services/m3dbnode/config/bootstrap.go | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/cmd/services/m3dbnode/config/bootstrap.go b/src/cmd/services/m3dbnode/config/bootstrap.go index 35a436f1b3..9cd22828ca 100644 --- a/src/cmd/services/m3dbnode/config/bootstrap.go +++ b/src/cmd/services/m3dbnode/config/bootstrap.go @@ -66,9 +66,19 @@ var ( uninitialized.UninitializedTopologyBootstrapperName, } + // bootstrapper order where peers is prefered over commitlog. + excludeCommitLogOrderedBootstrappers = []string{ + // Filesystem bootstrapping must be first. + bfs.FileSystemBootstrapperName, + // Commitlog excluded. + peers.PeersBootstrapperName, + uninitialized.UninitializedTopologyBootstrapperName, + } + validBootstrapModes = []BootstrapMode{ DefaultBootstrapMode, PreferPeersBootstrapMode, + ExcludeCommitLogBootstrapMode, } errReadBootstrapModeInvalid = errors.New("bootstrap mode invalid") @@ -82,6 +92,8 @@ const ( DefaultBootstrapMode BootstrapMode = iota // PreferPeersBootstrapMode executes peers before commitlog bootstrapper. PreferPeersBootstrapMode + // ExcludeCommitLogBootstrapMode executes all default bootstrappers except commitlog. + ExcludeCommitLogBootstrapMode ) // UnmarshalYAML unmarshals an BootstrapMode into a valid type from string. @@ -114,6 +126,8 @@ func (m BootstrapMode) String() string { return "default" case PreferPeersBootstrapMode: return "prefer_peers" + case ExcludeCommitLogBootstrapMode: + return "exclude_commitlog" } return "unknown" } @@ -375,8 +389,15 @@ func (bsc BootstrapConfiguration) peersConfig() BootstrapPeersConfiguration { } func (bsc BootstrapConfiguration) orderedBootstrappers() []string { - if bsc.BootstrapMode != nil && *bsc.BootstrapMode == PreferPeersBootstrapMode { - return preferPeersOrderedBootstrappers + if bsc.BootstrapMode != nil { + switch *bsc.BootstrapMode { + case DefaultBootstrapMode: + return defaultOrderedBootstrappers + case PreferPeersBootstrapMode: + return preferPeersOrderedBootstrappers + case ExcludeCommitLogBootstrapMode: + return excludeCommitLogOrderedBootstrappers + } } return defaultOrderedBootstrappers } From 0fc5de13b07b00147be505e311af5f7f83188477 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Thu, 29 Oct 2020 07:27:43 -0400 Subject: [PATCH 55/55] comment --- src/cmd/services/m3dbnode/config/bootstrap.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd/services/m3dbnode/config/bootstrap.go b/src/cmd/services/m3dbnode/config/bootstrap.go index 9cd22828ca..108b9dc824 100644 --- a/src/cmd/services/m3dbnode/config/bootstrap.go +++ b/src/cmd/services/m3dbnode/config/bootstrap.go @@ -66,7 +66,7 @@ var ( uninitialized.UninitializedTopologyBootstrapperName, } - // bootstrapper order where peers is prefered over commitlog. + // bootstrapper order where commitlog is excluded. excludeCommitLogOrderedBootstrappers = []string{ // Filesystem bootstrapping must be first. bfs.FileSystemBootstrapperName,