From ea22e4e67be67426bacb5e2bd48121ea422f9187 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Tue, 14 Apr 2020 19:29:45 -0700 Subject: [PATCH] yaml: all other fields Signed-off-by: Sugu Sougoumarane --- config/tablet/default.yaml | 29 +++++----- go/vt/vttablet/endtoend/config_test.go | 4 +- .../vttablet/tabletserver/messager/engine.go | 2 +- go/vt/vttablet/tabletserver/query_engine.go | 6 +- .../tabletserver/query_engine_test.go | 4 +- go/vt/vttablet/tabletserver/schema/engine.go | 2 +- .../tabletserver/schema/engine_test.go | 6 +- .../vttablet/tabletserver/tabletenv/config.go | 56 ++++++++----------- .../tabletserver/tabletenv/config_test.go | 34 +++++++++++ go/vt/vttablet/tabletserver/tx_engine.go | 2 +- go/vt/vttablet/tabletserver/tx_engine_test.go | 4 +- 11 files changed, 87 insertions(+), 62 deletions(-) diff --git a/config/tablet/default.yaml b/config/tablet/default.yaml index 87d88e855eb..44b07ea8779 100644 --- a/config/tablet/default.yaml +++ b/config/tablet/default.yaml @@ -98,17 +98,18 @@ messagePostponeParallelism: 4 # queryserver-config-message-postpone-c cacheResultFields: true # enable-query-plan-field-caching -enforce_strict_trans_tables -queryserver-config-strict-table-acl -queryserver-config-enable-table-acl-dry-run -queryserver-config-acl-exempt-acl -enable-tx-throttler -tx-throttler-config -tx-throttler-healthcheck-cells -enable_transaction_limit -enable_transaction_limit_dry_run -transaction_limit_per_user -transaction_limit_by_username -transaction_limit_by_principal -transaction_limit_by_component -transaction_limit_by_subcomponent +# The following flags are currently not supported. +# enforce_strict_trans_tables +# queryserver-config-strict-table-acl +# queryserver-config-enable-table-acl-dry-run +# queryserver-config-acl-exempt-acl +# enable-tx-throttler +# tx-throttler-config +# tx-throttler-healthcheck-cells +# enable_transaction_limit +# enable_transaction_limit_dry_run +# transaction_limit_per_user +# transaction_limit_by_username +# transaction_limit_by_principal +# transaction_limit_by_component +# transaction_limit_by_subcomponent diff --git a/go/vt/vttablet/endtoend/config_test.go b/go/vt/vttablet/endtoend/config_test.go index 4c61b6c84b1..026c2855e69 100644 --- a/go/vt/vttablet/endtoend/config_test.go +++ b/go/vt/vttablet/endtoend/config_test.go @@ -72,13 +72,13 @@ func TestConfigVars(t *testing.T) { val: currentConfig.Oltp.WarnRows, }, { tag: "QueryCacheCapacity", - val: currentConfig.QueryPlanCacheSize, + val: currentConfig.QueryCacheSize, }, { tag: "QueryTimeout", val: int(currentConfig.Oltp.QueryTimeoutSeconds * 1e9), }, { tag: "SchemaReloadTime", - val: int(currentConfig.SchemaReloadTime * 1e9), + val: int(currentConfig.SchemaReloadIntervalSeconds * 1e9), }, { tag: "StreamBufferSize", val: currentConfig.StreamBufferSize, diff --git a/go/vt/vttablet/tabletserver/messager/engine.go b/go/vt/vttablet/tabletserver/messager/engine.go index 4e7d8818bc7..0d275d7bdc7 100644 --- a/go/vt/vttablet/tabletserver/messager/engine.go +++ b/go/vt/vttablet/tabletserver/messager/engine.go @@ -66,7 +66,7 @@ func NewEngine(tsv TabletService, se *schema.Engine, vs VStreamer) *Engine { tsv: tsv, se: se, vs: vs, - postponeSema: sync2.NewSemaphore(tsv.Config().MessagePostponeCap, 0), + postponeSema: sync2.NewSemaphore(tsv.Config().MessagePostponeParallelism, 0), managers: make(map[string]*messageManager), } } diff --git a/go/vt/vttablet/tabletserver/query_engine.go b/go/vt/vttablet/tabletserver/query_engine.go index fa54ec70269..d46b24d6641 100644 --- a/go/vt/vttablet/tabletserver/query_engine.go +++ b/go/vt/vttablet/tabletserver/query_engine.go @@ -169,14 +169,14 @@ func NewQueryEngine(env tabletenv.Env, se *schema.Engine) *QueryEngine { env: env, se: se, tables: make(map[string]*schema.Table), - plans: cache.NewLRUCache(int64(config.QueryPlanCacheSize)), + plans: cache.NewLRUCache(int64(config.QueryCacheSize)), queryRuleSources: rules.NewMap(), } qe.conns = connpool.New(env, "ConnPool", config.OltpReadPool) qe.streamConns = connpool.New(env, "StreamConnPool", config.OlapReadPool) qe.consolidatorMode = config.Consolidator - qe.enableQueryPlanFieldCaching = config.EnableQueryPlanFieldCaching + qe.enableQueryPlanFieldCaching = config.CacheResultFields qe.consolidator = sync2.NewConsolidator() qe.txSerializer = txserializer.New(env) qe.streamQList = NewQueryList() @@ -203,7 +203,7 @@ func NewQueryEngine(env tabletenv.Env, se *schema.Engine) *QueryEngine { qe.warnResultSize = sync2.NewAtomicInt64(int64(config.Oltp.WarnRows)) qe.streamBufferSize = sync2.NewAtomicInt64(int64(config.StreamBufferSize)) - planbuilder.PassthroughDMLs = config.PassthroughDMLs + planbuilder.PassthroughDMLs = config.PassthroughDML qe.accessCheckerLogger = logutil.NewThrottledLogger("accessChecker", 1*time.Second) diff --git a/go/vt/vttablet/tabletserver/query_engine_test.go b/go/vt/vttablet/tabletserver/query_engine_test.go index fc1f4f7777a..d6fce353947 100644 --- a/go/vt/vttablet/tabletserver/query_engine_test.go +++ b/go/vt/vttablet/tabletserver/query_engine_test.go @@ -272,9 +272,9 @@ func TestStatsURL(t *testing.T) { qe.handleHTTPQueryRules(response, request) } -func newTestQueryEngine(queryPlanCacheSize int, idleTimeout time.Duration, strict bool, dbcfgs *dbconfigs.DBConfigs) *QueryEngine { +func newTestQueryEngine(queryCacheSize int, idleTimeout time.Duration, strict bool, dbcfgs *dbconfigs.DBConfigs) *QueryEngine { config := tabletenv.NewDefaultConfig() - config.QueryPlanCacheSize = queryPlanCacheSize + config.QueryCacheSize = queryCacheSize config.OltpReadPool.IdleTimeoutSeconds = int(idleTimeout / 1e9) config.OlapReadPool.IdleTimeoutSeconds = int(idleTimeout / 1e9) config.TxPool.IdleTimeoutSeconds = int(idleTimeout / 1e9) diff --git a/go/vt/vttablet/tabletserver/schema/engine.go b/go/vt/vttablet/tabletserver/schema/engine.go index 9ab05e4c049..0908f72b734 100644 --- a/go/vt/vttablet/tabletserver/schema/engine.go +++ b/go/vt/vttablet/tabletserver/schema/engine.go @@ -67,7 +67,7 @@ type Engine struct { // NewEngine creates a new Engine. func NewEngine(env tabletenv.Env) *Engine { - reloadTime := time.Duration(env.Config().SchemaReloadTime * 1e9) + reloadTime := time.Duration(env.Config().SchemaReloadIntervalSeconds * 1e9) se := &Engine{ env: env, // We need only one connection because the reloader is diff --git a/go/vt/vttablet/tabletserver/schema/engine_test.go b/go/vt/vttablet/tabletserver/schema/engine_test.go index 6fab38f850b..9faced3730c 100644 --- a/go/vt/vttablet/tabletserver/schema/engine_test.go +++ b/go/vt/vttablet/tabletserver/schema/engine_test.go @@ -298,10 +298,10 @@ func TestStatsURL(t *testing.T) { se.handleDebugSchema(response, request) } -func newEngine(queryPlanCacheSize int, reloadTime time.Duration, idleTimeout time.Duration, strict bool, db *fakesqldb.DB) *Engine { +func newEngine(queryCacheSize int, reloadTime time.Duration, idleTimeout time.Duration, strict bool, db *fakesqldb.DB) *Engine { config := tabletenv.NewDefaultConfig() - config.QueryPlanCacheSize = queryPlanCacheSize - config.SchemaReloadTime = float64(reloadTime) / 1e9 + config.QueryCacheSize = queryCacheSize + config.SchemaReloadIntervalSeconds = int(reloadTime) / 1e9 config.OltpReadPool.IdleTimeoutSeconds = int(idleTimeout / 1e9) config.OlapReadPool.IdleTimeoutSeconds = int(idleTimeout / 1e9) config.TxPool.IdleTimeoutSeconds = int(idleTimeout / 1e9) diff --git a/go/vt/vttablet/tabletserver/tabletenv/config.go b/go/vt/vttablet/tabletserver/tabletenv/config.go index 0b8358b4ef6..40e1ce235ae 100644 --- a/go/vt/vttablet/tabletserver/tabletenv/config.go +++ b/go/vt/vttablet/tabletserver/tabletenv/config.go @@ -80,19 +80,19 @@ func init() { flag.IntVar(&deprecatedMessagePoolPrefillParallelism, "queryserver-config-message-conn-pool-prefill-parallelism", 0, "DEPRECATED: Unused.") flag.IntVar(¤tConfig.TxPool.Size, "queryserver-config-transaction-cap", defaultConfig.TxPool.Size, "query server transaction cap is the maximum number of transactions allowed to happen at any given point of a time for a single vttablet. E.g. by setting transaction cap to 100, there are at most 100 transactions will be processed by a vttablet and the 101th transaction will be blocked (and fail if it cannot get connection within specified timeout)") flag.IntVar(¤tConfig.TxPool.PrefillParallelism, "queryserver-config-transaction-prefill-parallelism", defaultConfig.TxPool.PrefillParallelism, "query server transaction prefill parallelism, a non-zero value will prefill the pool using the specified parallism.") - flag.IntVar(¤tConfig.MessagePostponeCap, "queryserver-config-message-postpone-cap", defaultConfig.MessagePostponeCap, "query server message postpone cap is the maximum number of messages that can be postponed at any given time. Set this number to substantially lower than transaction cap, so that the transaction pool isn't exhausted by the message subsystem.") + flag.IntVar(¤tConfig.MessagePostponeParallelism, "queryserver-config-message-postpone-cap", defaultConfig.MessagePostponeParallelism, "query server message postpone cap is the maximum number of messages that can be postponed at any given time. Set this number to substantially lower than transaction cap, so that the transaction pool isn't exhausted by the message subsystem.") flag.IntVar(&deprecatedFoundRowsPoolSize, "client-found-rows-pool-size", 0, "DEPRECATED: queryserver-config-transaction-cap will be used instead.") flag.IntVar(¤tConfig.Oltp.TxTimeoutSeconds, "queryserver-config-transaction-timeout", defaultConfig.Oltp.TxTimeoutSeconds, "query server transaction timeout (in seconds), a transaction will be killed if it takes longer than this value") - flag.Float64Var(¤tConfig.TxShutDownGracePeriod, "transaction_shutdown_grace_period", defaultConfig.TxShutDownGracePeriod, "how long to wait (in seconds) for transactions to complete during graceful shutdown.") + flag.IntVar(¤tConfig.ShutdownGracePeriodSeconds, "transaction_shutdown_grace_period", defaultConfig.ShutdownGracePeriodSeconds, "how long to wait (in seconds) for transactions to complete during graceful shutdown.") flag.IntVar(¤tConfig.Oltp.MaxRows, "queryserver-config-max-result-size", defaultConfig.Oltp.MaxRows, "query server max result size, maximum number of rows allowed to return from vttablet for non-streaming queries.") flag.IntVar(¤tConfig.Oltp.WarnRows, "queryserver-config-warn-result-size", defaultConfig.Oltp.WarnRows, "query server result size warning threshold, warn if number of rows returned from vttablet for non-streaming queries exceeds this") flag.IntVar(&deprecatedMaxDMLRows, "queryserver-config-max-dml-rows", 0, "query server max dml rows per statement, maximum number of rows allowed to return at a time for an update or delete with either 1) an equality where clauses on primary keys, or 2) a subselect statement. For update and delete statements in above two categories, vttablet will split the original query into multiple small queries based on this configuration value. ") - flag.BoolVar(¤tConfig.PassthroughDMLs, "queryserver-config-passthrough-dmls", defaultConfig.PassthroughDMLs, "query server pass through all dml statements without rewriting") + flag.BoolVar(¤tConfig.PassthroughDML, "queryserver-config-passthrough-dmls", defaultConfig.PassthroughDML, "query server pass through all dml statements without rewriting") flag.BoolVar(&deprecateAllowUnsafeDMLs, "queryserver-config-allowunsafe-dmls", false, "deprecated") flag.IntVar(¤tConfig.StreamBufferSize, "queryserver-config-stream-buffer-size", defaultConfig.StreamBufferSize, "query server stream buffer size, the maximum number of bytes sent from vttablet for each stream call. It's recommended to keep this value in sync with vtgate's stream_buffer_size.") - flag.IntVar(¤tConfig.QueryPlanCacheSize, "queryserver-config-query-cache-size", defaultConfig.QueryPlanCacheSize, "query server query cache size, maximum number of queries to be cached. vttablet analyzes every incoming query and generate a query plan, these plans are being cached in a lru cache. This config controls the capacity of the lru cache.") - flag.Float64Var(¤tConfig.SchemaReloadTime, "queryserver-config-schema-reload-time", defaultConfig.SchemaReloadTime, "query server schema reload time, how often vttablet reloads schemas from underlying MySQL instance in seconds. vttablet keeps table schemas in its own memory and periodically refreshes it from MySQL. This config controls the reload time.") + flag.IntVar(¤tConfig.QueryCacheSize, "queryserver-config-query-cache-size", defaultConfig.QueryCacheSize, "query server query cache size, maximum number of queries to be cached. vttablet analyzes every incoming query and generate a query plan, these plans are being cached in a lru cache. This config controls the capacity of the lru cache.") + flag.IntVar(¤tConfig.SchemaReloadIntervalSeconds, "queryserver-config-schema-reload-time", defaultConfig.SchemaReloadIntervalSeconds, "query server schema reload time, how often vttablet reloads schemas from underlying MySQL instance in seconds. vttablet keeps table schemas in its own memory and periodically refreshes it from MySQL. This config controls the reload time.") flag.IntVar(¤tConfig.Oltp.QueryTimeoutSeconds, "queryserver-config-query-timeout", defaultConfig.Oltp.QueryTimeoutSeconds, "query server query timeout (in seconds), this is the query timeout in vttablet side. If a query takes more than this timeout, it will be killed.") flag.IntVar(¤tConfig.OltpReadPool.TimeoutSeconds, "queryserver-config-query-pool-timeout", defaultConfig.OltpReadPool.TimeoutSeconds, "query server query pool timeout (in seconds), it is how long vttablet waits for a connection from the query pool. If set to 0 (default) then the overall query timeout is used instead.") flag.IntVar(¤tConfig.TxPool.TimeoutSeconds, "queryserver-config-txpool-timeout", defaultConfig.TxPool.TimeoutSeconds, "query server transaction pool timeout, it is how long vttablet waits if tx pool is full") @@ -134,7 +134,7 @@ func init() { flag.BoolVar(¤tConfig.EnforceStrictTransTables, "enforce_strict_trans_tables", defaultConfig.EnforceStrictTransTables, "If true, vttablet requires MySQL to run with STRICT_TRANS_TABLES or STRICT_ALL_TABLES on. It is recommended to not turn this flag off. Otherwise MySQL may alter your supplied values before saving them to the database.") flag.BoolVar(&enableConsolidator, "enable-consolidator", true, "This option enables the query consolidator.") flag.BoolVar(&enableConsolidatorReplicas, "enable-consolidator-replicas", false, "This option enables the query consolidator only on replicas.") - flag.BoolVar(¤tConfig.EnableQueryPlanFieldCaching, "enable-query-plan-field-caching", defaultConfig.EnableQueryPlanFieldCaching, "This option fetches & caches fields (columns) when storing query plans") + flag.BoolVar(¤tConfig.CacheResultFields, "enable-query-plan-field-caching", defaultConfig.CacheResultFields, "This option fetches & caches fields (columns) when storing query plans") } // Init must be called after flag.Parse, and before doing any other operations. @@ -191,18 +191,19 @@ type TabletConfig struct { HotRowProtection HotRowProtectionConfig `json:"hotRowProtection,omitempty"` Consolidator string `json:"consolidator,omitempty"` HeartbeatIntervalMilliseconds int `json:"heartbeatIntervalMilliseconds,omitempty"` + ShutdownGracePeriodSeconds int `json:"shutdownGracePeriodSeconds,omitempty"` + PassthroughDML bool `json:"passthroughDML,omitempty"` + StreamBufferSize int `json:"streamBufferSize,omitempty"` + QueryCacheSize int `json:"queryCacheSize,omitempty"` + SchemaReloadIntervalSeconds int `json:"schemaReloadIntervalSeconds,omitempty"` + WatchReplication bool `json:"watchReplication,omitempty"` + TerseErrors bool `json:"terseErrors,omitempty"` + MessagePostponeParallelism int `json:"messagePostponeParallelism,omitempty"` + CacheResultFields bool `json:"cacheResultFields,omitempty"` - MessagePostponeCap int `json:"-"` - TxShutDownGracePeriod float64 `json:"-"` - PassthroughDMLs bool `json:"-"` - StreamBufferSize int `json:"-"` - QueryPlanCacheSize int `json:"-"` - SchemaReloadTime float64 `json:"-"` StrictTableACL bool `json:"-"` - TerseErrors bool `json:"-"` EnableTableACLDryRun bool `json:"-"` TableACLExemptACL string `json:"-"` - WatchReplication bool `json:"-"` TwoPCEnable bool `json:"-"` TwoPCCoordinatorAddress string `json:"-"` TwoPCAbandonAge float64 `json:"-"` @@ -213,8 +214,7 @@ type TabletConfig struct { TransactionLimitConfig `json:"-"` - EnforceStrictTransTables bool `json:"-"` - EnableQueryPlanFieldCaching bool `json:"-"` + EnforceStrictTransTables bool `json:"-"` } // ConnPoolConfig contains the config for a conn pool. @@ -356,21 +356,12 @@ var defaultConfig = TabletConfig{ // of them ready in MySQL and profit from a pipelining effect. MaxConcurrency: 5, }, - Consolidator: Enable, - MessagePostponeCap: 4, - TxShutDownGracePeriod: 0, - PassthroughDMLs: false, - QueryPlanCacheSize: 5000, - SchemaReloadTime: 30 * 60, - StreamBufferSize: 32 * 1024, - StrictTableACL: false, - TerseErrors: false, - EnableTableACLDryRun: false, - TableACLExemptACL: "", - WatchReplication: false, - TwoPCEnable: false, - TwoPCCoordinatorAddress: "", - TwoPCAbandonAge: 0, + Consolidator: Enable, + StreamBufferSize: 32 * 1024, + QueryCacheSize: 5000, + SchemaReloadIntervalSeconds: 30 * 60, + MessagePostponeParallelism: 4, + CacheResultFields: true, EnableTxThrottler: false, TxThrottlerConfig: defaultTxThrottlerConfig(), @@ -378,8 +369,7 @@ var defaultConfig = TabletConfig{ TransactionLimitConfig: defaultTransactionLimitConfig(), - EnforceStrictTransTables: true, - EnableQueryPlanFieldCaching: true, + EnforceStrictTransTables: true, } // defaultTxThrottlerConfig formats the default throttlerdata.Configuration diff --git a/go/vt/vttablet/tabletserver/tabletenv/config_test.go b/go/vt/vttablet/tabletserver/tabletenv/config_test.go index fa08ac46797..52073c43375 100644 --- a/go/vt/vttablet/tabletserver/tabletenv/config_test.go +++ b/go/vt/vttablet/tabletserver/tabletenv/config_test.go @@ -61,3 +61,37 @@ txPool: {} require.NoError(t, err) assert.Equal(t, cfg, gotCfg) } + +func TestDefaultConfig(t *testing.T) { + gotBytes, err := yaml.Marshal(NewDefaultConfig()) + require.NoError(t, err) + want := `cacheResultFields: true +consolidator: enable +hotRowProtection: + maxConcurrency: 5 + maxGlobalQueueSize: 1000 + maxQueueSize: 20 + mode: disable +messagePostponeParallelism: 4 +olapReadPool: + idleTimeoutSeconds: 1800 + size: 200 +oltp: + maxRpws: 10000 + queryTimeoutSeconds: 30 + txTimeoutSeconds: 30 +oltpReadPool: + idleTimeoutSeconds: 1800 + maxWaiters: 5000 + size: 16 +queryCacheSize: 5000 +schemaReloadIntervalSeconds: 1800 +streamBufferSize: 32768 +txPool: + idleTimeoutSeconds: 1800 + maxWaiters: 5000 + size: 20 + timeoutSeconds: 1 +` + assert.Equal(t, want, string(gotBytes)) +} diff --git a/go/vt/vttablet/tabletserver/tx_engine.go b/go/vt/vttablet/tabletserver/tx_engine.go index 018b461f9f7..b629e5b3487 100644 --- a/go/vt/vttablet/tabletserver/tx_engine.go +++ b/go/vt/vttablet/tabletserver/tx_engine.go @@ -100,7 +100,7 @@ func NewTxEngine(env tabletenv.Env) *TxEngine { config := env.Config() te := &TxEngine{ env: env, - shutdownGracePeriod: time.Duration(config.TxShutDownGracePeriod * 1e9), + shutdownGracePeriod: time.Duration(config.ShutdownGracePeriodSeconds * 1e9), } limiter := txlimiter.New(env) te.txPool = NewTxPool(env, limiter) diff --git a/go/vt/vttablet/tabletserver/tx_engine_test.go b/go/vt/vttablet/tabletserver/tx_engine_test.go index 53df128a124..5277843ab36 100644 --- a/go/vt/vttablet/tabletserver/tx_engine_test.go +++ b/go/vt/vttablet/tabletserver/tx_engine_test.go @@ -40,7 +40,7 @@ func TestTxEngineClose(t *testing.T) { config := tabletenv.NewDefaultConfig() config.TxPool.Size = 10 config.Oltp.TxTimeoutSeconds = 1 - config.TxShutDownGracePeriod = 0 + config.ShutdownGracePeriodSeconds = 0 te := NewTxEngine(tabletenv.NewTestEnv(config, dbcfgs, "TabletServerTest")) // Normal close. @@ -463,7 +463,7 @@ func setupTxEngine(db *fakesqldb.DB) *TxEngine { config := tabletenv.NewDefaultConfig() config.TxPool.Size = 10 config.Oltp.TxTimeoutSeconds = 1 - config.TxShutDownGracePeriod = 0 + config.ShutdownGracePeriodSeconds = 0 te := NewTxEngine(tabletenv.NewTestEnv(config, dbcfgs, "TabletServerTest")) return te }