yaml: phase 1

Makefile

@@ -144,28 +144,12 @@ java_test:
 install_protoc-gen-go:
 	go install github.com/golang/protobuf/protoc-gen-go
 
-# Find protoc compiler.
-# NOTE: We are *not* using the "protoc" binary (as suggested by the grpc Go
-#       quickstart for example). Instead, we run "protoc" via the Python
-#       wrapper script which is provided by the "grpcio-tools" PyPi package.
-#       (The package includes the compiler as library, but not as binary.
-#       Therefore, we have to use the wrapper script they provide.)
-ifneq ($(wildcard $(VTROOT)/dist/grpc/usr/local/lib/python2.7/site-packages/grpc_tools/protoc.py),)
-# IMPORTANT: The next line must not be indented.
-PROTOC_COMMAND := python -m grpc_tools.protoc
-endif
-
 PROTO_SRCS = $(wildcard proto/*.proto)
 PROTO_SRC_NAMES = $(basename $(notdir $(PROTO_SRCS)))
 PROTO_GO_OUTS = $(foreach name, $(PROTO_SRC_NAMES), go/vt/proto/$(name)/$(name).pb.go)
 
-# This rule rebuilds all the go and python files from the proto definitions for gRPC.
-proto: proto_banner $(PROTO_GO_OUTS)
-
-proto_banner:
-ifeq (,$(PROTOC_COMMAND))
-	$(error "Cannot find protoc compiler. Did bootstrap.sh succeed, and did you execute 'source dev.env'?")
-endif
+# This rule rebuilds all the go files from the proto definitions for gRPC.
+proto: $(PROTO_GO_OUTS)
 
 ifndef NOBANNER
 	echo $$(date): Compiling proto definitions

config/tablet/default.yaml

@@ -0,0 +1,115 @@
+tabletID: zone-1234
+
+init:
+  dbName:            # init_db_name_override
+  keyspace:          # init_keyspace
+  shard:             # init_shard
+  tabletType:        # init_tablet_type
+  timeoutSeconds: 60 # init_timeout
+
+db:
+  socket:     # db_socket
+  host:       # db_host
+  port: 0     # db_port
+  charSet:    # db_charset
+  flags: 0    # db_flags
+  flavor:     # db_flavor
+  sslCa:      # db_ssl_ca
+  sslCaPath:  # db_ssl_ca_path
+  sslCert:    # db_ssl_cert
+  sslKey:     # db_ssl_key
+  serverName: # db_server_name
+  connectTimeoutMilliseconds: 0 # db_connect_timeout_ms
+  app:
+    user: vt_app      # db_app_user
+    password:         # db_app_password
+    useSsl: true      # db_app_use_ssl
+    preferSocket: true
+  dba:
+    user: vt_dba      # db_dba_user
+    password:         # db_dba_password
+    useSsl: true      # db_dba_use_ssl
+    preferSocket: true
+  filtered:
+    user: vt_filtered # db_filtered_user
+    password:         # db_filtered_password
+    useSsl: true      # db_filtered_use_ssl
+    preferSocket: true
+  repl:
+    user: vt_repl     # db_repl_user
+    password:         # db_repl_password
+    useSsl: true      # db_repl_use_ssl
+    preferSocket: true
+  appdebug:
+    user: vt_appdebug # db_appdebug_user
+    password:         # db_appdebug_password
+    useSsl: true      # db_appdebug_use_ssl
+    preferSocket: true
+  allprivs:
+    user: vt_allprivs # db_allprivs_user
+    password:         # db_allprivs_password
+    useSsl: true      # db_allprivs_use_ssl
+    preferSocket: true
+
+oltpReadPool:
+  size: 16                 # queryserver-config-pool-size
+  timeoutSeconds: 0        # queryserver-config-query-pool-timeout
+  idleTimeoutSeconds: 1800 # queryserver-config-idle-timeout
+  prefillParallelism: 0    # queryserver-config-pool-prefill-parallelism
+  maxWaiters: 50000        # queryserver-config-query-pool-waiter-cap
+
+olapReadPool:
+  size: 200                # queryserver-config-stream-pool-size
+  timeoutSeconds: 0        # queryserver-config-query-pool-timeout
+  idleTimeoutSeconds: 1800 # queryserver-config-idle-timeout
+  prefillParallelism: 0    # queryserver-config-stream-pool-prefill-parallelism
+  maxWaiters: 0
+
+txPool:
+  size: 20                 # queryserver-config-transaction-cap
+  timeoutSeconds: 1        # queryserver-config-txpool-timeout
+  idleTimeoutSeconds: 1800 # queryserver-config-idle-timeout
+  prefillParallelism: 0    # queryserver-config-transaction-prefill-parallelism
+  maxWaiters: 50000        # queryserver-config-txpool-waiter-cap
+
+oltp:
+  queryTimeoutSeconds: 30 # queryserver-config-query-timeout
+  txTimeoutSeconds: 30    # queryserver-config-transaction-timeout
+  maxRows: 10000          # queryserver-config-max-result-size
+  warnRows: 0             # queryserver-config-warn-result-size
+
+hotRowProtection:
+  mode: disable|dryRun|enable # enable_hot_row_protection, enable_hot_row_protection_dry_run
+  # Default value is same as txPool.size.
+  maxQueueSize: 20            # hot_row_protection_max_queue_size
+  maxGlobalQueueSize: 1000    # hot_row_protection_max_global_queue_size
+  maxConcurrency: 5           # hot_row_protection_concurrent_transactions
+
+consolidator: enable|disable|notOnMaster # enable-consolidator, enable-consolidator-replicas
+heartbeatIntervalMilliseconds: 0         # heartbeat_enable, heartbeat_interval
+shutdownGracePeriodSeconds: 0            # transaction_shutdown_grace_period
+passthroughDML: false                    # queryserver-config-passthrough-dmls
+streamBufferSize: 32768                  # queryserver-config-stream-buffer-size
+queryCacheSize: 5000                     # queryserver-config-query-cache-size
+schemaReloadIntervalSeconds: 1800        # queryserver-config-schema-reload-time
+watchReplication: false                  # watch_replication_stream
+terseErrors: false                       # queryserver-config-terse-errors
+messagePostponeParallelism: 4            # queryserver-config-message-postpone-cap
+cacheResultFields: true                  # enable-query-plan-field-caching
+
+
+# The following flags are currently not supported.
+# enforce_strict_trans_tables
+# queryserver-config-strict-table-acl
+# queryserver-config-enable-table-acl-dry-run
+# queryserver-config-acl-exempt-acl
+# enable-tx-throttler
+# tx-throttler-config
+# tx-throttler-healthcheck-cells
+# enable_transaction_limit
+# enable_transaction_limit_dry_run
+# transaction_limit_per_user
+# transaction_limit_by_username
+# transaction_limit_by_principal
+# transaction_limit_by_component
+# transaction_limit_by_subcomponent

go.mod

@@ -162,6 +162,7 @@ require (
 	k8s.io/apimachinery v0.17.3
 	k8s.io/client-go v0.17.3
 	mvdan.cc/unparam v0.0.0-20191111180625-960b1ec0f2c2 // indirect
+	sigs.k8s.io/yaml v1.1.0
 	sourcegraph.com/sqs/pbtypes v1.0.0 // indirect
 	vitess.io/vitess/examples/are-you-alive v0.0.0-20200302220708-6b7695375ce9 // indirect
 )

go/cmd/vttablet/vttablet.go

@@ -19,8 +19,10 @@ package main
 
 import (
 	"flag"
+	"io/ioutil"
 
 	"golang.org/x/net/context"
+	"sigs.k8s.io/yaml"
 	"vitess.io/vitess/go/vt/dbconfigs"
 	"vitess.io/vitess/go/vt/log"
 	"vitess.io/vitess/go/vt/mysqlctl"
@@ -39,6 +41,7 @@ var (
 	tableACLConfig               = flag.String("table-acl-config", "", "path to table access checker config file; send SIGHUP to reload this file")
 	tableACLConfigReloadInterval = flag.Duration("table-acl-config-reload-interval", 0, "Ticker to reload ACLs")
 	tabletPath                   = flag.String("tablet-path", "", "tablet alias")
+	tabletConfig                 = flag.String("tablet_config", "", "YAML file config for tablet")
 
 	agent *tabletmanager.ActionAgent
 )
@@ -53,11 +56,23 @@ func main() {
 
 	servenv.ParseFlags("vttablet")
 
-	if err := tabletenv.VerifyConfig(); err != nil {
+	config := tabletenv.NewCurrentConfig()
+	if err := config.Verify(); err != nil {
 		log.Exitf("invalid config: %v", err)
 	}
 
 	tabletenv.Init()
+	if *tabletConfig != "" {
+		bytes, err := ioutil.ReadFile(*tabletConfig)
+		if err != nil {
+			log.Exitf("error reading config file %s: %v", *tabletConfig, err)
+		}
+		if err := yaml.Unmarshal(bytes, config); err != nil {
+			log.Exitf("error parsing config file %s: %v", bytes, err)
+		}
+		gotBytes, _ := yaml.Marshal(config)
+		log.Infof("Loaded config file %s successfully:\n%s", *tabletConfig, gotBytes)
+	}
 
 	servenv.Init()
 
@@ -102,7 +117,7 @@ func main() {
 
 	// creates and registers the query service
 	ts := topo.Open()
-	qsc := tabletserver.NewServer("", ts, *tabletAlias)
+	qsc := tabletserver.NewTabletServer("", config, ts, *tabletAlias)
 	servenv.OnRun(func() {
 		qsc.Register()
 		addStatusParts(qsc)

go/vt/tableacl/tableacl.go

@@ -331,10 +331,11 @@ func checkHealth(acl *tableACL) error {
 
 func init() {
 	servenv.OnRun(func() {
-		if !tabletenv.Config.StrictTableACL {
+		currentConfig := tabletenv.NewCurrentConfig()
+		if !currentConfig.StrictTableACL {
 			return
 		}
-		if tabletenv.Config.EnableTableACLDryRun {
+		if currentConfig.EnableTableACLDryRun {
 			return
 		}
 		health.DefaultAggregator.RegisterSimpleCheck("tableacl", func() error { return checkHealth(&currentTableACL) })

go/vt/vtexplain/vtexplain_vttablet.go

@@ -76,7 +76,7 @@ type explainTablet struct {
 func newTablet(opts *Options, t *topodatapb.Tablet) *explainTablet {
 	db := fakesqldb.New(nil)
 
-	config := tabletenv.Config
+	config := tabletenv.NewCurrentConfig()
 	if opts.ExecutionMode == ModeTwoPC {
 		config.TwoPCCoordinatorAddress = "XXX"
 		config.TwoPCAbandonAge = 1.0