Slack vitess r14.0.5 dsdefense (#214)

This PR adds the vitess backpressure / Transaction Throttler work to our v14 release. In short, it adds the following:

* vtgates parse a comment directive specifying the query priority. If missing, a default, configurable priority is assigned to the query.
* The throttler uses the priority to determine whether a given query can or cannot be throttled.
* The throttler will throttle if the query rate is above its estimate of max query rate and there is lag.
* The throttler can now throttle write statements outside of explicit BEGIN/COMMIT blocks.
* The throttler is still guarded by a CLI flag so if not passed, throttling is not enabled.
* Throttled queries return a different/separate MySQL error code.

In addition, some changes were backported from newer versions to v14 for easier handling of parsing of some config options.

config/tablet/default.yaml

@@ -117,6 +117,7 @@ cacheResultFields: true                  # enable-query-plan-field-caching
 # enable-tx-throttler
 # tx-throttler-config
 # tx-throttler-healthcheck-cells
+# tx-throttler-tablet-types
 # enable_transaction_limit
 # enable_transaction_limit_dry_run
 # transaction_limit_per_user

doc/ReplicationLagBasedThrottlingOfTransactions.md

@@ -30,7 +30,13 @@ If this is not specified a [default](https://github.com/vitessio/vitess/tree/mai
 * *tx-throttler-healthcheck-cells*
 
 A comma separated list of datacenter cells. The throttler will only monitor
-the non-RDONLY replicas found in these cells for replication lag.
+the replicas found in these cells for replication lag.
+
+* *tx-throttler-tablet-types*
+
+A comma separated list of tablet types. The throttler will only monitor tablets
+with these types. Only `replica` and/or `rdonly` types are supported. The default
+is `replica`.
 
 # Caveats and Known Issues
 * The throttler keeps trying to explore the maximum rate possible while keeping
@@ -39,4 +45,3 @@ lag limit may occasionally be slightly violated.
 
 * Transactions are considered homogeneous. There is currently no support
 for specifying how `expensive` a transaction is.
-

go/cmd/mysqlctl/mysqlctl.go

@@ -25,6 +25,8 @@ import (
 	"os"
 	"time"
 
+	"github.com/spf13/pflag"
+
 	"vitess.io/vitess/go/cmd"
 	"vitess.io/vitess/go/exit"
 	"vitess.io/vitess/go/flagutil"
@@ -248,7 +250,7 @@ func main() {
 		exit.Return(1)
 	}
 	dbconfigs.RegisterFlags(dbconfigs.Dba)
-	_flag.Parse()
+	_flag.Parse(pflag.NewFlagSet("mysqlctl", pflag.ExitOnError))
 
 	tabletAddr = netutil.JoinHostPort("localhost", int32(*port))
 

go/cmd/query_analyzer/query_analyzer.go

@@ -24,6 +24,8 @@ import (
 	"os"
 	"sort"
 
+	"github.com/spf13/pflag"
+
 	"vitess.io/vitess/go/exit"
 	"vitess.io/vitess/go/vt/log"
 	"vitess.io/vitess/go/vt/sqlparser"
@@ -59,7 +61,7 @@ func (a stats) Less(i, j int) bool { return a[i].Count > a[j].Count }
 
 func main() {
 	defer exit.Recover()
-	_flag.Parse()
+	_flag.Parse(pflag.NewFlagSet("query_analyzer", pflag.ExitOnError))
 	for _, filename := range _flag.Args() {
 		fmt.Printf("processing: %s\n", filename)
 		if err := processFile(filename); err != nil {

go/cmd/topo2topo/topo2topo.go

@@ -22,6 +22,8 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/spf13/pflag"
+
 	"vitess.io/vitess/go/exit"
 	"vitess.io/vitess/go/vt/log"
 	"vitess.io/vitess/go/vt/logutil"
@@ -53,7 +55,7 @@ func main() {
 	defer exit.RecoverAll()
 	defer logutil.Flush()
 
-	_flag.Parse()
+	_flag.Parse(pflag.NewFlagSet("topo2topo", pflag.ExitOnError))
 	args := _flag.Args()
 	if len(args) != 0 {
 		flag.Usage()

go/cmd/vtbench/vtbench.go

@@ -23,6 +23,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/spf13/pflag"
+
 	"vitess.io/vitess/go/exit"
 	"vitess.io/vitess/go/vt/dbconfigs"
 	"vitess.io/vitess/go/vt/log"
@@ -104,7 +106,7 @@ func main() {
 	defer exit.Recover()
 
 	flag.Lookup("logtostderr").Value.Set("true")
-	_flag.Parse()
+	_flag.Parse(pflag.NewFlagSet("vtbench", pflag.ExitOnError))
 
 	clientProto := vtbench.MySQL
 	switch *protocol {

go/cmd/vtclient/vtclient.go

@@ -31,6 +31,7 @@ import (
 	"time"
 
 	"github.com/olekukonko/tablewriter"
+	"github.com/spf13/pflag"
 
 	"vitess.io/vitess/go/vt/concurrency"
 	"vitess.io/vitess/go/vt/log"
@@ -147,7 +148,7 @@ func main() {
 }
 
 func run() (*results, error) {
-	_flag.Parse()
+	_flag.Parse(pflag.NewFlagSet("vtclient", pflag.ExitOnError))
 	args := _flag.Args()
 
 	if len(args) == 0 {

go/cmd/vtctlclient/main.go

@@ -25,6 +25,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/spf13/pflag"
+
 	"vitess.io/vitess/go/exit"
 	"vitess.io/vitess/go/trace"
 	"vitess.io/vitess/go/vt/log"
@@ -70,7 +72,7 @@ func checkDeprecations(args []string) {
 func main() {
 	defer exit.Recover()
 
-	_flag.Parse()
+	_flag.Parse(pflag.NewFlagSet("vtctlclient", pflag.ExitOnError))
 
 	closer := trace.StartTracing("vtctlclient")
 	defer trace.LogErrorsWhenClosing(closer)

go/cmd/vtgate/vtgate.go

@@ -142,7 +142,7 @@ func main() {
 		log.Exitf("tablet_types_to_wait should contain at least one serving tablet type")
 	}
 
-	err := CheckCellFlags(context.Background(), resilientServer, *cell, *vtgate.CellsToWatch)
+	err := CheckCellFlags(context.Background(), resilientServer, *cell, vtgate.CellsToWatch)
 	if err != nil {
 		log.Exitf("cells_to_watch validation failed: %v", err)
 	}

go/cmd/vtgr/main.go

@@ -18,6 +18,8 @@ import (
 	"flag"
 	"strings"
 
+	"github.com/spf13/pflag"
+
 	"vitess.io/vitess/go/vt/vtgr"
 
 	// Include deprecation warnings for soon-to-be-unsupported flag invocations.
@@ -26,7 +28,7 @@ import (
 
 func main() {
 	clustersToWatch := flag.String("clusters_to_watch", "", "Comma-separated list of keyspaces or keyspace/shards that this instance will monitor and repair. Defaults to all clusters in the topology. Example: \"ks1,ks2/-80\"")
-	_flag.Parse()
+	_flag.Parse(pflag.NewFlagSet("vtgr", pflag.ExitOnError))
 
 	// openTabletDiscovery will open up a connection to topo server
 	// and populate the tablets in memory

go/cmd/vttlstest/vttlstest.go

@@ -21,6 +21,8 @@ import (
 	"fmt"
 	"io"
 
+	"github.com/spf13/pflag"
+
 	"vitess.io/vitess/go/exit"
 	"vitess.io/vitess/go/vt/log"
 	"vitess.io/vitess/go/vt/logutil"
@@ -137,7 +139,7 @@ func main() {
 	_flag.SetUsage(flag.CommandLine, _flag.UsageOptions{
 		Preface: func(w io.Writer) { fmt.Fprint(w, doc) },
 	})
-	_flag.Parse()
+	_flag.Parse(pflag.NewFlagSet("vttlstest", pflag.ExitOnError))
 	args := _flag.Args()
 	if len(args) == 0 {
 		flag.Usage()

go/cmd/zk/zkcmd.go

@@ -33,6 +33,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/spf13/pflag"
 	"github.com/z-division/go-zookeeper/zk"
 	"golang.org/x/term"
 
@@ -140,10 +141,11 @@ func main() {
 	defer exit.Recover()
 	defer logutil.Flush()
 
-	_flag.SetUsage(flag.CommandLine, _flag.UsageOptions{
+	fs := pflag.NewFlagSet("zkcmd", pflag.ExitOnError)
+	_flag.SetUsage(flag.CommandLine, _flag.UsageOptions{ // TODO: hmmm
 		Epilogue: func(w io.Writer) { fmt.Fprint(w, doc) },
 	})
-	_flag.Parse()
+	_flag.Parse(fs)
 	args := _flag.Args()
 	if len(args) == 0 {
 		flag.Usage()

go/cmd/zkctl/zkctl.go

@@ -24,6 +24,8 @@ import (
 	"io"
 	"os"
 
+	"github.com/spf13/pflag"
+
 	"vitess.io/vitess/go/exit"
 	"vitess.io/vitess/go/vt/log"
 	"vitess.io/vitess/go/vt/logutil"
@@ -60,7 +62,7 @@ func main() {
 	defer exit.Recover()
 	defer logutil.Flush()
 
-	_flag.Parse()
+	_flag.Parse(pflag.NewFlagSet("zkctl", pflag.ExitOnError))
 	args := _flag.Args()
 
 	if len(args) == 0 {

go/cmd/zkctld/zkctld.go

@@ -25,6 +25,8 @@ import (
 	"os/signal"
 	"syscall"
 
+	"github.com/spf13/pflag"
+
 	"vitess.io/vitess/go/exit"
 	"vitess.io/vitess/go/vt/log"
 	"vitess.io/vitess/go/vt/logutil"
@@ -45,7 +47,7 @@ func main() {
 	defer exit.Recover()
 	defer logutil.Flush()
 
-	_flag.Parse()
+	_flag.Parse(pflag.NewFlagSet("zkctld", pflag.ExitOnError))
 
 	zkConfig := zkctl.MakeZkConfigFromString(*zkCfg, uint32(*myID))
 	zkd := zkctl.NewZkd(zkConfig)