Throttler: set timeouts on gRPC communication and on topo communicati…

…on (vitessio#14165)

Signed-off-by: Shlomi Noach <[email protected]>

go/vt/vttablet/tabletserver/throttle/throttler.go

@@ -714,6 +714,10 @@ func (throttler *Throttler) Operate(ctx context.Context) {
 
 func (throttler *Throttler) generateTabletHTTPProbeFunction(ctx context.Context, tmClient tmclient.TabletManagerClient, clusterName string, probe *mysql.Probe) (probeFunc func() *mysql.MySQLThrottleMetric) {
 	return func() *mysql.MySQLThrottleMetric {
+		// Some reasonable timeout, to ensure we release connections even if they're hanging (otherwise grpc-go keeps polling those connections forever)
+		ctx, cancel := context.WithTimeout(ctx, 4*mysqlCollectInterval)
+		defer cancel()
+
 		// Hit a tablet's `check-self` via HTTP, and convert its CheckResult JSON output into a MySQLThrottleMetric
 		mySQLThrottleMetric := mysql.NewMySQLThrottleMetric()
 		mySQLThrottleMetric.ClusterName = clusterName
@@ -872,6 +876,9 @@ func (throttler *Throttler) refreshMySQLInventory(ctx context.Context) error {
 			}
 			// The primary tablet is also in charge of collecting the shard's metrics
 			err := func() error {
+				ctx, cancel := context.WithTimeout(ctx, mysqlRefreshInterval)
+				defer cancel()
+
 				tabletAliases, err := throttler.ts.FindAllTabletAliasesInShard(ctx, throttler.keyspace, throttler.shard)
 				if err != nil {
 					return err