From 6c23d0df37dad857946975c5efa099e1ebecbe37 Mon Sep 17 00:00:00 2001 From: Zhenchi Date: Thu, 16 Dec 2021 13:27:42 +0800 Subject: [PATCH 1/3] server: add grpc server config for a suitable behavior Signed-off-by: Zhenchi --- config/config.go | 32 +++++++++++++++++++++----------- config/config.toml.example | 17 +++++++++++++++++ server/rpc_server.go | 12 +++++++++++- 3 files changed, 49 insertions(+), 12 deletions(-) diff --git a/config/config.go b/config/config.go index 53141fadd093a..c0f61a914fd66 100644 --- a/config/config.go +++ b/config/config.go @@ -439,12 +439,17 @@ func (s *Security) ClusterSecurity() tikvcfg.Security { // Status is the status section of the config. type Status struct { - StatusHost string `toml:"status-host" json:"status-host"` - MetricsAddr string `toml:"metrics-addr" json:"metrics-addr"` - StatusPort uint `toml:"status-port" json:"status-port"` - MetricsInterval uint `toml:"metrics-interval" json:"metrics-interval"` - ReportStatus bool `toml:"report-status" json:"report-status"` - RecordQPSbyDB bool `toml:"record-db-qps" json:"record-db-qps"` + StatusHost string `toml:"status-host" json:"status-host"` + MetricsAddr string `toml:"metrics-addr" json:"metrics-addr"` + StatusPort uint `toml:"status-port" json:"status-port"` + MetricsInterval uint `toml:"metrics-interval" json:"metrics-interval"` + ReportStatus bool `toml:"report-status" json:"report-status"` + RecordQPSbyDB bool `toml:"record-db-qps" json:"record-db-qps"` + GRPCKeepAliveTime uint `toml:"grpc-keepalive-time" json:"grpc-keepalive-time"` + GRPCKeepAliveTimeout uint `toml:"grpc-keepalive-timeout" json:"grpc-keepalive-timeout"` + GRPCConcurrentStreams uint `toml:"grpc-concurrent-streams" json:"grpc-concurrent-streams"` + GRPCInitialWindowSize int `toml:"grpc-initial-window-size" json:"grpc-initial-window-size"` + GRPCMaxSendMsgSize int `toml:"grpc-max-send-msg-size" json:"grpc-max-send-msg-size"` } // Performance is the performance section of the config. @@ -658,11 +663,16 @@ var defaultConf = Config{ EnableSlowLog: *NewAtomicBool(logutil.DefaultTiDBEnableSlowLog), }, Status: Status{ - ReportStatus: true, - StatusHost: DefStatusHost, - StatusPort: DefStatusPort, - MetricsInterval: 15, - RecordQPSbyDB: false, + ReportStatus: true, + StatusHost: DefStatusHost, + StatusPort: DefStatusPort, + MetricsInterval: 15, + RecordQPSbyDB: false, + GRPCKeepAliveTime: 10, + GRPCKeepAliveTimeout: 3, + GRPCConcurrentStreams: 1024, + GRPCInitialWindowSize: 2 * 1024 * 1024, + GRPCMaxSendMsgSize: 10 * 1024 * 1024, }, Performance: Performance{ MaxMemory: 0, diff --git a/config/config.toml.example b/config/config.toml.example index 0e91f903748f8..a78d93139aa99 100644 --- a/config/config.toml.example +++ b/config/config.toml.example @@ -238,6 +238,23 @@ metrics-interval = 15 # Record statements qps by database name if it is enabled. record-db-qps = false +# After a duration of this time in seconds if the server doesn't see any activity it pings +# the client to see if the transport is still alive. +grpc-keepalive-time = 10 + +# After having pinged for keepalive check, the server waits for a duration of timeout in seconds +# and if no activity is seen even after that the connection is closed. +grpc-keepalive-timeout = 3 + +# The number of max concurrent streams/requests on a client connection. +grpc-concurrent-streams = 1024 + +# Sets window size for stream. The default value is 2MB. +grpc-initial-window-size = 2097152 + +# Set maximum message length in bytes that gRPC can send. `-1` means unlimited. The default value is 10MB. +grpc-max-send-msg-size = 10485760 + [performance] # Max CPUs to use, 0 use number of CPUs in the machine. max-procs = 0 diff --git a/server/rpc_server.go b/server/rpc_server.go index 67965ac381f4d..674047781a6bd 100644 --- a/server/rpc_server.go +++ b/server/rpc_server.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "net" + "time" "github.com/pingcap/kvproto/pkg/coprocessor" "github.com/pingcap/kvproto/pkg/diagnosticspb" @@ -34,6 +35,7 @@ import ( "github.com/pingcap/tidb/util/memory" "go.uber.org/zap" "google.golang.org/grpc" + "google.golang.org/grpc/keepalive" "google.golang.org/grpc/peer" ) @@ -46,7 +48,15 @@ func NewRPCServer(config *config.Config, dom *domain.Domain, sm util.SessionMana } }() - s := grpc.NewServer() + s := grpc.NewServer( + grpc.KeepaliveParams(keepalive.ServerParameters{ + Time: time.Duration(config.Status.GRPCKeepAliveTime) * time.Second, + Timeout: time.Duration(config.Status.GRPCKeepAliveTimeout) * time.Second, + }), + grpc.MaxConcurrentStreams(uint32(config.Status.GRPCConcurrentStreams)), + grpc.InitialWindowSize(int32(config.Status.GRPCInitialWindowSize)), + grpc.MaxSendMsgSize(config.Status.GRPCMaxSendMsgSize), + ) rpcSrv := &rpcServer{ DiagnosticsServer: sysutil.NewDiagnosticsServer(config.Log.File.Filename), dom: dom, From a92c4207e5cc4656e9d71b4d12d7442978810323 Mon Sep 17 00:00:00 2001 From: Zhenchi Date: Thu, 16 Dec 2021 14:16:30 +0800 Subject: [PATCH 2/3] hide configs Signed-off-by: Zhenchi --- config/config.go | 29 ++++++++++++++++++----------- config/config.toml.example | 17 ----------------- 2 files changed, 18 insertions(+), 28 deletions(-) diff --git a/config/config.go b/config/config.go index c0f61a914fd66..2319d953286c7 100644 --- a/config/config.go +++ b/config/config.go @@ -439,17 +439,24 @@ func (s *Security) ClusterSecurity() tikvcfg.Security { // Status is the status section of the config. type Status struct { - StatusHost string `toml:"status-host" json:"status-host"` - MetricsAddr string `toml:"metrics-addr" json:"metrics-addr"` - StatusPort uint `toml:"status-port" json:"status-port"` - MetricsInterval uint `toml:"metrics-interval" json:"metrics-interval"` - ReportStatus bool `toml:"report-status" json:"report-status"` - RecordQPSbyDB bool `toml:"record-db-qps" json:"record-db-qps"` - GRPCKeepAliveTime uint `toml:"grpc-keepalive-time" json:"grpc-keepalive-time"` - GRPCKeepAliveTimeout uint `toml:"grpc-keepalive-timeout" json:"grpc-keepalive-timeout"` - GRPCConcurrentStreams uint `toml:"grpc-concurrent-streams" json:"grpc-concurrent-streams"` - GRPCInitialWindowSize int `toml:"grpc-initial-window-size" json:"grpc-initial-window-size"` - GRPCMaxSendMsgSize int `toml:"grpc-max-send-msg-size" json:"grpc-max-send-msg-size"` + StatusHost string `toml:"status-host" json:"status-host"` + MetricsAddr string `toml:"metrics-addr" json:"metrics-addr"` + StatusPort uint `toml:"status-port" json:"status-port"` + MetricsInterval uint `toml:"metrics-interval" json:"metrics-interval"` + ReportStatus bool `toml:"report-status" json:"report-status"` + RecordQPSbyDB bool `toml:"record-db-qps" json:"record-db-qps"` + // After a duration of this time in seconds if the server doesn't see any activity it pings + // the client to see if the transport is still alive. + GRPCKeepAliveTime uint `toml:"grpc-keepalive-time" json:"grpc-keepalive-time"` + // After having pinged for keepalive check, the server waits for a duration of timeout in seconds + // and if no activity is seen even after that the connection is closed. + GRPCKeepAliveTimeout uint `toml:"grpc-keepalive-timeout" json:"grpc-keepalive-timeout"` + // The number of max concurrent streams/requests on a client connection. + GRPCConcurrentStreams uint `toml:"grpc-concurrent-streams" json:"grpc-concurrent-streams"` + // Sets window size for stream. The default value is 2MB. + GRPCInitialWindowSize int `toml:"grpc-initial-window-size" json:"grpc-initial-window-size"` + // Set maximum message length in bytes that gRPC can send. `-1` means unlimited. The default value is 10MB. + GRPCMaxSendMsgSize int `toml:"grpc-max-send-msg-size" json:"grpc-max-send-msg-size"` } // Performance is the performance section of the config. diff --git a/config/config.toml.example b/config/config.toml.example index a78d93139aa99..0e91f903748f8 100644 --- a/config/config.toml.example +++ b/config/config.toml.example @@ -238,23 +238,6 @@ metrics-interval = 15 # Record statements qps by database name if it is enabled. record-db-qps = false -# After a duration of this time in seconds if the server doesn't see any activity it pings -# the client to see if the transport is still alive. -grpc-keepalive-time = 10 - -# After having pinged for keepalive check, the server waits for a duration of timeout in seconds -# and if no activity is seen even after that the connection is closed. -grpc-keepalive-timeout = 3 - -# The number of max concurrent streams/requests on a client connection. -grpc-concurrent-streams = 1024 - -# Sets window size for stream. The default value is 2MB. -grpc-initial-window-size = 2097152 - -# Set maximum message length in bytes that gRPC can send. `-1` means unlimited. The default value is 10MB. -grpc-max-send-msg-size = 10485760 - [performance] # Max CPUs to use, 0 use number of CPUs in the machine. max-procs = 0 From 774bfebf8c6f9d197de8753e7886a5236fdede30 Mon Sep 17 00:00:00 2001 From: Zhenchi Date: Thu, 16 Dec 2021 14:27:54 +0800 Subject: [PATCH 3/3] add tests Signed-off-by: Zhenchi --- config/config_test.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/config/config_test.go b/config/config_test.go index ab3edc62a0fc4..25c79dc40ebe3 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -261,6 +261,12 @@ deadlock-history-capacity = 123 deadlock-history-collect-retryable = true [top-sql] receiver-address = "127.0.0.1:10100" +[status] +grpc-keepalive-time = 20 +grpc-keepalive-timeout = 10 +grpc-concurrent-streams = 2048 +grpc-initial-window-size = 10240 +grpc-max-send-msg-size = 40960 `) require.NoError(t, err) @@ -318,6 +324,11 @@ receiver-address = "127.0.0.1:10100" require.False(t, conf.Experimental.EnableNewCharset) require.Equal(t, "127.0.0.1:10100", conf.TopSQL.ReceiverAddress) require.True(t, conf.Experimental.AllowsExpressionIndex) + require.Equal(t, uint(20), conf.Status.GRPCKeepAliveTime) + require.Equal(t, uint(10), conf.Status.GRPCKeepAliveTimeout) + require.Equal(t, uint(2048), conf.Status.GRPCConcurrentStreams) + require.Equal(t, 10240, conf.Status.GRPCInitialWindowSize) + require.Equal(t, 40960, conf.Status.GRPCMaxSendMsgSize) err = f.Truncate(0) require.NoError(t, err)