[cherry-pick-7.1] *: default resource group (pingcap#526)

* default resource group (pingcap#245) * default resource group Signed-off-by: disksing <[email protected]> * enable resource control Signed-off-by: disksing <[email protected]> * add comments Signed-off-by: disksing <[email protected]> * move DefaultResourceGroup to config Signed-off-by: disksing <[email protected]> --------- Signed-off-by: disksing <[email protected]> Co-authored-by: zzm <[email protected]> * Resource Control: fix segmentation violation when keyspace not set (pingcap#251) * fix segmentation violation when setting up ru Signed-off-by: David <[email protected]> * only setup resource control when cluster id not empty Signed-off-by: David <[email protected]> --------- Signed-off-by: David <[email protected]> * skip ur (pingcap#279) Signed-off-by: ystaticy <[email protected]> * disable resource group sql interface (pingcap#309) Signed-off-by: disksing <[email protected]> * *: update pd client and increase maxWaitDuration (pingcap#321) Signed-off-by: disksing <[email protected]> * update pd client and ru config (pingcap#325) Signed-off-by: disksing <[email protected]> * update max wait duration (pingcap#351) Signed-off-by: disksing <[email protected]> * update the pd client and client go (pingcap#419) Signed-off-by: nolouch <[email protected]> * update main Signed-off-by: disksing <[email protected]> --------- Signed-off-by: disksing <[email protected]> Signed-off-by: David <[email protected]> Signed-off-by: ystaticy <[email protected]> Signed-off-by: nolouch <[email protected]> Co-authored-by: zzm <[email protected]> Co-authored-by: David <[email protected]> Co-authored-by: ystaticy <[email protected]> Co-authored-by: ShuNing <[email protected]>
rleungx · Jul 28, 2023 · 2877fb2 · 2877fb2
1 parent acba176
commit 2877fb2
Show file tree

Hide file tree

Showing 9 changed files with 42 additions and 9 deletions.
diff --git a/config/config.go b/config/config.go
@@ -270,15 +270,16 @@ type Config struct {
 	// InitializeSQLFile is a file that will be executed after first bootstrap only.
 	// It can be used to set GLOBAL system variable values
 	InitializeSQLFile string `toml:"initialize-sql-file" json:"initialize-sql-file"`
-  
+
 	// Serverless related configs.
 	StandByMode          bool `toml:"standby" json:"standby"`
 	KeyspaceActivateMode bool `toml:"keyspace-activate" json:"keyspace-activate"`
 	MaxIdleSeconds       uint `toml:"max-idle-seconds" json:"max-idle-seconds"`
 	// ActivationTimeout specifies the maximum allowed time for tidb to activate from standby mode.
-	ActivationTimeout uint `toml:"activation-timeout" json:"activation-timeout"`
-  
+	ActivationTimeout          uint `toml:"activation-timeout" json:"activation-timeout"`
+	EnableRULimit              bool `toml:"enable-ru-limit" json:"enable-ru-limit"`
 	EnableAlterUserPessimistic bool `toml:"enable-alter-user-pessimistic" json:"enable-alter-user-pessimistic"`
+
 	// The following items are deprecated. We need to keep them here temporarily
 	// to support the upgrade process. They can be removed in future.
 
@@ -1069,6 +1070,7 @@ var defaultConf = Config{
 	EnableSafePointV2:                    false,
 	GCV1BlackList:                        []uint32{},
 	ResolveLocksByKeyspace:               true,
+	EnableRULimit:                        false,
 	EnableAlterUserPessimistic:           false,
 	EnableGlobalKill:                     true,
 	TrxSummary:                           DefaultTrxSummary(),

diff --git a/config/serverless.go b/config/serverless.go
@@ -33,3 +33,6 @@ func defaultBootstrapControl() BootstrapControl {
 		SkipPushdownBlacklist:   false,
 	}
 }
+
+// DefaultResourceGroup is the default resource group name for all txns and snapshots.
+var DefaultResourceGroup string
diff --git a/executor/infoschema_reader.go b/executor/infoschema_reader.go
@@ -184,7 +184,7 @@ func (e *memtableRetriever) retrieve(ctx context.Context, sctx sessionctx.Contex
 		case infoschema.ClusterTableMemoryUsageOpsHistory:
 			err = e.setDataForClusterMemoryUsageOpsHistory(sctx)
 		case infoschema.TableResourceGroups:
-			err = e.setDataFromResourceGroups()
+			// err = e.setDataFromResourceGroups() // Serverless: hides resource groups from users.
 		}
 		if err != nil {
 			return nil, err

diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go
@@ -2422,6 +2422,10 @@ var defaultSysVars = []*SysVar{
 		},
 	},
 	{Scope: ScopeGlobal, Name: TiDBEnableResourceControl, Value: BoolToOnOff(DefTiDBEnableResourceControl), Type: TypeBool, SetGlobal: func(ctx context.Context, vars *SessionVars, s string) error {
+		if TiDBOptOn(s) {
+			logutil.BgLogger().Info("refuse to enable resource control")
+			return errors.New("serverless clusters do not support resource control configurations")
+		}
 		if TiDBOptOn(s) != EnableResourceControl.Load() {
 			EnableResourceControl.Store(TiDBOptOn(s))
 			(*SetGlobalResourceControl.Load())(TiDBOptOn(s))

diff --git a/store/copr/coprocessor.go b/store/copr/coprocessor.go
@@ -33,6 +33,7 @@ import (
 	"github.com/pingcap/kvproto/pkg/errorpb"
 	"github.com/pingcap/kvproto/pkg/kvrpcpb"
 	"github.com/pingcap/kvproto/pkg/metapb"
+	"github.com/pingcap/tidb/config"
 	"github.com/pingcap/tidb/domain/infosync"
 	"github.com/pingcap/tidb/errno"
 	"github.com/pingcap/tidb/kv"
@@ -77,6 +78,7 @@ type CopClient struct {
 
 // Send builds the request and gets the coprocessor iterator response.
 func (c *CopClient) Send(ctx context.Context, req *kv.Request, variables interface{}, option *kv.ClientSendOption) kv.Response {
+	req.ResourceGroupName = config.DefaultResourceGroup
 	vars, ok := variables.(*tikv.Variables)
 	if !ok {
 		return copErrorResponse{errors.Errorf("unsupported variables:%+v", variables)}

diff --git a/store/driver/tikv_driver.go b/store/driver/tikv_driver.go
@@ -107,7 +107,14 @@ func TrySetupGlobalResourceController(ctx context.Context, serverID uint64, s kv
 		return errors.New("cannot setup up resource controller, should use tikv storage")
 	}
 
-	control, err := rmclient.NewResourceGroupController(ctx, serverID, store.GetPDClient(), nil, rmclient.WithMaxWaitDuration(time.Second*30))
+	ruConfig := rmclient.DefaultRequestUnitConfig()
+
+	opts := []rmclient.ResourceControlCreateOption{
+		rmclient.EnableSingleGroupByKeyspace(),
+		rmclient.WithMaxWaitDuration(time.Hour),
+	}
+
+	control, err := rmclient.NewResourceGroupController(ctx, serverID, store.GetPDClient(), &ruConfig, opts...)
 	if err != nil {
 		return err
 	}
@@ -392,13 +399,17 @@ func (s *tikvStore) Begin(opts ...tikv.TxnOption) (kv.Transaction, error) {
 	if err != nil {
 		return nil, derr.ToTiDBErr(err)
 	}
-	return txn_driver.NewTiKVTxn(txn), err
+	txn.SetResourceGroupName(tidb_config.DefaultResourceGroup)
+	tx := txn_driver.NewTiKVTxn(txn)
+	return tx, nil
 }
 
 // GetSnapshot gets a snapshot that is able to read any data which data is <= ver.
 // if ver is MaxVersion or > current max committed version, we will use current version for this snapshot.
 func (s *tikvStore) GetSnapshot(ver kv.Version) kv.Snapshot {
-	return txn_driver.NewSnapshot(s.KVStore.GetSnapshot(ver.Ver))
+	snap := s.KVStore.GetSnapshot(ver.Ver)
+	snap.SetResourceGroupName(tidb_config.DefaultResourceGroup)
+	return txn_driver.NewSnapshot(snap)
 }
 
 // CurrentVersion returns current max committed version with the given txnScope (local or global).

diff --git a/store/driver/txn/snapshot.go b/store/driver/txn/snapshot.go
@@ -137,7 +137,7 @@ func (s *tikvSnapshot) SetOption(opt int, val interface{}) {
 			s.KVSnapshot.SetScanBatchSize(size)
 		}
 	case kv.ResourceGroupName:
-		s.KVSnapshot.SetResourceGroupName(val.(string))
+		//s.KVSnapshot.SetResourceGroupName(val.(string))
 	case kv.LoadBasedReplicaReadThreshold:
 		s.KVSnapshot.SetLoadBasedReplicaReadThreshold(val.(time.Duration))
 	}

diff --git a/store/driver/txn/txn_driver.go b/store/driver/txn/txn_driver.go
@@ -275,7 +275,7 @@ func (txn *tikvTxn) SetOption(opt int, val interface{}) {
 	case kv.TxnSource:
 		txn.KVTxn.SetTxnSource(val.(uint64))
 	case kv.ResourceGroupName:
-		txn.KVTxn.SetResourceGroupName(val.(string))
+		//txn.KVTxn.SetResourceGroupName(val.(string))
 	case kv.LoadBasedReplicaReadThreshold:
 		txn.KVTxn.GetSnapshot().SetLoadBasedReplicaReadThreshold(val.(time.Duration))
 	}

diff --git a/tidb-server/main.go b/tidb-server/main.go
@@ -265,8 +265,19 @@ func main() {
 	err := registerStores()
 	mainErrHandler(err)
 
+	var keyspaceID uint32
+	if keyspaceMeta != nil {
+		keyspaceID = keyspaceMeta.GetId()
+		if config.GetGlobalConfig().EnableRULimit {
+			log.Info("setting up serverless resource control", zap.Uint32("keyspaceID", keyspaceID))
+			config.DefaultResourceGroup = strconv.FormatUint(uint64(keyspaceID), 10)
+			tikv.EnableResourceControl()
+		}
+	}
+
 	err = metricsutil.RegisterMetricsWithKeyspaceMeta(keyspaceMeta)
 	mainErrHandler(err)
+
 	if variable.EnableTmpStorageOnOOM.Load() {
 		config.GetGlobalConfig().UpdateTempStoragePath()
 		err = disk.InitializeTempDir()