util: add dynamic goroutine pool

build/linter/allrevive/analyzer.go

@@ -65,6 +65,7 @@ var defaultRules = []lint.Rule{
 	&rule.TimeNamingRule{},
 	//&rule.ContextKeysType{},
 	&rule.ContextAsArgumentRule{},
+	&rule.ReceiverNamingRule{},
 }
 
 var allRules = append([]lint.Rule{
@@ -118,6 +119,7 @@ var allRules = append([]lint.Rule{
 	&rule.TimeEqualRule{},
 	//&rule.BannedCharsRule{},
 	&rule.OptimizeOperandsOrderRule{},
+	&rule.ReceiverNamingRule{},
 }, defaultRules...)
 
 func run(pass *analysis.Pass) (any, error) {

build/nogo_config.json

@@ -161,6 +161,7 @@
       "util/etcd/": "util/etcd/ code",
       "util/expensivequery/": "util/expensivequery/ code",
       "util/filter/": "util/filter/ code",
+      "util/gpool/": "util/gpool/ code",
       "util/importer/": "util/importer/ code",
       "util/keydecoder/": "util/keydecoder/ code",
       "util/kvcache/": "util/kvcache/ code",
@@ -377,6 +378,7 @@
       "dumpling/export": "dumpling/export code",
       "lock/": "lock file",
       "errno/": "errno code",
+      "resourcemanage/": "resourcemanage code",
       "structure/": "structure code",
       "telemetry/": "telemetry code",
       "planner/memo": "planner code",

resourcemanager/BUILD.bazel

@@ -9,6 +9,7 @@ go_library(
     importpath = "github.com/pingcap/tidb/resourcemanager",
     visibility = ["//visibility:public"],
     deps = [
+        "//resourcemanager/limiter",
         "//resourcemanager/scheduler",
         "//resourcemanager/util",
         "//util",

resourcemanager/limiter/BUILD.bazel

@@ -0,0 +1,15 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "limiter",
+    srcs = [
+        "bbr_limiter.go",
+        "limiter.go",
+    ],
+    importpath = "github.com/pingcap/tidb/resourcemanager/limiter",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//resourcemanager/util",
+        "//util/cpu",
+    ],
+)

resourcemanager/limiter/bbr_limiter.go

@@ -0,0 +1,66 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package limiter
+
+import (
+	"time"
+
+	"github.com/pingcap/tidb/resourcemanager/util"
+	"github.com/pingcap/tidb/util/cpu"
+)
+
+// BBRLimiter is a limiter based on cpu usage.
+type BBRLimiter struct {
+	cpuThreshold int64
+}
+
+// NewBBRLimiter is to create a bbr limiter.
+func NewBBRLimiter(cpuThreshold int64) Limiter {
+	return &BBRLimiter{
+		cpuThreshold: cpuThreshold,
+	}
+}
+
+// Limit is to limit the goroutine pool.
+func (b *BBRLimiter) Limit(_ util.Component, p util.GorotinuePool) bool {
+	usage := cpu.GetCPUUsage() * 100
+	if usage < float64(b.cpuThreshold) {
+		// current cpu payload below the threshold
+		prevDropTime := p.LastTunerTs()
+		if prevDropTime.IsZero() {
+			// haven't start drop,
+			// accept current request
+			return false
+		}
+		if time.Since(prevDropTime) <= time.Second {
+			// just start drop one second ago,
+			// check current inflight count
+			inFlight := p.InFlight()
+			return inFlight > 1 && inFlight > p.MaxInFlight()
+		}
+		return false
+	}
+	// current cpu payload exceeds the threshold
+	inFlight := p.InFlight()
+	drop := inFlight > 1 && inFlight > p.MaxInFlight()
+	if drop {
+		prevDropTime := p.LastTunerTs()
+		if prevDropTime.IsZero() {
+			// already started drop, return directly
+			return drop
+		}
+	}
+	return drop
+}

resourcemanager/limiter/limiter.go

@@ -0,0 +1,31 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package limiter
+
+import (
+	"github.com/pingcap/tidb/resourcemanager/util"
+)
+
+// Limiter is a limiter interface
+type Limiter interface {
+	Limit(component util.Component, p util.GorotinuePool) bool
+}
+
+// NewLimiter is to create a default collection of limiter.
+func NewLimiter() []Limiter {
+	return []Limiter{
+		NewBBRLimiter(80),
+	}
+}

resourcemanager/pooltask/BUILD.bazel

@@ -1,8 +1,27 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "pooltask",
-    srcs = ["task.go"],
+    srcs = [
+        "stat.go",
+        "task.go",
+        "task_manager.go",
+        "task_manager_iterator.go",
+        "task_manager_scheduler.go",
+    ],
     importpath = "github.com/pingcap/tidb/resourcemanager/pooltask",
     visibility = ["//visibility:public"],
+    deps = [
+        "//util",
+        "//util/mathutil",
+        "//util/window",
+        "@org_golang_x_sys//cpu",
+        "@org_uber_go_atomic//:atomic",
+    ],
+)
+
+go_test(
+    name = "pooltask_test",
+    srcs = ["task_test.go"],
+    embed = [":pooltask"],
 )