From ccebde534a8b045ef19d5f20172e55b7da0fa47f Mon Sep 17 00:00:00 2001
From: Shenghui Wu <793703860@qq.com>
Date: Mon, 28 Nov 2022 20:46:00 +0800
Subject: [PATCH] util: set memory limit tuner to valid value after trigger
 tuning (#39343)

ref pingcap/tidb#37816
---
 util/gctuner/memory_limit_tuner.go      | 17 ++++++++++++-----
 util/gctuner/memory_limit_tuner_test.go | 10 ++++------
 2 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/util/gctuner/memory_limit_tuner.go b/util/gctuner/memory_limit_tuner.go
index 9dba1bd0bd546..1679e012579d0 100644
--- a/util/gctuner/memory_limit_tuner.go
+++ b/util/gctuner/memory_limit_tuner.go
@@ -38,6 +38,9 @@ type memoryLimitTuner struct {
 	nextGCTriggeredByMemoryLimit atomicutil.Bool
 }
 
+// fallbackPercentage indicates the fallback memory limit percentage when turning.
+const fallbackPercentage float64 = 1.1
+
 // tuning check the memory nextGC and judge whether this GC is trigger by memory limit.
 // Go runtime ensure that it will be called serially.
 func (t *memoryLimitTuner) tuning() {
@@ -61,7 +64,7 @@ func (t *memoryLimitTuner) tuning() {
 			go func() {
 				memory.MemoryLimitGCLast.Store(time.Now())
 				memory.MemoryLimitGCTotal.Add(1)
-				debug.SetMemoryLimit(math.MaxInt64)
+				debug.SetMemoryLimit(t.calcMemoryLimit(fallbackPercentage))
 				resetInterval := 1 * time.Minute // Wait 1 minute and set back, to avoid frequent GC
 				failpoint.Inject("testMemoryLimitTuner", func(val failpoint.Value) {
 					if val, ok := val.(bool); val && ok {
@@ -69,7 +72,7 @@ func (t *memoryLimitTuner) tuning() {
 					}
 				})
 				time.Sleep(resetInterval)
-				debug.SetMemoryLimit(t.calcMemoryLimit())
+				debug.SetMemoryLimit(t.calcMemoryLimit(t.GetPercentage()))
 				for !t.waitingReset.CompareAndSwap(true, false) {
 					continue
 				}
@@ -106,23 +109,27 @@ func (t *memoryLimitTuner) GetPercentage() float64 {
 // UpdateMemoryLimit updates the memory limit.
 // This function should be called when `tidb_server_memory_limit` or `tidb_server_memory_limit_gc_trigger` is modified.
 func (t *memoryLimitTuner) UpdateMemoryLimit() {
-	var memoryLimit = t.calcMemoryLimit()
+	var memoryLimit = t.calcMemoryLimit(t.GetPercentage())
 	if memoryLimit == math.MaxInt64 {
 		t.isTuning.Store(false)
+		memoryLimit = initGOMemoryLimitValue
 	} else {
 		t.isTuning.Store(true)
 	}
 	debug.SetMemoryLimit(memoryLimit)
 }
 
-func (t *memoryLimitTuner) calcMemoryLimit() int64 {
-	memoryLimit := int64(float64(memory.ServerMemoryLimit.Load()) * t.percentage.Load()) // `tidb_server_memory_limit` * `tidb_server_memory_limit_gc_trigger`
+func (*memoryLimitTuner) calcMemoryLimit(percentage float64) int64 {
+	memoryLimit := int64(float64(memory.ServerMemoryLimit.Load()) * percentage) // `tidb_server_memory_limit` * `tidb_server_memory_limit_gc_trigger`
 	if memoryLimit == 0 {
 		memoryLimit = math.MaxInt64
 	}
 	return memoryLimit
 }
 
+var initGOMemoryLimitValue int64
+
 func init() {
+	initGOMemoryLimitValue = debug.SetMemoryLimit(-1)
 	GlobalMemoryLimitTuner.Start()
 }
diff --git a/util/gctuner/memory_limit_tuner_test.go b/util/gctuner/memory_limit_tuner_test.go
index 47d1d8409d8b5..c6f63215c01dd 100644
--- a/util/gctuner/memory_limit_tuner_test.go
+++ b/util/gctuner/memory_limit_tuner_test.go
@@ -15,7 +15,6 @@
 package gctuner
 
 import (
-	"math"
 	"runtime"
 	"runtime/debug"
 	"testing"
@@ -76,10 +75,9 @@ func TestGlobalMemoryTuner(t *testing.T) {
 	checkNextGCEqualMemoryLimit := func() {
 		runtime.ReadMemStats(r)
 		nextGC := r.NextGC
-		memoryLimit := GlobalMemoryLimitTuner.calcMemoryLimit()
-		// In golang source, nextGC = memoryLimit - three parts memory. So check 90%~100% here.
+		memoryLimit := GlobalMemoryLimitTuner.calcMemoryLimit(GlobalMemoryLimitTuner.GetPercentage())
+		// In golang source, nextGC = memoryLimit - three parts memory.
 		require.True(t, nextGC < uint64(memoryLimit))
-		require.True(t, nextGC > uint64(memoryLimit)/10*9)
 	}
 
 	memory600mb := allocator.alloc(600 << 20)
@@ -91,7 +89,7 @@ func TestGlobalMemoryTuner(t *testing.T) {
 	require.True(t, gcNum < getNowGCNum())
 	// Test waiting for reset
 	time.Sleep(500 * time.Millisecond)
-	require.Equal(t, int64(math.MaxInt64), debug.SetMemoryLimit(-1))
+	require.Equal(t, GlobalMemoryLimitTuner.calcMemoryLimit(fallbackPercentage), debug.SetMemoryLimit(-1))
 	gcNum = getNowGCNum()
 	memory100mb := allocator.alloc(100 << 20)
 	time.Sleep(100 * time.Millisecond)
@@ -102,7 +100,7 @@ func TestGlobalMemoryTuner(t *testing.T) {
 	runtime.GC()
 	// Trigger GC in 80% again
 	time.Sleep(500 * time.Millisecond)
-	require.Equal(t, GlobalMemoryLimitTuner.calcMemoryLimit(), debug.SetMemoryLimit(-1))
+	require.Equal(t, GlobalMemoryLimitTuner.calcMemoryLimit(GlobalMemoryLimitTuner.GetPercentage()), debug.SetMemoryLimit(-1))
 	time.Sleep(100 * time.Millisecond)
 	gcNum = getNowGCNum()
 	checkNextGCEqualMemoryLimit()