statistics: get right max table id when to init stats (#58280)

.bazelversion

@@ -0,0 +1 @@
+7.4.1

pkg/statistics/handle/bootstrap.go

@@ -51,6 +51,11 @@ const (
 
 var maxTidRecord MaxTidRecord
 
+// GetMaxTidRecordForTest gets the max tid record for test.
+func GetMaxTidRecordForTest() int64 {
+	return maxTidRecord.tid.Load()
+}
+
 // MaxTidRecord is to record the max tid.
 type MaxTidRecord struct {
 	mu  sync.Mutex
@@ -84,7 +89,7 @@ func (*Handle) initStatsMeta4Chunk(cache statstypes.StatsCache, iter *chunk.Iter
 	maxTidRecord.mu.Lock()
 	defer maxTidRecord.mu.Unlock()
 	if maxTidRecord.tid.Load() < maxPhysicalID {
-		maxTidRecord.tid.Store(physicalID)
+		maxTidRecord.tid.Store(maxPhysicalID)
 	}
 }
 
@@ -352,7 +357,7 @@ func (h *Handle) initStatsHistogramsByPaging(is infoschema.InfoSchema, cache sta
 		if req.NumRows() == 0 {
 			break
 		}
-		h.initStatsHistograms4Chunk(is, cache, iter, isFullCache(cache, totalMemory))
+		h.initStatsHistograms4Chunk(is, cache, iter, IsFullCacheFunc(cache, totalMemory))
 	}
 	return nil
 }
@@ -376,7 +381,7 @@ func (h *Handle) initStatsHistogramsConcurrency(is infoschema.InfoSchema, cache
 }
 
 func (*Handle) initStatsTopN4Chunk(cache statstypes.StatsCache, iter *chunk.Iterator4Chunk, totalMemory uint64) {
-	if isFullCache(cache, totalMemory) {
+	if IsFullCacheFunc(cache, totalMemory) {
 		return
 	}
 	affectedIndexes := make(map[*statistics.Index]struct{})
@@ -483,20 +488,20 @@ func (h *Handle) initStatsTopNByPaging(cache statstypes.StatsCache, task initsta
 }
 
 func (h *Handle) initStatsTopNConcurrency(cache statstypes.StatsCache, totalMemory uint64) error {
-	if isFullCache(cache, totalMemory) {
+	if IsFullCacheFunc(cache, totalMemory) {
 		return nil
 	}
 	var maxTid = maxTidRecord.tid.Load()
 	tid := int64(0)
 	ls := initstats.NewRangeWorker("TopN", func(task initstats.Task) error {
-		if isFullCache(cache, totalMemory) {
+		if IsFullCacheFunc(cache, totalMemory) {
 			return nil
 		}
 		return h.initStatsTopNByPaging(cache, task, totalMemory)
 	}, uint64(maxTid), uint64(initStatsStep), initStatsPercentageInterval)
 	ls.LoadStats()
 	for tid <= maxTid {
-		if isFullCache(cache, totalMemory) {
+		if IsFullCacheFunc(cache, totalMemory) {
 			break
 		}
 		ls.SendTask(initstats.Task{
@@ -657,7 +662,7 @@ func initStatsBucketsSQLGen(isPaging bool) string {
 }
 
 func (h *Handle) initStatsBuckets(cache statstypes.StatsCache, totalMemory uint64) error {
-	if isFullCache(cache, totalMemory) {
+	if IsFullCacheFunc(cache, totalMemory) {
 		return nil
 	}
 	if config.GetGlobalConfig().Performance.ConcurrentlyInitStats {
@@ -728,13 +733,13 @@ func (h *Handle) initStatsBucketsByPaging(cache statstypes.StatsCache, task init
 }
 
 func (h *Handle) initStatsBucketsConcurrency(cache statstypes.StatsCache, totalMemory uint64) error {
-	if isFullCache(cache, totalMemory) {
+	if IsFullCacheFunc(cache, totalMemory) {
 		return nil
 	}
 	var maxTid = maxTidRecord.tid.Load()
 	tid := int64(0)
 	ls := initstats.NewRangeWorker("bucket", func(task initstats.Task) error {
-		if isFullCache(cache, totalMemory) {
+		if IsFullCacheFunc(cache, totalMemory) {
 			return nil
 		}
 		return h.initStatsBucketsByPaging(cache, task)
@@ -746,7 +751,7 @@ func (h *Handle) initStatsBucketsConcurrency(cache statstypes.StatsCache, totalM
 			EndTid:   tid + initStatsStep,
 		})
 		tid += initStatsStep
-		if isFullCache(cache, totalMemory) {
+		if IsFullCacheFunc(cache, totalMemory) {
 			break
 		}
 	}
@@ -851,6 +856,9 @@ func (h *Handle) InitStats(ctx context.Context, is infoschema.InfoSchema) (err e
 	return nil
 }
 
+// IsFullCacheFunc is whether the cache is full or not. but we can only change it when to test
+var IsFullCacheFunc = isFullCache
+
 func isFullCache(cache statstypes.StatsCache, total uint64) bool {
 	memQuota := variable.StatsCacheMemQuota.Load()
 	return (uint64(cache.MemConsumed()) >= total/4) || (cache.MemConsumed() >= memQuota && memQuota != 0)

pkg/statistics/handle/handletest/initstats/BUILD.bazel

@@ -0,0 +1,16 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
+
+go_test(
+    name = "initstats_test",
+    timeout = "short",
+    srcs = ["load_stats_test.go"],
+    flaky = True,
+    deps = [
+        "//pkg/config",
+        "//pkg/parser/model",
+        "//pkg/statistics/handle",
+        "//pkg/statistics/handle/types",
+        "//pkg/testkit",
+        "@com_github_stretchr_testify//require",
+    ],
+)

pkg/statistics/handle/handletest/initstats/load_stats_test.go

@@ -0,0 +1,104 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package initstats
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/pingcap/tidb/pkg/config"
+	"github.com/pingcap/tidb/pkg/parser/model"
+	"github.com/pingcap/tidb/pkg/statistics/handle"
+	"github.com/pingcap/tidb/pkg/statistics/handle/types"
+	"github.com/pingcap/tidb/pkg/testkit"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConcurrentlyInitStatsWithMemoryLimit(t *testing.T) {
+	restore := config.RestoreFunc()
+	defer restore()
+	config.UpdateGlobal(func(conf *config.Config) {
+		conf.Performance.LiteInitStats = false
+		conf.Performance.ConcurrentlyInitStats = true
+	})
+	handle.IsFullCacheFunc = func(cache types.StatsCache, total uint64) bool {
+		return true
+	}
+	testConcurrentlyInitStats(t)
+}
+
+func TestConcurrentlyInitStatsWithoutMemoryLimit(t *testing.T) {
+	restore := config.RestoreFunc()
+	defer restore()
+	config.UpdateGlobal(func(conf *config.Config) {
+		conf.Performance.LiteInitStats = false
+		conf.Performance.ConcurrentlyInitStats = true
+	})
+	handle.IsFullCacheFunc = func(cache types.StatsCache, total uint64) bool {
+		return false
+	}
+	testConcurrentlyInitStats(t)
+}
+
+func testConcurrentlyInitStats(t *testing.T) {
+	store, dom := testkit.CreateMockStoreAndDomain(t)
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec("use test")
+	tk.MustExec("set global tidb_analyze_column_options='ALL'")
+	tk.MustExec("create table t1 (a int, b int, c int, primary key(c))")
+	tk.MustExec("insert into t1 values (1,1,1),(2,2,2),(3,3,3),(4,4,4),(5,5,5),(6,7,8)")
+	tk.MustExec("analyze table t1")
+	for i := 2; i < 10; i++ {
+		tk.MustExec(fmt.Sprintf("create table t%v (a int, b int, c int, primary key(c))", i))
+		tk.MustExec(fmt.Sprintf("insert into t%v select * from t1", i))
+		tk.MustExec(fmt.Sprintf("analyze table t%v all columns", i))
+	}
+	h := dom.StatsHandle()
+	is := dom.InfoSchema()
+	h.Clear()
+	require.Equal(t, h.MemConsumed(), int64(0))
+	require.NoError(t, h.InitStats(context.Background(), is))
+	for i := 1; i < 10; i++ {
+		tbl, err := is.TableByName(context.Background(), model.NewCIStr("test"), model.NewCIStr(fmt.Sprintf("t%v", i)))
+		require.NoError(t, err)
+		stats, ok := h.StatsCache.Get(tbl.Meta().ID)
+		require.True(t, ok)
+		for _, col := range stats.GetColSlice() {
+			require.True(t, col.IsAllEvicted())
+			require.False(t, col.IsFullLoad())
+		}
+	}
+	for i := 1; i < 10; i++ {
+		tk.MustQuery(fmt.Sprintf("explain select * from t%v where a = 1", i)).CheckNotContain("pseudo")
+	}
+	for i := 1; i < 10; i++ {
+		tk.MustQuery(fmt.Sprintf("explain select * from t%v where b = 1", i)).CheckNotContain("pseudo")
+	}
+	for i := 1; i < 10; i++ {
+		tk.MustQuery(fmt.Sprintf("explain select * from t%v where c >= 1", i)).CheckNotContain("pseudo")
+	}
+	for i := 1; i < 10; i++ {
+		tbl, err := is.TableByName(context.Background(), model.NewCIStr("test"), model.NewCIStr(fmt.Sprintf("t%v", i)))
+		require.NoError(t, err)
+		stats, ok := h.StatsCache.Get(tbl.Meta().ID)
+		require.True(t, ok)
+		for _, col := range stats.GetColSlice() {
+			require.True(t, col.IsFullLoad())
+			require.False(t, col.IsAllEvicted())
+		}
+	}
+	require.Equal(t, int64(126), handle.GetMaxTidRecordForTest())
+}