diff --git a/.bazelversion b/.bazelversion new file mode 100644 index 0000000000000..815da58b7a9ed --- /dev/null +++ b/.bazelversion @@ -0,0 +1 @@ +7.4.1 diff --git a/pkg/statistics/handle/bootstrap.go b/pkg/statistics/handle/bootstrap.go index 22837aa7dec97..0cc355385565c 100644 --- a/pkg/statistics/handle/bootstrap.go +++ b/pkg/statistics/handle/bootstrap.go @@ -51,6 +51,11 @@ const ( var maxTidRecord MaxTidRecord +// GetMaxTidRecordForTest gets the max tid record for test. +func GetMaxTidRecordForTest() int64 { + return maxTidRecord.tid.Load() +} + // MaxTidRecord is to record the max tid. type MaxTidRecord struct { mu sync.Mutex @@ -84,7 +89,7 @@ func (*Handle) initStatsMeta4Chunk(cache statstypes.StatsCache, iter *chunk.Iter maxTidRecord.mu.Lock() defer maxTidRecord.mu.Unlock() if maxTidRecord.tid.Load() < maxPhysicalID { - maxTidRecord.tid.Store(physicalID) + maxTidRecord.tid.Store(maxPhysicalID) } } @@ -352,7 +357,7 @@ func (h *Handle) initStatsHistogramsByPaging(is infoschema.InfoSchema, cache sta if req.NumRows() == 0 { break } - h.initStatsHistograms4Chunk(is, cache, iter, isFullCache(cache, totalMemory)) + h.initStatsHistograms4Chunk(is, cache, iter, IsFullCacheFunc(cache, totalMemory)) } return nil } @@ -376,7 +381,7 @@ func (h *Handle) initStatsHistogramsConcurrency(is infoschema.InfoSchema, cache } func (*Handle) initStatsTopN4Chunk(cache statstypes.StatsCache, iter *chunk.Iterator4Chunk, totalMemory uint64) { - if isFullCache(cache, totalMemory) { + if IsFullCacheFunc(cache, totalMemory) { return } affectedIndexes := make(map[*statistics.Index]struct{}) @@ -483,20 +488,20 @@ func (h *Handle) initStatsTopNByPaging(cache statstypes.StatsCache, task initsta } func (h *Handle) initStatsTopNConcurrency(cache statstypes.StatsCache, totalMemory uint64) error { - if isFullCache(cache, totalMemory) { + if IsFullCacheFunc(cache, totalMemory) { return nil } var maxTid = maxTidRecord.tid.Load() tid := int64(0) ls := initstats.NewRangeWorker("TopN", func(task initstats.Task) error { - if isFullCache(cache, totalMemory) { + if IsFullCacheFunc(cache, totalMemory) { return nil } return h.initStatsTopNByPaging(cache, task, totalMemory) }, uint64(maxTid), uint64(initStatsStep), initStatsPercentageInterval) ls.LoadStats() for tid <= maxTid { - if isFullCache(cache, totalMemory) { + if IsFullCacheFunc(cache, totalMemory) { break } ls.SendTask(initstats.Task{ @@ -657,7 +662,7 @@ func initStatsBucketsSQLGen(isPaging bool) string { } func (h *Handle) initStatsBuckets(cache statstypes.StatsCache, totalMemory uint64) error { - if isFullCache(cache, totalMemory) { + if IsFullCacheFunc(cache, totalMemory) { return nil } if config.GetGlobalConfig().Performance.ConcurrentlyInitStats { @@ -728,13 +733,13 @@ func (h *Handle) initStatsBucketsByPaging(cache statstypes.StatsCache, task init } func (h *Handle) initStatsBucketsConcurrency(cache statstypes.StatsCache, totalMemory uint64) error { - if isFullCache(cache, totalMemory) { + if IsFullCacheFunc(cache, totalMemory) { return nil } var maxTid = maxTidRecord.tid.Load() tid := int64(0) ls := initstats.NewRangeWorker("bucket", func(task initstats.Task) error { - if isFullCache(cache, totalMemory) { + if IsFullCacheFunc(cache, totalMemory) { return nil } return h.initStatsBucketsByPaging(cache, task) @@ -746,7 +751,7 @@ func (h *Handle) initStatsBucketsConcurrency(cache statstypes.StatsCache, totalM EndTid: tid + initStatsStep, }) tid += initStatsStep - if isFullCache(cache, totalMemory) { + if IsFullCacheFunc(cache, totalMemory) { break } } @@ -851,6 +856,9 @@ func (h *Handle) InitStats(ctx context.Context, is infoschema.InfoSchema) (err e return nil } +// IsFullCacheFunc is whether the cache is full or not. but we can only change it when to test +var IsFullCacheFunc = isFullCache + func isFullCache(cache statstypes.StatsCache, total uint64) bool { memQuota := variable.StatsCacheMemQuota.Load() return (uint64(cache.MemConsumed()) >= total/4) || (cache.MemConsumed() >= memQuota && memQuota != 0) diff --git a/pkg/statistics/handle/handletest/initstats/BUILD.bazel b/pkg/statistics/handle/handletest/initstats/BUILD.bazel new file mode 100644 index 0000000000000..9c614a18cbbe8 --- /dev/null +++ b/pkg/statistics/handle/handletest/initstats/BUILD.bazel @@ -0,0 +1,16 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_test") + +go_test( + name = "initstats_test", + timeout = "short", + srcs = ["load_stats_test.go"], + flaky = True, + deps = [ + "//pkg/config", + "//pkg/parser/model", + "//pkg/statistics/handle", + "//pkg/statistics/handle/types", + "//pkg/testkit", + "@com_github_stretchr_testify//require", + ], +) diff --git a/pkg/statistics/handle/handletest/initstats/load_stats_test.go b/pkg/statistics/handle/handletest/initstats/load_stats_test.go new file mode 100644 index 0000000000000..37d0c45a9d030 --- /dev/null +++ b/pkg/statistics/handle/handletest/initstats/load_stats_test.go @@ -0,0 +1,104 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package initstats + +import ( + "context" + "fmt" + "testing" + + "github.com/pingcap/tidb/pkg/config" + "github.com/pingcap/tidb/pkg/parser/model" + "github.com/pingcap/tidb/pkg/statistics/handle" + "github.com/pingcap/tidb/pkg/statistics/handle/types" + "github.com/pingcap/tidb/pkg/testkit" + "github.com/stretchr/testify/require" +) + +func TestConcurrentlyInitStatsWithMemoryLimit(t *testing.T) { + restore := config.RestoreFunc() + defer restore() + config.UpdateGlobal(func(conf *config.Config) { + conf.Performance.LiteInitStats = false + conf.Performance.ConcurrentlyInitStats = true + }) + handle.IsFullCacheFunc = func(cache types.StatsCache, total uint64) bool { + return true + } + testConcurrentlyInitStats(t) +} + +func TestConcurrentlyInitStatsWithoutMemoryLimit(t *testing.T) { + restore := config.RestoreFunc() + defer restore() + config.UpdateGlobal(func(conf *config.Config) { + conf.Performance.LiteInitStats = false + conf.Performance.ConcurrentlyInitStats = true + }) + handle.IsFullCacheFunc = func(cache types.StatsCache, total uint64) bool { + return false + } + testConcurrentlyInitStats(t) +} + +func testConcurrentlyInitStats(t *testing.T) { + store, dom := testkit.CreateMockStoreAndDomain(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("set global tidb_analyze_column_options='ALL'") + tk.MustExec("create table t1 (a int, b int, c int, primary key(c))") + tk.MustExec("insert into t1 values (1,1,1),(2,2,2),(3,3,3),(4,4,4),(5,5,5),(6,7,8)") + tk.MustExec("analyze table t1") + for i := 2; i < 10; i++ { + tk.MustExec(fmt.Sprintf("create table t%v (a int, b int, c int, primary key(c))", i)) + tk.MustExec(fmt.Sprintf("insert into t%v select * from t1", i)) + tk.MustExec(fmt.Sprintf("analyze table t%v all columns", i)) + } + h := dom.StatsHandle() + is := dom.InfoSchema() + h.Clear() + require.Equal(t, h.MemConsumed(), int64(0)) + require.NoError(t, h.InitStats(context.Background(), is)) + for i := 1; i < 10; i++ { + tbl, err := is.TableByName(context.Background(), model.NewCIStr("test"), model.NewCIStr(fmt.Sprintf("t%v", i))) + require.NoError(t, err) + stats, ok := h.StatsCache.Get(tbl.Meta().ID) + require.True(t, ok) + for _, col := range stats.GetColSlice() { + require.True(t, col.IsAllEvicted()) + require.False(t, col.IsFullLoad()) + } + } + for i := 1; i < 10; i++ { + tk.MustQuery(fmt.Sprintf("explain select * from t%v where a = 1", i)).CheckNotContain("pseudo") + } + for i := 1; i < 10; i++ { + tk.MustQuery(fmt.Sprintf("explain select * from t%v where b = 1", i)).CheckNotContain("pseudo") + } + for i := 1; i < 10; i++ { + tk.MustQuery(fmt.Sprintf("explain select * from t%v where c >= 1", i)).CheckNotContain("pseudo") + } + for i := 1; i < 10; i++ { + tbl, err := is.TableByName(context.Background(), model.NewCIStr("test"), model.NewCIStr(fmt.Sprintf("t%v", i))) + require.NoError(t, err) + stats, ok := h.StatsCache.Get(tbl.Meta().ID) + require.True(t, ok) + for _, col := range stats.GetColSlice() { + require.True(t, col.IsFullLoad()) + require.False(t, col.IsAllEvicted()) + } + } + require.Equal(t, int64(126), handle.GetMaxTidRecordForTest()) +}