From 917044c64694ac4f13aeb0852d2f44d1902d822c Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Wed, 1 Feb 2023 13:21:32 +0800 Subject: [PATCH 1/8] util: add InTest to check whether to be in test Signed-off-by: Weizhen Wang --- Makefile | 2 +- resourcemanager/util/BUILD.bazel | 1 + resourcemanager/util/shard_pool_map.go | 3 ++- util/intest/BUILD.bazel | 11 +++++++++++ util/intest/common.go | 20 ++++++++++++++++++++ util/intest/intest.go | 20 ++++++++++++++++++++ 6 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 util/intest/BUILD.bazel create mode 100644 util/intest/common.go create mode 100644 util/intest/intest.go diff --git a/Makefile b/Makefile index 6afde9f6420ec..09c4b78090c90 100644 --- a/Makefile +++ b/Makefile @@ -411,7 +411,7 @@ bazel_test: failpoint-enable bazel_ci_prepare bazel_coverage_test: failpoint-enable bazel_ci_prepare bazel $(BAZEL_GLOBAL_CONFIG) coverage $(BAZEL_CMD_CONFIG) \ - --build_event_json_file=bazel_1.json --@io_bazel_rules_go//go/config:cover_format=go_cover --define gotags=deadlock \ + --build_event_json_file=bazel_1.json --@io_bazel_rules_go//go/config:cover_format=go_cover --define gotags=deadlock,intest \ -- //... -//cmd/... -//tests/graceshutdown/... \ -//tests/globalkilltest/... -//tests/readonlytest/... -//br/pkg/task:task_test -//tests/realtikvtest/... diff --git a/resourcemanager/util/BUILD.bazel b/resourcemanager/util/BUILD.bazel index 1c5396db6049b..4bd4fd53994f5 100644 --- a/resourcemanager/util/BUILD.bazel +++ b/resourcemanager/util/BUILD.bazel @@ -10,6 +10,7 @@ go_library( importpath = "github.com/pingcap/tidb/resourcemanager/util", visibility = ["//visibility:public"], deps = [ + "//util/intest", "@com_github_pingcap_errors//:errors", "@org_uber_go_atomic//:atomic", ], diff --git a/resourcemanager/util/shard_pool_map.go b/resourcemanager/util/shard_pool_map.go index 371365af031e1..3b3975b3b933f 100644 --- a/resourcemanager/util/shard_pool_map.go +++ b/resourcemanager/util/shard_pool_map.go @@ -18,6 +18,7 @@ import ( "sync" "github.com/pingcap/errors" + "github.com/pingcap/tidb/util/intest" ) const shard = 8 @@ -69,7 +70,7 @@ func newPoolMap() poolMap { func (p *poolMap) Add(key string, pool *PoolContainer) error { p.mu.Lock() defer p.mu.Unlock() - if _, contain := p.poolMap[key]; contain { + if _, contain := p.poolMap[key]; contain && !intest.InTest { return errors.New("pool is already exist") } p.poolMap[key] = pool diff --git a/util/intest/BUILD.bazel b/util/intest/BUILD.bazel new file mode 100644 index 0000000000000..8eb8b75c1ca25 --- /dev/null +++ b/util/intest/BUILD.bazel @@ -0,0 +1,11 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "intest", + srcs = [ + "common.go", + "intest.go", #keep + ], + importpath = "github.com/pingcap/tidb/util/intest", + visibility = ["//visibility:public"], +) diff --git a/util/intest/common.go b/util/intest/common.go new file mode 100644 index 0000000000000..6e5a6abb17dd4 --- /dev/null +++ b/util/intest/common.go @@ -0,0 +1,20 @@ +// Copyright 2023 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !intest + +package intest + +// InTest checks if the code is running in test. +const InTest = false diff --git a/util/intest/intest.go b/util/intest/intest.go new file mode 100644 index 0000000000000..a96d2fbf737ef --- /dev/null +++ b/util/intest/intest.go @@ -0,0 +1,20 @@ +// Copyright 2023 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build intest + +package intest + +// InTest checks if the code is running in test. +const InTest = true From f0a10f3e98aaf9a4cdac66ccd2d70e06aa744856 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Wed, 1 Feb 2023 13:29:26 +0800 Subject: [PATCH 2/8] update Signed-off-by: Weizhen Wang --- ddl/BUILD.bazel | 1 + ddl/sanity_check.go | 4 ++-- kv/BUILD.bazel | 1 + kv/txn.go | 4 ++-- server/BUILD.bazel | 1 + server/mock_conn.go | 4 ++-- session/BUILD.bazel | 1 + session/bootstrap.go | 12 +++--------- session/session.go | 4 ++-- 9 files changed, 15 insertions(+), 17 deletions(-) diff --git a/ddl/BUILD.bazel b/ddl/BUILD.bazel index d3636a9d64b2c..daa9c7eea4b29 100644 --- a/ddl/BUILD.bazel +++ b/ddl/BUILD.bazel @@ -107,6 +107,7 @@ go_library( "//util/gcutil", "//util/generic", "//util/hack", + "//util/intest", "//util/logutil", "//util/mathutil", "//util/mock", diff --git a/ddl/sanity_check.go b/ddl/sanity_check.go index e005eee6a9856..2dcc17a8131f4 100644 --- a/ddl/sanity_check.go +++ b/ddl/sanity_check.go @@ -16,7 +16,6 @@ package ddl import ( "context" - "flag" "fmt" "strings" @@ -26,6 +25,7 @@ import ( "github.com/pingcap/tidb/parser/ast" "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/util/intest" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/mathutil" "github.com/pingcap/tidb/util/sqlexec" @@ -182,7 +182,7 @@ func (ctx *delRangeCntCtx) deduplicateIdxCnt(indexIDs []int64) int { // It's only check during the test environment, so it would panic directly. // These checks may be controlled by configuration in the future. func (d *ddl) checkHistoryJobInTest(ctx sessionctx.Context, historyJob *model.Job) { - if !(flag.Lookup("test.v") != nil || flag.Lookup("check.v") != nil) { + if !intest.InTest { return } diff --git a/kv/BUILD.bazel b/kv/BUILD.bazel index 992d99d382e42..8b345f574336a 100644 --- a/kv/BUILD.bazel +++ b/kv/BUILD.bazel @@ -30,6 +30,7 @@ go_library( "//types", "//util/codec", "//util/dbterror", + "//util/intest", "//util/logutil", "//util/memory", "//util/set", diff --git a/kv/txn.go b/kv/txn.go index 035f2aa662eca..fbbe6bdf27137 100644 --- a/kv/txn.go +++ b/kv/txn.go @@ -17,7 +17,6 @@ package kv import ( "context" "errors" - "flag" "fmt" "math" "math/rand" @@ -26,6 +25,7 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/tidb/parser/terror" + "github.com/pingcap/tidb/util/intest" "github.com/pingcap/tidb/util/logutil" "github.com/tikv/client-go/v2/oracle" "go.uber.org/zap" @@ -206,7 +206,7 @@ func setRequestSourceForInnerTxn(ctx context.Context, txn Transaction) { } // panic in test mode in case there are requests without source in the future. // log warnings in production mode. - if flag.Lookup("test.v") != nil || flag.Lookup("check.v") != nil { + if intest.InTest { panic("unexpected no source type context, if you see this error, " + "the `RequestSourceTypeKey` is missing in your context") } else { diff --git a/server/BUILD.bazel b/server/BUILD.bazel index 0d1303bf53993..d1139f0a94138 100644 --- a/server/BUILD.bazel +++ b/server/BUILD.bazel @@ -78,6 +78,7 @@ go_library( "//util/fastrand", "//util/gcutil", "//util/hack", + "//util/intest", "//util/logutil", "//util/memory", "//util/pdapi", diff --git a/server/mock_conn.go b/server/mock_conn.go index 36ad6503db812..620cae702707a 100644 --- a/server/mock_conn.go +++ b/server/mock_conn.go @@ -18,7 +18,6 @@ import ( "bufio" "bytes" "context" - "flag" "math/rand" "testing" @@ -29,6 +28,7 @@ import ( tmysql "github.com/pingcap/tidb/parser/mysql" "github.com/pingcap/tidb/util/arena" "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/intest" "github.com/stretchr/testify/require" ) @@ -80,7 +80,7 @@ func (mc *mockConn) ID() uint64 { func CreateMockServer(t *testing.T, store kv.Storage) *Server { if !RunInGoTest { // If CreateMockServer is called in another package, RunInGoTest is not initialized. - RunInGoTest = flag.Lookup("test.v") != nil || flag.Lookup("check.v") != nil + RunInGoTest = intest.InTest } tidbdrv := NewTiDBDriver(store) cfg := config.NewConfig() diff --git a/session/BUILD.bazel b/session/BUILD.bazel index 8e567503a6377..d6404e04c4bbf 100644 --- a/session/BUILD.bazel +++ b/session/BUILD.bazel @@ -72,6 +72,7 @@ go_library( "//util/collate", "//util/dbterror", "//util/execdetails", + "//util/intest", "//util/kvcache", "//util/logutil", "//util/logutil/consistency", diff --git a/session/bootstrap.go b/session/bootstrap.go index ed65bb0720cf0..d7e1643abbdeb 100644 --- a/session/bootstrap.go +++ b/session/bootstrap.go @@ -21,7 +21,6 @@ package session import ( "context" "encoding/hex" - "flag" "fmt" "io/ioutil" osuser "os/user" @@ -51,6 +50,7 @@ import ( "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/dbterror" + "github.com/pingcap/tidb/util/intest" "github.com/pingcap/tidb/util/logutil" utilparser "github.com/pingcap/tidb/util/parser" "github.com/pingcap/tidb/util/sqlexec" @@ -2407,12 +2407,6 @@ func doBootstrapSQLFile(s Session) { } } -// inTestSuite checks if we are bootstrapping in the context of tests. -// There are some historical differences in behavior between tests and non-tests. -func inTestSuite() bool { - return flag.Lookup("test.v") != nil || flag.Lookup("check.v") != nil -} - // doDMLWorks executes DML statements in bootstrap stage. // All the statements run in a single transaction. func doDMLWorks(s Session) { @@ -2457,11 +2451,11 @@ func doDMLWorks(s Session) { vVal = variable.On } case variable.TiDBMemOOMAction: - if inTestSuite() { + if intest.InTest { vVal = variable.OOMActionLog } case variable.TiDBEnableAutoAnalyze: - if inTestSuite() { + if intest.InTest { vVal = variable.Off } // For the following sysvars, we change the default diff --git a/session/session.go b/session/session.go index 11226483660bb..4699425046a2a 100644 --- a/session/session.go +++ b/session/session.go @@ -25,7 +25,6 @@ import ( "encoding/hex" "encoding/json" stderrs "errors" - "flag" "fmt" "math" "math/rand" @@ -92,6 +91,7 @@ import ( "github.com/pingcap/tidb/util/collate" "github.com/pingcap/tidb/util/dbterror" "github.com/pingcap/tidb/util/execdetails" + "github.com/pingcap/tidb/util/intest" "github.com/pingcap/tidb/util/kvcache" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/logutil/consistency" @@ -4217,7 +4217,7 @@ func (s *session) setRequestSource(ctx context.Context, stmtLabel string, stmtNo } // panic in test mode in case there are requests without source in the future. // log warnings in production mode. - if flag.Lookup("test.v") != nil || flag.Lookup("check.v") != nil { + if intest.InTest { panic("unexpected no source type context, if you see this error, " + "the `RequestSourceTypeKey` is missing in your context") } else { From fa94809fc79aab11bac5c7d12511b0e539b7ae73 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Wed, 1 Feb 2023 13:32:40 +0800 Subject: [PATCH 3/8] update Signed-off-by: Weizhen Wang --- testkit/testkit.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testkit/testkit.go b/testkit/testkit.go index 56e02fef5e688..75355d075fbe1 100644 --- a/testkit/testkit.go +++ b/testkit/testkit.go @@ -33,6 +33,7 @@ import ( "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/intest" "github.com/pingcap/tidb/util/mathutil" "github.com/pingcap/tidb/util/sqlexec" "github.com/stretchr/testify/assert" @@ -56,6 +57,7 @@ type TestKit struct { // NewTestKit returns a new *TestKit. func NewTestKit(t testing.TB, store kv.Storage) *TestKit { + require.True(t, intest.InTest, "you should add --tags=intest when to test") runtime.GOMAXPROCS(mathutil.Min(16, runtime.GOMAXPROCS(0))) tk := &TestKit{ require: require.New(t), From 8cbfd76164e53a5cbabed26f4cabb55604bf10b8 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Wed, 1 Feb 2023 13:41:45 +0800 Subject: [PATCH 4/8] update Signed-off-by: Weizhen Wang --- tools/check/ut.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/check/ut.go b/tools/check/ut.go index 08b31d243a157..32f85b422109b 100644 --- a/tools/check/ut.go +++ b/tools/check/ut.go @@ -837,7 +837,7 @@ func skipDIR(pkg string) bool { func buildTestBinary(pkg string) error { // go test -c - cmd := exec.Command("go", "test", "-c", "-vet", "off", "-o", testFileName(pkg)) + cmd := exec.Command("go", "test", "-c", "-vet", "off", "--tags=intest", "-o", testFileName(pkg)) if coverprofile != "" { cmd.Args = append(cmd.Args, "-cover") } From e6df61a98af033af69b1a13e4785a5ac9be1d2bb Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Wed, 1 Feb 2023 13:45:21 +0800 Subject: [PATCH 5/8] update Signed-off-by: Weizhen Wang --- Makefile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 09c4b78090c90..1cfd93fc662e7 100644 --- a/Makefile +++ b/Makefile @@ -444,27 +444,27 @@ bazel_golangcilinter: -- run $$($(PACKAGE_DIRECTORIES)) --config ./.golangci.yaml bazel_brietest: failpoint-enable bazel_ci_prepare - bazel $(BAZEL_GLOBAL_CONFIG) test $(BAZEL_CMD_CONFIG) --test_arg=-with-real-tikv --define gotags=deadlock \ + bazel $(BAZEL_GLOBAL_CONFIG) test $(BAZEL_CMD_CONFIG) --test_arg=-with-real-tikv --define gotags=deadlock,intest \ -- //tests/realtikvtest/brietest/... bazel_pessimistictest: failpoint-enable bazel_ci_prepare - bazel $(BAZEL_GLOBAL_CONFIG) test $(BAZEL_CMD_CONFIG) --test_arg=-with-real-tikv --define gotags=deadlock \ + bazel $(BAZEL_GLOBAL_CONFIG) test $(BAZEL_CMD_CONFIG) --test_arg=-with-real-tikv --define gotags=deadlock,intest \ -- //tests/realtikvtest/pessimistictest/... bazel_sessiontest: failpoint-enable bazel_ci_prepare - bazel $(BAZEL_GLOBAL_CONFIG) test $(BAZEL_CMD_CONFIG) --test_arg=-with-real-tikv --define gotags=deadlock \ + bazel $(BAZEL_GLOBAL_CONFIG) test $(BAZEL_CMD_CONFIG) --test_arg=-with-real-tikv --define gotags=deadlock,intest \ -- //tests/realtikvtest/sessiontest/... bazel_statisticstest: failpoint-enable bazel_ci_prepare - bazel $(BAZEL_GLOBAL_CONFIG) test $(BAZEL_CMD_CONFIG) --test_arg=-with-real-tikv --define gotags=deadlock \ + bazel $(BAZEL_GLOBAL_CONFIG) test $(BAZEL_CMD_CONFIG) --test_arg=-with-real-tikv --define gotags=deadlock,intest \ -- //tests/realtikvtest/statisticstest/... bazel_txntest: failpoint-enable bazel_ci_prepare - bazel $(BAZEL_GLOBAL_CONFIG) test $(BAZEL_CMD_CONFIG) --test_arg=-with-real-tikv --define gotags=deadlock \ + bazel $(BAZEL_GLOBAL_CONFIG) test $(BAZEL_CMD_CONFIG) --test_arg=-with-real-tikv --define gotags=deadlock,intest \ -- //tests/realtikvtest/txntest/... bazel_addindextest: failpoint-enable bazel_ci_prepare - bazel $(BAZEL_GLOBAL_CONFIG) test $(BAZEL_CMD_CONFIG) --test_arg=-with-real-tikv --define gotags=deadlock \ + bazel $(BAZEL_GLOBAL_CONFIG) test $(BAZEL_CMD_CONFIG) --test_arg=-with-real-tikv --define gotags=deadlock,intest \ -- //tests/realtikvtest/addindextest/... bazel_lint: bazel_prepare From a6cd8e506e0a43ceea6a61f05b07fe368f19939f Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Wed, 1 Feb 2023 13:55:17 +0800 Subject: [PATCH 6/8] update Signed-off-by: Weizhen Wang --- testkit/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/testkit/BUILD.bazel b/testkit/BUILD.bazel index c28ef0614eb04..3e33b8bd3f555 100644 --- a/testkit/BUILD.bazel +++ b/testkit/BUILD.bazel @@ -31,6 +31,7 @@ go_library( "//util/breakpoint", "//util/chunk", "//util/gctuner", + "//util/intest", "//util/mathutil", "//util/sqlexec", "@com_github_pingcap_errors//:errors", From 07c73dbbb3b5bdd1c04c37c2bc12b5ec52441975 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Wed, 1 Feb 2023 14:15:28 +0800 Subject: [PATCH 7/8] update Signed-off-by: Weizhen Wang --- resourcemanager/util/shard_pool_map_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/resourcemanager/util/shard_pool_map_test.go b/resourcemanager/util/shard_pool_map_test.go index bb09e2fbd8a24..48f0c3b7d3122 100644 --- a/resourcemanager/util/shard_pool_map_test.go +++ b/resourcemanager/util/shard_pool_map_test.go @@ -19,6 +19,7 @@ import ( "sync/atomic" "testing" + "github.com/pingcap/tidb/util/intest" "github.com/stretchr/testify/require" ) @@ -29,7 +30,9 @@ func TestShardPoolMap(t *testing.T) { id := strconv.FormatInt(int64(i), 10) require.NoError(t, pm.Add(id, &PoolContainer{Pool: NewMockGPool(id), Component: DDL})) } - require.Error(t, pm.Add("1", &PoolContainer{Pool: NewMockGPool("1"), Component: DDL})) + if !intest.InTest { + require.Error(t, pm.Add("1", &PoolContainer{Pool: NewMockGPool("1"), Component: DDL})) + } var cnt atomic.Int32 pm.Iter(func(pool *PoolContainer) { cnt.Add(1) From 1caa5af01660333975e47bfa2ea7fe4cec25d492 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Wed, 1 Feb 2023 14:29:03 +0800 Subject: [PATCH 8/8] update Signed-off-by: Weizhen Wang --- resourcemanager/util/BUILD.bazel | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/resourcemanager/util/BUILD.bazel b/resourcemanager/util/BUILD.bazel index 4bd4fd53994f5..37283d38f7e07 100644 --- a/resourcemanager/util/BUILD.bazel +++ b/resourcemanager/util/BUILD.bazel @@ -20,5 +20,8 @@ go_test( name = "util_test", srcs = ["shard_pool_map_test.go"], embed = [":util"], - deps = ["@com_github_stretchr_testify//require"], + deps = [ + "//util/intest", + "@com_github_stretchr_testify//require", + ], )