*: improve planner/core to make test faster (pingcap#41416)

ref pingcap#41377
ghazalfamilyusa · Feb 15, 2023 · 2f9e7b2 · 2f9e7b2
1 parent 2de1d17
commit 2f9e7b2
Show file tree

Hide file tree

Showing 79 changed files with 12,533 additions and 12,272 deletions.
diff --git a/planner/core/BUILD.bazel b/planner/core/BUILD.bazel
@@ -97,6 +97,7 @@ go_library(
         "//parser/opcode",
         "//parser/terror",
         "//parser/types",
+        "//planner/core/internal",
         "//planner/funcdep",
         "//planner/property",
         "//planner/util",
@@ -169,14 +170,12 @@ go_test(
         "expression_rewriter_test.go",
         "expression_test.go",
         "find_best_task_test.go",
-        "flat_plan_test.go",
         "fragment_test.go",
         "indexmerge_intersection_test.go",
         "indexmerge_path_test.go",
         "indexmerge_test.go",
         "integration_partition_test.go",
         "integration_test.go",
-        "logical_plan_test.go",
         "logical_plan_trace_test.go",
         "logical_plans_test.go",
         "main_test.go",
@@ -200,17 +199,12 @@ go_test(
         "plan_to_pb_test.go",
         "planbuilder_test.go",
         "point_get_plan_test.go",
-        "predicate_simplification_test.go",
         "prepare_test.go",
         "preprocess_test.go",
-        "rule_derive_topn_from_window_test.go",
-        "rule_inject_extra_projection_test.go",
         "rule_join_reorder_dp_test.go",
         "rule_join_reorder_test.go",
         "rule_result_reorder_test.go",
-        "stats_test.go",
         "stringer_test.go",
-        "window_push_down_test.go",
     ],
     data = glob(["testdata/**"]),
     embed = [":core"],
@@ -236,6 +230,7 @@ go_test(
         "//parser/mysql",
         "//parser/terror",
         "//planner",
+        "//planner/core/internal",
         "//planner/property",
         "//planner/util",
         "//session",
@@ -245,8 +240,6 @@ go_test(
         "//sessiontxn",
         "//statistics",
         "//statistics/handle",
-        "//store/mockstore",
-        "//store/mockstore/unistore",
         "//table",
         "//testkit",
         "//testkit/ddlhelper",
@@ -275,12 +268,10 @@ go_test(
         "@com_github_golang_snappy//:snappy",
         "@com_github_pingcap_errors//:errors",
         "@com_github_pingcap_failpoint//:failpoint",
-        "@com_github_pingcap_kvproto//pkg/metapb",
         "@com_github_pingcap_tipb//go-tipb",
         "@com_github_prometheus_client_golang//prometheus",
         "@com_github_prometheus_client_model//go",
         "@com_github_stretchr_testify//require",
-        "@com_github_tikv_client_go_v2//testutils",
         "@org_golang_x_exp//slices",
         "@org_uber_go_goleak//:goleak",
     ],

diff --git a/planner/core/binary_plan_test.go b/planner/core/binary_plan_test.go
@@ -19,14 +19,12 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"regexp"
 	"strings"
 	"testing"
 
 	"github.com/golang/snappy"
 	"github.com/pingcap/tidb/config"
 	"github.com/pingcap/tidb/parser/auth"
-	"github.com/pingcap/tidb/planner/core"
 	"github.com/pingcap/tidb/sessionctx/variable"
 	"github.com/pingcap/tidb/testkit"
 	"github.com/pingcap/tidb/testkit/testdata"
@@ -36,117 +34,6 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func simplifyAndCheckBinaryPlan(t *testing.T, pb *tipb.ExplainData) {
-	if pb.Main != nil {
-		simplifyAndCheckBinaryOperator(t, pb.Main, pb.WithRuntimeStats)
-	}
-	for _, cte := range pb.Ctes {
-		if cte != nil {
-			simplifyAndCheckBinaryOperator(t, cte, pb.WithRuntimeStats)
-		}
-	}
-}
-
-func simplifyAndCheckBinaryOperator(t *testing.T, pb *tipb.ExplainOperator, withRuntimeStats bool) {
-	if withRuntimeStats {
-		if pb.TaskType == tipb.TaskType_root {
-			require.NotEmpty(t, pb.RootBasicExecInfo)
-		} else if pb.TaskType != tipb.TaskType_unknown {
-			require.NotEmpty(t, pb.CopExecInfo)
-		}
-	}
-	pb.RootBasicExecInfo = ""
-	pb.RootGroupExecInfo = nil
-	pb.CopExecInfo = ""
-	match, err := regexp.MatchString("((Table|Index).*Scan)|CTEFullScan|Point_Get", pb.Name)
-	if err == nil && match {
-		require.NotNil(t, pb.AccessObjects)
-	}
-	// AccessObject field is an interface and json.Unmarshall can't handle it, so we don't check it against the json output.
-	pb.AccessObjects = nil
-	// MemoryBytes and DiskBytes are not stable sometimes.
-	pb.MemoryBytes = 0
-	pb.DiskBytes = 0
-	if len(pb.Children) > 0 {
-		for _, op := range pb.Children {
-			if op != nil {
-				simplifyAndCheckBinaryOperator(t, op, withRuntimeStats)
-			}
-		}
-	}
-}
-
-func TestBinaryPlanInExplainAndSlowLog(t *testing.T) {
-	// Prepare the slow log
-	originCfg := config.GetGlobalConfig()
-	newCfg := *originCfg
-	f, err := os.CreateTemp("", "tidb-slow-*.log")
-	require.NoError(t, err)
-	newCfg.Log.SlowQueryFile = f.Name()
-	config.StoreGlobalConfig(&newCfg)
-	defer func() {
-		config.StoreGlobalConfig(originCfg)
-		require.NoError(t, f.Close())
-		require.NoError(t, os.Remove(newCfg.Log.SlowQueryFile))
-	}()
-	require.NoError(t, logutil.InitLogger(newCfg.Log.ToLogConfig()))
-	store := testkit.CreateMockStore(t)
-	tk := testkit.NewTestKit(t, store)
-	tk.MustExec("use test")
-	tk.MustExec("set tidb_cost_model_version=2")
-	// If we don't set this, it will be false sometimes and the cost in the result will be different.
-	tk.MustExec("set @@tidb_enable_chunk_rpc=true")
-	tk.MustExec(fmt.Sprintf("set @@tidb_slow_query_file='%v'", f.Name()))
-	tk.MustExec("set tidb_slow_log_threshold=0;")
-	defer func() {
-		tk.MustExec("set tidb_slow_log_threshold=300;")
-	}()
-
-	var input []string
-	var output []struct {
-		SQL        string
-		BinaryPlan *tipb.ExplainData
-	}
-	planSuiteData := core.GetBinaryPlanSuiteData()
-	planSuiteData.LoadTestCases(t, &input, &output)
-
-	for i, test := range input {
-		comment := fmt.Sprintf("case:%v sql:%s", i, test)
-		if len(test) < 7 || test[:7] != "explain" {
-			tk.MustExec(test)
-			testdata.OnRecord(func() {
-				output[i].SQL = test
-				output[i].BinaryPlan = nil
-			})
-			continue
-		}
-		result := testdata.ConvertRowsToStrings(tk.MustQuery(test).Rows())
-		require.Equal(t, len(result), 1, comment)
-		s := result[0]
-
-		// assert that the binary plan in the slow log is the same as the result in the EXPLAIN statement
-		slowLogResult := testdata.ConvertRowsToStrings(tk.MustQuery("select binary_plan from information_schema.slow_query " +
-			`where query = "` + test + `;" ` +
-			"order by time desc limit 1").Rows())
-		require.Lenf(t, slowLogResult, 1, comment)
-		require.Equal(t, s, slowLogResult[0], comment)
-
-		b, err := base64.StdEncoding.DecodeString(s)
-		require.NoError(t, err)
-		b, err = snappy.Decode(nil, b)
-		require.NoError(t, err)
-		binary := &tipb.ExplainData{}
-		err = binary.Unmarshal(b)
-		require.NoError(t, err)
-		testdata.OnRecord(func() {
-			output[i].SQL = test
-			output[i].BinaryPlan = binary
-		})
-		simplifyAndCheckBinaryPlan(t, binary)
-		require.Equal(t, output[i].BinaryPlan, binary)
-	}
-}
-
 func TestBinaryPlanSwitch(t *testing.T) {
 	originCfg := config.GetGlobalConfig()
 	newCfg := *originCfg

diff --git a/planner/core/casetest/BUILD.bazel b/planner/core/casetest/BUILD.bazel
@@ -0,0 +1,71 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
+
+go_test(
+    name = "casetest_test",
+    timeout = "short",
+    srcs = [
+        "binary_plan_test.go",
+        "cbo_test.go",
+        "enforce_mpp_test.go",
+        "expression_rewriter_test.go",
+        "flat_plan_test.go",
+        "integration_partition_test.go",
+        "integration_test.go",
+        "main_test.go",
+        "partition_pruner_test.go",
+        "physical_plan_test.go",
+        "plan_test.go",
+        "point_get_plan_test.go",
+        "predicate_simplification_test.go",
+        "rule_derive_topn_from_window_test.go",
+        "rule_inject_extra_projection_test.go",
+        "rule_join_reorder_test.go",
+        "rule_result_reorder_test.go",
+        "stats_test.go",
+        "window_push_down_test.go",
+    ],
+    data = glob(["testdata/**"]),
+    flaky = True,
+    shard_count = 50,
+    deps = [
+        "//config",
+        "//domain",
+        "//executor",
+        "//expression",
+        "//expression/aggregation",
+        "//infoschema",
+        "//kv",
+        "//parser",
+        "//parser/ast",
+        "//parser/model",
+        "//parser/mysql",
+        "//parser/terror",
+        "//planner",
+        "//planner/core",
+        "//planner/core/internal",
+        "//planner/property",
+        "//session",
+        "//sessionctx/stmtctx",
+        "//sessionctx/variable",
+        "//sessiontxn",
+        "//statistics",
+        "//statistics/handle",
+        "//testkit",
+        "//testkit/external",
+        "//testkit/testdata",
+        "//testkit/testmain",
+        "//testkit/testsetup",
+        "//types",
+        "//util",
+        "//util/collate",
+        "//util/hint",
+        "//util/logutil",
+        "//util/mock",
+        "//util/plancodec",
+        "@com_github_golang_snappy//:snappy",
+        "@com_github_pingcap_failpoint//:failpoint",
+        "@com_github_pingcap_tipb//go-tipb",
+        "@com_github_stretchr_testify//require",
+        "@org_uber_go_goleak//:goleak",
+    ],
+)