makefile(all): add new cases to run_group and check cases in makefile (…

…#8794) (#8924)

close #8932

Makefile

@@ -311,8 +311,8 @@ check-static: tools/bin/golangci-lint
 	cd dm && ../tools/bin/golangci-lint run --timeout 10m0s
 
 check: check-copyright generate_mock go-generate fmt check-static tidy terror_check errdoc \
-	check-merge-conflicts check-ticdc-dashboard check-diff-line-width \
-	swagger-spec check-makefiles check_engine_integration_test
+	check-merge-conflicts check-ticdc-dashboard check-diff-line-width swagger-spec check-makefiles \
+	check_cdc_integration_test check_dm_integration_test check_engine_integration_test 
 	@git --no-pager diff --exit-code || (echo "Please add changed files!" && false)
 
 fast_check: check-copyright fmt check-static tidy terror_check errdoc \
@@ -537,6 +537,13 @@ check_third_party_binary_for_engine:
 
 check_engine_integration_test:
 	./engine/test/utils/check_case.sh
+	./engine/test/integration_tests/run_group.sh "check others"
+
+check_dm_integration_test:
+	./dm/tests/run_group.sh "check others"
+
+check_cdc_integration_test:
+	./tests/integration_tests/run_group.sh check "others"
 
 bin/mc:
 	./scripts/download-mc.sh

dm/checker/checker.go

@@ -28,6 +28,7 @@ import (
 	"github.com/pingcap/tidb/br/pkg/lightning/importer"
 	"github.com/pingcap/tidb/br/pkg/lightning/importer/opts"
 	"github.com/pingcap/tidb/br/pkg/lightning/mydump"
+	"github.com/pingcap/tidb/br/pkg/lightning/precheck"
 	"github.com/pingcap/tidb/dumpling/export"
 	"github.com/pingcap/tidb/parser/mysql"
 	"github.com/pingcap/tidb/types"
@@ -477,28 +478,28 @@ func (c *Checker) Init(ctx context.Context) (err error) {
 				info.totalDataSize.Load(), targetInfoGetter))
 		}
 		if _, ok := c.checkingItems[config.LightningEmptyRegionChecking]; ok {
-			lChecker, err := builder.BuildPrecheckItem(importer.CheckTargetClusterEmptyRegion)
+			lChecker, err := builder.BuildPrecheckItem(precheck.CheckTargetClusterEmptyRegion)
 			if err != nil {
 				return err
 			}
 			c.checkList = append(c.checkList, checker.NewLightningEmptyRegionChecker(lChecker))
 		}
 		if _, ok := c.checkingItems[config.LightningRegionDistributionChecking]; ok {
-			lChecker, err := builder.BuildPrecheckItem(importer.CheckTargetClusterRegionDist)
+			lChecker, err := builder.BuildPrecheckItem(precheck.CheckTargetClusterRegionDist)
 			if err != nil {
 				return err
 			}
 			c.checkList = append(c.checkList, checker.NewLightningRegionDistributionChecker(lChecker))
 		}
 		if _, ok := c.checkingItems[config.LightningDownstreamVersionChecking]; ok {
-			lChecker, err := builder.BuildPrecheckItem(importer.CheckTargetClusterVersion)
+			lChecker, err := builder.BuildPrecheckItem(precheck.CheckTargetClusterVersion)
 			if err != nil {
 				return err
 			}
 			c.checkList = append(c.checkList, checker.NewLightningClusterVersionChecker(lChecker))
 		}
 		if _, ok := c.checkingItems[config.LightningMutexFeatureChecking]; ok {
-			lChecker, err := builder.BuildPrecheckItem(importer.CheckTargetUsingCDCPITR)
+			lChecker, err := builder.BuildPrecheckItem(precheck.CheckTargetUsingCDCPITR)
 			if err != nil {
 				return err
 			}

dm/pkg/checker/lightning.go

@@ -19,13 +19,14 @@ import (
 
 	"github.com/docker/go-units"
 	"github.com/pingcap/tidb/br/pkg/lightning/importer"
+	"github.com/pingcap/tidb/br/pkg/lightning/precheck"
 	"github.com/pingcap/tiflow/dm/pkg/log"
 )
 
 func convertLightningPrecheck(
 	ctx context.Context,
 	dmResult *Result,
-	lightningPrechecker importer.PrecheckItem,
+	lightningPrechecker precheck.Checker,
 	failLevel State,
 	instruction string,
 ) {
@@ -45,11 +46,11 @@ func convertLightningPrecheck(
 
 // LightningEmptyRegionChecker checks whether there are too many empty regions in the cluster.
 type LightningEmptyRegionChecker struct {
-	inner importer.PrecheckItem
+	inner precheck.Checker
 }
 
 // NewLightningEmptyRegionChecker creates a new LightningEmptyRegionChecker.
-func NewLightningEmptyRegionChecker(lightningChecker importer.PrecheckItem) RealChecker {
+func NewLightningEmptyRegionChecker(lightningChecker precheck.Checker) RealChecker {
 	return &LightningEmptyRegionChecker{inner: lightningChecker}
 }
 
@@ -77,11 +78,11 @@ func (c *LightningEmptyRegionChecker) Check(ctx context.Context) *Result {
 
 // LightningRegionDistributionChecker checks whether the region distribution is balanced.
 type LightningRegionDistributionChecker struct {
-	inner importer.PrecheckItem
+	inner precheck.Checker
 }
 
 // NewLightningRegionDistributionChecker creates a new LightningRegionDistributionChecker.
-func NewLightningRegionDistributionChecker(lightningChecker importer.PrecheckItem) RealChecker {
+func NewLightningRegionDistributionChecker(lightningChecker precheck.Checker) RealChecker {
 	return &LightningRegionDistributionChecker{inner: lightningChecker}
 }
 
@@ -109,11 +110,11 @@ func (c *LightningRegionDistributionChecker) Check(ctx context.Context) *Result
 
 // LightningClusterVersionChecker checks whether the cluster version is compatible with Lightning.
 type LightningClusterVersionChecker struct {
-	inner importer.PrecheckItem
+	inner precheck.Checker
 }
 
 // NewLightningClusterVersionChecker creates a new LightningClusterVersionChecker.
-func NewLightningClusterVersionChecker(lightningChecker importer.PrecheckItem) RealChecker {
+func NewLightningClusterVersionChecker(lightningChecker precheck.Checker) RealChecker {
 	return &LightningClusterVersionChecker{inner: lightningChecker}
 }
 
@@ -207,11 +208,11 @@ func (c *LightningFreeSpaceChecker) Check(ctx context.Context) *Result {
 
 // LightningCDCPiTRChecker checks whether the cluster has running CDC PiTR tasks.
 type LightningCDCPiTRChecker struct {
-	inner importer.PrecheckItem
+	inner precheck.Checker
 }
 
 // NewLightningCDCPiTRChecker creates a new LightningCDCPiTRChecker.
-func NewLightningCDCPiTRChecker(lightningChecker importer.PrecheckItem) RealChecker {
+func NewLightningCDCPiTRChecker(lightningChecker precheck.Checker) RealChecker {
 	c, ok := lightningChecker.(*importer.CDCPITRCheckItem)
 	if ok {
 		c.Instruction = "physical import mode is not compatible with them. Please switch to logical import mode then try again."

dm/pkg/checker/lightning_test.go

@@ -17,7 +17,7 @@ import (
 	"context"
 	"testing"
 
-	"github.com/pingcap/tidb/br/pkg/lightning/importer"
+	"github.com/pingcap/tidb/br/pkg/lightning/precheck"
 	"github.com/pingcap/tiflow/pkg/errors"
 	"github.com/stretchr/testify/require"
 )
@@ -28,17 +28,17 @@ type mockPrecheckItem struct {
 	msg  string
 }
 
-func (m mockPrecheckItem) Check(ctx context.Context) (*importer.CheckResult, error) {
+func (m mockPrecheckItem) Check(ctx context.Context) (*precheck.CheckResult, error) {
 	if m.err != nil {
 		return nil, m.err
 	}
-	return &importer.CheckResult{
+	return &precheck.CheckResult{
 		Passed:  m.pass,
 		Message: m.msg,
 	}, nil
 }
 
-func (m mockPrecheckItem) GetCheckItemID() importer.CheckItemID {
+func (m mockPrecheckItem) GetCheckItemID() precheck.CheckItemID {
 	return "mock"
 }
 

dm/tests/lightning_load_task/conf/diff_config1.toml

@@ -0,0 +1,35 @@
+# diff Configuration.
+
+check-thread-count = 4
+
+export-fix-sql = true
+
+check-struct-only = false
+
+[task]
+    output-dir = "/tmp/ticdc_dm_test/output"
+
+    source-instances = ["mysql1", "mysql2"]
+
+    target-instance = "tidb0"
+
+    target-check-tables = ["load_task1.t?*"]
+
+[data-sources]
+[data-sources.mysql1]
+host = "127.0.0.1"
+port = 3306
+user = "root"
+password = "123456"
+
+[data-sources.mysql2]
+host = "127.0.0.1"
+port = 3307
+user = "root"
+password = "123456"
+
+[data-sources.tidb0]
+host = "127.0.0.1"
+port = 4000
+user = "test"
+password = "123456"

dm/tests/lightning_load_task/conf/diff_config2.toml

@@ -0,0 +1,35 @@
+# diff Configuration.
+
+check-thread-count = 4
+
+export-fix-sql = true
+
+check-struct-only = false
+
+[task]
+    output-dir = "/tmp/ticdc_dm_test/output"
+
+    source-instances = ["mysql1", "mysql2"]
+
+    target-instance = "tidb0"
+
+    target-check-tables = ["load_task2.t?*"]
+
+[data-sources]
+[data-sources.mysql1]
+host = "127.0.0.1"
+port = 3306
+user = "root"
+password = "123456"
+
+[data-sources.mysql2]
+host = "127.0.0.1"
+port = 3307
+user = "root"
+password = "123456"
+
+[data-sources.tidb0]
+host = "127.0.0.1"
+port = 4000
+user = "test"
+password = "123456"

dm/tests/lightning_load_task/conf/diff_config3.toml

@@ -0,0 +1,29 @@
+# diff Configuration.
+
+check-thread-count = 4
+
+export-fix-sql = true
+
+check-struct-only = false
+
+[task]
+    output-dir = "/tmp/ticdc_dm_test/output"
+
+    source-instances = ["mysql1"]
+
+    target-instance = "tidb0"
+
+    target-check-tables = ["load_task3.t?*"]
+
+[data-sources]
+[data-sources.mysql1]
+host = "127.0.0.1"
+port = 3307
+user = "root"
+password = "123456"
+
+[data-sources.tidb0]
+host = "127.0.0.1"
+port = 4000
+user = "test"
+password = "123456"

dm/tests/lightning_load_task/conf/diff_config4.toml

@@ -0,0 +1,29 @@
+# diff Configuration.
+
+check-thread-count = 4
+
+export-fix-sql = true
+
+check-struct-only = false
+
+[task]
+    output-dir = "/tmp/ticdc_dm_test/output"
+
+    source-instances = ["mysql1"]
+
+    target-instance = "tidb0"
+
+    target-check-tables = ["load_task4.t?*"]
+
+[data-sources]
+[data-sources.mysql1]
+host = "127.0.0.1"
+port = 3306
+user = "root"
+password = "123456"
+
+[data-sources.tidb0]
+host = "127.0.0.1"
+port = 4000
+user = "test"
+password = "123456"

dm/tests/lightning_load_task/conf/dm-master.toml

@@ -0,0 +1,6 @@
+# Master Configuration.
+master-addr = ":8261"
+advertise-addr = "127.0.0.1:8261"
+
+rpc-timeout = "30s"
+auto-compaction-retention = "3s"

dm/tests/lightning_load_task/conf/dm-task-standalone.yaml

@@ -0,0 +1,41 @@
+---
+name: load_task1
+task-mode: all
+is-sharding: false
+meta-schema: "dm_meta"
+heartbeat-update-interval: 1
+heartbeat-report-interval: 1
+
+target-database:
+  host: "127.0.0.1"
+  port: 4000
+  user: "test"
+  password: "/Q7B9DizNLLTTfiZHv9WoEAKamfpIUs="
+
+mysql-instances:
+  - source-id: "mysql-replica-01"
+    block-allow-list:  "instance"
+    mydumper-config-name: "global"
+    loader-config-name: "global"
+    syncer-config-name: "global"
+
+block-allow-list:
+  instance:
+    do-dbs: ["load_task1"]
+
+mydumpers:
+  global:
+    threads: 4
+    chunk-filesize: 64
+    skip-tz-utc: true
+    extra-args: ""
+
+loaders:
+  global:
+    pool-size: 16
+    dir: "./dumped_data"
+
+syncers:
+  global:
+    worker-count: 16
+    batch: 100

dm/tests/lightning_load_task/conf/dm-task.yaml

@@ -40,7 +40,6 @@ loaders:
   global:
     pool-size: 16
     dir: "./dumped_data"
-    import-mode: "sql"
 
 syncers:
   global:

dm/tests/lightning_load_task/conf/dm-task2-standalone.yaml

@@ -0,0 +1,41 @@
+---
+name: load_task2
+task-mode: all
+is-sharding: false
+meta-schema: "dm_meta"
+heartbeat-update-interval: 1
+heartbeat-report-interval: 1
+
+target-database:
+  host: "127.0.0.1"
+  port: 4000
+  user: "test"
+  password: "/Q7B9DizNLLTTfiZHv9WoEAKamfpIUs="
+
+mysql-instances:
+  - source-id: "mysql-replica-01"
+    block-allow-list:  "instance"
+    mydumper-config-name: "global"
+    loader-config-name: "global"
+    syncer-config-name: "global"
+
+block-allow-list:
+  instance:
+    do-dbs: ["load_task2"]
+
+mydumpers:
+  global:
+    threads: 4
+    chunk-filesize: 64
+    skip-tz-utc: true
+    extra-args: ""
+
+loaders:
+  global:
+    pool-size: 16
+    dir: "./dumped_data"
+
+syncers:
+  global:
+    worker-count: 16
+    batch: 100