makefile(all): add new cases to run_group and check cases in makefile (#8794)

Makefile

@@ -299,9 +299,15 @@ check-static: tools/bin/golangci-lint
 	tools/bin/golangci-lint run --timeout 10m0s --skip-dirs "^dm/","^tests/"
 	#cd dm && ../tools/bin/golangci-lint run --timeout 10m0s
 
+<<<<<<< HEAD
 check: check-copyright fmt check-static tidy terror_check errdoc \
 	check-merge-conflicts check-ticdc-dashboard check-diff-line-width \
 	swagger-spec check-makefiles check_engine_integration_test
+=======
+check: check-copyright generate_mock go-generate fmt check-static tidy terror_check errdoc \
+	check-merge-conflicts check-ticdc-dashboard check-diff-line-width swagger-spec check-makefiles \
+	check_cdc_integration_test check_dm_integration_test check_engine_integration_test 
+>>>>>>> 65217445e8 (makefile(all): add new cases to run_group and check cases in makefile (#8794))
 	@git --no-pager diff --exit-code || (echo "Please add changed files!" && false)
 
 integration_test_coverage: tools/bin/gocovmerge tools/bin/goveralls
@@ -514,6 +520,13 @@ check_third_party_binary_for_engine:
 
 check_engine_integration_test:
 	./engine/test/utils/check_case.sh
+	./engine/test/integration_tests/run_group.sh "check others"
+
+check_dm_integration_test:
+	./dm/tests/run_group.sh "check others"
+
+check_cdc_integration_test:
+	./tests/integration_tests/run_group.sh check "others"
 
 bin/mc:
 	./scripts/download-mc.sh

dm/checker/checker.go

@@ -25,7 +25,15 @@ import (
 	"time"
 
 	_ "github.com/go-sql-driver/mysql" // for mysql
+<<<<<<< HEAD
 	"github.com/pingcap/tidb/br/pkg/lightning/restore"
+=======
+	"github.com/pingcap/tidb/br/pkg/lightning/checkpoints"
+	"github.com/pingcap/tidb/br/pkg/lightning/importer"
+	"github.com/pingcap/tidb/br/pkg/lightning/importer/opts"
+	"github.com/pingcap/tidb/br/pkg/lightning/mydump"
+	"github.com/pingcap/tidb/br/pkg/lightning/precheck"
+>>>>>>> 65217445e8 (makefile(all): add new cases to run_group and check cases in makefile (#8794))
 	"github.com/pingcap/tidb/dumpling/export"
 	"github.com/pingcap/tidb/parser/mysql"
 	"github.com/pingcap/tidb/util/dbutil"
@@ -355,26 +363,48 @@ func (c *Checker) Init(ctx context.Context) (err error) {
 			return err
 		}
 		if _, ok := c.checkingItems[config.LightningEmptyRegionChecking]; ok {
+<<<<<<< HEAD
 			lChecker, err := builder.BuildPrecheckItem(restore.CheckTargetClusterEmptyRegion)
+=======
+			lChecker, err := builder.BuildPrecheckItem(precheck.CheckTargetClusterEmptyRegion)
+>>>>>>> 65217445e8 (makefile(all): add new cases to run_group and check cases in makefile (#8794))
 			if err != nil {
 				return err
 			}
 			c.checkList = append(c.checkList, checker.NewLightningEmptyRegionChecker(lChecker))
 		}
 		if _, ok := c.checkingItems[config.LightningRegionDistributionChecking]; ok {
+<<<<<<< HEAD
 			lChecker, err := builder.BuildPrecheckItem(restore.CheckTargetClusterRegionDist)
+=======
+			lChecker, err := builder.BuildPrecheckItem(precheck.CheckTargetClusterRegionDist)
+>>>>>>> 65217445e8 (makefile(all): add new cases to run_group and check cases in makefile (#8794))
 			if err != nil {
 				return err
 			}
 			c.checkList = append(c.checkList, checker.NewLightningRegionDistributionChecker(lChecker))
 		}
 		if _, ok := c.checkingItems[config.LightningDownstreamVersionChecking]; ok {
+<<<<<<< HEAD
 			lChecker, err := builder.BuildPrecheckItem(restore.CheckTargetClusterVersion)
+=======
+			lChecker, err := builder.BuildPrecheckItem(precheck.CheckTargetClusterVersion)
+>>>>>>> 65217445e8 (makefile(all): add new cases to run_group and check cases in makefile (#8794))
 			if err != nil {
 				return err
 			}
 			c.checkList = append(c.checkList, checker.NewLightningClusterVersionChecker(lChecker))
 		}
+<<<<<<< HEAD
+=======
+		if _, ok := c.checkingItems[config.LightningMutexFeatureChecking]; ok {
+			lChecker, err := builder.BuildPrecheckItem(precheck.CheckTargetUsingCDCPITR)
+			if err != nil {
+				return err
+			}
+			c.checkList = append(c.checkList, checker.NewLightningCDCPiTRChecker(lChecker))
+		}
+>>>>>>> 65217445e8 (makefile(all): add new cases to run_group and check cases in makefile (#8794))
 	}
 
 	c.tctx.Logger.Info(c.displayCheckingItems())

dm/pkg/checker/lightning.go

@@ -16,13 +16,24 @@ package checker
 import (
 	"context"
 
+<<<<<<< HEAD
 	"github.com/pingcap/tidb/br/pkg/lightning/restore"
+=======
+	"github.com/docker/go-units"
+	"github.com/pingcap/tidb/br/pkg/lightning/importer"
+	"github.com/pingcap/tidb/br/pkg/lightning/precheck"
+	"github.com/pingcap/tiflow/dm/pkg/log"
+>>>>>>> 65217445e8 (makefile(all): add new cases to run_group and check cases in makefile (#8794))
 )
 
 func convertLightningPrecheck(
 	ctx context.Context,
 	dmResult *Result,
+<<<<<<< HEAD
 	lightningPrechecker restore.PrecheckItem,
+=======
+	lightningPrechecker precheck.Checker,
+>>>>>>> 65217445e8 (makefile(all): add new cases to run_group and check cases in makefile (#8794))
 	failLevel State,
 	instruction string,
 ) {
@@ -42,11 +53,19 @@ func convertLightningPrecheck(
 
 // LightningEmptyRegionChecker checks whether there are too many empty regions in the cluster.
 type LightningEmptyRegionChecker struct {
+<<<<<<< HEAD
 	inner restore.PrecheckItem
 }
 
 // NewLightningEmptyRegionChecker creates a new LightningEmptyRegionChecker.
 func NewLightningEmptyRegionChecker(lightningChecker restore.PrecheckItem) RealChecker {
+=======
+	inner precheck.Checker
+}
+
+// NewLightningEmptyRegionChecker creates a new LightningEmptyRegionChecker.
+func NewLightningEmptyRegionChecker(lightningChecker precheck.Checker) RealChecker {
+>>>>>>> 65217445e8 (makefile(all): add new cases to run_group and check cases in makefile (#8794))
 	return &LightningEmptyRegionChecker{inner: lightningChecker}
 }
 
@@ -74,11 +93,19 @@ func (c *LightningEmptyRegionChecker) Check(ctx context.Context) *Result {
 
 // LightningRegionDistributionChecker checks whether the region distribution is balanced.
 type LightningRegionDistributionChecker struct {
+<<<<<<< HEAD
 	inner restore.PrecheckItem
 }
 
 // NewLightningRegionDistributionChecker creates a new LightningRegionDistributionChecker.
 func NewLightningRegionDistributionChecker(lightningChecker restore.PrecheckItem) RealChecker {
+=======
+	inner precheck.Checker
+}
+
+// NewLightningRegionDistributionChecker creates a new LightningRegionDistributionChecker.
+func NewLightningRegionDistributionChecker(lightningChecker precheck.Checker) RealChecker {
+>>>>>>> 65217445e8 (makefile(all): add new cases to run_group and check cases in makefile (#8794))
 	return &LightningRegionDistributionChecker{inner: lightningChecker}
 }
 
@@ -106,11 +133,19 @@ func (c *LightningRegionDistributionChecker) Check(ctx context.Context) *Result
 
 // LightningClusterVersionChecker checks whether the cluster version is compatible with Lightning.
 type LightningClusterVersionChecker struct {
+<<<<<<< HEAD
 	inner restore.PrecheckItem
 }
 
 // NewLightningClusterVersionChecker creates a new LightningClusterVersionChecker.
 func NewLightningClusterVersionChecker(lightningChecker restore.PrecheckItem) RealChecker {
+=======
+	inner precheck.Checker
+}
+
+// NewLightningClusterVersionChecker creates a new LightningClusterVersionChecker.
+func NewLightningClusterVersionChecker(lightningChecker precheck.Checker) RealChecker {
+>>>>>>> 65217445e8 (makefile(all): add new cases to run_group and check cases in makefile (#8794))
 	return &LightningClusterVersionChecker{inner: lightningChecker}
 }
 
@@ -135,3 +170,110 @@ func (c *LightningClusterVersionChecker) Check(ctx context.Context) *Result {
 	)
 	return result
 }
+<<<<<<< HEAD
+=======
+
+// LightningFreeSpaceChecker checks whether the cluster has enough free space.
+type LightningFreeSpaceChecker struct {
+	sourceDataSize int64
+	infoGetter     importer.TargetInfoGetter
+}
+
+// NewLightningFreeSpaceChecker creates a new LightningFreeSpaceChecker.
+func NewLightningFreeSpaceChecker(sourceDataSize int64, getter importer.TargetInfoGetter) RealChecker {
+	return &LightningFreeSpaceChecker{
+		sourceDataSize: sourceDataSize,
+		infoGetter:     getter,
+	}
+}
+
+// Name implements the RealChecker interface.
+func (c *LightningFreeSpaceChecker) Name() string {
+	return "lightning_free_space"
+}
+
+// Check implements the RealChecker interface.
+func (c *LightningFreeSpaceChecker) Check(ctx context.Context) *Result {
+	result := &Result{
+		Name:  c.Name(),
+		Desc:  "check whether the downstream has enough free space to store the data to be migrated",
+		State: StateFailure,
+	}
+	storeInfo, err := c.infoGetter.GetStorageInfo(ctx)
+	if err != nil {
+		markCheckError(result, err)
+		return result
+	}
+	clusterAvail := uint64(0)
+	for _, store := range storeInfo.Stores {
+		clusterAvail += uint64(store.Status.Available)
+	}
+	if clusterAvail < uint64(c.sourceDataSize) {
+		result.State = StateFailure
+		result.Errors = append(result.Errors, &Error{
+			Severity: StateFailure,
+			ShortErr: fmt.Sprintf("Downstream doesn't have enough space, available is %s, but we need %s",
+				units.BytesSize(float64(clusterAvail)), units.BytesSize(float64(c.sourceDataSize))),
+		})
+		result.Instruction = "you can try to scale-out TiKV storage or TiKV instance to gain more storage space"
+		return result
+	}
+
+	replConfig, err := c.infoGetter.GetReplicationConfig(ctx)
+	if err != nil {
+		markCheckError(result, err)
+		return result
+	}
+	safeSize := uint64(c.sourceDataSize) * replConfig.MaxReplicas * 2
+	if clusterAvail < safeSize {
+		result.State = StateWarning
+		result.Errors = append(result.Errors, &Error{
+			Severity: StateWarning,
+			ShortErr: fmt.Sprintf("Cluster may not have enough space, available is %s, but we need %s",
+				units.BytesSize(float64(clusterAvail)), units.BytesSize(float64(safeSize))),
+		})
+		result.Instruction = "you can try to scale-out TiKV storage or TiKV instance to gain more storage space"
+		return result
+	}
+	result.State = StateSuccess
+	return result
+}
+
+// LightningCDCPiTRChecker checks whether the cluster has running CDC PiTR tasks.
+type LightningCDCPiTRChecker struct {
+	inner precheck.Checker
+}
+
+// NewLightningCDCPiTRChecker creates a new LightningCDCPiTRChecker.
+func NewLightningCDCPiTRChecker(lightningChecker precheck.Checker) RealChecker {
+	c, ok := lightningChecker.(*importer.CDCPITRCheckItem)
+	if ok {
+		c.Instruction = "physical import mode is not compatible with them. Please switch to logical import mode then try again."
+	} else {
+		log.L().DPanic("lightningChecker is not CDCPITRCheckItem")
+	}
+	return &LightningCDCPiTRChecker{inner: lightningChecker}
+}
+
+// Name implements the RealChecker interface.
+func (c *LightningCDCPiTRChecker) Name() string {
+	return "lightning_downstream_mutex_features"
+}
+
+// Check implements the RealChecker interface.
+func (c *LightningCDCPiTRChecker) Check(ctx context.Context) *Result {
+	result := &Result{
+		Name:  c.Name(),
+		Desc:  "check whether the downstream has tasks incompatible with physical import mode",
+		State: StateFailure,
+	}
+	convertLightningPrecheck(
+		ctx,
+		result,
+		c.inner,
+		StateFailure,
+		`you can switch to logical import mode which has no requirements on this`,
+	)
+	return result
+}
+>>>>>>> 65217445e8 (makefile(all): add new cases to run_group and check cases in makefile (#8794))

dm/pkg/checker/lightning_test.go

@@ -17,7 +17,11 @@ import (
 	"context"
 	"testing"
 
+<<<<<<< HEAD
 	"github.com/pingcap/tidb/br/pkg/lightning/restore"
+=======
+	"github.com/pingcap/tidb/br/pkg/lightning/precheck"
+>>>>>>> 65217445e8 (makefile(all): add new cases to run_group and check cases in makefile (#8794))
 	"github.com/pingcap/tiflow/pkg/errors"
 	"github.com/stretchr/testify/require"
 )
@@ -28,17 +32,29 @@ type mockPrecheckItem struct {
 	msg  string
 }
 
+<<<<<<< HEAD
 func (m mockPrecheckItem) Check(ctx context.Context) (*restore.CheckResult, error) {
 	if m.err != nil {
 		return nil, m.err
 	}
 	return &restore.CheckResult{
+=======
+func (m mockPrecheckItem) Check(ctx context.Context) (*precheck.CheckResult, error) {
+	if m.err != nil {
+		return nil, m.err
+	}
+	return &precheck.CheckResult{
+>>>>>>> 65217445e8 (makefile(all): add new cases to run_group and check cases in makefile (#8794))
 		Passed:  m.pass,
 		Message: m.msg,
 	}, nil
 }
 
+<<<<<<< HEAD
 func (m mockPrecheckItem) GetCheckItemID() restore.CheckItemID {
+=======
+func (m mockPrecheckItem) GetCheckItemID() precheck.CheckItemID {
+>>>>>>> 65217445e8 (makefile(all): add new cases to run_group and check cases in makefile (#8794))
 	return "mock"
 }
 

dm/tests/lightning_load_task/conf/diff_config1.toml

@@ -0,0 +1,35 @@
+# diff Configuration.
+
+check-thread-count = 4
+
+export-fix-sql = true
+
+check-struct-only = false
+
+[task]
+    output-dir = "/tmp/ticdc_dm_test/output"
+
+    source-instances = ["mysql1", "mysql2"]
+
+    target-instance = "tidb0"
+
+    target-check-tables = ["load_task1.t?*"]
+
+[data-sources]
+[data-sources.mysql1]
+host = "127.0.0.1"
+port = 3306
+user = "root"
+password = "123456"
+
+[data-sources.mysql2]
+host = "127.0.0.1"
+port = 3307
+user = "root"
+password = "123456"
+
+[data-sources.tidb0]
+host = "127.0.0.1"
+port = 4000
+user = "test"
+password = "123456"