add noschema back to keep compatibility

pkg/lightning/config/config.go

@@ -232,15 +232,17 @@ type CSVConfig struct {
 }
 
 type MydumperRuntime struct {
-	ReadBlockSize    ByteSize            `toml:"read-block-size" json:"read-block-size"`
-	BatchSize        ByteSize            `toml:"batch-size" json:"batch-size"`
-	BatchImportRatio float64             `toml:"batch-import-ratio" json:"batch-import-ratio"`
-	SourceDir        string              `toml:"data-source-dir" json:"data-source-dir"`
-	CharacterSet     string              `toml:"character-set" json:"character-set"`
-	CSV              CSVConfig           `toml:"csv" json:"csv"`
-	MaxRegionSize    ByteSize            `toml:"max-region-size" json:"max-region-size"`
-	Filter           []string            `toml:"filter" json:"filter"`
-	FileRouters      []*FileRouteRule    `toml:"files" json:"files"`
+	ReadBlockSize    ByteSize         `toml:"read-block-size" json:"read-block-size"`
+	BatchSize        ByteSize         `toml:"batch-size" json:"batch-size"`
+	BatchImportRatio float64          `toml:"batch-import-ratio" json:"batch-import-ratio"`
+	SourceDir        string           `toml:"data-source-dir" json:"data-source-dir"`
+	CharacterSet     string           `toml:"character-set" json:"character-set"`
+	CSV              CSVConfig        `toml:"csv" json:"csv"`
+	MaxRegionSize    ByteSize         `toml:"max-region-size" json:"max-region-size"`
+	Filter           []string         `toml:"filter" json:"filter"`
+	FileRouters      []*FileRouteRule `toml:"files" json:"files"`
+	// Deprecated: only used to keep the compatibility.
+	NoSchema         bool                `toml:"no-schema" json:"no-schema"`
 	CaseSensitive    bool                `toml:"case-sensitive" json:"case-sensitive"`
 	StrictFormat     bool                `toml:"strict-format" json:"strict-format"`
 	DefaultFileRules bool                `toml:"default-file-rules" json:"default-file-rules"`
@@ -409,6 +411,7 @@ func (cfg *Config) LoadFromGlobal(global *GlobalConfig) error {
 	cfg.TiDB.Psw = global.TiDB.Psw
 	cfg.TiDB.StatusPort = global.TiDB.StatusPort
 	cfg.TiDB.PdAddr = global.TiDB.PdAddr
+	cfg.Mydumper.NoSchema = global.Mydumper.NoSchema
 	cfg.Mydumper.SourceDir = global.Mydumper.SourceDir
 	cfg.Mydumper.Filter = global.Mydumper.Filter
 	cfg.TikvImporter.Addr = global.TikvImporter.Addr

pkg/lightning/config/global.go

@@ -51,7 +51,9 @@ type GlobalTiDB struct {
 }
 
 type GlobalMydumper struct {
-	SourceDir     string           `toml:"data-source-dir" json:"data-source-dir"`
+	SourceDir string `toml:"data-source-dir" json:"data-source-dir"`
+	// Deprecated
+	NoSchema      bool             `toml:"no-schema" json:"no-schema"`
 	Filter        []string         `toml:"filter" json:"filter"`
 	IgnoreColumns []*IgnoreColumns `toml:"ignore-columns" json:"ignore-columns"`
 }
@@ -155,6 +157,7 @@ func LoadGlobalConfig(args []string, extraFlags func(*flag.FlagSet)) (*GlobalCon
 	backend := flagext.ChoiceVar(fs, "backend", "", `delivery backend: importer, tidb, local (default importer)`, "", "importer", "tidb", "local")
 	sortedKVDir := fs.String("sorted-kv-dir", "", "path for KV pairs when local backend enabled")
 	enableCheckpoint := fs.Bool("enable-checkpoint", true, "whether to enable checkpoints")
+	noSchema := fs.Bool("no-schema", false, "ignore schema files, get schema directly from TiDB instead")
 	checksum := flagext.ChoiceVar(fs, "checksum", "", "compare checksum after importing.", "", "required", "optional", "off", "true", "false")
 	analyze := flagext.ChoiceVar(fs, "analyze", "", "analyze table after importing", "", "required", "optional", "off", "true", "false")
 	checkRequirements := fs.Bool("check-requirements", true, "check cluster version before starting")
@@ -243,6 +246,9 @@ func LoadGlobalConfig(args []string, extraFlags func(*flag.FlagSet)) (*GlobalCon
 	if !*enableCheckpoint {
 		cfg.Checkpoint.Enable = false
 	}
+	if *noSchema {
+		cfg.Mydumper.NoSchema = true
+	}
 	if *checksum != "" {
 		_ = cfg.PostRestore.Checksum.FromStringValue(*checksum)
 	}

pkg/lightning/restore/check_info.go

@@ -483,6 +483,8 @@ func (rc *Controller) SchemaIsValid(ctx context.Context, tableInfo *md.MDTableMe
 			for _, col := range colsFromTiDB {
 				colMap[col] = struct{}{}
 			}
+			// tidb_rowid can be ignored in check
+			colMap[model.ExtraHandleName.String()] = struct{}{}
 			for _, col := range colsFromDataFile {
 				if _, ok := colMap[col]; !ok {
 					msgs = append(msgs, fmt.Sprintf("TiDB schema `%s`.`%s` doesn't have column %s, "+

pkg/lightning/restore/restore.go

@@ -2116,12 +2116,18 @@ func (rc *Controller) DataCheck(ctx context.Context) error {
 	}
 	if len(checkPointCriticalMsgs) != 0 {
 		rc.checkTemplate.Collect(Critical, false, strings.Join(checkPointCriticalMsgs, ","))
+	} else {
+		rc.checkTemplate.Collect(Critical, true, "checkpoints are valid")
 	}
 	if len(dataCriticalMsgs) != 0 {
 		rc.checkTemplate.Collect(Critical, false, strings.Join(dataCriticalMsgs, ","))
+	} else {
+		rc.checkTemplate.Collect(Critical, true, "table has no data before import")
 	}
 	if len(schemaCriticalMsgs) != 0 {
 		rc.checkTemplate.Collect(Critical, false, strings.Join(schemaCriticalMsgs, ","))
+	} else {
+		rc.checkTemplate.Collect(Critical, true, "table schemas are valid")
 	}
 	return nil
 }

tests/lightning_no_schema/run.sh

@@ -16,15 +16,15 @@
 set -eu
 
 run_sql "DROP DATABASE IF EXISTS noschema;"
-echo yes | run_lightning -d "tests/$TEST_NAME/schema-data"
+echo yes | run_lightning --no-schema=1 -d "tests/$TEST_NAME/schema-data"
 run_sql "show databases"
 check_not_contains "noschema"
 
 run_sql "create database noschema;"
 run_sql "create table noschema.t (x int primary key);"
 
 # Starting importing
-echo yes | run_lightning 
+echo yes | run_lightning --no-schema=1
 
 run_sql "SELECT sum(x) FROM noschema.t;"
 check_contains 'sum(x): 120'