From ce5fa5f86eb17760da837c2c562fea772dc59e52 Mon Sep 17 00:00:00 2001 From: WangXiangUSTC Date: Thu, 22 Aug 2019 00:44:09 +0800 Subject: [PATCH] backend: update mysql backend to tidb backend (#228) * update mysql backend to tidb backend * fix test --- lightning/backend/{mysql.go => tidb.go} | 64 +++++++++---------- .../backend/{mysql_test.go => tidb_test.go} | 2 +- lightning/config/config.go | 6 +- lightning/restore/restore.go | 4 +- tests/csv/run.sh | 2 +- tests/various_types/run.sh | 2 +- tidb-lightning.toml | 2 +- 7 files changed, 41 insertions(+), 41 deletions(-) rename lightning/backend/{mysql.go => tidb.go} (75%) rename lightning/backend/{mysql_test.go => tidb_test.go} (98%) diff --git a/lightning/backend/mysql.go b/lightning/backend/tidb.go similarity index 75% rename from lightning/backend/mysql.go rename to lightning/backend/tidb.go index 776dd6821..1ab858ccd 100644 --- a/lightning/backend/mysql.go +++ b/lightning/backend/tidb.go @@ -32,34 +32,34 @@ import ( "github.com/pingcap/tidb-lightning/lightning/verification" ) -type mysqlRow string +type tidbRow string -type mysqlRows []mysqlRow +type tidbRows []tidbRow -type mysqlEncoder struct { +type tidbEncoder struct { mode mysql.SQLMode } -type mysqlBackend struct { +type tidbBackend struct { db *sql.DB } -// NewMySQLBackend creates a new MySQL backend using the given database. +// NewTiDBBackend creates a new TiDB backend using the given database. // // The backend does not take ownership of `db`. Caller should close `db` // manually after the backend expired. -func NewMySQLBackend(db *sql.DB) Backend { - return MakeBackend(&mysqlBackend{db: db}) +func NewTiDBBackend(db *sql.DB) Backend { + return MakeBackend(&tidbBackend{db: db}) } -func (row mysqlRow) ClassifyAndAppend(data *Rows, checksum *verification.KVChecksum, _ *Rows, _ *verification.KVChecksum) { - rows := (*data).(mysqlRows) - *data = mysqlRows(append(rows, row)) +func (row tidbRow) ClassifyAndAppend(data *Rows, checksum *verification.KVChecksum, _ *Rows, _ *verification.KVChecksum) { + rows := (*data).(tidbRows) + *data = tidbRows(append(rows, row)) cs := verification.MakeKVChecksum(uint64(len(row)), 1, 0) checksum.Add(&cs) } -func (rows mysqlRows) SplitIntoChunks(splitSize int) []Rows { +func (rows tidbRows) SplitIntoChunks(splitSize int) []Rows { if len(rows) == 0 { return nil } @@ -80,11 +80,11 @@ func (rows mysqlRows) SplitIntoChunks(splitSize int) []Rows { return append(res, rows[i:]) } -func (rows mysqlRows) Clear() Rows { +func (rows tidbRows) Clear() Rows { return rows[:0] } -func (enc mysqlEncoder) appendSQLBytes(sb *strings.Builder, value []byte) { +func (enc tidbEncoder) appendSQLBytes(sb *strings.Builder, value []byte) { sb.Grow(2 + len(value)) sb.WriteByte('\'') if enc.mode.HasNoBackslashEscapesMode() { @@ -124,7 +124,7 @@ func (enc mysqlEncoder) appendSQLBytes(sb *strings.Builder, value []byte) { // appendSQL appends the SQL representation of the Datum into the string builder. // Note that we cannot use Datum.ToString since it doesn't perform SQL escaping. -func (enc mysqlEncoder) appendSQL(sb *strings.Builder, datum *types.Datum) error { +func (enc tidbEncoder) appendSQL(sb *strings.Builder, datum *types.Datum) error { switch datum.Kind() { case types.KindNull: sb.WriteString("NULL") @@ -192,9 +192,9 @@ func (enc mysqlEncoder) appendSQL(sb *strings.Builder, datum *types.Datum) error return nil } -func (mysqlEncoder) Close() {} +func (tidbEncoder) Close() {} -func (enc mysqlEncoder) Encode(logger log.Logger, row []types.Datum, _ int64, _ []int) (Row, error) { +func (enc tidbEncoder) Encode(logger log.Logger, row []types.Datum, _ int64, _ []int) (Row, error) { var encoded strings.Builder encoded.Grow(8 * len(row)) encoded.WriteByte('(') @@ -203,7 +203,7 @@ func (enc mysqlEncoder) Encode(logger log.Logger, row []types.Datum, _ int64, _ encoded.WriteByte(',') } if err := enc.appendSQL(&encoded, &field); err != nil { - logger.Error("mysql encode failed", + logger.Error("tidb encode failed", zap.Array("original", rowArrayMarshaler(row)), zap.Int("originalCol", i), log.ShortError(err), @@ -212,52 +212,52 @@ func (enc mysqlEncoder) Encode(logger log.Logger, row []types.Datum, _ int64, _ } } encoded.WriteByte(')') - return mysqlRow(encoded.String()), nil + return tidbRow(encoded.String()), nil } -func (be *mysqlBackend) Close() { +func (be *tidbBackend) Close() { // *Not* going to close `be.db`. The db object is normally borrowed from a // TidbManager, so we let the manager to close it. } -func (be *mysqlBackend) MakeEmptyRows() Rows { - return mysqlRows(nil) +func (be *tidbBackend) MakeEmptyRows() Rows { + return tidbRows(nil) } -func (be *mysqlBackend) RetryImportDelay() time.Duration { +func (be *tidbBackend) RetryImportDelay() time.Duration { return 0 } -func (be *mysqlBackend) MaxChunkSize() int { +func (be *tidbBackend) MaxChunkSize() int { return 1048576 } -func (be *mysqlBackend) ShouldPostProcess() bool { +func (be *tidbBackend) ShouldPostProcess() bool { return false } -func (be *mysqlBackend) NewEncoder(_ table.Table, mode mysql.SQLMode) Encoder { - return mysqlEncoder{mode: mode} +func (be *tidbBackend) NewEncoder(_ table.Table, mode mysql.SQLMode) Encoder { + return tidbEncoder{mode: mode} } -func (be *mysqlBackend) OpenEngine(context.Context, uuid.UUID) error { +func (be *tidbBackend) OpenEngine(context.Context, uuid.UUID) error { return nil } -func (be *mysqlBackend) CloseEngine(context.Context, uuid.UUID) error { +func (be *tidbBackend) CloseEngine(context.Context, uuid.UUID) error { return nil } -func (be *mysqlBackend) CleanupEngine(context.Context, uuid.UUID) error { +func (be *tidbBackend) CleanupEngine(context.Context, uuid.UUID) error { return nil } -func (be *mysqlBackend) ImportEngine(context.Context, uuid.UUID) error { +func (be *tidbBackend) ImportEngine(context.Context, uuid.UUID) error { return nil } -func (be *mysqlBackend) WriteRows(ctx context.Context, _ uuid.UUID, tableName string, columnNames []string, _ uint64, r Rows) error { - rows := r.(mysqlRows) +func (be *tidbBackend) WriteRows(ctx context.Context, _ uuid.UUID, tableName string, columnNames []string, _ uint64, r Rows) error { + rows := r.(tidbRows) if len(rows) == 0 { return nil } diff --git a/lightning/backend/mysql_test.go b/lightning/backend/tidb_test.go similarity index 98% rename from lightning/backend/mysql_test.go rename to lightning/backend/tidb_test.go index 350072e31..77e875064 100644 --- a/lightning/backend/mysql_test.go +++ b/lightning/backend/tidb_test.go @@ -37,7 +37,7 @@ func (s *mysqlSuite) SetUpTest(c *C) { c.Assert(err, IsNil) s.mockDB = mock - s.backend = kv.NewMySQLBackend(db) + s.backend = kv.NewTiDBBackend(db) } func (s *mysqlSuite) TearDownTest(c *C) { diff --git a/lightning/config/config.go b/lightning/config/config.go index a51c5058f..8c0aeb53b 100644 --- a/lightning/config/config.go +++ b/lightning/config/config.go @@ -38,8 +38,8 @@ const ( // NormalMode defines mode of normal for tikv. NormalMode = "normal" - // BackendMySQL is a constant for choosing the "MySQL" backend in the configuration. - BackendMySQL = "mysql" + // BackendTiDB is a constant for choosing the "TiDB" backend in the configuration. + BackendTiDB = "tidb" // BackendImporter is a constant for choosing the "Importer" backend in the configuration. BackendImporter = "importer" @@ -308,7 +308,7 @@ func (cfg *Config) Adjust() error { cfg.TikvImporter.Backend = strings.ToLower(cfg.TikvImporter.Backend) switch cfg.TikvImporter.Backend { - case BackendMySQL, BackendImporter: + case BackendTiDB, BackendImporter: default: return errors.Errorf("invalid config: unsupported `tikv-importer.backend` (%s)", cfg.TikvImporter.Backend) } diff --git a/lightning/restore/restore.go b/lightning/restore/restore.go index fa8127b10..4f3fb24e8 100644 --- a/lightning/restore/restore.go +++ b/lightning/restore/restore.go @@ -171,8 +171,8 @@ func NewRestoreController(ctx context.Context, dbMetas []*mydump.MDDatabaseMeta, if err != nil { return nil, err } - case config.BackendMySQL: - backend = kv.NewMySQLBackend(tidbMgr.db) + case config.BackendTiDB: + backend = kv.NewTiDBBackend(tidbMgr.db) default: return nil, errors.New("unknown backend: " + cfg.TikvImporter.Backend) } diff --git a/tests/csv/run.sh b/tests/csv/run.sh index 8585f0163..94ab78563 100755 --- a/tests/csv/run.sh +++ b/tests/csv/run.sh @@ -2,7 +2,7 @@ set -eu -for BACKEND in importer mysql; do +for BACKEND in importer tidb; do run_sql 'DROP DATABASE IF EXISTS csv' diff --git a/tests/various_types/run.sh b/tests/various_types/run.sh index 787c172b3..53d2a74ea 100755 --- a/tests/various_types/run.sh +++ b/tests/various_types/run.sh @@ -17,7 +17,7 @@ set -eu -for BACKEND in importer mysql; do +for BACKEND in importer tidb; do run_sql 'DROP DATABASE IF EXISTS vt;' run_lightning config --backend $BACKEND diff --git a/tidb-lightning.toml b/tidb-lightning.toml index cb4c0ffe1..85270d61f 100644 --- a/tidb-lightning.toml +++ b/tidb-lightning.toml @@ -61,7 +61,7 @@ driver = "file" #keep-after-success = false [tikv-importer] -# Delivery backend, can be "importer" or "mysql". +# Delivery backend, can be "importer" or "tidb". backend = "importer" # Address of tikv-importer when the backend is 'importer' addr = "127.0.0.1:8287"