diff --git a/go.mod b/go.mod index f91b95ba..f52e7262 100644 --- a/go.mod +++ b/go.mod @@ -22,9 +22,6 @@ require ( github.com/pingcap/tidb-tools v4.0.9-0.20201127090955-2707c97b3853+incompatible github.com/prometheus/client_golang v1.5.1 github.com/prometheus/client_model v0.2.0 - github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b // indirect - github.com/siddontang/go-log v0.0.0-20190221022429-1e957dd83bed // indirect - github.com/siddontang/go-mysql v0.0.0-20200222075837-12e89848f047 github.com/soheilhy/cmux v0.1.4 github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index 61581c3d..c79de3a1 100644 --- a/go.sum +++ b/go.sum @@ -444,7 +444,6 @@ github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIf github.com/pingcap/log v0.0.0-20201112100606-8f1e84a3abc8/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4 h1:ERrF0fTuIOnwfGbt71Ji3DKbOEaP189tjym50u8gpC8= github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= -github.com/pingcap/parser v0.0.0-20190506092653-e336082eb825/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/parser v0.0.0-20210513020953-ae2c4497c07b h1:eLuDQ6eJCEKCbGwhGrkjzagwev1GJGU2Y2kFkAsBzV0= github.com/pingcap/parser v0.0.0-20210513020953-ae2c4497c07b/go.mod h1:xZC8I7bug4GJ5KtHhgAikjTfU4kBv1Sbo3Pf1MZ6lVw= github.com/pingcap/sysutil v0.0.0-20200206130906-2bfa6dc40bcd/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= @@ -455,7 +454,6 @@ github.com/pingcap/tidb v1.1.0-beta.0.20210517044538-8ad868f801fc/go.mod h1:MTGi github.com/pingcap/tidb-dashboard v0.0.0-20210312062513-eef5d6404638/go.mod h1:OzFN8H0EDMMqeulPhPMw2i2JaiZWOKFQ7zdRPhENNgo= github.com/pingcap/tidb-tools v4.0.9-0.20201127090955-2707c97b3853+incompatible h1:ceznmu/lLseGHP/jKyOa/3u/5H3wtLLLqkH2V3ssSjg= github.com/pingcap/tidb-tools v4.0.9-0.20201127090955-2707c97b3853+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= -github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= github.com/pingcap/tipb v0.0.0-20210422074242-57dd881b81b1 h1:Kcp3jIcQrqG+pT1JQ0oWyRncVKQtDgnMFzRt3zJBaBo= github.com/pingcap/tipb v0.0.0-20210422074242-57dd881b81b1/go.mod h1:nsEhnMokcn7MRqd2J60yxpn/ac3ZH8A6GOJ9NslabUo= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -504,30 +502,19 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= -github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shirou/gopsutil v2.19.10+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil v3.21.2+incompatible h1:U+YvJfjCh6MslYlIAXvPtzhW3YZEtc9uncueUNpD/0A= github.com/shirou/gopsutil v3.21.2+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= -github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 h1:bUGsEnyNbVPw06Bs80sCeARAlK8lhwqGyi6UT8ymuGk= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd h1:ug7PpSOB5RBPK1Kg6qskGBoP3Vnj/aNYFTznWvlkGo0= github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= -github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 h1:xT+JlYxNGqyT+XcU8iUrN18JYed2TvG9yN5ULG2jATM= -github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw= -github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07/go.mod h1:yFdBgwXP24JziuRl2NMUahT7nGLNOKi1SIiFxMttVD4= -github.com/siddontang/go-log v0.0.0-20190221022429-1e957dd83bed h1:KMgQoLJGCq1IoZpLZE3AIffh9veYWoVlsvA4ib55TMM= -github.com/siddontang/go-log v0.0.0-20190221022429-1e957dd83bed/go.mod h1:yFdBgwXP24JziuRl2NMUahT7nGLNOKi1SIiFxMttVD4= -github.com/siddontang/go-mysql v0.0.0-20200222075837-12e89848f047 h1:boyJ8EgQN/aC3grvx8QUoJrptt7RvneezSJSCbW25a4= -github.com/siddontang/go-mysql v0.0.0-20200222075837-12e89848f047/go.mod h1:+W4RCzesQDI11HvIkaDjS8yM36SpAnGNQ7jmTLn5BnU= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= diff --git a/v4/export/dump.go b/v4/export/dump.go index 618db9a4..f771e728 100755 --- a/v4/export/dump.go +++ b/v4/export/dump.go @@ -213,12 +213,6 @@ func (d *Dumper) Dump() (dumpErr error) { } }) - // get estimate total count - err = d.getEstimateTotalRowsCount(tctx, metaConn) - if err != nil { - tctx.L().Error("fail to get estimate total count", zap.Error(err)) - } - if conf.SQL == "" { if err = d.dumpDatabases(writerCtx, metaConn, taskChan); err != nil && !errors.ErrorEqual(err, context.Canceled) { return err @@ -338,6 +332,12 @@ func (d *Dumper) dumpTableData(tctx *tcontext.Context, conn *sql.Conn, meta Tabl if conf.NoData { return nil } + + // Update total rows + fieldName, _ := pickupPossibleField(meta, conn) + c := estimateCount(tctx, meta.DatabaseName(), meta.TableName(), conn, fieldName, conf) + AddCounter(estimateTotalRowsCounter, conf.Labels, float64(c)) + if conf.Rows == UnspecifiedSize { return d.sequentialDumpTable(tctx, conn, meta, taskChan) } @@ -451,11 +451,9 @@ func (d *Dumper) concurrentDumpTable(tctx *tcontext.Context, conn *sql.Conn, met (conf.ServerInfo.ServerVersion.Compare(*tableSampleVersion) >= 0 || (conf.ServerInfo.HasTiKV && conf.ServerInfo.ServerVersion.Compare(*gcSafePointVersion) >= 0)) { err := d.concurrentDumpTiDBTables(tctx, conn, meta, taskChan) - if err == nil { - return err - } - // don't retry on context error - if errors.ErrorEqual(err, context.Canceled) || errors.ErrorEqual(err, context.DeadlineExceeded) { + // don't retry on context error and successful tasks + if errors.ErrorEqual(err, context.Canceled) || errors.ErrorEqual(err, context.DeadlineExceeded) || + err == nil { return err } tctx.L().Warn("fallback to concurrent dump tables using rows due to tidb error", @@ -582,7 +580,7 @@ func (d *Dumper) selectMinAndMaxIntValue(conn *sql.Conn, db, tbl, field string) } func (d *Dumper) concurrentDumpTiDBTables(tctx *tcontext.Context, conn *sql.Conn, meta TableMeta, taskChan chan<- Task) error { - db, tbl, hasImplicitRowID := meta.DatabaseName(), meta.TableName(), meta.HasImplicitRowID() + db, tbl := meta.DatabaseName(), meta.TableName() var ( handleColNames []string @@ -592,7 +590,7 @@ func (d *Dumper) concurrentDumpTiDBTables(tctx *tcontext.Context, conn *sql.Conn if d.conf.ServerInfo.ServerVersion.Compare(*tableSampleVersion) >= 0 { tctx.L().Debug("dumping TiDB tables with TABLESAMPLE", zap.String("database", db), zap.String("table", tbl)) - handleColNames, handleVals, err = selectTiDBTableSample(tctx, conn, db, tbl, hasImplicitRowID) + handleColNames, handleVals, err = selectTiDBTableSample(tctx, conn, meta) } else { tctx.L().Debug("dumping TiDB tables with TABLE REGIONS", zap.String("database", db), zap.String("table", tbl)) @@ -600,7 +598,7 @@ func (d *Dumper) concurrentDumpTiDBTables(tctx *tcontext.Context, conn *sql.Conn partitions, err = GetPartitionNames(conn, db, tbl) if err == nil { if len(partitions) == 0 { - handleColNames, handleVals, err = selectTiDBTableRegion(tctx, conn, db, tbl, hasImplicitRowID) + handleColNames, handleVals, err = selectTiDBTableRegion(tctx, conn, meta) } else { return d.concurrentDumpTiDBPartitionTables(tctx, conn, meta, taskChan, partitions) } @@ -613,7 +611,7 @@ func (d *Dumper) concurrentDumpTiDBTables(tctx *tcontext.Context, conn *sql.Conn } func (d *Dumper) concurrentDumpTiDBPartitionTables(tctx *tcontext.Context, conn *sql.Conn, meta TableMeta, taskChan chan<- Task, partitions []string) error { - db, tbl, hasImplicitRowID := meta.DatabaseName(), meta.TableName(), meta.HasImplicitRowID() + db, tbl := meta.DatabaseName(), meta.TableName() tctx.L().Debug("dumping TiDB tables with TABLE REGIONS for partition table", zap.String("database", db), zap.String("table", tbl), zap.Strings("partitions", partitions)) @@ -621,7 +619,7 @@ func (d *Dumper) concurrentDumpTiDBPartitionTables(tctx *tcontext.Context, conn totalChunk := 0 cachedHandleVals := make([][][]string, len(partitions)) - handleColNames, _, err := selectTiDBRowKeyFields(conn, db, tbl, hasImplicitRowID, checkTiDBTableRegionPkFields) + handleColNames, _, err := selectTiDBRowKeyFields(conn, meta, checkTiDBTableRegionPkFields) if err != nil { return err } @@ -676,13 +674,13 @@ func (d *Dumper) L() log.Logger { return d.tctx.L() } -func selectTiDBTableSample(tctx *tcontext.Context, conn *sql.Conn, dbName, tableName string, hasImplicitRowID bool) (pkFields []string, pkVals [][]string, err error) { - pkFields, pkColTypes, err := selectTiDBRowKeyFields(conn, dbName, tableName, hasImplicitRowID, nil) +func selectTiDBTableSample(tctx *tcontext.Context, conn *sql.Conn, meta TableMeta) (pkFields []string, pkVals [][]string, err error) { + pkFields, pkColTypes, err := selectTiDBRowKeyFields(conn, meta, nil) if err != nil { return nil, nil, errors.Trace(err) } - query := buildTiDBTableSampleQuery(pkFields, dbName, tableName) + query := buildTiDBTableSampleQuery(pkFields, meta.DatabaseName(), meta.TableName()) rows, err := conn.QueryContext(tctx, query) if err != nil { return nil, nil, errors.Trace(err) @@ -721,11 +719,11 @@ func buildTiDBTableSampleQuery(pkFields []string, dbName, tblName string) string return fmt.Sprintf(template, pks, escapeString(dbName), escapeString(tblName), pks) } -func selectTiDBRowKeyFields(conn *sql.Conn, dbName, tableName string, hasImplicitRowID bool, checkPkFields func([]string, []string) error) (pkFields, pkColTypes []string, err error) { - if hasImplicitRowID { +func selectTiDBRowKeyFields(conn *sql.Conn, meta TableMeta, checkPkFields func([]string, []string) error) (pkFields, pkColTypes []string, err error) { + if meta.HasImplicitRowID() { pkFields, pkColTypes = []string{"_tidb_rowid"}, []string{"BIGINT"} } else { - pkFields, pkColTypes, err = GetPrimaryKeyAndColumnTypes(conn, dbName, tableName) + pkFields, pkColTypes, err = GetPrimaryKeyAndColumnTypes(conn, meta) if err == nil { if checkPkFields != nil { err = checkPkFields(pkFields, pkColTypes) @@ -746,8 +744,8 @@ func checkTiDBTableRegionPkFields(pkFields, pkColTypes []string) (err error) { return } -func selectTiDBTableRegion(tctx *tcontext.Context, conn *sql.Conn, dbName, tableName string, hasImplicitRowID bool) (pkFields []string, pkVals [][]string, err error) { - pkFields, _, err = selectTiDBRowKeyFields(conn, dbName, tableName, hasImplicitRowID, checkTiDBTableRegionPkFields) +func selectTiDBTableRegion(tctx *tcontext.Context, conn *sql.Conn, meta TableMeta) (pkFields []string, pkVals [][]string, err error) { + pkFields, _, err = selectTiDBRowKeyFields(conn, meta, checkTiDBTableRegionPkFields) if err != nil { return } @@ -760,6 +758,7 @@ func selectTiDBTableRegion(tctx *tcontext.Context, conn *sql.Conn, dbName, table tableRegionSQL = "SELECT START_KEY,tidb_decode_key(START_KEY) from INFORMATION_SCHEMA.TIKV_REGION_STATUS s WHERE s.DB_NAME = ? AND s.TABLE_NAME = ? AND IS_INDEX = 0 ORDER BY START_KEY;" tidbRowID = "_tidb_rowid=" ) + dbName, tableName := meta.DatabaseName(), meta.TableName() logger := tctx.L().With(zap.String("database", dbName), zap.String("table", tableName)) err = simpleQueryWithArgs(conn, func(rows *sql.Rows) error { rowID++ diff --git a/v4/export/dump_test.go b/v4/export/dump_test.go index f166513d..edfc16c3 100644 --- a/v4/export/dump_test.go +++ b/v4/export/dump_test.go @@ -59,9 +59,6 @@ func (s *testSQLSuite) TestDumpTableMeta(c *C) { c.Assert(err, IsNil) defer db.Close() - database := "foo" - table := "bar" - tctx, cancel := tcontext.Background().WithLogger(appLogger).WithCancel() defer cancel() conn, err := db.Conn(tctx) diff --git a/v4/export/sql.go b/v4/export/sql.go index b67e7c8c..f1b56381 100644 --- a/v4/export/sql.go +++ b/v4/export/sql.go @@ -160,14 +160,17 @@ func ListAllDatabasesTables(tctx *tcontext.Context, db *sql.Conn, databaseNames for _, schema := range databaseNames { dbTables[schema] = make([]*TableInfo, 0) } - if err := simpleQueryWithArgs(db, func(rows *sql.Rows) error { - var sqlAvgRowLength sql.NullInt64 - if err := rows.Scan(&schema, &table, &tableTypeStr, &sqlAvgRowLength); err != nil { - return errors.Trace(err) + if err = simpleQueryWithArgs(db, func(rows *sql.Rows) error { + var ( + sqlAvgRowLength sql.NullInt64 + err2 error + ) + if err2 = rows.Scan(&schema, &table, &tableTypeStr, &sqlAvgRowLength); err != nil { + return errors.Trace(err2) } - tableType, err = ParseTableType(tableTypeStr) - if err != nil { - return errors.Trace(err) + tableType, err2 = ParseTableType(tableTypeStr) + if err2 != nil { + return errors.Trace(err2) } if sqlAvgRowLength.Valid { @@ -185,7 +188,7 @@ func ListAllDatabasesTables(tctx *tcontext.Context, db *sql.Conn, databaseNames } } else { queryTemplate := "SHOW TABLE STATUS FROM `%s`" - selectedTableType := make(map[TableType]struct{}, 0) + selectedTableType := make(map[TableType]struct{}) for _, tableType = range tableTypes { selectedTableType[tableType] = struct{}{} } @@ -345,63 +348,47 @@ func GetColumnTypes(db *sql.Conn, fields, database, table string) ([]*sql.Column } // GetPrimaryKeyAndColumnTypes gets all primary columns and their types in ordinal order -func GetPrimaryKeyAndColumnTypes(conn *sql.Conn, database, table string) ([]string, []string, error) { - query := - `SELECT c.COLUMN_NAME, DATA_TYPE FROM -INFORMATION_SCHEMA.COLUMNS c INNER JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE k ON -c.column_name = k.column_name and -c.table_schema = k.table_schema and -c.table_name = k.table_name and -c.table_schema = ? and -c.table_name = ? -WHERE COLUMN_KEY = 'PRI' -ORDER BY k.ORDINAL_POSITION;` - var colNames, colTypes []string - if err := simpleQueryWithArgs(conn, func(rows *sql.Rows) error { - var colName, colType string - if err := rows.Scan(&colName, &colType); err != nil { - return errors.Trace(err) - } - colNames = append(colNames, colName) - colTypes = append(colTypes, strings.ToUpper(colType)) - return nil - }, query, database, table); err != nil { - return nil, nil, errors.Annotatef(err, "sql: %s", query) +func GetPrimaryKeyAndColumnTypes(conn *sql.Conn, meta TableMeta) ([]string, []string, error) { + var ( + colNames, colTypes []string + err error + ) + colNames, err = GetPrimaryKeyColumns(conn, meta.DatabaseName(), meta.TableName()) + if err != nil { + return nil, nil, err + } + colName2Type := string2Map(meta.ColumnNames(), meta.ColumnTypes()) + colTypes = make([]string, len(colNames)) + for i, colName := range colNames { + colTypes[i] = colName2Type[colName] } return colNames, colTypes, nil } // GetPrimaryKeyColumns gets all primary columns in ordinal order func GetPrimaryKeyColumns(db *sql.Conn, database, table string) ([]string, error) { - priKeyColsQuery := "SELECT column_name FROM information_schema.KEY_COLUMN_USAGE " + - "WHERE table_schema = ? AND table_name = ? AND CONSTRAINT_NAME = 'PRIMARY' order by ORDINAL_POSITION;" - rows, err := db.QueryContext(context.Background(), priKeyColsQuery, database, table) + priKeyColsQuery := fmt.Sprintf("SHOW INDEX FROM `%s`.`%s`", escapeString(database), escapeString(table)) + rows, err := db.QueryContext(context.Background(), priKeyColsQuery) if err != nil { return nil, errors.Annotatef(err, "sql: %s", priKeyColsQuery) } - defer rows.Close() - var cols []string - var col string - for rows.Next() { - err = rows.Scan(&col) - if err != nil { - return nil, errors.Annotatef(err, "sql: %s", priKeyColsQuery) - } - cols = append(cols, col) - } - if err = rows.Err(); err != nil { + results, err := GetSpecifiedColumnValuesAndClose(rows, "KEY_NAME", "COLUMN_NAME") + if err != nil { return nil, errors.Annotatef(err, "sql: %s", priKeyColsQuery) } + cols := make([]string, 0, len(results)) + for _, oneRow := range results { + keyName, columnName := oneRow[0], oneRow[1] + if keyName == "PRIMARY" { + cols = append(cols, columnName) + } + } return cols, nil } func getNumericIndex(db *sql.Conn, meta TableMeta) (string, error) { database, table := meta.DatabaseName(), meta.TableName() - colNames, colTypes := meta.ColumnNames(), meta.ColumnTypes() - colName2Type := make(map[string]string, len(colNames)) - for i := range colNames { - colName2Type[colNames[i]] = colTypes[i] - } + colName2Type := string2Map(meta.ColumnNames(), meta.ColumnTypes()) keyQuery := fmt.Sprintf("SHOW INDEX FROM `%s`.`%s`", escapeString(database), escapeString(table)) rows, err := db.QueryContext(context.Background(), keyQuery) if err != nil { diff --git a/v4/export/sql_test.go b/v4/export/sql_test.go index d60bfafc..4b6a0f42 100644 --- a/v4/export/sql_test.go +++ b/v4/export/sql_test.go @@ -16,7 +16,15 @@ import ( "github.com/pingcap/errors" ) -var _ = Suite(&testSQLSuite{}) +var ( + _ = Suite(&testSQLSuite{}) + showIndexHeaders = []string{"Table", "Non_unique", "Key_name", "Seq_in_index", "Column_name", "Collation", "Cardinality", "Sub_part", "Packed", "Null", "Index_type", "Comment", "Index_comment"} +) + +const ( + database = "foo" + table = "bar" +) type testSQLSuite struct{} @@ -72,33 +80,33 @@ func (s *testSQLSuite) TestBuildSelectAllQuery(c *C) { // Test TiDB server. mockConf.ServerInfo.ServerType = ServerTypeTiDB - orderByClause, err := buildOrderByClause(mockConf, conn, "test", "t", true) + orderByClause, err := buildOrderByClause(mockConf, conn, database, table, true) c.Assert(err, IsNil) mock.ExpectQuery("SHOW COLUMNS FROM"). WillReturnRows(sqlmock.NewRows([]string{"Field", "Type", "Null", "Key", "Default", "Extra"}). AddRow("id", "int(11)", "NO", "PRI", nil, "")) - selectedField, _, err := buildSelectField(conn, "test", "t", false) + selectedField, _, err := buildSelectField(conn, database, table, false) c.Assert(err, IsNil) - q := buildSelectQuery("test", "t", selectedField, "", "", orderByClause) - c.Assert(q, Equals, "SELECT * FROM `test`.`t` ORDER BY `_tidb_rowid`") + q := buildSelectQuery(database, table, selectedField, "", "", orderByClause) + c.Assert(q, Equals, fmt.Sprintf("SELECT * FROM `%s`.`%s` ORDER BY `_tidb_rowid`", database, table)) - mock.ExpectQuery("SELECT column_name FROM information_schema.KEY_COLUMN_USAGE"). - WithArgs("test", "t"). - WillReturnRows(sqlmock.NewRows([]string{"column_name"}).AddRow("id")) + mock.ExpectQuery(fmt.Sprintf("SHOW INDEX FROM `%s`.`%s`", database, table)). + WillReturnRows(sqlmock.NewRows(showIndexHeaders). + AddRow(table, 0, "PRIMARY", 1, "id", "A", 0, nil, nil, "", "BTREE", "", "")) - orderByClause, err = buildOrderByClause(mockConf, conn, "test", "t", false) + orderByClause, err = buildOrderByClause(mockConf, conn, database, table, false) c.Assert(err, IsNil) mock.ExpectQuery("SHOW COLUMNS FROM"). WillReturnRows(sqlmock.NewRows([]string{"Field", "Type", "Null", "Key", "Default", "Extra"}). AddRow("id", "int(11)", "NO", "PRI", nil, "")) - selectedField, _, err = buildSelectField(conn, "test", "t", false) + selectedField, _, err = buildSelectField(conn, database, table, false) c.Assert(err, IsNil) - q = buildSelectQuery("test", "t", selectedField, "", "", orderByClause) - c.Assert(q, Equals, "SELECT * FROM `test`.`t` ORDER BY `id`") + q = buildSelectQuery(database, table, selectedField, "", "", orderByClause) + c.Assert(q, Equals, fmt.Sprintf("SELECT * FROM `%s`.`%s` ORDER BY `id`", database, table)) c.Assert(mock.ExpectationsWereMet(), IsNil) // Test other servers. @@ -108,20 +116,20 @@ func (s *testSQLSuite) TestBuildSelectAllQuery(c *C) { for _, serverTp := range otherServers { mockConf.ServerInfo.ServerType = serverTp cmt := Commentf("server type: %s", serverTp) - mock.ExpectQuery("SELECT column_name FROM information_schema.KEY_COLUMN_USAGE"). - WithArgs("test", "t"). - WillReturnRows(sqlmock.NewRows([]string{"column_name"}).AddRow("id")) - orderByClause, err := buildOrderByClause(mockConf, conn, "test", "t", false) + mock.ExpectQuery(fmt.Sprintf("SHOW INDEX FROM `%s`.`%s`", database, table)). + WillReturnRows(sqlmock.NewRows(showIndexHeaders). + AddRow(table, 0, "PRIMARY", 1, "id", "A", 0, nil, nil, "", "BTREE", "", "")) + orderByClause, err := buildOrderByClause(mockConf, conn, database, table, false) c.Assert(err, IsNil, cmt) mock.ExpectQuery("SHOW COLUMNS FROM"). WillReturnRows(sqlmock.NewRows([]string{"Field", "Type", "Null", "Key", "Default", "Extra"}). AddRow("id", "int(11)", "NO", "PRI", nil, "")) - selectedField, _, err = buildSelectField(conn, "test", "t", false) + selectedField, _, err = buildSelectField(conn, database, table, false) c.Assert(err, IsNil) - q = buildSelectQuery("test", "t", selectedField, "", "", orderByClause) - c.Assert(q, Equals, "SELECT * FROM `test`.`t` ORDER BY `id`", cmt) + q = buildSelectQuery(database, table, selectedField, "", "", orderByClause) + c.Assert(q, Equals, fmt.Sprintf("SELECT * FROM `%s`.`%s` ORDER BY `id`", database, table), cmt) err = mock.ExpectationsWereMet() c.Assert(err, IsNil, cmt) c.Assert(mock.ExpectationsWereMet(), IsNil, cmt) @@ -131,11 +139,10 @@ func (s *testSQLSuite) TestBuildSelectAllQuery(c *C) { for _, serverTp := range otherServers { mockConf.ServerInfo.ServerType = serverTp cmt := Commentf("server type: %s", serverTp) - mock.ExpectQuery("SELECT column_name FROM information_schema.KEY_COLUMN_USAGE"). - WithArgs("test", "t"). - WillReturnRows(sqlmock.NewRows([]string{"column_name"})) + mock.ExpectQuery(fmt.Sprintf("SHOW INDEX FROM `%s`.`%s`", database, table)). + WillReturnRows(sqlmock.NewRows(showIndexHeaders)) - orderByClause, err := buildOrderByClause(mockConf, conn, "test", "t", false) + orderByClause, err := buildOrderByClause(mockConf, conn, database, table, false) c.Assert(err, IsNil, cmt) mock.ExpectQuery("SHOW COLUMNS FROM"). @@ -144,8 +151,8 @@ func (s *testSQLSuite) TestBuildSelectAllQuery(c *C) { selectedField, _, err = buildSelectField(conn, "test", "t", false) c.Assert(err, IsNil) - q := buildSelectQuery("test", "t", selectedField, "", "", orderByClause) - c.Assert(q, Equals, "SELECT * FROM `test`.`t`", cmt) + q := buildSelectQuery(database, table, selectedField, "", "", orderByClause) + c.Assert(q, Equals, fmt.Sprintf("SELECT * FROM `%s`.`%s`", database, table), cmt) err = mock.ExpectationsWereMet() c.Assert(err, IsNil, cmt) c.Assert(mock.ExpectationsWereMet(), IsNil) @@ -163,8 +170,8 @@ func (s *testSQLSuite) TestBuildSelectAllQuery(c *C) { selectedField, _, err := buildSelectField(conn, "test", "t", false) c.Assert(err, IsNil) - q := buildSelectQuery("test", "t", selectedField, "", "", "") - c.Assert(q, Equals, "SELECT * FROM `test`.`t`", cmt) + q := buildSelectQuery(database, table, selectedField, "", "", "") + c.Assert(q, Equals, fmt.Sprintf("SELECT * FROM `%s`.`%s`", database, table), cmt) c.Assert(mock.ExpectationsWereMet(), IsNil, cmt) } } @@ -182,40 +189,38 @@ func (s *testSQLSuite) TestBuildOrderByClause(c *C) { // Test TiDB server. mockConf.ServerInfo.ServerType = ServerTypeTiDB - orderByClause, err := buildOrderByClause(mockConf, conn, "test", "t", true) + orderByClause, err := buildOrderByClause(mockConf, conn, database, table, true) c.Assert(err, IsNil) c.Assert(orderByClause, Equals, orderByTiDBRowID) - mock.ExpectQuery("SELECT column_name FROM information_schema.KEY_COLUMN_USAGE"). - WithArgs("test", "t"). - WillReturnRows(sqlmock.NewRows([]string{"column_name"}).AddRow("id")) + mock.ExpectQuery(fmt.Sprintf("SHOW INDEX FROM `%s`.`%s`", database, table)). + WillReturnRows(sqlmock.NewRows(showIndexHeaders).AddRow(table, 0, "PRIMARY", 1, "id", "A", 0, nil, nil, "", "BTREE", "", "")) - orderByClause, err = buildOrderByClause(mockConf, conn, "test", "t", false) + orderByClause, err = buildOrderByClause(mockConf, conn, database, table, false) c.Assert(err, IsNil) c.Assert(orderByClause, Equals, "ORDER BY `id`") // Test table with primary key. - mock.ExpectQuery("SELECT column_name FROM information_schema.KEY_COLUMN_USAGE"). - WithArgs("test", "t"). - WillReturnRows(sqlmock.NewRows([]string{"column_name"}).AddRow("id")) - orderByClause, err = buildOrderByClause(mockConf, conn, "test", "t", false) + mock.ExpectQuery(fmt.Sprintf("SHOW INDEX FROM `%s`.`%s`", database, table)). + WillReturnRows(sqlmock.NewRows(showIndexHeaders).AddRow(table, 0, "PRIMARY", 1, "id", "A", 0, nil, nil, "", "BTREE", "", "")) + orderByClause, err = buildOrderByClause(mockConf, conn, database, table, false) c.Assert(err, IsNil) c.Assert(orderByClause, Equals, "ORDER BY `id`") // Test table with joint primary key. - mock.ExpectQuery("SELECT column_name FROM information_schema.KEY_COLUMN_USAGE"). - WithArgs("test", "t"). - WillReturnRows(sqlmock.NewRows([]string{"column_name"}).AddRow("id").AddRow("name")) - orderByClause, err = buildOrderByClause(mockConf, conn, "test", "t", false) + mock.ExpectQuery(fmt.Sprintf("SHOW INDEX FROM `%s`.`%s`", database, table)). + WillReturnRows(sqlmock.NewRows(showIndexHeaders). + AddRow(table, 0, "PRIMARY", 1, "id", "A", 0, nil, nil, "", "BTREE", "", ""). + AddRow(table, 0, "PRIMARY", 2, "name", "A", 0, nil, nil, "", "BTREE", "", "")) + orderByClause, err = buildOrderByClause(mockConf, conn, database, table, false) c.Assert(err, IsNil) c.Assert(orderByClause, Equals, "ORDER BY `id`,`name`") // Test table without primary key. - mock.ExpectQuery("SELECT column_name FROM information_schema.KEY_COLUMN_USAGE"). - WithArgs("test", "t"). - WillReturnRows(sqlmock.NewRows([]string{"column_name"})) + mock.ExpectQuery(fmt.Sprintf("SHOW INDEX FROM `%s`.`%s`", database, table)). + WillReturnRows(sqlmock.NewRows(showIndexHeaders)) - orderByClause, err = buildOrderByClause(mockConf, conn, "test", "t", false) + orderByClause, err = buildOrderByClause(mockConf, conn, database, table, false) c.Assert(err, IsNil) c.Assert(orderByClause, Equals, "") @@ -224,7 +229,7 @@ func (s *testSQLSuite) TestBuildOrderByClause(c *C) { for _, hasImplicitRowID := range []bool{false, true} { cmt := Commentf("current hasImplicitRowID: ", hasImplicitRowID) - orderByClause, err := buildOrderByClause(mockConf, conn, "test", "t", hasImplicitRowID) + orderByClause, err := buildOrderByClause(mockConf, conn, database, table, hasImplicitRowID) c.Assert(err, IsNil, cmt) c.Assert(orderByClause, Equals, "", cmt) } @@ -399,11 +404,6 @@ func (s *testSQLSuite) TestBuildTableSampleQueries(c *C) { ServerVersion: tableSampleVersion, } - const ( - database = "foo" - table = "bar" - ) - testCases := []struct { handleColNames []string handleColTypes []string @@ -420,7 +420,7 @@ func (s *testSQLSuite) TestBuildTableSampleQueries(c *C) { }, { []string{"a"}, - []string{"bigint"}, + []string{"BIGINT"}, [][]driver.Value{{1}}, []string{"`a`<1", "`a`>=1"}, false, @@ -428,7 +428,7 @@ func (s *testSQLSuite) TestBuildTableSampleQueries(c *C) { // check whether dumpling can turn to dump whole table { []string{"a"}, - []string{"bigint"}, + []string{"BIGINT"}, [][]driver.Value{}, nil, false, @@ -436,21 +436,21 @@ func (s *testSQLSuite) TestBuildTableSampleQueries(c *C) { // check whether dumpling can turn to dump whole table { []string{"_tidb_rowid"}, - []string{"bigint"}, + []string{"BIGINT"}, [][]driver.Value{}, nil, true, }, { []string{"_tidb_rowid"}, - []string{"bigint"}, + []string{"BIGINT"}, [][]driver.Value{{1}}, []string{"`_tidb_rowid`<1", "`_tidb_rowid`>=1"}, true, }, { []string{"a"}, - []string{"bigint"}, + []string{"BIGINT"}, [][]driver.Value{ {1}, {2}, @@ -461,14 +461,14 @@ func (s *testSQLSuite) TestBuildTableSampleQueries(c *C) { }, { []string{"a", "b"}, - []string{"bigint", "bigint"}, + []string{"BIGINT", "BIGINT"}, [][]driver.Value{{1, 2}}, []string{"`a`<1 or(`a`=1 and `b`<2)", "`a`>1 or(`a`=1 and `b`>=2)"}, false, }, { []string{"a", "b"}, - []string{"bigint", "bigint"}, + []string{"BIGINT", "BIGINT"}, [][]driver.Value{ {1, 2}, {3, 4}, @@ -484,7 +484,7 @@ func (s *testSQLSuite) TestBuildTableSampleQueries(c *C) { }, { []string{"a", "b", "c"}, - []string{"bigint", "bigint", "bigint"}, + []string{"BIGINT", "BIGINT", "BIGINT"}, [][]driver.Value{ {1, 2, 3}, {4, 5, 6}, @@ -498,7 +498,7 @@ func (s *testSQLSuite) TestBuildTableSampleQueries(c *C) { }, { []string{"a", "b", "c"}, - []string{"bigint", "bigint", "bigint"}, + []string{"BIGINT", "BIGINT", "BIGINT"}, [][]driver.Value{ {1, 2, 3}, {1, 4, 5}, @@ -512,7 +512,7 @@ func (s *testSQLSuite) TestBuildTableSampleQueries(c *C) { }, { []string{"a", "b", "c"}, - []string{"bigint", "bigint", "bigint"}, + []string{"BIGINT", "BIGINT", "BIGINT"}, [][]driver.Value{ {1, 2, 3}, {1, 2, 8}, @@ -527,7 +527,7 @@ func (s *testSQLSuite) TestBuildTableSampleQueries(c *C) { // special case: avoid return same samples { []string{"a", "b", "c"}, - []string{"bigint", "bigint", "bigint"}, + []string{"BIGINT", "BIGINT", "BIGINT"}, [][]driver.Value{ {1, 2, 3}, {1, 2, 3}, @@ -542,7 +542,7 @@ func (s *testSQLSuite) TestBuildTableSampleQueries(c *C) { // special case: numbers has bigger lexicographically order but lower number { []string{"a", "b", "c"}, - []string{"bigint", "bigint", "bigint"}, + []string{"BIGINT", "BIGINT", "BIGINT"}, [][]driver.Value{ {12, 2, 3}, {111, 4, 5}, @@ -557,7 +557,7 @@ func (s *testSQLSuite) TestBuildTableSampleQueries(c *C) { // test string fields { []string{"a", "b", "c"}, - []string{"bigint", "bigint", "varchar"}, + []string{"BIGINT", "BIGINT", "varchar"}, [][]driver.Value{ {1, 2, "3"}, {1, 4, "5"}, @@ -571,7 +571,7 @@ func (s *testSQLSuite) TestBuildTableSampleQueries(c *C) { }, { []string{"a", "b", "c", "d"}, - []string{"bigint", "bigint", "bigint", "bigint"}, + []string{"BIGINT", "BIGINT", "BIGINT", "BIGINT"}, [][]driver.Value{ {1, 2, 3, 4}, {5, 6, 7, 8}, @@ -625,22 +625,24 @@ func (s *testSQLSuite) TestBuildTableSampleQueries(c *C) { quotaCols = append(quotaCols, wrapBackTicks(col)) } selectFields := strings.Join(quotaCols, ",") - meta := &tableMeta{ - database: database, - table: table, + meta := &mockTableIR{ + dbName: database, + tblName: table, selectedField: selectFields, hasImplicitRowID: testCase.hasTiDBRowID, - specCmts: []string{ + colTypes: handleColTypes, + colNames: handleColNames, + specCmt: []string{ "/*!40101 SET NAMES binary*/;", }, } if !testCase.hasTiDBRowID { - rows := sqlmock.NewRows([]string{"COLUMN_NAME", "DATA_TYPE"}) - for i := range handleColNames { - rows.AddRow(handleColNames[i], handleColTypes[i]) + rows := sqlmock.NewRows(showIndexHeaders) + for i, handleColName := range handleColNames { + rows.AddRow(table, 0, "PRIMARY", i, handleColName, "A", 0, nil, nil, "", "BTREE", "", "") } - mock.ExpectQuery("SELECT c.COLUMN_NAME, DATA_TYPE FROM").WithArgs(database, table).WillReturnRows(rows) + mock.ExpectQuery(fmt.Sprintf("SHOW INDEX FROM `%s`.`%s`", database, table)).WillReturnRows(rows) } rows := sqlmock.NewRows(handleColNames) @@ -651,12 +653,12 @@ func (s *testSQLSuite) TestBuildTableSampleQueries(c *C) { // special case, no enough value to split chunks if len(handleVals) == 0 { if !testCase.hasTiDBRowID { - rows = sqlmock.NewRows([]string{"COLUMN_NAME"}) - for i := range handleColNames { - rows.AddRow(handleColNames[i]) + rows = sqlmock.NewRows(showIndexHeaders) + for i, handleColName := range handleColNames { + rows.AddRow(table, 0, "PRIMARY", i, handleColName, "A", 0, nil, nil, "", "BTREE", "", "") } - mock.ExpectQuery("SELECT column_name FROM information_schema.KEY_COLUMN_USAGE WHERE table_schema").WithArgs(database, table).WillReturnRows(rows) - mock.ExpectQuery("SHOW INDEX FROM").WillReturnRows(sqlmock.NewRows([]string{"Table", "Non_unique", "Key_name", "Seq_in_index", "Column_name", "Collation", "Cardinality", "Sub_part", "Packed", "Null", "Index_type", "Comment", "Index_comment"})) + mock.ExpectQuery(fmt.Sprintf("SHOW INDEX FROM `%s`.`%s`", database, table)).WillReturnRows(rows) + mock.ExpectQuery("SHOW INDEX FROM").WillReturnRows(sqlmock.NewRows(showIndexHeaders)) } else { d.conf.Rows = 200000 mock.ExpectQuery("EXPLAIN SELECT `_tidb_rowid`"). @@ -786,7 +788,7 @@ func (s *testSQLSuite) TestBuildRegionQueriesWithoutPartition(c *C) { {"7480000000000000FF3300000000000000F8", "7480000000000000FF3300000000000000F8"}, }, []string{"a"}, - []string{"bigint"}, + []string{"BIGINT"}, []string{ "", }, @@ -797,7 +799,7 @@ func (s *testSQLSuite) TestBuildRegionQueriesWithoutPartition(c *C) { {"7480000000000000FF3300000000000000F8", "7480000000000000FF3300000000000000F8"}, }, []string{"_tidb_rowid"}, - []string{"bigint"}, + []string{"BIGINT"}, []string{ "", }, @@ -811,7 +813,7 @@ func (s *testSQLSuite) TestBuildRegionQueriesWithoutPartition(c *C) { {"7480000000000000FF335F728000000000FF2BF2010000000000FA", "tableID=51, _tidb_rowid=2880001"}, }, []string{"a"}, - []string{"bigint"}, + []string{"BIGINT"}, []string{ "`a`<960001", "`a`>=960001 and `a`<1920001", @@ -830,7 +832,7 @@ func (s *testSQLSuite) TestBuildRegionQueriesWithoutPartition(c *C) { {"7480000000000000FF335F728000000000FF2BF2010000000000FA", "tableID=51, _tidb_rowid=2880001"}, }, []string{"_tidb_rowid"}, - []string{"bigint"}, + []string{"BIGINT"}, []string{ "`_tidb_rowid`<960001", "`_tidb_rowid`>=960001 and `_tidb_rowid`<1920001", @@ -849,13 +851,15 @@ func (s *testSQLSuite) TestBuildRegionQueriesWithoutPartition(c *C) { // Test build tasks through table region taskChan := make(chan Task, 128) - meta := &tableMeta{ - database: database, - table: table, + meta := &mockTableIR{ + dbName: database, + tblName: table, selectedField: "*", selectedLen: len(handleColNames), hasImplicitRowID: testCase.hasTiDBRowID, - specCmts: []string{ + colTypes: handleColTypes, + colNames: handleColNames, + specCmt: []string{ "/*!40101 SET NAMES binary*/;", }, } @@ -864,11 +868,11 @@ func (s *testSQLSuite) TestBuildRegionQueriesWithoutPartition(c *C) { WithArgs(database, table).WillReturnRows(sqlmock.NewRows([]string{"PARTITION_NAME"}).AddRow(nil)) if !testCase.hasTiDBRowID { - rows := sqlmock.NewRows([]string{"COLUMN_NAME", "DATA_TYPE"}) - for i := range handleColNames { - rows.AddRow(handleColNames[i], handleColTypes[i]) + rows := sqlmock.NewRows(showIndexHeaders) + for i, handleColName := range handleColNames { + rows.AddRow(table, 0, "PRIMARY", i, handleColName, "A", 0, nil, nil, "", "BTREE", "", "") } - mock.ExpectQuery("SELECT c.COLUMN_NAME, DATA_TYPE FROM").WithArgs(database, table).WillReturnRows(rows) + mock.ExpectQuery(fmt.Sprintf("SHOW INDEX FROM `%s`.`%s`", database, table)).WillReturnRows(rows) } rows := sqlmock.NewRows([]string{"START_KEY", "tidb_decode_key(START_KEY)"}) @@ -881,12 +885,12 @@ func (s *testSQLSuite) TestBuildRegionQueriesWithoutPartition(c *C) { orderByClause := buildOrderByClauseString(handleColNames) // special case, no enough value to split chunks if !testCase.hasTiDBRowID && len(regionResults) <= 1 { - rows = sqlmock.NewRows([]string{"COLUMN_NAME"}) - for i := range handleColNames { - rows.AddRow(handleColNames[i]) + rows = sqlmock.NewRows(showIndexHeaders) + for i, handleColName := range handleColNames { + rows.AddRow(table, 0, "PRIMARY", i, handleColName, "A", 0, nil, nil, "", "BTREE", "", "") } - mock.ExpectQuery("SELECT column_name FROM information_schema.KEY_COLUMN_USAGE WHERE table_schema").WithArgs(database, table).WillReturnRows(rows) - mock.ExpectQuery("SHOW INDEX FROM").WillReturnRows(sqlmock.NewRows([]string{"Table", "Non_unique", "Key_name", "Seq_in_index", "Column_name", "Collation", "Cardinality", "Sub_part", "Packed", "Null", "Index_type", "Comment", "Index_comment"})) + mock.ExpectQuery(fmt.Sprintf("SHOW INDEX FROM `%s`.`%s`", database, table)).WillReturnRows(rows) + mock.ExpectQuery("SHOW INDEX FROM").WillReturnRows(sqlmock.NewRows(showIndexHeaders)) } c.Assert(d.concurrentDumpTable(tctx, conn, meta, taskChan), IsNil) c.Assert(mock.ExpectationsWereMet(), IsNil) @@ -922,8 +926,6 @@ func (s *testSQLSuite) TestBuildRegionQueriesWithPartitions(c *C) { ServerType: ServerTypeTiDB, ServerVersion: gcSafePointVersion, } - database := "foo" - table := "bar" partitions := []string{"p0", "p1", "p2"} testCases := []struct { @@ -950,7 +952,7 @@ func (s *testSQLSuite) TestBuildRegionQueriesWithPartitions(c *C) { }, }, []string{"_tidb_rowid"}, - []string{"bigint"}, + []string{"BIGINT"}, [][]string{ {""}, {""}, {""}, }, @@ -979,7 +981,7 @@ func (s *testSQLSuite) TestBuildRegionQueriesWithPartitions(c *C) { }, }, []string{"_tidb_rowid"}, - []string{"bigint"}, + []string{"BIGINT"}, [][]string{ { "`_tidb_rowid`<10001", @@ -1016,7 +1018,7 @@ func (s *testSQLSuite) TestBuildRegionQueriesWithPartitions(c *C) { }, }, []string{"a"}, - []string{"bigint"}, + []string{"BIGINT"}, [][]string{ { @@ -1045,13 +1047,15 @@ func (s *testSQLSuite) TestBuildRegionQueriesWithPartitions(c *C) { // Test build tasks through table region taskChan := make(chan Task, 128) - meta := &tableMeta{ - database: database, - table: table, + meta := &mockTableIR{ + dbName: database, + tblName: table, selectedField: "*", selectedLen: len(handleColNames), hasImplicitRowID: testCase.hasTiDBRowID, - specCmts: []string{ + colTypes: handleColTypes, + colNames: handleColNames, + specCmt: []string{ "/*!40101 SET NAMES binary*/;", }, } @@ -1064,11 +1068,11 @@ func (s *testSQLSuite) TestBuildRegionQueriesWithPartitions(c *C) { WithArgs(database, table).WillReturnRows(rows) if !testCase.hasTiDBRowID { - rows = sqlmock.NewRows([]string{"COLUMN_NAME", "DATA_TYPE"}) - for i := range handleColNames { - rows.AddRow(handleColNames[i], handleColTypes[i]) + rows = sqlmock.NewRows(showIndexHeaders) + for i, handleColName := range handleColNames { + rows.AddRow(table, 0, "PRIMARY", i, handleColName, "A", 0, nil, nil, "", "BTREE", "", "") } - mock.ExpectQuery("SELECT c.COLUMN_NAME, DATA_TYPE FROM").WithArgs(database, table).WillReturnRows(rows) + mock.ExpectQuery(fmt.Sprintf("SHOW INDEX FROM `%s`.`%s`", database, table)).WillReturnRows(rows) } for i, partition := range partitions { diff --git a/v4/export/status.go b/v4/export/status.go index b54aff46..7c368783 100644 --- a/v4/export/status.go +++ b/v4/export/status.go @@ -3,7 +3,6 @@ package export import ( - "database/sql" "fmt" "time" @@ -60,18 +59,3 @@ func calculateTableCount(m DatabaseTables) int { } return cnt } - -func (d *Dumper) getEstimateTotalRowsCount(tctx *tcontext.Context, conn *sql.Conn) error { - conf := d.conf - var totalCount uint64 - for db, tables := range conf.Tables { - for _, m := range tables { - if m.Type == TableTypeBase { - c := estimateCount(tctx, db, m.Name, conn, "", conf) - totalCount += c - } - } - } - AddCounter(estimateTotalRowsCounter, conf.Labels, float64(totalCount)) - return nil -} diff --git a/v4/export/test_util.go b/v4/export/test_util.go index d9a8f2cb..ec06b06c 100644 --- a/v4/export/test_util.go +++ b/v4/export/test_util.go @@ -68,9 +68,9 @@ type mockTableIR struct { colTypes []string colNames []string escapeBackSlash bool + hasImplicitRowID bool rowErr error rows *sql.Rows - hasImplicitRowID bool SQLRowIter } diff --git a/v4/export/util.go b/v4/export/util.go index 6df177c9..5f29119c 100644 --- a/v4/export/util.go +++ b/v4/export/util.go @@ -66,3 +66,11 @@ func sameStringArray(a, b []string) bool { } return true } + +func string2Map(a, b []string) map[string]string { + a2b := make(map[string]string, len(a)) + for i, str := range a { + a2b[str] = b[i] + } + return a2b +} diff --git a/v4/export/writer_util_test.go b/v4/export/writer_util_test.go index 62b06826..4ee88405 100644 --- a/v4/export/writer_util_test.go +++ b/v4/export/writer_util_test.go @@ -50,6 +50,7 @@ func (s *testWriteSuite) SetUpSuite(_ *C) { } InitMetricsVector(s.mockCfg.Labels) } + func (s *testWriteSuite) TearDownTest(c *C) { RemoveLabelValuesWithTaskInMetrics(s.mockCfg.Labels)