From 0a471962d1a18c98b042a75db101f79053663b34 Mon Sep 17 00:00:00 2001 From: buchuitoudegou <756541536@qq.com> Date: Thu, 7 Apr 2022 11:01:59 +0800 Subject: [PATCH 1/5] fix: rows count exceeds definition panic --- br/pkg/lightning/backend/tidb/tidb.go | 10 ++++++++++ br/pkg/lightning/backend/tidb/tidb_test.go | 9 +++++++++ 2 files changed, 19 insertions(+) diff --git a/br/pkg/lightning/backend/tidb/tidb.go b/br/pkg/lightning/backend/tidb/tidb.go index e7afb91b4a9fb..73d8c3ecd25d6 100644 --- a/br/pkg/lightning/backend/tidb/tidb.go +++ b/br/pkg/lightning/backend/tidb/tidb.go @@ -308,11 +308,21 @@ func (enc *tidbEncoder) Encode(logger log.Logger, row []types.Datum, _ int64, co // See: tests/generated_columns/data/gencol.various_types.0.sql this sql has no columns, so encodeLoop will fill the // column permutation with default, thus enc.columnCnt > len(row). if len(row) < enc.columnCnt { + // 1. if len(row) < enc.columnCnt: data in row cannot populate the insert statement, because + // there are enc.columnCnt elements to insert but fewer columns in row logger.Error("column count mismatch", zap.Ints("column_permutation", columnPermutation), zap.Array("data", kv.RowArrayMarshaler(row))) return emptyTiDBRow, errors.Errorf("column count mismatch, expected %d, got %d", enc.columnCnt, len(row)) } + if len(row) > len(enc.columnIdx) { + // 2. if len(row) > len(columnIdx): raw row data has more columns than those + // in the table + logger.Error("column count mismatch", zap.Ints("column_count", enc.columnIdx), + zap.Array("data", kv.RowArrayMarshaler(row))) + return emptyTiDBRow, errors.Errorf("column count mismatch, at most %d but got %d", len(enc.columnIdx), len(row)) + } + var encoded strings.Builder encoded.Grow(8 * len(row)) encoded.WriteByte('(') diff --git a/br/pkg/lightning/backend/tidb/tidb_test.go b/br/pkg/lightning/backend/tidb/tidb_test.go index e31ef43bd29e5..85cf6be99cf57 100644 --- a/br/pkg/lightning/backend/tidb/tidb_test.go +++ b/br/pkg/lightning/backend/tidb/tidb_test.go @@ -19,6 +19,7 @@ import ( "database/sql" "database/sql/driver" "fmt" + "regexp" "testing" "github.com/DATA-DOG/go-sqlmock" @@ -594,6 +595,14 @@ func encodeRowsTiDB(t *testing.T, b backend.Backend, tbl table.Table) kv.Rows { require.NoError(t, err) row.ClassifyAndAppend(&dataRows, &dataChecksum, &indexRows, &indexChecksum) + + rawRow := make([]types.Datum, 0) + for i := 0; i < 15; i++ { + rawRow = append(rawRow, types.NewIntDatum(0)) + } + row, err = encoder.Encode(logger, rawRow, 1, []int{0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, "12.csv", 0) + require.NotNil(t, err) + require.Regexp(t, regexp.MustCompile(".*column count mismatch, at most.*"), err.Error()) return dataRows } From 50f35c5fe14b5f57d5f2902c1a8af1cd4b265850 Mon Sep 17 00:00:00 2001 From: buchuitoudegou <756541536@qq.com> Date: Thu, 7 Apr 2022 13:50:11 +0800 Subject: [PATCH 2/5] fix: comment --- br/pkg/lightning/backend/tidb/tidb_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/br/pkg/lightning/backend/tidb/tidb_test.go b/br/pkg/lightning/backend/tidb/tidb_test.go index 85cf6be99cf57..5b3d06a88c0f2 100644 --- a/br/pkg/lightning/backend/tidb/tidb_test.go +++ b/br/pkg/lightning/backend/tidb/tidb_test.go @@ -19,7 +19,6 @@ import ( "database/sql" "database/sql/driver" "fmt" - "regexp" "testing" "github.com/DATA-DOG/go-sqlmock" @@ -602,7 +601,7 @@ func encodeRowsTiDB(t *testing.T, b backend.Backend, tbl table.Table) kv.Rows { } row, err = encoder.Encode(logger, rawRow, 1, []int{0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, "12.csv", 0) require.NotNil(t, err) - require.Regexp(t, regexp.MustCompile(".*column count mismatch, at most.*"), err.Error()) + require.Contains(t, err.Error(), "column count mismatch, at most") return dataRows } From 0293a9922ccf6b6863cdccb89e384857130f422f Mon Sep 17 00:00:00 2001 From: buchuitoudegou <756541536@qq.com> Date: Fri, 8 Apr 2022 10:26:24 +0800 Subject: [PATCH 3/5] fix: preallocate slice with capacity --- br/pkg/lightning/backend/tidb/tidb_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/br/pkg/lightning/backend/tidb/tidb_test.go b/br/pkg/lightning/backend/tidb/tidb_test.go index 5b3d06a88c0f2..8fa50fd489e9d 100644 --- a/br/pkg/lightning/backend/tidb/tidb_test.go +++ b/br/pkg/lightning/backend/tidb/tidb_test.go @@ -595,7 +595,7 @@ func encodeRowsTiDB(t *testing.T, b backend.Backend, tbl table.Table) kv.Rows { row.ClassifyAndAppend(&dataRows, &dataChecksum, &indexRows, &indexChecksum) - rawRow := make([]types.Datum, 0) + rawRow := make([]types.Datum, 14) for i := 0; i < 15; i++ { rawRow = append(rawRow, types.NewIntDatum(0)) } From 46f79113bf1d5c6b9579215832f90510bce79b87 Mon Sep 17 00:00:00 2001 From: buchuitoudegou <756541536@qq.com> Date: Wed, 13 Apr 2022 11:21:09 +0800 Subject: [PATCH 4/5] fix: comment --- br/pkg/lightning/backend/tidb/tidb_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/br/pkg/lightning/backend/tidb/tidb_test.go b/br/pkg/lightning/backend/tidb/tidb_test.go index 8fa50fd489e9d..5b3d06a88c0f2 100644 --- a/br/pkg/lightning/backend/tidb/tidb_test.go +++ b/br/pkg/lightning/backend/tidb/tidb_test.go @@ -595,7 +595,7 @@ func encodeRowsTiDB(t *testing.T, b backend.Backend, tbl table.Table) kv.Rows { row.ClassifyAndAppend(&dataRows, &dataChecksum, &indexRows, &indexChecksum) - rawRow := make([]types.Datum, 14) + rawRow := make([]types.Datum, 0) for i := 0; i < 15; i++ { rawRow = append(rawRow, types.NewIntDatum(0)) } From c72180ee5611f17064db5c21fa18a156421f063f Mon Sep 17 00:00:00 2001 From: buchuitoudegou <756541536@qq.com> Date: Wed, 13 Apr 2022 15:39:20 +0800 Subject: [PATCH 5/5] fix: it error --- br/tests/lightning_routes/run.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/br/tests/lightning_routes/run.sh b/br/tests/lightning_routes/run.sh index efe801dc829d8..1db0ce2035021 100755 --- a/br/tests/lightning_routes/run.sh +++ b/br/tests/lightning_routes/run.sh @@ -21,5 +21,3 @@ check_contains 'sum(x): 1296' run_sql 'SHOW TABLES IN routes_a1;' check_not_contains 'Tables_in_routes_a1: t2' -run_sql 'SHOW DATABASES;' -check_not_contains 'Database: routes_a0'