From 706f8a9d9c3a7911500286dc311145c595e177bd Mon Sep 17 00:00:00 2001 From: glorv Date: Wed, 16 Feb 2022 10:19:39 +0800 Subject: [PATCH] lightning: skip restore table when source files are empty (#28189) close pingcap/tidb#28144 --- br/pkg/lightning/restore/restore.go | 4 +- br/pkg/lightning/restore/table_restore.go | 16 +-- .../data/incr.empty_table-schema.sql | 1 + .../data1/incr.empty_table-schema.sql | 1 + .../data1/incr.empty_table2-schema.sql | 1 + br/tests/lightning_incremental/run.sh | 98 ++++++++++--------- 6 files changed, 59 insertions(+), 62 deletions(-) create mode 100644 br/tests/lightning_incremental/data/incr.empty_table-schema.sql create mode 100644 br/tests/lightning_incremental/data1/incr.empty_table-schema.sql create mode 100644 br/tests/lightning_incremental/data1/incr.empty_table2-schema.sql diff --git a/br/pkg/lightning/restore/restore.go b/br/pkg/lightning/restore/restore.go index ec6a9c94e6281..df888dd81bd24 100644 --- a/br/pkg/lightning/restore/restore.go +++ b/br/pkg/lightning/restore/restore.go @@ -1467,6 +1467,9 @@ func (rc *Controller) restoreTables(ctx context.Context) (finalErr error) { if err != nil { return errors.Trace(err) } + if cp.Status < checkpoints.CheckpointStatusAllWritten && len(tableMeta.DataFiles) == 0 { + continue + } igCols, err := rc.cfg.Mydumper.IgnoreColumns.GetIgnoreColumns(dbInfo.Name, tableInfo.Name, rc.cfg.Mydumper.CaseSensitive) if err != nil { return errors.Trace(err) @@ -1526,7 +1529,6 @@ func (tr *TableRestore) restoreTable( cp *checkpoints.TableCheckpoint, ) (bool, error) { // 1. Load the table info. - select { case <-ctx.Done(): return false, ctx.Err() diff --git a/br/pkg/lightning/restore/table_restore.go b/br/pkg/lightning/restore/table_restore.go index 056435d9af7cf..fa50c1179f939 100644 --- a/br/pkg/lightning/restore/table_restore.go +++ b/br/pkg/lightning/restore/table_restore.go @@ -202,17 +202,6 @@ func (tr *TableRestore) restoreEngines(pCtx context.Context, rc *Controller, cp tr.logger.Error("fail to restoreEngines because indexengine is nil") return errors.Errorf("table %v index engine checkpoint not found", tr.tableName) } - // If there is an index engine only, it indicates no data needs to restore. - // So we can change status to imported directly and avoid opening engine. - if len(cp.Engines) == 1 { - if err := rc.saveStatusCheckpoint(pCtx, tr.tableName, indexEngineID, nil, checkpoints.CheckpointStatusImported); err != nil { - return errors.Trace(err) - } - if err := rc.saveStatusCheckpoint(pCtx, tr.tableName, checkpoints.WholeTableEngineID, nil, checkpoints.CheckpointStatusIndexImported); err != nil { - return errors.Trace(err) - } - return nil - } ctx, cancel := context.WithCancel(pCtx) defer cancel() @@ -683,10 +672,7 @@ func (tr *TableRestore) postProcess( forcePostProcess bool, metaMgr tableMetaMgr, ) (bool, error) { - // there are no data in this table, no need to do post process - // this is important for tables that are just the dump table of views - // because at this stage, the table was already deleted and replaced by the related view - if !rc.backend.ShouldPostProcess() || len(cp.Engines) == 1 { + if !rc.backend.ShouldPostProcess() { return false, nil } diff --git a/br/tests/lightning_incremental/data/incr.empty_table-schema.sql b/br/tests/lightning_incremental/data/incr.empty_table-schema.sql new file mode 100644 index 0000000000000..881156cb99fd5 --- /dev/null +++ b/br/tests/lightning_incremental/data/incr.empty_table-schema.sql @@ -0,0 +1 @@ +CREATE TABLE `empty_table` (id int primary key); diff --git a/br/tests/lightning_incremental/data1/incr.empty_table-schema.sql b/br/tests/lightning_incremental/data1/incr.empty_table-schema.sql new file mode 100644 index 0000000000000..881156cb99fd5 --- /dev/null +++ b/br/tests/lightning_incremental/data1/incr.empty_table-schema.sql @@ -0,0 +1 @@ +CREATE TABLE `empty_table` (id int primary key); diff --git a/br/tests/lightning_incremental/data1/incr.empty_table2-schema.sql b/br/tests/lightning_incremental/data1/incr.empty_table2-schema.sql new file mode 100644 index 0000000000000..610412d940815 --- /dev/null +++ b/br/tests/lightning_incremental/data1/incr.empty_table2-schema.sql @@ -0,0 +1 @@ +CREATE TABLE `empty_table2` (id int primary key, s varchar(16)); diff --git a/br/tests/lightning_incremental/run.sh b/br/tests/lightning_incremental/run.sh index f04630055936a..4cdd5a53ec74b 100644 --- a/br/tests/lightning_incremental/run.sh +++ b/br/tests/lightning_incremental/run.sh @@ -18,60 +18,66 @@ set -eu check_cluster_version 4 0 0 "incremental restore" || exit 0 -DB_NAME=incr +run_lightning_and_check_meta() { + run_lightning --backend local "$@" + # check metadata table is not exist + run_sql "SHOW DATABASES like 'lightning_metadata';" + check_not_contains "Database: lightning_metadata" +} -for backend in importer local; do - run_sql "DROP DATABASE IF EXISTS incr;" - run_lightning --backend $backend +DB_NAME=incr - for tbl in auto_random pk_auto_inc rowid_uk_inc uk_auto_inc; do - run_sql "SELECT count(*) from incr.$tbl" - check_contains "count(*): 3" - done +run_sql "DROP DATABASE IF EXISTS incr;" +run_sql "DROP DATABASE IF EXISTS lightning_metadata;" +run_lightning_and_check_meta - for tbl in auto_random pk_auto_inc rowid_uk_inc uk_auto_inc; do - if [ "$tbl" = "auto_random" ]; then - run_sql "SELECT id & b'000001111111111111111111111111111111111111111111111111111111111' as inc FROM incr.$tbl" - else - run_sql "SELECT id as inc FROM incr.$tbl" - fi - check_contains 'inc: 1' - check_contains 'inc: 2' - check_contains 'inc: 3' - done +for tbl in auto_random pk_auto_inc rowid_uk_inc uk_auto_inc; do + run_sql "SELECT count(*) from incr.$tbl" + check_contains "count(*): 3" +done - for tbl in pk_auto_inc rowid_uk_inc; do - run_sql "SELECT group_concat(v) from incr.$tbl group by 'all';" - check_contains "group_concat(v): a,b,c" - done +for tbl in auto_random pk_auto_inc rowid_uk_inc uk_auto_inc; do + if [ "$tbl" = "auto_random" ]; then + run_sql "SELECT id & b'000001111111111111111111111111111111111111111111111111111111111' as inc FROM incr.$tbl" + else + run_sql "SELECT id as inc FROM incr.$tbl" + fi + check_contains 'inc: 1' + check_contains 'inc: 2' + check_contains 'inc: 3' +done - run_sql "SELECT sum(pk) from incr.uk_auto_inc;" - check_contains "sum(pk): 6" +for tbl in pk_auto_inc rowid_uk_inc; do + run_sql "SELECT group_concat(v) from incr.$tbl group by 'all';" + check_contains "group_concat(v): a,b,c" +done - # incrementally import all data in data1 - run_lightning --backend $backend -d "tests/$TEST_NAME/data1" +run_sql "SELECT sum(pk) from incr.uk_auto_inc;" +check_contains "sum(pk): 6" - for tbl in auto_random pk_auto_inc rowid_uk_inc uk_auto_inc; do - run_sql "SELECT count(*) from incr.$tbl" - check_contains "count(*): 6" - done +# incrementally import all data in data1 +run_lightning_and_check_meta -d "tests/$TEST_NAME/data1" - for tbl in auto_random pk_auto_inc rowid_uk_inc uk_auto_inc; do - if [ "$tbl" = "auto_random" ]; then - run_sql "SELECT id & b'000001111111111111111111111111111111111111111111111111111111111' as inc FROM incr.$tbl" - else - run_sql "SELECT id as inc FROM incr.$tbl" - fi - check_contains 'inc: 4' - check_contains 'inc: 5' - check_contains 'inc: 6' - done +for tbl in auto_random pk_auto_inc rowid_uk_inc uk_auto_inc; do + run_sql "SELECT count(*) from incr.$tbl" + check_contains "count(*): 6" +done - for tbl in pk_auto_inc rowid_uk_inc; do - run_sql "SELECT group_concat(v) from incr.$tbl group by 'all';" - check_contains "group_concat(v): a,b,c,d,e,f" - done +for tbl in auto_random pk_auto_inc rowid_uk_inc uk_auto_inc; do + if [ "$tbl" = "auto_random" ]; then + run_sql "SELECT id & b'000001111111111111111111111111111111111111111111111111111111111' as inc FROM incr.$tbl" + else + run_sql "SELECT id as inc FROM incr.$tbl" + fi + check_contains 'inc: 4' + check_contains 'inc: 5' + check_contains 'inc: 6' +done - run_sql "SELECT sum(pk) from incr.uk_auto_inc;" - check_contains "sum(pk): 21" +for tbl in pk_auto_inc rowid_uk_inc; do + run_sql "SELECT group_concat(v) from incr.$tbl group by 'all';" + check_contains "group_concat(v): a,b,c,d,e,f" done + +run_sql "SELECT sum(pk) from incr.uk_auto_inc;" +check_contains "sum(pk): 21"