Skip to content
This repository has been archived by the owner on Nov 24, 2023. It is now read-only.

dumpling: properly enable EscapeBackslash with upstream SQL mode #1142

Merged
merged 1 commit into from
Oct 13, 2020
Merged

dumpling: properly enable EscapeBackslash with upstream SQL mode #1142

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions dumpling/dumpling.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ func (m *Dumpling) Init(ctx context.Context) error {
var err error
m.dumpConfig, err = m.constructArgs()
m.detectSQLMode()
m.logger.Info("create dumpling", zap.Stringer("config", m.dumpConfig))
return err
}

Expand Down Expand Up @@ -211,7 +212,6 @@ func (m *Dumpling) constructArgs() (*export.Config, error) {
return nil, err
}
dumpConfig.TableFilter = tableFilter
dumpConfig.EscapeBackslash = true
dumpConfig.CompleteInsert = true // always keep column name in `INSERT INTO` statements.
dumpConfig.Logger = m.logger.Logger

Expand Down Expand Up @@ -254,16 +254,15 @@ func (m *Dumpling) constructArgs() (*export.Config, error) {
dumpConfig.PosAfterConnect = true
}

m.logger.Info("create dumpling", zap.Stringer("config", dumpConfig))

if !cfg.CaseSensitive {
dumpConfig.TableFilter = filter.CaseInsensitive(dumpConfig.TableFilter)
}

return dumpConfig, nil
}

// detectSQLMode tries to detect SQL mode from upstream. If success, write it to LoaderConfig
// detectSQLMode tries to detect SQL mode from upstream. If success, write it to LoaderConfig.
// Because loader will use this SQL mode, we need to treat disable `EscapeBackslash` when NO_BACKSLASH_ESCAPES
func (m *Dumpling) detectSQLMode() {
db, err := sql.Open("mysql", m.dumpConfig.GetDSN(""))
if err != nil {
Expand All @@ -277,4 +276,9 @@ func (m *Dumpling) detectSQLMode() {
}
m.logger.Info("found upstream SQL mode", zap.String("SQL mode", sqlMode))
m.cfg.LoaderConfig.SQLMode = sqlMode
if strings.Contains(sqlMode, "NO_BACKSLASH_ESCAPES") {
Copy link
Member

@csuzhangxc csuzhangxc Oct 12, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we may also use HasNoBackslashEscapesMode from TiDB instead.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If it's Aurora upstream, some SQL mode can't be supported using TiDB parser

m.dumpConfig.EscapeBackslash = false
} else {
m.dumpConfig.EscapeBackslash = true
}
}
7 changes: 4 additions & 3 deletions tests/full_mode/data/db1.prepare.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,10 @@ create database `full_mode`;
use `full_mode`;
create table t1 (id int, name varchar(20), primary key(`id`));
insert into t1 (id, name) values (1, 'arya'), (2, 'catelyn');
insert into t1 (id, name) values (3, 'Eddard Stark');
update t1 set name = 'Arya Stark' where id = 1;
update t1 set name = 'Catelyn Stark' where name = 'catelyn';
insert into t1 (id, name) values (3, 'Eddard
Stark');
update t1 set name = 'Arya S\\\\tark' where id = 1;
update t1 set name = 'Catelyn S\"\n\ttark' where name = 'catelyn';

-- test multi column index with generated column
alter table t1 add column info json;
Expand Down
3 changes: 3 additions & 0 deletions tests/full_mode/run.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,8 @@ function run() {
fail_acquire_global_lock
escape_schema

run_sql_both_source "SET @@GLOBAL.SQL_MODE='NO_BACKSLASH_ESCAPES'"

run_sql_file $cur/data/db1.prepare.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1
check_contains 'Query OK, 2 rows affected'
run_sql_file $cur/data/db2.prepare.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2
Expand Down Expand Up @@ -149,6 +151,7 @@ function run() {
echo "check dump files have been cleaned"
ls $WORK_DIR/worker1/dumped_data.test && exit 1 || echo "worker1 auto removed dump files"
ls $WORK_DIR/worker2/dumped_data.test && exit 1 || echo "worker2 auto removed dump files"
run_sql_both_source "SET @@GLOBAL.SQL_MODE='ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION'"
}

cleanup_data full_mode
Expand Down