From f56f55f6071bec6d611218c4d9fb52347fda9f32 Mon Sep 17 00:00:00 2001 From: lance6716 Date: Fri, 31 Mar 2023 16:58:55 +0800 Subject: [PATCH] This is an automated cherry-pick of #8705 Signed-off-by: ti-chi-bot --- dm/syncer/dml.go | 7 +++++++ dm/tests/expression_filter/conf/dm-task2.yaml | 11 +++++++++++ dm/tests/expression_filter/data/db1.increment2.sql | 9 +++++++++ dm/tests/expression_filter/data/db1.prepare2.sql | 11 ++++++++++- dm/tests/expression_filter/run.sh | 13 +++++++++++++ 5 files changed, 50 insertions(+), 1 deletion(-) diff --git a/dm/syncer/dml.go b/dm/syncer/dml.go index d2d247f29a6..34fe831c50d 100644 --- a/dm/syncer/dml.go +++ b/dm/syncer/dml.go @@ -43,6 +43,13 @@ type genDMLParam struct { extendData [][]interface{} // all data include extend data } +<<<<<<< HEAD +======= +// latin1Decider is not usually ISO8859_1 in MySQL. +// ref https://dev.mysql.com/doc/refman/8.0/en/charset-we-sets.html +var latin1Decoder = charmap.Windows1252.NewDecoder() + +>>>>>>> fe2f347c52 (syncer(dm): fix wrong MySQL latin1 decoder (#8705)) // extractValueFromData adjust the values obtained from go-mysql so that // - the values can be correctly converted to TiDB datum // - the values are in the correct type that go-sql-driver/mysql uses. diff --git a/dm/tests/expression_filter/conf/dm-task2.yaml b/dm/tests/expression_filter/conf/dm-task2.yaml index 9b441b5e191..3707e6bf2da 100644 --- a/dm/tests/expression_filter/conf/dm-task2.yaml +++ b/dm/tests/expression_filter/conf/dm-task2.yaml @@ -45,7 +45,18 @@ expression-filter: table: "t5" update-old-value-expr: "c = 1" update-new-value-expr: "c = 2" +<<<<<<< HEAD +======= + only_muller: + schema: "expr_filter" + table: "t6" + insert-value-expr: "name != 'Müller…'" + e02: + schema: expr_filter + table: t7 + update-new-value-expr: "r = 'a'" +>>>>>>> fe2f347c52 (syncer(dm): fix wrong MySQL latin1 decoder (#8705)) black-white-list: # compatible with deprecated config instance: diff --git a/dm/tests/expression_filter/data/db1.increment2.sql b/dm/tests/expression_filter/data/db1.increment2.sql index 5757086d47b..73f7115eb0f 100644 --- a/dm/tests/expression_filter/data/db1.increment2.sql +++ b/dm/tests/expression_filter/data/db1.increment2.sql @@ -41,5 +41,14 @@ update t5 set should_skip = 1, c = 2 where c = 1; insert into t5 values (4, 1, 1); -- check this `should_skip = 1` row must be updated to `should_skip = 0` in TiDB update t5 set should_skip = 0, c = 3 where c = 1; +<<<<<<< HEAD +======= +insert into t6 (id, name, msg) values (1, 'Müller…', 'Müller…'), (2, 'X Æ A-12', 'X Æ A-12'); +alter table t6 add column name2 varchar(20) character set latin1 default 'Müller…'; + +-- test https://github.com/pingcap/tiflow/issues/7774 +UPDATE t7 SET s = s + 1 WHERE a = 1; + +>>>>>>> fe2f347c52 (syncer(dm): fix wrong MySQL latin1 decoder (#8705)) -- trigger a flush alter table t5 add column dummy int; \ No newline at end of file diff --git a/dm/tests/expression_filter/data/db1.prepare2.sql b/dm/tests/expression_filter/data/db1.prepare2.sql index 56753cdd9f8..12d91e4517a 100644 --- a/dm/tests/expression_filter/data/db1.prepare2.sql +++ b/dm/tests/expression_filter/data/db1.prepare2.sql @@ -6,4 +6,13 @@ create table t2 (id int primary key, should_skip int, c int, gen int as (id + 1) -); \ No newline at end of file +<<<<<<< HEAD +); +======= +); + +create table t6 (id int, name varchar(20), msg text, primary key(`id`)) character set latin1; +insert into t6 (id, name, msg) values (0, 'Müller…', 'Müller…'); +CREATE TABLE t7 (a BIGINT PRIMARY KEY, r VARCHAR(10), s INT); +INSERT INTO t7 VALUES (1, 'a', 2); +>>>>>>> fe2f347c52 (syncer(dm): fix wrong MySQL latin1 decoder (#8705)) diff --git a/dm/tests/expression_filter/run.sh b/dm/tests/expression_filter/run.sh index c5662e6ae89..6abc732becc 100755 --- a/dm/tests/expression_filter/run.sh +++ b/dm/tests/expression_filter/run.sh @@ -62,6 +62,19 @@ function complex_behaviour() { run_sql_tidb "select count(8) from expr_filter.t5 where should_skip = 1" check_contains "count(8): 0" +<<<<<<< HEAD +======= + run_sql_tidb "select count(9) from expr_filter.t6 where name = 'Müller…' and msg = 'Müller…' and name2 = 'Müller…'" + check_contains "count(9): 2" + run_sql_tidb "select count(10) from expr_filter.t6 where name != 'Müller…'" + check_contains "count(10): 0" + + run_sql_tidb "select count(11) from expr_filter.t7 where r = 'a' and s = 2" + check_contains "count(11): 1" + run_sql_tidb "select count(12) from expr_filter.t7 where r = 'a' and s = 3" + check_contains "count(12): 0" + +>>>>>>> fe2f347c52 (syncer(dm): fix wrong MySQL latin1 decoder (#8705)) insert_num=$(grep -o '"number of filtered insert"=[0-9]\+' $WORK_DIR/worker1/log/dm-worker.log | grep -o '[0-9]\+' | awk '{n += $1}; END{print n}') [ $insert_num -eq 5 ] update_num=$(grep -o '"number of filtered update"=[0-9]\+' $WORK_DIR/worker1/log/dm-worker.log | grep -o '[0-9]\+' | awk '{n += $1}; END{print n}')