syncer(dm): fix wrong MySQL latin1 decoder (#8705)

dm/syncer/dml.go

@@ -43,6 +43,13 @@ type genDMLParam struct {
 	extendData      [][]interface{}  // all data include extend data
 }
 
+<<<<<<< HEAD
+=======
+// latin1Decider is not usually ISO8859_1 in MySQL.
+// ref https://dev.mysql.com/doc/refman/8.0/en/charset-we-sets.html
+var latin1Decoder = charmap.Windows1252.NewDecoder()
+
+>>>>>>> fe2f347c52 (syncer(dm): fix wrong MySQL latin1 decoder (#8705))
 // extractValueFromData adjust the values obtained from go-mysql so that
 // - the values can be correctly converted to TiDB datum
 // - the values are in the correct type that go-sql-driver/mysql uses.

dm/tests/expression_filter/conf/dm-task2.yaml

@@ -45,7 +45,18 @@ expression-filter:
     table: "t5"
     update-old-value-expr: "c = 1"
     update-new-value-expr: "c = 2"
+<<<<<<< HEAD
 
+=======
+  only_muller:
+    schema: "expr_filter"
+    table: "t6"
+    insert-value-expr: "name != 'Müller…'"
+  e02:
+    schema: expr_filter
+    table: t7
+    update-new-value-expr: "r = 'a'"
+>>>>>>> fe2f347c52 (syncer(dm): fix wrong MySQL latin1 decoder (#8705))
 
 black-white-list: # compatible with deprecated config
   instance:

dm/tests/expression_filter/data/db1.increment2.sql

@@ -41,5 +41,14 @@ update t5 set should_skip = 1, c = 2 where c = 1;
 insert into t5 values (4, 1, 1); -- check this `should_skip = 1` row must be updated to `should_skip = 0` in TiDB
 update t5 set should_skip = 0, c = 3 where c = 1;
 
+<<<<<<< HEAD
+=======
+insert into t6 (id, name, msg) values (1, 'Müller…', 'Müller…'), (2, 'X Æ A-12', 'X Æ A-12');
+alter table t6 add column name2 varchar(20) character set latin1 default 'Müller…';
+
+-- test https://github.com/pingcap/tiflow/issues/7774
+UPDATE t7 SET s = s + 1 WHERE a = 1;
+
+>>>>>>> fe2f347c52 (syncer(dm): fix wrong MySQL latin1 decoder (#8705))
 -- trigger a flush
 alter table t5 add column dummy int;

dm/tests/expression_filter/data/db1.prepare2.sql

@@ -6,4 +6,13 @@ create table t2 (id int primary key,
     should_skip int,
     c int,
     gen int as (id + 1)
-);
+<<<<<<< HEAD
+);
+=======
+);
+
+create table t6 (id int, name varchar(20), msg text, primary key(`id`)) character set latin1;
+insert into t6 (id, name, msg) values (0, 'Müller…', 'Müller…');
+CREATE TABLE t7 (a BIGINT PRIMARY KEY, r VARCHAR(10), s INT);
+INSERT INTO t7 VALUES (1, 'a', 2);
+>>>>>>> fe2f347c52 (syncer(dm): fix wrong MySQL latin1 decoder (#8705))

dm/tests/expression_filter/run.sh

@@ -62,6 +62,19 @@ function complex_behaviour() {
 	run_sql_tidb "select count(8) from expr_filter.t5 where should_skip = 1"
 	check_contains "count(8): 0"
 
+<<<<<<< HEAD
+=======
+	run_sql_tidb "select count(9) from expr_filter.t6 where name = 'Müller…' and msg = 'Müller…' and name2 = 'Müller…'"
+	check_contains "count(9): 2"
+	run_sql_tidb "select count(10) from expr_filter.t6 where name != 'Müller…'"
+	check_contains "count(10): 0"
+
+	run_sql_tidb "select count(11) from expr_filter.t7 where r = 'a' and s = 2"
+	check_contains "count(11): 1"
+	run_sql_tidb "select count(12) from expr_filter.t7 where r = 'a' and s = 3"
+	check_contains "count(12): 0"
+
+>>>>>>> fe2f347c52 (syncer(dm): fix wrong MySQL latin1 decoder (#8705))
 	insert_num=$(grep -o '"number of filtered insert"=[0-9]\+' $WORK_DIR/worker1/log/dm-worker.log | grep -o '[0-9]\+' | awk '{n += $1}; END{print n}')
 	[ $insert_num -eq 5 ]
 	update_num=$(grep -o '"number of filtered update"=[0-9]\+' $WORK_DIR/worker1/log/dm-worker.log | grep -o '[0-9]\+' | awk '{n += $1}; END{print n}')