test(engine): add DM sync_collation integration test

deployments/engine/docker-compose/config/tidb.toml

@@ -0,0 +1 @@
+new_collations_enabled_on_first_bootstrap = true

deployments/engine/docker-compose/dm_databases.yaml

@@ -16,8 +16,6 @@ services:
         container_name: dm_upstream_mysql2
         ports:
             - "3307:3306"
-        volumes:
-            - ./config/mysql.cnf:/etc/mysql/conf.d/mysql2.cnf
         environment:
             MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
 
@@ -50,10 +48,13 @@ services:
         ports:
             - "4000:4000"
             - "10080:10080"
+        volumes:
+            - ./config/tidb.toml:/tidb.toml
         command:
             - --store=tikv
             - --path=dm_downstream_pd:2379
             - --advertise-address=dm_downstream_tidb
+            - --config=/tidb.toml
         depends_on:
             - "dm_downstream_tikv"
         restart: on-failure

engine/test/integration_tests/dm_case_sensitive/run.sh

@@ -26,7 +26,7 @@ function run() {
 
 	create_job_json=$(base64 -w0 $CUR_DIR/conf/job.yaml | jq -Rs '{ type: "DM", config: . }')
 	echo "create_job_json: $create_job_json"
-	job_id=$(curl -X POST -H "Content-Type: application/json" -d "$create_job_json" "http://127.0.0.1:10245/api/v1/jobs?tenant_id=dm_full_mode&project_id=dm_full_mode" | jq -r .id)
+	job_id=$(curl -X POST -H "Content-Type: application/json" -d "$create_job_json" "http://127.0.0.1:10245/api/v1/jobs?tenant_id=dm_case_sensitive&project_id=dm_case_sensitive" | jq -r .id)
 	echo "job_id: $job_id"
 
 	# wait for job finished

engine/test/integration_tests/dm_collation/conf/job.yaml

@@ -0,0 +1,26 @@
+name: test
+task-mode: all
+collation_compatible: "strict"
+target-database:
+  host: host.docker.internal
+  port: 4000
+  user: root
+  password: ''
+upstreams:
+  - db-config:
+      host: host.docker.internal
+      port: 3306
+      user: root
+      password: ''
+    source-id: mysql-01
+    block-allow-list: global
+  - db-config:
+      host: host.docker.internal
+      port: 3307
+      user: root
+      password: ''
+    source-id: mysql-02
+    block-allow-list: global
+block-allow-list:
+  global:
+    do-dbs: ["sync_collation*"]

engine/test/integration_tests/dm_collation/data/db1.increment.sql

@@ -0,0 +1,13 @@
+drop database if exists `sync_collation_increment`;
+create database `sync_collation_increment` character set utf8;
+use `sync_collation_increment`;
+create table t1 (id int, name varchar(20), primary key(`id`)) character set utf8;
+insert into t1 (id, name) values (1, 'Aa'), (2, 'aA');
+create table t2 (id int, name varchar(20) character set utf8, primary key(`id`)) character set latin1 collate latin1_bin;
+insert into t2 (id, name) values (1, 'Aa'), (2, 'aA');
+set collation_server = utf8_general_ci;
+drop database if exists `sync_collation_server`;
+create database `sync_collation_server`;
+use `sync_collation_server`;
+create table t1 (id int, name varchar(20), primary key(`id`));
+insert into t1 (id, name) values (1, 'Aa'), (2, 'aA');

engine/test/integration_tests/dm_collation/data/db1.prepare.sql

@@ -0,0 +1,7 @@
+drop database if exists `sync_collation`;
+create database `sync_collation` character set utf8;
+use `sync_collation`;
+create table t1 (id int, name varchar(20), primary key(`id`)) character set utf8;
+insert into t1 (id, name) values (1, 'Aa'), (2, 'aA');
+create table t2 (id int, name varchar(20) character set utf8, primary key(`id`)) character set latin1 collate latin1_bin;
+insert into t2 (id, name) values (1, 'Aa'), (2, 'aA');

engine/test/integration_tests/dm_collation/data/db2.increment.sql

@@ -0,0 +1,13 @@
+drop database if exists `sync_collation_increment2`;
+create database `sync_collation_increment2` character set utf8mb4 collate utf8mb4_general_ci;
+use `sync_collation_increment2`;
+create table t1 (id int, name varchar(20), primary key(`id`)) character set utf8mb4 collate utf8mb4_general_ci;
+insert into t1 (id, name) values (1, 'Aa'), (2, 'aA');
+create table t2 (id int, name varchar(20) character set utf8mb4 collate utf8mb4_general_ci, primary key(`id`)) character set latin1 collate latin1_bin;
+insert into t2 (id, name) values (1, 'Aa'), (2, 'aA');
+set collation_server = utf8mb4_general_ci;
+drop database if exists `sync_collation_server2`;
+create database `sync_collation_server2`;
+use `sync_collation_server2`;
+create table t1 (id int, name varchar(20), primary key(`id`));
+insert into t1 (id, name) values (1, 'Aa'), (2, 'aA');

engine/test/integration_tests/dm_collation/data/db2.prepare.sql

@@ -0,0 +1,7 @@
+drop database if exists `sync_collation2`;
+create database `sync_collation2` character set utf8mb4 collate utf8mb4_general_ci;
+use `sync_collation2`;
+create table t1 (id int, name varchar(20), primary key(`id`)) character set utf8mb4 collate utf8mb4_general_ci;
+insert into t1 (id, name) values (1, 'Aa'), (2, 'aA');
+create table t2 (id int, name varchar(20) character set utf8mb4 collate utf8mb4_general_ci, primary key(`id`)) character set latin1 collate latin1_bin;
+insert into t2 (id, name) values (1, 'Aa'), (2, 'aA');

engine/test/integration_tests/dm_collation/run.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+set -eu
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+WORK_DIR=$OUT_DIR/$TEST_NAME
+CONFIG="$DOCKER_COMPOSE_DIR/3m3e.yaml $DOCKER_COMPOSE_DIR/dm_databases.yaml"
+TABLE_NUM=500
+
+function run() {
+	rm -rf $WORK_DIR && mkdir -p $WORK_DIR
+
+	start_engine_cluster $CONFIG
+	wait_mysql_online.sh --port 3306
+	wait_mysql_online.sh --port 3307
+	wait_mysql_online.sh --port 4000
+
+	# change default charset and collation for MySQL 8.0
+	run_sql --port 3307 "set global character_set_server='utf8mb4';set global collation_server='utf8mb4_bin';"
+
+	# prepare data
+
+	run_sql_file $CUR_DIR/data/db1.prepare.sql
+	run_sql_file --port 3307 $CUR_DIR/data/db2.prepare.sql
+
+	# create job
+
+	create_job_json=$(base64 -w0 $CUR_DIR/conf/job.yaml | jq -Rs '{ type: "DM", config: . }')
+	echo "create_job_json: $create_job_json"
+	job_id=$(curl -X POST -H "Content-Type: application/json" -d "$create_job_json" "http://127.0.0.1:10245/api/v1/jobs?tenant_id=dm_case_sensitive&project_id=dm_case_sensitive" | jq -r .id)
+	echo "job_id: $job_id"
+
+	# wait for dump and load finished
+
+	exec_with_retry --count 30 "curl \"http://127.0.0.1:10245/api/v1/jobs/$job_id/status\" | tee /dev/stderr | jq -e '.TaskStatus.\"mysql-01\".Status.Unit == 12 and .TaskStatus.\"mysql-02\".Status.Unit == 12'"
+
+	# check data
+
+	exec_with_retry 'run_sql --port 4000 "select count(1) from sync_collation.t1 where name =' "'aa'" '\G" | grep -Fq "count(1): 2"'
+	exec_with_retry 'run_sql --port 4000 "select count(1) from sync_collation.t2 where name =' "'aa'" '\G" | grep -Fq "count(1): 2"'
+	exec_with_retry 'run_sql --port 4000 "select count(1) from sync_collation2.t1 where name =' "'aa'" '\G" | grep -Fq "count(1): 2"'
+	exec_with_retry 'run_sql --port 4000 "select count(1) from sync_collation2.t2 where name =' "'aa'" '\G" | grep -Fq "count(1): 2"'
+
+	# insert increment data
+
+	run_sql_file $CUR_DIR/data/db1.increment.sql
+	run_sql_file --port 3307 $CUR_DIR/data/db2.increment.sql
+
+	# check data
+
+	exec_with_retry 'run_sql --port 4000 "select count(1) from sync_collation_increment.t1 where name =' "'aa'" '\G" | grep -Fq "count(1): 2"'
+	exec_with_retry 'run_sql --port 4000 "select count(1) from sync_collation_increment.t2 where name =' "'aa'" '\G" | grep -Fq "count(1): 2"'
+	exec_with_retry 'run_sql --port 4000 "select count(1) from sync_collation_increment2.t1 where name =' "'aa'" '\G" | grep -Fq "count(1): 2"'
+	exec_with_retry 'run_sql --port 4000 "select count(1) from sync_collation_increment2.t2 where name =' "'aa'" '\G" | grep -Fq "count(1): 2"'
+	exec_with_retry 'run_sql --port 4000 "select count(1) from sync_collation_server.t1 where name =' "'aa'" '\G" | grep -Fq "count(1): 2"'
+	exec_with_retry 'run_sql --port 4000 "select count(1) from sync_collation_server2.t1 where name =' "'aa'" '\G" | grep -Fq "count(1): 2"'
+}
+
+trap "stop_engine_cluster $CONFIG" EXIT
+run $*
+# TODO: handle log properly
+# check_logs $WORK_DIR
+echo "[$(date)] <<<<<< run test case $TEST_NAME success! >>>>>>"

engine/test/utils/run_sql

@@ -37,9 +37,9 @@ while [[ ${1} ]]; do
 done
 
 if [ -z ${password} ]; then
-	cmd="mysql -u${user} -h${host} -P${port} --default-character-set utf8mb4 -e '${sql}'"
+	cmd="mysql -u${user} -h${host} -P${port} --default-character-set utf8mb4 -e \"${sql}\""
 else
-	cmd="mysql -u${user} -h${host} -P${port} -p${password} --default-character-set utf8mb4 -e '${sql}'"
+	cmd="mysql -u${user} -h${host} -P${port} -p${password} --default-character-set utf8mb4 -e \"${sql}\""
 fi
 
 echo "will execute: $cmd" >&2