Update drainer config (#745)

* Update drainer config * Add new configuration items to binlog * Apply suggestions from code review Co-Authored-By: liubo0127 <[email protected]>
pingcap · Apr 30, 2019 · c4ea46d · c4ea46d
1 parent dabf60b
commit c4ea46d
Show file tree

Hide file tree

Showing 7 changed files with 46 additions and 26 deletions.
diff --git a/conf/drainer-cluster.toml b/conf/drainer-cluster.toml
@@ -3,8 +3,14 @@
 # the interval time (in seconds) of detect pumps' status
 detect-interval = 10
 
+# Use the specified compressor algorithm to compress payload between pump and drainer
+# compressor = "gzip"
+
 # syncer Configuration.
 [syncer]
+# Assume the upstream sql-mode.
+# If this is setted , drainer will use the sql-mode to parse DDL statment
+# sql-mode = "STRICT_TRANS_TABLES,NO_ENGINE_SUBSTITUTION
 
 # disable sync these schema
 ignore-schemas = "INFORMATION_SCHEMA,PERFORMANCE_SCHEMA,mysql"
@@ -17,26 +23,33 @@ txn-batch = 20
 # to get higher throughput by higher concurrent write to the downstream
 worker-count = 16
 
+# whether to disable the SQL feature of splitting a single binlog event.
+# If it is set to "true", binlog events are restored to a single transaction for synchronization based on the order of binlogs.
+# If the downstream service is MySQL, set it to "False".
 disable-dispatch = false
 
 # safe mode will split update to delete and insert
 safe-mode = false
 
 # downstream storage, equal to --dest-db-type
-# valid values are "mysql", "pb", "tidb", "flash", "kafka"
+# valid values are "mysql", "file", "tidb", "flash", "kafka"
 db-type = "mysql"
 
-##replicate-do-db priority over replicate-do-table if have same db name
-##and we support regex expression , start with '~' declare use regex expression.
-#
-#replicate-do-db = ["~^b.*","s1"]
-#[[syncer.replicate-do-table]]
-#db-name ="test"
-#tbl-name = "log"
+# replicate-do-db priority over replicate-do-table if have same db name
+# and we support regex expression , start with '~' declare use regex expression.
+# replicate-do-db = ["~^b.*","s1"]
+# [[syncer.replicate-do-table]]
+# db-name ="test"
+# tbl-name = "log"
+
+# [[syncer.replicate-do-table]]
+# db-name ="test"
+# tbl-name = "~^a.*"
 
-#[[syncer.replicate-do-table]]
-#db-name ="test"
-#tbl-name = "~^a.*"
+# disable sync these table
+# [[syncer.ignore-table]]
+# db-name = "test"
+# tbl-name = "log"
 
 # the downstream mysql protocol database
 [syncer.to]
@@ -45,15 +58,12 @@ user = "root"
 password = ""
 port = 3306
 
-# Uncomment this if you want to use pb or sql as db-type.
-# Compress compresses output file, like pb and sql file. Now it supports "gzip" algorithm only.
-# Values can be "gzip". Leave it empty to disable compression.
-#[syncer.to]
-#dir = "data.drainer"
-#compression = "gzip"
+# Uncomment this if you want to use file as db-type.
+# [syncer.to]
+# dir = "data.drainer"
 
 # when db-type is kafka, you can uncomment this to config the down stream kafka, it will be the globle config kafka default
-#[syncer.to]
+# [syncer.to]
 # only need config one of zookeeper-addrs and kafka-addrs, will get kafka address if zookeeper-addrs is configed.
 # zookeeper-addrs = "127.0.0.1:2181"
 # kafka-addrs = "127.0.0.1:9092"

diff --git a/conf/drainer-kafka.toml b/conf/drainer-kafka.toml
@@ -23,7 +23,7 @@ disable-dispatch = false
 safe-mode = false
 
 # downstream storage, equal to --dest-db-type
-# valid values are "mysql", "pb", "tidb", "flash", "kafka"
+# valid values are "mysql", "file", "tidb", "flash", "kafka"
 db-type = "mysql"
 
 ##replicate-do-db priority over replicate-do-table if have same db name
@@ -45,12 +45,9 @@ user = "root"
 password = ""
 port = 3306
 
-# Uncomment this if you want to use pb or sql as db-type.
-# Compress compresses output file, like pb and sql file. Now it supports "gzip" algorithm only. 
-# Values can be "gzip". Leave it empty to disable compression. 
+# Uncomment this if you want to use file as db-type.
 #[syncer.to]
 #dir = "data.drainer"
-#compression = "gzip"
 
 # when db-type is kafka, you can uncomment this to config the down stream kafka, it will be the globle config kafka default
 #[syncer.to]

diff --git a/conf/pump-cluster.yml b/conf/pump-cluster.yml
@@ -18,3 +18,7 @@ security:
 
   # Path of file that contains X509 key in PEM format for connection with cluster components.
   # ssl-key: "/path/to/drainer-key.pem"
+
+storage:
+  # Set to true (by default) to guarantee reliability by ensuring binlog data is flushed to the disk.
+  # sync-log: true
diff --git a/group_vars/all.yml b/group_vars/all.yml
@@ -49,5 +49,8 @@ pump_cert_dir: "{{ deploy_dir }}/conf/ssl"
 # pump binlog to pump sock
 pump_socket: "{{ status_dir }}/pump.sock"
 
+# binlog version, "kafka" or "cluster":
+binlog_version: "cluster"
+
 # drainer
 drainer_port: 8249
diff --git a/inventory.ini b/inventory.ini
@@ -78,9 +78,6 @@ set_hostname = False
 ## binlog trigger
 enable_binlog = False
 
-# binlog version, "kafka" or "cluster":
-binlog_version = "cluster"
-
 # kafka cluster address for monitoring, example:
 # kafka_addrs = "192.168.0.11:9092,192.168.0.12:9092,192.168.0.13:9092"
 kafka_addrs = ""

diff --git a/roles/pump_cluster/templates/pump.toml.j2 b/roles/pump_cluster/templates/pump.toml.j2
@@ -8,3 +8,8 @@
 {% for item, value in pump_conf.security | dictsort -%}
 {{ item }} = {{ value | to_json }}
 {% endfor %}
+
+[storage]
+{% for item, value in pump_conf.storage | dictsort -%}
+{{ item }} = {{ value | to_json }}
+{% endfor %}
diff --git a/roles/pump_cluster/vars/default.yml b/roles/pump_cluster/vars/default.yml
@@ -18,3 +18,7 @@ security:
 
   # Path of file that contains X509 key in PEM format for connection with cluster components.
   ssl-key: ""
+
+storage:
+  # Set to true (by default) to guarantee reliability by ensuring binlog data is flushed to the disk.
+  # sync-log: true