diff --git a/conf/drainer-cluster.toml b/conf/drainer-cluster.toml index 352547dff..81cf2a081 100644 --- a/conf/drainer-cluster.toml +++ b/conf/drainer-cluster.toml @@ -3,8 +3,14 @@ # the interval time (in seconds) of detect pumps' status detect-interval = 10 +# Use the specified compressor algorithm to compress payload between pump and drainer +# compressor = "gzip" + # syncer Configuration. [syncer] +# Assume the upstream sql-mode. +# If this is setted , drainer will use the sql-mode to parse DDL statment +# sql-mode = "STRICT_TRANS_TABLES,NO_ENGINE_SUBSTITUTION # disable sync these schema ignore-schemas = "INFORMATION_SCHEMA,PERFORMANCE_SCHEMA,mysql" @@ -17,26 +23,33 @@ txn-batch = 20 # to get higher throughput by higher concurrent write to the downstream worker-count = 16 +# whether to disable the SQL feature of splitting a single binlog event. +# If it is set to "true", binlog events are restored to a single transaction for synchronization based on the order of binlogs. +# If the downstream service is MySQL, set it to "False". disable-dispatch = false # safe mode will split update to delete and insert safe-mode = false # downstream storage, equal to --dest-db-type -# valid values are "mysql", "pb", "tidb", "flash", "kafka" +# valid values are "mysql", "file", "tidb", "flash", "kafka" db-type = "mysql" -##replicate-do-db priority over replicate-do-table if have same db name -##and we support regex expression , start with '~' declare use regex expression. -# -#replicate-do-db = ["~^b.*","s1"] -#[[syncer.replicate-do-table]] -#db-name ="test" -#tbl-name = "log" +# replicate-do-db priority over replicate-do-table if have same db name +# and we support regex expression , start with '~' declare use regex expression. +# replicate-do-db = ["~^b.*","s1"] +# [[syncer.replicate-do-table]] +# db-name ="test" +# tbl-name = "log" + +# [[syncer.replicate-do-table]] +# db-name ="test" +# tbl-name = "~^a.*" -#[[syncer.replicate-do-table]] -#db-name ="test" -#tbl-name = "~^a.*" +# disable sync these table +# [[syncer.ignore-table]] +# db-name = "test" +# tbl-name = "log" # the downstream mysql protocol database [syncer.to] @@ -45,15 +58,12 @@ user = "root" password = "" port = 3306 -# Uncomment this if you want to use pb or sql as db-type. -# Compress compresses output file, like pb and sql file. Now it supports "gzip" algorithm only. -# Values can be "gzip". Leave it empty to disable compression. -#[syncer.to] -#dir = "data.drainer" -#compression = "gzip" +# Uncomment this if you want to use file as db-type. +# [syncer.to] +# dir = "data.drainer" # when db-type is kafka, you can uncomment this to config the down stream kafka, it will be the globle config kafka default -#[syncer.to] +# [syncer.to] # only need config one of zookeeper-addrs and kafka-addrs, will get kafka address if zookeeper-addrs is configed. # zookeeper-addrs = "127.0.0.1:2181" # kafka-addrs = "127.0.0.1:9092" diff --git a/conf/drainer-kafka.toml b/conf/drainer-kafka.toml index 74e2d1fdc..9df556d5e 100644 --- a/conf/drainer-kafka.toml +++ b/conf/drainer-kafka.toml @@ -23,7 +23,7 @@ disable-dispatch = false safe-mode = false # downstream storage, equal to --dest-db-type -# valid values are "mysql", "pb", "tidb", "flash", "kafka" +# valid values are "mysql", "file", "tidb", "flash", "kafka" db-type = "mysql" ##replicate-do-db priority over replicate-do-table if have same db name @@ -45,12 +45,9 @@ user = "root" password = "" port = 3306 -# Uncomment this if you want to use pb or sql as db-type. -# Compress compresses output file, like pb and sql file. Now it supports "gzip" algorithm only. -# Values can be "gzip". Leave it empty to disable compression. +# Uncomment this if you want to use file as db-type. #[syncer.to] #dir = "data.drainer" -#compression = "gzip" # when db-type is kafka, you can uncomment this to config the down stream kafka, it will be the globle config kafka default #[syncer.to] diff --git a/conf/pump-cluster.yml b/conf/pump-cluster.yml index 0dba6f736..3d986c899 100644 --- a/conf/pump-cluster.yml +++ b/conf/pump-cluster.yml @@ -18,3 +18,7 @@ security: # Path of file that contains X509 key in PEM format for connection with cluster components. # ssl-key: "/path/to/drainer-key.pem" + +storage: + # Set to true (by default) to guarantee reliability by ensuring binlog data is flushed to the disk. + # sync-log: true diff --git a/group_vars/all.yml b/group_vars/all.yml index 7c109352f..03be7ba13 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -49,5 +49,8 @@ pump_cert_dir: "{{ deploy_dir }}/conf/ssl" # pump binlog to pump sock pump_socket: "{{ status_dir }}/pump.sock" +# binlog version, "kafka" or "cluster": +binlog_version: "cluster" + # drainer drainer_port: 8249 diff --git a/inventory.ini b/inventory.ini index c6527dbb4..4d15d54f6 100644 --- a/inventory.ini +++ b/inventory.ini @@ -78,9 +78,6 @@ set_hostname = False ## binlog trigger enable_binlog = False -# binlog version, "kafka" or "cluster": -binlog_version = "cluster" - # kafka cluster address for monitoring, example: # kafka_addrs = "192.168.0.11:9092,192.168.0.12:9092,192.168.0.13:9092" kafka_addrs = "" diff --git a/roles/pump_cluster/templates/pump.toml.j2 b/roles/pump_cluster/templates/pump.toml.j2 index d43e6f870..89781ab08 100644 --- a/roles/pump_cluster/templates/pump.toml.j2 +++ b/roles/pump_cluster/templates/pump.toml.j2 @@ -8,3 +8,8 @@ {% for item, value in pump_conf.security | dictsort -%} {{ item }} = {{ value | to_json }} {% endfor %} + +[storage] +{% for item, value in pump_conf.storage | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} diff --git a/roles/pump_cluster/vars/default.yml b/roles/pump_cluster/vars/default.yml index c9e883434..a1049d77f 100644 --- a/roles/pump_cluster/vars/default.yml +++ b/roles/pump_cluster/vars/default.yml @@ -18,3 +18,7 @@ security: # Path of file that contains X509 key in PEM format for connection with cluster components. ssl-key: "" + +storage: + # Set to true (by default) to guarantee reliability by ensuring binlog data is flushed to the disk. + # sync-log: true