diff --git a/include/mysql/raft_listener_queue_if.h b/include/mysql/raft_listener_queue_if.h index ea2ea031c4de..4ceeff904167 100644 --- a/include/mysql/raft_listener_queue_if.h +++ b/include/mysql/raft_listener_queue_if.h @@ -26,7 +26,7 @@ enum class RaftListenerCallbackType { GET_EXECUTED_GTIDS = 15, SET_BINLOG_DURABILITY = 16, RAFT_CONFIG_CHANGE = 17, - + HANDLE_DUMP_THREADS = 18, // Note: Please update CallbackTypeToString() below when adding/removing elems // here }; @@ -41,6 +41,7 @@ class RaftListenerCallbackArg { bool val_bool; uint32_t val_uint; std::pair master_instance; + std::string master_uuid; std::string val_str; std::map val_sys_var_uint; std::pair val_opid; @@ -152,6 +153,8 @@ class RaftListenerQueueIf { return "SET_BINLOG_DURABILITY"; case RaftListenerCallbackType::RAFT_CONFIG_CHANGE: return "RAFT_CONFIG_CHANGE"; + case RaftListenerCallbackType::HANDLE_DUMP_THREADS: + return "HANDLE_DUMP_THREADS"; default: return {}; } diff --git a/mysql-test/include/check-testcase.test b/mysql-test/include/check-testcase.test index 06bc37f5cf38..9ca32ceccc2b 100644 --- a/mysql-test/include/check-testcase.test +++ b/mysql-test/include/check-testcase.test @@ -31,7 +31,7 @@ if ($tmp) { let $rpl_enable_raft= `SELECT COUNT(*) FROM INFORMATION_SCHEMA.PLUGINS WHERE PLUGIN_NAME = 'RPL_RAFT'`; if ($rpl_enable_raft) { - --echo Source_Host ::1 + --echo Source_Host 0000:0000:0000:0000:0000:0000:0000:0001 } if (!$rpl_enable_raft) { diff --git a/mysql-test/include/mtr_check.sql b/mysql-test/include/mtr_check.sql index 493bb3818192..b0e8e7ce4716 100644 --- a/mysql-test/include/mtr_check.sql +++ b/mysql-test/include/mtr_check.sql @@ -108,6 +108,7 @@ BEGIN SELECT * FROM performance_schema.global_variables WHERE variable_name NOT IN ('timestamp', 'server_uuid', 'gtid_executed', 'gtid_purged', + 'gtid_purged_for_tailing', 'gtid_committed', 'group_replication_group_name', 'keyring_file_data', @@ -247,4 +248,3 @@ BEGIN END$$ DELIMITER ; - diff --git a/mysql-test/include/raft_config.py b/mysql-test/include/raft_config.py new file mode 100755 index 000000000000..1597ada58e50 --- /dev/null +++ b/mysql-test/include/raft_config.py @@ -0,0 +1,73 @@ +#!/usr/local/bin/python3 + +import os +import sys +import json +import socket +import re + +def execute_sql(server_id, query): + socket = f'{socket_path}/mysqld.{server_id}.sock' + stream = os.popen( + f'mysql --no-defaults -S {socket} -u root -D test -sNe "{query}"' + ) + return stream.read().strip() + +def ip(): + stream = os.popen( + f'hostname -i' + ) + return stream.read().strip() + +num_servers = int(sys.argv[1]) +socket_path = sys.argv[2] + +hostname = socket.gethostname() +hostname = hostname.replace('.facebook.com', '') +dc = hostname.split(".")[1] +region = re.sub('[0-9]+$', '', dc) +ip = ip() + +config={} + +for i in range(1, num_servers + 1): + uuid = execute_sql(i, "select @@global.server_uuid") + server_id = int(execute_sql(i, "select @@global.server_id")) + port = execute_sql(i, "select @@global.port") + + config["bootstrap"] = True + config["commit_rule"] = {"mode": 2} + config["replicaset_name"] = "mysql.replicaset.0" + server_config = {} + server_config["region"] = region + server_config["hostname"] = hostname + server_config["ip_port"] = f"[{ip}]:{port}" + server_config["uuid"] = uuid + server_config["backed_by_database"] = True + server_config["voter_type"] = 0 + server_config["server_id"] = server_id + config["server_config"] = server_config + + server_props = [] + + for j in range(1, num_servers + 1): + uuid = execute_sql(j, "select @@global.server_uuid") + server_id = int(execute_sql(j, "select @@global.server_id")) + port = execute_sql(j, "select @@global.port") + server_prop = {} + server_prop["region"] = region + server_prop["hostname"] = hostname + server_prop["ip_port"] = f"[{ip}]:{port}" + server_prop["uuid"] = uuid + server_prop["server_id"] = server_id + server_prop["backed_by_database"] = True + server_prop["voter_type"] = 0 + server_props.append(server_prop) + + config["raft_topology"] = {"raft_server_properties": server_props} + config["voter_distribution"] = {region: num_servers} + config["enable_flexiraft"] = False + + config_str = json.dumps(config) + config_str = config_str.replace('"', r'\"') + execute_sql(i, f"set @@global.rpl_raft_topology_config_json='{config_str}'") diff --git a/mysql-test/include/rpl_end.inc b/mysql-test/include/rpl_end.inc index 83efbe40f0ab..196f02abba27 100644 --- a/mysql-test/include/rpl_end.inc +++ b/mysql-test/include/rpl_end.inc @@ -150,11 +150,6 @@ if ($rpl_group_replication) } } -if ($rpl_enable_raft) -{ - --source include/rpl_end_raft.inc -} - if (!$rpl_group_replication) { if (!$rpl_skip_stop_slave) @@ -219,6 +214,11 @@ if (!$rpl_group_replication) } } +if ($rpl_enable_raft) +{ + --source include/rpl_end_raft.inc +} + # Restore the server state by deleting all channels if ($rpl_multi_source) { diff --git a/mysql-test/include/rpl_end_raft.inc b/mysql-test/include/rpl_end_raft.inc index 9647469aa977..60b8bb154ae6 100644 --- a/mysql-test/include/rpl_end_raft.inc +++ b/mysql-test/include/rpl_end_raft.inc @@ -1,3 +1,17 @@ +# Suppress known warnings and reset binlogs on all raft followers. +--disable_query_log +connection server_1; +call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); +connection server_2; +call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); +connection server_3; +call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); +connection server_4; +call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); +connection server_5; +call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); +--enable_query_log + let $_rpl_server= $rpl_server_count; while ($_rpl_server) { @@ -10,7 +24,10 @@ while ($_rpl_server) disable_query_log; SET GLOBAL RPL_RAFT_ON= 0; SET GLOBAL RPL_RAFT_CONFIG_JSON= default; + SET GLOBAL RPL_RAFT_TOPOLOGY_CONFIG_JSON= default; + SET GLOBAL RPL_RAFT_SKIP_SMC_UPDATES= default; SET GLOBAL RPL_RAFT_NEW_LEADER_UUID= default; + SET GLOBAL RPL_RAFT_START_ELECTION= default; SET GLOBAL READ_ONLY_ERROR_MSG_EXTRA= default; exec rm -fr $fs_wal_path; if (!$plugin_was_early_installed) diff --git a/mysql-test/include/rpl_init_raft.inc b/mysql-test/include/rpl_init_raft.inc index f6957dea29dc..4ff24f7a6e9f 100644 --- a/mysql-test/include/rpl_init_raft.inc +++ b/mysql-test/include/rpl_init_raft.inc @@ -32,11 +32,24 @@ while ($_rpl_server) --let $ipv6_host_port= `SELECT CONCAT('[::1]', ':', (SELECT @@GLOBAL.PORT), ',')` --let $instances= `SELECT CONCAT('$instances', '$ipv6_host_port')` + let $date= `SELECT DATE_FORMAT(NOW(), '%m-%d-%Y')`; + + eval SET @@GLOBAL.RPL_RAFT_SKIP_SMC_UPDATES= 'ON,$date'; + --dec $_rpl_server } --let $instances= `SELECT TRIM(TRAILING ',' FROM '$instances')` --let $instances= `SELECT CONCAT('$instances', '"')` +let $socket_dir= `SELECT SUBSTRING(VARIABLE_VALUE, 1, + LOCATE(SUBSTRING_INDEX( + VARIABLE_VALUE, '/', -1), + VARIABLE_VALUE) - 2) + FROM performance_schema.global_variables + WHERE VARIABLE_NAME = 'socket'`; + +exec ./include/raft_config.py $rpl_server_count $socket_dir; + --let $_rpl_server= $rpl_server_count while ($_rpl_server) { diff --git a/mysql-test/r/mysqld--help-notwin.result b/mysql-test/r/mysqld--help-notwin.result index 2b67a8656b15..32e3abd644fa 100644 --- a/mysql-test/r/mysqld--help-notwin.result +++ b/mysql-test/r/mysqld--help-notwin.result @@ -8,6 +8,9 @@ The following options may be given as the first argument: Also read groups with concat(group, suffix) --login-path=# Read this path from the login file. + --abort-on-raft-purge-error + Any error in raft plugin to purge files will abort the + server --abort-slave-event-count=# Option used by mysql-test for debugging and testing of replication. @@ -1386,6 +1389,9 @@ The following options may be given as the first argument: --read-rnd-buffer-size=# When reading rows in sorted order after a sort, the rows are read through this buffer to avoid a disk seeks + --recover-raft-log Temprary variable to control recovery of raft log by + removing partial trxs. This should be removed later. + (Defaults to on; use --skip-recover-raft-log to disable.) --regexp-stack-limit=# Stack size limit for regular expressions matches --regexp-time-limit=# @@ -2702,6 +2708,7 @@ The following options may be given as the first argument: library is selected. Variables (--variable-name=value) +abort-on-raft-purge-error FALSE abort-slave-event-count 0 activate-all-roles-on-login FALSE admin-address (No default value) @@ -3088,6 +3095,7 @@ read-only FALSE read-only-error-msg-extra read-only-slave TRUE read-rnd-buffer-size 262144 +recover-raft-log TRUE regexp-stack-limit 8000000 regexp-time-limit 32 relay-log relaylog diff --git a/mysql-test/suite/binlog_nogtid/r/binlog_persist_only_variables.result b/mysql-test/suite/binlog_nogtid/r/binlog_persist_only_variables.result index d004afb8211f..744f9c8d5fdc 100644 --- a/mysql-test/suite/binlog_nogtid/r/binlog_persist_only_variables.result +++ b/mysql-test/suite/binlog_nogtid/r/binlog_persist_only_variables.result @@ -26,7 +26,7 @@ VARIABLE_NAME LIKE '%source%') AND 'binlog_file_basedir', 'binlog_index_basedir', 'skip_flush_master_info', 'skip_flush_relay_worker_info', 'read_only_slave', 'rpl_semi_sync_source_crash_if_active_trxs', -'rpl_skip_tx_api', +'rpl_skip_tx_api', 'gtid_purged_for_tailing', 'reset_seconds_behind_master', 'group_replication_plugin_hooks', 'innodb_master_thread_disabled_debug', 'innodb_replication_delay')) AND (VARIABLE_NAME NOT LIKE 'rocksdb%') diff --git a/mysql-test/suite/binlog_nogtid/r/binlog_persist_variables.result b/mysql-test/suite/binlog_nogtid/r/binlog_persist_variables.result index e5c5ad4ea290..f707a5ccbb76 100644 --- a/mysql-test/suite/binlog_nogtid/r/binlog_persist_variables.result +++ b/mysql-test/suite/binlog_nogtid/r/binlog_persist_variables.result @@ -24,6 +24,7 @@ VARIABLE_NAME LIKE '%slave%' OR VARIABLE_NAME LIKE '%source%') AND (VARIABLE_NAME NOT IN ('innodb_api_enable_binlog', 'binlog_file_basedir', 'binlog_index_basedir', +'gtid_purged_for_tailing', 'skip_flush_master_info', 'skip_flush_relay_worker_info', 'read_only_slave', 'rpl_semi_sync_source_crash_if_active_trxs', 'reset_seconds_behind_master', 'group_replication_plugin_hooks', diff --git a/mysql-test/suite/binlog_nogtid/t/binlog_persist_only_variables.test b/mysql-test/suite/binlog_nogtid/t/binlog_persist_only_variables.test index 432baf516a54..62eb10f825b2 100644 --- a/mysql-test/suite/binlog_nogtid/t/binlog_persist_only_variables.test +++ b/mysql-test/suite/binlog_nogtid/t/binlog_persist_only_variables.test @@ -56,7 +56,7 @@ INSERT INTO rplvars (varname, varvalue) 'binlog_file_basedir', 'binlog_index_basedir', 'skip_flush_master_info', 'skip_flush_relay_worker_info', 'read_only_slave', 'rpl_semi_sync_source_crash_if_active_trxs', - 'rpl_skip_tx_api', + 'rpl_skip_tx_api', 'gtid_purged_for_tailing', 'reset_seconds_behind_master', 'group_replication_plugin_hooks', 'innodb_master_thread_disabled_debug', 'innodb_replication_delay')) AND (VARIABLE_NAME NOT LIKE 'rocksdb%') diff --git a/mysql-test/suite/binlog_nogtid/t/binlog_persist_variables.test b/mysql-test/suite/binlog_nogtid/t/binlog_persist_variables.test index 0770c4e938df..4f1dd84f7114 100644 --- a/mysql-test/suite/binlog_nogtid/t/binlog_persist_variables.test +++ b/mysql-test/suite/binlog_nogtid/t/binlog_persist_variables.test @@ -55,6 +55,7 @@ INSERT INTO rplvars (varname, varvalue) VARIABLE_NAME LIKE '%source%') AND (VARIABLE_NAME NOT IN ('innodb_api_enable_binlog', 'binlog_file_basedir', 'binlog_index_basedir', + 'gtid_purged_for_tailing', 'skip_flush_master_info', 'skip_flush_relay_worker_info', 'read_only_slave', 'rpl_semi_sync_source_crash_if_active_trxs', 'reset_seconds_behind_master', 'group_replication_plugin_hooks', diff --git a/mysql-test/suite/perfschema/r/relaylog.result b/mysql-test/suite/perfschema/r/relaylog.result index 67d1e9092770..a1523c913f7a 100644 --- a/mysql-test/suite/perfschema/r/relaylog.result +++ b/mysql-test/suite/perfschema/r/relaylog.result @@ -72,6 +72,7 @@ wait/synch/mutex/sql/MYSQL_BIN_LOG::LOCK_done MANY wait/synch/mutex/sql/MYSQL_BIN_LOG::LOCK_flush_queue MANY wait/synch/mutex/sql/MYSQL_BIN_LOG::LOCK_index MANY wait/synch/mutex/sql/MYSQL_BIN_LOG::LOCK_log MANY +wait/synch/mutex/sql/MYSQL_BIN_LOG::LOCK_lost_gtids_for_tailing MANY wait/synch/mutex/sql/MYSQL_BIN_LOG::LOCK_non_xid_trxs MANY wait/synch/mutex/sql/MYSQL_BIN_LOG::LOCK_sync MANY wait/synch/mutex/sql/MYSQL_BIN_LOG::LOCK_sync_queue MANY @@ -103,6 +104,7 @@ wait/synch/mutex/sql/MYSQL_RELAY_LOG::LOCK_commit NONE NONE NONE NONE NONE wait/synch/mutex/sql/MYSQL_RELAY_LOG::LOCK_index MANY MANY MANY MANY MANY wait/synch/mutex/sql/MYSQL_RELAY_LOG::LOCK_log MANY MANY MANY MANY MANY wait/synch/mutex/sql/MYSQL_RELAY_LOG::LOCK_log_end_pos NONE NONE NONE NONE NONE +wait/synch/mutex/sql/MYSQL_RELAY_LOG::LOCK_lost_gtids_for_tailing NONE NONE NONE NONE NONE wait/synch/mutex/sql/MYSQL_RELAY_LOG::LOCK_sync NONE NONE NONE NONE NONE wait/synch/mutex/sql/MYSQL_RELAY_LOG::LOCK_xids NONE NONE NONE NONE NONE "============ Performance schema on slave ============" @@ -174,6 +176,7 @@ wait/synch/mutex/sql/MYSQL_BIN_LOG::LOCK_done MANY wait/synch/mutex/sql/MYSQL_BIN_LOG::LOCK_flush_queue MANY wait/synch/mutex/sql/MYSQL_BIN_LOG::LOCK_index MANY wait/synch/mutex/sql/MYSQL_BIN_LOG::LOCK_log MANY +wait/synch/mutex/sql/MYSQL_BIN_LOG::LOCK_lost_gtids_for_tailing MANY wait/synch/mutex/sql/MYSQL_BIN_LOG::LOCK_non_xid_trxs MANY wait/synch/mutex/sql/MYSQL_BIN_LOG::LOCK_sync MANY wait/synch/mutex/sql/MYSQL_BIN_LOG::LOCK_sync_queue MANY @@ -221,6 +224,7 @@ wait/synch/mutex/sql/MYSQL_RELAY_LOG::LOCK_commit NONE wait/synch/mutex/sql/MYSQL_RELAY_LOG::LOCK_index MANY wait/synch/mutex/sql/MYSQL_RELAY_LOG::LOCK_log MANY wait/synch/mutex/sql/MYSQL_RELAY_LOG::LOCK_log_end_pos MANY +wait/synch/mutex/sql/MYSQL_RELAY_LOG::LOCK_lost_gtids_for_tailing MANY wait/synch/mutex/sql/MYSQL_RELAY_LOG::LOCK_sync NONE wait/synch/mutex/sql/MYSQL_RELAY_LOG::LOCK_xids MANY include/rpl_end.inc diff --git a/mysql-test/suite/rpl_raft/include/raft_3_node.inc b/mysql-test/suite/rpl_raft/include/raft_3_node.inc index 4d02155dcbb1..6c5a0e2790ca 100644 --- a/mysql-test/suite/rpl_raft/include/raft_3_node.inc +++ b/mysql-test/suite/rpl_raft/include/raft_3_node.inc @@ -73,12 +73,55 @@ let $show_metadata_event= 1; # See: show_events.inc let $dont_decrement_filename= 1; - --let $include_filename= raft_3_node.inc --source include/end_include_file.inc - # Set the default connection to 'master'. Do this after # end_include_file.inc, so that it gets printed to the query log. --let $rpl_connection_name= master --source include/rpl_connection.inc + +# Create connections to server 4 and 5 (these are not in the ring) +let $rpl_server_number= 4; +let $rpl_connection_name= server_4; +source include/rpl_connect.inc; + +let $rpl_server_number= 5; +let $rpl_connection_name= server_5; +source include/rpl_connect.inc; + +connection server_1; +show status like 'rpl_raft_role'; + +connection server_2; +show status like 'rpl_raft_role'; +let $exec_gtid_set= query_get_value(SHOW MASTER STATUS, Executed_Gtid_Set, 1); +reset master; +--disable_query_log +eval set global gtid_purged='$exec_gtid_set'; +--enable_query_log + +connection server_3; +show status like 'rpl_raft_role'; +let $exec_gtid_set= query_get_value(SHOW MASTER STATUS, Executed_Gtid_Set, 1); +reset master; +--disable_query_log +eval set global gtid_purged='$exec_gtid_set'; +--enable_query_log + +connection server_4; +let $exec_gtid_set= query_get_value(SHOW MASTER STATUS, Executed_Gtid_Set, 1); +--disable_query_log +reset master; +eval set global gtid_purged='$exec_gtid_set'; +--enable_query_log + +connection server_5; +let $exec_gtid_set= query_get_value(SHOW MASTER STATUS, Executed_Gtid_Set, 1); +--disable_query_log +reset master; +eval set global gtid_purged='$exec_gtid_set'; +--enable_query_log + +# switch back to server_1 +connection server_1; diff --git a/mysql-test/suite/rpl_raft/include/raft_leadership.inc b/mysql-test/suite/rpl_raft/include/raft_leadership.inc new file mode 100644 index 000000000000..07fa8d79535e --- /dev/null +++ b/mysql-test/suite/rpl_raft/include/raft_leadership.inc @@ -0,0 +1,43 @@ +--let $include_filename= raft_leadership.inc +--source include/begin_include_file.inc + +connection server_1; +let $master_raft_uuid= `SELECT VARIABLE_VALUE FROM + performance_schema.global_status WHERE VARIABLE_NAME = "RPL_RAFT_PEER_UUID"`; +let $master_raft_role= `SELECT VARIABLE_VALUE FROM + performance_schema.global_status WHERE VARIABLE_NAME = "RPL_RAFT_ROLE"`; + +let $num_times = 0; +while ($master_raft_role != 'LEADER') +{ + # Find the current raft leader + let $_rpl_server= $rpl_server_count; + while ($_rpl_server) + { + let $rpl_connection_name= server_$_rpl_server; + source include/rpl_connection.inc; + let $_is_raft_leader= `SELECT VARIABLE_VALUE FROM + performance_schema.global_status WHERE VARIABLE_NAME = "RPL_RAFT_ROLE"`; + if ($_is_raft_leader == 'LEADER') + { + if ($num_times == 0) + { + --disable_query_log + eval SET @@global.rpl_raft_new_leader_uuid= '$master_raft_uuid'; + --enable_query_log + inc $num_times; + } + } + dec $_rpl_server; + } + + connection server_1; + let $master_raft_role= `SELECT VARIABLE_VALUE FROM + performance_schema.global_status WHERE VARIABLE_NAME = "RPL_RAFT_ROLE"`; +} + +if ($rpl_debug) +{ + --echo "Raft leadership transferred to true master" +} +--source include/end_include_file.inc diff --git a/mysql-test/suite/rpl_raft/my.cnf b/mysql-test/suite/rpl_raft/my.cnf index e0bfd87e95bc..7ddd302a1472 100644 --- a/mysql-test/suite/rpl_raft/my.cnf +++ b/mysql-test/suite/rpl_raft/my.cnf @@ -26,6 +26,7 @@ rpl_wait_for_semi_sync_ack=1 # To bypass some strict raft config checks only for MTR override_enable_raft_check=1 disallow_raft=0 +rpl_raft_skip_setup_replica_if_unnecessary=0 [mysqld.2] # Run the -slave.sh script before starting this process @@ -56,6 +57,7 @@ rpl_wait_for_semi_sync_ack=1 # To bypass some strict raft config checks only for MTR override_enable_raft_check=1 disallow_raft=0 +rpl_raft_skip_setup_replica_if_unnecessary=0 [mysqld.3] # Run the -slave.sh script before starting this process @@ -86,6 +88,7 @@ rpl_wait_for_semi_sync_ack=1 # To bypass some strict raft config checks only for MTR override_enable_raft_check=1 disallow_raft=0 +rpl_raft_skip_setup_replica_if_unnecessary=0 [mysqld.4] # Run the -slave.sh script before starting this process @@ -115,6 +118,7 @@ rpl_wait_for_semi_sync_ack=1 # To bypass some strict raft config checks only for MTR override_enable_raft_check=1 disallow_raft=0 +rpl_raft_skip_setup_replica_if_unnecessary=0 [mysqld.5] # Run the -slave.sh script before starting this process @@ -144,6 +148,7 @@ rpl_wait_for_semi_sync_ack=1 # To bypass some strict raft config checks only for MTR override_enable_raft_check=1 disallow_raft=0 +rpl_raft_skip_setup_replica_if_unnecessary=0 [ENV] MASTER_MYPORT=@mysqld.1.port diff --git a/mysql-test/suite/rpl_raft/r/raft_io_thread.result b/mysql-test/suite/rpl_raft/r/raft_io_thread.result index 162985d260db..d80ad371ef83 100644 --- a/mysql-test/suite/rpl_raft/r/raft_io_thread.result +++ b/mysql-test/suite/rpl_raft/r/raft_io_thread.result @@ -1,11 +1,24 @@ -include/master-slave.inc +include/raft_3_node.inc +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. Warnings: Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. [connection master] -include/rpl_connect.inc [creating server_3] include/rpl_connect.inc [creating server_4] include/rpl_connect.inc [creating server_5] +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role LEADER +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; SET @save.enable_raft_plugin=@@global.enable_raft_plugin; SET @@global.enable_raft_plugin=0; START SLAVE IO_THREAD; diff --git a/mysql-test/suite/rpl_raft/r/rpl_raft_apply_events.result b/mysql-test/suite/rpl_raft/r/rpl_raft_apply_events.result index 90a0fa2309c1..8db599d2c99f 100644 --- a/mysql-test/suite/rpl_raft/r/rpl_raft_apply_events.result +++ b/mysql-test/suite/rpl_raft/r/rpl_raft_apply_events.result @@ -8,6 +8,17 @@ Note #### Storing MySQL user name or password information in the master info rep [connection master] include/rpl_connect.inc [creating server_4] include/rpl_connect.inc [creating server_5] +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role LEADER +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; flush binary logs; "server_1 is the initial leader" "Writing data on leader" @@ -19,13 +30,13 @@ insert into t1 values(); include/sync_slave_sql_with_master.inc include/show_binlog_events.inc Log_name Pos Event_type Server_id End_log_pos Info -# # Previous_gtids # # Gtid_set +# # Previous_gtids # # # # Metadata # # Prev HLC_TIME: # # # Gtid # # SET @@SESSION.GTID_NEXT= 'Gtid_set' -# # Metadata # # HLC_TIME: # Raft term: 1 Raft Index: 3 +# # Metadata # # HLC_TIME: # Raft term: 1 Raft Index: 2 # # Query # # use `test`; create table t1 (a int primary key auto_increment) engine = innodb # # Gtid # # SET @@SESSION.GTID_NEXT= 'Gtid_set' -# # Metadata # # HLC_TIME: # Raft term: 1 Raft Index: 4 +# # Metadata # # HLC_TIME: # Raft term: 1 Raft Index: 3 # # Query # # BEGIN # # Table_map # # table_id: # (test.t1) # # Write_rows # # table_id: # flags: STMT_END_F diff --git a/mysql-test/suite/rpl_raft/r/rpl_raft_auto_purge_apply_logs.result b/mysql-test/suite/rpl_raft/r/rpl_raft_auto_purge_apply_logs.result index 454dd20a4cb1..a11bf823335c 100644 --- a/mysql-test/suite/rpl_raft/r/rpl_raft_auto_purge_apply_logs.result +++ b/mysql-test/suite/rpl_raft/r/rpl_raft_auto_purge_apply_logs.result @@ -14,9 +14,11 @@ Rpl_raft_role LEADER show status like 'rpl_raft_role'; Variable_name Value Rpl_raft_role FOLLOWER +reset master; show status like 'rpl_raft_role'; Variable_name Value Rpl_raft_role FOLLOWER +reset master; create table t1 (a int primary key); "Inserting rows into t1 on server_1" include/sync_slave_sql_with_master.inc @@ -67,6 +69,7 @@ include/sync_slave_sql_with_master.inc "Verifying apply logs on follower: server_2" show binary logs; Log_name File_size Encrypted +apply-logs-13001.000001 # # apply-logs-13001.000002 # # apply-logs-13001.000003 # # apply-logs-13001.000004 # # @@ -89,10 +92,10 @@ apply-logs-13001.000020 # # apply-logs-13001.000021 # # apply-logs-13001.000022 # # apply-logs-13001.000023 # # -apply-logs-13001.000024 # # "Verifying apply logs on follower: server_3" show binary logs; Log_name File_size Encrypted +apply-logs-13002.000001 # # apply-logs-13002.000002 # # apply-logs-13002.000003 # # apply-logs-13002.000004 # # @@ -115,7 +118,6 @@ apply-logs-13002.000020 # # apply-logs-13002.000021 # # apply-logs-13002.000022 # # apply-logs-13002.000023 # # -apply-logs-13002.000024 # # "Sleep for 70s to expire the logs" select sleep(70); sleep(70) @@ -142,7 +144,6 @@ apply-logs-13001.000021 # # apply-logs-13001.000022 # # apply-logs-13001.000023 # # apply-logs-13001.000024 # # -apply-logs-13001.000025 # # "Verifying apply logs on follower: server_3" select count(*) from t1; count(*) @@ -158,7 +159,6 @@ apply-logs-13002.000021 # # apply-logs-13002.000022 # # apply-logs-13002.000023 # # apply-logs-13002.000024 # # -apply-logs-13002.000025 # # "Transfering leadership: server_1 -> server_2" include/raft_promote_to_leader.inc select sleep(10); @@ -248,7 +248,6 @@ apply-logs-13002.000042 # # apply-logs-13002.000043 # # apply-logs-13002.000044 # # apply-logs-13002.000045 # # -apply-logs-13002.000046 # # "Transfering leadership: server_2 -> server_1" include/raft_promote_to_leader.inc show status like 'rpl_raft_role'; @@ -338,7 +337,6 @@ apply-logs-13002.000063 # # apply-logs-13002.000064 # # apply-logs-13002.000065 # # apply-logs-13002.000066 # # -apply-logs-13002.000067 # # drop table t1; include/sync_slave_sql_with_master.inc include/sync_slave_sql_with_master.inc diff --git a/mysql-test/suite/rpl_raft/r/rpl_raft_auto_purge_apply_logs_restart.result b/mysql-test/suite/rpl_raft/r/rpl_raft_auto_purge_apply_logs_restart.result index acf4ba01d666..011bb994c934 100644 --- a/mysql-test/suite/rpl_raft/r/rpl_raft_auto_purge_apply_logs_restart.result +++ b/mysql-test/suite/rpl_raft/r/rpl_raft_auto_purge_apply_logs_restart.result @@ -8,16 +8,17 @@ Note #### Storing MySQL user name or password information in the master info rep [connection master] include/rpl_connect.inc [creating server_4] include/rpl_connect.inc [creating server_5] -call mtr.add_suppression("Engine has seen trxs till file .*"); show status like 'rpl_raft_role'; Variable_name Value Rpl_raft_role LEADER show status like 'rpl_raft_role'; Variable_name Value Rpl_raft_role FOLLOWER +reset master; show status like 'rpl_raft_role'; Variable_name Value Rpl_raft_role FOLLOWER +reset master; create table t1 (a int primary key) engine=innodb; "Inserting rows into t1 on server_1" include/sync_slave_sql_with_master.inc @@ -53,6 +54,7 @@ count(*) 12 show binary logs; Log_name File_size Encrypted +apply-logs-13001.000001 # # apply-logs-13001.000002 # # apply-logs-13001.000003 # # apply-logs-13001.000004 # # @@ -67,13 +69,13 @@ apply-logs-13001.000012 # # apply-logs-13001.000013 # # apply-logs-13001.000014 # # apply-logs-13001.000015 # # -apply-logs-13001.000016 # # "Verifying apply logs on follower: server_3" select count(*) from t1; count(*) 12 show binary logs; Log_name File_size Encrypted +apply-logs-13002.000001 # # apply-logs-13002.000002 # # apply-logs-13002.000003 # # apply-logs-13002.000004 # # @@ -88,7 +90,6 @@ apply-logs-13002.000012 # # apply-logs-13002.000013 # # apply-logs-13002.000014 # # apply-logs-13002.000015 # # -apply-logs-13002.000016 # # "Restarting follower: server_2" include/rpl_restart_server.inc [server_number=2 parameters: --skip-slave-start] show status like 'rpl_raft_role'; @@ -103,7 +104,7 @@ Note 3083 Replication thread(s) for channel '' are already runnning. "Verifying apply logs on follower: server_2" show binary logs; Log_name File_size Encrypted -apply-logs-13001.000019 # # +apply-logs-13001.000018 # # select count(*) from t1; count(*) 12 @@ -124,15 +125,14 @@ count(*) 13 show binary logs; Log_name File_size Encrypted +apply-logs-13001.000018 # # apply-logs-13001.000019 # # -apply-logs-13001.000020 # # "Verifying apply logs on follower: server_3" select count(*) from t1; count(*) 13 show binary logs; Log_name File_size Encrypted -apply-logs-13002.000008 # # apply-logs-13002.000009 # # apply-logs-13002.000010 # # apply-logs-13002.000011 # # @@ -141,7 +141,6 @@ apply-logs-13002.000013 # # apply-logs-13002.000014 # # apply-logs-13002.000015 # # apply-logs-13002.000016 # # -apply-logs-13002.000017 # # drop table t1; include/sync_slave_sql_with_master.inc include/sync_slave_sql_with_master.inc diff --git a/mysql-test/suite/rpl_raft/r/rpl_raft_basic.result b/mysql-test/suite/rpl_raft/r/rpl_raft_basic.result index 2ef016860dde..e70f1c229925 100644 --- a/mysql-test/suite/rpl_raft/r/rpl_raft_basic.result +++ b/mysql-test/suite/rpl_raft/r/rpl_raft_basic.result @@ -8,6 +8,17 @@ Note #### Storing MySQL user name or password information in the master info rep [connection master] include/rpl_connect.inc [creating server_4] include/rpl_connect.inc [creating server_5] +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role LEADER +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; show variables like 'read_only'; Variable_name Value read_only OFF diff --git a/mysql-test/suite/rpl_raft/r/rpl_raft_dump_raft_logs.result b/mysql-test/suite/rpl_raft/r/rpl_raft_dump_raft_logs.result new file mode 100644 index 000000000000..b7839aa96add --- /dev/null +++ b/mysql-test/suite/rpl_raft/r/rpl_raft_dump_raft_logs.result @@ -0,0 +1,299 @@ +include/raft_3_node.inc +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +[connection master] +include/rpl_connect.inc [creating server_4] +include/rpl_connect.inc [creating server_5] +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role LEADER +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; +RESET MASTER; +RESET SLAVE; +SET @@GLOBAL.ENABLE_RAFT_PLUGIN = 0; +CHANGE MASTER TO MASTER_HOST = '::1', MASTER_PORT = SERVER_MYPORT_1, MASTER_USER = 'root', MASTER_CONNECT_RETRY = 1, MASTER_AUTO_POSITION = 1; +Warnings: +Note 1759 Sending passwords in plain text without SSL/TLS is extremely insecure. +Note 1760 Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +START SLAVE; +RESET MASTER; +RESET SLAVE; +SET @@GLOBAL.ENABLE_RAFT_PLUGIN = 0; +CHANGE MASTER TO MASTER_HOST = '::1', MASTER_PORT = SERVER_MYPORT_2, MASTER_USER = 'root', MASTER_CONNECT_RETRY = 1, MASTER_AUTO_POSITION = 1; +Warnings: +Note 1759 Sending passwords in plain text without SSL/TLS is extremely insecure. +Note 1760 Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +START SLAVE; +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role LEADER +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role +create table t1 (a int primary key) engine = innodb; +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +SELECT "LOGNAME" LIKE "binary-logs%"; +"LOGNAME" LIKE "binary-logs%" +1 +SELECT "LOGNAME" LIKE "binary-logs%"; +"LOGNAME" LIKE "binary-logs%" +1 +insert into t1 values(1); +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +select * from t1; +a +1 +select * from t1; +a +1 +select * from t1; +a +1 +select * from t1; +a +1 +include/raft_promote_to_leader.inc +insert into t1 values(2); +insert into t1 values(3); +insert into t1 values(4); +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +select * from t1; +a +1 +2 +3 +4 +select * from t1; +a +1 +2 +3 +4 +select * from t1; +a +1 +2 +3 +4 +select * from t1; +a +1 +2 +3 +4 +SELECT "LOGNAME" LIKE "binary-logs%"; +"LOGNAME" LIKE "binary-logs%" +1 +SELECT "LOGNAME" LIKE "binary-logs%"; +"LOGNAME" LIKE "binary-logs%" +1 +include/raft_promote_to_leader.inc +flush binary logs; +insert into t1 values(5); +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +select * from t1; +a +1 +2 +3 +4 +5 +select * from t1; +a +1 +2 +3 +4 +5 +select * from t1; +a +1 +2 +3 +4 +5 +select * from t1; +a +1 +2 +3 +4 +5 +set @@global.debug='+d,dump_wait_before_find_next_log'; +insert into t1 values(6); +insert into t1 values(7); +flush binary logs; +insert into t1 values(8); +insert into t1 values(9); +flush binary logs; +insert into t1 values(10); +insert into t1 values(11); +set debug_sync= 'now wait_for signal.reached'; +purge raft logs to 'LOGNAME'; +set debug_sync= 'now signal signal.done'; +set @@global.debug='-d,dump_wait_before_find_next_log'; +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +select * from t1; +a +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +select * from t1; +a +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +select * from t1; +a +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +select * from t1; +a +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +include/stop_slave.inc +include/rpl_restart_server.inc [server_number=2] +include/rpl_restart_server.inc [server_number=1] +include/raft_promote_to_leader.inc +START SLAVE IO_THREAD; +START SLAVE IO_THREAD; +insert into t1 values(12); +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +select * from t1; +a +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +select * from t1; +a +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +select * from t1; +a +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +select * from t1; +a +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +drop table t1; +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +stop slave; +reset slave all; +stop slave; +reset slave all; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl_raft/r/rpl_raft_dump_truncation.result b/mysql-test/suite/rpl_raft/r/rpl_raft_dump_truncation.result new file mode 100644 index 000000000000..36778e4160aa --- /dev/null +++ b/mysql-test/suite/rpl_raft/r/rpl_raft_dump_truncation.result @@ -0,0 +1,103 @@ +include/raft_3_node.inc +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +[connection master] +include/rpl_connect.inc [creating server_4] +include/rpl_connect.inc [creating server_5] +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role LEADER +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; +RESET MASTER; +RESET SLAVE; +SET @@GLOBAL.ENABLE_RAFT_PLUGIN = 0; +CHANGE MASTER TO MASTER_HOST = '::1', MASTER_PORT = SERVER_MYPORT_1, MASTER_USER = 'root', MASTER_CONNECT_RETRY = 1, MASTER_AUTO_POSITION = 1; +Warnings: +Note 1759 Sending passwords in plain text without SSL/TLS is extremely insecure. +Note 1760 Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +START SLAVE; +RESET MASTER; +RESET SLAVE; +SET @@GLOBAL.ENABLE_RAFT_PLUGIN = 0; +CHANGE MASTER TO MASTER_HOST = '::1', MASTER_PORT = SERVER_MYPORT_2, MASTER_USER = 'root', MASTER_CONNECT_RETRY = 1, MASTER_AUTO_POSITION = 1; +Warnings: +Note 1759 Sending passwords in plain text without SSL/TLS is extremely insecure. +Note 1760 Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +START SLAVE; +create table t1 (a int primary key) engine = innodb; +insert into t1 values(0); +insert into t1 values(1); +insert into t1 values(2); +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +"Stopping followers" +"Executing trx" +insert into t1 values(3); +"Stopping leader" +"Resuming followers" +"Waiting for leader election" +select * from t1; +a +0 +1 +2 +select * from t1; +a +0 +1 +2 +"Resuming old leader" +ERROR HY000: Got error 1 - 'Operation not permitted' during COMMIT +include/raft_promote_to_leader.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +select * from t1; +a +0 +1 +2 +select * from t1; +a +0 +1 +2 +select * from t1; +a +0 +1 +2 +select * from t1; +a +0 +1 +2 +select * from t1; +a +0 +1 +2 +drop table t1; +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +stop slave; +reset slave all; +stop slave; +reset slave all; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl_raft/r/rpl_raft_file_rotation_error.result b/mysql-test/suite/rpl_raft/r/rpl_raft_file_rotation_error.result index 00c2f3157b23..69c39f33c1d5 100644 --- a/mysql-test/suite/rpl_raft/r/rpl_raft_file_rotation_error.result +++ b/mysql-test/suite/rpl_raft/r/rpl_raft_file_rotation_error.result @@ -6,10 +6,19 @@ Warnings: Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. [connection master] -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); -call mtr.add_suppression("Failed to rotate binary log"); include/rpl_connect.inc [creating server_4] include/rpl_connect.inc [creating server_5] +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role LEADER +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; show variables like 'read_only'; Variable_name Value read_only OFF diff --git a/mysql-test/suite/rpl_raft/r/rpl_raft_gtid_purged_for_tailing.result b/mysql-test/suite/rpl_raft/r/rpl_raft_gtid_purged_for_tailing.result new file mode 100644 index 000000000000..7ea4cc8c4b8c --- /dev/null +++ b/mysql-test/suite/rpl_raft/r/rpl_raft_gtid_purged_for_tailing.result @@ -0,0 +1,70 @@ +include/raft_3_node.inc +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +[connection master] +include/rpl_connect.inc [creating server_4] +include/rpl_connect.inc [creating server_5] +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role LEADER +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; +create table t1 (a int primary key); +insert into t1 values (1); +flush logs; +flush logs; +purge raft logs before now(); +Warnings: +Warning 1868 file ./binary-logs-13000.000004 was not purged because it is the active log file. +Warning 1868 file ./binary-logs-13000.000003 was not purged because it is the active log file. +"Verifying gtid_purged_for_tailing and gtid_purged is same in leader" +select @@global.gtid_purged_for_tailing; +@@global.gtid_purged_for_tailing +uuid1:1-2 +select @@global.gtid_purged; +@@global.gtid_purged +uuid1:1-2 +purge raft logs before now(); +Warnings: +Warning 1868 file ./binary-logs-13001.000004 was not purged because it is the active log file. +Warning 1868 file ./binary-logs-13001.000003 was not purged because it is the active log file. +"Verifying table gtid_purged_for_tailing is not empty and gtid_purged is empty" +select @@global.gtid_purged_for_tailing; +@@global.gtid_purged_for_tailing +uuid1:1-2 +select @@global.gtid_purged; +@@global.gtid_purged + +SET @@GLOBAL.RPL_RAFT_NEW_LEADER_UUID="uuid2"; +[connection server_2] +"Check gtid_purged_for_tailing again after follower becoming leader" +select @@global.gtid_purged_for_tailing; +@@global.gtid_purged_for_tailing +uuid1:1-2 +select @@global.gtid_purged; +@@global.gtid_purged +uuid1:1-2 +"Check gtid_purged_for_tailing again after leader becoming follower" +select @@global.gtid_purged_for_tailing; +@@global.gtid_purged_for_tailing +uuid1:1-2 +select @@global.gtid_purged_for_tailing; +@@global.gtid_purged_for_tailing +uuid1:1-2 +SET @@GLOBAL.RPL_RAFT_NEW_LEADER_UUID="uuid1"; +[connection server_1] +drop table t1; +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/rpl_end.inc +reset master; diff --git a/mysql-test/suite/rpl_raft/r/rpl_raft_leader_election_crash.result b/mysql-test/suite/rpl_raft/r/rpl_raft_leader_election_crash.result new file mode 100644 index 000000000000..4abc5b82fe19 --- /dev/null +++ b/mysql-test/suite/rpl_raft/r/rpl_raft_leader_election_crash.result @@ -0,0 +1,87 @@ +include/raft_3_node.inc +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +[connection master] +include/rpl_connect.inc [creating server_4] +include/rpl_connect.inc [creating server_5] +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role LEADER +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; +"server_1 is the initial leader" +select variable_value from performance_schema.global_status where variable_name = 'Rpl_raft_role'; +variable_value +LEADER +"Stopping slave applier on all peers" +stop slave sql_thread; +stop slave sql_thread; +stop slave sql_thread; +"Writing data on leader" +create table t1 (a int primary key auto_increment) engine = innodb; +insert into t1 values(); +insert into t1 values(); +insert into t1 values(); +select * from t1; +a +1 +2 +3 +"Setting debug symbol so that server_2 crashed on becoming a leader after switching logs" +set global debug="+d,crash_after_point_binlog_to_binlog"; +"Stopping sql appliers on server_2" +stop slave sql_thread +insert into t1 values(); +insert into t1 values(); +insert into t1 values(); +select * from t1; +a +1 +2 +3 +4 +5 +6 +"Transfering leadership: server_1 -> server_2" +set @@global.rpl_raft_new_leader_uuid = 'uuid2'; +select sleep(1); +sleep(1) +0 +"Restarting server_2" +include/rpl_start_server.inc [server_number=2] +"Checking table values in server_2" +connection server_2 +select * from t1; +a +1 +2 +3 +4 +5 +6 +select sleep(20); +sleep(20) +0 +"Make server_1 the leader" +connection server_1 +sleep(10) +0 +sleep(10) +0 +Warnings: +Note 3083 Replication thread(s) for channel '' are already runnning. +Warnings: +Note 3083 Replication thread(s) for channel '' are already runnning. +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/rpl_end.inc diff --git a/mysql-test/suite/rpl_raft/r/rpl_raft_leader_election_simple.result b/mysql-test/suite/rpl_raft/r/rpl_raft_leader_election_simple.result index 867c7c4ca39b..53667d376826 100644 --- a/mysql-test/suite/rpl_raft/r/rpl_raft_leader_election_simple.result +++ b/mysql-test/suite/rpl_raft/r/rpl_raft_leader_election_simple.result @@ -8,6 +8,17 @@ Note #### Storing MySQL user name or password information in the master info rep [connection master] include/rpl_connect.inc [creating server_4] include/rpl_connect.inc [creating server_5] +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role LEADER +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; "server_1 is the initial leader" select variable_value from performance_schema.global_status where variable_name = 'Rpl_raft_role'; variable_value diff --git a/mysql-test/suite/rpl_raft/r/rpl_raft_out_of_order_raft_log.result b/mysql-test/suite/rpl_raft/r/rpl_raft_out_of_order_raft_log.result new file mode 100644 index 000000000000..f686ffd7a70d --- /dev/null +++ b/mysql-test/suite/rpl_raft/r/rpl_raft_out_of_order_raft_log.result @@ -0,0 +1,48 @@ +include/raft_3_node.inc +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +[connection master] +include/rpl_connect.inc [creating server_4] +include/rpl_connect.inc [creating server_5] +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role LEADER +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; +call mtr.add_suppression("Out of order opid found"); +create table t1(a int primary key) engine = innodb; +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +stop slave sql_thread; +insert into t1 values(1); +insert into t1 values(2); +insert into t1 values(3); +insert into t1 values(4); +flush binary logs; +set @sql_log_bin=0; +create table offsets(a int); +load data local infile 'MYSQLD_DATADIR/offsets' into table offsets FIELDS TERMINATED BY '\t' LINES TERMINATED BY '\n'; +set @sql_log_bin=1; +set @sql_log_bin=0; +drop table offsets; +set @sql_log_bin=1; +include/start_slave_sql.inc +include/wait_for_slave_sql_error.inc [errno=13121] +include/rpl_stop_server.inc [server_number=2] +flush binary logs; +include/rpl_start_server.inc [server_number=2 parameters: --skip-slave-start=1] +include/start_slave.inc +drop table t1; +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/rpl_end.inc diff --git a/mysql-test/suite/rpl_raft/r/rpl_raft_purge_logs.result b/mysql-test/suite/rpl_raft/r/rpl_raft_purge_logs.result index 2274a7bfd156..fb004e8b6053 100644 --- a/mysql-test/suite/rpl_raft/r/rpl_raft_purge_logs.result +++ b/mysql-test/suite/rpl_raft/r/rpl_raft_purge_logs.result @@ -14,9 +14,11 @@ Rpl_raft_role LEADER show status like 'rpl_raft_role'; Variable_name Value Rpl_raft_role FOLLOWER +reset master; show status like 'rpl_raft_role'; Variable_name Value Rpl_raft_role FOLLOWER +reset master; create table t1 (a int primary key); "Inserting rows into t1 on server_1" select count(*) from t1; @@ -78,6 +80,7 @@ binary-logs-13000.000005 # # flush binary logs; show binary logs; Log_name File_size Encrypted +apply-logs-13001.000001 # # apply-logs-13001.000002 # # apply-logs-13001.000003 # # purge binary logs to 'apply-logs-13001.000002'; diff --git a/mysql-test/suite/rpl_raft/r/rpl_raft_purge_raft_logs.result b/mysql-test/suite/rpl_raft/r/rpl_raft_purge_raft_logs.result index d9fa6872d5cd..27dd5905c851 100644 --- a/mysql-test/suite/rpl_raft/r/rpl_raft_purge_raft_logs.result +++ b/mysql-test/suite/rpl_raft/r/rpl_raft_purge_raft_logs.result @@ -14,9 +14,13 @@ Rpl_raft_role LEADER show status like 'rpl_raft_role'; Variable_name Value Rpl_raft_role FOLLOWER +reset master; show status like 'rpl_raft_role'; Variable_name Value Rpl_raft_role FOLLOWER +reset master; +CALL mtr.add_suppression(".*MYSQL_BIN_LOG::purge_logs was called with file *"); +CALL mtr.add_suppression(".*MYSQL_BIN_LOG::purge_logs raft plugin failed *"); create table t1 (a int primary key); "Case 1: Insert on leader and purge on leader and follower" "Inserting rows into t1 on leader: server_1" @@ -42,13 +46,14 @@ binary-logs-13000.000006 # No binary-logs-13000.000007 # No binary-logs-13000.000008 # No binary-logs-13000.000009 # No -purge raft logs to 'binary-logs-13000.000009'; +binary-logs-13000.000010 # No +purge raft logs to 'binary-logs-13000.000010'; Warnings: -Warning 1868 file ./binary-logs-13000.000008 was not purged because it is the active log file. +Warning 1868 file ./binary-logs-13000.000009 was not purged because it is the active log file. show raft logs; Log_name File_size Encrypted -binary-logs-13000.000008 # No binary-logs-13000.000009 # No +binary-logs-13000.000010 # No "Verifying table on leader: server_1" select count(*) from t1; count(*) @@ -65,13 +70,14 @@ binary-logs-13001.000006 # binary-logs-13001.000007 # binary-logs-13001.000008 # binary-logs-13001.000009 # -purge raft logs to 'binary-logs-13001.000009'; +binary-logs-13001.000010 # +purge raft logs to 'binary-logs-13001.000010'; Warnings: -Warning 1868 file ./binary-logs-13001.000008 was not purged because it is the active log file. +Warning 1868 file ./binary-logs-13001.000009 was not purged because it is the active log file. show raft logs; Log_name File_size -binary-logs-13001.000008 # binary-logs-13001.000009 # +binary-logs-13001.000010 # "Verifying table on follower: server_2" select count(*) from t1; count(*) @@ -91,20 +97,20 @@ include/sync_slave_sql_with_master.inc "Purging raft logs on leader: server_1" show raft logs; Log_name File_size Encrypted -binary-logs-13000.000008 # No binary-logs-13000.000009 # No binary-logs-13000.000010 # No binary-logs-13000.000011 # No binary-logs-13000.000012 # No binary-logs-13000.000013 # No binary-logs-13000.000014 # No -purge raft logs to 'binary-logs-13000.000014'; +binary-logs-13000.000015 # No +purge raft logs to 'binary-logs-13000.000015'; Warnings: -Warning 1868 file ./binary-logs-13000.000013 was not purged because it is the active log file. +Warning 1868 file ./binary-logs-13000.000014 was not purged because it is the active log file. show raft logs; Log_name File_size Encrypted -binary-logs-13000.000013 # No binary-logs-13000.000014 # No +binary-logs-13000.000015 # No "Verifying table on leader: server_1" select count(*) from t1; count(*) @@ -112,20 +118,20 @@ count(*) "Purging raft logs on follower: server_2" show raft logs; Log_name File_size -binary-logs-13001.000008 # binary-logs-13001.000009 # binary-logs-13001.000010 # binary-logs-13001.000011 # binary-logs-13001.000012 # binary-logs-13001.000013 # binary-logs-13001.000014 # -purge raft logs to 'binary-logs-13001.000014'; +binary-logs-13001.000015 # +purge raft logs to 'binary-logs-13001.000015'; Warnings: -Warning 1868 file ./binary-logs-13001.000013 was not purged because it is the active log file. +Warning 1868 file ./binary-logs-13001.000014 was not purged because it is the active log file. show raft logs; Log_name File_size -binary-logs-13001.000013 # binary-logs-13001.000014 # +binary-logs-13001.000015 # "Verifying table on follower: server_2" select count(*) from t1; count(*) @@ -153,7 +159,6 @@ sleep(10) "Purging raft logs on leader: server_1" show raft logs; Log_name File_size Encrypted -binary-logs-13000.000013 # No binary-logs-13000.000014 # No binary-logs-13000.000015 # No binary-logs-13000.000016 # No @@ -165,13 +170,14 @@ binary-logs-13000.000021 # No binary-logs-13000.000022 # No binary-logs-13000.000023 # No binary-logs-13000.000024 # No -purge raft logs to 'binary-logs-13000.000024'; +binary-logs-13000.000025 # No +purge raft logs to 'binary-logs-13000.000025'; Warnings: -Warning 1868 file ./binary-logs-13000.000023 was not purged because it is the active log file. +Warning 1868 file ./binary-logs-13000.000024 was not purged because it is the active log file. show raft logs; Log_name File_size Encrypted -binary-logs-13000.000023 # No binary-logs-13000.000024 # No +binary-logs-13000.000025 # No "Verifying table on leader: server_1" select count(*) from t1; count(*) @@ -182,7 +188,6 @@ sleep(10) "Purging raft logs on follower: server_2. This should not purge all logs and include a warning" show raft logs; Log_name File_size -binary-logs-13001.000013 # binary-logs-13001.000014 # binary-logs-13001.000015 # binary-logs-13001.000016 # @@ -194,17 +199,18 @@ binary-logs-13001.000021 # binary-logs-13001.000022 # binary-logs-13001.000023 # binary-logs-13001.000024 # +binary-logs-13001.000025 # purge raft logs to 'binary-logs-13001.000023'; Warnings: -Warning 1868 file ./binary-logs-13001.000019 was not purged because it is the active log file. +Warning 1868 file ./binary-logs-13001.000020 was not purged because it is the active log file. show raft logs; Log_name File_size -binary-logs-13001.000019 # binary-logs-13001.000020 # binary-logs-13001.000021 # binary-logs-13001.000022 # binary-logs-13001.000023 # binary-logs-13001.000024 # +binary-logs-13001.000025 # "Verifying table on follower: server_2" select count(*) from t1; count(*) @@ -214,13 +220,13 @@ start slave sql_thread; include/sync_slave_sql_with_master.inc include/sync_slave_sql_with_master.inc "Purging raft logs on follower: server_2" -purge raft logs to 'binary-logs-13001.000024'; +purge raft logs to 'binary-logs-13001.000025'; Warnings: -Warning 1868 file ./binary-logs-13001.000023 was not purged because it is the active log file. +Warning 1868 file ./binary-logs-13001.000024 was not purged because it is the active log file. show raft logs; Log_name File_size -binary-logs-13001.000023 # binary-logs-13001.000024 # +binary-logs-13001.000025 # "Verifying table on follower: server_2 after starting sql_thread" select count(*) from t1; count(*) @@ -254,7 +260,6 @@ include/sync_slave_sql_with_master.inc "Purging raft logs on leader: server_2" show raft logs; Log_name File_size Encrypted -binary-logs-13001.000023 # No binary-logs-13001.000024 # No binary-logs-13001.000025 # No binary-logs-13001.000026 # No @@ -263,15 +268,16 @@ binary-logs-13001.000028 # No binary-logs-13001.000029 # No binary-logs-13001.000030 # No binary-logs-13001.000031 # No -purge raft logs to 'binary-logs-13001.000030'; +binary-logs-13001.000032 # No +purge raft logs to 'binary-logs-13001.000031'; show raft logs; Log_name File_size Encrypted -binary-logs-13001.000030 # No binary-logs-13001.000031 # No +binary-logs-13001.000032 # No show binary logs; Log_name File_size Encrypted -binary-logs-13001.000030 # # binary-logs-13001.000031 # # +binary-logs-13001.000032 # # "Verifying table on leader: server_1" select count(*) from t1; count(*) @@ -279,7 +285,6 @@ count(*) "Purging raft logs on follower: server_1" show raft logs; Log_name File_size -binary-logs-13000.000023 # binary-logs-13000.000024 # binary-logs-13000.000025 # binary-logs-13000.000026 # @@ -288,29 +293,39 @@ binary-logs-13000.000028 # binary-logs-13000.000029 # binary-logs-13000.000030 # binary-logs-13000.000031 # -purge raft logs to 'binary-logs-13000.000031'; +binary-logs-13000.000032 # +purge raft logs to 'binary-logs-13000.000032'; Warnings: -Warning 1868 file ./binary-logs-13000.000030 was not purged because it is the active log file. +Warning 1868 file ./binary-logs-13000.000031 was not purged because it is the active log file. show raft logs; Log_name File_size -binary-logs-13000.000030 # binary-logs-13000.000031 # +binary-logs-13000.000032 # "Verifying table on follower: server_1" select count(*) from t1; count(*) 26 "Purging raft logs on follower: server_3" -purge raft logs to 'binary-logs-13002.000031'; +purge raft logs to 'binary-logs-13002.000032'; Warnings: -Warning 1868 file ./binary-logs-13002.000030 was not purged because it is the active log file. +Warning 1868 file ./binary-logs-13002.000031 was not purged because it is the active log file. show raft logs; Log_name File_size -binary-logs-13002.000030 # binary-logs-13002.000031 # +binary-logs-13002.000032 # "Verifying table on follower: server_3" select count(*) from t1; count(*) 26 +"Purging non existing file on leader: server_2" +flush logs; +flush logs; +select sleep(10); +sleep(10) +0 +set session debug="+d,simulate_raft_plugin_purge_error"; +purge raft logs to 'binary-logs-13001.000033'; +ERROR HY000: Unknown error during log purge "Transfering leadership: server_2 -> server_1" include/raft_promote_to_leader.inc select sleep(10); diff --git a/mysql-test/suite/rpl_raft/r/rpl_raft_purge_raft_logs_before_date.result b/mysql-test/suite/rpl_raft/r/rpl_raft_purge_raft_logs_before_date.result index 8a4fa3dbd843..9bd7ac730a02 100644 --- a/mysql-test/suite/rpl_raft/r/rpl_raft_purge_raft_logs_before_date.result +++ b/mysql-test/suite/rpl_raft/r/rpl_raft_purge_raft_logs_before_date.result @@ -14,9 +14,11 @@ Rpl_raft_role LEADER show status like 'rpl_raft_role'; Variable_name Value Rpl_raft_role FOLLOWER +reset master; show status like 'rpl_raft_role'; Variable_name Value Rpl_raft_role FOLLOWER +reset master; create table t1 (a int primary key); "Case 1: Insert on leader and purge on leader and follower" "Inserting rows into t1 on leader: server_1" @@ -44,15 +46,14 @@ binary-logs-13000.000005 # No binary-logs-13000.000006 # No binary-logs-13000.000007 # No binary-logs-13000.000008 # No -binary-logs-13000.000009 # No purge raft logs before now(); Warnings: -Warning 1868 file ./binary-logs-13000.000009 was not purged because it is the active log file. Warning 1868 file ./binary-logs-13000.000008 was not purged because it is the active log file. +Warning 1868 file ./binary-logs-13000.000007 was not purged because it is the active log file. show raft logs; Log_name File_size Encrypted +binary-logs-13000.000007 # No binary-logs-13000.000008 # No -binary-logs-13000.000009 # No "Verifying table on leader: server_1" select count(*) from t1; count(*) @@ -68,15 +69,14 @@ binary-logs-13001.000005 # binary-logs-13001.000006 # binary-logs-13001.000007 # binary-logs-13001.000008 # -binary-logs-13001.000009 # purge raft logs before now(); Warnings: -Warning 1868 file ./binary-logs-13001.000009 was not purged because it is the active log file. Warning 1868 file ./binary-logs-13001.000008 was not purged because it is the active log file. +Warning 1868 file ./binary-logs-13001.000007 was not purged because it is the active log file. show raft logs; Log_name File_size +binary-logs-13001.000007 # binary-logs-13001.000008 # -binary-logs-13001.000009 # "Verifying table on follower: server_2" select count(*) from t1; count(*) @@ -98,21 +98,21 @@ sleep(10) 0 show raft logs; Log_name File_size Encrypted +binary-logs-13000.000007 # No binary-logs-13000.000008 # No binary-logs-13000.000009 # No binary-logs-13000.000010 # No binary-logs-13000.000011 # No binary-logs-13000.000012 # No binary-logs-13000.000013 # No -binary-logs-13000.000014 # No purge raft logs before now(); Warnings: -Warning 1868 file ./binary-logs-13000.000014 was not purged because it is the active log file. Warning 1868 file ./binary-logs-13000.000013 was not purged because it is the active log file. +Warning 1868 file ./binary-logs-13000.000012 was not purged because it is the active log file. show raft logs; Log_name File_size Encrypted +binary-logs-13000.000012 # No binary-logs-13000.000013 # No -binary-logs-13000.000014 # No "Verifying table on leader: server_1" select count(*) from t1; count(*) @@ -120,21 +120,21 @@ count(*) "Purging raft logs on follower: server_2" show raft logs; Log_name File_size +binary-logs-13001.000007 # binary-logs-13001.000008 # binary-logs-13001.000009 # binary-logs-13001.000010 # binary-logs-13001.000011 # binary-logs-13001.000012 # binary-logs-13001.000013 # -binary-logs-13001.000014 # purge raft logs before now(); Warnings: -Warning 1868 file ./binary-logs-13001.000014 was not purged because it is the active log file. Warning 1868 file ./binary-logs-13001.000013 was not purged because it is the active log file. +Warning 1868 file ./binary-logs-13001.000012 was not purged because it is the active log file. show raft logs; Log_name File_size +binary-logs-13001.000012 # binary-logs-13001.000013 # -binary-logs-13001.000014 # "Verifying table on follower: server_2" select count(*) from t1; count(*) @@ -162,6 +162,7 @@ sleep(10) 0 show raft logs; Log_name File_size Encrypted +binary-logs-13000.000012 # No binary-logs-13000.000013 # No binary-logs-13000.000014 # No binary-logs-13000.000015 # No @@ -173,15 +174,14 @@ binary-logs-13000.000020 # No binary-logs-13000.000021 # No binary-logs-13000.000022 # No binary-logs-13000.000023 # No -binary-logs-13000.000024 # No purge raft logs before now(); Warnings: -Warning 1868 file ./binary-logs-13000.000024 was not purged because it is the active log file. Warning 1868 file ./binary-logs-13000.000023 was not purged because it is the active log file. +Warning 1868 file ./binary-logs-13000.000022 was not purged because it is the active log file. show raft logs; Log_name File_size Encrypted +binary-logs-13000.000022 # No binary-logs-13000.000023 # No -binary-logs-13000.000024 # No "Verifying table on leader: server_1" select count(*) from t1; count(*) @@ -192,6 +192,7 @@ sleep(10) "Purging raft logs on follower: server_2. This should not purge all logs and include a warning" show raft logs; Log_name File_size +binary-logs-13001.000012 # binary-logs-13001.000013 # binary-logs-13001.000014 # binary-logs-13001.000015 # @@ -203,16 +204,15 @@ binary-logs-13001.000020 # binary-logs-13001.000021 # binary-logs-13001.000022 # binary-logs-13001.000023 # -binary-logs-13001.000024 # purge raft logs before now(); show raft logs; Log_name File_size +binary-logs-13001.000018 # binary-logs-13001.000019 # binary-logs-13001.000020 # binary-logs-13001.000021 # binary-logs-13001.000022 # binary-logs-13001.000023 # -binary-logs-13001.000024 # "Verifying table on follower: server_2" select count(*) from t1; count(*) @@ -224,12 +224,12 @@ include/sync_slave_sql_with_master.inc "Purging raft logs on follower: server_2" purge raft logs before now(); Warnings: -Warning 1868 file ./binary-logs-13001.000024 was not purged because it is the active log file. Warning 1868 file ./binary-logs-13001.000023 was not purged because it is the active log file. +Warning 1868 file ./binary-logs-13001.000022 was not purged because it is the active log file. show raft logs; Log_name File_size +binary-logs-13001.000022 # binary-logs-13001.000023 # -binary-logs-13001.000024 # "Verifying table on follower: server_2 after starting sql_thread" select count(*) from t1; count(*) @@ -266,6 +266,7 @@ sleep(10) "Purging raft logs on leader: server_2" show raft logs; Log_name File_size Encrypted +binary-logs-13001.000022 # No binary-logs-13001.000023 # No binary-logs-13001.000024 # No binary-logs-13001.000025 # No @@ -274,19 +275,18 @@ binary-logs-13001.000027 # No binary-logs-13001.000028 # No binary-logs-13001.000029 # No binary-logs-13001.000030 # No -binary-logs-13001.000031 # No purge raft logs before now(); Warnings: -Warning 1868 file ./binary-logs-13001.000031 was not purged because it is the active log file. Warning 1868 file ./binary-logs-13001.000030 was not purged because it is the active log file. +Warning 1868 file ./binary-logs-13001.000029 was not purged because it is the active log file. show raft logs; Log_name File_size Encrypted +binary-logs-13001.000029 # No binary-logs-13001.000030 # No -binary-logs-13001.000031 # No show binary logs; Log_name File_size Encrypted +binary-logs-13001.000029 # # binary-logs-13001.000030 # # -binary-logs-13001.000031 # # "Verifying table on leader: server_1" select count(*) from t1; count(*) @@ -294,6 +294,7 @@ count(*) "Purging raft logs on follower: server_1" show raft logs; Log_name File_size +binary-logs-13000.000022 # binary-logs-13000.000023 # binary-logs-13000.000024 # binary-logs-13000.000025 # @@ -302,27 +303,26 @@ binary-logs-13000.000027 # binary-logs-13000.000028 # binary-logs-13000.000029 # binary-logs-13000.000030 # -binary-logs-13000.000031 # purge raft logs before now(); Warnings: -Warning 1868 file ./binary-logs-13000.000031 was not purged because it is the active log file. Warning 1868 file ./binary-logs-13000.000030 was not purged because it is the active log file. +Warning 1868 file ./binary-logs-13000.000029 was not purged because it is the active log file. show raft logs; Log_name File_size +binary-logs-13000.000029 # binary-logs-13000.000030 # -binary-logs-13000.000031 # "Verifying table on follower: server_1" select count(*) from t1; count(*) 26 "Purging raft logs on follower: server_3" -purge raft logs to 'binary-logs-13002.000031'; +purge raft logs to 'binary-logs-13002.000030'; Warnings: -Warning 1868 file ./binary-logs-13002.000030 was not purged because it is the active log file. +Warning 1868 file ./binary-logs-13002.000029 was not purged because it is the active log file. show raft logs; Log_name File_size +binary-logs-13002.000029 # binary-logs-13002.000030 # -binary-logs-13002.000031 # "Verifying table on follower: server_3" select count(*) from t1; count(*) diff --git a/mysql-test/suite/rpl_raft/r/rpl_raft_purged_gtids_dump_threads.result b/mysql-test/suite/rpl_raft/r/rpl_raft_purged_gtids_dump_threads.result new file mode 100644 index 000000000000..86d61f1c478e --- /dev/null +++ b/mysql-test/suite/rpl_raft/r/rpl_raft_purged_gtids_dump_threads.result @@ -0,0 +1,55 @@ +include/raft_3_node.inc +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +[connection master] +include/rpl_connect.inc [creating server_4] +include/rpl_connect.inc [creating server_5] +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role LEADER +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; +RESET MASTER; +RESET SLAVE; +SET @@GLOBAL.ENABLE_RAFT_PLUGIN = 0; +CHANGE MASTER TO MASTER_HOST = '::1', MASTER_PORT = SERVER_MYPORT_1, MASTER_USER = 'root', MASTER_CONNECT_RETRY = 1, MASTER_AUTO_POSITION = 1; +Warnings: +Note 1759 Sending passwords in plain text without SSL/TLS is extremely insecure. +Note 1760 Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +START SLAVE; +create table t1 (a int) engine = innodb; +insert into t1 values(1); +insert into t1 values(2); +include/sync_slave_sql_with_master.inc +include/raft_promote_to_leader.inc +STOP SLAVE; +change master to master_host='localhost', master_port=port2, master_auto_position=1, master_user='root'; +Warnings: +Note 1759 Sending passwords in plain text without SSL/TLS is extremely insecure. +Note 1760 Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +include/start_slave.inc +insert into t1 values(3); +include/sync_slave_sql_with_master.inc +select * from t1; +a +1 +2 +3 +include/raft_promote_to_leader.inc +drop table t1; +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/stop_slave.inc +reset slave all; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl_raft/r/rpl_raft_recover_raft_log.result b/mysql-test/suite/rpl_raft/r/rpl_raft_recover_raft_log.result new file mode 100644 index 000000000000..f80e8ea0d6d9 --- /dev/null +++ b/mysql-test/suite/rpl_raft/r/rpl_raft_recover_raft_log.result @@ -0,0 +1,88 @@ +include/raft_3_node.inc +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +[connection master] +include/rpl_connect.inc [creating server_4] +include/rpl_connect.inc [creating server_5] +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role LEADER +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; +create table t1 (a int primary key) engine = innodb; +insert into t1 values(1); +insert into t1 values(2); +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +select * from t1; +a +1 +2 +select * from t1; +a +1 +2 +select sleep(10); +sleep(10) +0 +"raft file: binary-logs-13001.000002" +"raft file pos: 1132" +"Restarting server_2" +include/rpl_restart_server.inc [server_number=2] +start slave sql_thread; +Warnings: +Note 3083 Replication thread(s) for channel '' are already runnning. +insert into t1 values(3); +insert into t1 values(4); +insert into t1 values(5); +insert into t1 values(6); +insert into t1 values(7); +insert into t1 values(8); +insert into t1 values(9); +insert into t1 values(10); +insert into t1 values(11); +insert into t1 values(12); +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +select * from t1; +a +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +select * from t1; +a +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +drop table t1; +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/rpl_end.inc diff --git a/mysql-test/suite/rpl_raft/r/rpl_raft_restart.result b/mysql-test/suite/rpl_raft/r/rpl_raft_restart.result index 9689b1bf128d..0f47f2d9824b 100644 --- a/mysql-test/suite/rpl_raft/r/rpl_raft_restart.result +++ b/mysql-test/suite/rpl_raft/r/rpl_raft_restart.result @@ -8,6 +8,17 @@ Note #### Storing MySQL user name or password information in the master info rep [connection master] include/rpl_connect.inc [creating server_4] include/rpl_connect.inc [creating server_5] +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role LEADER +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; create table t1 (a int primary key) engine = innodb; insert into t1 values(1); insert into t1 values(2); diff --git a/mysql-test/suite/rpl_raft/r/rpl_raft_show_raft_logs.result b/mysql-test/suite/rpl_raft/r/rpl_raft_show_raft_logs.result index 802a2e80ad98..93d5d9ea399c 100644 --- a/mysql-test/suite/rpl_raft/r/rpl_raft_show_raft_logs.result +++ b/mysql-test/suite/rpl_raft/r/rpl_raft_show_raft_logs.result @@ -14,9 +14,11 @@ Rpl_raft_role LEADER show status like 'rpl_raft_role'; Variable_name Value Rpl_raft_role FOLLOWER +reset master; show status like 'rpl_raft_role'; Variable_name Value Rpl_raft_role FOLLOWER +reset master; create table t1 (a int primary key); "Case 1: Simple insert and verify raft logs" "Inserting rows into t1 on leader: server_1" diff --git a/mysql-test/suite/rpl_raft/r/rpl_raft_show_slave_status.result b/mysql-test/suite/rpl_raft/r/rpl_raft_show_slave_status.result new file mode 100644 index 000000000000..f8c18f271428 --- /dev/null +++ b/mysql-test/suite/rpl_raft/r/rpl_raft_show_slave_status.result @@ -0,0 +1,25 @@ +include/raft_3_node.inc +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +[connection master] +include/rpl_connect.inc [creating server_4] +include/rpl_connect.inc [creating server_5] +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role LEADER +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; +SET @@GLOBAL.RPL_RAFT_NEW_LEADER_UUID="uuid2"; +[connection server_2] +set @@GLOBAL.RPL_RAFT_NEW_LEADER_UUID="uuid1"; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl_raft/r/rpl_raft_slave_out_of_order_commit.result b/mysql-test/suite/rpl_raft/r/rpl_raft_slave_out_of_order_commit.result index fd8dd0f90e3d..9521c4d4db44 100644 --- a/mysql-test/suite/rpl_raft/r/rpl_raft_slave_out_of_order_commit.result +++ b/mysql-test/suite/rpl_raft/r/rpl_raft_slave_out_of_order_commit.result @@ -8,6 +8,17 @@ Note #### Storing MySQL user name or password information in the master info rep [connection master] include/rpl_connect.inc [creating server_4] include/rpl_connect.inc [creating server_5] +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role LEADER +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; create database d1; create database d2; create table d1.t1(a int primary key) engine = innodb; @@ -16,6 +27,7 @@ insert into d1.t1 values(1); insert into d2.t2 values(1); include/sync_slave_sql_with_master.inc include/sync_slave_sql_with_master.inc +SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED; begin; update d1.t1 set a = a + 100; update d1.t1 set a = a + 1; diff --git a/mysql-test/suite/rpl_raft/r/rpl_raft_wait_for_ack.result b/mysql-test/suite/rpl_raft/r/rpl_raft_wait_for_ack.result index f0dbb86f8c7f..ee2b64fcf080 100644 --- a/mysql-test/suite/rpl_raft/r/rpl_raft_wait_for_ack.result +++ b/mysql-test/suite/rpl_raft/r/rpl_raft_wait_for_ack.result @@ -8,6 +8,17 @@ Note #### Storing MySQL user name or password information in the master info rep [connection master] include/rpl_connect.inc [creating server_4] include/rpl_connect.inc [creating server_5] +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role LEADER +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; RESET MASTER; RESET SLAVE; SET @@GLOBAL.ENABLE_RAFT_PLUGIN = 0; diff --git a/mysql-test/suite/rpl_raft/r/rpl_stepdown_trim.result b/mysql-test/suite/rpl_raft/r/rpl_stepdown_trim.result new file mode 100644 index 000000000000..023bb26246ea --- /dev/null +++ b/mysql-test/suite/rpl_raft/r/rpl_stepdown_trim.result @@ -0,0 +1,65 @@ +include/raft_3_node.inc +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +[connection master] +include/rpl_connect.inc [creating server_4] +include/rpl_connect.inc [creating server_5] +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role LEADER +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +reset master; +create table t1 (a int primary key) engine=innodb; +insert into t1 values(1); +insert into t1 values(2); +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +insert into t1 values (3), (4), (5); +SET net_write_timeout = 1; +insert into t1 values (6), (7), (8); +"Now triggering DMP by stopping current MASTER" +"Now triggering DMP by reenabling current SLAVES" +"Will restart previous master to trigger trimming in 15 seconds" +ERROR HY000: Got error 1 - 'Operation not permitted' during COMMIT +include/raft_leadership.inc +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role LEADER +show variables like 'read_only'; +Variable_name Value +read_only OFF +select * from t1; +a +1 +2 +3 +4 +5 +select * from t1; +a +1 +2 +3 +4 +5 +select * from t1; +a +1 +2 +3 +4 +5 +drop table t1; +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/rpl_end.inc diff --git a/mysql-test/suite/rpl_raft/t/raft_io_thread.test b/mysql-test/suite/rpl_raft/t/raft_io_thread.test index ee7da3e332c8..87558c75f477 100644 --- a/mysql-test/suite/rpl_raft/t/raft_io_thread.test +++ b/mysql-test/suite/rpl_raft/t/raft_io_thread.test @@ -1,38 +1,7 @@ ---source include/master-slave.inc - ---disable_query_log -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - -# Create connections to server 3 -let $rpl_server_number= 3; -let $rpl_connection_name= server_3; -source include/rpl_connect.inc; ---disable_query_log -connection server_3; -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - -# Create connections to server 4 -let $rpl_server_number= 4; -let $rpl_connection_name= server_4; -source include/rpl_connect.inc; ---disable_query_log -connection server_4; -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - -# Create connections to server 5 -let $rpl_server_number= 5; -let $rpl_connection_name= server_5; -source include/rpl_connect.inc; ---disable_query_log -connection server_5; -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log +source ../include/raft_3_node.inc; # Save variable value ---connection slave +--connection server_2 SET @save.enable_raft_plugin=@@global.enable_raft_plugin; SET @@global.enable_raft_plugin=0; @@ -65,5 +34,5 @@ START SLAVE IO_THREAD; STOP SLAVE IO_THREAD; SET @@global.enable_raft_plugin=@save.enable_raft_plugin; ---connection master +--connection server_1 --source include/rpl_end.inc diff --git a/mysql-test/suite/rpl_raft/t/rpl_raft_apply_events.test b/mysql-test/suite/rpl_raft/t/rpl_raft_apply_events.test index 86cac49ad38a..5053577fee4c 100644 --- a/mysql-test/suite/rpl_raft/t/rpl_raft_apply_events.test +++ b/mysql-test/suite/rpl_raft/t/rpl_raft_apply_events.test @@ -1,26 +1,5 @@ source ../include/raft_3_node.inc; ---disable_query_log -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - -# Create connections to server 4 and 5 (these are not in the ring) -let $rpl_server_number= 4; -let $rpl_connection_name= server_4; -source include/rpl_connect.inc; ---disable_query_log -connection server_4; -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - -let $rpl_server_number= 5; -let $rpl_connection_name= server_5; -source include/rpl_connect.inc; ---disable_query_log -connection server_5; -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - connection server_1; let $uuid1= `select variable_value from performance_schema.global_status where variable_name = 'Rpl_raft_peer_uuid'`; connection server_2; diff --git a/mysql-test/suite/rpl_raft/t/rpl_raft_auto_purge_apply_logs.test b/mysql-test/suite/rpl_raft/t/rpl_raft_auto_purge_apply_logs.test index e0a75d83b577..cafc3638746b 100644 --- a/mysql-test/suite/rpl_raft/t/rpl_raft_auto_purge_apply_logs.test +++ b/mysql-test/suite/rpl_raft/t/rpl_raft_auto_purge_apply_logs.test @@ -1,36 +1,6 @@ source include/have_debug.inc; source ../include/raft_3_node.inc; ---disable_query_log -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - -# Create connections to server 4 and 5 (these are not in the ring) -let $rpl_server_number= 4; -let $rpl_connection_name= server_4; -source include/rpl_connect.inc; ---disable_query_log -connection server_4; -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - -let $rpl_server_number= 5; -let $rpl_connection_name= server_5; -source include/rpl_connect.inc; ---disable_query_log -connection server_5; -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - -connection server_1; -show status like 'rpl_raft_role'; - -connection server_2; -show status like 'rpl_raft_role'; - -connection server_3; -show status like 'rpl_raft_role'; - connection server_1; create table t1 (a int primary key); diff --git a/mysql-test/suite/rpl_raft/t/rpl_raft_auto_purge_apply_logs_restart.test b/mysql-test/suite/rpl_raft/t/rpl_raft_auto_purge_apply_logs_restart.test index b084b83a8d38..10875a34330a 100644 --- a/mysql-test/suite/rpl_raft/t/rpl_raft_auto_purge_apply_logs_restart.test +++ b/mysql-test/suite/rpl_raft/t/rpl_raft_auto_purge_apply_logs_restart.test @@ -1,36 +1,5 @@ source include/have_debug.inc; source ../include/raft_3_node.inc; ---disable_query_log -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - -# Create connections to server 4 and 5 (these are not in the ring) -let $rpl_server_number= 4; -let $rpl_connection_name= server_4; -source include/rpl_connect.inc; ---disable_query_log -connection server_4; -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - -let $rpl_server_number= 5; -let $rpl_connection_name= server_5; -source include/rpl_connect.inc; ---disable_query_log -connection server_5; -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - -call mtr.add_suppression("Engine has seen trxs till file .*"); - -connection server_1; -show status like 'rpl_raft_role'; - -connection server_2; -show status like 'rpl_raft_role'; - -connection server_3; -show status like 'rpl_raft_role'; connection server_1; create table t1 (a int primary key) engine=innodb; diff --git a/mysql-test/suite/rpl_raft/t/rpl_raft_basic.test b/mysql-test/suite/rpl_raft/t/rpl_raft_basic.test index c665050faae6..c215706d9354 100644 --- a/mysql-test/suite/rpl_raft/t/rpl_raft_basic.test +++ b/mysql-test/suite/rpl_raft/t/rpl_raft_basic.test @@ -1,26 +1,5 @@ source ../include/raft_3_node.inc; ---disable_query_log -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - -# Create connections to server 4 and 5 (these are not in the ring) -let $rpl_server_number= 4; -let $rpl_connection_name= server_4; -source include/rpl_connect.inc; ---disable_query_log -connection server_4; -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - -let $rpl_server_number= 5; -let $rpl_connection_name= server_5; -source include/rpl_connect.inc; ---disable_query_log -connection server_5; -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - connection server_1; show variables like 'read_only'; show variables like 'super_read_only'; diff --git a/mysql-test/suite/rpl_raft/t/rpl_raft_disallow_start_sql.test b/mysql-test/suite/rpl_raft/t/rpl_raft_disallow_start_sql.test new file mode 100644 index 000000000000..d89790dc2276 --- /dev/null +++ b/mysql-test/suite/rpl_raft/t/rpl_raft_disallow_start_sql.test @@ -0,0 +1,48 @@ +source ../include/raft_3_node.inc; + +echo "server_1 is the initial leader"; +# Reject append entries on server 3 +connection server_3; +set @@global.rpl_raft_reject_append_entries = 1; + +connection server_2; +set @@global.rpl_raft_start_election = 1; + +--sleep 5 + +connection server_1; +let $running1 = query_get_value(SHOW SLAVE STATUS, Slave_SQL_Running, 1); + +echo "server1 running state : $running1"; + +connection server_3; +let $running3 = query_get_value(SHOW SLAVE STATUS, Slave_SQL_Running, 1); + +echo "server3 running state : $running3"; + +--error ER_RAFT_OPERATION_INCOMPATIBLE +start slave sql_thread; + +set @@global.rpl_raft_reject_append_entries = 0; + +--sleep 2 + +let $running3_2 = query_get_value(SHOW SLAVE STATUS, Slave_SQL_Running, 1); + +echo "server3 running state now $running3_2"; + +stop slave sql_thread; + +start slave sql_thread; + +connection server_1; +set @@global.rpl_raft_start_election = 1; + +--sleep 3 + +let $sync_slave_connection= server_2; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_3; +source include/sync_slave_sql_with_master.inc; + +source include/rpl_end.inc; diff --git a/mysql-test/suite/rpl_raft/t/rpl_raft_dump_raft_logs-slave.opt b/mysql-test/suite/rpl_raft/t/rpl_raft_dump_raft_logs-slave.opt new file mode 100644 index 000000000000..fa88d39211c9 --- /dev/null +++ b/mysql-test/suite/rpl_raft/t/rpl_raft_dump_raft_logs-slave.opt @@ -0,0 +1 @@ +--initialize --innodb_page_size=16k diff --git a/mysql-test/suite/rpl_raft/t/rpl_raft_dump_raft_logs.test b/mysql-test/suite/rpl_raft/t/rpl_raft_dump_raft_logs.test new file mode 100644 index 000000000000..2c607601a4fb --- /dev/null +++ b/mysql-test/suite/rpl_raft/t/rpl_raft_dump_raft_logs.test @@ -0,0 +1,245 @@ +source ../include/raft_3_node.inc; +source include/have_debug_sync.inc; + +# Connect server 4 and 5 using COM_BINLOG_DUMP_GTID +# server4 will tail the leader +connection server_4; +RESET MASTER; +RESET SLAVE; +SET @@GLOBAL.ENABLE_RAFT_PLUGIN = 0; +replace_result $SERVER_MYPORT_1 SERVER_MYPORT_1; +eval CHANGE MASTER TO MASTER_HOST = '::1', MASTER_PORT = $SERVER_MYPORT_1, MASTER_USER = 'root', MASTER_CONNECT_RETRY = 1, MASTER_AUTO_POSITION = 1; +START SLAVE; + +# server5 will tail a follower +connection server_5; +RESET MASTER; +RESET SLAVE; +SET @@GLOBAL.ENABLE_RAFT_PLUGIN = 0; +replace_result $SERVER_MYPORT_2 SERVER_MYPORT_2; +eval CHANGE MASTER TO MASTER_HOST = '::1', MASTER_PORT = $SERVER_MYPORT_2, MASTER_USER = 'root', MASTER_CONNECT_RETRY = 1, MASTER_AUTO_POSITION = 1; +START SLAVE; + +# Check raft roles, 4 and 5 should be empty +connection server_1; +show status like 'rpl_raft_role'; +connection server_2; +show status like 'rpl_raft_role'; +connection server_3; +show status like 'rpl_raft_role'; +connection server_4; +show status like 'rpl_raft_role'; +connection server_5; +show status like 'rpl_raft_role'; + +# Create a schema and sync it across replicas +connection server_1; +create table t1 (a int primary key) engine = innodb; + +let $sync_slave_connection= server_2; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_3; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_4; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_5; +source include/sync_slave_sql_with_master.inc; + +# Check if server 4 and 5 are tailing raft logs +connection server_4; +let $logname= query_get_value(SHOW SLAVE STATUS, Relay_Master_Log_File, 1); +replace_result $logname LOGNAME; +eval SELECT "$logname" LIKE "binary-logs%"; +connection server_5; +let $logname= query_get_value(SHOW SLAVE STATUS, Relay_Master_Log_File, 1); +replace_result $logname LOGNAME; +eval SELECT "$logname" LIKE "binary-logs%"; + +# Execute a regular DML (insert) +connection server_1; +insert into t1 values(1); +let $sync_slave_connection= server_2; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_3; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_4; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_5; +source include/sync_slave_sql_with_master.inc; + +connection server_2; +select * from t1; +connection server_3; +select * from t1; +connection server_4; +select * from t1; +connection server_5; +select * from t1; + +# Transfer leadership from server1 to server2 +let $rpl_raft_leader_number= 2; +source ../include/raft_promote_to_leader.inc; + +# Execute some more trxs +connection server_2; +let $wait_condition= select @@global.read_only = 0; +source include/wait_condition.inc; +insert into t1 values(2); +insert into t1 values(3); +insert into t1 values(4); +let $sync_slave_connection= server_1; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_3; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_4; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_5; +source include/sync_slave_sql_with_master.inc; + +connection server_1; +select * from t1; +connection server_3; +select * from t1; +connection server_4; +select * from t1; +connection server_5; +select * from t1; + +# Check if server 4 and 5 are still tailing raft logs post election +connection server_4; +let $logname= query_get_value(SHOW SLAVE STATUS, Relay_Master_Log_File, 1); +replace_result $logname LOGNAME; +eval SELECT "$logname" LIKE "binary-logs%"; +connection server_5; +let $logname= query_get_value(SHOW SLAVE STATUS, Relay_Master_Log_File, 1); +replace_result $logname LOGNAME; +eval SELECT "$logname" LIKE "binary-logs%"; + +# Transfer leadership from server2 to server1 +let $rpl_raft_leader_number= 1; +source ../include/raft_promote_to_leader.inc; + +# Check if binlog flushing works as expected +connection server_1; +let $wait_condition= select @@global.read_only = 0; +source include/wait_condition.inc; +flush binary logs; +insert into t1 values(5); +let $sync_slave_connection= server_2; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_3; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_4; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_5; +source include/sync_slave_sql_with_master.inc; + +connection server_2; +select * from t1; +connection server_3; +select * from t1; +connection server_4; +select * from t1; +connection server_5; +select * from t1; + +# Check interaction of dump logs with purge +connection server_4; +let $logname= query_get_value(SHOW SLAVE STATUS, Relay_Master_Log_File, 1); +connection server_1; +set @@global.debug='+d,dump_wait_before_find_next_log'; +insert into t1 values(6); +insert into t1 values(7); +flush binary logs; +insert into t1 values(8); +insert into t1 values(9); +flush binary logs; +insert into t1 values(10); +insert into t1 values(11); +set debug_sync= 'now wait_for signal.reached'; +replace_result $logname LOGNAME; +eval purge raft logs to '$logname'; +set debug_sync= 'now signal signal.done'; +set @@global.debug='-d,dump_wait_before_find_next_log'; +let $sync_slave_connection= server_2; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_3; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_4; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_5; +source include/sync_slave_sql_with_master.inc; + +connection server_2; +select * from t1; +connection server_3; +select * from t1; +connection server_4; +select * from t1; +connection server_5; +select * from t1; + + +# Check if clean restart affects dump threads +connection server_2; +source include/stop_slave.inc; +let $rpl_server_number= 2; +source include/rpl_restart_server.inc; + +connection server_1; +let $rpl_server_number= 1; +source include/rpl_restart_server.inc; + +let $rpl_raft_leader_number= 1; +source ../include/raft_promote_to_leader.inc; + +disable_warnings; +connection server_4; +START SLAVE IO_THREAD; +connection server_5; +START SLAVE IO_THREAD; +enable_warnings; + +connection server_1; +let $wait_condition= select @@global.read_only = 0; +source include/wait_condition.inc; +insert into t1 values(12); +let $sync_slave_connection= server_2; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_3; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_4; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_5; +source include/sync_slave_sql_with_master.inc; + +connection server_2; +select * from t1; +connection server_3; +select * from t1; +connection server_4; +select * from t1; +connection server_5; +select * from t1; + +# Cleanup +connection server_1; +drop table t1; + +let $sync_slave_connection= server_2; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_3; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_4; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_5; +source include/sync_slave_sql_with_master.inc; + +connection server_4; +stop slave; +reset slave all; +connection server_5; +stop slave; +reset slave all; + +source include/rpl_end.inc; diff --git a/mysql-test/suite/rpl_raft/t/rpl_raft_dump_truncation.test b/mysql-test/suite/rpl_raft/t/rpl_raft_dump_truncation.test new file mode 100644 index 000000000000..8078a70de6d3 --- /dev/null +++ b/mysql-test/suite/rpl_raft/t/rpl_raft_dump_truncation.test @@ -0,0 +1,145 @@ +source ../include/raft_3_node.inc; +connection server_1; + +# Connect server 4 and 5 using COM_BINLOG_DUMP_GTID +# server4 will tail the leader +connection server_4; +RESET MASTER; +RESET SLAVE; +SET @@GLOBAL.ENABLE_RAFT_PLUGIN = 0; +replace_result $SERVER_MYPORT_1 SERVER_MYPORT_1; +eval CHANGE MASTER TO MASTER_HOST = '::1', MASTER_PORT = $SERVER_MYPORT_1, MASTER_USER = 'root', MASTER_CONNECT_RETRY = 1, MASTER_AUTO_POSITION = 1; +START SLAVE; + +# server5 will tail a follower +connection server_5; +RESET MASTER; +RESET SLAVE; +SET @@GLOBAL.ENABLE_RAFT_PLUGIN = 0; +replace_result $SERVER_MYPORT_2 SERVER_MYPORT_2; +eval CHANGE MASTER TO MASTER_HOST = '::1', MASTER_PORT = $SERVER_MYPORT_2, MASTER_USER = 'root', MASTER_CONNECT_RETRY = 1, MASTER_AUTO_POSITION = 1; +START SLAVE; + +# Create a schema and sync it across replicas +connection server_1; +create table t1 (a int primary key) engine = innodb; +insert into t1 values(0); +insert into t1 values(1); +insert into t1 values(2); + +let $sync_slave_connection= server_2; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_3; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_4; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_5; +source include/sync_slave_sql_with_master.inc; + +echo "Stopping followers"; +connection server_2; +let $server2_pid_file= `select @@global.pid_file`; +exec kill -STOP `cat $server2_pid_file`; +connection server_3; +let $server3_pid_file= `select @@global.pid_file`; +exec kill -STOP `cat $server3_pid_file`; + +sleep 5; + +echo "Executing trx"; +connection server_1; +send insert into t1 values(3); + +echo "Stopping leader"; +connection server_1_1; +let $server1_pid_file= `select @@global.pid_file`; +exec kill -STOP `cat $server1_pid_file`; + +sleep 5; + +echo "Resuming followers"; +exec kill -CONT `cat $server2_pid_file`; +exec kill -CONT `cat $server3_pid_file`; + +echo "Waiting for leader election"; +connection server_2; +let $server2_is_leader = `select variable_value = "LEADER" from performance_schema.global_status where variable_name = 'rpl_raft_role'`; +connection server_3; +let $server3_is_leader = `select variable_value = "LEADER" from performance_schema.global_status where variable_name = 'rpl_raft_role'`; +let $election_done = `SELECT $server2_is_leader + $server3_is_leader`; +while (!$election_done) { + sleep 1; + connection server_2; + let $server2_is_leader = `select variable_value = "LEADER" from performance_schema.global_status where variable_name = 'rpl_raft_role'`; + connection server_3; + let $server3_is_leader = `select variable_value = "LEADER" from performance_schema.global_status where variable_name = 'rpl_raft_role'`; + let $election_done = `SELECT $server2_is_leader + $server3_is_leader`; +} + +connection server_2; +select * from t1; +connection server_3; +select * from t1; + +echo "Resuming old leader"; +exec kill -CONT `cat $server1_pid_file`; + +connection server_1; +error 1180; +reap; + +let $rpl_raft_leader_number = 1; +source ../include/raft_promote_to_leader.inc; +let $wait_condition= select @@global.read_only = 0; +source include/wait_condition.inc; + +let $sync_slave_connection= server_2; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_3; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_4; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_5; +source include/sync_slave_sql_with_master.inc; + +connection server_1; +select * from t1; +connection server_2; +select * from t1; +connection server_3; +select * from t1; +connection server_4; +select * from t1; +connection server_5; +select * from t1; + +# Cleanup +connection server_1; +drop table t1; + +let $sync_slave_connection= server_2; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_3; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_4; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_5; +source include/sync_slave_sql_with_master.inc; + +# Suppress known warnings. +--disable_query_log +connection server_1; +call mtr.add_suppression("Run function 'before_flush' in plugin 'RPL_RAFT' failed"); +call mtr.add_suppression("Error writing file 'binary-logs-13000'.*"); +call mtr.add_suppression("Commit consensus error set for 1 threads in the group"); +call mtr.add_suppression("Run function 'before_commit' in plugin 'RPL_RAFT' failed"); +--enable_query_log + +connection server_4; +stop slave; +reset slave all; +connection server_5; +stop slave; +reset slave all; + +source include/rpl_end.inc; diff --git a/mysql-test/suite/rpl_raft/t/rpl_raft_file_rotation_error.test b/mysql-test/suite/rpl_raft/t/rpl_raft_file_rotation_error.test index 6abb345385ec..66c3250eeca5 100644 --- a/mysql-test/suite/rpl_raft/t/rpl_raft_file_rotation_error.test +++ b/mysql-test/suite/rpl_raft/t/rpl_raft_file_rotation_error.test @@ -1,24 +1,5 @@ source include/have_debug.inc; source ../include/raft_3_node.inc; -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); -call mtr.add_suppression("Failed to rotate binary log"); - -# Create connections to server 4 and 5 (these are not in the ring) -let $rpl_server_number= 4; -let $rpl_connection_name= server_4; -source include/rpl_connect.inc; ---disable_query_log -connection server_4; -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - -let $rpl_server_number= 5; -let $rpl_connection_name= server_5; -source include/rpl_connect.inc; ---disable_query_log -connection server_5; -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log connection server_1; show variables like 'read_only'; @@ -122,4 +103,9 @@ source include/sync_slave_sql_with_master.inc; let $sync_slave_connection= server_3; source include/sync_slave_sql_with_master.inc; +--disable_query_log +connection server_1; +call mtr.add_suppression("Failed to rotate binary log"); +--enable_query_log + source include/rpl_end.inc; diff --git a/mysql-test/suite/rpl_raft/t/rpl_raft_gtid_purged_for_tailing.test b/mysql-test/suite/rpl_raft/t/rpl_raft_gtid_purged_for_tailing.test new file mode 100644 index 000000000000..87941944e593 --- /dev/null +++ b/mysql-test/suite/rpl_raft/t/rpl_raft_gtid_purged_for_tailing.test @@ -0,0 +1,98 @@ + +source ../include/raft_3_node.inc; + +connection server_1; +let $uuid1= `select variable_value from performance_schema.global_status where variable_name = 'Rpl_raft_peer_uuid'`; +create table t1 (a int primary key); +insert into t1 values (1); +flush logs; +flush logs; + +sleep 5; + +purge raft logs before now(); + +echo "Verifying gtid_purged_for_tailing and gtid_purged is same in leader"; + +replace_result $uuid1 uuid1; +select @@global.gtid_purged_for_tailing; + +replace_result $uuid1 uuid1; +select @@global.gtid_purged; + +connection server_2; +purge raft logs before now(); + +echo "Verifying table gtid_purged_for_tailing is not empty and gtid_purged is empty"; + +replace_result $uuid1 uuid1; +select @@global.gtid_purged_for_tailing; + +replace_result $uuid1 uuid1; +select @@global.gtid_purged; + +let $uuid2 = `select @@server_uuid`; + +connection server_1; +replace_result $uuid2 uuid2; +eval SET @@GLOBAL.RPL_RAFT_NEW_LEADER_UUID="$uuid2"; + +connection server_2; +# wait for new leader to become writable +let $rpl_connection_name = server_2; +source include/rpl_connection.inc; +let $max_sleep = 300; +while (!$_leader_writeable) { + let $_leader_writeable = `SELECT @@GLOBAL.READ_ONLY = + 0 AND @@GLOBAL.SUPER_READ_ONLY = 0`; + sleep 1; + let $max_sleep = $max_sleep - 1; + if ($max_sleep == 0) { + die "Timeout while waiting for read_only = 0 on leader"; + } +} + +echo "Check gtid_purged_for_tailing again after follower becoming leader"; +replace_result $uuid1 uuid1; +select @@global.gtid_purged_for_tailing; + +replace_result $uuid1 uuid1; +select @@global.gtid_purged; + +connection server_1; +echo "Check gtid_purged_for_tailing again after leader becoming follower"; +replace_result $uuid1 uuid1; +select @@global.gtid_purged_for_tailing; + +replace_result $uuid1 uuid1; +select @@global.gtid_purged_for_tailing; + +# clear and reset +connection server_2; +replace_result $uuid1 uuid1; +eval SET @@GLOBAL.RPL_RAFT_NEW_LEADER_UUID="$uuid1"; + +connection server_1; +# wait for new leader to become writable +let $rpl_connection_name = server_1; +source include/rpl_connection.inc; +let $max_sleep = 300; +while (!$_leader_writeable2) { + let $_leader_writeable2 = `SELECT @@GLOBAL.READ_ONLY = + 0 AND @@GLOBAL.SUPER_READ_ONLY = 0`; + sleep 1; + let $max_sleep = $max_sleep - 1; + if ($max_sleep == 0) { + die "Timeout while waiting for read_only = 0 on leader"; + } +} +drop table t1; + +let $sync_slave_connection= server_2; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_3; +source include/sync_slave_sql_with_master.inc; + +source include/rpl_end.inc; + +reset master; diff --git a/mysql-test/suite/rpl_raft/t/rpl_raft_leader_election_crash.test b/mysql-test/suite/rpl_raft/t/rpl_raft_leader_election_crash.test new file mode 100644 index 000000000000..2ad1b30c632b --- /dev/null +++ b/mysql-test/suite/rpl_raft/t/rpl_raft_leader_election_crash.test @@ -0,0 +1,83 @@ +source include/have_debug.inc; +source include/not_parallel.inc; +source ../include/raft_3_node.inc; + +connection server_1; +let $uuid1= `select variable_value from performance_schema.global_status where variable_name = 'Rpl_raft_peer_uuid'`; + +connection server_2; +let $uuid2= `select variable_value from performance_schema.global_status where variable_name = 'Rpl_raft_peer_uuid'`; + +connection server_3; +let $uuid3= `select variable_value from performance_schema.global_status where variable_name = 'Rpl_raft_peer_uuid'`; + +echo "server_1 is the initial leader"; +connection server_1; +select variable_value from performance_schema.global_status where variable_name = 'Rpl_raft_role'; + +echo "Stopping slave applier on all peers"; +connection server_1; +stop slave sql_thread; +connection server_2; +stop slave sql_thread; +connection server_3; +stop slave sql_thread; + +echo "Writing data on leader"; +connection server_1; +create table t1 (a int primary key auto_increment) engine = innodb; +insert into t1 values(); +insert into t1 values(); +insert into t1 values(); +select * from t1; + +echo "Setting debug symbol so that server_2 crashed on becoming a leader after switching logs"; +connection server_2; +set global debug="+d,crash_after_point_binlog_to_binlog"; + +echo "Stopping sql appliers on server_2" +stop slave sql_thread; + +connection server_1; +insert into t1 values(); +insert into t1 values(); +insert into t1 values(); +select * from t1; +echo "Transfering leadership: server_1 -> server_2"; +--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +replace_result $uuid2 uuid2; +eval set @@global.rpl_raft_new_leader_uuid = '$uuid2'; + +select sleep(1); +echo "Restarting server_2"; +let $rpl_server_number = 2; +--source include/rpl_start_server.inc +echo "Checking table values in server_2" +connection server_2; +select * from t1; + +select sleep(20); + +# Cleanup +--disable_query_log + +echo "Make server_1 the leader" +connection server_1; +set global rpl_raft_start_election = 1; +select sleep(10); +drop table t1; +set global rpl_raft_start_election = default; +select sleep(10); + +connection server_2; +start slave sql_thread; + +connection server_3; +start slave sql_thread; + +let $sync_slave_connection= server_2; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_3; +source include/sync_slave_sql_with_master.inc; + +source include/rpl_end.inc; diff --git a/mysql-test/suite/rpl_raft/t/rpl_raft_leader_election_simple.test b/mysql-test/suite/rpl_raft/t/rpl_raft_leader_election_simple.test index 00a0c005e7e8..ae7ce2bea63d 100644 --- a/mysql-test/suite/rpl_raft/t/rpl_raft_leader_election_simple.test +++ b/mysql-test/suite/rpl_raft/t/rpl_raft_leader_election_simple.test @@ -1,27 +1,5 @@ source ../include/raft_3_node.inc; ---disable_query_log -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - -# Create connections to server 4 and 5 (these are not in the ring) -let $rpl_server_number= 4; -let $rpl_connection_name= server_4; -source include/rpl_connect.inc; ---disable_query_log -connection server_4; -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - -let $rpl_server_number= 5; -let $rpl_connection_name= server_5; -source include/rpl_connect.inc; ---disable_query_log -connection server_5; -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - - connection server_1; let $uuid1= `select variable_value from performance_schema.global_status where variable_name = 'Rpl_raft_peer_uuid'`; connection server_2; diff --git a/mysql-test/suite/rpl_raft/t/rpl_raft_out_of_order_raft_log-master.opt b/mysql-test/suite/rpl_raft/t/rpl_raft_out_of_order_raft_log-master.opt new file mode 100644 index 000000000000..e7d69f419e4a --- /dev/null +++ b/mysql-test/suite/rpl_raft/t/rpl_raft_out_of_order_raft_log-master.opt @@ -0,0 +1 @@ +--local_infile=1 diff --git a/mysql-test/suite/rpl_raft/t/rpl_raft_out_of_order_raft_log-slave.opt b/mysql-test/suite/rpl_raft/t/rpl_raft_out_of_order_raft_log-slave.opt new file mode 100644 index 000000000000..e7d69f419e4a --- /dev/null +++ b/mysql-test/suite/rpl_raft/t/rpl_raft_out_of_order_raft_log-slave.opt @@ -0,0 +1 @@ +--local_infile=1 diff --git a/mysql-test/suite/rpl_raft/t/rpl_raft_out_of_order_raft_log.test b/mysql-test/suite/rpl_raft/t/rpl_raft_out_of_order_raft_log.test new file mode 100644 index 000000000000..d42f4062bc3b --- /dev/null +++ b/mysql-test/suite/rpl_raft/t/rpl_raft_out_of_order_raft_log.test @@ -0,0 +1,96 @@ +source ../include/raft_3_node.inc; + +call mtr.add_suppression("Out of order opid found"); + +connection server_1; +create table t1(a int primary key) engine = innodb; +let $sync_slave_connection= server_2; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_3; +source include/sync_slave_sql_with_master.inc; + +# stop sql thread on server2, we'll 1st reorder some trx and then start it again +connection server_2; +stop slave sql_thread; + +# execute some transactions and rotate the file +connection server_1; +insert into t1 values(1); +insert into t1 values(2); +insert into t1 values(3); +insert into t1 values(4); +flush binary logs; + +# mess with raft log on server2 and reorder the last two trxs +connection server_2; +let $MYSQLD_DATADIR = `select @@datadir`; +let $raft_log = query_get_value("SHOW SLAVE STATUS", "Relay_Log_File", 1); + +# store offsets of last 3 Xid events in a file +exec $MYSQL_BINLOG -vvv $MYSQLD_DATADIR/$raft_log | grep Xid | tr -s [:blank:] | cut -d' ' -f7 | tail -n3 > $MYSQLD_DATADIR/offsets; + +connection server_1; +set @sql_log_bin=0; +create table offsets(a int); +replace_result $MYSQLD_DATADIR MYSQLD_DATADIR; +eval load data local infile '$MYSQLD_DATADIR/offsets' into table offsets FIELDS TERMINATED BY '\t' LINES TERMINATED BY '\n'; +set @sql_log_bin=1; + +let $offset1= `select a from offsets limit 1`; +let $offset2= `select a from offsets limit 1, 1`; +let $size1= `select (select a from offsets limit 1, 1) - (select a from offsets limit 0, 1)`; +let $size2= `select (select a from offsets limit 2, 1) - (select a from offsets limit 1, 1)`; + +set @sql_log_bin=0; +drop table offsets; +set @sql_log_bin=1; + +# make a copy of orig raft log file +exec cp $MYSQLD_DATADIR/$raft_log $MYSQLD_DATADIR/orig_$raft_log; + +# create new raft log with last two trx out of order +exec dd if=$MYSQLD_DATADIR/$raft_log of=$MYSQLD_DATADIR/prefix_raft_log skip=0 count=$offset1 iflag=skip_bytes,count_bytes; +exec dd if=$MYSQLD_DATADIR/$raft_log of=$MYSQLD_DATADIR/penultimate_trx_raft_log skip=$offset1 count=$size1 iflag=skip_bytes,count_bytes; +exec dd if=$MYSQLD_DATADIR/$raft_log of=$MYSQLD_DATADIR/last_trx_raft_log skip=$offset2 count=$size2 iflag=skip_bytes,count_bytes; +exec cat $MYSQLD_DATADIR/prefix_raft_log $MYSQLD_DATADIR/last_trx_raft_log $MYSQLD_DATADIR/penultimate_trx_raft_log > $MYSQLD_DATADIR/$raft_log; + +# start sql thread +connection server_2; +source include/start_slave_sql.inc; + +# sql thread should fail +let $slave_sql_errno= convert_error(ER_SLAVE_RELAY_LOG_READ_FAILURE); +source include/wait_for_slave_sql_error.inc; + +# replace with original raft log and start again +exec mv $MYSQLD_DATADIR/orig_$raft_log $MYSQLD_DATADIR/$raft_log; + +let $rpl_server_number= 2; +source include/rpl_stop_server.inc; + +connection server_1; +flush binary logs; + +connection server_2; +let $rpl_server_number= 2; +let $rpl_server_parameters=--skip-slave-start=1; +source include/rpl_start_server.inc; + +connection server_2; +source include/start_slave.inc; + +remove_file $MYSQLD_DATADIR/offsets; +remove_file $MYSQLD_DATADIR/prefix_raft_log; +remove_file $MYSQLD_DATADIR/penultimate_trx_raft_log; +remove_file $MYSQLD_DATADIR/last_trx_raft_log; + +# cleanup +connection server_1; +drop table t1; +let $sync_slave_connection= server_2; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_3; +source include/sync_slave_sql_with_master.inc; + + +source include/rpl_end.inc; diff --git a/mysql-test/suite/rpl_raft/t/rpl_raft_purge_logs.test b/mysql-test/suite/rpl_raft/t/rpl_raft_purge_logs.test index 716c43379408..dbafcc1895c5 100644 --- a/mysql-test/suite/rpl_raft/t/rpl_raft_purge_logs.test +++ b/mysql-test/suite/rpl_raft/t/rpl_raft_purge_logs.test @@ -1,33 +1,4 @@ source ../include/raft_3_node.inc; ---disable_query_log -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - -# Create connections to server 4 and 5 (these are not in the ring) -let $rpl_server_number= 4; -let $rpl_connection_name= server_4; -source include/rpl_connect.inc; ---disable_query_log -connection server_4; -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - -let $rpl_server_number= 5; -let $rpl_connection_name= server_5; -source include/rpl_connect.inc; ---disable_query_log -connection server_5; -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - -connection server_1; -show status like 'rpl_raft_role'; - -connection server_2; -show status like 'rpl_raft_role'; - -connection server_3; -show status like 'rpl_raft_role'; connection server_1; create table t1 (a int primary key); diff --git a/mysql-test/suite/rpl_raft/t/rpl_raft_purge_raft_logs.test b/mysql-test/suite/rpl_raft/t/rpl_raft_purge_raft_logs.test index f8e2a253d421..788dd26f18b2 100644 --- a/mysql-test/suite/rpl_raft/t/rpl_raft_purge_raft_logs.test +++ b/mysql-test/suite/rpl_raft/t/rpl_raft_purge_raft_logs.test @@ -1,35 +1,8 @@ source include/have_debug.inc; source ../include/raft_3_node.inc; ---disable_query_log -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - -# Create connections to server 4 and 5 (these are not in the ring) -let $rpl_server_number= 4; -let $rpl_connection_name= server_4; -source include/rpl_connect.inc; ---disable_query_log -connection server_4; -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - -let $rpl_server_number= 5; -let $rpl_connection_name= server_5; -source include/rpl_connect.inc; ---disable_query_log -connection server_5; -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - -connection server_1; -show status like 'rpl_raft_role'; - -connection server_2; -show status like 'rpl_raft_role'; - -connection server_3; -show status like 'rpl_raft_role'; +CALL mtr.add_suppression(".*MYSQL_BIN_LOG::purge_logs was called with file *"); +CALL mtr.add_suppression(".*MYSQL_BIN_LOG::purge_logs raft plugin failed *"); connection server_1; create table t1 (a int primary key); @@ -265,6 +238,16 @@ source include/show_raft_logs.inc; echo "Verifying table on follower: server_3"; select count(*) from t1; +echo "Purging non existing file on leader: server_2"; +connection server_2; +flush logs; +let $binlog= query_get_value(SHOW MASTER STATUS, File, 1); +flush logs; +select sleep(10); +set session debug="+d,simulate_raft_plugin_purge_error"; +--error ER_LOG_PURGE_UNKNOWN_ERR +eval purge raft logs to '$binlog'; + connection server_2; echo "Transfering leadership: server_2 -> server_1"; let $rpl_raft_leader_number= 1; diff --git a/mysql-test/suite/rpl_raft/t/rpl_raft_purge_raft_logs_before_date.test b/mysql-test/suite/rpl_raft/t/rpl_raft_purge_raft_logs_before_date.test index cc474f6b9b00..c804ee107fe3 100644 --- a/mysql-test/suite/rpl_raft/t/rpl_raft_purge_raft_logs_before_date.test +++ b/mysql-test/suite/rpl_raft/t/rpl_raft_purge_raft_logs_before_date.test @@ -1,36 +1,6 @@ source include/have_debug.inc; source ../include/raft_3_node.inc; ---disable_query_log -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - -# Create connections to server 4 and 5 (these are not in the ring) -let $rpl_server_number= 4; -let $rpl_connection_name= server_4; -source include/rpl_connect.inc; ---disable_query_log -connection server_4; -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - -let $rpl_server_number= 5; -let $rpl_connection_name= server_5; -source include/rpl_connect.inc; ---disable_query_log -connection server_5; -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - -connection server_1; -show status like 'rpl_raft_role'; - -connection server_2; -show status like 'rpl_raft_role'; - -connection server_3; -show status like 'rpl_raft_role'; - connection server_1; create table t1 (a int primary key); diff --git a/mysql-test/suite/rpl_raft/t/rpl_raft_purged_gtids_dump_threads.test b/mysql-test/suite/rpl_raft/t/rpl_raft_purged_gtids_dump_threads.test new file mode 100644 index 000000000000..ab50e28435bf --- /dev/null +++ b/mysql-test/suite/rpl_raft/t/rpl_raft_purged_gtids_dump_threads.test @@ -0,0 +1,74 @@ +# Test auto position based tailing after election. After election we purge +# apply logs that make gtid_purged == gtid_executed even though all transcations +# exist in the raft log. + +source ../include/raft_3_node.inc; +source include/have_debug_sync.inc; +let $use_gtids= 1; + +# server4 will tail the leader +connection server_4; +RESET MASTER; +RESET SLAVE; +SET @@GLOBAL.ENABLE_RAFT_PLUGIN = 0; +replace_result $SERVER_MYPORT_1 SERVER_MYPORT_1; +eval CHANGE MASTER TO MASTER_HOST = '::1', MASTER_PORT = $SERVER_MYPORT_1, MASTER_USER = 'root', MASTER_CONNECT_RETRY = 1, MASTER_AUTO_POSITION = 1; +START SLAVE; + +# Execute some transactions +connection server_1; +let $uuid1= `select variable_value from performance_schema.global_status where variable_name = 'Rpl_raft_peer_uuid'`; +create table t1 (a int) engine = innodb; +insert into t1 values(1); +insert into t1 values(2); + +let $sync_slave_connection= server_4; +source include/sync_slave_sql_with_master.inc; + +# Trasfer leadership to server_2 (this will clear out the apply logs and make +# gtid_purged == gtid_executed +connection server_2; +let $uuid2= `select variable_value from performance_schema.global_status where variable_name = 'Rpl_raft_peer_uuid'`; +let $port2= `select @@global.port`; + +connection server_1; +let $rpl_raft_leader_number= 2; +source ../include/raft_promote_to_leader.inc; + +# Make server_4 tail server_2 +connection server_4; +STOP SLAVE; +replace_result $port2 port2; +eval change master to master_host='localhost', master_port=$port2, master_auto_position=1, master_user='root'; +source include/start_slave.inc; +--let $uuid4= `SELECT @@GLOBAL.SERVER_UUID` + +connection server_2; +# Note: This insert statement is required to move the last_acked to correct position. +insert into t1 values(3); +let $sync_slave_connection= server_4; +source include/sync_slave_sql_with_master.inc; + +connection server_4; +select * from t1; + +# Cleanup +connection server_2; +let $rpl_raft_leader_number= 1; +source ../include/raft_promote_to_leader.inc; + +connection server_1; +drop table t1; + +let $sync_slave_connection= server_2; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_3; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_4; +source include/sync_slave_sql_with_master.inc; + +connection server_4; +source include/stop_slave.inc; +reset slave all; + +source include/rpl_end.inc; diff --git a/mysql-test/suite/rpl_raft/t/rpl_raft_recover_raft_log-slave.opt b/mysql-test/suite/rpl_raft/t/rpl_raft_recover_raft_log-slave.opt new file mode 100644 index 000000000000..fa88d39211c9 --- /dev/null +++ b/mysql-test/suite/rpl_raft/t/rpl_raft_recover_raft_log-slave.opt @@ -0,0 +1 @@ +--initialize --innodb_page_size=16k diff --git a/mysql-test/suite/rpl_raft/t/rpl_raft_recover_raft_log.test b/mysql-test/suite/rpl_raft/t/rpl_raft_recover_raft_log.test new file mode 100644 index 000000000000..dcba4122f621 --- /dev/null +++ b/mysql-test/suite/rpl_raft/t/rpl_raft_recover_raft_log.test @@ -0,0 +1,77 @@ +source ../include/raft_3_node.inc; +connection server_1; +connection server_1; +create table t1 (a int primary key) engine = innodb; +insert into t1 values(1); +insert into t1 values(2); + +let $sync_slave_connection= server_2; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_3; +source include/sync_slave_sql_with_master.inc; + +connection server_2; +select * from t1; + +connection server_3; +select * from t1; + +connection server_2; +select sleep(10); + +# Expand this test later to cover multiple scenarios +let $server2_datadir = `select @@datadir`; +let $server2_raft_file = query_get_value("SHOW SLAVE STATUS", "Relay_Log_File", 1); +let $server2_raft_file_pos = query_get_value("SHOW SLAVE STATUS", "Relay_Log_Pos", 1); +echo "raft file: $server2_raft_file"; +echo "raft file pos: $server2_raft_file_pos"; +let $half = `select ROUND($server2_raft_file_pos / 2)`; +exec truncate -s $half $server2_datadir/$server2_raft_file; + +echo "Restarting server_2"; +let $rpl_server_number = 2; +source include/rpl_restart_server.inc; + +connection server_2; +# wait for raft plugin initialzed +sleep 10; +start slave sql_thread; + +connection server_1; +insert into t1 values(3); +insert into t1 values(4); +insert into t1 values(5); +insert into t1 values(6); +insert into t1 values(7); +insert into t1 values(8); +insert into t1 values(9); +insert into t1 values(10); +insert into t1 values(11); +insert into t1 values(12); + +let $sync_slave_connection= server_2; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_3; +source include/sync_slave_sql_with_master.inc; + +connection server_2; +select * from t1; + +connection server_3; +select * from t1; + +# cleanup +connection server_1; +drop table t1; + +let $sync_slave_connection= server_2; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_3; +source include/sync_slave_sql_with_master.inc; + +# Suppress known warnings. +--disable_query_log +connection server_1; +call mtr.add_suppression("Error reading GTIDs from relaylog:.*"); +--enable_query_log +source include/rpl_end.inc; diff --git a/mysql-test/suite/rpl_raft/t/rpl_raft_restart.test b/mysql-test/suite/rpl_raft/t/rpl_raft_restart.test index e9158c44d5c7..3a764222e5f8 100644 --- a/mysql-test/suite/rpl_raft/t/rpl_raft_restart.test +++ b/mysql-test/suite/rpl_raft/t/rpl_raft_restart.test @@ -1,28 +1,5 @@ source ../include/raft_3_node.inc; ---disable_query_log -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); -call mtr.add_suppression("Engine has seen trxs till file .*"); ---enable_query_log - -# Create connections to server 4 and 5 (these are not in the ring) -let $rpl_server_number= 4; -let $rpl_connection_name= server_4; -source include/rpl_connect.inc; ---disable_query_log -connection server_4; -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - -let $rpl_server_number= 5; -let $rpl_connection_name= server_5; -source include/rpl_connect.inc; ---disable_query_log -connection server_5; -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - - connection server_1; create table t1 (a int primary key) engine = innodb; insert into t1 values(1); diff --git a/mysql-test/suite/rpl_raft/t/rpl_raft_show_raft_logs.test b/mysql-test/suite/rpl_raft/t/rpl_raft_show_raft_logs.test index 4af1cdc25fb1..bfc32eb490c3 100644 --- a/mysql-test/suite/rpl_raft/t/rpl_raft_show_raft_logs.test +++ b/mysql-test/suite/rpl_raft/t/rpl_raft_show_raft_logs.test @@ -1,35 +1,5 @@ source ../include/raft_3_node.inc; ---disable_query_log -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - -# Create connections to server 4 and 5 (these are not in the ring) -let $rpl_server_number= 4; -let $rpl_connection_name= server_4; -source include/rpl_connect.inc; ---disable_query_log -connection server_4; -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - -let $rpl_server_number= 5; -let $rpl_connection_name= server_5; -source include/rpl_connect.inc; ---disable_query_log -connection server_5; -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - -connection server_1; -show status like 'rpl_raft_role'; - -connection server_2; -show status like 'rpl_raft_role'; - -connection server_3; -show status like 'rpl_raft_role'; - connection server_1; create table t1 (a int primary key); diff --git a/mysql-test/suite/rpl_raft/t/rpl_raft_show_slave_status.test b/mysql-test/suite/rpl_raft/t/rpl_raft_show_slave_status.test new file mode 100644 index 000000000000..0a8b56c49b84 --- /dev/null +++ b/mysql-test/suite/rpl_raft/t/rpl_raft_show_slave_status.test @@ -0,0 +1,44 @@ +source ../include/raft_3_node.inc; + +connection server_2; +let $master_uuid2 = query_get_value(SHOW SLAVE STATUS, Master_UUID, 1); +let $uuid2 = `select @@server_uuid`; + +connection server_1; +let $uuid1 = `select @@server_uuid`; +if ($master_uuid2 != $uuid1) +{ + echo 'Master_UUID: $master_uuid2' <> 'server_uuid: $uuid1'; + die Failed because Master_UUID is not equal to server_uuid; +} +replace_result $uuid2 uuid2; +eval SET @@GLOBAL.RPL_RAFT_NEW_LEADER_UUID="$uuid2"; + +# wait for new leader to become writable +connection server_2; +let $rpl_connection_name = server_2; +source include/rpl_connection.inc; +let $max_sleep = 300; +while (!$_leader_writeable) { + let $_leader_writeable = `SELECT @@GLOBAL.READ_ONLY = + 0 AND @@GLOBAL.SUPER_READ_ONLY = 0`; + sleep 1; + let $max_sleep = $max_sleep - 1; + if ($max_sleep == 0) { + die "Timeout while waiting for read_only = 0 on leader"; + } +} + +connection server_1; +let $master_uuid1 = query_get_value(SHOW SLAVE STATUS, Master_UUID, 1); +if ($master_uuid1 != $uuid2) +{ + echo 'Master_UUID: $master_uuid1' <> 'server_uuid: $uuid2'; + die Failed because Master_UUID is not equal to server_uuid; +} + +connection server_2; +replace_result $uuid1 uuid1; +eval set @@GLOBAL.RPL_RAFT_NEW_LEADER_UUID="$uuid1"; + +source include/rpl_end.inc; diff --git a/mysql-test/suite/rpl_raft/t/rpl_raft_slave_out_of_order_commit.test b/mysql-test/suite/rpl_raft/t/rpl_raft_slave_out_of_order_commit.test index 3166953b3ecb..bb596409c2fa 100644 --- a/mysql-test/suite/rpl_raft/t/rpl_raft_slave_out_of_order_commit.test +++ b/mysql-test/suite/rpl_raft/t/rpl_raft_slave_out_of_order_commit.test @@ -2,27 +2,6 @@ source ../include/raft_3_node.inc; source include/have_binlog_format_row.inc; source include/only_mts_slave_parallel_type_dependency.inc; ---disable_query_log -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - -# Create connections to server 4 and 5 (these are not in the ring) -let $rpl_server_number= 4; -let $rpl_connection_name= server_4; -source include/rpl_connect.inc; ---disable_query_log -connection server_4; -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - -let $rpl_server_number= 5; -let $rpl_connection_name= server_5; -source include/rpl_connect.inc; ---disable_query_log -connection server_5; -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - # create tables in two different databases connection server_1; create database d1; @@ -38,18 +17,19 @@ source include/sync_slave_sql_with_master.inc; # block writes on d1.t1 in server_2 connection server_2; +SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED; begin; update d1.t1 set a = a + 100; let $s2_uuid= `select variable_value from performance_schema.global_status where variable_name = 'Rpl_raft_peer_uuid'`; connection server_1; -update d1.t1 set a = a + 1; # this will be blocked on sever2 +update d1.t1 set a = a + 1; # this will be blocked on server2 insert into d2.t2 values(2); let $s1_uuid= `select variable_value from performance_schema.global_status where variable_name = 'Rpl_raft_peer_uuid'`; # wait for 2nd insert for d2.t2 to go through connection server_2; -let $wait_condition= select count(*) = 2 from d2.t2; +let $wait_condition = select count(*) = 2 from d2.t2; let $wait_timeout= 120; source include/wait_condition.inc; select * from d2.t2; diff --git a/mysql-test/suite/rpl_raft/t/rpl_raft_wait_for_ack.test b/mysql-test/suite/rpl_raft/t/rpl_raft_wait_for_ack.test index bebcaa08931c..519d454762f9 100644 --- a/mysql-test/suite/rpl_raft/t/rpl_raft_wait_for_ack.test +++ b/mysql-test/suite/rpl_raft/t/rpl_raft_wait_for_ack.test @@ -1,29 +1,6 @@ -# TODO(pgl) Excluding this test case till the time we have debug raft plugin in tools directory. -source include/have_nodebug.inc; source include/have_debug.inc; source ../include/raft_3_node.inc; ---disable_query_log -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - -# Create connections to server 4 and 5 (these are not in the ring) -let $rpl_server_number= 4; -let $rpl_connection_name= server_4; -source include/rpl_connect.inc; ---disable_query_log -connection server_4; -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - -let $rpl_server_number= 5; -let $rpl_connection_name= server_5; -source include/rpl_connect.inc; ---disable_query_log -connection server_5; -call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*"); ---enable_query_log - # Connect server 4 and 5 using COM_BINLOG_DUMP_GTID connection server_4; RESET MASTER; diff --git a/mysql-test/suite/rpl_raft/t/rpl_stepdown_trim.test b/mysql-test/suite/rpl_raft/t/rpl_stepdown_trim.test new file mode 100644 index 000000000000..621bc8e01b3f --- /dev/null +++ b/mysql-test/suite/rpl_raft/t/rpl_stepdown_trim.test @@ -0,0 +1,82 @@ +source ../include/raft_3_node.inc; + +connection server_1; +--disable_query_log +call mtr.add_suppression("Run function 'before_commit' in plugin 'RPL_RAFT' failed"); +--enable_query_log + +connection server_1; +--let $outfile1= `SELECT @@pid_file` +connection server_2; +--let $outfile2= `SELECT @@pid_file` +connection server_3; +--let $outfile3= `SELECT @@pid_file` + +connection server_1; +# InnoDB explicitly so that rollbacks are supported +create table t1 (a int primary key) engine=innodb; +insert into t1 values(1); +insert into t1 values(2); + +let $sync_slave_connection= server_2; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_3; +source include/sync_slave_sql_with_master.inc; + +connection server_1; +insert into t1 values (3), (4), (5); + +# stop the 2 slaves so that they don't receive the last +# trxs from the master. +--exec kill -STOP `cat $outfile2` +--exec kill -STOP `cat $outfile3` + +# This makes sure that the slaves have stopped +--sleep 5 + +connection server_1; +SET net_write_timeout = 1; +send insert into t1 values (6), (7), (8); +--echo "Now triggering DMP by stopping current MASTER" +--echo "Now triggering DMP by reenabling current SLAVES" + +# Now stop the master and reenable the slaves so that +# the master stops and the slaves re-enable to start election +--exec kill -STOP `cat $outfile1` +--exec kill -CONT `cat $outfile2` +--exec kill -CONT `cat $outfile3` + +--sleep 5 +--echo "Will restart previous master to trigger trimming in 15 seconds" +--exec kill -CONT `cat $outfile1` + +--error 1180 +reap + +let $rpl_server_count= 3; +--source ../include/raft_leadership.inc +--sleep 5 + +connection server_1; +show status like 'rpl_raft_role'; +let $wait_condition= select @@global.read_only = 0; +source include/wait_condition.inc; +show variables like 'read_only'; + +connection server_2; +select * from t1; + +connection server_3; +select * from t1; + +connection server_1; +select * from t1; + +drop table t1; + +let $sync_slave_connection= server_2; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_3; +source include/sync_slave_sql_with_master.inc; + +source include/rpl_end.inc; diff --git a/mysql-test/suite/sys_vars/r/abort_on_raft_purge_error_basic.result b/mysql-test/suite/sys_vars/r/abort_on_raft_purge_error_basic.result new file mode 100644 index 000000000000..67972558ffee --- /dev/null +++ b/mysql-test/suite/sys_vars/r/abort_on_raft_purge_error_basic.result @@ -0,0 +1,19 @@ +Default value of abort_on_raft_purge_error is false +SELECT @@global.abort_on_raft_purge_error; +@@global.abort_on_raft_purge_error +0 +SELECT @@session.abort_on_raft_purge_error; +ERROR HY000: Variable 'abort_on_raft_purge_error' is a GLOBAL variable +Expected error 'Variable is a GLOBAL variable' +SET @@global.abort_on_raft_purge_error = true; +SELECT @@global.abort_on_raft_purge_error; +@@global.abort_on_raft_purge_error +1 +SET @@global.abort_on_raft_purge_error = false; +SELECT @@global.abort_on_raft_purge_error; +@@global.abort_on_raft_purge_error +0 +SET @@global.abort_on_raft_purge_error = default; +SELECT @@global.abort_on_raft_purge_error; +@@global.abort_on_raft_purge_error +0 diff --git a/mysql-test/suite/sys_vars/r/gtid_purged_for_tailing_basic.result b/mysql-test/suite/sys_vars/r/gtid_purged_for_tailing_basic.result new file mode 100644 index 000000000000..f8b649dfb64d --- /dev/null +++ b/mysql-test/suite/sys_vars/r/gtid_purged_for_tailing_basic.result @@ -0,0 +1,24 @@ +SET @start_global_value = @@global.gtid_purged_for_tailing; +SELECT @start_global_value; +@start_global_value + +select @@global.gtid_purged_for_tailing; +@@global.gtid_purged_for_tailing + +select @@session.gtid_purged_for_tailing; +ERROR HY000: Variable 'gtid_purged_for_tailing' is a GLOBAL variable +show global variables like 'gtid_purged_for_tailing'; +Variable_name Value +gtid_purged_for_tailing +show session variables like 'gtid_purged_for_tailing'; +Variable_name Value +gtid_purged_for_tailing +select * from performance_schema.global_variables where variable_name='gtid_purged_for_tailing'; +VARIABLE_NAME VARIABLE_VALUE +gtid_purged_for_tailing +select * from performance_schema.session_variables where variable_name='gtid_purged_for_tailing'; +VARIABLE_NAME VARIABLE_VALUE +gtid_purged_for_tailing +SET @@global.gtid_purged_for_tailing='11111111-1111-1111-1111-111111111111:2'; +ERROR HY000: Variable 'gtid_purged_for_tailing' is a read only variable +Expected error 'Read only variable' diff --git a/mysql-test/suite/sys_vars/r/recover_raft_log_basic.result b/mysql-test/suite/sys_vars/r/recover_raft_log_basic.result new file mode 100644 index 000000000000..f0fcccf0af76 --- /dev/null +++ b/mysql-test/suite/sys_vars/r/recover_raft_log_basic.result @@ -0,0 +1,42 @@ +SET @start_recover_raft_log = @@global.recover_raft_log; +SELECT @start_recover_raft_log; +@start_recover_raft_log +1 +SET @@global.recover_raft_log = DEFAULT; +SELECT @@global.recover_raft_log; +@@global.recover_raft_log +1 +SET @@global.recover_raft_log = false; +SELECT @@global.recover_raft_log; +@@global.recover_raft_log +0 +SET @@global.recover_raft_log = true; +SELECT @@global.recover_raft_log; +@@global.recover_raft_log +1 +SET @@global.recover_raft_log = 1; +SELECT @@global.recover_raft_log; +@@global.recover_raft_log +1 +SET @@global.recover_raft_log = 0; +SELECT @@global.recover_raft_log; +@@global.recover_raft_log +0 +SET @@global.recover_raft_log = -1; +ERROR 42000: Variable 'recover_raft_log' can't be set to the value of '-1' +SELECT @@global.recover_raft_log; +@@global.recover_raft_log +0 +SET @@global.recover_raft_log = 100; +ERROR 42000: Variable 'recover_raft_log' can't be set to the value of '100' +SELECT @@global.recover_raft_log; +@@global.recover_raft_log +0 +SET @@session.recover_raft_log = 10; +ERROR HY000: Variable 'recover_raft_log' is a GLOBAL variable and should be set with SET GLOBAL +SELECT @@session.recover_raft_log; +ERROR HY000: Variable 'recover_raft_log' is a GLOBAL variable +SET @@global.recover_raft_log = @start_recover_raft_log; +SELECT @@global.recover_raft_log; +@@global.recover_raft_log +1 diff --git a/mysql-test/suite/sys_vars/t/abort_on_raft_purge_error_basic.test b/mysql-test/suite/sys_vars/t/abort_on_raft_purge_error_basic.test new file mode 100644 index 000000000000..b1f8c32b7254 --- /dev/null +++ b/mysql-test/suite/sys_vars/t/abort_on_raft_purge_error_basic.test @@ -0,0 +1,26 @@ +-- source include/load_sysvars.inc + +#### +# Verify default value false +#### +--echo Default value of abort_on_raft_purge_error is false +SELECT @@global.abort_on_raft_purge_error; + +#### +# Verify that this is not a session variable +#### +--Error ER_INCORRECT_GLOBAL_LOCAL_VAR +SELECT @@session.abort_on_raft_purge_error; +--echo Expected error 'Variable is a GLOBAL variable' + +#### +## Verify that the variable can be updated correctly +#### +SET @@global.abort_on_raft_purge_error = true; +SELECT @@global.abort_on_raft_purge_error; + +SET @@global.abort_on_raft_purge_error = false; +SELECT @@global.abort_on_raft_purge_error; + +SET @@global.abort_on_raft_purge_error = default; +SELECT @@global.abort_on_raft_purge_error; diff --git a/mysql-test/suite/sys_vars/t/gtid_purged_for_tailing_basic.test b/mysql-test/suite/sys_vars/t/gtid_purged_for_tailing_basic.test new file mode 100644 index 000000000000..c93f7566e337 --- /dev/null +++ b/mysql-test/suite/sys_vars/t/gtid_purged_for_tailing_basic.test @@ -0,0 +1,26 @@ + +SET @start_global_value = @@global.gtid_purged_for_tailing; +SELECT @start_global_value; + +# +# exists as global only +# +select @@global.gtid_purged_for_tailing; +--error ER_INCORRECT_GLOBAL_LOCAL_VAR +select @@session.gtid_purged_for_tailing; +show global variables like 'gtid_purged_for_tailing'; +show session variables like 'gtid_purged_for_tailing'; +select * from performance_schema.global_variables where variable_name='gtid_purged_for_tailing'; +select * from performance_schema.session_variables where variable_name='gtid_purged_for_tailing'; + +# +# show that it's read only +# +--error ER_INCORRECT_GLOBAL_LOCAL_VAR +SET @@global.gtid_purged_for_tailing='11111111-1111-1111-1111-111111111111:2'; +--echo Expected error 'Read only variable' + + +# +# See rpl_raft_gtid_purged_for_tailing.test for a comprehensive test case. +# diff --git a/mysql-test/suite/sys_vars/t/recover_raft_log_basic.test b/mysql-test/suite/sys_vars/t/recover_raft_log_basic.test new file mode 100644 index 000000000000..01b4517b2bb6 --- /dev/null +++ b/mysql-test/suite/sys_vars/t/recover_raft_log_basic.test @@ -0,0 +1,32 @@ +SET @start_recover_raft_log = @@global.recover_raft_log; +SELECT @start_recover_raft_log; + +SET @@global.recover_raft_log = DEFAULT; +SELECT @@global.recover_raft_log; + +SET @@global.recover_raft_log = false; +SELECT @@global.recover_raft_log; + +SET @@global.recover_raft_log = true; +SELECT @@global.recover_raft_log; + +SET @@global.recover_raft_log = 1; +SELECT @@global.recover_raft_log; + +SET @@global.recover_raft_log = 0; +SELECT @@global.recover_raft_log; + +--Error ER_WRONG_VALUE_FOR_VAR +SET @@global.recover_raft_log = -1; +SELECT @@global.recover_raft_log; +--Error ER_WRONG_VALUE_FOR_VAR +SET @@global.recover_raft_log = 100; +SELECT @@global.recover_raft_log; + +--ERROR ER_GLOBAL_VARIABLE +SET @@session.recover_raft_log = 10; +--ERROR ER_INCORRECT_GLOBAL_LOCAL_VAR +SELECT @@session.recover_raft_log; + +SET @@global.recover_raft_log = @start_recover_raft_log; +SELECT @@global.recover_raft_log; diff --git a/mysql-test/t/all_persisted_variables.test b/mysql-test/t/all_persisted_variables.test index 76d314ed0ab5..1a13ba4ef1df 100644 --- a/mysql-test/t/all_persisted_variables.test +++ b/mysql-test/t/all_persisted_variables.test @@ -56,6 +56,7 @@ let $total_excluded_vars=`SELECT COUNT(*) FROM performance_schema.global_variabl 'disable_raft_log_repointing', 'group_replication_plugin_hooks', 'gtid_committed', +'gtid_purged_for_tailing', 'have_boringssl', 'innodb_buffer_pool_populate', 'innodb_sync_pool_size', diff --git a/scripts/mysqld_safe.sh b/scripts/mysqld_safe.sh old mode 100644 new mode 100755 index dc63874a9cdc..242d13278b45 --- a/scripts/mysqld_safe.sh +++ b/scripts/mysqld_safe.sh @@ -493,7 +493,7 @@ else fi if test -z "$MYSQL_HOME" -then +then MYSQL_HOME=$MY_BASEDIR_VERSION fi export MYSQL_HOME @@ -599,7 +599,7 @@ then # mysqld does not add ".err" to "--log-error=foo."; it considers a # trailing "." as an extension - + if expr "$err_log" : '.*\.[^/]*$' > /dev/null then : @@ -975,6 +975,11 @@ fast_restart=0 max_fast_restarts=5 # flag whether a usable sleep command exists have_sleep=1 + +cur_retry_times=0 +max_restart_a_day=10 +prev_date=`date +%D` + while true do start_time=`date +%M%S` @@ -1042,6 +1047,41 @@ do fi + # stop retrying too ofen and rocksdb filling up txlogs + # + # 1. Reset current retry times in a new day + cur_date=`date +%D` + if test $cur_date != $prev_date + then + cur_retry_times=0 + prev_date=`date +%D` + fi + + # 2. Sleep 2,4,8,16... seconds between restart + # Sleep 24 hours if reaching 10 retries in a day + cur_retry_times=`expr $cur_retry_times + 1` + if test ! -f /tmp/disable_mysqld_restart_throttle \ + && test "$cur_retry_times" -gt "$max_restart_a_day" + then + log_notice "Throttling restart after 10 restarts: Sleep 1 day" + sleep 86400 + if [ $? = 137 ]; + then + log_notice "Sleep was likely interrupted from outside. Will exit loop" + break + fi + else + sleep_time=$((1<<${cur_retry_times})) + log_notice "Throttling restart after $cur_retry_times restarts: Sleep $sleep_time seconds" + sleep $sleep_time + if [ $? = 137 ]; + then + log_notice "Sleep was likely interrupted from outside. Will exit loop" + break + fi + fi + + # Note: the following code is not needed after exponential backoff # sanity check if time reading is sane and there's sleep if test $end_time -gt 0 -a $have_sleep -gt 0 then @@ -1079,8 +1119,8 @@ do log_notice "Number of processes running now: $numofproces" I=1 while test "$I" -le "$numofproces" - do - PROC=`ps xaww | grep "$ledir/$MYSQLD\>" | grep -v "grep" | grep "pid-file=$pid_file" | sed -n '$p'` + do + PROC=`ps xaww | grep "$ledir/$MYSQLD\>" | grep -v "grep" | grep "pid-file=$pid_file" | sed -n '$p'` for T in $PROC do diff --git a/sql/binlog.cc b/sql/binlog.cc index e10521fbab1c..d9c1a4eb3ee2 100644 --- a/sql/binlog.cc +++ b/sql/binlog.cc @@ -56,6 +56,7 @@ #include #include #include +#include #include "dur_prop.h" #include "libbinlogevents/include/compression/base.h" @@ -105,6 +106,7 @@ #include "sql/psi_memory_key.h" #include "sql/query_options.h" #include "sql/replication.h" +#include "sql/rpl_binlog_sender.h" #include "sql/rpl_filter.h" #include "sql/rpl_gtid.h" #include "sql/rpl_handler.h" // RUN_HOOK @@ -184,7 +186,10 @@ const char *hlc_wait_timeout_ms = "hlc_wait_timeout_ms"; ulong rpl_read_size; bool rpl_semi_sync_source_enabled = false; +latency_histogram histogram_raft_trx_wait; + MYSQL_BIN_LOG mysql_bin_log(&sync_binlog_period); +Dump_log dump_log; static int binlog_init(void *p); static int binlog_start_trans_and_stmt(THD *thd, Log_event *start_event); @@ -203,7 +208,8 @@ static bool binlog_recover(Binlog_file_reader *binlog_file_reader, my_off_t *valid_pos, Gtid *binlog_max_gtid, char *engine_binlog_file, my_off_t *engine_binlog_pos, - const std::string &cur_binlog_file); + const std::string &cur_binlog_file, + my_off_t *first_gtid_start_pos); static void binlog_prepare_row_images(const THD *thd, TABLE *table, bool is_update); static bool is_loggable_xa_prepare(THD *thd); @@ -1440,6 +1446,9 @@ static int binlog_init(void *p) { binlog_hton->recover_binlog_pos = binlog_dummy_recover_binlog_pos; binlog_hton->recover = binlog_dummy_recover; binlog_hton->flags = HTON_NOT_USER_SELECTABLE | HTON_HIDDEN; + + latency_histogram_init(&histogram_raft_trx_wait, "125us"); + return 0; } @@ -1821,9 +1830,9 @@ bool MYSQL_BIN_LOG::write_transaction(THD *thd, binlog_cache_data *cache_data, thd->commit_consensus_error = false; // TODO(luqun): perf concern? merge in plugin? cache_data->get_cache()->copy_to(temp_binlog_cache.get()); - ret = RUN_HOOK(raft_replication, before_flush, - (thd, temp_binlog_cache->get_io_cache(), - RaftReplicateMsgOpType::OP_TYPE_TRX)); + ret = RUN_HOOK_STRICT(raft_replication, before_flush, + (thd, temp_binlog_cache->get_io_cache(), + RaftReplicateMsgOpType::OP_TYPE_TRX)); DBUG_EXECUTE_IF("fail_binlog_flush_raft", { ret = 1; }); @@ -3410,11 +3419,12 @@ static bool binlog_savepoint_rollback_can_release_mdl(handlerton *, THD *thd) { */ class Adjust_offset : public Do_THD_Impl { public: - Adjust_offset(my_off_t value) : m_purge_offset(value) {} + Adjust_offset(my_off_t value, bool is_relay_log) + : m_purge_offset(value), m_relay_log(is_relay_log) {} void operator()(THD *thd) override { - LOG_INFO *linfo; + LOG_INFO *linfo = thd->current_linfo; mysql_mutex_lock(&thd->LOCK_thd_data); - if ((linfo = thd->current_linfo)) { + if (linfo && (!enable_raft_plugin || linfo->is_relay_log == m_relay_log)) { /* Index file offset can be less that purge offset only if we just started reading the index file. In that case @@ -3430,6 +3440,25 @@ class Adjust_offset : public Do_THD_Impl { private: my_off_t m_purge_offset; + bool m_relay_log; +}; + +class Adjust_linfo_in_dump_thread : public Do_THD_Impl { + public: + explicit Adjust_linfo_in_dump_thread(bool is_relay_log) { + m_relay_log = is_relay_log; + } + virtual void operator()(THD *thd) override { + LOG_INFO *linfo = thd->current_linfo; + if (linfo && linfo->is_used_by_dump_thd) { + mysql_mutex_lock(&thd->LOCK_thd_data); + linfo->is_relay_log = m_relay_log; + mysql_mutex_unlock(&thd->LOCK_thd_data); + } + } + + private: + bool m_relay_log; }; /* @@ -3437,7 +3466,7 @@ class Adjust_offset : public Do_THD_Impl { SYNOPSIS adjust_linfo_offsets() - purge_offset Number of bytes removed from start of log index file + purge_offset Number of bytes removed from start of log index file NOTES - This is called when doing a PURGE when we delete lines from the @@ -3452,11 +3481,17 @@ class Adjust_offset : public Do_THD_Impl { in the binary log file with flush_relay_log_info. Now they sync is done for next read. */ -static void adjust_linfo_offsets(my_off_t purge_offset) { - Adjust_offset adjust_offset(purge_offset); +static void adjust_linfo_offsets(my_off_t purge_offset, bool is_relay_log) { + Adjust_offset adjust_offset(purge_offset, is_relay_log); Global_THD_manager::get_instance()->do_for_all_thd(&adjust_offset); } +static void adjust_linfo_in_dump_threads(bool is_relay_log) { + Adjust_linfo_in_dump_thread adjust_linfo_in_dump_thread(is_relay_log); + Global_THD_manager::get_instance()->do_for_all_thd( + &adjust_linfo_in_dump_thread); +} + /** This class implements Call back function for do_for_all_thd(). It is called for each thd in thd list to count @@ -4087,7 +4122,7 @@ bool show_binlog_events(THD *thd, MYSQL_BIN_LOG *binary_log) { Protocol *protocol = thd->get_protocol(); List field_list; std::string errmsg; - LOG_INFO linfo; + LOG_INFO linfo(binary_log->is_relay_log); DBUG_TRACE; @@ -4335,6 +4370,7 @@ MYSQL_BIN_LOG::MYSQL_BIN_LOG(uint *sync_period, bool relay_log) m_binlog_file(new Binlog_ofile()), m_key_LOCK_log(key_LOG_LOCK_log), bytes_written(0), + lost_gtid_for_tailing(""), file_id(1), sync_period_ptr(sync_period), sync_counter(0), @@ -4379,6 +4415,7 @@ void MYSQL_BIN_LOG::cleanup() { mysql_mutex_destroy(&LOCK_binlog_end_pos); mysql_mutex_destroy(&LOCK_xids); mysql_mutex_destroy(&LOCK_non_xid_trxs); + mysql_mutex_destroy(&LOCK_lost_gtids_for_tailing); mysql_cond_destroy(&update_cond); mysql_cond_destroy(&m_prep_xids_cond); mysql_cond_destroy(&non_xid_trxs_cond); @@ -4406,6 +4443,8 @@ void MYSQL_BIN_LOG::init_pthread_objects() { mysql_mutex_init(m_key_LOCK_binlog_end_pos, &LOCK_binlog_end_pos, MY_MUTEX_INIT_FAST); mysql_mutex_init(m_key_LOCK_xids, &LOCK_xids, MY_MUTEX_INIT_FAST); + mysql_mutex_init(m_key_LOCK_lost_gtids_for_tailing, + &LOCK_lost_gtids_for_tailing, MY_MUTEX_INIT_FAST); mysql_cond_init(m_key_update_cond, &update_cond); mysql_cond_init(m_key_prep_xids_cond, &m_prep_xids_cond); mysql_cond_init(m_key_non_xid_trxs_cond, &non_xid_trxs_cond); @@ -4828,7 +4867,8 @@ int MYSQL_BIN_LOG::init_index_file() { // NO_LINT_DEBUG sql_print_information( "Binlog apply index file exists. Recovering mysqld " - "based on binlog apply index file"); + "based on binlog apply index file: %s", + opt_applylog_index_name); index_file_name = opt_applylog_index_name; log_file_name = opt_apply_logname; is_apply_log = true; @@ -4836,7 +4876,8 @@ int MYSQL_BIN_LOG::init_index_file() { // NO_LINT_DEBUG sql_print_information( "Binlog apply index file does not exist. Recovering " - "mysqld based on binlog index file"); + "mysqld based on binlog index file: %s", + opt_binlog_index_name); index_file_name = opt_binlog_index_name; log_file_name = opt_bin_logname; } @@ -5533,7 +5574,7 @@ bool MYSQL_BIN_LOG::find_first_log_not_in_gtid_set(char *binlog_file_name, /* * This is a limited version of init_gtid_sets which is only * called from binlog_change_to_apply. - * Needs to be called under LOCK_log and LOCK_index held. + * Needs to be called LOCK_index held. * The previous_gtid_set_map is cleared and reinitialized from * the index file contents. */ @@ -5544,7 +5585,6 @@ bool MYSQL_BIN_LOG::init_prev_gtid_sets_map() { std::pair iterator; DBUG_ENTER("MYSQL_BIN_LOG::init_prev_gtid_sets_map"); - mysql_mutex_assert_owner(&LOCK_log); mysql_mutex_assert_owner(&LOCK_index); // clear the map as it is being reset @@ -5593,6 +5633,7 @@ bool MYSQL_BIN_LOG::init_prev_gtid_sets_map() { } end: + update_lost_gtid_for_tailing(); DBUG_PRINT("info", ("returning %d", error)); DBUG_RETURN(error != 0 ? true : false); } @@ -5798,8 +5839,16 @@ bool MYSQL_BIN_LOG::init_gtid_sets(Gtid_set *all_gtids, Gtid_set *lost_gtids, "Could not get the transaction log file name from the engine. " "Using the latest for initializing mysqld state"); - log_file_to_read.assign(last_binlog_file_with_gtids); - max_pos = this->engine_binlog_pos; + // In innodb, engine_binlog_file is populated _only_ when there are + // trxs to recover (i.e trxs are in prepared state) during startup + // and engine recovery. Hence, engine_binlog_file being empty indicates + // that engine's state is up-to-date with the latest binlog file and we + // can include all gtids in the latest binlog file for calculating + // executed-gtid-set + if (strlen(engine_binlog_file) == 0) + max_pos = ULLONG_MAX; + else + max_pos = this->first_gtid_start_pos; } else { // Initializing gtid_sets based on engine binlog position is fine since // idempotent recovery will fill in the holes @@ -5938,7 +5987,9 @@ bool MYSQL_BIN_LOG::init_gtid_sets(Gtid_set *all_gtids, Gtid_set *lost_gtids, } } } + end: + update_lost_gtid_for_tailing(); if (all_gtids) all_gtids->dbug_print("all_gtids"); if (lost_gtids) lost_gtids->dbug_print("lost_gtids"); if (need_lock) { @@ -5969,7 +6020,8 @@ bool MYSQL_BIN_LOG::open_binlog( const char *log_name, const char *new_name, ulong max_size_arg, bool null_created_arg, bool need_lock_index, bool need_sid_lock, Format_description_log_event *extra_description_event, - uint32 new_index_number, RaftRotateInfo *raft_rotate_info) { + uint32 new_index_number, RaftRotateInfo *raft_rotate_info, + bool need_end_log_pos_lock) { // lock_index must be acquired *before* sid_lock. assert(need_sid_lock || !need_lock_index); DBUG_TRACE; @@ -6026,6 +6078,21 @@ bool MYSQL_BIN_LOG::open_binlog( bool write_file_name_to_index_file = false; bool raft_noop = raft_rotate_info && raft_rotate_info->noop; + /* + + * Which rotates are initiated by the plugin and happen + * in the raft listener queue thread? + * + * 1. Any rotate which is a post_append (i.e. normal rotate) + * 2. Any rotate which the plugin asks the mysql to initiate + * via injecting an operation in the listener queue. + * i.e. 2.1 - No-Op messages + * 2.2 - Config Change messages. + */ + bool rotate_in_listener_context = + raft_rotate_info && + (raft_rotate_info->post_append || raft_rotate_info->noop || + raft_rotate_info->config_change_rotate); /* This must be before goto err. */ #ifndef NDEBUG binary_log_debug::debug_pretend_version_50034_in_binlog = @@ -6069,6 +6136,10 @@ bool MYSQL_BIN_LOG::open_binlog( if (!s.is_valid()) goto err; s.dont_set_created = null_created_arg; + // Since start time of listener thread is not correct, + // we need to explicitly set event timestamp otherwise, + // the mysqlbinlog output will show UTC-0 for FD/PGTID/MD event + if (rotate_in_listener_context) s.common_header->when.tv_sec = my_time(0); /* Set LOG_EVENT_RELAY_LOG_F flag for relay log's FD */ if (is_relay_log && !raft_noop) s.set_relay_log_event(); if (write_event_to_binlog(&s)) goto err; @@ -6148,6 +6219,9 @@ bool MYSQL_BIN_LOG::open_binlog( DBUG_PRINT("info", ("Generating PREVIOUS_GTIDS for %s file.", is_relay_log ? "relaylog" : "binlog")); Previous_gtids_log_event prev_gtids_ev(previous_logged_gtids); + + if (rotate_in_listener_context) + prev_gtids_ev.common_header->when.tv_sec = my_time(0); // TODO(pgl) : confirm if raft_noop check required here. if (!raft_noop && is_relay_log) prev_gtids_ev.set_relay_log_event(); if (need_sid_lock) sid_lock->unlock(); @@ -6166,6 +6240,8 @@ bool MYSQL_BIN_LOG::open_binlog( current_hlc = mysql_bin_log.get_current_hlc(); } Metadata_log_event metadata_ev(current_hlc); + if (rotate_in_listener_context) + metadata_ev.common_header->when.tv_sec = my_time(0); if (raft_rotate_info) { metadata_ev.set_raft_prev_opid(raft_rotate_info->rotate_opid.first, raft_rotate_info->rotate_opid.second); @@ -6244,7 +6320,7 @@ bool MYSQL_BIN_LOG::open_binlog( close_purge_index_file(); - update_binlog_end_pos(); + update_binlog_end_pos(need_end_log_pos_lock); my_free(previous_gtid_set_buffer); return false; @@ -6267,7 +6343,8 @@ bool MYSQL_BIN_LOG::open_binlog( } bool MYSQL_BIN_LOG::open_existing_binlog(const char *log_name, - ulong max_size_arg) { + ulong max_size_arg, + bool need_end_log_pos_lock) { DBUG_ENTER("MYSQL_BIN_LOG::open_existing_binlog(const char *, ...)"); DBUG_PRINT("enter", ("name: %s", log_name)); @@ -6295,8 +6372,8 @@ bool MYSQL_BIN_LOG::open_existing_binlog(const char *log_name, DBUG_RETURN(1); } - if (!(this->name = - my_strdup(key_memory_MYSQL_LOG_name, log_name, MYF(MY_WME)))) { + my_free(name); + if (!(name = my_strdup(key_memory_MYSQL_LOG_name, log_name, MYF(MY_WME)))) { // NO_LINT_DEBUG sql_print_error("Could not allocate name %s (error %d)", log_name, errno); DBUG_RETURN(1); @@ -6306,6 +6383,8 @@ bool MYSQL_BIN_LOG::open_existing_binlog(const char *log_name, Binlog_ofile::open_existing(m_key_file_log, existing_file, MYF(MY_WME)); if (!binlog_file) goto err; + // release current point before assign + delete m_binlog_file; m_binlog_file = binlog_file.release(); file = mysql_file_open(m_key_file_log, existing_file, O_CREAT | O_WRONLY, @@ -6329,7 +6408,7 @@ bool MYSQL_BIN_LOG::open_existing_binlog(const char *log_name, if (offset > 0) m_binlog_file->seek(offset); max_size = max_size_arg; - update_binlog_end_pos(); + update_binlog_end_pos(need_end_log_pos_lock); atomic_log_state = LOG_OPENED; DBUG_RETURN(0); // Success @@ -6635,7 +6714,13 @@ void MYSQL_BIN_LOG::get_current_log_without_lock_log(LOG_INFO *linfo) { int MYSQL_BIN_LOG::raw_get_current_log(LOG_INFO *linfo) { strmake(linfo->log_file_name, log_file_name, sizeof(linfo->log_file_name) - 1); - linfo->pos = m_binlog_file->position(); + // raft write data directly into IO_CACHE, thus + // Binlog_ofile's m_position member isn't updated. + if (enable_raft_plugin && !is_apply_log) + linfo->pos = atomic_binlog_end_pos.load(); + else + linfo->pos = m_binlog_file->position(); + linfo->encrypted_header_size = m_binlog_file->get_encrypted_header_size(); return 0; } @@ -7139,6 +7224,7 @@ bool MYSQL_BIN_LOG::reset_logs(THD *thd, bool delete_only) { } err: + update_lost_gtid_for_tailing(); if (name == nullptr) name = const_cast(save_name); // restore old file-name sid_lock->unlock(); @@ -7282,7 +7368,7 @@ int MYSQL_BIN_LOG::remove_logs_from_index(LOG_INFO *log_info, // now update offsets in index file for running threads if (need_update_threads) - adjust_linfo_offsets(log_info->index_file_start_offset); + adjust_linfo_offsets(log_info->index_file_start_offset, is_relay_log); return 0; err: @@ -7351,6 +7437,8 @@ int MYSQL_BIN_LOG::purge_logs(const char *to_log, bool included, log_file_name_container delete_list; std::pair file_index_pair; std::string safe_purge_file; + int raft_plugin_error = 0; + DBUG_TRACE; DBUG_PRINT("info", ("to_log= %s", to_log)); @@ -7385,8 +7473,10 @@ int MYSQL_BIN_LOG::purge_logs(const char *to_log, bool included, // Nothing to purge if file index is 0 if (!included && file_index_pair.second == 0) goto err; - error = RUN_HOOK(raft_replication, purge_logs, - (current_thd, file_index_pair.second)); + error = RUN_HOOK_STRICT(raft_replication, purge_logs, + (current_thd, file_index_pair.second)); + + DBUG_EXECUTE_IF("simulate_raft_plugin_purge_error", error = 1;); if (error) { // NO_LINT_DEBUG @@ -7394,6 +7484,7 @@ int MYSQL_BIN_LOG::purge_logs(const char *to_log, bool included, "MYSQL_BIN_LOG::purge_logs raft plugin failed in " "purge_logs(). file-name: %s", to_log); + raft_plugin_error = 1; goto err; } @@ -7496,6 +7587,10 @@ int MYSQL_BIN_LOG::purge_logs(const char *to_log, bool included, global_sid_lock->unlock(); } + if (enable_raft_plugin && is_relay_log) { + error = init_prev_gtid_sets_map(); + } + err: if (need_lock_index) mysql_mutex_unlock(&LOCK_index); @@ -7505,10 +7600,12 @@ int MYSQL_BIN_LOG::purge_logs(const char *to_log, bool included, */ error = error ? error : error_index; if (error && should_abort_on_binlog_error()) { - exec_binlog_error_action_abort( - "Either disk is full, file system is read only or " - "there was an encryption error while opening the binlog. " - "Aborting the server."); + if (!raft_plugin_error || abort_on_raft_purge_error) { + exec_binlog_error_action_abort( + "Either disk is full, file system is read only or " + "there was an encryption error while opening the binlog. " + "Aborting the server."); + } } return error; } @@ -8139,6 +8236,10 @@ int MYSQL_BIN_LOG::new_file_impl( // skip rotate event append implies rotate event has already // been appended to relay log by plugin bool skip_re_append = raft_rotate_info && raft_rotate_info->post_append; + bool rotate_in_listener_context = + raft_rotate_info && + (raft_rotate_info->post_append || raft_rotate_info->noop || + raft_rotate_info->config_change_rotate); if (skip_re_append) { assert(is_relay_log); } @@ -8249,6 +8350,7 @@ int MYSQL_BIN_LOG::new_file_impl( Rotate_log_event r( new_name + dirname_length(new_name), 0, LOG_EVENT_OFFSET, is_relay_log && !no_op ? Rotate_log_event::RELAY_LOG : 0); + if (rotate_in_listener_context) r.common_header->when.tv_sec = my_time(0); if (rotate_via_raft) { temp_binlog_cache = std::make_unique(); @@ -8276,6 +8378,8 @@ int MYSQL_BIN_LOG::new_file_impl( me.set_raft_rotate_tag(Metadata_log_event::RRET_SIMPLE_ROTATE); } + if (rotate_in_listener_context) + me.common_header->when.tv_sec = my_time(0); // checksum has to be turned off, because the raft plugin // will patch the events and generate the final checksum. me.common_footer->checksum_alg = binary_log::BINLOG_CHECKSUM_ALG_OFF; @@ -8315,13 +8419,13 @@ int MYSQL_BIN_LOG::new_file_impl( DBUG_EXECUTE_IF("simulate_before_flush_error_on_new_file", error = 1;); if (!error) - error = - RUN_HOOK(raft_replication, before_flush, - (current_thd, temp_binlog_cache->get_io_cache(), op_type)); + error = RUN_HOOK_STRICT( + raft_replication, before_flush, + (current_thd, temp_binlog_cache->get_io_cache(), op_type)); // time to safely readjust the cur_log_ext back to expected value if (!error) { - error = RUN_HOOK(raft_replication, before_commit, (current_thd)); + error = RUN_HOOK_STRICT(raft_replication, before_commit, (current_thd)); if (!error) { // If there was no error, there is a guarantee that this rotate @@ -8433,7 +8537,7 @@ int MYSQL_BIN_LOG::new_file_impl( if (!error && rotate_via_raft && !no_op) { // not trapping return code, because this is the existing // pattern in most places of after_commit hook (TODO) - (void)RUN_HOOK(raft_replication, after_commit, (current_thd)); + (void)RUN_HOOK_STRICT(raft_replication, after_commit, (current_thd)); } if (!error && enable_raft_plugin && !no_op && (is_relay_log || rotate_via_raft)) { @@ -9714,6 +9818,15 @@ int MYSQL_BIN_LOG::open_binlog(const char *opt_name) { goto err; } + // We found atleast one binlog file in the binlog index file + // Note that this is useful only when raft is enabled - even during clean + // shutdown, raft plugin can rollback last batch of trxs from the engine + // (after writing to binlog). hence on restart, this binlog file can have + // uncommitted trxs and should not be marked as 'cleanly closed'. In other + // words, 'LOG_EVENT_BINLOG_IN_USE_F' is not a reliable indicator anymore + // that a binlog file does not contain uncommitted trxs. + open_binlog_found = true; + /* If the binary log was not properly closed it means that the server may have crashed. In that case, we need to call @@ -9739,9 +9852,9 @@ int MYSQL_BIN_LOG::open_binlog(const char *opt_name) { const std::string cur_log_file = log_name + dirname_length(log_name); error = binlog_recover(&binlog_file_reader, &valid_pos, &engine_binlog_max_gtid, engine_binlog_file, - &engine_binlog_pos, cur_log_file); + &engine_binlog_pos, cur_log_file, + &this->first_gtid_start_pos); binlog_size = binlog_file_reader.ifile()->length(); - open_binlog_found = true; } else { /* * If we are here, it implies either mysqld was shutdown cleanly or @@ -9768,7 +9881,6 @@ int MYSQL_BIN_LOG::open_binlog(const char *opt_name) { */ error = ha_recover(&xids, &engine_binlog_max_gtid, tmp_binlog_file, &tmp_binlog_pos); - open_binlog_found = false; } delete ev; @@ -10413,15 +10525,26 @@ void MYSQL_BIN_LOG::process_consensus_queue(THD *queue_head) { if (thd->commit_error == THD::CE_NONE) last_thd = thd; if (last_thd) { - error = RUN_HOOK(raft_replication, before_commit, (last_thd)); + auto start_time = my_timer_now(); + + error = RUN_HOOK_STRICT(raft_replication, before_commit, (last_thd)); + + if (!this->is_apply_log) { + auto wait_time = my_timer_since(start_time); + latency_histogram_increment(&histogram_raft_trx_wait, wait_time, 1); + } if (error) set_commit_consensus_error(queue_head); if (!error && opt_raft_signal_async_dump_threads == AFTER_CONSENSUS && enable_raft_plugin && rpl_wait_for_semi_sync_ack) { const char *log_file = nullptr; my_off_t log_pos = 0; - queue_head->get_trans_fixed_pos((const char **)&log_file, &log_pos); - signal_semi_sync_ack(log_file, log_pos); + if (mysql_bin_log.is_apply_log) { + last_thd->get_trans_relay_log_pos((const char **)&log_file, &log_pos); + } else { + last_thd->get_trans_fixed_pos((const char **)&log_file, &log_pos); + } + dump_log.signal_semi_sync_ack(log_file, log_pos); } } } @@ -10622,10 +10745,14 @@ void MYSQL_BIN_LOG::process_after_commit_stage_queue(THD *thd, THD *first) { */ Thd_backup_and_restore switch_thd(thd, head); bool all = head->get_transaction()->m_flags.real_commit; - error = error || RUN_HOOK(transaction, after_commit, (head, all)); - if (enable_raft_plugin) - (void)RUN_HOOK(raft_replication, after_commit, (head)); + // Call semi-sync plugin only when raft is not enabled + if (!enable_raft_plugin) + error = error || RUN_HOOK(transaction, after_commit, (head, all)); + else + error = + error || RUN_HOOK_STRICT(raft_replication, after_commit, (head)); + if (!enable_raft_plugin) { my_off_t pos; head->get_trans_pos(nullptr, &pos, nullptr, nullptr); @@ -10646,8 +10773,12 @@ void MYSQL_BIN_LOG::process_after_commit_stage_queue(THD *thd, THD *first) { rpl_wait_for_semi_sync_ack) { my_off_t log_pos; const char *log_file = nullptr; - last_thd->get_trans_fixed_pos((const char **)&log_file, &log_pos); - signal_semi_sync_ack(log_file, log_pos); + if (mysql_bin_log.is_apply_log) { + thd->get_trans_relay_log_pos((const char **)&log_file, &log_pos); + } else { + thd->get_trans_fixed_pos((const char **)&log_file, &log_pos); + } + dump_log.signal_semi_sync_ack(log_file, log_pos); } } @@ -10658,16 +10789,9 @@ void MYSQL_BIN_LOG::process_after_commit_stage_queue(THD *thd, THD *first) { @param queue_head Head of the queue */ void MYSQL_BIN_LOG::set_commit_consensus_error(THD *queue_head) { - uint32 thread_count = 0; - for (THD *thd = queue_head; thd != NULL; thd = thd->next_to_commit) { - thread_count++; thd->commit_consensus_error = true; } - - // NO_LINT_DEBUG - sql_print_error("Commit consensus error set for %u threads in the group", - thread_count); } /** @@ -10761,7 +10885,13 @@ int MYSQL_BIN_LOG::flush_cache_to_file(my_off_t *end_pos_var) { thd->commit_error = THD::CE_FLUSH_ERROR; return ER_ERROR_ON_WRITE; } - *end_pos_var = m_binlog_file->position(); + + // raft write data directly into IO_CACHE, thus + // Binlog_ofile's m_position member isn't updated. + if (enable_raft_plugin && !is_apply_log) + *end_pos_var = atomic_binlog_end_pos.load(); + else + *end_pos_var = m_binlog_file->position(); return 0; } @@ -10863,18 +10993,22 @@ int MYSQL_BIN_LOG::finish_commit(THD *thd) { if (!enable_raft_plugin) (void)RUN_HOOK(transaction, after_commit, (thd, all)); else - (void)RUN_HOOK(raft_replication, after_commit, (thd)); + (void)RUN_HOOK_STRICT(raft_replication, after_commit, (thd)); thd->get_transaction()->m_flags.run_hooks = false; } if (enable_raft_plugin && thd->commit_error == THD::CE_NONE) { - int error = RUN_HOOK(raft_replication, after_commit, (thd)); + int error = RUN_HOOK_STRICT(raft_replication, after_commit, (thd)); if (!error && opt_raft_signal_async_dump_threads == AFTER_ENGINE_COMMIT && enable_raft_plugin && rpl_wait_for_semi_sync_ack) { const char *log_file = nullptr; my_off_t log_pos = 0; - thd->get_trans_fixed_pos((const char **)&log_file, &log_pos); - signal_semi_sync_ack(log_file, log_pos); + if (mysql_bin_log.is_apply_log) { + thd->get_trans_relay_log_pos((const char **)&log_file, &log_pos); + } else { + thd->get_trans_fixed_pos((const char **)&log_file, &log_pos); + } + dump_log.signal_semi_sync_ack(log_file, log_pos); } thd->get_transaction()->m_flags.run_hooks = false; } @@ -11080,7 +11214,7 @@ int MYSQL_BIN_LOG::register_log_entities(THD *thd, int context, bool need_lock, arg.log_prefix = name; arg.log_name = log_file_name; arg.cur_log_ext = &raft_cur_log_ext; - arg.endpos_log_name = binlog_file_name; + arg.endpos_log_name = log_file_name; arg.endpos = (ulonglong *)&atomic_binlog_end_pos; arg.signal_cnt = &signal_cnt; arg.lock_log = &LOCK_log; @@ -11090,7 +11224,7 @@ int MYSQL_BIN_LOG::register_log_entities(THD *thd, int context, bool need_lock, arg.context = context; arg.is_relay_log = is_relay_log; - int err = RUN_HOOK(raft_replication, setup_flush, (thd, &arg)); + int err = RUN_HOOK_STRICT(raft_replication, setup_flush, (thd, &arg)); if (need_lock) mysql_mutex_unlock(&LOCK_log); @@ -11207,9 +11341,10 @@ int ask_server_to_register_with_raft(Raft_Registration_Item item) { } THD *thd = current_thd; - err = RUN_HOOK(raft_replication, register_paths, - (thd, server_uuid, s_wal_dir, s_log_dir, log_bin_basename, - glob_hostname, (uint64_t)mysqld_port)); + err = RUN_HOOK_STRICT( + raft_replication, register_paths, + (thd, server_uuid, s_wal_dir, s_log_dir, log_bin_basename, + glob_hostname, (uint64_t)mysqld_port)); break; } default: @@ -11555,6 +11690,37 @@ int rotate_binlog_file(THD *thd) { DBUG_RETURN(error); } +bool block_all_dump_threads() { + block_dump_threads = true; + kill_all_dump_threads(); + + uint count = 50; + while (count-- && + Global_THD_manager::get_instance()->get_num_thread_binlog_client()) + std::this_thread::sleep_for(std::chrono::milliseconds{100}); + + if (Global_THD_manager::get_instance()->get_num_thread_binlog_client()) { + // NO_LINT_DEBUG + sql_print_error("Dump thread count did not reach 0 after 5 secs!"); + block_dump_threads = false; + return false; + } + + return true; +} + +void unblock_all_dump_threads() { block_dump_threads = false; } + +int handle_dump_threads(bool block) { + DBUG_TRACE; + int err = 0; + if (block) + err = block_all_dump_threads() ? 0 : 1; + else + unblock_all_dump_threads(); + return err; +} + int binlog_change_to_apply() { DBUG_ENTER("binlog_change_to_apply"); @@ -11567,8 +11733,9 @@ int binlog_change_to_apply() { int error = 0; LOG_INFO linfo; mysql_mutex_lock(mysql_bin_log.get_log_lock()); + const bool is_locked = dump_log.lock(); mysql_bin_log.lock_index(); - + mysql_bin_log.lock_binlog_end_pos(); mysql_bin_log.close(LOG_CLOSE_INDEX, /*need_lock_log=*/false, /*need_lock_index=*/false); @@ -11590,10 +11757,14 @@ int binlog_change_to_apply() { /*null_created_arg=*/false, /*need_lock_index=*/false, /*need_sid_lock=*/true, - /*extra_description_event=*/NULL)) { + /*extra_description_event=*/NULL, + /*new_index_number =*/0, + /*raft_rotate_info =*/nullptr, + /*need_end_log_pos_lock =*/false)) { error = 1; goto err; } + dump_log.switch_log(/* relay_log= */ true, /* should_lock= */ false); // Purge all apply logs before the last log, because they // are from the previous epoch of being a FOLLOWER, and they @@ -11614,8 +11785,9 @@ int binlog_change_to_apply() { } err: - + mysql_bin_log.unlock_binlog_end_pos(); mysql_bin_log.unlock_index(); + dump_log.unlock(is_locked); mysql_mutex_unlock(mysql_bin_log.get_log_lock()); DBUG_RETURN(error); @@ -11634,9 +11806,28 @@ int binlog_change_to_binlog(THD *thd) { uint64_t prev_hlc = 0; std::vector lognames; + // Flush logs to ensure that storage engine has flushed and fsynced the last + // batch of transactions. This is important because the act of switching trx + // logs from "apply-logs-*" to "binary-logs-*" looks like a rotation to other + // parts of the system and rotation is always a 'sync' point + ha_flush_logs(NULL); + mysql_mutex_lock(mysql_bin_log.get_log_lock()); + const bool is_locked = dump_log.lock(); mysql_bin_log.lock_index(); + Master_info *active_mi; + if (!get_and_lock_master_info(&active_mi) || !active_mi || !active_mi->rli) { + error = 1; + // NO_LINT_DEBUG + sql_print_error("active_mi or rli is not set"); + mysql_bin_log.unlock_index(); + dump_log.unlock(is_locked); + mysql_mutex_unlock(mysql_bin_log.get_log_lock()); + return error; + } + active_mi->rli->relay_log.lock_binlog_end_pos(); + // Get the index file name std::string indexfn = mysql_bin_log.get_index_fname(); @@ -11728,15 +11919,22 @@ int binlog_change_to_binlog(THD *thd) { mysql_bin_log.is_apply_log = false; mysql_bin_log.apply_file_count.store(0); + dump_log.switch_log(/* relay_log= */ false, /* should_lock= */ false); + // Register new log to raft // Previous mysql_bin_log.close(LOG_CLOSE_INDEX) will also close binlog and // its IO_CACHE. mysql_bin_log.register_log_entities(thd, /*context=*/0, /*need_lock=*/false, /*is_relay_log=*/false); err: + active_mi->rli->relay_log.unlock_binlog_end_pos(); + unlock_master_info(active_mi); mysql_bin_log.unlock_index(); + dump_log.unlock(is_locked); mysql_mutex_unlock(mysql_bin_log.get_log_lock()); + DBUG_EXECUTE_IF("crash_after_point_binlog_to_binlog", DBUG_SUICIDE();); + DBUG_RETURN(error); } @@ -11770,6 +11968,40 @@ int MYSQL_BIN_LOG::get_lognames_from_index(bool need_lock, return error; } +Dump_log::Dump_log() { + if (enable_raft_plugin && mysql_bin_log.is_apply_log) { + Master_info *active_mi = nullptr; + if (!get_and_lock_master_info(&active_mi)) { + // NO_LINT_DEBUG + sql_print_error("active_mi or rli is not set"); + } + assert(active_mi && active_mi->rli); + log_ = &active_mi->rli->relay_log; + unlock_master_info(active_mi); + } else { + log_ = &mysql_bin_log; + } +} + +void Dump_log::switch_log(bool relay_log, bool should_lock) { + bool is_locked = false; + if (should_lock) is_locked = lock(); + mysql_mutex_assert_owner(log_->get_binlog_end_pos_lock()); + log_->update_binlog_end_pos(/* need_lock= */ false); + Master_info *active_mi = nullptr; + if (!get_and_lock_master_info(&active_mi)) { + // NO_LINT_DEBUG + sql_print_error("active_mi or rli is not set"); + } + assert(active_mi && active_mi->rli); + log_ = relay_log ? &active_mi->rli->relay_log : &mysql_bin_log; + unlock_master_info(active_mi); + // Now let's update the dump thread's linfos + log_->reset_semi_sync_last_acked(); + adjust_linfo_in_dump_threads(relay_log); + if (should_lock) unlock(is_locked); +} + // Given a file name of the form 'binlog-file-name.index', it extracts the // and and returns it as a pair // Example: @@ -11891,7 +12123,8 @@ static bool binlog_recover(Binlog_file_reader *binlog_file_reader, my_off_t *valid_pos, Gtid *binlog_max_gtid, char *engine_binlog_file, my_off_t *engine_binlog_pos, - const std::string &cur_binlog_file) { + const std::string &cur_binlog_file, + my_off_t *first_gtid_start_pos) { bool res = false; binlog::tools::Iterator it(binlog_file_reader); it.set_copy_event_buffer(); @@ -11965,9 +12198,11 @@ static bool binlog_recover(Binlog_file_reader *binlog_file_reader, current_gtid.set(gev->get_sidno(true), gev->get_gno()); else current_gtid.clear(); - if (first_gtid_start == 0) + if (first_gtid_start == 0) { first_gtid_start = ev->common_header->log_pos - ev->common_header->data_written; + *first_gtid_start_pos = first_gtid_start; + } } default: { break; @@ -12043,8 +12278,12 @@ static bool binlog_recover(Binlog_file_reader *binlog_file_reader, 2. A binlog rotation ensures that the previous binlogs and engine's transaction logs are flushed and made durable. Hence all previous transactions are made durable. + + If enable_raft_plugin is set, then we skip trimming binlog. This is + because the trim is handled inside the raft plugin when this node + rejoins the raft ring */ - if (opt_trim_binlog) { + if (opt_trim_binlog && !enable_raft_plugin) { set_valid_pos(valid_pos, cur_binlog_file, first_gtid_start, engine_binlog_file, *engine_binlog_pos); } @@ -12057,10 +12296,11 @@ static bool binlog_recover(Binlog_file_reader *binlog_file_reader, } void MYSQL_BIN_LOG::report_missing_purged_gtids( - const Gtid_set *slave_executed_gtid_set, std::string &errmsg) { + const Gtid_set *lost_gtid_set, const Gtid_set *slave_executed_gtid_set, + std::string &errmsg) { DBUG_TRACE; THD *thd = current_thd; - Gtid_set gtid_missing(gtid_state->get_lost_gtids()->get_sid_map()); + Gtid_set gtid_missing(lost_gtid_set->get_sid_map()); gtid_missing.add_gtid_set(gtid_state->get_lost_gtids()); gtid_missing.remove_gtid_set(slave_executed_gtid_set); @@ -12179,6 +12419,141 @@ void MYSQL_BIN_LOG::report_missing_gtids( my_free(slave_executed_gtids); } +/** + * Recover raft log. This is primarily for relay logs in the raft world since + * trx logs (binary logs or apply logs) are already recovered by mysqld as part + * of trx log recovery. This method tries to get rid of partial trxs in the tal + * of the raft log. Much has been borrowed from + * MYSQL_BIN_LOG::open_binlog(const char *opt_name) and + * binlog_recover(). Refactoring the components is rather hard and + * adds unnecessary complexity with additional params and if() {} else {} + * branches. Hence a separate method. + */ +int MYSQL_BIN_LOG::raft_log_recover() { + int error = 0; + Log_event *ev = 0; + char log_name[FN_REFLEN]; + my_off_t valid_pos = 0; + my_off_t binlog_size = 0; + LOG_INFO log_info; + bool pending_gtid = false; + std::string error_message; + int status = 0; + bool in_transaction = false; + if (!mysql_bin_log.is_apply_log) + goto err; // raft log already recovered as part of trx log recovery + + if (!my_b_inited(&index_file)) { + error_message = "Index file is not inited in recover_raft_log"; + error = 1; + goto err; + } + + if ((status = + find_log_pos(&log_info, NullS, true /*need_lock_index=true*/))) { + if (status != LOG_INFO_EOF) { + error_message = "find_log_pos() failed in recover_raft_log with error: " + + std::to_string(error); + error = 1; + } + goto err; + } + + do { + strmake(log_name, log_info.log_file_name, sizeof(log_name) - 1); + } while (!(status = find_next_log(&log_info, true /*need_lock_index=true*/))); + + if (status != LOG_INFO_EOF) { + error_message = "find_log_pos() failed in recover_raft_log with error: " + + std::to_string(error); + error = 1; + goto err; + } + + { + Binlog_file_reader binlog_file_reader(opt_source_verify_checksum); + if (binlog_file_reader.open(log_name)) { + error = 1; + error_message = "open_binlog_file() failed in recover_raft_log with "; + goto err; + } + binlog_size = binlog_file_reader.ifile()->length(); + // This logic is borrowed from binlog_recover() which has to do + // additional things and refactoring it will simply add more branches. Hence + // the code duplication + while ((ev = binlog_file_reader.read_event_object())) { + if (ev->get_type_code() == binary_log::QUERY_EVENT && + !strcmp(static_cast(ev)->query, "BEGIN")) { + in_transaction = true; + } else if (ev->get_type_code() == binary_log::QUERY_EVENT && + !strcmp(static_cast(ev)->query, "COMMIT")) { + assert(in_transaction == true); + in_transaction = false; + } else if (is_gtid_event(ev)) { + pending_gtid = true; + } else if (ev->get_type_code() == binary_log::XID_EVENT || + (ev->get_type_code() == binary_log::QUERY_EVENT && + !strcmp(static_cast(ev)->query, + "COMMIT"))) { + if (!in_transaction) { + // When we see a commit message, we should already be parsing a valid + // transaction + error_message = + "Saw a XID/COMMIT event without a begin. Corrupted log: " + + std::string(log_name); + error = 1; + delete ev; + break; + } + in_transaction = false; + } + if (!(ev->get_type_code() == binary_log::METADATA_EVENT && + pending_gtid)) { + if (!in_transaction && !is_gtid_event(ev)) { + valid_pos = binlog_file_reader.position(); + pending_gtid = false; + } + } + + delete ev; + } + } + + // No partial trxs found in the raft log or error parsing the log + if (error || (valid_pos == 0 || valid_pos >= binlog_size)) goto err; + + // NO_LINT_DEBUG + sql_print_information( + "Raft log %s with a size of %llu will be trimmed to " + "%llu bytes based on valid transactions in the file", + log_name, binlog_size, valid_pos); + + { + std::unique_ptr ofile( + Binlog_ofile::open_existing(key_file_binlog, log_name, MYF(MY_WME))); + if (!ofile) { + error_message = + "Failed to remove partial transactions from raft log file "; + error = 1; + goto err; + } + if (ofile->truncate(valid_pos)) { + error_message = + "Failed to remove partial transactions from raft log file " + + std::string(log_name); + error = 1; + goto err; + } + } + +err: + if (error && !error_message.empty()) + // NO_LINT_DEBUG + sql_print_error("%s", error_message.c_str()); + + return error; +} + void MYSQL_BIN_LOG::update_binlog_end_pos(bool need_lock) { if (need_lock) lock_binlog_end_pos(); @@ -12191,13 +12566,17 @@ void MYSQL_BIN_LOG::update_binlog_end_pos(bool need_lock) { if (need_lock) unlock_binlog_end_pos(); } -inline void MYSQL_BIN_LOG::update_binlog_end_pos(const char *file, - my_off_t pos) { - lock_binlog_end_pos(); - if (is_active(file) && (pos > atomic_binlog_end_pos)) +inline void MYSQL_BIN_LOG::update_binlog_end_pos(const char *file, my_off_t pos, + bool need_lock) { + if (need_lock) + lock_binlog_end_pos(); + else + mysql_mutex_assert_owner(&LOCK_binlog_end_pos); + if (is_active(file) && (pos > atomic_binlog_end_pos)) { atomic_binlog_end_pos = pos; + } signal_update(); - unlock_binlog_end_pos(); + if (need_lock) unlock_binlog_end_pos(); } my_off_t MYSQL_BIN_LOG::get_binlog_end_pos() const { @@ -12231,7 +12610,7 @@ my_off_t MYSQL_BIN_LOG::get_last_acked_pos(bool *wait_for_ack, /* Use by raft plugin */ void signal_semi_sync_ack(const std::string &file, uint pos) { - mysql_bin_log.signal_semi_sync_ack(file.c_str(), pos); + dump_log.signal_semi_sync_ack(file.c_str(), pos); } void MYSQL_BIN_LOG::signal_semi_sync_ack(const char *const log_file, @@ -12259,7 +12638,7 @@ void MYSQL_BIN_LOG::signal_semi_sync_ack(const char *const log_file, lock_binlog_end_pos(); if (acked > last_acked.load()) { last_acked = acked; - signal_update(); + update_binlog_end_pos(log_file, log_pos, false); } unlock_binlog_end_pos(); } @@ -12267,7 +12646,10 @@ void MYSQL_BIN_LOG::signal_semi_sync_ack(const char *const log_file, void MYSQL_BIN_LOG::reset_semi_sync_last_acked() { lock_binlog_end_pos(); /* binary log is rotated and all trxs in previous binlog are already committed - * to the storage engine */ + * to the storage engine. + * Note: when in raft mode we cannot init the coords without consulting the + * plugin, so we reset the coords + */ if (strlen(log_file_name)) { last_acked = {extract_file_index(log_file_name).second, 0}; } else { @@ -12391,8 +12773,10 @@ bool show_raft_status(THD *thd) { std::vector> var_value_pairs; std::vector>::const_iterator itr; - int error = RUN_HOOK(raft_replication, show_raft_status, - (current_thd, &var_value_pairs)); + mysql_mutex_lock(&LOCK_status); + int error = RUN_HOOK_STRICT(raft_replication, show_raft_status, + (current_thd, &var_value_pairs)); + mysql_mutex_unlock(&LOCK_status); if (error) { errmsg = "Failure to run plugin hook"; goto err; @@ -14822,6 +15206,28 @@ int trim_logged_gtid(const std::vector &trimmed_gtids) { return error; } +int get_committed_gtids(const std::vector >ids, + std::vector *committed_gtids) { + global_sid_lock->rdlock(); + + for (const auto >id_s : gtids) { + if (gtid_s.empty()) continue; + + Gtid gtid; + enum_return_status st = gtid.parse(global_sid_map, gtid_s.c_str()); + if (st != RETURN_STATUS_OK) { + global_sid_lock->unlock(); + return st; + } + + if (gtid_state->get_executed_gtids()->contains_gtid(gtid)) + committed_gtids->push_back(gtid_s); + } + global_sid_lock->unlock(); + + return 0; +} + struct st_mysql_storage_engine binlog_storage_engine = { MYSQL_HANDLERTON_INTERFACE_VERSION}; diff --git a/sql/binlog.h b/sql/binlog.h index d51b4d4d0369..3014377fe25d 100644 --- a/sql/binlog.h +++ b/sql/binlog.h @@ -108,6 +108,8 @@ struct Binlog_user_var_event { bool unsigned_flag; }; +extern latency_histogram histogram_raft_trx_wait; + /* The enum defining the server's action when a trx fails inside ordered commit * due to an error related to consensus (raft plugin) */ enum enum_commit_consensus_error_actions { @@ -145,6 +147,9 @@ enum enum_raft_signal_async_dump_threads_options { #define LOG_CLOSE_TO_BE_OPENED 2 #define LOG_CLOSE_STOP_EVENT 4 +bool block_all_dump_threads(); +void unblock_all_dump_threads(); + /* Note that we destroy the lock mutex in the destructor here. This means that object instances cannot be destroyed/go out of scope @@ -157,13 +162,17 @@ struct LOG_INFO { bool fatal; // if the purge happens to give us a negative offset int entry_index; // used in purge_logs(), calculatd in find_log_pos(). int encrypted_header_size; - LOG_INFO() + bool is_relay_log; // is this info pointing to a relay log? + bool is_used_by_dump_thd; // is this info being used by a dump thread? + LOG_INFO(bool relay_log = false, bool used_by_dump_thd = false) : index_file_offset(0), index_file_start_offset(0), pos(0), fatal(false), entry_index(0), - encrypted_header_size(0) { + encrypted_header_size(0), + is_relay_log(relay_log), + is_used_by_dump_thd(used_by_dump_thd) { memset(log_file_name, 0, FN_REFLEN); } }; @@ -397,6 +406,7 @@ class MYSQL_BIN_LOG : public TC_LOG { /** The instrumentation key to use for @ LOCK_xids. */ PSI_mutex_key m_key_LOCK_xids; PSI_mutex_key m_key_LOCK_non_xid_trxs; + PSI_mutex_key m_key_LOCK_lost_gtids_for_tailing; /** The instrumentation key to use for @ update_cond. */ PSI_cond_key m_key_update_cond; /** The instrumentation key to use for @ prep_xids_cond. */ @@ -419,6 +429,7 @@ class MYSQL_BIN_LOG : public TC_LOG { mysql_mutex_t LOCK_binlog_end_pos; mysql_mutex_t LOCK_xids; mysql_mutex_t LOCK_non_xid_trxs; + mysql_mutex_t LOCK_lost_gtids_for_tailing; mysql_cond_t update_cond; std::atomic atomic_binlog_end_pos; @@ -452,6 +463,10 @@ class MYSQL_BIN_LOG : public TC_LOG { directories change. */ Gtid_set_map previous_gtid_set_map; + /* + Used by sys_var gtid_purged_for_tailing + */ + std::string lost_gtid_for_tailing; /* crash_safe_index_file is temp file used for guaranteeing index file crash safe when master server restarts. @@ -654,7 +669,8 @@ class MYSQL_BIN_LOG : public TC_LOG { PSI_mutex_key key_LOCK_flush_queue, PSI_mutex_key key_LOCK_log, PSI_mutex_key key_LOCK_binlog_end_pos, PSI_mutex_key key_LOCK_sync, PSI_mutex_key key_LOCK_sync_queue, PSI_mutex_key key_LOCK_xids, - PSI_mutex_key key_LOCK_non_xid_trxs, PSI_cond_key key_COND_done, + PSI_mutex_key key_LOCK_non_xid_trxs, + PSI_mutex_key key_LOCK_lost_gtids_for_tailing, PSI_cond_key key_COND_done, PSI_cond_key key_COND_flush_queue, PSI_cond_key key_update_cond, PSI_cond_key key_prep_xids_cond, PSI_cond_key key_non_xid_trxs_cond, PSI_file_key key_file_log, PSI_file_key key_file_log_index, @@ -674,6 +690,7 @@ class MYSQL_BIN_LOG : public TC_LOG { m_key_LOCK_sync = key_LOCK_sync; m_key_LOCK_xids = key_LOCK_xids; m_key_LOCK_non_xid_trxs = key_LOCK_non_xid_trxs; + m_key_LOCK_lost_gtids_for_tailing = key_LOCK_lost_gtids_for_tailing; m_key_update_cond = key_update_cond; m_key_prep_xids_cond = key_prep_xids_cond; m_key_non_xid_trxs_cond = key_non_xid_trxs_cond; @@ -757,6 +774,25 @@ class MYSQL_BIN_LOG : public TC_LOG { */ bool init_prev_gtid_sets_map(); + void get_lost_gtids(Gtid_set *gtids) { + gtids->clear(); + mysql_mutex_lock(&LOCK_index); + const auto it = previous_gtid_set_map.begin(); + if (it != previous_gtid_set_map.end() && !it->second.empty()) + gtids->add_gtid_encoding((const uchar *)it->second.c_str(), + it->second.length()); + mysql_mutex_unlock(&LOCK_index); + } + + void update_lost_gtid_for_tailing() { + mysql_mutex_assert_owner(&LOCK_index); + mysql_mutex_lock(&LOCK_lost_gtids_for_tailing); + lost_gtid_for_tailing = ""; + const auto it = previous_gtid_set_map.begin(); + if (it != previous_gtid_set_map.end()) lost_gtid_for_tailing = it->second; + mysql_mutex_unlock(&LOCK_lost_gtids_for_tailing); + } + enum_read_gtids_from_binlog_status read_gtids_from_binlog( const char *filename, Gtid_set *all_gtids, Gtid_set *prev_gtids, Gtid *first_gtid, Sid_map *sid_map, bool verify_checksum, @@ -1086,10 +1122,13 @@ class MYSQL_BIN_LOG : public TC_LOG { } void update_binlog_end_pos(bool need_lock = true); - void update_binlog_end_pos(const char *file, my_off_t pos); + void update_binlog_end_pos(const char *file, my_off_t pos, + bool need_lock = true); int wait_for_update(const struct timespec *timeout); + int raft_log_recover(); + public: /** register binlog/relay (its IO_CACHE) and mutexes to plugin. Sharing the pointers with the plugin enables the plugin to @@ -1131,23 +1170,29 @@ class MYSQL_BIN_LOG : public TC_LOG { after the RESET MASTER TO command is called. @param raft_rotate_info rotate related information passed in by listener callbacks + @param need_end_log_pos_lock If true, LOCK_binlog_end_pos is acquired; + otherwise LOCK_binlog_end_pos must be taken by the caller. */ bool open_binlog(const char *log_name, const char *new_name, ulong max_size_arg, bool null_created_arg, bool need_lock_index, bool need_sid_lock, Format_description_log_event *extra_description_event, uint32 new_index_number = 0, - RaftRotateInfo *raft_rotate_info = nullptr); + RaftRotateInfo *raft_rotate_info = nullptr, + bool need_end_log_pos_lock = true); /** Open an existing binlog/relaylog file @param log_name Name of binlog @param max_size The size at which this binlog will be rotated. + @param need_end_log_pos_lock If true, LOCK_binlog_end_pos is acquired; + otherwise LOCK_binlog_end_pos must be taken by the caller. @retval false on success, true on error */ - bool open_existing_binlog(const char *log_name, ulong max_size_arg); + bool open_existing_binlog(const char *log_name, ulong max_size_arg, + bool need_end_log_pos_lock = true); bool open_index_file(const char *index_file_name_arg, const char *log_name, bool need_lock_index); @@ -1422,10 +1467,12 @@ class MYSQL_BIN_LOG : public TC_LOG { This function will be called from mysql_binlog_send() function. + @param lost_gtid_set GTID set of missing gtids @param slave_executed_gtid_set GTID set executed by slave @param errmsg Pointer to the error message */ - void report_missing_purged_gtids(const Gtid_set *slave_executed_gtid_set, + void report_missing_purged_gtids(const Gtid_set *lost_gtid_set, + const Gtid_set *slave_executed_gtid_set, std::string &errmsg); /** @@ -1456,6 +1503,15 @@ class MYSQL_BIN_LOG : public TC_LOG { return &previous_gtid_set_map; } + void get_lost_gtid_for_tailing(Gtid_set *gtids) { + gtids->clear(); + mysql_mutex_lock(&LOCK_lost_gtids_for_tailing); + if (!lost_gtid_for_tailing.empty()) + gtids->add_gtid_encoding((const uchar *)lost_gtid_for_tailing.c_str(), + lost_gtid_for_tailing.length()); + mysql_mutex_unlock(&LOCK_lost_gtids_for_tailing); + } + /* It is called by the threads (e.g. dump thread, applier thread) which want to read hot log without LOCK_log protection. @@ -1500,6 +1556,78 @@ struct LOAD_FILE_INFO { extern MYSQL_PLUGIN_IMPORT MYSQL_BIN_LOG mysql_bin_log; +/** + * Encapsulation over binlog or relay log for dumping raft logs during + * COM_BINLOG_DUMP and COM_BINLOG_DUMP_GTID. + */ +class Dump_log { + public: + // RAII class to handle locking for Dump_log + class Locker { + public: + explicit Locker(Dump_log *dump_log) { + dump_log_ = dump_log; + should_lock_ = dump_log_->lock(); + } + + ~Locker() { + if (should_lock_) dump_log_->unlock(should_lock_); + } + + private: + bool should_lock_ = false; + Dump_log *dump_log_ = nullptr; + }; + + Dump_log(); + + void switch_log(bool relay_log, bool should_lock = true); + + MYSQL_BIN_LOG *get_log(bool should_lock = true) { + bool is_locked = false; + if (should_lock) is_locked = lock(); + auto ret = log_; + if (should_lock) unlock(is_locked); + return ret; + } + + void signal_semi_sync_ack(const char *const log_file, + const my_off_t log_pos) { + Locker lock(this); + log_->signal_semi_sync_ack(log_file, log_pos); + } + + void reset_semi_sync_last_acked() { + Locker lock(this); + log_->reset_semi_sync_last_acked(); + } + + void get_lost_gtids(Gtid_set *gtid_set) { + Locker lock(this); + log_->get_lost_gtid_for_tailing(gtid_set); + } + + // Avoid using this and try to use Dump_log::Locker class instead + bool lock() { + // NOTE: we lock only when we're in raft mode. That's why we're returning a + // bool to indicate whether we locked or not. We pass this bool to unlock + // method to unlock only then the mutex was actually locked. + const bool should_lock = enable_raft_plugin; + if (should_lock) log_mutex_.lock(); + return should_lock; + } + + // Avoid using this and try to use Dump_log::Locker class instead + void unlock(bool is_locked) { + if (is_locked) log_mutex_.unlock(); + } + + private: + MYSQL_BIN_LOG *log_; + std::mutex log_mutex_; +}; +extern MYSQL_PLUGIN_IMPORT Dump_log dump_log; + /** Check if the the transaction is empty. @@ -1561,6 +1689,8 @@ bool show_raft_status(THD *thd); bool get_and_lock_master_info(Master_info **master_info); void unlock_master_info(Master_info *master_info); int trim_logged_gtid(const std::vector &trimmed_gtids); +int get_committed_gtids(const std::vector >ids, + std::vector *committed_gtids); extern const char *log_bin_index; extern const char *log_bin_basename; @@ -1575,6 +1705,11 @@ extern bool rpl_semi_sync_source_enabled; */ int raft_config_change(std::string config_change); +/** + * Block/unblock dump threads + */ +int handle_dump_threads(bool block); + /** Rotates the binary log file. Helper method invoked by raft plugin through raft listener queue. diff --git a/sql/log_event.cc b/sql/log_event.cc index cc432659cdcb..908758819c9e 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -4715,9 +4715,29 @@ void Query_log_event::detach_temp_tables_worker(THD *thd_arg, Query_log_event::do_apply_event() */ int Query_log_event::do_apply_event(Relay_log_info const *rli) { + // Note: We're using event's future_event_relay_log_pos instead of + // rli->get_event_relay_log_pos() because rli is only updated in + // do_update_pos() which is called after applying the event and we might need + // to use this pos during application (e.g. during commit) + Relay_log_info *rli_ptr = const_cast(rli); + thd->set_trans_relay_log_pos(rli_ptr->get_event_relay_log_name(), + future_event_relay_log_pos); return do_apply_event(rli, query, q_len); } +int Query_log_event::do_apply_event_worker(Slave_worker *w) { + // Note: We're using event's future_event_relay_log_pos instead of + // rli->get_event_relay_log_pos() because rli is only updated in + // do_update_pos() which is called after applying the event and we might + // need to use this pos during application (e.g. during commit) + Slave_job_group *ptr_g = w->c_rli->gaq->get_job_group(mts_group_idx); + thd->set_trans_relay_log_pos(ptr_g && ptr_g->group_relay_log_name + ? ptr_g->group_relay_log_name + : w->get_group_relay_log_name(), + future_event_relay_log_pos); + return do_apply_event(w, query, q_len); +} + /* is_silent_error @@ -5621,11 +5641,15 @@ void Format_description_log_event::print( "closed properly.\n"); } + if (is_raft_log_event()) { + my_b_printf( + head, "# This binlog was generated when this host was in raft mode.\n"); + } + if (is_relay_log_event()) { my_b_printf(head, "# This Format_description_event appears in a relay log " "and was generated by the slave thread.\n"); - return; } if (!is_artificial_event() && created) { @@ -5643,9 +5667,16 @@ void Format_description_log_event::print( } if (temp_buf && print_event_info->base64_output_mode != BASE64_OUTPUT_NEVER && !print_event_info->short_form) { - if (print_event_info->base64_output_mode != BASE64_OUTPUT_DECODE_ROWS) - my_b_printf(head, "BINLOG '\n"); - print_base64(head, print_event_info, false); + /* + BUG#20980932 MYSQLBINLOG GENERATES 'ROLLBACK' AFTER FD EVENT CAUSING 1782 + ERROR + Format_description_log_event of relay log is not output as a 'BINLOG' + */ + if (!is_relay_log_event()) { + if (print_event_info->base64_output_mode != BASE64_OUTPUT_DECODE_ROWS) + my_b_printf(head, "BINLOG '\n"); + print_base64(head, print_event_info, false); + } print_event_info->printed_fd_event = true; /* @@ -5941,12 +5972,19 @@ bool Rotate_log_event::write(Basic_ostream *ostream) { write_footer(ostream)); } -int Rotate_log_event::do_apply_event(Relay_log_info const *) { +int Rotate_log_event::do_apply_event(Relay_log_info const *rli) { if (!enable_raft_plugin) return 0; + // Note: We're using event's future_event_relay_log_pos instead of + // rli->get_event_relay_log_pos() because rli is only updated in + // do_update_pos() which is called after applying the event and we might need + // to use this pos during application (e.g. during commit) + Relay_log_info *rli_ptr = const_cast(rli); + thd->set_trans_relay_log_pos(rli_ptr->get_event_relay_log_name(), + future_event_relay_log_pos); int64_t term, index; thd->get_trans_marker(&term, &index); - return RUN_HOOK(raft_replication, after_commit, (thd)); + return RUN_HOOK_STRICT(raft_replication, after_commit, (thd)); } /* @@ -6497,6 +6535,9 @@ int Xid_apply_log_event::do_apply_event_worker(Slave_worker *w) { goto err; } + thd->set_trans_relay_log_pos(w->get_group_relay_log_name(), + w->get_group_relay_log_pos()); + DBUG_PRINT( "mts", ("do_apply group master %s %llu group relay %s %llu event %s %llu.", @@ -6636,6 +6677,8 @@ int Xid_apply_log_event::do_apply_event(Relay_log_info const *rli) { strmake(new_group_relay_log_name, rli_ptr->get_group_relay_log_name(), FN_REFLEN - 1); new_group_relay_log_pos = rli_ptr->get_group_relay_log_pos(); + thd->set_trans_relay_log_pos(rli_ptr->get_group_relay_log_name(), + rli_ptr->get_group_relay_log_pos()); /* Rollback positions in memory just before commit. Position values will be reset to their new values only on successful commit operation. @@ -7802,6 +7845,10 @@ int Execute_load_query_log_event::do_apply_event(Relay_log_info const *rli) { return error; } +int Execute_load_query_log_event::do_apply_event_worker(Slave_worker *w) { + return do_apply_event(w); +} + /***************************************************************************** Load_query_generator is used to generate the LOAD DATA statement for binlog ******************************************************************************/ @@ -14852,6 +14899,8 @@ bool Metadata_log_event::write_data_body(Basic_ostream *ostream) { if (write_raft_term_and_index(ostream)) DBUG_RETURN(1); + if (write_raft_str(ostream)) DBUG_RETURN(1); + if (write_raft_prev_opid(ostream)) DBUG_RETURN(1); if (write_rotate_tag(ostream)) DBUG_RETURN(1); @@ -15029,6 +15078,7 @@ uint32 Metadata_log_event::write_data_body(uchar *obuffer) { length += write_hlc_time(obuffer + length); length += write_prev_hlc_time(obuffer + length); length += write_raft_term_and_index(obuffer + length); + length += write_raft_str(obuffer + length); length += write_rotate_tag(obuffer + length); DBUG_RETURN(length); } diff --git a/sql/log_event.h b/sql/log_event.h index b7d9cab05597..a093736906d9 100644 --- a/sql/log_event.h +++ b/sql/log_event.h @@ -1032,6 +1032,9 @@ class Log_event { } void set_relay_log_event() { common_header->flags |= LOG_EVENT_RELAY_LOG_F; } void set_raft_log_event() { common_header->flags |= LOG_EVENT_RAFT_LOG_F; } + bool is_raft_log_event() const { + return common_header->flags & LOG_EVENT_RAFT_LOG_F; + } bool is_artificial_event() const { return common_header->flags & LOG_EVENT_ARTIFICIAL_F; } @@ -1655,6 +1658,7 @@ class Query_log_event : public virtual binary_log::Query_event, #if defined(MYSQL_SERVER) enum_skip_reason do_shall_skip(Relay_log_info *rli) override; int do_apply_event(Relay_log_info const *rli) override; + int do_apply_event_worker(Slave_worker *w) override; int do_update_pos(Relay_log_info *rli) override; void prepare_dep(Relay_log_info *rli, std::shared_ptr &ev) override; @@ -2462,6 +2466,7 @@ class Execute_load_query_log_event private: #if defined(MYSQL_SERVER) int do_apply_event(Relay_log_info const *rli) override; + int do_apply_event_worker(Slave_worker *w) override; #endif }; diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 0f94b51deaa3..b8e143205e30 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -1085,6 +1085,7 @@ static PSI_mutex_key key_BINLOG_LOCK_sync; static PSI_mutex_key key_BINLOG_LOCK_sync_queue; static PSI_mutex_key key_BINLOG_LOCK_xids; static PSI_mutex_key key_BINLOG_LOCK_non_xid_trxs; +static PSI_mutex_key key_BINLOG_LOCK_lost_gtids_for_tailing; static PSI_rwlock_key key_rwlock_global_sid_lock; PSI_rwlock_key key_rwlock_gtid_mode_lock; static PSI_rwlock_key key_rwlock_LOCK_system_variables_hash; @@ -1232,10 +1233,13 @@ ulong opt_commit_consensus_error_action = 0; bool enable_raft_plugin = 0; bool disallow_raft = 1; // raft is not allowed by default bool override_enable_raft_check = false; +bool abort_on_raft_purge_error = false; ulonglong apply_log_retention_num = 0; ulonglong apply_log_retention_duration = 0; bool disable_raft_log_repointing = 0; ulong opt_raft_signal_async_dump_threads = 0; +bool recover_raft_log = false; + /* Apply log related variables for raft "_ptr" variables are system variables that should not be free by us */ char *opt_apply_logname = nullptr, *opt_apply_logname_ptr = nullptr; @@ -1914,6 +1918,10 @@ bool replica_preserve_commit_order_supplied = false; char *opt_general_logname, *opt_slow_logname, *opt_bin_logname; char *opt_gap_lock_logname; +/* status variables for raft trx wait times */ +SHOW_VAR latency_histogram_raft_trx_wait[NUMBER_OF_HISTOGRAM_BINS + 1]; +ulonglong histogram_raft_trx_wait_values[NUMBER_OF_HISTOGRAM_BINS]; + /* True if expire_logs_days and binlog_expire_logs_seconds is set explictly. @@ -2540,9 +2548,11 @@ class Call_close_conn : public Do_THD_Impl { static void close_connections(void) { DBUG_TRACE; - // NO_LINT_DEBUG - sql_print_information("Sending shutdown call to raft plugin"); - RUN_HOOK(raft_replication, before_shutdown, (nullptr)); + if (enable_raft_plugin) { + // NO_LINT_DEBUG + sql_print_information("Sending shutdown call to raft plugin"); + RUN_HOOK_STRICT(raft_replication, before_shutdown, (nullptr)); + } (void)RUN_HOOK(server_state, before_server_shutdown, (nullptr)); @@ -2843,6 +2853,7 @@ static void clean_up(bool print_message) { release_keyring_handles(); keyring_lockable_deinit(); + free_latency_histogram_sysvars(latency_histogram_raft_trx_wait); /* make sure that handlers finish up @@ -5416,17 +5427,17 @@ int init_common_variables() { initialization, and can not be set in the MYSQL_BIN_LOG constructor (called before main()). */ - mysql_bin_log.set_psi_keys(key_BINLOG_LOCK_index, key_BINLOG_LOCK_commit, - key_BINLOG_LOCK_commit_queue, key_BINLOG_LOCK_done, - key_BINLOG_LOCK_flush_queue, key_BINLOG_LOCK_log, - key_BINLOG_LOCK_binlog_end_pos, - key_BINLOG_LOCK_sync, key_BINLOG_LOCK_sync_queue, - key_BINLOG_LOCK_xids, key_BINLOG_LOCK_non_xid_trxs, - key_BINLOG_COND_done, key_BINLOG_COND_flush_queue, - key_BINLOG_update_cond, key_BINLOG_prep_xids_cond, - key_BINLOG_non_xid_trxs_cond, key_file_binlog, - key_file_binlog_index, key_file_binlog_cache, - key_file_binlog_index_cache); + mysql_bin_log.set_psi_keys( + key_BINLOG_LOCK_index, key_BINLOG_LOCK_commit, + key_BINLOG_LOCK_commit_queue, key_BINLOG_LOCK_done, + key_BINLOG_LOCK_flush_queue, key_BINLOG_LOCK_log, + key_BINLOG_LOCK_binlog_end_pos, key_BINLOG_LOCK_sync, + key_BINLOG_LOCK_sync_queue, key_BINLOG_LOCK_xids, + key_BINLOG_LOCK_non_xid_trxs, key_BINLOG_LOCK_lost_gtids_for_tailing, + key_BINLOG_COND_done, key_BINLOG_COND_flush_queue, key_BINLOG_update_cond, + key_BINLOG_prep_xids_cond, key_BINLOG_non_xid_trxs_cond, key_file_binlog, + key_file_binlog_index, key_file_binlog_cache, + key_file_binlog_index_cache); #endif /* @@ -5436,7 +5447,6 @@ int init_common_variables() { inited before MY_INIT(). So we do it here. */ mysql_bin_log.init_pthread_objects(); - /* TODO: remove this when my_time_t is 64 bit compatible */ if (!is_time_t_valid_for_timestamp(server_start_time)) { LogErr(ERROR_LEVEL, ER_UNSUPPORTED_DATE); @@ -7468,7 +7478,7 @@ static int init_server_components() { if (mysql_bin_log.init_gtid_sets( executed_gtids, NULL /* lost_gtid */, opt_source_verify_checksum, true /*true=need lock*/, NULL /*trx_parser*/, NULL /*partial_trx*/, - &prev_hlc)) + &prev_hlc, /*startup=*/true)) unireg_abort(1); /* @@ -7597,7 +7607,7 @@ static int init_server_components() { rpl_source_io_monitor = new Source_IO_monitor(); udf_load_service.init(); - mysql_bin_log.reset_semi_sync_last_acked(); + dump_log.reset_semi_sync_last_acked(); /* Initialize the optimizer cost module */ init_optimizer_cost_module(true); @@ -8391,7 +8401,8 @@ int mysqld_main(int argc, char **argv) if (mysql_bin_log.init_gtid_sets( >ids_in_binlog, &purged_gtids_from_binlog, opt_source_verify_checksum, true /*true=need lock*/, - nullptr /*trx_parser*/, nullptr /*partial_trx*/, &prev_hlc)) + nullptr /*trx_parser*/, nullptr /*partial_trx*/, &prev_hlc, + /*startup=*/true)) unireg_abort(MYSQLD_ABORT_EXIT); // Update the instance's HLC clock to be greater than or equal to the HLC @@ -9912,6 +9923,22 @@ static int show_jemalloc_tcache_bytes(THD *thd, SHOW_VAR *var, char *buff) { } #endif /* HAVE_JEMALLOC */ +static int show_latency_histogram_raft_trx_wait(THD * /*thd*/, SHOW_VAR *var, + char * /*buff*/) { + size_t i; + for (i = 0; i < NUMBER_OF_HISTOGRAM_BINS; ++i) + histogram_raft_trx_wait_values[i] = + latency_histogram_get_count(&histogram_raft_trx_wait, i); + + prepare_latency_histogram_vars(&histogram_raft_trx_wait, + latency_histogram_raft_trx_wait, + histogram_raft_trx_wait_values); + var->type = SHOW_ARRAY; + var->value = (char *)&latency_histogram_raft_trx_wait; + + return 0; +} + static int show_queries(THD *thd, SHOW_VAR *var, char *) { var->type = SHOW_LONGLONG; var->value = (char *)&thd->query_id; @@ -11034,7 +11061,9 @@ SHOW_VAR status_vars[] = { SHOW_LONGLONG, SHOW_SCOPE_GLOBAL}, {"Json_contains_legacy_count", (char *)&json_contains_legacy_count, SHOW_LONGLONG, SHOW_SCOPE_GLOBAL}, - + {"Rpl_raft_trx_wait_histogram", + (char *)&show_latency_histogram_raft_trx_wait, SHOW_FUNC, + SHOW_SCOPE_GLOBAL}, {NullS, NullS, SHOW_LONG, SHOW_SCOPE_ALL}}; void add_terminator(vector *options) { @@ -12038,8 +12067,11 @@ static int generate_apply_file_gvars() { } if (!opt_applylog_index_name_ptr && opt_apply_logname) { - opt_applylog_index_name = const_cast(rpl_make_log_name( - PSI_NOT_INSTRUMENTED, NULL, opt_apply_logname, ".index")); + // generate relate path for opt_applylog_index_name + char buff[FN_REFLEN]; + fn_format(buff, opt_apply_logname, mysql_data_home, ".index", + MY_UNPACK_FILENAME | MY_REPLACE_EXT); + opt_applylog_index_name = my_strdup(PSI_NOT_INSTRUMENTED, buff, MYF(0)); } DBUG_RETURN(0); @@ -12867,6 +12899,7 @@ PSI_mutex_key key_RELAYLOG_LOCK_log_end_pos; PSI_mutex_key key_RELAYLOG_LOCK_sync; PSI_mutex_key key_RELAYLOG_LOCK_xids; PSI_mutex_key key_RELAYLOG_LOCK_non_xid_trxs; +PSI_mutex_key key_RELAYLOG_LOCK_lost_gtids_for_tailing; PSI_mutex_key key_gtid_ensure_index_mutex; PSI_mutex_key key_hlc_wait_mutex; PSI_mutex_key key_object_cache_mutex; // TODO need to initialize @@ -12903,6 +12936,8 @@ static PSI_mutex_info all_server_mutexes[]= { &key_BINLOG_LOCK_sync_queue, "MYSQL_BIN_LOG::LOCK_sync_queue", 0, 0, PSI_DOCUMENT_ME}, { &key_BINLOG_LOCK_xids, "MYSQL_BIN_LOG::LOCK_xids", 0, 0, PSI_DOCUMENT_ME}, { &key_BINLOG_LOCK_non_xid_trxs, "MYSQL_BIN_LOG::LOCK_non_xid_trxs", 0, 0, PSI_DOCUMENT_ME}, + { &key_BINLOG_LOCK_lost_gtids_for_tailing, + "MYSQL_BIN_LOG::LOCK_lost_gtids_for_tailing", 0, 0, PSI_DOCUMENT_ME}, { &key_RELAYLOG_LOCK_commit, "MYSQL_RELAY_LOG::LOCK_commit", 0, 0, PSI_DOCUMENT_ME}, { &key_RELAYLOG_LOCK_index, "MYSQL_RELAY_LOG::LOCK_index", 0, 0, PSI_DOCUMENT_ME}, { &key_RELAYLOG_LOCK_log, "MYSQL_RELAY_LOG::LOCK_log", 0, 0, PSI_DOCUMENT_ME}, @@ -12910,6 +12945,8 @@ static PSI_mutex_info all_server_mutexes[]= { &key_RELAYLOG_LOCK_sync, "MYSQL_RELAY_LOG::LOCK_sync", 0, 0, PSI_DOCUMENT_ME}, { &key_RELAYLOG_LOCK_xids, "MYSQL_RELAY_LOG::LOCK_xids", 0, 0, PSI_DOCUMENT_ME}, { &key_RELAYLOG_LOCK_non_xid_trxs, "MYSQL_RELAY_LOG::LOCK_xids", 0, 0, PSI_DOCUMENT_ME}, + { &key_RELAYLOG_LOCK_lost_gtids_for_tailing, + "MYSQL_RELAY_LOG::LOCK_lost_gtids_for_tailing",0, 0, PSI_DOCUMENT_ME}, { &key_hash_filo_lock, "hash_filo::lock", 0, 0, PSI_DOCUMENT_ME}, { &Gtid_set::key_gtid_executed_free_intervals_mutex, "Gtid_set::gtid_executed::free_intervals_mutex", 0, 0, PSI_DOCUMENT_ME}, { &key_LOCK_crypt, "LOCK_crypt", PSI_FLAG_SINGLETON, 0, PSI_DOCUMENT_ME}, diff --git a/sql/mysqld.h b/sql/mysqld.h index 4dc7cda8a02f..fb92aeddd0de 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -450,8 +450,10 @@ extern bool disable_raft_log_repointing; extern ulong opt_raft_signal_async_dump_threads; extern bool disallow_raft; extern bool override_enable_raft_check; +extern bool abort_on_raft_purge_error; extern ulonglong apply_log_retention_num; extern ulonglong apply_log_retention_duration; +extern bool recover_raft_log; /* Apply log related variables for raft */ extern char *opt_apply_logname; extern char *opt_applylog_index_name; @@ -715,6 +717,7 @@ extern PSI_mutex_key key_RELAYLOG_LOCK_log_end_pos; extern PSI_mutex_key key_RELAYLOG_LOCK_sync; extern PSI_mutex_key key_RELAYLOG_LOCK_non_xid_trxs; extern PSI_mutex_key key_RELAYLOG_LOCK_xids; +extern PSI_mutex_key key_RELAYLOG_LOCK_lost_gtids_for_tailing; extern PSI_mutex_key key_gtid_ensure_index_mutex; extern PSI_mutex_key key_mta_temp_table_LOCK; extern PSI_mutex_key key_mta_gaq_LOCK; diff --git a/sql/rpl_binlog_sender.cc b/sql/rpl_binlog_sender.cc index 3c87872fa866..7f82b10907e8 100644 --- a/sql/rpl_binlog_sender.cc +++ b/sql/rpl_binlog_sender.cc @@ -48,6 +48,7 @@ #include "mysql/psi/mysql_file.h" #include "mysql/psi/mysql_mutex.h" #include "mysql/psi/mysql_socket.h" +#include "sql/binlog.h" #include "sql/binlog_reader.h" #include "sql/debug_sync.h" // debug_sync_set_action #include "sql/derror.h" // ER_THD @@ -70,6 +71,8 @@ #include "typelib.h" #include "unsafe_string_append.h" +std::atomic block_dump_threads{false}; + #ifndef NDEBUG static uint binlog_dump_count = 0; #endif @@ -350,12 +353,16 @@ bool Binlog_sender::set_dscp(void) { void Binlog_sender::init() { DBUG_TRACE; THD *thd = m_thd; - + auto raw_log = dump_log.get_log(false); thd->push_diagnostics_area(&m_diag_area); init_heartbeat_period(); m_last_event_sent_ts = now_in_nanosecs(); - + if (enable_raft_plugin && block_dump_threads) { + set_unknown_error("Binlog dump threads are blocked!"); + return; + } mysql_mutex_lock(&thd->LOCK_thd_data); + m_linfo = LOG_INFO(raw_log->is_relay_log, /* is_used_by_dump_thd = */ true); thd->current_linfo = &m_linfo; mysql_mutex_unlock(&thd->LOCK_thd_data); @@ -368,7 +375,7 @@ void Binlog_sender::init() { /* Save original query. Will be unset in cleanup */ m_orig_query = thd->query(); - if (!mysql_bin_log.is_open()) { + if (!raw_log->is_open()) { set_fatal_error("Binary log is not open"); return; } @@ -502,8 +509,11 @@ bool is_semi_sync_slave() { void Binlog_sender::run() { DBUG_TRACE; - init(); + { + Dump_log::Locker lock(&dump_log); + init(); + } m_is_semi_sync_slave = is_semi_sync_slave(); unsigned int max_event_size = @@ -550,28 +560,47 @@ void Binlog_sender::run() { "wait_for continue_dump_thread no_clear_event"; assert(!debug_sync_set_action(m_thd, STRING_WITH_LEN(act))); };); - mysql_bin_log.lock_index(); - if (!mysql_bin_log.is_open()) { - if (mysql_bin_log.open_index_file(mysql_bin_log.get_index_fname(), - log_file, false)) { + + // Note: This part is tricky and should be touched if you really know + // what you're doing. We're locking dump log to get the raw log + // pointer, then we're locking lock_index before unlocking the dump + // log. We're taking the lock in the same sequence as when log is + // switched in binlog_change_to_binlog() and binlog_change_to_apply() + // to avoid deadlocks. This locking pattern ensures that we're working + // with the correct raw log and that there is no race between getting + // the raw log and log switching. Log switching will be blocked until + // we release the binlog end pos lock before waiting for signal in + // wait_for_update_bin_log(). + const bool is_locked = dump_log.lock(); + MYSQL_BIN_LOG *raw_log = dump_log.get_log(false); + raw_log->lock_index(); + dump_log.unlock(is_locked); + if (!raw_log->is_open()) { + if (raw_log->open_index_file(raw_log->get_index_fname(), log_file, + false)) { set_fatal_error( "Binary log is not open and failed to open index file " "to retrieve next file."); - mysql_bin_log.unlock_index(); + raw_log->unlock_index(); break; } is_index_file_reopened_on_binlog_disable = true; } - int error = mysql_bin_log.find_next_log(&m_linfo, false); - mysql_bin_log.unlock_index(); + DBUG_EXECUTE_IF("dump_wait_before_find_next_log", { + const char act[] = "now signal signal.reached wait_for signal.done"; + assert(opt_debug_sync_timeout > 0); + assert(!debug_sync_set_action(m_thd, STRING_WITH_LEN(act))); + };); + int error = raw_log->find_next_log(&m_linfo, false); + raw_log->unlock_index(); if (unlikely(error)) { DBUG_EXECUTE_IF("waiting_for_disable_binlog", { const char act[] = "now signal consumed_binlog"; assert(!debug_sync_set_action(m_thd, STRING_WITH_LEN(act))); };); if (is_index_file_reopened_on_binlog_disable) - mysql_bin_log.close(LOG_CLOSE_INDEX, true /*need_lock_log=true*/, - true /*need_lock_index=true*/); + raw_log->close(LOG_CLOSE_INDEX, true /*need_lock_log=true*/, + true /*need_lock_index=true*/); set_fatal_error("could not find next log"); break; } @@ -669,19 +698,17 @@ int Binlog_sender::get_binlog_end_pos(File_reader *reader, my_off_t *end_pos) { to wait for updates on the binary log (Binlog_sender::wait_new_event()). */ bool wait_for_ack = !m_is_semi_sync_slave; + MYSQL_BIN_LOG *raw_log = dump_log.get_log(true); *end_pos = - mysql_bin_log.get_last_acked_pos(&wait_for_ack, m_linfo.log_file_name); - + raw_log->get_last_acked_pos(&wait_for_ack, m_linfo.log_file_name); /* If this is a cold binlog file, we are done getting the end pos */ - if (unlikely(!wait_for_ack && - !mysql_bin_log.is_active(m_linfo.log_file_name))) { + if (unlikely(!wait_for_ack && !raw_log->is_active(m_linfo.log_file_name))) { *end_pos = 0; return 0; } - DBUG_PRINT("info", ("Reading file %s, seek pos %llu, end_pos is %llu", m_linfo.log_file_name, read_pos, *end_pos)); - DBUG_PRINT("info", ("Active file is %s", mysql_bin_log.get_log_fname())); + DBUG_PRINT("info", ("Active file is %s", raw_log->get_log_fname())); if (read_pos < *end_pos) return 0; @@ -925,22 +952,32 @@ inline bool Binlog_sender::skip_event(const uchar *event_ptr, int Binlog_sender::wait_new_events(my_off_t log_pos) { int ret = 0; PSI_stage_info old_stage; - - mysql_bin_log.lock_binlog_end_pos(); + // Note: This part is tricky and should be touched if you really know + // what you're doing. We're locking dump log to get the raw log + // pointer, then we're locking end log pos before unlocking the dump + // log. We're taking the lock in the same sequence as when log is + // switched in binlog_change_to_binlog() and binlog_change_to_apply() + // to avoid deadlocks. This locking pattern ensures that we're working + // with the correct raw log and that there is no race between getting + // the raw log and log switching. Log switching will be blocked until + // we release the binlog end pos lock before waiting for signal in + // wait_for_update_bin_log(). + bool is_locked = dump_log.lock(); + MYSQL_BIN_LOG *raw_log = dump_log.get_log(false); + raw_log->lock_binlog_end_pos(); + dump_log.unlock(is_locked); /* If the binary log was updated before reaching this waiting point, there is no need to wait. */ bool wait_for_ack = !m_is_semi_sync_slave; - if (mysql_bin_log.get_last_acked_pos(&wait_for_ack, m_linfo.log_file_name) > + if (raw_log->get_last_acked_pos(&wait_for_ack, m_linfo.log_file_name) > log_pos || - (!wait_for_ack && !mysql_bin_log.is_active(m_linfo.log_file_name))) { - mysql_bin_log.unlock_binlog_end_pos(); + (!wait_for_ack && !raw_log->is_active(m_linfo.log_file_name))) { + raw_log->unlock_binlog_end_pos(); return ret; } - - m_thd->ENTER_COND(mysql_bin_log.get_log_cond(), - mysql_bin_log.get_binlog_end_pos_lock(), + m_thd->ENTER_COND(raw_log->get_log_cond(), raw_log->get_binlog_end_pos_lock(), &stage_source_has_sent_all_binlog_to_replica, &old_stage); if (m_heartbeat_period.count() > 0) @@ -948,7 +985,7 @@ int Binlog_sender::wait_new_events(my_off_t log_pos) { else ret = wait_without_heartbeat(); - mysql_bin_log.unlock_binlog_end_pos(); + raw_log->unlock_binlog_end_pos(); m_thd->EXIT_COND(&old_stage); return ret; } @@ -959,11 +996,13 @@ inline int Binlog_sender::wait_with_heartbeat(my_off_t log_pos) { #endif struct timespec ts; int ret; - ulong signal_cnt = mysql_bin_log.signal_cnt; + auto raw_log = dump_log.get_log(false); + + ulong signal_cnt = raw_log->signal_cnt; do { set_timespec_nsec(&ts, m_heartbeat_period.count()); - ret = mysql_bin_log.wait_for_update(&ts); + ret = raw_log->wait_for_update(&ts); if (!is_timeout(ret)) break; #ifndef NDEBUG @@ -976,13 +1015,13 @@ inline int Binlog_sender::wait_with_heartbeat(my_off_t log_pos) { } #endif if (send_heartbeat_event(log_pos, true)) return 1; - } while (signal_cnt == mysql_bin_log.signal_cnt && !m_thd->killed); - + } while (signal_cnt == raw_log->signal_cnt && !m_thd->killed); return ret ? 1 : 0; } inline int Binlog_sender::wait_without_heartbeat() { - return mysql_bin_log.wait_for_update(nullptr); + auto raw_log = dump_log.get_log(false); + return raw_log->wait_for_update(nullptr); } void Binlog_sender::init_heartbeat_period() { @@ -1004,9 +1043,9 @@ int Binlog_sender::check_start_file() { char index_entry_name[FN_REFLEN]; char *name_ptr = nullptr; std::string errmsg; - + auto raw_log = dump_log.get_log(false); if (m_start_file[0] != '\0') { - mysql_bin_log.make_log_name(index_entry_name, m_start_file); + raw_log->make_log_name(index_entry_name, m_start_file); name_ptr = index_entry_name; } else if (m_using_gtid_protocol) { /* @@ -1071,15 +1110,24 @@ int Binlog_sender::check_start_file() { will not find one and an error ER_MASTER_HAS_PURGED_REQUIRED_GTIDS is thrown from there. */ - if (!gtid_state->get_lost_gtids()->is_subset(m_exclude_gtid)) { - mysql_bin_log.report_missing_purged_gtids(m_exclude_gtid, errmsg); + Gtid_set *lost_gtids = const_cast(gtid_state->get_lost_gtids()); + + Sid_map gtids_lost_sid_map(nullptr); + Gtid_set gtids(>ids_lost_sid_map, nullptr); + if (enable_raft_plugin) { + Gtid_set *lost_gtid = >ids; + raw_log->get_lost_gtid_for_tailing(lost_gtid); + } + if (!lost_gtids->is_subset(m_exclude_gtid)) { + mysql_bin_log.report_missing_purged_gtids(lost_gtids, m_exclude_gtid, + errmsg); global_sid_lock->unlock(); set_fatal_error(errmsg.c_str()); return 1; } global_sid_lock->unlock(); Gtid first_gtid = {0, 0}; - if (mysql_bin_log.find_first_log_not_in_gtid_set( + if (raw_log->find_first_log_not_in_gtid_set( index_entry_name, m_exclude_gtid, &first_gtid, errmsg)) { set_fatal_error(errmsg.c_str()); return 1; @@ -1107,7 +1155,7 @@ int Binlog_sender::check_start_file() { then starts from the first file in index file. */ - if (mysql_bin_log.find_log_pos(&m_linfo, name_ptr, true)) { + if (raw_log->find_log_pos(&m_linfo, name_ptr, true)) { set_fatal_error( "Could not find first log file name in binary log " "index file"); @@ -1425,7 +1473,7 @@ int Binlog_sender::send_heartbeat_event_v1(my_off_t log_pos, // value in the HB event otherwise we use now() time_t ts = 0; if (send_timestamp) { - ts = mysql_bin_log.last_master_timestamp.load(); + ts = dump_log.get_log(false)->last_master_timestamp.load(); if (ts == 0) ts = time(nullptr); } int4store(header, ts); diff --git a/sql/rpl_binlog_sender.h b/sql/rpl_binlog_sender.h index 65131acaf215..78d18654a784 100644 --- a/sql/rpl_binlog_sender.h +++ b/sql/rpl_binlog_sender.h @@ -41,6 +41,7 @@ class String; class THD; extern uint rpl_send_buffer_size; +extern std::atomic block_dump_threads; /** The major logic of dump thread is implemented in this class. It sends diff --git a/sql/rpl_handler.cc b/sql/rpl_handler.cc index 249198cda4b5..fb6aaafc37cf 100644 --- a/sql/rpl_handler.cc +++ b/sql/rpl_handler.cc @@ -75,7 +75,8 @@ extern int raft_reset_slave(THD *thd); extern int raft_change_master( - THD *thd, const std::pair &master_instance); + THD *thd, const std::pair &master_instance, + const std::string &master_uuid); extern int rotate_binlog_file(THD *thd); extern int raft_stop_sql_thread(THD *thd); extern int raft_stop_io_thread(THD *thd); @@ -478,6 +479,50 @@ void delegates_update_lock_type() { if (binlog_relay_io_delegate) binlog_relay_io_delegate->update_lock_type(); } +/* + This macro is used by raft Delegate methods to call into raft plugin + The only difference is that this is a 'stricter' version which will return + failure if the plugin hooks were not called + */ +#define FOREACH_OBSERVER_STRICT(r, f, args) \ + Prealloced_array plugins(PSI_NOT_INSTRUMENTED); \ + read_lock(); \ + Observer_info_iterator iter = observer_info_iter(); \ + Observer_info *info = iter++; \ + for (; info; info = iter++) { \ + plugin_ref plugin = my_plugin_lock(0, &info->plugin); \ + if (!plugin) { \ + /* plugin is not intialized or deleted, this is not an error */ \ + enable_raft_plugin ? (r) = 1 : (r) = 0; \ + break; \ + } \ + plugins.push_back(plugin); \ + if (((Observer *)info->observer)->f && \ + ((Observer *)info->observer)->f args) { \ + r = 1; \ + LogEvent() \ + .prio(ERROR_LEVEL) \ + .errcode(ER_RPL_PLUGIN_FUNCTION_FAILED) \ + .subsys(LOG_SUBSYSTEM_TAG) \ + .function(#f) \ + .message("Run function '" #f "' in plugin '%s' failed", \ + info->plugin_int->name.str); \ + break; \ + } \ + /* Plugin is successfully called, set return status to 0 \ + * indicating success */ \ + (r) = 0; \ + } \ + unlock(); \ + /* \ + Unlock plugins should be done after we released the Delegate lock \ + to avoid possible deadlock when this is the last user of the \ + plugin, and when we unlock the plugin, it will try to \ + deinitialize the plugin, which will try to lock the Delegate in \ + order to remove the observers. \ + */ \ + if (!plugins.empty()) plugin_unlock_list(0, &plugins[0], plugins.size()); + /* This macro is used by almost all the Delegate methods to iterate over all the observers running given callback function of the @@ -612,6 +657,13 @@ int Trans_delegate::before_commit(THD *thd, bool all, (all || !thd->get_transaction()->is_active(Transaction_ctx::SESSION)); if (is_real_trans) param.flags |= TRANS_IS_REAL_TRANS; + if (mysql_bin_log.is_apply_log) + thd->get_trans_relay_log_pos(¶m.log_file, ¶m.log_pos); + else + thd->get_trans_fixed_pos(¶m.log_file, ¶m.log_pos); + + DBUG_PRINT("enter", + ("log_file: %s, log_pos: %llu", param.log_file, param.log_pos)); int ret = 0; FOREACH_OBSERVER(ret, before_commit, (¶m)); plugin_foreach(thd, se_before_commit, MYSQL_STORAGE_ENGINE_PLUGIN, ¶m); @@ -811,7 +863,6 @@ int Trans_delegate::after_commit(THD *thd, bool all) { bool is_real_trans = (all || !thd->get_transaction()->is_active(Transaction_ctx::SESSION)); if (is_real_trans) param.flags |= TRANS_IS_REAL_TRANS; - thd->get_trans_fixed_pos(¶m.log_file, ¶m.log_pos); param.server_id = thd->server_id; param.rpl_channel_type = thd->rpl_thd_ctx.get_rpl_channel_type(); @@ -836,7 +887,10 @@ int Trans_delegate::after_rollback(THD *thd, bool all) { bool is_real_trans = (all || !thd->get_transaction()->is_active(Transaction_ctx::SESSION)); if (is_real_trans) param.flags |= TRANS_IS_REAL_TRANS; - thd->get_trans_fixed_pos(¶m.log_file, ¶m.log_pos); + if (mysql_bin_log.is_apply_log) + thd->get_trans_relay_log_pos(¶m.log_file, ¶m.log_pos); + else + thd->get_trans_fixed_pos(¶m.log_file, ¶m.log_pos); param.server_id = thd->server_id; param.rpl_channel_type = thd->rpl_thd_ctx.get_rpl_channel_type(); @@ -1270,8 +1324,7 @@ int Raft_replication_delegate::before_flush(THD *thd, IO_CACHE *io_cache, int ret = 0; - thd->set_trans_marker(-1, -1); - FOREACH_OBSERVER(ret, before_flush, (¶m, io_cache, op_type)); + FOREACH_OBSERVER_STRICT(ret, before_flush, (¶m, io_cache, op_type)); DBUG_PRINT("return", ("term: %ld, index: %ld", param.term, param.index)); @@ -1291,7 +1344,7 @@ int Raft_replication_delegate::before_commit(THD *thd) { DBUG_PRINT("enter", ("term: %ld, index: %ld", param.term, param.index)); int ret = 0; - FOREACH_OBSERVER(ret, before_commit, (¶m)); + FOREACH_OBSERVER_STRICT(ret, before_commit, (¶m)); DEBUG_SYNC(thd, "after_call_after_sync_observer"); DBUG_RETURN(ret); @@ -1302,7 +1355,7 @@ int Raft_replication_delegate::setup_flush( DBUG_ENTER("Raft_replication_delegate::setup_flush"); int ret = 0; - FOREACH_OBSERVER(ret, setup_flush, (arg)); + FOREACH_OBSERVER_STRICT(ret, setup_flush, (arg)); DBUG_RETURN(ret); } @@ -1311,7 +1364,7 @@ int Raft_replication_delegate::before_shutdown(THD * /* thd */) { DBUG_ENTER("Raft_replication_delegate::before_shutdown"); int ret = 0; - FOREACH_OBSERVER(ret, before_shutdown, ()); + FOREACH_OBSERVER_STRICT(ret, before_shutdown, ()); DBUG_RETURN(ret); } @@ -1324,9 +1377,10 @@ int Raft_replication_delegate::register_paths( DBUG_ENTER("Raft_replication_delegate::register_paths"); int ret = 0; - FOREACH_OBSERVER(ret, register_paths, - (&raft_listener_queue, s_uuid, wal_dir_parent, - log_dir_parent, raft_log_path_prefix, s_hostname, port)); + FOREACH_OBSERVER_STRICT( + ret, register_paths, + (&raft_listener_queue, s_uuid, wal_dir_parent, log_dir_parent, + raft_log_path_prefix, s_hostname, port)); DBUG_RETURN(ret); } @@ -1337,8 +1391,15 @@ int Raft_replication_delegate::after_commit(THD *thd) { thd->get_trans_marker(¶m.term, ¶m.index); + const char *file = nullptr; + my_off_t pos = 0; + if (mysql_bin_log.is_apply_log) + thd->get_trans_relay_log_pos(&file, &pos); + else + thd->get_trans_fixed_pos(&file, &pos); + int ret = 0; - FOREACH_OBSERVER(ret, after_commit, (¶m)); + FOREACH_OBSERVER_STRICT(ret, after_commit, (¶m)); DBUG_RETURN(ret); } @@ -1347,7 +1408,7 @@ int Raft_replication_delegate::purge_logs(THD *thd, uint64_t file_ext) { Raft_replication_param param; param.purge_file_ext = file_ext; int ret = 0; - FOREACH_OBSERVER(ret, purge_logs, (¶m)); + FOREACH_OBSERVER_STRICT(ret, purge_logs, (¶m)); // Set the safe purge file that was sent back by the plugin thd->set_safe_purge_file(param.purge_file); @@ -1361,7 +1422,7 @@ int Raft_replication_delegate::show_raft_status( DBUG_ENTER("Raft_replication_delegate::show_raft_status"); Raft_replication_param param; int ret = 0; - FOREACH_OBSERVER(ret, show_raft_status, (var_value_pairs)); + FOREACH_OBSERVER_STRICT(ret, show_raft_status, (var_value_pairs)); DBUG_RETURN(ret); } @@ -1756,8 +1817,8 @@ extern "C" void *process_raft_queue(void *) { break; } case RaftListenerCallbackType::CHANGE_MASTER: { - result.error = - raft_change_master(current_thd, element.arg.master_instance); + result.error = raft_change_master( + current_thd, element.arg.master_instance, element.arg.master_uuid); if (!result.error && !element.arg.val_str.empty()) { Item_string item(element.arg.val_str.c_str(), element.arg.val_str.length(), @@ -1767,6 +1828,13 @@ extern "C" void *process_raft_queue(void *) { } break; } + + case RaftListenerCallbackType::GET_COMMITTED_GTIDS: { + result.error = + get_committed_gtids(element.arg.trim_gtids, &result.gtids); + break; + } + case RaftListenerCallbackType::GET_EXECUTED_GTIDS: { char *buffer; global_sid_lock->wrlock(); @@ -1785,6 +1853,10 @@ extern "C" void *process_raft_queue(void *) { result.error = raft_config_change(std::move(element.arg.val_str)); break; } + case RaftListenerCallbackType::HANDLE_DUMP_THREADS: { + result.error = handle_dump_threads(element.arg.val_bool); + break; + } default: // placate the compiler result.error = 0; diff --git a/sql/rpl_handler.h b/sql/rpl_handler.h index 714b9cfddb11..790080bec717 100644 --- a/sql/rpl_handler.h +++ b/sql/rpl_handler.h @@ -468,6 +468,12 @@ extern Raft_replication_delegate *raft_replication_delegate; #define RUN_HOOK(group, hook, args) \ (group##_delegate->is_empty() ? 0 : group##_delegate->hook args) +/* + This is same as RUN_HOOK, but return 1 if there are no observers +*/ +#define RUN_HOOK_STRICT(group, hook, args) \ + (group##_delegate->is_empty() ? 1 : group##_delegate->hook args) + #define NO_HOOK(group) (group##_delegate->is_empty()) int launch_hook_trans_begin(THD *thd, TABLE_LIST *table); diff --git a/sql/rpl_replica.cc b/sql/rpl_replica.cc index c30937c5457e..45dee3772416 100644 --- a/sql/rpl_replica.cc +++ b/sql/rpl_replica.cc @@ -196,6 +196,16 @@ bool reset_seconds_behind_master = true; const char *relay_log_index = nullptr; const char *relay_log_basename = nullptr; + +/* When raft has done a TermAdvancement, it starts + * the SQL thread. During the receive of No-Ops it + * does the log repointing and then starts the SQL + * thread. During this phase, no external actor should + * be able to start the SQL thread. This boolean is + * set to true when raft has stopped the SQL thread. + */ +std::atomic sql_thread_stopped_by_raft(false); + std::weak_ptr global_applier_reader; /* MTS load-ballancing parameter. @@ -623,6 +633,12 @@ int init_replica() { goto err; } + for (mi_map::iterator it = channel_map.begin(); it != channel_map.end(); + it++) { + Master_info *mi = it->second; + mi->rli->relay_log.raft_log_recover(); + } + /* Loop through the channel_map and start slave threads for each channel. */ @@ -1364,10 +1380,35 @@ int rli_relay_log_raft_reset( mysql_mutex_lock(&mi->data_lock); mysql_mutex_lock(&mi->rli->data_lock); - if (mi->rli->check_info() == REPOSITORY_DOES_NOT_EXIST) { + enum_return_check check_return_mi = mi->check_info(); + enum_return_check check_return_rli = mi->rli->check_info(); + + // If the master.info file does not exist, or if it exists, + // but the inited has never happened (most likely due to an + // error), try mi_init_info + if (check_return_mi == REPOSITORY_DOES_NOT_EXIST || !mi->inited) { // NO_LINT_DEBUG sql_print_information( - "Relay log info repository doesn't exist, creating one now"); + "rli_relay_log_raft_reset: Master info " + "repository doesn't exist or not inited." + " Calling mi_init_info"); + if (mi->mi_init_info()) { + // NO_LINT_DEBUG + sql_print_error( + "rli_relay_log_raft_reset: Failed to initialize " + "the master info structure"); + error = 1; + goto end; + } + } + + if (check_return_rli == REPOSITORY_DOES_NOT_EXIST) { + // NO_LINT_DEBUG + // NO_LINT_DEBUG + sql_print_information( + "rli_relay_log_raft_reset: Relay log info repository" + " doesn't exist or not inited. Calling" + " load_mi_and_rli_from_repositories "); // TODO: Check these additional params (skip_received_gtid_set_recovery) if (load_mi_and_rli_from_repositories( mi, @@ -1508,14 +1549,31 @@ int load_mi_and_rli_from_repositories(Master_info *mi, bool ignore_if_no_info, */ check_return = mi->check_info(); if (check_return == ERROR_CHECKING_REPOSITORY) { + if (enable_raft_plugin) { + // NO_LINT_DEBUG + sql_print_error( + "load_mi_and_rli_from_repositories: mi repository " + "check returns ERROR_CHECKING_REPOSITORY"); + } init_error = 1; goto end; } if (!ignore_if_no_info || check_return != REPOSITORY_DOES_NOT_EXIST) { if ((thread_mask & SLAVE_IO) != 0) { + if (enable_raft_plugin) { + // NO_LINT_DEBUG + sql_print_information( + "load_mi_and_rli_from_repositories: mi_init_info called"); + } if (!mi->inited || force_load) { if (mi->mi_init_info()) { + if (enable_raft_plugin) { + // NO_LINT_DEBUG + sql_print_error( + "load_mi_and_rli_from_repositories: mi_init_info returned " + "error"); + } init_error = 1; } } @@ -1524,13 +1582,30 @@ int load_mi_and_rli_from_repositories(Master_info *mi, bool ignore_if_no_info, check_return = mi->rli->check_info(); if (check_return == ERROR_CHECKING_REPOSITORY) { + if (enable_raft_plugin) { + // NO_LINT_DEBUG + sql_print_error( + "load_mi_and_rli_from_repositories: rli repository check returns" + " ERROR_CHECKING_REPOSITORY"); + } init_error = 1; goto end; } if (!ignore_if_no_info || check_return != REPOSITORY_DOES_NOT_EXIST) { if ((thread_mask & SLAVE_SQL) != 0 || !(mi->rli->inited)) { + if (enable_raft_plugin) { + // NO_LINT_DEBUG + sql_print_information( + "load_mi_and_rli_from_repositories: rli_init_info called"); + } if (!mi->rli->inited || force_load) { if (mi->rli->rli_init_info(skip_received_gtid_set_recovery)) { + if (enable_raft_plugin) { + // NO_LINT_DEBUG + sql_print_error( + "load_mi_and_rli_from_repositories: rli_init_info returned " + "error"); + } init_error = 1; } else { /* @@ -1589,13 +1664,16 @@ int raft_reset_slave(THD *) { channel_map.rdlock(); Master_info *mi = channel_map.get_default_channel_mi(); + if (!mi) { + channel_map.unlock(); + DBUG_RETURN(error); + } mysql_mutex_lock(&mi->data_lock); strmake(mi->host, "\0", sizeof(mi->host) - 1); mi->port = 0; mi->inited = false; mysql_mutex_lock(&mi->rli->data_lock); mi->rli->inited = false; - mi->flush_info(true); /** Clear the retrieved gtid set for this channel. */ @@ -1605,7 +1683,7 @@ int raft_reset_slave(THD *) { mysql_mutex_unlock(&mi->rli->data_lock); mysql_mutex_unlock(&mi->data_lock); - + remove_info(mi); // no longer a slave. will be set again during change master is_slave = false; channel_map.unlock(); @@ -1614,28 +1692,52 @@ int raft_reset_slave(THD *) { // TODO: currently we're only setting host port int raft_change_master( - THD *, const std::pair &master_instance) { + THD *, const std::pair &master_instance, + const std::string &master_uuid) { DBUG_ENTER("raft_change_master"); int error = 0; channel_map.rdlock(); Master_info *mi = channel_map.get_default_channel_mi(); - if (!mi) goto end; + if (!mi) { + channel_map.unlock(); + DBUG_RETURN(error); + } mysql_mutex_lock(&mi->data_lock); strmake(mi->host, const_cast(master_instance.first.c_str()), sizeof(mi->host) - 1); mi->port = master_instance.second; + assert(master_uuid.length() == UUID_LENGTH); + strncpy(mi->master_uuid, master_uuid.c_str(), UUID_LENGTH); + mi->master_uuid[UUID_LENGTH] = 0; mi->set_auto_position(true); mi->init_master_log_pos(); + + int thread_mask_stopped_threads; + /* + Before load_mi_and_rli_from_repositories() call, get a bit mask to indicate + stopped threads in thread_mask_stopped_threads. Since the third argguement + is 1, thread_mask when the function returns stands for stopped threads. + */ + init_thread_mask(&thread_mask_stopped_threads, mi, 1); + mysql_mutex_lock(&mi->rli->data_lock); + // Call mi->init_info() and/or mi->rli->init_info() if itn't configured + if (load_mi_and_rli_from_repositories(mi, false, thread_mask_stopped_threads, + false, /*need_lock*/ false)) { + error = ER_MASTER_INFO; + my_error(ER_MASTER_INFO, MYF(0)); + goto end; + } mi->inited = true; mi->flush_info(true); - mysql_mutex_unlock(&mi->data_lock); // changing to a slave. set the is_slave flag is_slave = true; end: + mysql_mutex_unlock(&mi->rli->data_lock); + mysql_mutex_unlock(&mi->data_lock); channel_map.unlock(); DBUG_RETURN(error); } @@ -2320,6 +2422,12 @@ bool start_slave_threads(bool need_lock_slave, bool wait_for_start, if (!mi->inited || !mi->rli->inited) { int error = (!mi->inited ? ER_SLAVE_MI_INIT_REPOSITORY : ER_SLAVE_RLI_INIT_REPOSITORY); + + if (enable_raft_plugin) { + // NO_LINT_DEBUG + sql_print_error("start_slave_threads: error: %d mi_inited: %d", error, + mi->inited); + } Rpl_info *info = (!mi->inited ? mi : static_cast(mi->rli)); const char *prefix = current_thd ? ER_THD_NONCONST(current_thd, error) : ER_DEFAULT_NONCONST(error); @@ -5255,6 +5363,32 @@ static int exec_relay_log_event(THD *thd, Relay_log_info *rli, } if (ev) { + if (enable_raft_plugin && + ev->get_type_code() == binary_log::METADATA_EVENT) { + Metadata_log_event *mev = static_cast(ev); + if (mev->does_exist(binary_log::Metadata_event::Metadata_event_types:: + RAFT_TERM_INDEX_TYPE)) { + const int64_t term = mev->get_raft_term(); + const int64_t index = mev->get_raft_index(); + if (rli->last_opid.first != -1 && rli->last_opid.second != -1 && + (index != rli->last_opid.second + 1 || + term < rli->last_opid.first)) { + char msg[1024]; + snprintf( + msg, sizeof(msg), + "Out of order opid found last opid=%ld:%ld, current opid=%ld:%ld", + rli->last_opid.first, rli->last_opid.second, term, index); + rli->report(ERROR_LEVEL, ER_SLAVE_RELAY_LOG_READ_FAILURE, + ER_THD(thd, ER_SLAVE_RELAY_LOG_READ_FAILURE), msg); + rli->abort_slave = 1; + mysql_mutex_unlock(&rli->data_lock); + delete ev; + return 1; + } + rli->last_opid = std::make_pair(term, index); + } + } + if (rli->is_row_format_required()) { bool info_error{false}; binary_log::Log_event_basic_info log_event_info; @@ -6518,6 +6652,10 @@ bool mts_recovery_groups(Relay_log_info *rli) { return false; } + // raft replication always have GTID_MODE=ON, thus ignore positions + if (enable_raft_plugin) { + return false; + } /* Save relay log position to compare with worker's position. */ @@ -7336,6 +7474,8 @@ extern "C" void *handle_slave_sql(void *arg) { else rli->current_mts_submode = new Mts_submode_dependency(); + rli->last_opid = std::make_pair(-1, -1); + const auto replica_preserve_commit_order = get_slave_preserve_commit_order(); // Only use replica preserve commit order if more than 1 worker exists @@ -9423,7 +9563,7 @@ uint sql_replica_skip_counter; */ bool start_slave(THD *thd, LEX_SLAVE_CONNECTION *connection_param, LEX_MASTER_INFO *master_param, int thread_mask_input, - Master_info *mi, bool set_mts_settings) { + Master_info *mi, bool set_mts_settings, bool invoked_by_raft) { bool is_error = false; int thread_mask; @@ -9470,9 +9610,32 @@ bool start_slave(THD *thd, LEX_SLAVE_CONNECTION *connection_param, if (thread_mask) // some threads are stopped, start them { + // If raft is doing some critical operations to block out threads, + // we disallow slave sql start till raft has restarted the slave + // thread. + if (enable_raft_plugin && !invoked_by_raft && sql_thread_stopped_by_raft) { + unlock_slave_threads(mi); + + mi->channel_unlock(); + + // NO_LINT_DEBUG + sql_print_information( + "Did not allow start_slave as raft has stopped SQL threads"); + my_error(ER_RAFT_OPERATION_INCOMPATIBLE, MYF(0), + "start slave not allowed when raft has stopped SQL threads"); + return true; + } + + (void)invoked_by_raft; if (load_mi_and_rli_from_repositories(mi, false, thread_mask)) { is_error = true; my_error(ER_MASTER_INFO, MYF(0)); + + if (enable_raft_plugin) { + // NO_LINT_DEBUG + sql_print_error( + "start_slave: error as load_mi_and_rli_from_repositories failed"); + } } else if (*mi->host || !(thread_mask & SLAVE_IO)) { /* If we will start IO thread we need to take care of possible @@ -9653,6 +9816,11 @@ int raft_stop_sql_thread(THD *thd) { res = stop_slave(thd, mi, /*net_report=*/0, /*for_one_channel=*/true, &push_temp_table_warning); + if (!res) { + // set this flag to prevent other non-raft actors from + // starting sql thread during critical raft operations + sql_thread_stopped_by_raft = true; + } end: channel_map.unlock(); @@ -9678,7 +9846,12 @@ int raft_start_sql_thread(THD *thd) { } res = start_slave(thd, &lex_connection, &lex_mi, thd->lex->slave_thd_opt, mi, - /*set_mts_settings=*/true); + /*set_mts_settings=*/true, true /*invoked_by_raft*/); + if (!res) { + // reset this flag to let other non-raft actors + // to stop and start sql threads. + sql_thread_stopped_by_raft = false; + } end: channel_map.unlock(); diff --git a/sql/rpl_replica.h b/sql/rpl_replica.h index ee72e7205ce2..eda348d59bdd 100644 --- a/sql/rpl_replica.h +++ b/sql/rpl_replica.h @@ -527,7 +527,8 @@ bool start_slave(THD *thd); int stop_slave(THD *thd); bool start_slave(THD *thd, LEX_SLAVE_CONNECTION *connection_param, LEX_MASTER_INFO *master_param, int thread_mask_input, - Master_info *mi, bool set_mts_settings); + Master_info *mi, bool set_mts_settings, + bool invoked_by_raft = false); int stop_slave(THD *thd, Master_info *mi, bool net_report, bool for_one_channel, bool *push_temp_table_warning); /* diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index 7218a5138c13..24118dc2c7a4 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -210,9 +210,9 @@ Relay_log_info::Relay_log_info(bool is_slave_recovery, PSI_NOT_INSTRUMENTED, PSI_NOT_INSTRUMENTED, key_RELAYLOG_LOCK_log, key_RELAYLOG_LOCK_log_end_pos, key_RELAYLOG_LOCK_sync, PSI_NOT_INSTRUMENTED, key_RELAYLOG_LOCK_xids, - key_RELAYLOG_LOCK_non_xid_trxs, PSI_NOT_INSTRUMENTED, - PSI_NOT_INSTRUMENTED, key_RELAYLOG_update_cond, PSI_NOT_INSTRUMENTED, - key_RELAYLOG_non_xid_trxs_cond, key_file_relaylog, + key_RELAYLOG_LOCK_non_xid_trxs, key_RELAYLOG_LOCK_lost_gtids_for_tailing, + PSI_NOT_INSTRUMENTED, PSI_NOT_INSTRUMENTED, key_RELAYLOG_update_cond, + PSI_NOT_INSTRUMENTED, key_RELAYLOG_non_xid_trxs_cond, key_file_relaylog, key_file_relaylog_index, key_file_relaylog_cache, key_file_relaylog_index_cache); #endif @@ -1553,7 +1553,12 @@ int Relay_log_info::rli_init_info(bool skip_received_gtid_set_recovery) { stopped when there were replication initialization errors, now it is not and so init_info() must be aware of previous failures. */ - if (error_on_rli_init_info) goto err; + if (error_on_rli_init_info) { + // In raft mode, these error codes are critical. Hence we should + // not chew them. + if (enable_raft_plugin) error = 1; + goto err; + } if (inited) { return recovery_parallel_workers ? mts_recovery_groups(this) : 0; @@ -1934,7 +1939,9 @@ int Relay_log_info::remove_logged_gtids( DBUG_PRINT("info", ("Removing gtid(sidno:%d, gno:%ld) from rli logged gtids", gtid.sidno, gtid.gno)); + get_sid_lock()->wrlock(); gtid_set->_remove_gtid(gtid); + get_sid_lock()->unlock(); } } diff --git a/sql/rpl_rli.h b/sql/rpl_rli.h index 131074d6d4b1..4702e21b4806 100644 --- a/sql/rpl_rli.h +++ b/sql/rpl_rli.h @@ -339,6 +339,8 @@ class Relay_log_info : public Rpl_info { // Last gtid seen by coordinator thread. char last_gtid[Gtid::MAX_TEXT_LENGTH + 1]; + std::pair last_opid = std::make_pair(-1, -1); + /* The following variables are safe to read any time */ /* diff --git a/sql/rpl_source.cc b/sql/rpl_source.cc index 2c14eaf5dc9b..3aa100abd3e3 100644 --- a/sql/rpl_source.cc +++ b/sql/rpl_source.cc @@ -1322,6 +1322,26 @@ void kill_zombie_dump_threads(THD *thd) { } } +class Kill_dump_thread : public Do_THD_Impl { + public: + virtual void operator()(THD *thd) override { + if (thd->get_command() == COM_BINLOG_DUMP || + thd->get_command() == COM_BINLOG_DUMP_GTID) { + mysql_mutex_lock(&thd->LOCK_thd_data); + thd->awake(THD::KILL_CONNECTION); + mysql_mutex_unlock(&thd->LOCK_thd_data); + } + } +}; + +/* + Kill all Binlog_dump threads. +*/ +void kill_all_dump_threads() { + Kill_dump_thread kill_dump_thread; + Global_THD_manager::get_instance()->do_for_all_thd_copy(&kill_dump_thread); +} + /** Execute a RESET MASTER statement. @@ -1332,7 +1352,7 @@ void kill_zombie_dump_threads(THD *thd) { @retval false success @retval true error */ -bool reset_master(THD *thd, bool unlock_global_read_lock) { +bool reset_master(THD *thd, bool unlock_global_read_lock, bool force) { bool ret = false; /* @@ -1345,6 +1365,26 @@ bool reset_master(THD *thd, bool unlock_global_read_lock) { */ thd->set_skip_readonly_check(); + /* + No RESET MASTER commands are allowed while Raft replication is running + */ + if (enable_raft_plugin) { + if (!force && !override_enable_raft_check) { + // NO_LINT_DEBUG + sql_print_information( + "Did not allow reset_master as enable_raft_plugin is ON"); + my_error(ER_CANT_RESET_MASTER, MYF(0), + "reset master not allowed when enable_raft_plugin is ON"); + ret = true; + goto end; + } else { + // NO_LINT_DEBUG + sql_print_information( + "Allow reset_master in enable_raft_plugin mode as force " + "or override_enable_raft_check is set"); + } + } + /* No RESET MASTER commands are allowed while Group Replication is running unless executed during a clone operation as part of the process. diff --git a/sql/rpl_source.h b/sql/rpl_source.h index a1ec572420d5..62774a68aef4 100644 --- a/sql/rpl_source.h +++ b/sql/rpl_source.h @@ -87,6 +87,7 @@ bool show_master_offset(THD *thd, snapshot_info_st &ss_info, bool *need_ok); bool show_master_status(THD *thd); bool show_binlogs(THD *thd, bool with_gtid = false); void kill_zombie_dump_threads(THD *thd); +void kill_all_dump_threads(); uint find_gtid_position_helper(const char *gtid_string, char *log_name, my_off_t >id_pos); @@ -149,7 +150,7 @@ bool com_binlog_dump(THD *thd, char *packet, size_t packet_length); void mysql_binlog_send(THD *thd, char *log_ident, my_off_t pos, Gtid_set *gtid_set, uint32 flags); -bool reset_master(THD *thd, bool unlock_read_lock); +bool reset_master(THD *thd, bool unlock_read_lock, bool force = false); std::vector get_all_replica_statistics(); int get_current_replication_lag(); diff --git a/sql/sql_class.h b/sql/sql_class.h index 41e1c6cac139..4bece2cce8ac 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -1950,6 +1950,11 @@ class THD : public MDL_context_owner, NET net; // client connection descriptor String packet; // dynamic buffer for network I/O + /** + * Relay log positions for the transaction + */ + std::pair m_trans_relay_log_pos; + /* The term and index that need to be communicated across different raft * plugin hooks. These fields are not protected by locks since they are * accessed by the same THD serially during different stages of ordered commit @@ -2993,6 +2998,16 @@ class THD : public MDL_context_owner, return; } + void get_trans_relay_log_pos(const char **file_var, my_off_t *pos_var) const { + if (file_var) *file_var = m_trans_relay_log_pos.first.c_str(); + if (pos_var) *pos_var = m_trans_relay_log_pos.second; + } + + void set_trans_relay_log_pos(const std::string &file, my_off_t pos) { + m_trans_relay_log_pos.first = file; + m_trans_relay_log_pos.second = pos; + } + /**@}*/ /* Get the trans marker i.e (term, index) tuple stashed in this THD */ diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 4447074f72fd..4d759126df6b 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -4353,9 +4353,10 @@ static Sys_var_int32 Sys_regexp_stack_limit( static bool update_rpl_wait_for_semi_sync_ack(sys_var *, THD *, enum_var_type) { if (!rpl_wait_for_semi_sync_ack) { - mysql_bin_log.lock_binlog_end_pos(); - mysql_bin_log.signal_update(); - mysql_bin_log.unlock_binlog_end_pos(); + auto raw_log = dump_log.get_log(/*should_lock*/ true); + raw_log->lock_binlog_end_pos(); + raw_log->signal_update(); + raw_log->unlock_binlog_end_pos(); } return false; } @@ -7421,6 +7422,15 @@ static Sys_var_gtid_purged Sys_gtid_purged( NOT_IN_BINLOG, ON_CHECK(check_gtid_purged)); export sys_var *Sys_gtid_purged_ptr = &Sys_gtid_purged; +Gtid_set *gtid_purged_for_tailing; +static Sys_var_gtid_purged_for_tailing Sys_gtid_purged_for_tailing( + "gtid_purged_for_tailing", + "The set of GTIDs that existed in previous, purged binary logs in " + "non-raft mode. The set of GTIDs that existed in previous, purged " + "raft logs in raft mode; ", + READ_ONLY GLOBAL_VAR(gtid_purged_for_tailing), NO_CMD_LINE, DEFAULT(NULL), + NO_MUTEX_GUARD, NOT_IN_BINLOG); + static Sys_var_gtid_owned Sys_gtid_owned( "gtid_owned", "The global variable lists all GTIDs owned by all threads. " @@ -8679,14 +8689,24 @@ static bool validate_enable_raft(sys_var * /*self */, THD *, set_var *var) { err = true; } + if (!err) { + // NO_LINT_DEBUG + sql_print_information("Killing and blocking binlog dump threads"); + err = !block_all_dump_threads(); + } + return err; } -static bool log_enable_raft_change(sys_var * /*self */, THD *thd, - enum_var_type) { +static bool update_enable_raft_change(sys_var * /*self */, THD *thd, + enum_var_type) { const char *user = "unknown"; const char *host = "unknown"; + // NO_LINT_DEBUG + sql_print_information("Unblocking dump threads"); + unblock_all_dump_threads(); + if (thd && thd->get_user_connect()) { user = (const_cast(thd->get_user_connect()))->user; host = (const_cast(thd->get_user_connect()))->host; @@ -8707,7 +8727,13 @@ static Sys_var_bool Sys_enable_raft_plugin( "plugin when it is enabled", GLOBAL_VAR(enable_raft_plugin), CMD_LINE(OPT_ARG), DEFAULT(false), NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(validate_enable_raft), - ON_UPDATE(log_enable_raft_change)); + ON_UPDATE(update_enable_raft_change)); + +static Sys_var_bool Sys_abort_on_raft_purge_error( + "abort_on_raft_purge_error", + "Any error in raft plugin to purge files will abort the server", + GLOBAL_VAR(abort_on_raft_purge_error), CMD_LINE(OPT_ARG), DEFAULT(false), + NO_MUTEX_GUARD, NOT_IN_BINLOG); static Sys_var_bool Sys_disallow_raft( "disallow_raft", @@ -8862,6 +8888,13 @@ static Sys_var_ulonglong Sys_apply_log_retention_duration( GLOBAL_VAR(apply_log_retention_duration), CMD_LINE(OPT_ARG), VALID_RANGE(0, ULONG_LONG_MAX), DEFAULT(15), BLOCK_SIZE(1)); +static Sys_var_bool Sys_recover_raft_log( + "recover_raft_log", + "Temprary variable to control recovery of raft log by removing partial " + "trxs. This should be removed later.", + GLOBAL_VAR(recover_raft_log), CMD_LINE(OPT_ARG), DEFAULT(true), + NO_MUTEX_GUARD, NOT_IN_BINLOG); + /* Free global_write_statistics if sys_var is set to 0 */ static bool update_write_stats_count(sys_var *, THD *, enum_var_type) { if (write_stats_count == 0) { diff --git a/sql/sys_vars.h b/sql/sys_vars.h index a04c464f5110..436536c570b1 100644 --- a/sql/sys_vars.h +++ b/sql/sys_vars.h @@ -54,6 +54,7 @@ #include "mysql/udf_registration_types.h" #include "mysqld_error.h" #include "sql/auth/sql_security_ctx.h" +#include "sql/binlog.h" // dump_log #include "sql/debug_sync.h" // debug_sync_update #include "sql/handler.h" #include "sql/item.h" // Item @@ -2782,6 +2783,86 @@ class Sys_var_gtid_purged : public sys_var { } }; +/** + Class for @@global.gtid_purged_for_tailing. +*/ +class Sys_var_gtid_purged_for_tailing : public sys_var { + public: + Sys_var_gtid_purged_for_tailing( + const char *name_arg, const char *comment, int flag_args, ptrdiff_t off, + size_t, CMD_LINE getopt, const char *def_val, PolyLock *lock = 0, + enum binlog_status_enum binlog_status_arg = VARIABLE_NOT_IN_BINLOG, + on_check_function on_check_func = 0, + on_update_function on_update_func = 0, const char *substitute = 0, + int parse_flag = PARSE_NORMAL) + : sys_var(&all_sys_vars, name_arg, comment, flag_args, off, getopt.id, + getopt.arg_type, SHOW_CHAR, (intptr)def_val, lock, + binlog_status_arg, on_check_func, on_update_func, substitute, + parse_flag) {} + + bool session_update(THD *, set_var *) override { + assert(false); + return true; + } + + void session_save_default(THD *, set_var *var) override { + my_error(ER_NO_DEFAULT, MYF(0), var->var->name.str); + } + + bool global_update(THD *, set_var *) override { + assert(false); + return true; + } + + void global_save_default(THD *, set_var *var) override { + /* gtid_purged does not have default value */ + my_error(ER_NO_DEFAULT, MYF(0), var->var->name.str); + } + + bool check_update_type(Item_result) override { + assert(false); + return true; + } + + void saved_value_to_string(THD *, set_var *var, char *) override { + my_error(ER_NO_DEFAULT, MYF(0), var->var->name.str); + } + + bool do_check(THD *, set_var *) override { + assert(false); + return true; + } + + const uchar *global_value_ptr(THD *thd, LEX_STRING *) override { + DBUG_TRACE; + char *buf = nullptr; + if (opt_bin_log) { + Sid_map gtids_lost_sid_map(nullptr); + Gtid_set gs(>ids_lost_sid_map, nullptr); + dump_log.get_lost_gtids(&gs); + buf = reinterpret_cast(thd->alloc(gs.get_string_length() + 1)); + if (buf == nullptr) + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + else + gs.to_string(buf); + } else { + /* + When binlog is off, report @@GLOBAL.GTID_PURGED_FOR_TAILING + from executed_gtids. Same as GTID_PURGED. + */ + global_sid_lock->wrlock(); + const Gtid_set *gs = gtid_state->get_executed_gtids(); + buf = reinterpret_cast(thd->alloc(gs->get_string_length() + 1)); + if (buf == nullptr) + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + else + gs->to_string(buf); + global_sid_lock->unlock(); + } + return reinterpret_cast(buf); + } +}; + class Sys_var_gtid_owned : Sys_var_charptr_func { public: Sys_var_gtid_owned(const char *name_arg, const char *comment_arg)