DBZ-8196 Tutorial for Vitess 2.7

tutorial/README.md

@@ -378,7 +378,7 @@ docker-compose -f docker-compose-cassandra.yaml down
 
 ```shell 
 # Start the topology as defined in https://debezium.io/documentation/reference/stable/tutorial.html
-export DEBEZIUM_VERSION=2.1
+export DEBEZIUM_VERSION=2.7
 
 docker-compose -f docker-compose-vitess.yaml up --build
 

tutorial/debezium-vitess-init/Dockerfile

@@ -1,7 +1,7 @@
 # Use a temporary layer for the build stage.
-FROM vitess/base:v12.0.6 AS base
+FROM vitess/base:v19.0.4 AS base
 
-FROM vitess/lite:v12.0.6
+FROM vitess/lite:v19.0.4
 
 USER root
 

tutorial/debezium-vitess-init/local/env.sh

@@ -15,12 +15,17 @@ fi
 
 # mysqld might be in /usr/sbin which will not be in the default PATH
 PATH="/usr/sbin:$PATH"
-for binary in mysqld etcd etcdctl curl vtctlclient vttablet vtgate vtctld mysqlctl; do
+for binary in mysqld etcd etcdctl curl vtctlclient vtctldclient vttablet vtgate vtctld mysqlctl; do
   command -v "$binary" > /dev/null || fail "${binary} is not installed in PATH. See https://vitess.io/docs/get-started/local/ for install instructions."
 done;
 
+# vtctlclient has a separate alias setup below
+for binary in vttablet vtgate vtctld mysqlctl vtorc vtctl; do
+  alias $binary="$binary --config-file-not-found-handling=ignore"
+done;
+
 ETCD_SERVER="localhost:2379"
-TOPOLOGY_FLAGS="-topo_implementation etcd2 -topo_global_server_address $ETCD_SERVER -topo_global_root /vitess/global"
+TOPOLOGY_FLAGS="--topo_implementation etcd2 --topo_global_server_address $ETCD_SERVER --topo_global_root /vitess/global"
 mkdir -p "${VTDATAROOT}/etcd"
 
 mkdir -p "${VTDATAROOT}/tmp"
@@ -30,7 +35,8 @@ mkdir -p "${VTDATAROOT}/tmp"
 # such as ~/.my.cnf
 
 alias mysql="command mysql -h 127.0.0.1 -P 15306"
-alias vtctlclient="command vtctlclient -server localhost:15999 -log_dir ${VTDATAROOT}/tmp -alsologtostderr"
+alias vtctlclient="command vtctlclient --server localhost:15999 --log_dir ${VTDATAROOT}/tmp --alsologtostderr --config-file-not-found-handling=ignore --grpc_auth_static_client_creds grpc_static_client_auth.json "
+alias vtctldclient="command vtctldclient --server localhost:15999 --grpc_auth_static_client_creds grpc_static_client_auth.json "
 
 # Make sure aliases are expanded in non-interactive shell
 shopt -s expand_aliases

tutorial/debezium-vitess-init/local/initial_cluster.sh

@@ -8,36 +8,37 @@ CELL=zone1 ./scripts/etcd-up.sh
 # start vtctld
 CELL=zone1 ./scripts/vtctld-up.sh
 
+vtctldclient CreateKeyspace --durability-policy=semi_sync customer || fail "Failed to create and configure customer keyspace"
+vtctldclient CreateKeyspace --durability-policy=semi_sync inventory || fail "Failed to create and configure inventory keyspace"
+
 # start vttablets for unsharded keyspace customer
 for i in 100 101 102; do
 	CELL=zone1 TABLET_UID=$i ./scripts/mysqlctl-up.sh
 	CELL=zone1 KEYSPACE=customer TABLET_UID=$i ./scripts/vttablet-up.sh
 done
 
-# set one of the replicas to master
-vtctlclient -grpc_auth_static_client_creds grpc_static_client_auth.json InitShardMaster -force customer/0 zone1-100
+# start vtorc, which handles electing shard primaries (no need for InitShardPrimary manual step)
+source ./scripts/vtorc-up.sh
+
 
 # start vttablets for sharded keyspace inventory
 for i in 200 201 202; do
 	CELL=zone1 TABLET_UID=$i ./scripts/mysqlctl-up.sh
 	SHARD=-80 CELL=zone1 KEYSPACE=inventory TABLET_UID=$i ./scripts/vttablet-up.sh
 done
 
-
 for i in 300 301 302; do
 	CELL=zone1 TABLET_UID=$i ./scripts/mysqlctl-up.sh
 	SHARD=80- CELL=zone1 KEYSPACE=inventory TABLET_UID=$i ./scripts/vttablet-up.sh
 done
 
-# set one of the replicas to master
-vtctlclient -grpc_auth_static_client_creds grpc_static_client_auth.json InitShardMaster -force inventory/-80 zone1-200
-vtctlclient -grpc_auth_static_client_creds grpc_static_client_auth.json InitShardMaster -force inventory/80- zone1-300
+
 
 # create seq table and unsharded table in the unsharded keyspace, sharded tables in sharded keyspace
-vtctlclient -grpc_auth_static_client_creds grpc_static_client_auth.json ApplySchema -sql-file create_tables_unsharded_customer.sql customer
-vtctlclient -grpc_auth_static_client_creds grpc_static_client_auth.json ApplyVSchema -vschema_file vschema_tables_unsharded_customer.json customer
-vtctlclient -grpc_auth_static_client_creds grpc_static_client_auth.json ApplySchema -sql-file create_tables_sharded_inventory.sql inventory
-vtctlclient -grpc_auth_static_client_creds grpc_static_client_auth.json ApplyVSchema -vschema_file vschema_tables_sharded_inventory.json inventory
+vtctlclient ApplySchema -- --sql-file create_tables_unsharded_customer.sql customer
+vtctlclient ApplyVSchema -- --vschema_file vschema_tables_unsharded_customer.json customer
+vtctlclient ApplySchema -- --sql-file create_tables_sharded_inventory.sql inventory
+vtctlclient ApplyVSchema -- --vschema_file vschema_tables_sharded_inventory.json inventory
 
 # start vtgate
 CELL=zone1 ./scripts/vtgate-up.sh

tutorial/debezium-vitess-init/local/scripts/etcd-up.sh

@@ -3,30 +3,25 @@
 source ./env.sh
 
 cell=${CELL:-'test'}
-export ETCDCTL_API=2
+
+echo "Starting etcd..."
 
 # Check that etcd is not already running
 curl "http://${ETCD_SERVER}" > /dev/null 2>&1 && fail "etcd is already running. Exiting."
 
-etcd --enable-v2=true --data-dir "${VTDATAROOT}/etcd/"  --listen-client-urls "http://${ETCD_SERVER}" --advertise-client-urls "http://${ETCD_SERVER}" > "${VTDATAROOT}"/tmp/etcd.out 2>&1 &
+etcd --data-dir "${VTDATAROOT}/etcd/"  --listen-client-urls "http://${ETCD_SERVER}" --advertise-client-urls "http://${ETCD_SERVER}" > "${VTDATAROOT}"/tmp/etcd.out 2>&1 &
 PID=$!
 echo $PID > "${VTDATAROOT}/tmp/etcd.pid"
 sleep 5
 
-echo "add /vitess/global"
-etcdctl --endpoints "http://${ETCD_SERVER}" mkdir /vitess/global &
-
-echo "add /vitess/$cell"
-etcdctl --endpoints "http://${ETCD_SERVER}" mkdir /vitess/$cell &
-
 # And also add the CellInfo description for the cell.
 # If the node already exists, it's fine, means we used existing data.
 echo "add $cell CellInfo"
 set +e
 # shellcheck disable=SC2086
-vtctl $TOPOLOGY_FLAGS AddCellInfo \
-  -root /vitess/$cell \
-  -server_address "${ETCD_SERVER}" \
+vtctl $TOPOLOGY_FLAGS VtctldCommand AddCellInfo \
+  --root /vitess/$cell \
+  --server-address "${ETCD_SERVER}" \
   $cell
 set -e
 

tutorial/debezium-vitess-init/local/scripts/mysqlctl-up.sh

@@ -20,7 +20,7 @@ if [ -d $VTDATAROOT/$tablet_dir ]; then
 fi
 
 mysqlctl \
- -log_dir $VTDATAROOT/tmp \
- -tablet_uid $uid \
- -mysql_port $mysql_port \
+ --log_dir $VTDATAROOT/tmp \
+ --tablet_uid $uid \
+ --mysql_port $mysql_port \
  $action

tutorial/debezium-vitess-init/local/scripts/vtctld-up.sh

@@ -9,16 +9,14 @@ echo "Starting vtctld..."
 # shellcheck disable=SC2086
 vtctld \
  $TOPOLOGY_FLAGS \
- -cell $cell \
- -workflow_manager_init \
- -workflow_manager_use_election \
- -service_map 'grpc-vtctl' \
- -backup_storage_implementation file \
- -file_backup_storage_root $VTDATAROOT/backups \
- -log_dir $VTDATAROOT/tmp \
- -port $vtctld_web_port \
- -grpc_port $grpc_port \
- -pid_file $VTDATAROOT/tmp/vtctld.pid \
- -grpc_auth_mode static\
- -grpc_auth_static_password_file grpc_static_auth.json\
+ --cell $cell \
+ --service_map 'grpc-vtctl,grpc-vtctld' \
+ --backup_storage_implementation file \
+ --file_backup_storage_root $VTDATAROOT/backups \
+ --log_dir $VTDATAROOT/tmp \
+ --port $vtctld_web_port \
+ --grpc_port $grpc_port \
+ --pid_file $VTDATAROOT/tmp/vtctld.pid \
+ --grpc_auth_mode static\
+ --grpc_auth_static_password_file grpc_static_auth.json\
   > $VTDATAROOT/tmp/vtctld.out 2>&1 &

tutorial/debezium-vitess-init/local/scripts/vtgate-up.sh

@@ -12,20 +12,20 @@ mysql_server_socket_path="/tmp/mysql.sock"
 # shellcheck disable=SC2086
 vtgate \
   $TOPOLOGY_FLAGS \
-  -log_dir $VTDATAROOT/tmp \
-  -log_queries_to_file $VTDATAROOT/tmp/vtgate_querylog.txt \
-  -port $web_port \
-  -grpc_port $grpc_port \
-  -mysql_server_port $mysql_server_port \
-  -mysql_server_socket_path $mysql_server_socket_path \
-  -cell $cell \
-  -cells_to_watch $cell \
-  -tablet_types_to_wait MASTER,REPLICA \
-  -service_map 'grpc-vtgateservice' \
-  -pid_file $VTDATAROOT/tmp/vtgate.pid \
-  -mysql_auth_server_impl none \
-  -grpc_auth_mode static\
-  -grpc_auth_static_password_file grpc_static_auth.json\
+  --log_dir $VTDATAROOT/tmp \
+  --log_queries_to_file $VTDATAROOT/tmp/vtgate_querylog.txt \
+  --port $web_port \
+  --grpc_port $grpc_port \
+  --mysql_server_port $mysql_server_port \
+  --mysql_server_socket_path $mysql_server_socket_path \
+  --cell $cell \
+  --cells_to_watch $cell \
+  --tablet_types_to_wait MASTER,REPLICA \
+  --service_map 'grpc-vtgateservice' \
+  --pid_file $VTDATAROOT/tmp/vtgate.pid \
+  --mysql_auth_server_impl none \
+  --grpc_auth_mode static\
+  --grpc_auth_static_password_file grpc_static_auth.json\
   > $VTDATAROOT/tmp/vtgate.out 2>&1 &
 
 # Block waiting for vtgate to be listening

tutorial/debezium-vitess-init/local/scripts/vttablet-up.sh

@@ -20,27 +20,29 @@ if [[ "${uid: -1}" -gt 1 ]]; then
 fi
 
 echo "Starting vttablet for $alias..."
+
 # shellcheck disable=SC2086
 vttablet \
  $TOPOLOGY_FLAGS \
- -log_dir $VTDATAROOT/tmp \
- -log_queries_to_file $VTDATAROOT/tmp/$tablet_logfile \
- -tablet-path $alias \
- -tablet_hostname "$tablet_hostname" \
- -init_keyspace $keyspace \
- -init_shard $shard \
- -init_tablet_type $tablet_type \
- -health_check_interval 5s \
- -enable_semi_sync \
- -enable_replication_reporter \
- -backup_storage_implementation file \
- -file_backup_storage_root $VTDATAROOT/backups \
- -restore_from_backup \
- -port $port \
- -grpc_port $grpc_port \
- -service_map 'grpc-queryservice,grpc-tabletmanager,grpc-updatestream' \
- -pid_file $VTDATAROOT/$tablet_dir/vttablet.pid \
- -vtctld_addr http://$hostname:$vtctld_web_port/ \
+ --log_dir $VTDATAROOT/tmp \
+ --log_queries_to_file $VTDATAROOT/tmp/$tablet_logfile \
+ --tablet-path $alias \
+ --tablet_hostname "$tablet_hostname" \
+ --init_keyspace $keyspace \
+ --init_shard $shard \
+ --init_tablet_type $tablet_type \
+ --health_check_interval 5s \
+ --backup_storage_implementation file \
+ --watch_replication_stream \
+ --file_backup_storage_root $VTDATAROOT/backups \
+ --restore_from_backup \
+ --port $port \
+ --grpc_port $grpc_port \
+ --service_map 'grpc-queryservice,grpc-tabletmanager,grpc-updatestream' \
+ --pid_file $VTDATAROOT/$tablet_dir/vttablet.pid \
+ --heartbeat_enable \
+ --heartbeat_interval=250ms \
+ --heartbeat_on_demand_duration=5s \
  > $VTDATAROOT/$tablet_dir/vttablet.out 2>&1 &
 
 # Block waiting for the tablet to be listening
@@ -53,3 +55,5 @@ done
 
 # check one last time
 curl -I "http://$hostname:$port/debug/status" || fail "tablet could not be started!"
+
+echo -e "vttablet for $alias is running!"

tutorial/docker-compose-vitess.yaml

@@ -1,4 +1,3 @@
-version: '2'
 services:
   zookeeper:
     image: quay.io/debezium/zookeeper:${DEBEZIUM_VERSION}
@@ -10,7 +9,7 @@ services:
     image: quay.io/debezium/kafka:${DEBEZIUM_VERSION}
     ports:
       - 9092:9092
-    links:
+    depends_on:
       - zookeeper
     environment:
       - ZOOKEEPER_CONNECT=zookeeper:2181
@@ -26,7 +25,7 @@ services:
     image: quay.io/debezium/connect:${DEBEZIUM_VERSION}
     ports:
       - 8083:8083
-    links:
+    depends_on:
       - kafka
       - vitess
     environment: