Merge branch 'upstream-master' into vtshovel-poc

docker/bootstrap/Dockerfile.mariadb

@@ -1,18 +1,23 @@
 FROM vitess/bootstrap:common
 
 # Install MariaDB 10
-RUN apt-get update -y \
+RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --recv-keys 9334A25F8507EFA5 && break; done && \
+    add-apt-repository 'deb http://repo.percona.com/apt stretch main' && \
+    { \
+        echo debconf debconf/frontend select Noninteractive; \
+        echo percona-server-server-5.7 percona-server-server/root_password password 'unused'; \
+        echo percona-server-server-5.7 percona-server-server/root_password_again password 'unused'; \
+    } | debconf-set-selections && \
+    apt-get update -y \
  && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
       bzip2 \
       mariadb-server \
       libmariadbclient-dev \
       libdbd-mysql-perl \
       rsync \
       libev4 \
- && rm -rf /var/lib/apt/lists/* \
- && wget https://www.percona.com/downloads/XtraBackup/Percona-XtraBackup-2.4.13/binary/debian/stretch/x86_64/percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb \
- && dpkg -i percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb \
- && rm -f percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb
+      percona-xtrabackup-24 \
+ && rm -rf /var/lib/apt/lists/*
 
 # Bootstrap Vitess
 WORKDIR /vt/src/vitess.io/vitess

docker/bootstrap/Dockerfile.mysql56

@@ -3,12 +3,16 @@ FROM vitess/bootstrap:common
 # Install MySQL 5.6
 RUN for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver pool.sks-keyservers.net 5072E1F5 && break; done && \
     add-apt-repository 'deb http://repo.mysql.com/apt/debian/ stretch mysql-5.6' && \
+    for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --recv-keys 9334A25F8507EFA5 && break; done && \
+    echo 'deb http://repo.percona.com/apt stretch main' > /etc/apt/sources.list.d/percona.list && \
+    { \
+        echo debconf debconf/frontend select Noninteractive; \
+        echo percona-server-server-5.6 percona-server-server/root_password password 'unused'; \
+        echo percona-server-server-5.6 percona-server-server/root_password_again password 'unused'; \
+    } | debconf-set-selections && \
     apt-get update -y && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server libmysqlclient-dev libdbd-mysql-perl rsync libev4 && \
-    rm -rf /var/lib/apt/lists/* && \
-    wget https://www.percona.com/downloads/XtraBackup/Percona-XtraBackup-2.4.13/binary/debian/stretch/x86_64/percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb && \
-    dpkg -i percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb && \
-    rm -f percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb
+    DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server libmysqlclient-dev libdbd-mysql-perl rsync libev4 percona-xtrabackup-24 && \
+    rm -rf /var/lib/apt/lists/*
 
 # Bootstrap Vitess
 WORKDIR /vt/src/vitess.io/vitess

docker/bootstrap/Dockerfile.mysql57

@@ -4,12 +4,16 @@ FROM vitess/bootstrap:common
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gnupg dirmngr ca-certificates && \
     for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver ha.pool.sks-keyservers.net 5072E1F5 && break; done && \
     add-apt-repository 'deb http://repo.mysql.com/apt/debian/ stretch mysql-5.7' && \
+    for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --recv-keys 9334A25F8507EFA5 && break; done && \
+    echo 'deb http://repo.percona.com/apt stretch main' > /etc/apt/sources.list.d/percona.list && \
+    { \
+        echo debconf debconf/frontend select Noninteractive; \
+        echo percona-server-server-5.7 percona-server-server/root_password password 'unused'; \
+        echo percona-server-server-5.7 percona-server-server/root_password_again password 'unused'; \
+    } | debconf-set-selections && \
     apt-get update -y && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server libmysqlclient-dev libdbd-mysql-perl rsync libev4 && \
-    rm -rf /var/lib/apt/lists/* && \
-    wget https://www.percona.com/downloads/XtraBackup/Percona-XtraBackup-2.4.13/binary/debian/stretch/x86_64/percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb && \
-    dpkg -i percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb && \
-    rm -f percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb
+    DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server libmysqlclient-dev libdbd-mysql-perl rsync libev4 percona-xtrabackup-24 && \
+    rm -rf /var/lib/apt/lists/*
 
 # Bootstrap Vitess
 WORKDIR /vt/src/vitess.io/vitess

docker/bootstrap/Dockerfile.mysql80

@@ -3,12 +3,16 @@ FROM vitess/bootstrap:common
 # Install MySQL 8.0
 RUN for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver ha.pool.sks-keyservers.net 8C718D3B5072E1F5 && break; done && \
     add-apt-repository 'deb http://repo.mysql.com/apt/debian/ stretch mysql-8.0' && \
+    for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --recv-keys 9334A25F8507EFA5 && break; done && \
+    echo 'deb http://repo.percona.com/apt stretch main' > /etc/apt/sources.list.d/percona.list && \
+    { \
+        echo debconf debconf/frontend select Noninteractive; \
+        echo percona-server-server-8.0 percona-server-server/root_password password 'unused'; \
+        echo percona-server-server-8.0 percona-server-server/root_password_again password 'unused'; \
+    } | debconf-set-selections && \
     apt-get update -y && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server libmysqlclient-dev libdbd-mysql-perl rsync libev4 && \
-    rm -rf /var/lib/apt/lists/* && \
-    wget https://www.percona.com/downloads/XtraBackup/Percona-XtraBackup-8.0.4/binary/debian/stretch/x86_64/percona-xtrabackup-80_8.0.4-1.stretch_amd64.deb && \
-    dpkg -i percona-xtrabackup-80_8.0.4-1.stretch_amd64.deb && \
-    rm -f percona-xtrabackup-80_8.0.4-1.stretch_amd64.deb
+    DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server libmysqlclient-dev libdbd-mysql-perl rsync libev4 libcurl4-openssl-dev percona-xtrabackup-80 && \
+    rm -rf /var/lib/apt/lists/*
 
 # Bootstrap Vitess
 WORKDIR /vt/src/vitess.io/vitess

docker/bootstrap/Dockerfile.percona

@@ -10,11 +10,8 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --r
     } | debconf-set-selections && \
     apt-get update && \
     apt-get install -y --no-install-recommends \
-        percona-server-server-5.6 libperconaserverclient18.1-dev rsync libev4 && \
-    rm -rf /var/lib/apt/lists/* && \
-    wget https://www.percona.com/downloads/XtraBackup/Percona-XtraBackup-2.4.13/binary/debian/stretch/x86_64/percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb && \
-    dpkg -i percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb && \
-    rm -f percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb
+        percona-server-server-5.6 libperconaserverclient18.1-dev rsync libev4 percona-xtrabackup-24 && \
+    rm -rf /var/lib/apt/lists/*
 
 # Bootstrap Vitess
 WORKDIR /vt/src/vitess.io/vitess

docker/bootstrap/Dockerfile.percona57

@@ -11,7 +11,7 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --r
     apt-get update && \
     apt-get install -y --no-install-recommends \
         percona-server-server-5.7 \
-	libperconaserverclient18.1-dev && \
+	libperconaserverclient18.1-dev percona-xtrabackup-24 && \
     rm -rf /var/lib/apt/lists/*
 
 # Bootstrap Vitess

docker/bootstrap/Dockerfile.percona80

@@ -18,10 +18,16 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --r
 	libdbd-mysql-perl \
 	rsync \
 	libev4 \
-    && rm -rf /var/lib/apt/lists/* \
-    && wget https://www.percona.com/downloads/XtraBackup/Percona-XtraBackup-8.0.4/binary/debian/stretch/x86_64/percona-xtrabackup-80_8.0.4-1.stretch_amd64.deb \
-    && dpkg -i percona-xtrabackup-80_8.0.4-1.stretch_amd64.deb \
-    && rm -f percona-xtrabackup-80_8.0.4-1.stretch_amd64.deb
+#    && rm -f /etc/apt/sources.list.d/percona.list \
+    && echo 'deb http://repo.percona.com/apt stretch main' > /etc/apt/sources.list.d/percona.list \
+#    { \
+#        echo debconf debconf/frontend select Noninteractive; \
+#        echo percona-server-server-8.0 percona-server-server/root_password password 'unused'; \
+#        echo percona-server-server-8.0 percona-server-server/root_password_again password 'unused'; \
+#    } | debconf-set-selections \
+    && apt-get update -y \
+    && apt-get install -y --no-install-recommends percona-xtrabackup-80 \
+    && rm -rf /var/lib/apt/lists/*
 
 # Bootstrap Vitess
 WORKDIR /vt/src/vitess.io/vitess

examples/compose/README.md

@@ -5,6 +5,30 @@ To understand it better, you can run it.
 
 First you will need to [install docker-compose](https://docs.docker.com/compose/install/).
 
+### Programatically create Vitess configuration for Docker
+To create a configuration to your specifications, run vtcompose. Creates corresponding docker-compose file, vschema files per keyspace, and loads schemas.
+```
+vitess/examples/compose$ go run vtcompose.go
+```
+
+Use `-h` or `--help` to get list of flags with descriptions.
+
+Flags available:
+* **baseDockerComposeFile** - Specifies starting docker-compose yaml file.
+* **baseVschemaFile** - Specifies starting vschema json file.
+* **topologyFlags** - Specifies Vitess topology flags config
+* **webPort** - Specifies web port to be used.
+* **gRpcPort** - Specifies gRPC port to be used.
+* **mySqlPort** - Specifies mySql port to be used.
+* **cell** - `Specifies Vitess cell name to be used.
+* **keyspaceData** - List of `keyspace_name:num_of_shards:num_of_replica_tablets:schema_file_names:<optional>lookup_keyspace_name` separated by ' '. 
+    * This is where you specify most of the data for the program to build your vSchema and docker-compose files.
+    * Use `0` for `num_of_shards` to specify an unsharded keyspace
+    ```
+    e.g "test_keyspace:2:1:create_messages.sql,create_tokens.sql:lookup_keyspace lookup_keyspace:1:1:create_tokens_token_lookup.sql,create_messages_message_lookup.sql"
+    ```
+
+
 ### Start the cluster
 To start Consul(which saves the topology config), vtctld, vtgate and a few vttablets with MySQL running on them.
 ```
@@ -19,15 +43,17 @@ vitess/examples/compose$ docker-compose logs -f vtgate
 ```
 
 ### Load the schema
-We need to create a few tables into our new cluster. To do that, we can run the `ApplySchema` command.
-```
-vitess/examples/compose$ ./lvtctl.sh ApplySchema -sql "$(cat create_test_table.sql)" test_keyspace
-```
+***Note: Should not be needed if VtCompose was used.***
 
-### Create Vschema
-Create Vschema
+We need to create a few tables into our new cluster. To do that, we can run the `ApplySchema` command.	
+```	
+vitess/examples/compose$ ./lvtctl.sh ApplySchema -sql "$(cat create_test_table.sql)" test_keyspace	
 ```
-vitess/examples/compose$ ./lvtctl.sh ApplyVschema -vschema '{"tables": {"messages": {} } }' test_keyspace
+
+### Create Vschema	
+Create Vschema (should not be needed if VtCompose was used)
+```	
+vitess/examples/compose$ ./lvtctl.sh ApplyVschema -vschema '{"tables": {"messages": {} } }' test_keyspace	
 ```
 
 ### Run the client to insert and read some data
@@ -40,7 +66,7 @@ vitess/examples/compose$ ./client.sh
 ### Connect to vgate and run queries
 vtgate responds to the MySQL protocol, so we can connect to it using the default MySQL client command line.
 ```
-vitess/examples/compose$ ./lmysql.sh --port=15306 --host=127.0.0.1
+vitess/examples/compose$ mysql --port=15306 --host=127.0.0.1
 ```
 
 
@@ -113,15 +139,15 @@ vitess/examples/compose$ ./lfixrepl.sh
 ```
 
 ### Apply Vschema
-Apply Vschema for the unsharded keyspace
-```
-vitess/examples/compose$ ./lvtctl.sh ApplyVschema -vschema '{"sharded":false, "tables": {"*": {} } }' external_db_name
+Apply Vschema for the unsharded keyspace	
+```	
+vitess/examples/compose$ ./lvtctl.sh ApplyVschema -vschema '{"sharded":false, "tables": {"*": {} } }' external_db_name	
 ```
 
 ### Connect to vgate and run queries
 vtgate responds to the MySQL protocol, so we can connect to it using the default MySQL client command line.
 ```sh
-vitess/examples/compose$ ./lmysql.sh --port=15306 --host=<host of machine containers are running in e.g. 127.0.0.1, docker-machine ip e.t.c>
+vitess/examples/compose$ mysql --port=15306 --host=<host of machine containers are running in e.g. 127.0.0.1, docker-machine ip e.t.c>
 
 mysql> show databases;
 +--------------------+
@@ -157,7 +183,6 @@ mysql> show tables;
 ## Helper Scripts
 The following helper scripts are included to help you perform various actions easily
 * vitess/examples/compose/lvtctl.sh
-* vitess/examples/compose/lmysql.sh
 * vitess/examples/compose/lfixrepl.sh
 
 You may run them as below

examples/compose/dbcli.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+usage () {
+  echo "Starts a session on a sideloaded vttablet."
+  echo "Note that this is a direct MySQL connection; if you actually want to work with Vitess, connect via the vtgate with:"
+  echo "  mysql --port=15306 --host=127.0.0.1"
+  echo
+  echo "Usage: $0 <tablet alias> [<keyspace>]"
+  echo "  Don't forget the 'vt_' before the keyspace!"
+}
+
+if [ $# -lt 1 ]; then
+  usage
+  exit -1
+fi
+
+keyspace=${2:-vt_test_keyspace}
+long_alias=`printf "%010d" $1`
+docker-compose exec vttablet$1 mysql -uvt_dba -S /vt/vtdataroot/vt_${long_alias}/mysql.sock $keyspace