Merge pull request #5866 from planetscale/config_cleanup

config.json cleanup

.github/workflows/check_make_visitor.yml

@@ -0,0 +1,35 @@
+name: check_make_visitor
+on: [push, pull_request]
+jobs:
+
+  build:
+    name: Check Make Visitor
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.13
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+    - name: Run make minimaltools
+      run: |
+        make minimaltools
+
+    - name: check_make_visitor
+      run: |
+        misc/git/hooks/visitorgen
+

.github/workflows/cluster_endtoend.yml

@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        name: [11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]
+        name: [11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]
 
     steps:
     - name: Set up Go

.github/workflows/misc_test_docker.yml

@@ -0,0 +1,20 @@
+name: misc test
+on: [push, pull_request]
+jobs:
+
+  build:
+    name: Misc Test
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.13
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Run Misc test which requires docker
+      run: |
+        go run test.go -docker=true -shard 25

Makefile

@@ -324,11 +324,11 @@ packages: docker_base
 
 tools:
 	echo $$(date): Installing dependencies
-	BUILD_PYTHON=0 ./bootstrap.sh
+	./bootstrap.sh
 
 minimaltools:
 	echo $$(date): Installing minimal dependencies
-	BUILD_PYTHON=0 BUILD_JAVA=0 BUILD_CONSUL=0 ./bootstrap.sh
+	BUILD_JAVA=0 BUILD_CONSUL=0 ./bootstrap.sh
 
 dependency_check:
 	./tools/dependency_check.sh

bootstrap.sh

@@ -23,7 +23,6 @@ source ./dev.env
 # 0. Initialization and helper methods.
 # 1. Installation of dependencies.
 
-BUILD_PYTHON=${BUILD_PYTHON:-1}
 BUILD_JAVA=${BUILD_JAVA:-1}
 BUILD_CONSUL=${BUILD_CONSUL:-1}
 
@@ -89,34 +88,6 @@ function get_arch() {
   uname -m
 }
 
-
-# Install the gRPC Python library (grpcio) and the protobuf gRPC Python plugin (grpcio-tools) from PyPI.
-# Dependencies like the Python protobuf package will be installed automatically.
-function install_grpc() {
-  local version="$1"
-  local dist="$2"
-
-  # Python requires a very recent version of virtualenv.
-  # We also require a recent version of pip, as we use it to
-  # upgrade the other tools.
-  # For instance, setuptools doesn't work with pip 6.0:
-  # https://github.com/pypa/setuptools/issues/945
-  # (and setuptools is used by grpc install).
-  grpc_virtualenv="$dist/usr/local"
-  $VIRTUALENV -v "$grpc_virtualenv"
-  PIP=$grpc_virtualenv/bin/pip
-  $PIP install --upgrade pip
-  $PIP install --upgrade --ignore-installed virtualenv
-  $PIP install mysql-connector-python
-
-  grpcio_ver=$version
-  $PIP install --upgrade grpcio=="$grpcio_ver" grpcio-tools=="$grpcio_ver"
-}
-
-if [ "$BUILD_PYTHON" == 1 ] ; then
-    install_dep "gRPC" "1.16.0" "$VTROOT/dist/grpc" install_grpc
-fi
-
 # Install protoc.
 function install_protoc() {
   local version="$1"
@@ -220,74 +191,5 @@ if [ "$BUILD_CONSUL" == 1 ] ; then
   install_dep "Consul" "1.4.0" "$VTROOT/dist/consul" install_consul
 fi
 
-# Install py-mock.
-function install_pymock() {
-  local version="$1"
-  local dist="$2"
-
-  # For some reason, it seems like setuptools won't create directories even with the --prefix argument
-  mkdir -p lib/python2.7/site-packages
-  PYTHONPATH=$(prepend_path "$PYTHONPATH" "$dist/lib/python2.7/site-packages")
-  export PYTHONPATH
-
-  pushd "$VTROOT/third_party/py" >/dev/null
-  tar -xzf "mock-$version.tar.gz"
-  cd "mock-$version"
-  $PYTHON ./setup.py install --prefix="$dist"
-  cd ..
-  rm -r "mock-$version"
-  popd >/dev/null
-}
-pymock_version=1.0.1
-if [ "$BUILD_PYTHON" == 1 ] ; then
-    install_dep "py-mock" "$pymock_version" "$VTROOT/dist/py-mock-$pymock_version" install_pymock
-fi
-
-# Download Selenium (necessary to run test/vtctld_web_test.py).
-function install_selenium() {
-  local version="$1"
-  local dist="$2"
-
-  PYTHONPATH='' $VIRTUALENV "$dist"
-  PIP="$dist/bin/pip"
-  # PYTHONPATH is removed for `pip install` because otherwise it can pick up go/dist/grpc/usr/local/lib/python2.7/site-packages
-  # instead of go/dist/selenium/lib/python3.5/site-packages and then can't find module 'pip._vendor.requests'
-  PYTHONPATH='' $PIP install selenium
-}
-if [ "$BUILD_PYTHON" == 1 ] ; then
-    install_dep "Selenium" "latest" "$VTROOT/dist/selenium" install_selenium
-fi
-
-# Download chromedriver (necessary to run test/vtctld_web_test.py).
-function install_chromedriver() {
-  local version="$1"
-  local dist="$2"
-
-  if [ "$(arch)" == "aarch64" ] ; then
-      os=$(cat /etc/*release | grep "^ID=" | cut -d '=' -f 2)
-      case $os in
-          ubuntu|debian)
-              sudo apt-get update -y && sudo apt install -y --no-install-recommends unzip libglib2.0-0 libnss3 libx11-6
-	      ;;
-	  centos|fedora)
-	      sudo yum update -y && yum install -y libX11 unzip wget
-	      ;;
-      esac
-      echo "For Arm64, using prebuilt binary from electron (https://github.com/electron/electron/) of version 76.0.3809.126"
-      wget https://github.com/electron/electron/releases/download/v6.0.3/chromedriver-v6.0.3-linux-arm64.zip
-      unzip -o -q chromedriver-v6.0.3-linux-arm64.zip -d "$dist"
-      rm chromedriver-v6.0.3-linux-arm64.zip
-  else
-      curl -sL "https://chromedriver.storage.googleapis.com/$version/chromedriver_linux64.zip" > chromedriver_linux64.zip
-      unzip -o -q chromedriver_linux64.zip -d "$dist"
-      rm chromedriver_linux64.zip
-  fi
-}
-install_dep "chromedriver" "73.0.3683.20" "$VTROOT/dist/chromedriver" install_chromedriver
-
-if [ "$BUILD_PYTHON" == 1 ] ; then
-  PYTHONPATH='' $PIP install mysql-connector-python
-fi
-
 echo
 echo "bootstrap finished - run 'make build' to compile"

go/test/endtoend/README.md

@@ -15,6 +15,7 @@ These tests almost always launch a topology service, a few mysqld instances, a f
 All the end to end test are placed under path go/test/endtoend. 
 The main purpose of grouping them together is to make sure we have single place for reference and to combine similar test to run them in the same cluster and save test running time.  
 
+
 ### Setup
 All the tests should be launching a real cluster just like the production setup and execute the tests on that setup followed by a teardown of all the services.
 
@@ -31,22 +32,13 @@ In general the cluster is build in following order
 
 A good example to refer will be  go/test/endtoend/clustertest
 
-## Progress
-So far we have converted the following Python end to end test cases
-- Keyspace tests
-- mysqlctl tests
-- sharded tests
-- tabletmanager tests
-- vtgate v3 tests
-- Inital sharding
-- resharding
-- vsplit
-- reparent
-
-### In-progress 
-- Backup
-- Encryption
-
-After a Python test is migrated in Go it will be removed from end to end ci test run by updating the shard value to 5 in `test/config.json`
+### Pre-Requisite 
+Make sure you have vitess binary available in bin folder. If not, then you can run `./bootstrap.sh` follow by `make build` & `source build.env`.
 
-
+To make it easier to re-run test please add following to you bash profile. 
+
+```
+export VTROOT=/<vitess path>/vitess
+export VTDATAROOT=${VTROOT}/vtdataroot
+export PATH=${VTROOT}/bin:${PATH}
+```

test/client_test.sh

@@ -37,9 +37,6 @@ vtctlclient -server localhost:15999 RebuildVSchemaGraph
 
 CELL=test ./scripts/vtgate-up.sh
 
-echo "Run Python client script.."
-$VTROOT/test/client.sh
-
 echo "Run Go client script..."
 go run $VTROOT/test/client.go -server=localhost:15991