Merge branch 'master' into jacques_skip_verify

Signed-off-by: Jacques Grove <[email protected]>

.github/workflows/cluster_endtoend.yml

@@ -40,6 +40,7 @@ jobs:
         fi
 
     - name: sharded cluster_endtoend
+      timeout-minutes: 30
       run: |
         source build.env
         eatmydata -- go run test.go -docker=false -print-log -follow -shard ${{matrix.name}}

.github/workflows/cluster_initial_sharding_multi.yml

@@ -0,0 +1,20 @@
+name: cluster_initial_sharding_multi
+on: [push, pull_request]
+jobs:
+
+  build:
+    name: cluster initial sharding multi
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.13
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Run initial sharding multi
+      run: |
+        go run test.go -print-log initial_sharding_multi

.github/workflows/e2e_race.yml

@@ -30,5 +30,6 @@ jobs:
         make minimaltools
 
     - name: e2e_race
+      timeout-minutes: 30
       run: |
         make e2e_test_race

.github/workflows/endtoend.yml

@@ -34,5 +34,6 @@ jobs:
         make build
 
     - name: endtoend
+      timeout-minutes: 30
       run: |
         tools/e2e_test_runner.sh

.github/workflows/legacy_local_example.yml

@@ -0,0 +1,54 @@
+name: local_example
+on: [push, pull_request]
+jobs:
+
+  build:
+    name: Legacy local example using ${{ matrix.topo }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        topo: [etcd,k8s]
+
+    steps:
+
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.13
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        if [ ${{matrix.os}} = "ubuntu-latest" ]; then
+          sudo apt-get update
+          sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+          sudo service mysql stop
+          sudo service etcd stop
+          sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+          sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        elif [ ${{matrix.os}} = "macos-latest" ]; then
+          brew install [email protected] make unzip etcd curl git wget
+        fi
+        go mod download
+
+    - name: Run make minimaltools
+      run: |
+        make minimaltools
+
+    - name: Build
+      run: |
+        make build
+
+    - name: local_example
+      timeout-minutes: 30
+      run: |
+        export TOPO=${{matrix.topo}}
+        if [ ${{matrix.os}} = "macos-latest" ]; then
+          export PATH="/usr/local/opt/[email protected]/bin:$PATH"
+        fi
+        # Make sure that testing is entirely non-reliant on config
+        mv config config-moved
+        eatmydata -- test/legacy_local_example.sh

.github/workflows/local_example.yml

@@ -43,6 +43,7 @@ jobs:
         make build
 
     - name: local_example
+      timeout-minutes: 30
       run: |
         export TOPO=${{matrix.topo}}
         if [ ${{matrix.os}} = "macos-latest" ]; then

.github/workflows/sonar_analysis.yml

@@ -2,7 +2,7 @@ name: sonar_analysis
 on:
   push:
     branches:
-      - 'master'
+      - 'sonartest'
 jobs:
 
   build:

.github/workflows/unit.yml

@@ -85,5 +85,6 @@ jobs:
         make tools
 
     - name: unit
+      timeout-minutes: 30
       run: |
         eatmydata -- make unit_test

.github/workflows/unit_race.yml

@@ -34,5 +34,6 @@ jobs:
         make tools
 
     - name: unit_race
+      timeout-minutes: 30
       run: |
         eatmydata -- make unit_test_race

Makefile

@@ -74,14 +74,6 @@ install: build
 	# binaries
 	mkdir -p "$${PREFIX}/bin"
 	cp "$${VTROOT}/bin/"{mysqlctld,vtctld,vtctlclient,vtgate,vttablet,vtworker,vtbackup} "$${PREFIX}/bin/"
-	# config files
-	mkdir -p "$${PREFIX}/src/vitess.io/vitess"
-	cp -R config "$${PREFIX}/src/vitess.io/vitess/"
-	# also symlink config files in the old location
-	ln -sf src/vitess.io/vitess/config "$${PREFIX}/config"
-	# vtctld web UI files
-	mkdir -p "$${PREFIX}/src/vitess.io/vitess/web/vtctld2"
-	cp -R web/vtctld2/app "$${PREFIX}/src/vitess.io/vitess/web/vtctld2/"
 
 parser:
 	make -C go/vt/sqlparser
@@ -289,12 +281,6 @@ docker_test:
 docker_unit_test:
 	go run test.go -flavor $(flavor) unit
 
-# This can be used to rebalance the total average runtime of each group of
-# tests in Travis. The results are saved in test/config.json, which you can
-# then commit and push.
-rebalance_tests:
-	go run test.go -rebalance 5
-
 # Release a version.
 # This will generate a tar.gz file into the releases folder with the current source
 release: docker_base
@@ -324,7 +310,7 @@ tools:
 
 minimaltools:
 	echo $$(date): Installing minimal dependencies
-	BUILD_JAVA=0 BUILD_CONSUL=0 ./bootstrap.sh
+	BUILD_CHROME=0 BUILD_JAVA=0 BUILD_CONSUL=0 ./bootstrap.sh
 
 dependency_check:
 	./tools/dependency_check.sh

bootstrap.sh

@@ -25,6 +25,7 @@ source ./dev.env
 
 BUILD_JAVA=${BUILD_JAVA:-1}
 BUILD_CONSUL=${BUILD_CONSUL:-1}
+BUILD_CHROME=${BUILD_CHROME:-1}
 
 #
 # 0. Initialization and helper methods.
@@ -172,13 +173,13 @@ function install_k3s() {
 
   case $(uname) in
     Linux)  local platform=linux;;
-    *)   echo "ERROR: unsupported platform. K3s only supports running on Linux"; exit 1;;
+    *)   echo "WARNING: unsupported platform. K3s only supports running on Linux, the k8s topology will not be available for local examples."; return;;
   esac
 
   case $(get_arch) in
       aarch64)  local target="-arm64";;
       x86_64)  local target="";;
-      *)   echo "ERROR: unsupported architecture"; exit 1;;
+      *)   echo "WARNING: unsupported architecture, the k8s topology will not be available for local examples."; return;;
   esac
 
   download_url=https://github.com/rancher/k3s/releases/download
@@ -191,7 +192,6 @@ function install_k3s() {
 }
 command -v  k3s || install_dep "k3s" "v1.0.0" "$VTROOT/dist/k3s" install_k3s
 
-
 # Download and install consul, link consul binary into our root.
 function install_consul() {
   local version="$1"
@@ -218,5 +218,40 @@ if [ "$BUILD_CONSUL" == 1 ] ; then
   install_dep "Consul" "1.4.0" "$VTROOT/dist/consul" install_consul
 fi
 
+# Download chromedriver
+function install_chromedriver() {
+  local version="$1"
+  local dist="$2"
+
+  case $(uname) in
+    Linux)  local platform=linux;;
+    *)   echo "Platform not supported for vtctl-web tests. Skipping chromedriver install."; return;;
+  esac
+
+  if [ "$(arch)" == "aarch64" ] ; then
+      os=$(cat /etc/*release | grep "^ID=" | cut -d '=' -f 2)
+      case $os in
+          ubuntu|debian)
+              sudo apt-get update -y && sudo apt install -y --no-install-recommends unzip libglib2.0-0 libnss3 libx11-6
+	      ;;
+	  centos|fedora)
+	      sudo yum update -y && yum install -y libX11 unzip wget
+	      ;;
+      esac
+      echo "For Arm64, using prebuilt binary from electron (https://github.com/electron/electron/) of version 76.0.3809.126"
+      wget https://github.com/electron/electron/releases/download/v6.0.3/chromedriver-v6.0.3-linux-arm64.zip
+      unzip -o -q chromedriver-v6.0.3-linux-arm64.zip -d "$dist"
+      rm chromedriver-v6.0.3-linux-arm64.zip
+  else
+      curl -sL "https://chromedriver.storage.googleapis.com/$version/chromedriver_linux64.zip" > chromedriver_linux64.zip
+      unzip -o -q chromedriver_linux64.zip -d "$dist"
+      rm chromedriver_linux64.zip
+  fi
+}
+
+if [ "$BUILD_CHROME" == 1 ] ; then
+	install_dep "chromedriver" "73.0.3683.20" "$VTROOT/dist/chromedriver" install_chromedriver
+fi
+
 echo
 echo "bootstrap finished - run 'make build' to compile"

build.env

@@ -17,7 +17,7 @@
 source ./tools/shell_functions.inc
 
 go version >/dev/null 2>&1 || fail "Go is not installed or is not in \$PATH. See https://vitess.io/contributing/build-from-source for install instructions."
-goversion_min 1.13 || fail "Go version reported: ${go version}. Version 1.13+ required. See https://vitess.io/contributing/build-from-source for install instructions."
+goversion_min 1.13 || fail "Go version reported: `go version`. Version 1.13+ required. See https://vitess.io/contributing/build-from-source for install instructions."
 
 mkdir -p dist
 mkdir -p bin

docker/bootstrap/Dockerfile.common

@@ -14,12 +14,6 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins
     libtool \
     make \
     openjdk-8-jdk \
-    python-crypto \
-    python-dev \
-    python-mysqldb \
-    python-pip \
-    ruby \
-    ruby-dev \
     software-properties-common \
     virtualenv \
     unzip \
@@ -46,9 +40,7 @@ ENV USER vitess
 # Copy files needed for bootstrap
 COPY bootstrap.sh dev.env build.env go.mod go.sum /vt/src/vitess.io/vitess/
 COPY config /vt/src/vitess.io/vitess/config
-COPY third_party /vt/src/vitess.io/vitess/third_party
 COPY tools /vt/src/vitess.io/vitess/tools
-COPY travis /vt/src/vitess.io/vitess/travis
 
 # Create vitess user
 RUN groupadd -r vitess && useradd -r -g vitess vitess && \