fix: retry starting zookeeper if it failed on onebox environment (#453)

run.sh

@@ -334,7 +334,7 @@ function run_test()
 {
     local test_modules=""
     local clear_flags="1"
-    local on_traivs=""
+    local on_travis=""
     while [[ $# > 0 ]]; do
         key="$1"
         case $key in
@@ -370,7 +370,10 @@ function run_test()
     start_time=`date +%s`
 
     ./run.sh clear_onebox #clear the onebox before test
-    ./run.sh start_onebox -w
+    if ! ./run.sh start_onebox -w; then
+        echo "ERROR: unable to continue on testing because starting onebox failed"
+        exit 1
+    fi
 
     for module in `echo $test_modules`; do
         pushd $ROOT/src/builder/bin/$module
@@ -619,9 +622,12 @@ function run_start_onebox()
         echo "ERROR: some onebox processes are running, start failed"
         exit 1
     fi
-    ln -s -f ${SERVER_PATH}/pegasus_server
+    ln -s -f "${SERVER_PATH}/pegasus_server" "${ROOT}"
 
-    run_start_zk
+    if ! run_start_zk; then
+        echo "ERROR: unable to setup onebox because zookeeper can not be started"
+        exit 1
+    fi
 
     if [ $USE_PRODUCT_CONFIG == "true" ]; then
         [ -z "${CONFIG_FILE}" ] && CONFIG_FILE=${ROOT}/src/server/config.ini

scripts/start_zk.sh

@@ -16,53 +16,52 @@ then
     exit 1
 fi
 
-mkdir -p $INSTALL_DIR
-if [ $? -ne 0 ]
+if ! mkdir -p "$INSTALL_DIR";
 then
-    echo "ERROR: mkdir $PREFIX failed"
+    echo "ERROR: mkdir $INSTALL_DIR failed"
     exit 1
 fi
 
-cd $INSTALL_DIR
+cd "$INSTALL_DIR" || exit
 
 if [ ! -f zookeeper-3.4.6.tar.gz ]; then
     echo "Downloading zookeeper..."
-    download_url="http://git.n.xiaomi.com/pegasus/packages/raw/master/zookeeper-3.4.6.tar.gz"
-    wget -T 5 -t 1 $download_url
-    if [ $? -ne 0 ]; then
-        download_url="https://github.com/XiaoMi/pegasus-common/releases/download/deps/zookeeper-3.4.6.tar.gz"
-        wget -T 5 -t 1 $download_url
-        if [ $? -ne 0 ]; then
-            echo "ERROR: download zookeeper failed"
-            exit 1
-        fi
+    download_url="https://github.com/XiaoMi/pegasus-common/releases/download/deps/zookeeper-3.4.6.tar.gz"
+    if ! wget -T 5 -t 1 $download_url; then
+        echo "ERROR: download zookeeper failed"
+        exit 1
     fi
 fi
 
 if [ ! -d zookeeper-3.4.6 ]; then
     echo "Decompressing zookeeper..."
-    tar xf zookeeper-3.4.6.tar.gz
-    if [ $? -ne 0 ]; then
+    if ! tar xf zookeeper-3.4.6.tar.gz; then
         echo "ERROR: decompress zookeeper failed"
         exit 1
     fi
 fi
 
-ZOOKEEPER_HOME=`pwd`/zookeeper-3.4.6
+ZOOKEEPER_HOME=$(pwd)/zookeeper-3.4.6
 ZOOKEEPER_PORT=$PORT
 
-cp $ZOOKEEPER_HOME/conf/zoo_sample.cfg $ZOOKEEPER_HOME/conf/zoo.cfg
-sed -i "s@dataDir=/tmp/zookeeper@dataDir=$ZOOKEEPER_HOME/data@" $ZOOKEEPER_HOME/conf/zoo.cfg
-sed -i "s@clientPort=2181@clientPort=$ZOOKEEPER_PORT@" $ZOOKEEPER_HOME/conf/zoo.cfg
+cp "$ZOOKEEPER_HOME"/conf/zoo_sample.cfg "$ZOOKEEPER_HOME"/conf/zoo.cfg
+sed -i "s@dataDir=/tmp/zookeeper@dataDir=$ZOOKEEPER_HOME/data@" "$ZOOKEEPER_HOME"/conf/zoo.cfg
+sed -i "s@clientPort=2181@clientPort=$ZOOKEEPER_PORT@" "$ZOOKEEPER_HOME"/conf/zoo.cfg
 
-mkdir -p $ZOOKEEPER_HOME/data
-$ZOOKEEPER_HOME/bin/zkServer.sh start
-sleep 1
+mkdir -p "$ZOOKEEPER_HOME"/data
+"$ZOOKEEPER_HOME"/bin/zkServer.sh start
 
-if echo ruok | nc localhost $ZOOKEEPER_PORT | grep -q imok; then
-    echo "Zookeeper started at port $ZOOKEEPER_PORT"
-    exit 0
-else
-    echo "ERROR: start zookeeper failed"
-    exit 1
-fi
+zk_check_count=0
+while true; do
+    sleep 1 # wait until zookeeper bootstrapped
+    if echo ruok | nc localhost "$ZOOKEEPER_PORT" | grep -q imok; then
+        echo "Zookeeper started at port $ZOOKEEPER_PORT"
+        exit 0
+    fi
+    zk_check_count=$((zk_check_count+1))
+    echo "ERROR: starting zookeeper has failed ${zk_check_count} times"
+    if [ $zk_check_count -gt 30 ]; then
+        echo "ERROR: failed to start zookeeper in 30 seconds"
+        exit 1
+    fi
+done