From 53d24401cf4423e03a3e699015a77260fc1bd0f9 Mon Sep 17 00:00:00 2001
From: Kang <zhoukangcn@gmail.com>
Date: Tue, 9 Apr 2024 20:58:07 +0800
Subject: [PATCH] [#2811] feat(docker): update start up srcipts for Doris
 Container (#2812)

### What changes were proposed in this pull request?

There's a very small chance of GitHub CI failure. to avoid this

- add check for BE status
- add retry for adding BE

### Why are the changes needed?

Fix: #2811

### Does this PR introduce _any_ user-facing change?

N/A

### How was this patch tested?

CI

Co-authored-by: Qi Yu <yuqi@datastrato.com>
---
 dev/docker/doris/start.sh    | 60 +++++++++++++++++++++++++++++++-----
 docs/docker-image-details.md |  3 ++
 2 files changed, 56 insertions(+), 7 deletions(-)

diff --git a/dev/docker/doris/start.sh b/dev/docker/doris/start.sh
index dec026d866f..b0b84a826bd 100644
--- a/dev/docker/doris/start.sh
+++ b/dev/docker/doris/start.sh
@@ -18,7 +18,7 @@ df -h
 THRESHOLD=6
 
 DISK_FREE=`df -BG | grep '/$' | tr -s ' ' | cut -d ' ' -f4 | grep -o '[0-9]*'`
-if [ "$DISK_FREE" -lt "$THRESHOLD" ]
+if [ "$DISK_FREE" -le "$THRESHOLD" ]
 then
     echo "ERROR: Doris FE (version < 2.1.0) can not start with less than ${THRESHOLD}G disk space."
     exit 1
@@ -38,7 +38,7 @@ echo "add priority_networks = ${PRIORITY_NETWORKS} to fe.conf & be.conf"
 echo "priority_networks = ${PRIORITY_NETWORKS}" >> ${DORIS_FE_HOME}/conf/fe.conf
 echo "priority_networks = ${PRIORITY_NETWORKS}" >> ${DORIS_BE_HOME}/conf/be.conf
 
-# start doris fe and be
+# start doris fe and be in daemon mode
 ${DORIS_FE_HOME}/bin/start_fe.sh --daemon
 ${DORIS_BE_HOME}/bin/start_be.sh --daemon
 
@@ -48,13 +48,16 @@ for i in {1..10}; do
   sleep 5
   # check http://localhost:8030/api/bootstrap return contains content '"msg":"success"' to see if the fe is ready
   url="http://localhost:8030/api/bootstrap"
+
+  echo "check fe for the $i times"
   response=$(curl --silent --request GET $url)
 
   if echo "$response" | grep -q "\"msg\":\"success\""; then
     fe_started=true
+    echo "Doris fe started successfully, response: $response"
     break
   else
-    echo "waiting for the fe to start"
+    echo "check failed, response: $response"
   fi
 done
 
@@ -65,13 +68,56 @@ if [ "$fe_started" = false ]; then
   exit 1
 fi
 
+# check for be started
+be_started=false
+for i in {1..10}; do
+  sleep 5
+  # check localhost:8040/api/health return contains content '"status": "OK"' to see if the be is ready
+  url="localhost:8040/api/health"
+
+  echo "check be for the $i times"
+  response=$(curl --silent --request GET $url)
+
+  if echo "$response" | grep -q "\"status\": \"OK\""; then
+    be_started=true
+    echo "Doris be started successfully, response: $response"
+    break
+  else
+    echo "check failed, response: $response"
+  fi
+done
+
+if [ "$be_started" = false ]; then
+  echo "Doris be failed to start"
+
+  cat ${DORIS_BE_HOME}/log/*
+  exit 1
+fi
+
+
 # add the be to the fe
-mysql -h127.0.0.1 -P9030 -uroot -e "ALTER SYSTEM ADD BACKEND '${CONTAINER_IP}:9050'"
+be_added=false
+for i in {1..10}; do
+  echo "add Doris BE to FE for the $i times"
+
+  mysql -h127.0.0.1 -P9030 -uroot -e "ALTER SYSTEM ADD BACKEND '${CONTAINER_IP}:9050'"
+  if [ $? -ne 0 ]; then
+    echo "Failed to add the BE to the FE"
+  else
+    be_added=true
+    echo "Doris BE added to FE successfully"
+    break
+  fi
+
+  sleep 1
+done
+
+if [ "$be_added" = false ]; then
+  echo "Doris BE failed to add to FE"
+  cat ${DORIS_FE_HOME}/log/fe.* ${DORIS_BE_HOME}/log/*
 
-if [ $? -ne 0 ]; then
-  echo "Failed to add the BE to the FE"
   exit 1
 fi
 
 # persist the container
-tail -f ${DORIS_FE_HOME}/log/fe.log ${DORIS_BE_HOME}/log/be.log
+tail -f ${DORIS_FE_HOME}/log/fe.log ${DORIS_BE_HOME}/log/be.INFO
diff --git a/docs/docker-image-details.md b/docs/docker-image-details.md
index 52ea9b7858f..a72a43e7ed3 100644
--- a/docs/docker-image-details.md
+++ b/docs/docker-image-details.md
@@ -166,6 +166,9 @@ You can use this image to test Apache Doris.
 
 Changelog
 
+- gravitino-ci-doris:0.1.2
+    - Add a check for the status of Doris BE, add retry for adding BE nodes.
+
 - gravitino-ci-doris:0.1.1
     - Optimize `start.sh`, add disk space check before starting Doris, exit when FE or BE start failed, add log to stdout