[Enhancement] Upgrade staros v3.3 rc0 (backport #43887) (#43969)

Co-authored-by: Kevin Cai <[email protected]>

conf/asan_suppressions.conf

@@ -12,9 +12,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 leak:brpc
-# Aws cpp sdk, thread local static variables
-leak:s2n_defend_if_forked
-# Aws cpp sdk, thread local static variables, the above function is inlined and is optimized out
-leak:s2n_drbg_instantiate
 leak:libjvm.so
 leak:libzip.so

docker/dockerfiles/dev-env/dev-env.Dockerfile

@@ -19,10 +19,10 @@ ARG prebuild_maven=true
 # value: true | false
 # default: false
 ARG predownload_thirdparty=false
-ARG thirdparty_url=https://cdn-thirdparty.starrocks.com/starrocks-thirdparty-main-20230720.tar
+ARG thirdparty_url=https://cdn-thirdparty.starrocks.com/starrocks-thirdparty-main-20240411.tar
 ARG commit_id
 # check thirdparty/starlet-artifacts-version.sh, to get the right tag
-ARG starlet_tag=v3.2-rc9
+ARG starlet_tag=v3.3-rc0
 # build for which linux distro: centos7|ubuntu
 ARG distro=ubuntu
 # Token to access artifacts in private github repositories.

fe/fe-core/pom.xml

@@ -41,7 +41,7 @@ under the License.
         <iceberg.version>1.5.0</iceberg.version>
         <awsv2.version>2.17.257</awsv2.version>
         <paimon.version>0.7.0-incubating</paimon.version>
-        <staros.version>3.2-rc9</staros.version>
+        <staros.version>3.3-rc0</staros.version>
         <python>python</python>
     </properties>
 

run-be-ut.sh

@@ -224,6 +224,9 @@ export LD_LIBRARY_PATH=$STARROCKS_HOME/lib/hadoop/native:$LD_LIBRARY_PATH
 THIRDPARTY_HADOOP_HOME=${STARROCKS_THIRDPARTY}/installed/hadoop/share/hadoop
 if [[ -d ${THIRDPARTY_HADOOP_HOME} ]] ; then
     export HADOOP_CLASSPATH=${THIRDPARTY_HADOOP_HOME}/common/*:${THIRDPARTY_HADOOP_HOME}/common/lib/*:${THIRDPARTY_HADOOP_HOME}/hdfs/*:${THIRDPARTY_HADOOP_HOME}/hdfs/lib/*
+    # get rid of StackOverflowError on the process reaper thread, which has a small stack size.
+    # https://bugs.openjdk.org/browse/JDK-8153057
+    export LIBHDFS_OPTS="$LIBHDFS_OPTS -Djdk.lang.processReaperUseDefaultStackSize=true"
 else
     # exclude HdfsFileSystemTest related test case if no hadoop env found
     echo "[INFO] Can't find available HADOOP common lib, disable HdfsFileSystemTest related test!"

thirdparty/download-thirdparty.sh

@@ -393,35 +393,13 @@ else
 fi
 
 cd $TP_SOURCE_DIR/$AWS_SDK_CPP_SOURCE
-if [ ! -f $PATCHED_MARK ] && [ $AWS_SDK_CPP_SOURCE = "aws-sdk-cpp-1.9.179" ]; then
+if [ $AWS_SDK_CPP_SOURCE = "aws-sdk-cpp-1.11.267" ]; then
     if [ ! -f prefetch_crt_dep_ok ]; then
         bash ./prefetch_crt_dependency.sh
         touch prefetch_crt_dep_ok
     fi
-    patch -p0 < $TP_PATCH_DIR/aws-sdk-cpp-1.9.179.patch    
-    # Fix crt BB, refer to https://github.com/aws/s2n-tls/issues/3166
-    patch -p1 -f -i $TP_PATCH_DIR/aws-sdk-cpp-patch-1.9.179-s2n-compile-error.patch
-    # refer to https://github.com/aws/aws-sdk-cpp/issues/1824
-    patch -p1 < $TP_PATCH_DIR/aws-sdk-cpp-patch-1.9.179-LINK_LIBRARIES_ALL.patch
-    touch $PATCHED_MARK
-    echo "Finished patching $AWS_SDK_CPP_SOURCE"
-else
-    echo "$AWS_SDK_CPP_SOURCE not patched"
 fi
 
-cd $TP_SOURCE_DIR/$AWS_SDK_CPP_SOURCE
-if [ ! -f $PATCHED_MARK ] && [ $AWS_SDK_CPP_SOURCE = "aws-sdk-cpp-1.10.36" ]; then
-    if [ ! -f prefetch_crt_dep_ok ]; then
-        bash ./prefetch_crt_dependency.sh
-        touch prefetch_crt_dep_ok
-    fi
-    # Fix InstanceProfile deadlock, refer to https://github.com/aws/aws-sdk-cpp/issues/2251
-    patch -p1 < $TP_PATCH_DIR/aws-sdk-cpp-1.10.36-instance-profile-deadlock.patch   
-    touch $PATCHED_MARK
-    echo "Finished patching $AWS_SDK_CPP_SOURCE"
-else
-    echo "$AWS_SDK_CPP_SOURCE not patched"
-fi
 
 # patch jemalloc_hook
 cd $TP_SOURCE_DIR/$JEMALLOC_SOURCE

thirdparty/starlet-artifacts-version.sh

@@ -9,4 +9,4 @@
 #   https://hub.docker.com/r/starrocks/starlet-artifacts-centos7/tags
 #
 # Update the following tag when STARLET releases a new version.
-export STARLET_ARTIFACTS_TAG=v3.2-rc9
+export STARLET_ARTIFACTS_TAG=v3.3-rc0

thirdparty/vars-aarch64.sh

@@ -52,7 +52,7 @@ JINDOSDK_SOURCE="jindosdk-4.6.8-linux-el7-aarch64"
 JINDOSDK_MD5SUM="27a4e2cd9a403c6e21079a866287d88b"
 
 # starcache
-STARCACHE_DOWNLOAD="https://cdn-thirdparty.starrocks.com/starcache/v3.2.3/starcache-centos7_arm64.tar.gz"
+STARCACHE_DOWNLOAD="https://cdn-thirdparty.starrocks.com/starcache/v3.3.0/starcache-centos7_arm64.tar.gz"
 STARCACHE_NAME="starcache.tar.gz"
 STARCACHE_SOURCE="starcache"
-STARCACHE_MD5SUM="d6a98b2945be5ed758a3690cb20ba422"
+STARCACHE_MD5SUM="adbf4b31877d57231f31cb07e68091cd"

thirdparty/vars-ubuntu22-aarch64.sh

@@ -22,7 +22,7 @@
 #####################################################
 
 # starcache
-STARCACHE_DOWNLOAD="https://cdn-thirdparty.starrocks.com/starcache/v3.2.3/starcache-ubuntu22_arm64.tar.gz"
+STARCACHE_DOWNLOAD="https://cdn-thirdparty.starrocks.com/starcache/v3.3.0/starcache-ubuntu22_arm64.tar.gz"
 STARCACHE_NAME="starcache.tar.gz"
 STARCACHE_SOURCE="starcache"
-STARCACHE_MD5SUM="350ed10c5c2c119718e49479e41b86a2"
+STARCACHE_MD5SUM="2dcdea4cfb164d173ac57925f0a0678e"

thirdparty/vars-ubuntu22-x86_64.sh

@@ -28,7 +28,7 @@ JINDOSDK_SOURCE="jindosdk-4.6.8-linux-ubuntu22-x86_64"
 JINDOSDK_MD5SUM="52236053391091591c2d09684791e810"
 
 # starcache
-STARCACHE_DOWNLOAD="https://cdn-thirdparty.starrocks.com/starcache/v3.2.3/starcache-ubuntu22_amd64.tar.gz"
+STARCACHE_DOWNLOAD="https://cdn-thirdparty.starrocks.com/starcache/v3.3.0/starcache-ubuntu22_amd64.tar.gz"
 STARCACHE_NAME="starcache.tar.gz"
 STARCACHE_SOURCE="starcache"
-STARCACHE_MD5SUM="e881d9a0896c82044cb98472fbb4a203"
+STARCACHE_MD5SUM="1e33c75c3e374c10cda7391c60f7d69d"

thirdparty/vars-x86_64.sh

@@ -52,7 +52,7 @@ JINDOSDK_SOURCE="jindosdk-4.6.8-linux"
 JINDOSDK_MD5SUM="5436e4fe39c4dfdc942e41821f1dd8a9"
 
 # starcache
-STARCACHE_DOWNLOAD="https://cdn-thirdparty.starrocks.com/starcache/v3.2.3/starcache-centos7_amd64.tar.gz"
+STARCACHE_DOWNLOAD="https://cdn-thirdparty.starrocks.com/starcache/v3.3.0/starcache-centos7_amd64.tar.gz"
 STARCACHE_NAME="starcache.tar.gz"
 STARCACHE_SOURCE="starcache"
-STARCACHE_MD5SUM="335d09cffef0160cda5bb1b7ca9160df"
+STARCACHE_MD5SUM="d2176508003f4462f896fc186f44342d"

thirdparty/vars.sh

@@ -320,10 +320,10 @@ GCS_CONNECTOR_SOURCE="gcs-connector-hadoop3-2.2.11-shaded"
 GCS_CONNECTOR_MD5SUM="51fd0eb5cb913a84e4ad8a5ed2069e21"
 
 # aws-sdk-cpp
-AWS_SDK_CPP_DOWNLOAD="https://github.com/aws/aws-sdk-cpp/archive/refs/tags/1.10.36.tar.gz"
-AWS_SDK_CPP_NAME="aws-sdk-cpp-1.10.36.tar.gz"
-AWS_SDK_CPP_SOURCE="aws-sdk-cpp-1.10.36"
-AWS_SDK_CPP_MD5SUM="8fed635c5ac98b448bc1a98cf7c97c70"
+AWS_SDK_CPP_DOWNLOAD="https://github.com/aws/aws-sdk-cpp/archive/refs/tags/1.11.267.tar.gz"
+AWS_SDK_CPP_NAME="aws-sdk-cpp-1.11.267.tar.gz"
+AWS_SDK_CPP_SOURCE="aws-sdk-cpp-1.11.267"
+AWS_SDK_CPP_MD5SUM="fdf43e7262f9d08968eb34f9ad18b8e7"
 
 # velocypack: A fast and compact format for serialization and storage
 VPACK_DOWNLOAD="https://github.com/arangodb/velocypack/archive/refs/tags/XYZ1.0.tar.gz"