diff --git a/conf/asan_suppressions.conf b/conf/asan_suppressions.conf index 1d9832991754d2..7b8a3340af66ae 100644 --- a/conf/asan_suppressions.conf +++ b/conf/asan_suppressions.conf @@ -12,9 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. leak:brpc -# Aws cpp sdk, thread local static variables -leak:s2n_defend_if_forked -# Aws cpp sdk, thread local static variables, the above function is inlined and is optimized out -leak:s2n_drbg_instantiate leak:libjvm.so leak:libzip.so diff --git a/docker/dockerfiles/dev-env/dev-env.Dockerfile b/docker/dockerfiles/dev-env/dev-env.Dockerfile index f3be607621a323..0a9bcc2220d4ea 100644 --- a/docker/dockerfiles/dev-env/dev-env.Dockerfile +++ b/docker/dockerfiles/dev-env/dev-env.Dockerfile @@ -19,10 +19,10 @@ ARG prebuild_maven=true # value: true | false # default: false ARG predownload_thirdparty=false -ARG thirdparty_url=https://cdn-thirdparty.starrocks.com/starrocks-thirdparty-main-20230720.tar +ARG thirdparty_url=https://cdn-thirdparty.starrocks.com/starrocks-thirdparty-main-20240411.tar ARG commit_id # check thirdparty/starlet-artifacts-version.sh, to get the right tag -ARG starlet_tag=v3.2-rc9 +ARG starlet_tag=v3.3-rc0 # build for which linux distro: centos7|ubuntu ARG distro=ubuntu # Token to access artifacts in private github repositories. diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index 5413e1dfcba67e..344077b673f8fb 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -41,7 +41,7 @@ under the License. 1.5.0 2.17.257 0.7.0-incubating - 3.2-rc9 + 3.3-rc0 python diff --git a/run-be-ut.sh b/run-be-ut.sh index e39944a469d69e..43953fcf56ab3a 100755 --- a/run-be-ut.sh +++ b/run-be-ut.sh @@ -224,6 +224,9 @@ export LD_LIBRARY_PATH=$STARROCKS_HOME/lib/hadoop/native:$LD_LIBRARY_PATH THIRDPARTY_HADOOP_HOME=${STARROCKS_THIRDPARTY}/installed/hadoop/share/hadoop if [[ -d ${THIRDPARTY_HADOOP_HOME} ]] ; then export HADOOP_CLASSPATH=${THIRDPARTY_HADOOP_HOME}/common/*:${THIRDPARTY_HADOOP_HOME}/common/lib/*:${THIRDPARTY_HADOOP_HOME}/hdfs/*:${THIRDPARTY_HADOOP_HOME}/hdfs/lib/* + # get rid of StackOverflowError on the process reaper thread, which has a small stack size. + # https://bugs.openjdk.org/browse/JDK-8153057 + export LIBHDFS_OPTS="$LIBHDFS_OPTS -Djdk.lang.processReaperUseDefaultStackSize=true" else # exclude HdfsFileSystemTest related test case if no hadoop env found echo "[INFO] Can't find available HADOOP common lib, disable HdfsFileSystemTest related test!" diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index 429a75fffdaace..602ddcd25b2023 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -393,35 +393,13 @@ else fi cd $TP_SOURCE_DIR/$AWS_SDK_CPP_SOURCE -if [ ! -f $PATCHED_MARK ] && [ $AWS_SDK_CPP_SOURCE = "aws-sdk-cpp-1.9.179" ]; then +if [ $AWS_SDK_CPP_SOURCE = "aws-sdk-cpp-1.11.267" ]; then if [ ! -f prefetch_crt_dep_ok ]; then bash ./prefetch_crt_dependency.sh touch prefetch_crt_dep_ok fi - patch -p0 < $TP_PATCH_DIR/aws-sdk-cpp-1.9.179.patch - # Fix crt BB, refer to https://github.com/aws/s2n-tls/issues/3166 - patch -p1 -f -i $TP_PATCH_DIR/aws-sdk-cpp-patch-1.9.179-s2n-compile-error.patch - # refer to https://github.com/aws/aws-sdk-cpp/issues/1824 - patch -p1 < $TP_PATCH_DIR/aws-sdk-cpp-patch-1.9.179-LINK_LIBRARIES_ALL.patch - touch $PATCHED_MARK - echo "Finished patching $AWS_SDK_CPP_SOURCE" -else - echo "$AWS_SDK_CPP_SOURCE not patched" fi -cd $TP_SOURCE_DIR/$AWS_SDK_CPP_SOURCE -if [ ! -f $PATCHED_MARK ] && [ $AWS_SDK_CPP_SOURCE = "aws-sdk-cpp-1.10.36" ]; then - if [ ! -f prefetch_crt_dep_ok ]; then - bash ./prefetch_crt_dependency.sh - touch prefetch_crt_dep_ok - fi - # Fix InstanceProfile deadlock, refer to https://github.com/aws/aws-sdk-cpp/issues/2251 - patch -p1 < $TP_PATCH_DIR/aws-sdk-cpp-1.10.36-instance-profile-deadlock.patch - touch $PATCHED_MARK - echo "Finished patching $AWS_SDK_CPP_SOURCE" -else - echo "$AWS_SDK_CPP_SOURCE not patched" -fi # patch jemalloc_hook cd $TP_SOURCE_DIR/$JEMALLOC_SOURCE diff --git a/thirdparty/patches/aws-sdk-cpp-1.10.36-instance-profile-deadlock.patch b/thirdparty/patches/aws-sdk-cpp-1.10.36-instance-profile-deadlock.patch deleted file mode 100644 index fe1efd926dce7f..00000000000000 --- a/thirdparty/patches/aws-sdk-cpp-1.10.36-instance-profile-deadlock.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff --git a/aws-cpp-sdk-core/source/auth/AWSCredentialsProvider.cpp b/aws-cpp-sdk-core/source/auth/AWSCredentialsProvider.cpp -index 084e4bca6e..6c8b059b2d 100644 ---- a/aws-cpp-sdk-core/source/auth/AWSCredentialsProvider.cpp -+++ b/aws-cpp-sdk-core/source/auth/AWSCredentialsProvider.cpp -@@ -259,7 +259,6 @@ AWSCredentials InstanceProfileCredentialsProvider::GetAWSCredentials() - - bool InstanceProfileCredentialsProvider::ExpiresSoon() const - { -- ReaderLockGuard guard(m_reloadLock); - auto profileIter = m_ec2MetadataConfigLoader->GetProfiles().find(Aws::Config::INSTANCE_PROFILE_KEY); - AWSCredentials credentials; - \ No newline at end of file diff --git a/thirdparty/patches/aws-sdk-cpp-1.9.179.patch b/thirdparty/patches/aws-sdk-cpp-1.9.179.patch deleted file mode 100644 index 6c0f4a50c0e07f..00000000000000 --- a/thirdparty/patches/aws-sdk-cpp-1.9.179.patch +++ /dev/null @@ -1,10 +0,0 @@ ---- crt/aws-crt-cpp/crt/s2n/CMakeLists.txt.orig 2022-01-22 12:00:14.626738952 +0800 -+++ crt/aws-crt-cpp/crt/s2n/CMakeLists.txt 2022-01-22 12:00:24.564666399 +0800 -@@ -182,6 +182,7 @@ - "${CMAKE_CURRENT_LIST_DIR}/tests/features/noop_main.c" - COMPILE_DEFINITIONS ${BIKE_R3_VPCLMUL_FLAGS}) - -+ set(BIKE_R3_VPCLMUL_SUPPORTED OFF) - if(BIKE_R3_AVX2_SUPPORTED OR BIKE_R3_AVX512_SUPPORTED OR BIKE_R3_PCLMUL_SUPPORTED OR BIKE_R3_VPCLMUL_SUPPORTED) - set(BIKE_R3_X86_64_OPT_SUPPORTED ON) - endif() diff --git a/thirdparty/patches/aws-sdk-cpp-patch-1.9.179-LINK_LIBRARIES_ALL.patch b/thirdparty/patches/aws-sdk-cpp-patch-1.9.179-LINK_LIBRARIES_ALL.patch deleted file mode 100644 index 16be3c35705d84..00000000000000 --- a/thirdparty/patches/aws-sdk-cpp-patch-1.9.179-LINK_LIBRARIES_ALL.patch +++ /dev/null @@ -1,22 +0,0 @@ -diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake -index 30c665341f..ac8a15a2be 100644 ---- a/cmake/dependencies.cmake -+++ b/cmake/dependencies.cmake -@@ -30,6 +30,17 @@ function(compute_links lib) - # For each direct link append it and its links - foreach(link ${links}) - if(TARGET ${link}) # Collect only target links -+ get_target_property(link_type ${link} TYPE) -+ if(link_type STREQUAL "INTERFACE_LIBRARY") -+ # A custom property LINK_LIBRARIES_ALL can't be set to INTERFACE libraries. -+ # We have to allow Threads::Threads, which might be an interface library. -+ if(link STREQUAL "Threads::Threads") -+ continue() -+ elseif() -+ message(STATUS "Library '${lib}' uses link '${link}'.") -+ message(FATAL_ERROR "Algorithm doesn't work with interface libraries.") -+ endif() -+ endif() - compute_links(${link}) - get_target_property(link_links_all ${link} LINK_LIBRARIES_ALL) - set_property(TARGET ${lib} APPEND PROPERTY diff --git a/thirdparty/patches/aws-sdk-cpp-patch-1.9.179-s2n-compile-error.patch b/thirdparty/patches/aws-sdk-cpp-patch-1.9.179-s2n-compile-error.patch deleted file mode 100644 index 7ee4ec9f567fac..00000000000000 --- a/thirdparty/patches/aws-sdk-cpp-patch-1.9.179-s2n-compile-error.patch +++ /dev/null @@ -1,109 +0,0 @@ -diff --git a/crt/aws-crt-cpp/crt/s2n/CMakeLists.txt b/crt/aws-crt-cpp/crt/s2n/CMakeLists.txt -index 705f840..20d2cf1 100644 ---- a/crt/aws-crt-cpp/crt/s2n/CMakeLists.txt -+++ b/crt/aws-crt-cpp/crt/s2n/CMakeLists.txt -@@ -162,28 +162,49 @@ else() - # First, check if the compiler supports the specific instruction set - # extensions. For example, gcc-4 doesn't fully support AVX-512, while - # gcc-7 doesn't support VPCLMUL extension. -- try_compile(BIKE_R3_AVX2_SUPPORTED -- ${CMAKE_BINARY_DIR} -- "${CMAKE_CURRENT_LIST_DIR}/tests/features/noop_main.c" -- COMPILE_DEFINITIONS "-mavx2") -+ set(BIKE_R3_AVX2_FLAGS "-mavx2") -+ try_compile( -+ BIKE_R3_AVX2_SUPPORTED -+ ${CMAKE_BINARY_DIR} -+ SOURCES -+ "${CMAKE_CURRENT_LIST_DIR}/tests/features/noop_main.c" -+ "${CMAKE_CURRENT_LIST_DIR}/pq-crypto/bike_r3/gf2x_mul_avx2.c" -+ CMAKE_FLAGS "-DINCLUDE_DIRECTORIES:STRING=${CMAKE_SOURCE_DIR}" -+ COMPILE_DEFINITIONS "${BIKE_R3_AVX2_FLAGS} -DS2N_BIKE_R3_AVX2" -+ ) - - set(BIKE_R3_AVX512_FLAGS "-mavx512f -mavx512bw -mavx512dq") -- try_compile(BIKE_R3_AVX512_SUPPORTED -- ${CMAKE_BINARY_DIR} -- "${CMAKE_CURRENT_LIST_DIR}/tests/features/noop_main.c" -- COMPILE_DEFINITIONS ${BIKE_R3_AVX512_FLAGS}) -+ try_compile( -+ BIKE_R3_AVX512_SUPPORTED -+ ${CMAKE_BINARY_DIR} -+ SOURCES -+ "${CMAKE_CURRENT_LIST_DIR}/tests/features/noop_main.c" -+ "${CMAKE_CURRENT_LIST_DIR}/pq-crypto/bike_r3/gf2x_mul_avx512.c" -+ CMAKE_FLAGS "-DINCLUDE_DIRECTORIES:STRING=${CMAKE_SOURCE_DIR}" -+ COMPILE_DEFINITIONS "${BIKE_R3_AVX512_FLAGS} -DS2N_BIKE_R3_AVX512" -+ ) - - set(BIKE_R3_PCLMUL_FLAGS "-mpclmul -msse2") -- try_compile(BIKE_R3_PCLMUL_SUPPORTED -- ${CMAKE_BINARY_DIR} -- "${CMAKE_CURRENT_LIST_DIR}/tests/features/noop_main.c" -- COMPILE_DEFINITIONS ${BIKE_R3_PCLMUL_FLAGS}) -+ try_compile( -+ BIKE_R3_PCLMUL_SUPPORTED -+ ${CMAKE_BINARY_DIR} -+ SOURCES -+ "${CMAKE_CURRENT_LIST_DIR}/tests/features/noop_main.c" -+ "${CMAKE_CURRENT_LIST_DIR}/pq-crypto/bike_r3/gf2x_mul_base_pclmul.c" -+ CMAKE_FLAGS "-DINCLUDE_DIRECTORIES:STRING=${CMAKE_SOURCE_DIR}" -+ COMPILE_DEFINITIONS "${BIKE_R3_PCLMUL_FLAGS} -DS2N_BIKE_R3_PCLMUL" -+ ) - - set(BIKE_R3_VPCLMUL_FLAGS "-mvpclmulqdq -mavx512f -mavx512bw -mavx512dq") -- try_compile(BIKE_R3_VPCLMUL_SUPPORTED -- ${CMAKE_BINARY_DIR} -- "${CMAKE_CURRENT_LIST_DIR}/tests/features/noop_main.c" -- COMPILE_DEFINITIONS ${BIKE_R3_VPCLMUL_FLAGS}) -+ try_compile( -+ BIKE_R3_VPCLMUL_SUPPORTED -+ ${CMAKE_BINARY_DIR} -+ SOURCES -+ "${CMAKE_CURRENT_LIST_DIR}/tests/features/noop_main.c" -+ "${CMAKE_CURRENT_LIST_DIR}/pq-crypto/bike_r3/gf2x_mul_base_vpclmul.c" -+ CMAKE_FLAGS "-DINCLUDE_DIRECTORIES:STRING=${CMAKE_SOURCE_DIR}" -+ COMPILE_DEFINITIONS "${BIKE_R3_VPCLMUL_FLAGS} -DS2N_BIKE_R3_VPCLMUL" -+ ) - - if(BIKE_R3_AVX2_SUPPORTED OR BIKE_R3_AVX512_SUPPORTED OR BIKE_R3_PCLMUL_SUPPORTED OR BIKE_R3_VPCLMUL_SUPPORTED) - set(BIKE_R3_X86_64_OPT_SUPPORTED ON) -@@ -335,31 +356,36 @@ if(BIKE_R3_X86_64_OPT_SUPPORTED) - # If any of the BIKE_R3 x86_64 optimizations is supported (this was checked - # earlier in the file), we add the required compile flags to files that - # contain the optimized code. -+ -+ message(STATUS "Enabling BIKE_R3 x86_64 optimizations") -+ - if(BIKE_R3_AVX2_SUPPORTED) -+ message(STATUS "\t Enabling BIKE_R3_AVX2") - FILE(GLOB BIKE_R3_AVX2_SRCS "pq-crypto/bike_r3/*_avx2.c") - set_source_files_properties(${BIKE_R3_AVX2_SRCS} PROPERTIES COMPILE_FLAGS -mavx2) - target_compile_options(${PROJECT_NAME} PUBLIC -DS2N_BIKE_R3_AVX2) - endif() - - if(BIKE_R3_AVX512_SUPPORTED) -+ message(STATUS "\t Enabling BIKE_R3_AVX512") - FILE(GLOB BIKE_R3_AVX512_SRCS "pq-crypto/bike_r3/*_avx512.c") - set_source_files_properties(${BIKE_R3_AVX512_SRCS} PROPERTIES COMPILE_FLAGS ${BIKE_R3_AVX512_FLAGS}) - target_compile_options(${PROJECT_NAME} PUBLIC -DS2N_BIKE_R3_AVX512) - endif() - - if(BIKE_R3_PCLMUL_SUPPORTED) -+ message(STATUS "\t Enabling BIKE_R3_PCLMUL") - FILE(GLOB BIKE_R3_PCLMUL_SRCS "pq-crypto/bike_r3/*_pclmul.c") - set_source_files_properties(${BIKE_R3_PCLMUL_SRCS} PROPERTIES COMPILE_FLAGS ${BIKE_R3_PCLMUL_FLAGS}) - target_compile_options(${PROJECT_NAME} PUBLIC -DS2N_BIKE_R3_PCLMUL) - endif() - - if(BIKE_R3_VPCLMUL_SUPPORTED) -+ message(STATUS "\t Enabling BIKE_R3_VPCLMUL") - FILE(GLOB BIKE_R3_VPCLMUL_SRCS "pq-crypto/bike_r3/*_vpclmul.c") - set_source_files_properties(${BIKE_R3_VPCLMUL_SRCS} PROPERTIES COMPILE_FLAGS ${BIKE_R3_VPCLMUL_FLAGS}) - target_compile_options(${PROJECT_NAME} PUBLIC -DS2N_BIKE_R3_VPCLMUL) - endif() -- -- message(STATUS "Enabling BIKE_R3 x86_64 optimizations") - endif() - - if(KYBER512R3_AVX2_BMI2_OPT_SUPPORTED) diff --git a/thirdparty/starlet-artifacts-version.sh b/thirdparty/starlet-artifacts-version.sh index 3d30ef56c0ef55..fc5d5692628309 100644 --- a/thirdparty/starlet-artifacts-version.sh +++ b/thirdparty/starlet-artifacts-version.sh @@ -9,4 +9,4 @@ # https://hub.docker.com/r/starrocks/starlet-artifacts-centos7/tags # # Update the following tag when STARLET releases a new version. -export STARLET_ARTIFACTS_TAG=v3.2-rc9 +export STARLET_ARTIFACTS_TAG=v3.3-rc0 diff --git a/thirdparty/vars-aarch64.sh b/thirdparty/vars-aarch64.sh index ebd6914e7d9af8..bfa8ac0ce6a850 100644 --- a/thirdparty/vars-aarch64.sh +++ b/thirdparty/vars-aarch64.sh @@ -52,7 +52,7 @@ JINDOSDK_SOURCE="jindosdk-4.6.8-linux-el7-aarch64" JINDOSDK_MD5SUM="27a4e2cd9a403c6e21079a866287d88b" # starcache -STARCACHE_DOWNLOAD="https://cdn-thirdparty.starrocks.com/starcache/v3.2.3/starcache-centos7_arm64.tar.gz" +STARCACHE_DOWNLOAD="https://cdn-thirdparty.starrocks.com/starcache/v3.3.0/starcache-centos7_arm64.tar.gz" STARCACHE_NAME="starcache.tar.gz" STARCACHE_SOURCE="starcache" -STARCACHE_MD5SUM="d6a98b2945be5ed758a3690cb20ba422" +STARCACHE_MD5SUM="adbf4b31877d57231f31cb07e68091cd" diff --git a/thirdparty/vars-ubuntu22-aarch64.sh b/thirdparty/vars-ubuntu22-aarch64.sh index db28f27111c618..2014f5fe9c50f8 100644 --- a/thirdparty/vars-ubuntu22-aarch64.sh +++ b/thirdparty/vars-ubuntu22-aarch64.sh @@ -22,7 +22,7 @@ ##################################################### # starcache -STARCACHE_DOWNLOAD="https://cdn-thirdparty.starrocks.com/starcache/v3.2.3/starcache-ubuntu22_arm64.tar.gz" +STARCACHE_DOWNLOAD="https://cdn-thirdparty.starrocks.com/starcache/v3.3.0/starcache-ubuntu22_arm64.tar.gz" STARCACHE_NAME="starcache.tar.gz" STARCACHE_SOURCE="starcache" -STARCACHE_MD5SUM="350ed10c5c2c119718e49479e41b86a2" +STARCACHE_MD5SUM="2dcdea4cfb164d173ac57925f0a0678e" diff --git a/thirdparty/vars-ubuntu22-x86_64.sh b/thirdparty/vars-ubuntu22-x86_64.sh index d6fef4809a324c..4c8327a0d4420a 100644 --- a/thirdparty/vars-ubuntu22-x86_64.sh +++ b/thirdparty/vars-ubuntu22-x86_64.sh @@ -28,7 +28,7 @@ JINDOSDK_SOURCE="jindosdk-4.6.8-linux-ubuntu22-x86_64" JINDOSDK_MD5SUM="52236053391091591c2d09684791e810" # starcache -STARCACHE_DOWNLOAD="https://cdn-thirdparty.starrocks.com/starcache/v3.2.3/starcache-ubuntu22_amd64.tar.gz" +STARCACHE_DOWNLOAD="https://cdn-thirdparty.starrocks.com/starcache/v3.3.0/starcache-ubuntu22_amd64.tar.gz" STARCACHE_NAME="starcache.tar.gz" STARCACHE_SOURCE="starcache" -STARCACHE_MD5SUM="e881d9a0896c82044cb98472fbb4a203" +STARCACHE_MD5SUM="1e33c75c3e374c10cda7391c60f7d69d" diff --git a/thirdparty/vars-x86_64.sh b/thirdparty/vars-x86_64.sh index 65c8c4d6e210e3..17de6b91cbf096 100644 --- a/thirdparty/vars-x86_64.sh +++ b/thirdparty/vars-x86_64.sh @@ -52,7 +52,7 @@ JINDOSDK_SOURCE="jindosdk-4.6.8-linux" JINDOSDK_MD5SUM="5436e4fe39c4dfdc942e41821f1dd8a9" # starcache -STARCACHE_DOWNLOAD="https://cdn-thirdparty.starrocks.com/starcache/v3.2.3/starcache-centos7_amd64.tar.gz" +STARCACHE_DOWNLOAD="https://cdn-thirdparty.starrocks.com/starcache/v3.3.0/starcache-centos7_amd64.tar.gz" STARCACHE_NAME="starcache.tar.gz" STARCACHE_SOURCE="starcache" -STARCACHE_MD5SUM="335d09cffef0160cda5bb1b7ca9160df" +STARCACHE_MD5SUM="d2176508003f4462f896fc186f44342d" diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index 73221ff62eb452..22f4eab704e59f 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -320,10 +320,10 @@ GCS_CONNECTOR_SOURCE="gcs-connector-hadoop3-2.2.11-shaded" GCS_CONNECTOR_MD5SUM="51fd0eb5cb913a84e4ad8a5ed2069e21" # aws-sdk-cpp -AWS_SDK_CPP_DOWNLOAD="https://github.com/aws/aws-sdk-cpp/archive/refs/tags/1.10.36.tar.gz" -AWS_SDK_CPP_NAME="aws-sdk-cpp-1.10.36.tar.gz" -AWS_SDK_CPP_SOURCE="aws-sdk-cpp-1.10.36" -AWS_SDK_CPP_MD5SUM="8fed635c5ac98b448bc1a98cf7c97c70" +AWS_SDK_CPP_DOWNLOAD="https://github.com/aws/aws-sdk-cpp/archive/refs/tags/1.11.267.tar.gz" +AWS_SDK_CPP_NAME="aws-sdk-cpp-1.11.267.tar.gz" +AWS_SDK_CPP_SOURCE="aws-sdk-cpp-1.11.267" +AWS_SDK_CPP_MD5SUM="fdf43e7262f9d08968eb34f9ad18b8e7" # velocypack: A fast and compact format for serialization and storage VPACK_DOWNLOAD="https://github.com/arangodb/velocypack/archive/refs/tags/XYZ1.0.tar.gz"