Upgrade toolchain to LLVM17 & Support C++23

.toolchain.yml

@@ -0,0 +1,18 @@
+# Copyright 2024 PingCAP, Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Hi, there!
+# This is passed via jenkins podTemplate argument.
+# Instead of trying injections, take a look at https://en.pingcap.com/careers/.
+image_tag_suffix: -llvm-17.0.6

CMakeLists.txt

@@ -168,7 +168,7 @@ endif ()
 
 include (cmake/cpu_features.cmake)
 
-set (CMAKE_CXX_STANDARD 20)
+set (CMAKE_CXX_STANDARD 23)
 set (CMAKE_CXX_EXTENSIONS 1) # https://cmake.org/cmake/help/latest/prop_tgt/CXX_EXTENSIONS.html#prop_tgt:CXX_EXTENSIONS
 set (CMAKE_CXX_STANDARD_REQUIRED ON)
 

README.md

@@ -37,7 +37,7 @@ And the following operating systems:
 The following packages are required:
 
 - CMake 3.21.0+
-- Clang 13.0.6+
+- Clang 17.0.0+ under Linux or AppleClang 15.0.0+ under MacOS
 - Rust
 - Python 3.0+
 - Ninja-Build or GNU Make
@@ -56,10 +56,10 @@ curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain none
 source $HOME/.cargo/env
 
 # Install LLVM, see https://apt.llvm.org for details
-# Clang will be available as /usr/bin/clang++-13
+# Clang will be available as /usr/bin/clang++-17
 wget https://apt.llvm.org/llvm.sh
 chmod +x llvm.sh
-sudo ./llvm.sh 13 all
+sudo ./llvm.sh 17 all
 
 # Install other dependencies
 sudo apt install -y cmake ninja-build zlib1g-dev libcurl4-openssl-dev ccache
@@ -128,10 +128,10 @@ brew install ninja cmake [email protected] ccache
 If your MacOS is higher or equal to 13.0, it should work out of the box because by default Apple clang is 14.0.0. But if your MacOS is lower than 13.0, you should install llvm clang manually.
 
 ```shell
-brew install llvm@15
+brew install llvm@17
 
 # check llvm version
-clang --version # should be 15.0.0 or higher
+clang --version # should be 17.0.0 or higher
 ```
 
 </details>
@@ -162,8 +162,8 @@ Note: In Linux, usually you need to explicitly specify to use LLVM.
 ```shell
 # In cmake-build-debug directory:
 cmake .. -GNinja -DCMAKE_BUILD_TYPE=DEBUG \
-  -DCMAKE_C_COMPILER=/usr/bin/clang-13 \
-  -DCMAKE_CXX_COMPILER=/usr/bin/clang++-13
+  -DCMAKE_C_COMPILER=/usr/bin/clang-17 \
+  -DCMAKE_CXX_COMPILER=/usr/bin/clang++-17
 ```
 
 In MacOS, if you install llvm clang, you need to explicitly specify to use llvm clang.

contrib/CMakeLists.txt

@@ -94,6 +94,11 @@ if (USE_INTERNAL_POCO_LIBRARY)
         set (ENABLE_DATA_ODBC 0 CACHE INTERNAL "") # TODO (build fail)
     endif ()
     add_subdirectory (poco)
+    target_compile_options(Net PUBLIC "-Wno-error=deprecated-declarations")
+    target_compile_options(Net PUBLIC "-Wno-deprecated-declarations")
+    target_no_warning(NetSSL deprecated-declarations)
+    target_no_warning(Foundation deprecated-declarations)
+    target_no_warning(Net deprecated-enum-enum-conversion)
     unset (CMAKE_DISABLE_FIND_PACKAGE_ZLIB)
     set (ENABLE_TESTS ${_save})
     set (CMAKE_CXX_FLAGS ${save_CMAKE_CXX_FLAGS})

contrib/boost-cmake/CMakeLists.txt

@@ -34,3 +34,4 @@ target_compile_definitions (boost_internal PUBLIC BOOST_BIND_GLOBAL_PLACEHOLDERS
 target_no_warning(boost_internal unused-but-set-variable)
 target_no_warning(boost_internal deprecated-builtins)
 target_no_warning(boost_internal non-virtual-dtor)
+target_no_warning(boost_internal deprecated-declarations)

contrib/boringssl-cmake/CMakeLists.txt

@@ -593,6 +593,7 @@ add_library(
   "${BORINGSSL_SOURCE_DIR}/crypto/x509v3/v3_skey.c"
   "${BORINGSSL_SOURCE_DIR}/crypto/x509v3/v3_utl.c"
 )
+target_compile_options(crypto PRIVATE "-Wno-unused-but-set-variable")
 
 add_library(
   ssl

contrib/grpc-cmake/CMakeLists.txt

@@ -1,3 +1,6 @@
+# We want to use C++23, but GRPC is not ready
+set (CMAKE_CXX_STANDARD 20)
+
 set(_gRPC_SOURCE_DIR "${TiFlash_SOURCE_DIR}/contrib/grpc")
 set(_gRPC_BINARY_DIR "${TiFlash_BINARY_DIR}/contrib/grpc")
 
@@ -65,6 +68,7 @@ endif ()
 # reject to start w/o it. While this is completelly different library.
 #
 #   [3]: https://packages.debian.org/bullseye/libnsl2
+
 if (NOT CMAKE_SYSTEM_NAME STREQUAL "SunOS")
   set(HAVE_LIBNSL OFF CACHE BOOL "" FORCE)
 endif()

contrib/libunwind-cmake/CMakeLists.txt

@@ -152,3 +152,5 @@ target_include_directories (unwind PUBLIC ${LIBUNWIND_SOURCE_DIR}/include)
 target_include_directories (unwind PRIVATE ${LIBUNWIND_SOURCE_DIR}/include/tdep)
 target_include_directories (unwind PRIVATE ${LIBUNWIND_SOURCE_DIR}/src)
 target_include_directories (unwind PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/include)
+target_compile_options (unwind PRIVATE "-Wno-single-bit-bitfield-constant-conversion")
+target_compile_options (unwind PRIVATE "-Wno-absolute-value")

contrib/protobuf-cmake/CMakeLists.txt

@@ -211,6 +211,7 @@ set(libprotoc_files
 
 add_library(libprotoc ${libprotoc_files})
 target_link_libraries(libprotoc libprotobuf)
+target_compile_options(libprotoc PRIVATE "-Wno-deprecated-pragma")
 add_library(protobuf::libprotoc ALIAS libprotoc)
 
 set(protoc_files ${protobuf_source_dir}/src/google/protobuf/compiler/main.cc)

dbms/src/Common/RemoteHostFilter.h

@@ -16,6 +16,7 @@
 
 #include <common/defines.h>
 
+#include <atomic>
 #include <mutex>
 #include <string>
 #include <unordered_set>

dbms/src/IO/BaseFile/WritableFile.h

@@ -14,6 +14,9 @@
 
 #pragma once
 
+#if __APPLE__ && __clang__
+#include <sys/_types/_off_t.h>
+#endif
 #include <memory>
 
 namespace DB

dbms/src/IO/Encryption/tests/gtest_encryption_test.cpp

@@ -175,7 +175,9 @@ try
         EncryptionMethod::Aes128Ctr,
         EncryptionMethod::Aes192Ctr,
         EncryptionMethod::Aes256Ctr,
+#if USE_GM_SSL
         EncryptionMethod::SM4Ctr,
+#endif
     };
     for (size_t i = 0; i < 10; ++i)
     {

dbms/src/Storages/DeltaMerge/StoragePool/StoragePool_fwd.h

@@ -14,6 +14,7 @@
 
 #pragma once
 
+#include <chrono>
 #include <memory>
 
 namespace DB::DM

dbms/src/Storages/KVStore/tests/gtest_kvstore.cpp

@@ -16,6 +16,8 @@
 #include <Storages/KVStore/MultiRaft/RegionsRangeIndex.h>
 #include <Storages/KVStore/tests/kvstore_helper.h>
 
+#include <regex>
+
 
 namespace DB::tests
 {
@@ -1542,9 +1544,8 @@ TEST_F(RegionKVStoreOldTest, RegionRange)
         catch (Exception & e)
         {
             const auto & res = e.message();
-            ASSERT_EQ(
-                res,
-                "void DB::RegionsRangeIndex::remove(const DB::RegionRange &, DB::RegionID): not found region_id=1");
+            std::regex msg_reg(".*: not found region_id=1");
+            ASSERT_TRUE(std::regex_match(res, msg_reg));
         }
 
         region_index.add(makeRegion(2, RecordKVFormat::genKey(1, 3), RecordKVFormat::genKey(1, 5)));
@@ -1558,9 +1559,8 @@ TEST_F(RegionKVStoreOldTest, RegionRange)
         catch (Exception & e)
         {
             const auto & res = e.message();
-            ASSERT_EQ(
-                res,
-                "void DB::RegionsRangeIndex::remove(const DB::RegionRange &, DB::RegionID): not found start key");
+            std::regex msg_reg(".*: not found start key");
+            ASSERT_TRUE(std::regex_match(res, msg_reg));
         }
 
         try
@@ -1573,9 +1573,8 @@ TEST_F(RegionKVStoreOldTest, RegionRange)
         catch (Exception & e)
         {
             const auto & res = e.message();
-            ASSERT_EQ(
-                res,
-                "void DB::RegionsRangeIndex::remove(const DB::RegionRange &, DB::RegionID): not found end key");
+            std::regex msg_reg(".*: not found end key");
+            ASSERT_TRUE(std::regex_match(res, msg_reg));
         }
 
         try
@@ -1588,10 +1587,8 @@ TEST_F(RegionKVStoreOldTest, RegionRange)
         catch (Exception & e)
         {
             const auto & res = e.message();
-            ASSERT_EQ(
-                res,
-                "void DB::RegionsRangeIndex::remove(const DB::RegionRange &, DB::RegionID): range of region_id=2 is "
-                "empty");
+            std::regex msg_reg(".*: range of region_id=2 is empty");
+            ASSERT_TRUE(std::regex_match(res, msg_reg));
         }
 
         try
@@ -1602,9 +1599,8 @@ TEST_F(RegionKVStoreOldTest, RegionRange)
         catch (Exception & e)
         {
             const auto & res = e.message();
-            ASSERT_EQ(
-                res,
-                "void DB::RegionsRangeIndex::remove(const DB::RegionRange &, DB::RegionID): not found region_id=2");
+            std::regex msg_reg(".*: not found region_id=2");
+            ASSERT_TRUE(std::regex_match(res, msg_reg));
         }
 
         region_index.clear();

dbms/src/Storages/S3/FileCache.cpp

@@ -32,7 +32,6 @@
 #include <chrono>
 #include <cmath>
 #include <filesystem>
-#include <fstream>
 
 namespace ProfileEvents
 {
@@ -56,6 +55,8 @@ namespace DB
 {
 using FileType = FileSegment::FileType;
 
+std::unique_ptr<FileCache> FileCache::global_file_cache_instance;
+
 FileCache::FileCache(PathCapacityMetricsPtr capacity_metrics_, const StorageRemoteCacheConfig & config_)
     : capacity_metrics(capacity_metrics_)
     , cache_dir(config_.getDTFileCacheDir())

dbms/src/Storages/S3/FileCache.h

@@ -228,7 +228,7 @@ class FileCache
 #endif
 
     inline static std::atomic<bool> global_file_cache_initialized{false};
-    inline static std::unique_ptr<FileCache> global_file_cache_instance;
+    static std::unique_ptr<FileCache> global_file_cache_instance;
 
     DISALLOW_COPY_AND_MOVE(FileCache);
 

dbms/src/TiDB/Schema/SchemaSyncService.cpp

@@ -60,7 +60,7 @@ void SchemaSyncService::addKeyspaceGCTasks()
     UInt64 num_add_tasks = 0;
     // Add new sync schema task for new keyspace.
     std::unique_lock<std::shared_mutex> lock(keyspace_map_mutex);
-    for (auto const iter : keyspaces)
+    for (auto const & iter : keyspaces)
     {
         auto keyspace = iter.first;
         if (keyspace_handle_map.contains(keyspace))

libs/libcommon/include/common/ThreadPool.h

@@ -15,7 +15,7 @@
 #pragma once
 
 #include <condition_variable>
-#include <cstdint>
+#include <exception>
 #include <functional>
 #include <mutex>
 #include <queue>

release-centos7-llvm/dockerfiles/misc/bake_llvm_base_aarch64.sh

@@ -30,18 +30,18 @@ function bake_llvm_base_aarch64() {
 
     # CMake
     source $SCRIPTPATH/install_cmake.sh
-    install_cmake "3.22.1" "aarch64"
+    install_cmake "3.24.2" "aarch64"
 
     # LLVM
     source $SCRIPTPATH/bootstrap_llvm.sh
-    bootstrap_llvm "13.0.0"
+    bootstrap_llvm "17.0.6"
     export CC=clang
     export CXX=clang++
     export LD=ld.lld
 
     # OpenSSL
     source $SCRIPTPATH/install_openssl.sh
-    install_openssl "1_1_1t"
+    install_openssl "1_1_1w"
     export OPENSSL_ROOT_DIR="/usr/local/opt/openssl"
 
     # Git

release-centos7-llvm/dockerfiles/misc/bake_llvm_base_amd64.sh

@@ -29,18 +29,18 @@ function bake_llvm_base_amd64() {
 
     # CMake
     source $SCRIPTPATH/install_cmake.sh
-    install_cmake "3.22.1" "x86_64"
+    install_cmake "3.24.2" "x86_64"
 
     # LLVM
     source $SCRIPTPATH/bootstrap_llvm.sh
-    bootstrap_llvm "13.0.0"
+    bootstrap_llvm "17.0.6"
     export CC=clang
     export CXX=clang++
     export LD=ld.lld
 
     # OpenSSL
     source $SCRIPTPATH/install_openssl.sh
-    install_openssl "1_1_1t"
+    install_openssl "1_1_1w"
     export OPENSSL_ROOT_DIR="/usr/local/opt/openssl"
 
     # Git

release-centos7-llvm/dockerfiles/misc/bootstrap_llvm.sh

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
+INSTALL_PREFIX=${INSTALL_PREFIX:-"/usr/local"}
 
 # Boostrap LLVM envriroment for CI/CD.
 # Require: git, ninja, cmake, compiler(devtoolset-10)
@@ -29,6 +29,7 @@ function bootstrap_llvm() {
 
     cmake -DCMAKE_BUILD_TYPE=Release \
         -GNinja \
+        -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \
         -DLLVM_ENABLE_PROJECTS="clang;lld" \
         -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi" \
         -DLLVM_TARGETS_TO_BUILD=Native \
@@ -44,7 +45,8 @@ function bootstrap_llvm() {
 
     cmake -DCMAKE_BUILD_TYPE=Release \
         -GNinja \
-        -DLLVM_ENABLE_PROJECTS="clang;lld;polly;clang-tools-extra" \
+        -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \
+        -DLLVM_ENABLE_PROJECTS="clang;lld;polly;clang-tools-extra;bolt" \
         -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;openmp;compiler-rt" \
         -DLLVM_TARGETS_TO_BUILD=Native \
         -DCOMPILER_RT_DEFAULT_TARGET_ONLY=ON \