From aced292114342c8d631905dd66a25c183a74d1a4 Mon Sep 17 00:00:00 2001 From: Ti Chi Robot Date: Tue, 7 Nov 2023 15:05:11 +0800 Subject: [PATCH] disagg: Bump aws sdk (#8247) (#8283) close pingcap/tiflash#8274 --- .gitmodules | 7 +- contrib/aws | 2 +- contrib/aws-c-auth | 2 +- contrib/aws-c-cal | 2 +- contrib/aws-c-common | 2 +- contrib/aws-c-compression | 2 +- contrib/aws-c-event-stream | 2 +- contrib/aws-c-http | 2 +- contrib/aws-c-io | 2 +- contrib/aws-c-mqtt | 2 +- contrib/aws-c-s3 | 2 +- contrib/aws-c-sdkutils | 2 +- contrib/aws-checksums | 2 +- ...y-to-check-if-there-is-anything-in-r.patch | 101 +++++++++++++++ ...rror-logging-and-404-for-HEAD-reques.patch | 62 +++++++++ contrib/aws-cmake/CMakeLists.txt | 75 ++++++++--- contrib/aws-crt-cpp | 2 +- contrib/aws-s2n-tls | 2 +- dbms/src/Interpreters/Context.cpp | 4 +- dbms/src/Interpreters/Context.h | 2 +- dbms/src/Server/Server.cpp | 26 ++-- .../KVStore/FFI/ProxyFFIStatusService.cpp | 118 +++++++++++++++--- dbms/src/Storages/KVStore/TMTContext.cpp | 4 +- dbms/src/Storages/KVStore/TMTContext.h | 2 + .../Universal/UniversalPageStorageService.cpp | 12 +- .../Universal/UniversalPageStorageService.h | 4 +- dbms/src/Storages/S3/S3Common.cpp | 4 +- dbms/src/Storages/S3/S3Common.h | 3 +- dbms/src/Storages/S3/S3GCManager.cpp | 8 ++ dbms/src/Storages/S3/S3GCManager.h | 2 + 30 files changed, 384 insertions(+), 78 deletions(-) create mode 100644 contrib/aws-cmake/0001-More-reliable-way-to-check-if-there-is-anything-in-r.patch create mode 100644 contrib/aws-cmake/0002-Reduce-verbose-error-logging-and-404-for-HEAD-reques.patch diff --git a/.gitmodules b/.gitmodules index 04fe954f623..021da59cf22 100644 --- a/.gitmodules +++ b/.gitmodules @@ -94,13 +94,16 @@ url = https://github.com/guanzhi/GmSSL.git [submodule "contrib/aws"] path = contrib/aws - url = https://github.com/JaySon-Huang/aws-sdk-cpp.git + url = https://github.com/aws/aws-sdk-cpp.git + # ignore dirty status since we apply patches to + # fix bug when using Poco client + ignore = dirty [submodule "contrib/aws-c-auth"] path = contrib/aws-c-auth url = https://github.com/awslabs/aws-c-auth.git [submodule "contrib/aws-c-cal"] path = contrib/aws-c-cal - url = https://github.com/ClickHouse/aws-c-cal.git + url = https://github.com/awslabs/aws-c-cal.git [submodule "contrib/aws-c-common"] path = contrib/aws-c-common url = https://github.com/awslabs/aws-c-common.git diff --git a/contrib/aws b/contrib/aws index 7587c2cb67e..a845f02457a 160000 --- a/contrib/aws +++ b/contrib/aws @@ -1 +1 @@ -Subproject commit 7587c2cb67e446bc42009f81acc77de6f40f5472 +Subproject commit a845f02457a9806d0678ffd11d47dfae5fbf7f21 diff --git a/contrib/aws-c-auth b/contrib/aws-c-auth index dd505b55fd4..c75e00804da 160000 --- a/contrib/aws-c-auth +++ b/contrib/aws-c-auth @@ -1 +1 @@ -Subproject commit dd505b55fd46222834f35c6e54165d8cbebbfaaa +Subproject commit c75e00804da02fef8275b29509bd7180cb7e4667 diff --git a/contrib/aws-c-cal b/contrib/aws-c-cal index 85dd7664b78..a916a84ec07 160000 --- a/contrib/aws-c-cal +++ b/contrib/aws-c-cal @@ -1 +1 @@ -Subproject commit 85dd7664b786a389c6fb1a6f031ab4bb2282133d +Subproject commit a916a84ec07d028fa7d8c09d4aecaa81df7e8a23 diff --git a/contrib/aws-c-common b/contrib/aws-c-common index 50695e94809..4c0a9f579d3 160000 --- a/contrib/aws-c-common +++ b/contrib/aws-c-common @@ -1 +1 @@ -Subproject commit 50695e948092579e1455623b5cc2ba7e511a3990 +Subproject commit 4c0a9f579d3064f086b42a2d39aaea721e7e71ca diff --git a/contrib/aws-c-compression b/contrib/aws-c-compression index b517b7decd0..99ec79ee297 160000 --- a/contrib/aws-c-compression +++ b/contrib/aws-c-compression @@ -1 +1 @@ -Subproject commit b517b7decd0dac30be2162f5186c250221c53aff +Subproject commit 99ec79ee2970f1a045d4ced1501b97ee521f2f85 diff --git a/contrib/aws-c-event-stream b/contrib/aws-c-event-stream index 2f9b60c42f9..08f24e384e5 160000 --- a/contrib/aws-c-event-stream +++ b/contrib/aws-c-event-stream @@ -1 +1 @@ -Subproject commit 2f9b60c42f90840ec11822acda3d8cdfa97a773d +Subproject commit 08f24e384e5be20bcffa42b49213d24dad7881ae diff --git a/contrib/aws-c-http b/contrib/aws-c-http index 99894610086..d777859b6da 160000 --- a/contrib/aws-c-http +++ b/contrib/aws-c-http @@ -1 +1 @@ -Subproject commit 99894610086372df1cf09d62b9eafca42eb53f5b +Subproject commit d777859b6da179b9098f87a2077fbf2129b574dc diff --git a/contrib/aws-c-io b/contrib/aws-c-io index f2ff573c191..c9cb77747d3 160000 --- a/contrib/aws-c-io +++ b/contrib/aws-c-io @@ -1 +1 @@ -Subproject commit f2ff573c191e1c4ea0248af5c08161356be3bc78 +Subproject commit c9cb77747d3fd2809cf3d9c43be7d5decc17e4b3 diff --git a/contrib/aws-c-mqtt b/contrib/aws-c-mqtt index 5cbde90916a..0cc50d15827 160000 --- a/contrib/aws-c-mqtt +++ b/contrib/aws-c-mqtt @@ -1 +1 @@ -Subproject commit 5cbde90916a1f9945e2a1ef36f3db58e1c976167 +Subproject commit 0cc50d1582719c9f3b55539139015fc86965bb6f diff --git a/contrib/aws-c-s3 b/contrib/aws-c-s3 index 91e03c1875f..1fe34646f6b 160000 --- a/contrib/aws-c-s3 +++ b/contrib/aws-c-s3 @@ -1 +1 @@ -Subproject commit 91e03c1875f8c28a71744d75b980336191f096e2 +Subproject commit 1fe34646f6bb2282491077b1d2407c724c0d5471 diff --git a/contrib/aws-c-sdkutils b/contrib/aws-c-sdkutils index 208a701fa01..a6fd80cf7c1 160000 --- a/contrib/aws-c-sdkutils +++ b/contrib/aws-c-sdkutils @@ -1 +1 @@ -Subproject commit 208a701fa01e99c7c8cc3dcebc8317da71362972 +Subproject commit a6fd80cf7c163062d31abb28f309e47330fbfc17 diff --git a/contrib/aws-checksums b/contrib/aws-checksums index ad53be196a2..321b805559c 160000 --- a/contrib/aws-checksums +++ b/contrib/aws-checksums @@ -1 +1 @@ -Subproject commit ad53be196a25bbefa3700a01187fdce573a7d2d0 +Subproject commit 321b805559c8e911be5bddba13fcbd222a3e2d3a diff --git a/contrib/aws-cmake/0001-More-reliable-way-to-check-if-there-is-anything-in-r.patch b/contrib/aws-cmake/0001-More-reliable-way-to-check-if-there-is-anything-in-r.patch new file mode 100644 index 00000000000..495f81fe90d --- /dev/null +++ b/contrib/aws-cmake/0001-More-reliable-way-to-check-if-there-is-anything-in-r.patch @@ -0,0 +1,101 @@ +From 92225a50ccbab5369fd40b99a310ef7fcaec1750 Mon Sep 17 00:00:00 2001 +From: JaySon-Huang +Date: Thu, 6 Apr 2023 12:54:23 +0800 +Subject: [PATCH 1/2] More reliable way to check if there is anything in result + IOStream + +Signed-off-by: JaySon-Huang +--- + src/aws-cpp-sdk-core/source/client/AWSJsonClient.cpp | 9 +++++---- + src/aws-cpp-sdk-core/source/client/AWSXmlClient.cpp | 6 +++--- + .../source/internal/AWSHttpResourceClient.cpp | 2 +- + 3 files changed, 9 insertions(+), 8 deletions(-) + +diff --git a/src/aws-cpp-sdk-core/source/client/AWSJsonClient.cpp b/src/aws-cpp-sdk-core/source/client/AWSJsonClient.cpp +index f42a306156..3cd26203f0 100644 +--- a/src/aws-cpp-sdk-core/source/client/AWSJsonClient.cpp ++++ b/src/aws-cpp-sdk-core/source/client/AWSJsonClient.cpp +@@ -115,7 +115,8 @@ JsonOutcome AWSJsonClient::MakeRequest(const Aws::Http::URI& uri, + {{TracingUtils::SMITHY_METHOD_DIMENSION, request.GetServiceRequestName()}, {TracingUtils::SMITHY_SERVICE_DIMENSION, this->GetServiceClientName()}}); + } + +- if (httpOutcome.GetResult()->GetResponseBody().tellp() > 0){ ++ if (httpOutcome.GetResult()->GetResponseBody().peek() != std::char_traits::eof()) ++ { + return smithy::components::tracing::TracingUtils::MakeCallWithTiming( + [&]() -> JsonOutcome { + return JsonOutcome(AmazonWebServiceResult(JsonValue(httpOutcome.GetResult()->GetResponseBody()), +@@ -154,7 +155,7 @@ JsonOutcome AWSJsonClient::MakeRequest(const Aws::Http::URI& uri, + {{TracingUtils::SMITHY_METHOD_DIMENSION, requestName}, {TracingUtils::SMITHY_SERVICE_DIMENSION, this->GetServiceClientName()}}); + } + +- if (httpOutcome.GetResult()->GetResponseBody().tellp() > 0) ++ if (httpOutcome.GetResult()->GetResponseBody().peek() != std::char_traits::eof()) + { + JsonValue jsonValue(httpOutcome.GetResult()->GetResponseBody()); + if (!jsonValue.WasParseSuccessful()) { +@@ -203,7 +204,7 @@ JsonOutcome AWSJsonClient::MakeEventStreamRequest(std::shared_ptrGetResponseBody().tellp() > 0) ++ if (httpOutcome.GetResult()->GetResponseBody().peek() != std::char_traits::eof()) + { + JsonValue jsonValue(httpOutcome.GetResult()->GetResponseBody()); + if (!jsonValue.WasParseSuccessful()) +@@ -229,7 +230,7 @@ AWSError AWSJsonClient::BuildAWSError( + bool retryable = httpResponse->GetClientErrorType() == CoreErrors::NETWORK_CONNECTION ? true : false; + error = AWSError(httpResponse->GetClientErrorType(), "", httpResponse->GetClientErrorMessage(), retryable); + } +- else if (!httpResponse->GetResponseBody() || httpResponse->GetResponseBody().tellp() < 1) ++ else if (!httpResponse->GetResponseBody() || httpResponse->GetResponseBody().peek() == std::char_traits::eof()) + { + auto responseCode = httpResponse->GetResponseCode(); + auto errorCode = AWSClient::GuessBodylessErrorType(responseCode); +diff --git a/src/aws-cpp-sdk-core/source/client/AWSXmlClient.cpp b/src/aws-cpp-sdk-core/source/client/AWSXmlClient.cpp +index 443ea31cbc..c122c5d5a1 100644 +--- a/src/aws-cpp-sdk-core/source/client/AWSXmlClient.cpp ++++ b/src/aws-cpp-sdk-core/source/client/AWSXmlClient.cpp +@@ -110,7 +110,7 @@ XmlOutcome AWSXMLClient::MakeRequest(const Aws::Http::URI& uri, + {{TracingUtils::SMITHY_METHOD_DIMENSION, request.GetServiceRequestName()}, {TracingUtils::SMITHY_SERVICE_DIMENSION, this->GetServiceClientName()}}); + } + +- if (httpOutcome.GetResult()->GetResponseBody().tellp() > 0) ++ if (httpOutcome.GetResult()->GetResponseBody().peek() != std::char_traits::eof()) + { + return smithy::components::tracing::TracingUtils::MakeCallWithTiming( + [&]() -> XmlOutcome { +@@ -152,7 +152,7 @@ XmlOutcome AWSXMLClient::MakeRequest(const Aws::Http::URI& uri, + {{TracingUtils::SMITHY_METHOD_DIMENSION, requestName}, {TracingUtils::SMITHY_SERVICE_DIMENSION, this->GetServiceClientName()}}); + } + +- if (httpOutcome.GetResult()->GetResponseBody().tellp() > 0) ++ if (httpOutcome.GetResult()->GetResponseBody().peek() != std::char_traits::eof()) + { + return smithy::components::tracing::TracingUtils::MakeCallWithTiming( + [&]() -> XmlOutcome { +@@ -182,7 +182,7 @@ AWSError AWSXMLClient::BuildAWSError(const std::shared_ptrGetClientErrorType() == CoreErrors::NETWORK_CONNECTION ? true : false; + error = AWSError(httpResponse->GetClientErrorType(), "", httpResponse->GetClientErrorMessage(), retryable); + } +- else if (!httpResponse->GetResponseBody() || httpResponse->GetResponseBody().tellp() < 1) ++ else if (!httpResponse->GetResponseBody() || httpResponse->GetResponseBody().peek() == std::char_traits::eof()) + { + auto responseCode = httpResponse->GetResponseCode(); + auto errorCode = AWSClient::GuessBodylessErrorType(responseCode); +diff --git a/src/aws-cpp-sdk-core/source/internal/AWSHttpResourceClient.cpp b/src/aws-cpp-sdk-core/source/internal/AWSHttpResourceClient.cpp +index 723747bbf1..8d84083ba3 100644 +--- a/src/aws-cpp-sdk-core/source/internal/AWSHttpResourceClient.cpp ++++ b/src/aws-cpp-sdk-core/source/internal/AWSHttpResourceClient.cpp +@@ -148,7 +148,7 @@ namespace Aws + AWS_LOGSTREAM_ERROR(m_logtag.c_str(), "Http request to retrieve credentials failed"); + return AWSError(CoreErrors::NETWORK_CONNECTION, true); // Retryable + } +- else if (m_errorMarshaller && response->GetResponseBody().tellp() > 0) ++ else if (m_errorMarshaller && response->GetResponseBody().peek() != std::char_traits::eof()) + { + return m_errorMarshaller->Marshall(*response); + } +-- +2.31.1 + diff --git a/contrib/aws-cmake/0002-Reduce-verbose-error-logging-and-404-for-HEAD-reques.patch b/contrib/aws-cmake/0002-Reduce-verbose-error-logging-and-404-for-HEAD-reques.patch new file mode 100644 index 00000000000..30dfdba9c4c --- /dev/null +++ b/contrib/aws-cmake/0002-Reduce-verbose-error-logging-and-404-for-HEAD-reques.patch @@ -0,0 +1,62 @@ +From 7e6f90112f21c6996e097012c0fe6bfc5c3445d3 Mon Sep 17 00:00:00 2001 +From: JaySon-Huang +Date: Wed, 17 May 2023 15:56:17 +0800 +Subject: [PATCH 2/2] Reduce verbose error logging and 404 for HEAD request + +--- + src/aws-cpp-sdk-core/source/client/AWSClient.cpp | 2 +- + src/aws-cpp-sdk-core/source/client/AWSXmlClient.cpp | 12 +++++++++++- + 2 files changed, 12 insertions(+), 2 deletions(-) + +diff --git a/src/aws-cpp-sdk-core/source/client/AWSClient.cpp b/src/aws-cpp-sdk-core/source/client/AWSClient.cpp +index 5d8a7a9e8a..932bf7d2c0 100644 +--- a/src/aws-cpp-sdk-core/source/client/AWSClient.cpp ++++ b/src/aws-cpp-sdk-core/source/client/AWSClient.cpp +@@ -209,7 +209,6 @@ bool AWSClient::AdjustClockSkew(HttpResponseOutcome& outcome, const char* signer + { + auto signer = GetSignerByName(signerName); + //detect clock skew and try to correct. +- AWS_LOGSTREAM_WARN(AWS_CLIENT_LOG_TAG, "If the signature check failed. This could be because of a time skew. Attempting to adjust the signer."); + + DateTime serverTime = GetServerTimeFromError(outcome.GetError()); + const auto signingTimestamp = signer->GetSigningTimestamp(); +@@ -224,6 +223,7 @@ bool AWSClient::AdjustClockSkew(HttpResponseOutcome& outcome, const char* signer + //only try again if clock skew was the cause of the error. + if (diff >= TIME_DIFF_MAX || diff <= TIME_DIFF_MIN) + { ++ AWS_LOGSTREAM_WARN(AWS_CLIENT_LOG_TAG, "If the signature check failed. This could be because of a time skew. Attempting to adjust the signer."); + diff = DateTime::Diff(serverTime, DateTime::Now()); + AWS_LOGSTREAM_INFO(AWS_CLIENT_LOG_TAG, "Computed time difference as " << diff.count() << " milliseconds. Adjusting signer with the skew."); + signer->SetClockSkew(diff); +diff --git a/src/aws-cpp-sdk-core/source/client/AWSXmlClient.cpp b/src/aws-cpp-sdk-core/source/client/AWSXmlClient.cpp +index c122c5d5a1..311b64f4c0 100644 +--- a/src/aws-cpp-sdk-core/source/client/AWSXmlClient.cpp ++++ b/src/aws-cpp-sdk-core/source/client/AWSXmlClient.cpp +@@ -13,6 +13,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -207,6 +208,15 @@ AWSError AWSXMLClient::BuildAWSError(const std::shared_ptrGetHeaders()); + error.SetResponseCode(httpResponse->GetResponseCode()); + error.SetRemoteHostIpAddress(httpResponse->GetOriginatingRequest().GetResolvedRemoteHost()); +- AWS_LOGSTREAM_ERROR(AWS_XML_CLIENT_LOG_TAG, error); ++ ++ if (httpResponse->GetOriginatingRequest().GetMethod() == HttpMethod::HTTP_HEAD && httpResponse->GetResponseCode() == HttpResponseCode::NOT_FOUND) ++ { ++ // ignore error logging for HEAD request with 404 response code, ususally it is caused by determining whether the object exists or not. ++ AWS_LOGSTREAM_DEBUG(AWS_XML_CLIENT_LOG_TAG, error); ++ } ++ else ++ { ++ AWS_LOGSTREAM_ERROR(AWS_XML_CLIENT_LOG_TAG, error); ++ } + return error; + } +-- +2.31.1 + diff --git a/contrib/aws-cmake/CMakeLists.txt b/contrib/aws-cmake/CMakeLists.txt index a729c6a48fb..5fe5efdc7d6 100644 --- a/contrib/aws-cmake/CMakeLists.txt +++ b/contrib/aws-cmake/CMakeLists.txt @@ -12,7 +12,14 @@ set(AWS_PUBLIC_COMPILE_DEFS) set(AWS_PRIVATE_COMPILE_DEFS) set(AWS_PRIVATE_LIBS) -list(APPEND AWS_PRIVATE_COMPILE_DEFS "-DENABLE_CURL_CLIENT") +# Versions. Note that we may update the patch under /aws-cmake to make it work +# after upgrading the aws-sdk-cpp version. +list(APPEND AWS_PUBLIC_COMPILE_DEFS "-DAWS_SDK_VERSION_MAJOR=1") +list(APPEND AWS_PUBLIC_COMPILE_DEFS "-DAWS_SDK_VERSION_MINOR=11") +list(APPEND AWS_PUBLIC_COMPILE_DEFS "-DAWS_SDK_VERSION_PATCH=186") + +# ignore the symbols defined in `aws/common/external/ittnotify.h` +list(APPEND AWS_PRIVATE_COMPILE_DEFS "-DENABLE_CURL_CLIENT -DINTEL_NO_ITTNOTIFY_API") if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG") list(APPEND AWS_PRIVATE_COMPILE_DEFS "-DDEBUG_BUILD") @@ -48,7 +55,6 @@ SET(AWS_S2N_TLS_DIR "${TiFlash_SOURCE_DIR}/contrib/aws-s2n-tls") SET(AWS_S3_DIR "${TiFlash_SOURCE_DIR}/contrib/aws-c-s3") SET(AWS_SDKUTILS_DIR "${TiFlash_SOURCE_DIR}/contrib/aws-c-sdkutils") - # aws-cpp-sdk-core file(GLOB AWS_SDK_CORE_SRC "${AWS_SDK_CORE_DIR}/source/*.cpp" @@ -64,15 +70,18 @@ file(GLOB AWS_SDK_CORE_SRC "${AWS_SDK_CORE_DIR}/source/external/cjson/*.cpp" "${AWS_SDK_CORE_DIR}/source/external/tinyxml2/*.cpp" "${AWS_SDK_CORE_DIR}/source/http/*.cpp" - "${AWS_SDK_CORE_DIR}/source/http/standard/*.cpp" "${AWS_SDK_CORE_DIR}/source/http/curl/*.cpp" + "${AWS_SDK_CORE_DIR}/source/http/standard/*.cpp" "${AWS_SDK_CORE_DIR}/source/internal/*.cpp" "${AWS_SDK_CORE_DIR}/source/monitoring/*.cpp" + "${AWS_SDK_CORE_DIR}/source/smithy/*.cpp" + "${AWS_SDK_CORE_DIR}/source/smithy/tracing/*.cpp" "${AWS_SDK_CORE_DIR}/source/utils/*.cpp" "${AWS_SDK_CORE_DIR}/source/utils/base64/*.cpp" + "${AWS_SDK_CORE_DIR}/source/utils/component-registry/*.cpp" "${AWS_SDK_CORE_DIR}/source/utils/crypto/*.cpp" - "${AWS_SDK_CORE_DIR}/source/utils/crypto/openssl/*.cpp" "${AWS_SDK_CORE_DIR}/source/utils/crypto/factory/*.cpp" + "${AWS_SDK_CORE_DIR}/source/utils/crypto/openssl/*.cpp" "${AWS_SDK_CORE_DIR}/source/utils/event/*.cpp" "${AWS_SDK_CORE_DIR}/source/utils/json/*.cpp" "${AWS_SDK_CORE_DIR}/source/utils/logging/*.cpp" @@ -83,6 +92,42 @@ file(GLOB AWS_SDK_CORE_SRC "${AWS_SDK_CORE_DIR}/source/utils/xml/*.cpp" ) +execute_process( + COMMAND grep "GetResponseBody().peek()" "${AWS_SDK_CORE_DIR}/source/client/AWSXmlClient.cpp" + RESULT_VARIABLE HAVE_APPLY_PATCH) +# grep - Normally, the exit status is 0 if selected lines are found and 1 otherwise. But the exit status is 2 if an error occurred. +if (HAVE_APPLY_PATCH EQUAL 1) + message(STATUS "aws patch not apply: ${HAVE_APPLY_PATCH}, patching...") + ## update the patch using `git format-patch` if you upgrade aws + set (AWS_PATCH_FILE "${TiFlash_SOURCE_DIR}/contrib/aws-cmake/0001-More-reliable-way-to-check-if-there-is-anything-in-r.patch") + # apply the patch + execute_process( + COMMAND git apply -v "${AWS_PATCH_FILE}" + WORKING_DIRECTORY "${AWS_SDK_CORE_DIR}" + COMMAND_ECHO STDOUT + RESULT_VARIABLE PATCH_SUCC) + if (NOT PATCH_SUCC EQUAL 0) + message(FATAL_ERROR "Can not apply aws patch ${AWS_PATCH_FILE}") + endif () + + set (AWS_PATCH_FILE "${TiFlash_SOURCE_DIR}/contrib/aws-cmake/0002-Reduce-verbose-error-logging-and-404-for-HEAD-reques.patch") + # apply the patch + execute_process( + COMMAND git apply -v "${AWS_PATCH_FILE}" + WORKING_DIRECTORY "${AWS_SDK_CORE_DIR}" + COMMAND_ECHO STDOUT + RESULT_VARIABLE PATCH_SUCC) + if (NOT PATCH_SUCC EQUAL 0) + message(FATAL_ERROR "Can not apply aws patch ${AWS_PATCH_FILE}") + else () + message(STATUS "aws patch done") + endif () +elseif (HAVE_APPLY_PATCH EQUAL 0) + message(STATUS "aws patch have been applied: ${HAVE_APPLY_PATCH}") +else () + message(FATAL_ERROR "Can not check the aws patch status") +endif () + if(OS_LINUX OR OS_DARWIN) file(GLOB AWS_SDK_CORE_NET_SRC "${AWS_SDK_CORE_DIR}/source/net/linux-shared/*.cpp") file(GLOB AWS_SDK_CORE_PLATFORM_SRC "${AWS_SDK_CORE_DIR}/source/platform/linux-shared/*.cpp") @@ -95,10 +140,6 @@ OPTION(USE_AWS_MEMORY_MANAGEMENT "Aws memory management" OFF) configure_file("${AWS_SDK_CORE_DIR}/include/aws/core/SDKConfig.h.in" "${CMAKE_CURRENT_BINARY_DIR}/include/aws/core/SDKConfig.h" @ONLY) -list(APPEND AWS_PUBLIC_COMPILE_DEFS "-DAWS_SDK_VERSION_MAJOR=1") -list(APPEND AWS_PUBLIC_COMPILE_DEFS "-DAWS_SDK_VERSION_MINOR=10") -list(APPEND AWS_PUBLIC_COMPILE_DEFS "-DAWS_SDK_VERSION_PATCH=36") - list(APPEND AWS_SOURCES ${AWS_SDK_CORE_SRC} ${AWS_SDK_CORE_NET_SRC} ${AWS_SDK_CORE_PLATFORM_SRC}) list(APPEND AWS_PUBLIC_INCLUDES @@ -140,6 +181,8 @@ file(GLOB AWS_CAL_SRC ) if (ENABLE_OPENSSL_ENCRYPTION) + # Note: We use `unix/*.c` for both OS_LINUX and OS_DARWIN + # `darwin/*.c` lead to compile errors file(GLOB AWS_CAL_OS_SRC "${AWS_CAL_DIR}/source/unix/*.c" ) @@ -204,9 +247,6 @@ list(APPEND AWS_PUBLIC_INCLUDES # aws-checksums file(GLOB AWS_CHECKSUMS_SRC "${AWS_CHECKSUMS_DIR}/source/*.c" - "${AWS_CHECKSUMS_DIR}/source/intel/*.c" - "${AWS_CHECKSUMS_DIR}/source/intel/asm/*.c" - "${AWS_CHECKSUMS_DIR}/source/arm/*.c" ) if(AWS_ARCH_INTEL AND AWS_HAVE_GCC_INLINE_ASM) @@ -220,14 +260,6 @@ if (AWS_ARCH_ARM64) "${AWS_CHECKSUMS_DIR}/source/arm/*.c" ) set_source_files_properties("${AWS_CHECKSUMS_DIR}/source/arm/crc32c_arm.c" PROPERTIES COMPILE_FLAGS -march=armv8-a+crc) -elseif (AWS_ARCH_ARM32) - if (AWS_ARM32_CRC) - file(GLOB AWS_CHECKSUMS_ARCH_SRC - "${AWS_CHECKSUMS_DIR}/source/arm/*.c" - "${AWS_CHECKSUMS_DIR}/source/arm/asm/*.c" - ) - set_source_files_properties(source/arm/crc32c_arm.c PROPERTIES COMPILE_FLAGS -march=armv8-a+crc) - endif() endif() list(APPEND AWS_SOURCES ${AWS_CHECKSUMS_SRC} ${AWS_CHECKSUMS_ARCH_SRC}) @@ -248,6 +280,8 @@ elseif (OS_DARWIN) file(GLOB AWS_IO_OS_SRC "${AWS_IO_DIR}/source/bsd/*.c" "${AWS_IO_DIR}/source/posix/*.c" + # Note darwin source files leads to compile errors, ignore them + # "${AWS_IO_DIR}/source/darwin/*.c" ) endif() @@ -290,13 +324,14 @@ file(GLOB AWS_CRT_SRC "${AWS_CRT_DIR}/source/auth/*.cpp" "${AWS_CRT_DIR}/source/crypto/*.cpp" "${AWS_CRT_DIR}/source/endpoints/*.cpp" - "${AWS_CRT_DIR}/source/external/*.cpp" "${AWS_CRT_DIR}/source/http/*.cpp" "${AWS_CRT_DIR}/source/io/*.cpp" ) list(APPEND AWS_SOURCES ${AWS_CRT_SRC}) list(APPEND AWS_PUBLIC_INCLUDES "${AWS_CRT_DIR}/include/") +configure_file("${AWS_CRT_DIR}/include/aws/crt/Config.h.in" + ${CMAKE_CURRENT_BINARY_DIR}/include/aws/crt/Config.h @ONLY) # aws-c-mqtt diff --git a/contrib/aws-crt-cpp b/contrib/aws-crt-cpp index 7ff9e0343c9..b381d68ce61 160000 --- a/contrib/aws-crt-cpp +++ b/contrib/aws-crt-cpp @@ -1 +1 @@ -Subproject commit 7ff9e0343c978fc54f440b98147c2f72d304f6d8 +Subproject commit b381d68ce6177443fb427f3e8fbcdc549b1de0bf diff --git a/contrib/aws-s2n-tls b/contrib/aws-s2n-tls index 0725d3c0bb5..4654fecb05c 160000 --- a/contrib/aws-s2n-tls +++ b/contrib/aws-s2n-tls @@ -1 +1 @@ -Subproject commit 0725d3c0bb5bc1383310e19dd94c821a9234d299 +Subproject commit 4654fecb05cd5aacbda262654eb95a3876183698 diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index af54d3c3742..d57f48c2396 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -1815,12 +1815,12 @@ UniversalPageStoragePtr Context::tryGetWriteNodePageStorage() const return nullptr; } -bool Context::trySyncAllDataToRemoteStore() const +bool Context::tryUploadAllDataToRemoteStore() const { auto lock = getLock(); if (shared->ctx_disagg->isDisaggregatedStorageMode() && shared->ps_write) { - shared->ps_write->setSyncAllData(); + shared->ps_write->setUploadAllData(); return true; } return false; diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 23f1305555b..759bcb25b66 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -452,7 +452,7 @@ class Context void initializeWriteNodePageStorageIfNeed(const PathPool & path_pool); UniversalPageStoragePtr getWriteNodePageStorage() const; UniversalPageStoragePtr tryGetWriteNodePageStorage() const; - bool trySyncAllDataToRemoteStore() const; + bool tryUploadAllDataToRemoteStore() const; void tryReleaseWriteNodePageStorageForTest(); SharedContextDisaggPtr getSharedContextDisagg() const; diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp index 61c20b97eed..dba36172e07 100644 --- a/dbms/src/Server/Server.cpp +++ b/dbms/src/Server/Server.cpp @@ -679,8 +679,10 @@ class Server::TcpHttpServersHolder LOG_INFO(log, "tcp_port_secure is closed because tls config is set"); } + // No TCP server is normal now because we only enable the TCP server + // under testing deployment if (servers.empty()) - LOG_WARNING(log, "No TCP and HTTP servers are created"); + LOG_INFO(log, "No TCP server is created"); } catch (const Poco::Net::NetException & e) { @@ -705,12 +707,13 @@ class Server::TcpHttpServersHolder server->start(); } + // terminate all TCP servers when receive exit signal void onExit() { auto & config = server.config(); - LOG_DEBUG(log, "Received termination signal."); - LOG_DEBUG(log, "Waiting for current connections to close."); + LOG_INFO(log, "Received termination signal, stopping server..."); + LOG_INFO(log, "Waiting for current connections to close."); int current_connections = 0; for (auto & server : servers) @@ -718,15 +721,11 @@ class Server::TcpHttpServersHolder server->stop(); current_connections += server->currentConnections(); } - String debug_msg = "Closed all listening sockets."; - - if (current_connections) - LOG_DEBUG(log, "{} Waiting for {} outstanding connections.", debug_msg, current_connections); - else - LOG_DEBUG(log, debug_msg); + String debug_msg = "Closed all listening sockets."; if (current_connections) { + LOG_INFO(log, "{} Waiting for {} outstanding connections.", debug_msg, current_connections); const int sleep_max_ms = 1000 * config.getInt("shutdown_wait_unfinished", 5); const int sleep_one_ms = 100; int sleep_current_ms = 0; @@ -741,18 +740,21 @@ class Server::TcpHttpServersHolder std::this_thread::sleep_for(std::chrono::milliseconds(sleep_one_ms)); } } + else + { + LOG_INFO(log, debug_msg); + } debug_msg = "Closed connections."; - if (current_connections) - LOG_DEBUG( + LOG_INFO( log, "{} But {} remains." " Tip: To increase wait time add to config: 60", debug_msg, current_connections); else - LOG_DEBUG(log, debug_msg); + LOG_INFO(log, debug_msg); } private: diff --git a/dbms/src/Storages/KVStore/FFI/ProxyFFIStatusService.cpp b/dbms/src/Storages/KVStore/FFI/ProxyFFIStatusService.cpp index a2b076d043a..ff561b89310 100644 --- a/dbms/src/Storages/KVStore/FFI/ProxyFFIStatusService.cpp +++ b/dbms/src/Storages/KVStore/FFI/ProxyFFIStatusService.cpp @@ -21,6 +21,8 @@ #include #include #include +#include +#include #include #include @@ -55,7 +57,8 @@ HttpRequestRes HandleHttpRequestSyncStatus( status = HttpRequestStatus::ErrorParam; return HttpRequestRes{ .status = status, - .res = CppStrWithView{.inner = GenRawCppPtr(), .view = BaseBuffView{nullptr, 0}}}; + .res = CppStrWithView{.inner = GenRawCppPtr(), .view = BaseBuffView{nullptr, 0}}, + }; } @@ -79,7 +82,8 @@ HttpRequestRes HandleHttpRequestSyncStatus( if (status != HttpRequestStatus::Ok) return HttpRequestRes{ .status = status, - .res = CppStrWithView{.inner = GenRawCppPtr(), .view = BaseBuffView{nullptr, 0}}}; + .res = CppStrWithView{.inner = GenRawCppPtr(), .view = BaseBuffView{nullptr, 0}}, + }; } auto & tmt = *server->tmt; @@ -143,8 +147,9 @@ HttpRequestRes HandleHttpRequestSyncStatus( return HttpRequestRes{ .status = status, .res = CppStrWithView{ - .inner = GenRawCppPtr(s, RawCppPtrTypeImpl::String), - .view = BaseBuffView{s->data(), s->size()}}}; + .inner = GenRawCppPtr(s, RawCppPtrTypeImpl::String), .view = BaseBuffView{s->data(), s->size()}, + }, + }; } // Return store status of this tiflash node @@ -158,13 +163,14 @@ HttpRequestRes HandleHttpRequestStoreStatus( auto * name = RawCppString::New(IntoStoreStatusName(server->tmt->getStoreStatus(std::memory_order_relaxed))); return HttpRequestRes{ .status = HttpRequestStatus::Ok, - .res = CppStrWithView{ - .inner = GenRawCppPtr(name, RawCppPtrTypeImpl::String), - .view = BaseBuffView{name->data(), name->size()}}}; + .res= CppStrWithView{ + .inner = GenRawCppPtr(name, RawCppPtrTypeImpl::String), .view = BaseBuffView{name->data(), name->size()}, + }, + }; } /// set a flag for upload all PageData to remote store from local UniPS -HttpRequestRes HandleHttpRequestRemoteStoreSync( +HttpRequestRes HandleHttpRequestRemoteReUpload( EngineStoreServerWrap * server, std::string_view, const std::string &, @@ -179,17 +185,95 @@ HttpRequestRes HandleHttpRequestRemoteStoreSync( magic_enum::enum_name(global_ctx.getSharedContextDisagg()->disaggregated_mode))); return HttpRequestRes{ .status = HttpRequestStatus::ErrorParam, - .res - = CppStrWithView{.inner = GenRawCppPtr(body, RawCppPtrTypeImpl::String), .view = BaseBuffView{body->data(), body->size()}}, + .res = CppStrWithView{ + .inner = GenRawCppPtr(body, RawCppPtrTypeImpl::String), + .view = BaseBuffView{body->data(), body->size()}, + }, }; } - bool flag_set = global_ctx.trySyncAllDataToRemoteStore(); + bool flag_set = global_ctx.tryUploadAllDataToRemoteStore(); auto * body = RawCppString::New(fmt::format(R"json({{"message":"flag_set={}"}})json", flag_set)); return HttpRequestRes{ .status = flag_set ? HttpRequestStatus::Ok : HttpRequestStatus::ErrorParam, - .res - = CppStrWithView{.inner = GenRawCppPtr(body, RawCppPtrTypeImpl::String), .view = BaseBuffView{body->data(), body->size()}}, + .res = CppStrWithView{ + .inner = GenRawCppPtr(body, RawCppPtrTypeImpl::String), + .view = BaseBuffView{body->data(), body->size()}, + }, + }; +} + +// get the remote gc owner info +HttpRequestRes HandleHttpRequestRemoteOwnerInfo( + EngineStoreServerWrap * server, + std::string_view, + const std::string &, + std::string_view, + std::string_view) +{ + const auto & owner = server->tmt->getS3GCOwnerManager(); + const auto owner_info = owner->getOwnerID(); + auto * body = RawCppString::New(fmt::format( + R"json({{"status":"{}","owner_id":"{}"}})json", + magic_enum::enum_name(owner_info.status), + owner_info.owner_id)); + return HttpRequestRes{ + .status = HttpRequestStatus::Ok, + .res = CppStrWithView{ + .inner = GenRawCppPtr(body, RawCppPtrTypeImpl::String), + .view = BaseBuffView{body->data(), body->size()}, + }, + }; +} + +// resign if this node is the remote gc owner +HttpRequestRes HandleHttpRequestRemoteOwnerResign( + EngineStoreServerWrap * server, + std::string_view, + const std::string &, + std::string_view, + std::string_view) +{ + const auto & owner = server->tmt->getS3GCOwnerManager(); + bool has_resign = owner->resignOwner(); + String msg = has_resign ? "Done" : "This node is not the remote gc owner, can't be resigned."; + auto * body = RawCppString::New(fmt::format(R"json({{"message":"{}"}})json", msg)); + return HttpRequestRes{ + .status = HttpRequestStatus::Ok, + .res = CppStrWithView{ + .inner = GenRawCppPtr(body, RawCppPtrTypeImpl::String), + .view = BaseBuffView{body->data(), body->size()}, + }, + }; +} + +// execute remote gc if this node is the remote gc owner +HttpRequestRes HandleHttpRequestRemoteGC( + EngineStoreServerWrap * server, + std::string_view, + const std::string &, + std::string_view, + std::string_view) +{ + const auto & owner = server->tmt->getS3GCOwnerManager(); + const auto owner_info = owner->getOwnerID(); + bool gc_is_executed = false; + if (owner_info.status == OwnerType::IsOwner) + { + server->tmt->getS3GCManager()->wake(); + gc_is_executed = true; + } + auto * body = RawCppString::New(fmt::format( + R"json({{"status":"{}","owner_id":"{}","execute":"{}"}})json", + magic_enum::enum_name(owner_info.status), + owner_info.owner_id, + gc_is_executed)); + return HttpRequestRes{ + .status = HttpRequestStatus::Ok, + .res = CppStrWithView{ + .inner = GenRawCppPtr(body, RawCppPtrTypeImpl::String), + .view = BaseBuffView{body->data(), body->size()}, + }, }; } @@ -203,7 +287,10 @@ using HANDLE_HTTP_URI_METHOD = HttpRequestRes (*)( static const std::map AVAILABLE_HTTP_URI = { {"/tiflash/sync-status/", HandleHttpRequestSyncStatus}, {"/tiflash/store-status", HandleHttpRequestStoreStatus}, - {"/tiflash/sync-remote-store", HandleHttpRequestRemoteStoreSync}, + {"/tiflash/remote/owner/info", HandleHttpRequestRemoteOwnerInfo}, + {"/tiflash/remote/owner/resign", HandleHttpRequestRemoteOwnerResign}, + {"/tiflash/remote/gc", HandleHttpRequestRemoteGC}, + {"/tiflash/remote/upload", HandleHttpRequestRemoteReUpload}, }; uint8_t CheckHttpUriAvailable(BaseBuffView path_) @@ -239,7 +326,8 @@ HttpRequestRes HandleHttpRequest( } return HttpRequestRes{ .status = HttpRequestStatus::ErrorParam, - .res = CppStrWithView{.inner = GenRawCppPtr(), .view = BaseBuffView{nullptr, 0}}}; + .res = CppStrWithView{.inner = GenRawCppPtr(), .view = BaseBuffView{nullptr, 0}}, + }; } } // namespace DB diff --git a/dbms/src/Storages/KVStore/TMTContext.cpp b/dbms/src/Storages/KVStore/TMTContext.cpp index 2d785b33a45..409397b6e32 100644 --- a/dbms/src/Storages/KVStore/TMTContext.cpp +++ b/dbms/src/Storages/KVStore/TMTContext.cpp @@ -190,8 +190,8 @@ TMTContext::TMTContext( magic_enum::enum_name(remote_gc_config.method)); } } - remote_gc_config.interval_seconds - = context.getSettingsRef().remote_gc_interval_seconds; // TODO: make it reloadable + // TODO: make it reloadable + remote_gc_config.interval_seconds = context.getSettingsRef().remote_gc_interval_seconds; remote_gc_config.verify_locks = context.getSettingsRef().remote_gc_verify_consistency > 0; // set the gc_method so that S3LockService can set tagging when create delmark S3::ClientFactory::instance().gc_method = remote_gc_config.method; diff --git a/dbms/src/Storages/KVStore/TMTContext.h b/dbms/src/Storages/KVStore/TMTContext.h index 0be5cbcc331..0cab37f847c 100644 --- a/dbms/src/Storages/KVStore/TMTContext.h +++ b/dbms/src/Storages/KVStore/TMTContext.h @@ -112,6 +112,8 @@ class TMTContext : private boost::noncopyable const OwnerManagerPtr & getS3GCOwnerManager() const; + const S3::S3GCManagerServicePtr & getS3GCManager() const { return s3gc_manager; } + S3::S3LockClientPtr getS3LockClient() const { return s3lock_client; } MPPTaskManagerPtr getMPPTaskManager(); diff --git a/dbms/src/Storages/Page/V3/Universal/UniversalPageStorageService.cpp b/dbms/src/Storages/Page/V3/Universal/UniversalPageStorageService.cpp index 29a22816402..992b54202fb 100644 --- a/dbms/src/Storages/Page/V3/Universal/UniversalPageStorageService.cpp +++ b/dbms/src/Storages/Page/V3/Universal/UniversalPageStorageService.cpp @@ -98,9 +98,9 @@ bool CheckpointUploadFunctor::operator()(const PS::V3::LocalCheckpointFiles & ch return remote_store->putCheckpointFiles(checkpoint, store_id, sequence); } -void UniversalPageStorageService::setSyncAllData() +void UniversalPageStorageService::setUploadAllData() { - sync_all_at_next_upload = true; + upload_all_at_next_upload = true; gc_handle->wake(); LOG_INFO(log, "sync_all flag is set, next checkpoint will upload all existing data"); } @@ -144,10 +144,10 @@ bool UniversalPageStorageService::uploadCheckpoint() return false; } auto s3lock_client = tmt.getS3LockClient(); - const bool force_sync = sync_all_at_next_upload.load(); - bool upload_done = uploadCheckpointImpl(store_info, s3lock_client, remote_store, force_sync); - if (force_sync && upload_done) - sync_all_at_next_upload = false; + const bool force_upload = upload_all_at_next_upload.load(); + bool upload_done = uploadCheckpointImpl(store_info, s3lock_client, remote_store, force_upload); + if (force_upload && upload_done) + upload_all_at_next_upload = false; // always return false to run at fixed rate return false; } diff --git a/dbms/src/Storages/Page/V3/Universal/UniversalPageStorageService.h b/dbms/src/Storages/Page/V3/Universal/UniversalPageStorageService.h index 8563ea4a6c0..04b4bf0a225 100644 --- a/dbms/src/Storages/Page/V3/Universal/UniversalPageStorageService.h +++ b/dbms/src/Storages/Page/V3/Universal/UniversalPageStorageService.h @@ -52,7 +52,7 @@ class UniversalPageStorageService final bool uploadCheckpoint(); // Set a flag for sync all data to remote store at next checkpoint - void setSyncAllData(); + void setUploadAllData(); UniversalPageStoragePtr getUniversalPageStorage() const { return uni_page_storage; } ~UniversalPageStorageService(); @@ -90,7 +90,7 @@ class UniversalPageStorageService final // Once this flag is set, all data will be synced to remote store at next time // `uploadCheckpoint` is called. - std::atomic_bool sync_all_at_next_upload{false}; + std::atomic_bool upload_all_at_next_upload{false}; Context & global_context; UniversalPageStoragePtr uni_page_storage; diff --git a/dbms/src/Storages/S3/S3Common.cpp b/dbms/src/Storages/S3/S3Common.cpp index 2ad7e6528d6..e929dd39fa5 100644 --- a/dbms/src/Storages/S3/S3Common.cpp +++ b/dbms/src/Storages/S3/S3Common.cpp @@ -755,10 +755,12 @@ bool ensureLifecycleRuleExist(const TiFlashS3Client & client, Int32 expire_days) { const auto & error = outcome.GetError(); // The life cycle is not added at all - if (error.GetExceptionName() == "NoSuchLifecycleConfiguration") + if (error.GetErrorType() == Aws::S3::S3Errors::RESOURCE_NOT_FOUND + || error.GetExceptionName() == "NoSuchLifecycleConfiguration") { break; } + LOG_WARNING( client.log, "GetBucketLifecycle fail, please check the bucket lifecycle configuration or create the lifecycle rule " diff --git a/dbms/src/Storages/S3/S3Common.h b/dbms/src/Storages/S3/S3Common.h index f7295901b69..191d5d0a29d 100644 --- a/dbms/src/Storages/S3/S3Common.h +++ b/dbms/src/Storages/S3/S3Common.h @@ -44,8 +44,9 @@ namespace DB::S3 inline String S3ErrorMessage(const Aws::S3::S3Error & e) { return fmt::format( - " s3error={} s3msg={} request_id={}", + " s3error={} s3exception_name={} s3msg={} request_id={}", magic_enum::enum_name(e.GetErrorType()), + e.GetExceptionName(), e.GetMessage(), e.GetRequestId()); } diff --git a/dbms/src/Storages/S3/S3GCManager.cpp b/dbms/src/Storages/S3/S3GCManager.cpp index 7c571953cc6..0c660e2ca28 100644 --- a/dbms/src/Storages/S3/S3GCManager.cpp +++ b/dbms/src/Storages/S3/S3GCManager.cpp @@ -765,4 +765,12 @@ void S3GCManagerService::shutdown() } } +void S3GCManagerService::wake() const +{ + if (timer) + { + timer->wake(); + } +} + } // namespace DB::S3 diff --git a/dbms/src/Storages/S3/S3GCManager.h b/dbms/src/Storages/S3/S3GCManager.h index 1d097bb4644..b411c93f049 100644 --- a/dbms/src/Storages/S3/S3GCManager.h +++ b/dbms/src/Storages/S3/S3GCManager.h @@ -165,6 +165,8 @@ class S3GCManagerService void shutdown(); + void wake() const; + private: Context & global_ctx; std::unique_ptr manager;