Add AWS SDK (#6828)

ref #6827

.gitmodules

@@ -92,3 +92,45 @@
 [submodule "contrib/GmSSL"]
 	path = contrib/GmSSL
 	url = https://github.com/guanzhi/GmSSL.git
+[submodule "contrib/aws"]
+	path = contrib/aws
+	url = [email protected]:aws/aws-sdk-cpp.git
+[submodule "contrib/aws-c-auth"]
+	path = contrib/aws-c-auth
+	url = [email protected]:awslabs/aws-c-auth.git
+[submodule "contrib/aws-c-cal"]
+	path = contrib/aws-c-cal
+	url = [email protected]:ClickHouse/aws-c-cal.git
+[submodule "contrib/aws-c-common"]
+	path = contrib/aws-c-common
+	url = [email protected]:awslabs/aws-c-common.git
+[submodule "contrib/aws-c-compression"]
+	path = contrib/aws-c-compression
+	url = [email protected]:awslabs/aws-c-compression.git
+[submodule "contrib/aws-c-event-stream"]
+	path = contrib/aws-c-event-stream
+	url = [email protected]:awslabs/aws-c-event-stream.git
+[submodule "contrib/aws-c-http"]
+	path = contrib/aws-c-http
+	url = [email protected]:awslabs/aws-c-http.git
+[submodule "contrib/aws-c-io"]
+	path = contrib/aws-c-io
+	url = [email protected]:awslabs/aws-c-io.git
+[submodule "contrib/aws-c-mqtt"]
+	path = contrib/aws-c-mqtt
+	url = [email protected]:awslabs/aws-c-mqtt.git
+[submodule "contrib/aws-c-s3"]
+	path = contrib/aws-c-s3
+	url = [email protected]:awslabs/aws-c-s3.git
+[submodule "contrib/aws-c-sdkutils"]
+	path = contrib/aws-c-sdkutils
+	url = [email protected]:awslabs/aws-c-sdkutils.git
+[submodule "contrib/aws-checksums"]
+	path = contrib/aws-checksums
+	url = [email protected]:awslabs/aws-checksums.git
+[submodule "contrib/aws-crt-cpp"]
+	path = contrib/aws-crt-cpp
+	url = [email protected]:awslabs/aws-crt-cpp.git
+[submodule "contrib/aws-s2n-tls"]
+	path = contrib/aws-s2n-tls
+	url = [email protected]:aws/s2n-tls.git

contrib/CMakeLists.txt

@@ -177,3 +177,5 @@ if (ARCH_AARCH64 AND OS_LINUX)
 endif ()
 
 add_subdirectory(magic_enum)
+
+add_subdirectory(aws-cmake)

contrib/aws-cmake/AwsFeatureTests.cmake

@@ -0,0 +1,114 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0.
+
+include(CheckCSourceRuns)
+
+option(USE_CPU_EXTENSIONS "Whenever possible, use functions optimized for CPUs with specific extensions (ex: SSE, AVX)." ON)
+
+# In the current (11/2/21) state of mingw64, the packaged gcc is not capable of emitting properly aligned avx2 instructions under certain circumstances.
+# This leads to crashes for windows builds using mingw64 when invoking the avx2-enabled versions of certain functions.  Until we can find a better
+# work-around, disable avx2 (and all other extensions) in mingw builds.
+#
+# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=54412
+#
+if (MINGW)
+    message(STATUS "MINGW detected!  Disabling avx2 and other CPU extensions")
+    set(USE_CPU_EXTENSIONS OFF)
+endif()
+
+if(NOT CMAKE_CROSSCOMPILING)
+    check_c_source_runs("
+    #include <stdbool.h>
+    bool foo(int a, int b, int *c) {
+        return __builtin_mul_overflow(a, b, c);
+    }
+
+    int main() {
+        int out;
+        if (foo(1, 2, &out)) {
+            return 0;
+        }
+
+        return 0;
+    }" AWS_HAVE_GCC_OVERFLOW_MATH_EXTENSIONS)
+
+    if (USE_CPU_EXTENSIONS)
+        check_c_source_runs("
+        int main() {
+        int foo = 42;
+        _mulx_u32(1, 2, &foo);
+        return foo != 2;
+        }" AWS_HAVE_MSVC_MULX)
+    endif()
+
+endif()
+
+check_c_source_compiles("
+    #include <Windows.h>
+    #if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP)
+    int main() {
+        return 0;
+    }
+    #else
+    it's not windows desktop
+    #endif
+" AWS_HAVE_WINAPI_DESKTOP)
+
+check_c_source_compiles("
+    int main() {
+#if !(defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86))
+#    error \"not intel\"
+#endif
+        return 0;
+    }
+" AWS_ARCH_INTEL)
+
+check_c_source_compiles("
+    int main() {
+#if !(defined(__aarch64__) || defined(_M_ARM64))
+#    error \"not arm64\"
+#endif
+        return 0;
+    }
+" AWS_ARCH_ARM64)
+
+check_c_source_compiles("
+    int main() {
+#if !(defined(__arm__) || defined(_M_ARM))
+#    error \"not arm\"
+#endif
+        return 0;
+    }
+" AWS_ARCH_ARM32)
+
+check_c_source_compiles("
+int main() {
+    int foo = 42, bar = 24;
+    __asm__ __volatile__(\"\":\"=r\"(foo):\"r\"(bar):\"memory\");
+}" AWS_HAVE_GCC_INLINE_ASM)
+
+check_c_source_compiles("
+#include <sys/auxv.h>
+int main() {
+#ifdef __linux__
+    getauxval(AT_HWCAP);
+    getauxval(AT_HWCAP2);
+#endif
+    return 0;
+}" AWS_HAVE_AUXV)
+
+string(REGEX MATCH "^(aarch64|arm)" ARM_CPU "${CMAKE_SYSTEM_PROCESSOR}")
+if(NOT LEGACY_COMPILER_SUPPORT OR ARM_CPU)
+    check_c_source_compiles("
+    #include <execinfo.h>
+    int main() {
+        backtrace(NULL, 0);
+        return 0;
+    }" AWS_HAVE_EXECINFO)
+endif()
+
+check_c_source_compiles("
+#include <linux/if_link.h>
+int main() {
+    return 1;
+}" AWS_HAVE_LINUX_IF_LINK_H)

contrib/aws-cmake/AwsSIMD.cmake

@@ -0,0 +1,74 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0.
+
+include(CheckCCompilerFlag)
+include(CheckIncludeFile)
+
+if (USE_CPU_EXTENSIONS)
+    if (MSVC)
+        check_c_compiler_flag("/arch:AVX2" HAVE_M_AVX2_FLAG)
+        if (HAVE_M_AVX2_FLAG)
+            set(AVX2_CFLAGS "/arch:AVX2")
+        endif()
+    else()
+        check_c_compiler_flag(-mavx2 HAVE_M_AVX2_FLAG)
+        if (HAVE_M_AVX2_FLAG)
+            set(AVX2_CFLAGS "-mavx -mavx2")
+        endif()
+    endif()
+
+
+    cmake_push_check_state()
+    set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${AVX2_CFLAGS}")
+
+    check_c_source_compiles("
+        #include <immintrin.h>
+        #include <emmintrin.h>
+        #include <string.h>
+
+        int main() {
+            __m256i vec;
+            memset(&vec, 0, sizeof(vec));
+
+            _mm256_shuffle_epi8(vec, vec);
+            _mm256_set_epi32(1,2,3,4,5,6,7,8);
+            _mm256_permutevar8x32_epi32(vec, vec);
+
+            return 0;
+        }"  HAVE_AVX2_INTRINSICS)
+
+    check_c_source_compiles("
+        #include <immintrin.h>
+        #include <string.h>
+
+        int main() {
+            __m256i vec;
+            memset(&vec, 0, sizeof(vec));
+            return (int)_mm256_extract_epi64(vec, 2);
+        }" HAVE_MM256_EXTRACT_EPI64)
+
+    cmake_pop_check_state()
+endif() # USE_CPU_EXTENSIONS
+
+macro(simd_add_definition_if target definition)
+    if(${definition})
+        target_compile_definitions(${target} PRIVATE -D${definition})
+    endif(${definition})
+endmacro(simd_add_definition_if)
+
+# Configure private preprocessor definitions for SIMD-related features
+# Does not set any processor feature codegen flags
+function(simd_add_definitions target)
+    simd_add_definition_if(${target} HAVE_AVX2_INTRINSICS)
+    simd_add_definition_if(${target} HAVE_MM256_EXTRACT_EPI64)
+endfunction(simd_add_definitions)
+
+# Adds source files only if AVX2 is supported. These files will be built with
+# avx2 intrinsics enabled.
+# Usage: simd_add_source_avx2(target file1.c file2.c ...)
+function(simd_add_source_avx2 target)
+    foreach(file ${ARGN})
+        target_sources(${target} PRIVATE ${file})
+        set_source_files_properties(${file} PROPERTIES COMPILE_FLAGS "${AVX2_CFLAGS}")
+    endforeach()
+endfunction(simd_add_source_avx2)

contrib/aws-cmake/AwsThreadAffinity.cmake

@@ -0,0 +1,50 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0.
+
+include(CheckSymbolExists)
+
+# Check if the platform supports setting thread affinity
+# (important for hitting full NIC entitlement on NUMA architectures)
+function(aws_set_thread_affinity_method target)
+
+    # Non-POSIX, Android, and Apple platforms do not support thread affinity.
+    if (NOT UNIX OR ANDROID OR APPLE)
+        target_compile_definitions(${target} PRIVATE
+            -DAWS_AFFINITY_METHOD=AWS_AFFINITY_METHOD_NONE)
+        return()
+    endif()
+
+    cmake_push_check_state()
+    list(APPEND CMAKE_REQUIRED_DEFINITIONS -D_GNU_SOURCE)
+    list(APPEND CMAKE_REQUIRED_LIBRARIES pthread)
+
+    set(headers "pthread.h")
+    # BSDs put nonportable pthread declarations in a separate header.
+    if(CMAKE_SYSTEM_NAME MATCHES BSD)
+        set(headers "${headers};pthread_np.h")
+    endif()
+
+    # Using pthread attrs is the preferred method, but is glibc-specific.
+    check_symbol_exists(pthread_attr_setaffinity_np "${headers}" USE_PTHREAD_ATTR_SETAFFINITY)
+    if (USE_PTHREAD_ATTR_SETAFFINITY)
+        target_compile_definitions(${target} PRIVATE
+            -DAWS_AFFINITY_METHOD=AWS_AFFINITY_METHOD_PTHREAD_ATTR)
+        return()
+    endif()
+
+    # This method is still nonportable, but is supported by musl and BSDs.
+    check_symbol_exists(pthread_setaffinity_np "${headers}" USE_PTHREAD_SETAFFINITY)
+    if (USE_PTHREAD_SETAFFINITY)
+        target_compile_definitions(${target} PRIVATE
+            -DAWS_AFFINITY_METHOD=AWS_AFFINITY_METHOD_PTHREAD)
+        return()
+    endif()
+
+    # If we got here, we expected thread affinity support but didn't find it.
+    # We still build with degraded NUMA performance, but show a warning.
+    message(WARNING "No supported method for setting thread affinity")
+    target_compile_definitions(${target} PRIVATE
+        -DAWS_AFFINITY_METHOD=AWS_AFFINITY_METHOD_NONE)
+
+    cmake_pop_check_state()
+endfunction()

contrib/aws-cmake/AwsThreadName.cmake

@@ -0,0 +1,61 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0.
+
+include(CheckSymbolExists)
+
+# Check how the platform supports setting thread name
+function(aws_set_thread_name_method target)
+
+    if (WINDOWS)
+        # On Windows we do a runtime check, instead of compile-time check
+        return()
+    elseif (APPLE)
+        # All Apple platforms we support have the same function, so no need for compile-time check.
+        return()
+    endif()
+
+    cmake_push_check_state()
+    list(APPEND CMAKE_REQUIRED_DEFINITIONS -D_GNU_SOURCE)
+    list(APPEND CMAKE_REQUIRED_LIBRARIES pthread)
+
+    # The start of the test program
+    set(c_source_start "
+        #define _GNU_SOURCE
+        #include <pthread.h>
+
+        #if defined(__FreeBSD__) || defined(__NETBSD__)
+        #include <pthread_np.h>
+        #endif
+
+        int main() {
+            pthread_t thread_id;
+        ")
+
+    # The end of the test program
+    set(c_source_end "}")
+
+    # pthread_setname_np() usually takes 2 args
+    check_c_source_compiles("
+        ${c_source_start}
+        pthread_setname_np(thread_id, \"asdf\");
+        ${c_source_end}"
+        PTHREAD_SETNAME_TAKES_2ARGS)
+    if (PTHREAD_SETNAME_TAKES_2ARGS)
+        target_compile_definitions(${target} PRIVATE -DAWS_PTHREAD_SETNAME_TAKES_2ARGS)
+        return()
+    endif()
+
+    # But on NetBSD it takes 3!
+    check_c_source_compiles("
+        ${c_source_start}
+        pthread_setname_np(thread_id, \"asdf\", NULL);
+        ${c_source_end}
+        " PTHREAD_SETNAME_TAKES_3ARGS)
+    if (PTHREAD_SETNAME_TAKES_3ARGS)
+        target_compile_definitions(${target} PRIVATE -DAWS_PTHREAD_SETNAME_TAKES_3ARGS)
+        return()
+    endif()
+
+    # And on many older/weirder platforms it's just not supported
+    cmake_pop_check_state()
+endfunction()