Update to more recent version of the AWS C, CRT and C++ SDK's

tensorstore/kvstore/s3_sdk/localstack_test.cc

@@ -300,7 +300,11 @@ class LocalStackFixture : public ::testing::Test {
     auto cfg = Aws::Client::ClientConfiguration{};
     cfg.endpointOverride = endpoint_url();
     cfg.region = Region();
-    auto create_client = std::make_shared<Aws::S3::S3Client>(Aws::Auth::AWSCredentials(), cfg);
+    auto create_client = std::make_shared<Aws::S3::S3Client>(
+      Aws::Auth::AWSCredentials(),
+      cfg,
+      Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always,
+      false);
 
     auto create_request = Aws::S3::Model::CreateBucketRequest{};
     create_request.SetBucket(Bucket());

tensorstore/kvstore/s3_sdk/s3_context.cc

@@ -16,6 +16,8 @@
 
 #include <cstdarg>
 #include <memory>
+#include <string_view>
+#include <vector>
 
 #include <aws/core/Aws.h>
 #include <aws/core/auth/AWSCredentialsProviderChain.h>
@@ -102,7 +104,7 @@ class CustomHttpClient : public AwsHttpClient {
         std::vector<char> buffer(absl::CordBuffer::kDefaultLimit);
         std::streampos original = body->tellg();
         while (body->read(buffer.data(), buffer.size()) || body->gcount() > 0) {
-          payload.Append(absl::string_view(buffer.data(), body->gcount()));
+          payload.Append(std::string_view(buffer.data(), body->gcount()));
         }
 
         if(payload.size() > k1MB) {
@@ -228,23 +230,35 @@ class AWSLogSystem : public AwsLogSystemInterface {
   void Log(AwsLogLevel log_level, const char* tag,
            const char* format, ...) override;
 
+  // Overridden, but prefer the safer LogStream
+  void vaLog(AwsLogLevel log_level, const char* tag,
+             const char* format, va_list args) override;
+
 private:
-  void LogMessage(AwsLogLevel log_level, const std::string & message);
+  void LogMessage(AwsLogLevel log_level, std::string_view message);
   AwsLogLevel log_level_;
 };
 
-
 void AWSLogSystem::Log(AwsLogLevel log_level, const char* tag,
            const char* format, ...) {
-  char buffer[256];
+  // https://www.open-std.org/JTC1/SC22/WG14/www/docs/n1570.pdf
+  // Section 7.16
   va_list args;
   va_start(args, format);
-  vsnprintf(buffer, 256, format, args);
+  vaLog(log_level, tag, format, args);
   va_end(args);
+}
+
+void AWSLogSystem::vaLog(AwsLogLevel log_level, const char* tag,
+           const char* format, va_list args) {
+  // https://www.open-std.org/JTC1/SC22/WG14/www/docs/n1570.pdf
+  // Section 7.16
+  char buffer[256];
+  vsnprintf(buffer, 256, format, args);
   LogMessage(log_level, buffer);
 }
 
-void AWSLogSystem::LogMessage(AwsLogLevel log_level, const std::string & message) {
+void AWSLogSystem::LogMessage(AwsLogLevel log_level, std::string_view message) {
   switch(log_level) {
     case AwsLogLevel::Info:
       ABSL_LOG(INFO) << message;

third_party/com_github_aws_c_auth/aws_c_auth.BUILD.bazel

@@ -0,0 +1,21 @@
+# Description:
+#   AWS C Auth
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])  # Apache 2.0
+
+exports_files(["LICENSE"])
+
+cc_library(
+    name = "aws_c_auth",
+    srcs = glob([
+        "include/aws/auth/**/*.h",
+        "source/*.c",
+    ]),
+    includes = ["include"],
+    deps = [
+        "@com_github_aws_c_http//:aws_c_http",
+        "@com_github_aws_c_sdkutils//:aws_c_sdkutils",
+    ],
+)

third_party/com_github_aws_c_auth/workspace.bzl

@@ -0,0 +1,34 @@
+# Copyright 2024 The TensorStore Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("//third_party:repo.bzl", "third_party_http_archive")
+load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
+
+def repo():
+    maybe(
+        third_party_http_archive,
+        name = "com_github_aws_c_auth",
+        sha256 = "f249a12a6ac319e929c005fb7efd5534c83d3af3a3a53722626ff60a494054bb",
+        strip_prefix = "aws-c-auth-0.7.22",
+        urls = [
+            "https://github.com/awslabs/aws-c-auth/archive/refs/tags/v0.7.22.tar.gz",
+        ],
+        build_file = Label("//third_party:com_github_aws_c_auth/aws_c_auth.BUILD.bazel"),
+        system_build_file = Label("//third_party:com_github_aws_c_auth/system.BUILD.bazel"),
+        cmake_name = "aws_c_auth",
+        cmake_target_mapping = {
+            "@com_github_aws_c_auth//:aws_c_auth": "aws_c_auth::aws_c_auth",
+        },
+        bazel_to_cmake = {},
+    )

third_party/com_github_aws_c_cal/aws_c_cal.BUILD.bazel

@@ -0,0 +1,30 @@
+# Description:
+#   AWS s2n tls
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])  # Apache 2.0
+
+cc_library(
+    name = "aws_c_cal",
+    srcs = glob([
+        "include/aws/cal/*.h",
+        "include/aws/cal/private/*.h",
+        "source/*.c"
+    ]) + select({
+        "@platforms//os:windows": glob([
+            "source/windows/*.c",
+        ]),
+        "@platforms//os:linux": glob([
+            "source/unix/*.c",
+        ]),
+        "@platforms//os:osx": glob([
+            "source/darwin/*.c",
+        ])
+    }),
+    includes = ["include"],
+    deps = [
+        "@com_github_aws_c_common//:aws_c_common",
+        "@com_google_boringssl//:crypto"
+    ]
+)

third_party/com_github_aws_c_cal/workspace.bzl

@@ -0,0 +1,35 @@
+# Copyright 2024 The TensorStore Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("//third_party:repo.bzl", "third_party_http_archive")
+load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
+
+
+def repo():
+    maybe(
+        third_party_http_archive,
+        name = "com_github_aws_c_cal",
+        sha256 = "9c51afbece6aa7a4a3e40b99c242884c1744d7f949a3f720cea41d247ac2d06a",
+        strip_prefix = "aws-c-cal-0.7.0",
+        urls = [
+            "https://github.com/awslabs/aws-c-cal/archive/refs/tags/v0.7.0.tar.gz",
+        ],
+        build_file = Label("//third_party:com_github_aws_c_cal/aws_c_cal.BUILD.bazel"),
+        #system_build_file = Label("//third_party:com_github_aws_c_cal/system.BUILD.bazel"),
+        cmake_name = "aws_c_cal",
+        cmake_target_mapping = {
+            "@com_github_aws_c_cal//:aws_c_cal": "aws_c_cal::aws_c_cal",
+        },
+        bazel_to_cmake = {},
+    )

third_party/com_github_aws_c_common/aws_c_common.BUILD.bazel

@@ -1,46 +1,96 @@
 # Description:
 #   AWS C Common
 
+load("@bazel_skylib//rules:write_file.bzl", "write_file")
+
 package(default_visibility = ["//visibility:public"])
 
 licenses(["notice"])  # Apache 2.0
 
 exports_files(["LICENSE"])
 
+write_file(
+    name = "write_config_h",
+    out = "include/aws/common/config.h",
+    newline = "auto",
+
+    content = [
+        "#ifndef AWS_COMMON_CONFIG_H",
+        "#define AWS_COMMON_CONFIG_H",
+        "",
+        "#define AWS_HAVE_GCC_OVERFLOW_MATH_EXTENSIONS",
+        "#define AWS_HAVE_GCC_INLINE_ASM",
+        "#define AWS_HAVE_POSIX_LARGE_FILE_SUPPORT",
+    ] + select({
+        "@platforms//os:linux": [
+            "#define AWS_HAVE_EXECINFO",
+            "#define AWS_HAVE_LINUX_IF_LINK_H",
+        ],
+        "@platforms//os:windows": [
+            "#define AWS_HAVE_WINAPI_DESKTOP",
+        ],
+    }) + [
+        "#undef AWS_HAVE_WINAPI_DESKTOP",
+        # TODO: improve with logic from AwsSIMD.cmake
+        # but this strictly requires configure style tests...
+        "#undef AWS_USE_CPU_EXTENSIONS",
+        "#undef AWS_HAVE_MSVC_INTRINSICS_X64",
+        "#undef AWS_HAVE_AVX2_INTRINSICS",
+        "#undef AWS_HAVE_AVX512_INTRINSICS",
+        "#undef AWS_HAVE_MM256_EXTRACT_EPI64",
+        "#undef AWS_HAVE_CLMUL",
+        "#undef AWS_HAVE_ARM32_CRC",
+        "#undef AWS_HAVE_ARMv8_1",
+        "#undef AWS_ARCH_ARM64",
+        "#undef AWS_ARCH_INTEL",
+        "#undef AWS_ARCH_INTEL_X64",
+        "",
+        "#endif"
+    ]
+)
+
 cc_library(
     name = "aws_c_common",
     srcs = glob([
         "include/aws/common/*.h",
+        "include/aws/common/external/*.h",
         "include/aws/common/private/*.h",
         "source/*.c",
+        "source/arch/generic/*.c",
+        "source/external/**/*.h",
+        "source/external/**/*.c",
     ]) + select({
         "@platforms//os:windows": glob([
             "source/windows/*.c",
         ]),
-        "//conditions:default": glob([
+        "@platforms//os:linux": glob([
+            "source/linux/*.c",
             "source/posix/*.c",
         ]),
+        "@platforms//os:osx": glob([
+            "source/posix/*.c",
+        ])
     }),
-    hdrs = [
-        "include/aws/common/config.h",
+    hdrs = glob([
+        "include/aws/common/*.h",
+        "include/aws/common/private/*.h",
+    ]) + [
+        ":write_config_h"
+    ],
+    defines = [
+        # TODO: improve this with logic from AwsThreadAffinity.cmake
+        "AWS_AFFINITY_METHOD=AWS_AFFINITY_METHOD_NONE",
+        # Disable macro tracing API
+        "INTEL_NO_ITTNOTIFY_API",
     ],
-    defines = [],
     includes = [
         "include",
+        "source/external",
+        "source/external/libcbor"
     ],
     textual_hdrs = glob([
         "include/**/*.inl",
     ]),
     deps = [],
 )
 
-genrule(
-    name = "config_h",
-    srcs = [
-        "include/aws/common/config.h.in",
-    ],
-    outs = [
-        "include/aws/common/config.h",
-    ],
-    cmd = "sed 's/cmakedefine/undef/g' $< > $@",
-)

third_party/com_github_aws_c_common/workspace.bzl

@@ -19,11 +19,10 @@ def repo():
     maybe(
         third_party_http_archive,
         name = "com_github_aws_c_common",
-        sha256 = "01c2a58553a37b3aa5914d9e0bf7bf14507ff4937bc5872a678892ca20fcae1f",
-        strip_prefix = "aws-c-common-0.4.29",
+        sha256 = "adf838daf6a60aa31268522105b03262d745f529bc981d3ac665424133d6f91b",
+        strip_prefix = "aws-c-common-0.9.23",
         urls = [
-            "https://storage.googleapis.com/mirror.tensorflow.org/github.com/awslabs/aws-c-common/archive/v0.4.29.tar.gz",
-            "https://github.com/awslabs/aws-c-common/archive/v0.4.29.tar.gz",
+            "https://github.com/awslabs/aws-c-common/archive/v0.9.23.tar.gz",
         ],
         build_file = Label("//third_party:com_github_aws_c_common/aws_c_common.BUILD.bazel"),
         system_build_file = Label("//third_party:com_github_aws_c_common/system.BUILD.bazel"),

third_party/com_github_aws_c_compression/aws_c_compression.BUILD.bazel

@@ -0,0 +1,20 @@
+# Description:
+#   AWS C Compression
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])  # Apache 2.0
+
+exports_files(["LICENSE"])
+
+cc_library(
+    name = "aws_c_compression",
+    srcs = glob([
+        "include/aws/compression/**/*.h",
+        "source/*.c",
+    ]),
+    includes = ["include"],
+    deps = [
+        "@com_github_aws_c_common//:aws_c_common",
+    ],
+)

third_party/com_github_aws_c_compression/workspace.bzl

@@ -0,0 +1,34 @@
+# Copyright 2024 The TensorStore Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("//third_party:repo.bzl", "third_party_http_archive")
+load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
+
+def repo():
+    maybe(
+        third_party_http_archive,
+        name = "com_github_aws_c_compression",
+        sha256 = "517c361f3b7fffca08efd5ad251a20489794f056eab0dfffacc6d5b341df8e86",
+        strip_prefix = "aws-c-compression-0.2.18",
+        urls = [
+            "https://github.com/awslabs/aws-c-compression/archive/v0.2.18.tar.gz",
+        ],
+        build_file = Label("//third_party:com_github_aws_c_compression/aws_c_compression.BUILD.bazel"),
+        system_build_file = Label("//third_party:com_github_aws_c_compression/system.BUILD.bazel"),
+        cmake_name = "aws_c_compression",
+        cmake_target_mapping = {
+            "@com_github_aws_c_compression//:aws_c_compression": "aws_c_compression::aws_c_compression",
+        },
+        bazel_to_cmake = {},
+    )

third_party/com_github_aws_c_event_stream/aws_c_event_stream.BUILD.bazel

@@ -21,5 +21,6 @@ cc_library(
     deps = [
         "@com_github_aws_c_common//:aws_c_common",
         "@com_github_aws_checksums//:aws_checksums",
+        "@com_github_aws_c_io//:aws_c_io",
     ],
 )