Implementation of AWS S3 as a caching backend

This commit implements the usage of S3 as a cache backend for bazel.
This backend acts in a similar fashion to the general HTTP and google
datastorage backends.

.gitattributes

@@ -4,3 +4,8 @@ site/* linguist-documentation
 
 # Files that should not use CRLF line endings, even on Windows.
 tools/genrule/genrule-setup.sh -text
+third_party/aws-sig-v4-test-suite/**/*.authz -text
+third_party/aws-sig-v4-test-suite/**/*.creq -text
+third_party/aws-sig-v4-test-suite/**/*.req -text
+third_party/aws-sig-v4-test-suite/**/*.sreq -text
+third_party/aws-sig-v4-test-suite/**/*.sts -text

scripts/bootstrap/compile.sh

@@ -44,7 +44,7 @@ if [ "$ERROR_PRONE_INDEX" -lt "$GUAVA_INDEX" ]; then
   LIBRARY_JARS="${LIBRARY_JARS_ARRAY[*]}"
 fi
 
-DIRS=$(echo src/{java_tools/singlejar/java/com/google/devtools/build/zip,main/java,tools/xcode-common/java/com/google/devtools/build/xcode/{common,util}} tools/java/runfiles third_party/java/dd_plist/java ${OUTPUT_DIR}/src)
+DIRS=$(echo src/{java_tools/singlejar/java/com/google/devtools/build/zip,main/java,tools/xcode-common/java/com/google/devtools/build/xcode/{common,util}} tools/java/runfiles third_party/java/dd_plist/java third_party/aws-sdk-auth-lite ${OUTPUT_DIR}/src)
 EXCLUDE_FILES="src/main/java/com/google/devtools/build/lib/server/GrpcServerImpl.java src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/testing/*"
 # Exclude whole directories under the bazel src tree that bazel itself
 # doesn't depend on.

site/docs/remote-caching.md

@@ -18,6 +18,7 @@ make builds significantly faster.
     * [nginx](#nginx)
     * [Bazel Remote Cache](#bazel-remote-cache)
     * [Google Cloud Storage](#google-cloud-storage)
+    * [AWS S3 Storage](#aws-s3-storage)
     * [Other servers](#other-servers)
 * [Authentication](#authentication)
 * [HTTP Caching Protocol](#http-caching-protocol)
@@ -98,6 +99,7 @@ include:
 * [nginx](#nginx)
 * [Bazel Remote Cache](#bazel-remote-cache)
 * [Google Cloud Storage](#google-cloud-storage)
+* [AWS S3 Storage](#aws-s3-storage)
 
 ### nginx
 
@@ -170,11 +172,59 @@ to/from your GCS bucket.
 5. You can configure Cloud Storage to automatically delete old files. To do so, see
 [Managing Object Lifecycles](https://cloud.google.com/storage/docs/managing-lifecycles).
 
+### AWS S3 Storage
+
+[AWS S3] is a fully managed object store which provides an
+HTTP API that is compatible with Bazel's remote caching protocol. It requires
+that you have an AWS account.
+
+To use S3 as the cache:
+
+1. [Create a bucket](https://docs.aws.amazon.com/AmazonS3/latest/gsg/CreatingABucket.html).
+Ensure that you select a bucket location that's closest to you, as network bandwidth
+is important for the remote cache.
+
+2. Generate the relevant auth token and pass it to Bazel for authentication.
+Store the key securely, as anyone with the key can read and write arbitrary data
+to/from your S3 bucket.
+
+3. Connect to S3 by adding the following flags to your Bazel command:
+   * Pass the following URL to Bazel by using the flag:
+     `--remote_http_cache=https://bucket-name.s3-website-region.amazonaws.com`
+     where `bucket-name` is the name of your storage bucket, and `region` is the
+     name of the chosen region. You will need to use the HTTP/HTTPS url that
+     points to your bucket. Presently `s3://` style urls are _not_ supported due
+     to a lack of any consistent standard.
+   * Pass the authentication key using the flags:
+     `--aws_access_key_id=$ACCESS_KEY`
+     `--aws_secret_access_key=$SECRET_KEY`
+     Alternatively, AWS credentials can be configured using standard methods
+     such as instance roles and environment variables. Use this flag to
+     authenticate with instance credentials: `--aws_default_credentials`
+     A specific credentials profile can be specified using `--aws_profile=$PROFILE`
+
+4. You can configure S3 to automatically delete old files. To do so, see
+[Object Lifecycle Management](https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html)
+
+#### Virtual hosted and accelerated buckets
+
+Buckets that have been configured under a virtual host name, or are using the s3
+cloudfront acceleration proxy require some additional configuration.
+
+These buckets cannot determine the region that should be used from the URL alone.
+to specify the region for these buckets directly use the
+`--aws_region=$AWS_REGION` flag.
+
+For those using s3 accelerated buckets, the remote http cache url will be of the
+following form:
+`--remote_http_cache=https://bucket-name.s3-accelerate.amazonaws.com` where
+`bucket-name` is the name of the bucket to use.
+
 ### Other servers
 
 You can set up any HTTP/1.1 server that supports PUT and GET as the cache's
 backend. Users have reported success with caching backends such as [Hazelcast],
-[Apache httpd], and [AWS S3].
+and [Apache httpd].
 
 ## Authentication
 

src/main/java/com/google/devtools/build/lib/authandtls/AuthAndTLSOptions.java

@@ -61,6 +61,49 @@ public class AuthAndTLSOptions extends OptionsBase {
   )
   public String googleCredentials;
 
+  @Option(
+    name = "aws_default_credentials",
+    defaultValue = "false",
+    category = "remote",
+    documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
+    effectTags = {OptionEffectTag.UNKNOWN},
+    help =
+        "Whether to use 'AWS Default Credentials' for authentication."
+            + "See https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html"
+            + " for details. Disabled by default."
+  )
+  public boolean useAwsDefaultCredentials;
+
+  @Option(
+    name = "aws_access_key_id",
+    defaultValue = "null",
+    category = "remote",
+    documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
+    effectTags = {OptionEffectTag.UNKNOWN},
+    help = "Use a specific AWS_ACCESS_KEY_ID for authentication"
+  )
+  public String awsAccessKeyId;
+
+  @Option(
+    name = "aws_secret_access_key",
+    defaultValue = "null",
+    category = "remote",
+    documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
+    effectTags = {OptionEffectTag.UNKNOWN},
+    help = "Use a specific AWS_SECRET_ACCESS_KEY for authentication"
+  )
+  public String awsSecretAccessKey;
+
+  @Option(
+    name = "aws_profile",
+    defaultValue = "null",
+    category = "remote",
+    documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
+    effectTags = {OptionEffectTag.UNKNOWN},
+    help = "Use a specific profile for credentials"
+  )
+  public String awsProfile;
+
   @Option(
     name = "tls_enabled",
     defaultValue = "false",

src/main/java/com/google/devtools/build/lib/authandtls/AwsAuthUtils.java

@@ -0,0 +1,55 @@
+// Copyright 2018 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.authandtls;
+
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.AWSCredentialsProviderChain;
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
+import com.amazonaws.auth.profile.ProfileCredentialsProvider;
+import com.google.common.collect.ImmutableList;
+
+import java.io.IOException;
+import java.util.List;
+
+/** Utility methods for using {@link AuthAndTLSOptions} with Amazon Web Services. */
+public final class AwsAuthUtils {
+
+  /**
+   * Create a new {@link com.amazonaws.auth.AWSCredentialsProvider} object.
+   *
+   * @throws IOException in case the credentials can't be constructed.
+   */
+  public static AWSCredentialsProvider newCredentials(AuthAndTLSOptions options) throws IOException {
+    final ImmutableList.Builder<AWSCredentialsProvider> creds = ImmutableList.builder();
+
+    if (options.awsAccessKeyId != null && options.awsSecretAccessKey != null) {
+      final BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(options.awsAccessKeyId, options.awsSecretAccessKey);
+      creds.add(new AWSStaticCredentialsProvider(basicAWSCredentials));
+    }
+
+    if (options.awsProfile != null) {
+      creds.add(new ProfileCredentialsProvider(options.awsProfile));
+    }
+
+    if (options.useAwsDefaultCredentials) {
+      creds.add(DefaultAWSCredentialsProviderChain.getInstance());
+    }
+
+    final List<AWSCredentialsProvider> providers = creds.build();
+    return providers.isEmpty() ? null : new AWSCredentialsProviderChain(providers);
+  }
+}

src/main/java/com/google/devtools/build/lib/authandtls/BUILD

@@ -15,6 +15,7 @@ java_library(
         "//third_party:guava",
         "//third_party:jsr305",
         "//third_party:netty",
+        "//third_party/aws-sdk-auth-lite",
         "//third_party/grpc:grpc-jar",
     ],
 )

src/main/java/com/google/devtools/build/lib/remote/BUILD

@@ -36,6 +36,7 @@ java_library(
         "//third_party:auth",
         "//third_party:guava",
         "//third_party:netty",
+        "//third_party/aws-sdk-auth-lite",
         "//third_party/grpc:grpc-jar",
         "//third_party/protobuf:protobuf_java",
         "//third_party/protobuf:protobuf_java_util",

src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java

@@ -260,7 +260,7 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
                 remoteOptions,
                 SimpleBlobStoreFactory.create(
                     remoteOptions,
-                    GoogleAuthUtils.newCredentials(authAndTlsOptions),
+                    authAndTlsOptions,
                     env.getWorkingDirectory()),
                 retrier,
                 digestUtil);

src/main/java/com/google/devtools/build/lib/remote/RemoteOptions.java

@@ -31,7 +31,7 @@ public final class RemoteOptions extends OptionsBase {
     documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
     effectTags = {OptionEffectTag.UNKNOWN},
     help =
-        "A base URL of a HTTP caching service. Both http:// and https:// are supported. BLOBs are "
+        "A base URL of a HTTP caching service. Both http:// https:// are supported. BLOBs are "
             + "stored with PUT and retrieved with GET. See remote/README.md for more information."
   )
   public String remoteHttpCache;
@@ -47,6 +47,18 @@ public final class RemoteOptions extends OptionsBase {
   )
   public String remoteCacheProxy;
 
+    @Option(
+      name = "remote_s3_region",
+      defaultValue = "null",
+      category = "remote",
+      documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
+      effectTags = {OptionEffectTag.UNKNOWN},
+      help =
+          "The specific region for an S3 bucket, used as a HTTP REST cache"
+              + ". See remote/README.md for more information."
+  )
+  public String awsS3Region;
+
   @Option(
       name = "remote_max_connections",
       defaultValue = "100",

src/main/java/com/google/devtools/build/lib/remote/SimpleBlobStoreFactory.java

@@ -16,10 +16,15 @@
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
+import com.amazonaws.auth.AWSCredentialsProvider;
 import com.google.auth.Credentials;
+import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions;
+import com.google.devtools.build.lib.authandtls.AwsAuthUtils;
+import com.google.devtools.build.lib.authandtls.GoogleAuthUtils;
 import com.google.devtools.build.lib.remote.blobstore.OnDiskBlobStore;
 import com.google.devtools.build.lib.remote.blobstore.SimpleBlobStore;
 import com.google.devtools.build.lib.remote.blobstore.http.HttpBlobStore;
+import com.google.devtools.build.lib.remote.blobstore.http.HttpCredentialsAdapter;
 import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.build.lib.vfs.PathFragment;
 import io.netty.channel.unix.DomainSocketAddress;
@@ -36,7 +41,7 @@ public final class SimpleBlobStoreFactory {
 
   private SimpleBlobStoreFactory() {}
 
-  public static SimpleBlobStore createRest(RemoteOptions options, Credentials creds) {
+  public static SimpleBlobStore createRest(RemoteOptions options, AuthAndTLSOptions authAndTLSOptions) {
     try {
       URI uri = URI.create(options.remoteHttpCache);
       int timeoutMillis = (int) TimeUnit.SECONDS.toMillis(options.remoteTimeout);
@@ -45,18 +50,33 @@ public static SimpleBlobStore createRest(RemoteOptions options, Credentials cred
         if (options.remoteCacheProxy.startsWith("unix:")) {
           return HttpBlobStore.create(
             new DomainSocketAddress(options.remoteCacheProxy.replaceFirst("^unix:", "")),
-              uri, timeoutMillis, options.remoteMaxConnections, creds);
+              uri, timeoutMillis, options.remoteMaxConnections, createHttpCredentialsAdapter(options, authAndTLSOptions));
         } else {
           throw new Exception("Remote cache proxy unsupported: " + options.remoteCacheProxy);
         }
       } else {
-        return HttpBlobStore.create(uri, timeoutMillis, options.remoteMaxConnections, creds);
+        return HttpBlobStore.create(uri, timeoutMillis, options.remoteMaxConnections, createHttpCredentialsAdapter(options, authAndTLSOptions));
       }
     } catch (Exception e) {
       throw new RuntimeException(e);
     }
   }
 
+  private static HttpCredentialsAdapter createHttpCredentialsAdapter(final RemoteOptions options, final AuthAndTLSOptions authAndTLSOptions) throws IOException {
+    final Credentials googlAuth = GoogleAuthUtils.newCredentials(authAndTLSOptions);
+    final AWSCredentialsProvider awsAuth = AwsAuthUtils.newCredentials(authAndTLSOptions);
+
+    if (googlAuth != null && awsAuth != null) {
+      throw new IOException("Both Google and AWS credentials provided for remote caching. Only one should be used");
+    } else if (googlAuth != null) {
+      return HttpCredentialsAdapter.fromGoogleCredentials(googlAuth);
+    } else if (awsAuth != null) {
+      return HttpCredentialsAdapter.fromAwsCredentails(options.awsS3Region, options.remoteHttpCache, "s3", awsAuth);
+    } else {
+      return null;
+    }
+  }
+
   public static SimpleBlobStore createDiskCache(Path workingDirectory, PathFragment diskCachePath)
       throws IOException {
     Path cacheDir = workingDirectory.getRelative(checkNotNull(diskCachePath));
@@ -67,10 +87,10 @@ public static SimpleBlobStore createDiskCache(Path workingDirectory, PathFragmen
   }
 
   public static SimpleBlobStore create(
-      RemoteOptions options, @Nullable Credentials creds, @Nullable Path workingDirectory)
+          RemoteOptions options, @Nullable AuthAndTLSOptions authAndTLSOptions, @Nullable Path workingDirectory)
       throws IOException {
     if (isRestUrlOptions(options)) {
-      return createRest(options, creds);
+      return createRest(options, authAndTLSOptions);
     }
     if (workingDirectory != null && isDiskCache(options)) {
       return createDiskCache(workingDirectory, options.diskCache);

src/main/java/com/google/devtools/build/lib/remote/blobstore/http/AbstractHttpHandler.java

@@ -13,8 +13,8 @@
 // limitations under the License.
 package com.google.devtools.build.lib.remote.blobstore.http;
 
-import com.google.auth.Credentials;
 import com.google.common.base.Charsets;
+import com.google.common.collect.Multimap;
 import com.google.common.io.BaseEncoding;
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.channel.ChannelOutboundHandler;
@@ -27,16 +27,14 @@
 import java.net.SocketAddress;
 import java.net.URI;
 import java.nio.channels.ClosedChannelException;
-import java.util.List;
-import java.util.Map;
 
 /** Common functionality shared by concrete classes. */
 abstract class AbstractHttpHandler<T extends HttpObject> extends SimpleChannelInboundHandler<T>
     implements ChannelOutboundHandler {
 
-  private final Credentials credentials;
+  private final HttpCredentialsAdapter credentials;
 
-  public AbstractHttpHandler(Credentials credentials) {
+  public AbstractHttpHandler(HttpCredentialsAdapter credentials) {
     this.credentials = credentials;
   }
 
@@ -63,19 +61,10 @@ protected void addCredentialHeaders(HttpRequest request, URI uri) throws IOExcep
       request.headers().set(HttpHeaderNames.AUTHORIZATION, "Basic " + value);
       return;
     }
-    if (credentials == null || !credentials.hasRequestMetadata()) {
+    if (credentials == null || !credentials.hasRequestHeaders()) {
       return;
     }
-    Map<String, List<String>> authHeaders = credentials.getRequestMetadata(uri);
-    if (authHeaders == null || authHeaders.isEmpty()) {
-      return;
-    }
-    for (Map.Entry<String, List<String>> entry : authHeaders.entrySet()) {
-      String name = entry.getKey();
-      for (String value : entry.getValue()) {
-        request.headers().add(name, value);
-      }
-    }
+    credentials.setRequestHeaders(request);
   }
 
   protected String constructPath(URI uri, String hash, boolean isCas) {
@@ -90,7 +79,13 @@ protected String constructPath(URI uri, String hash, boolean isCas) {
   }
 
   protected String constructHost(URI uri) {
-    return uri.getHost() + ":" + uri.getPort();
+    final int port = uri.getPort();
+    final String scheme = uri.getScheme();
+    if ((scheme.equalsIgnoreCase("http") && port == 80) || (scheme.equalsIgnoreCase("https") && port == 443)) {
+      return uri.getHost();
+    } else {
+      return uri.getHost() + ":" + uri.getPort();
+    }
   }
 
   @Override