diff --git a/.gitattributes b/.gitattributes index 35e4e72eeb31a1..88e53f4670b235 100644 --- a/.gitattributes +++ b/.gitattributes @@ -4,3 +4,8 @@ site/* linguist-documentation # Files that should not use CRLF line endings, even on Windows. tools/genrule/genrule-setup.sh -text +third_party/aws-sig-v4-test-suite/**/*.authz -text +third_party/aws-sig-v4-test-suite/**/*.creq -text +third_party/aws-sig-v4-test-suite/**/*.req -text +third_party/aws-sig-v4-test-suite/**/*.sreq -text +third_party/aws-sig-v4-test-suite/**/*.sts -text diff --git a/scripts/bootstrap/compile.sh b/scripts/bootstrap/compile.sh index 06a98546cc7e32..4e968a1cbf4f86 100755 --- a/scripts/bootstrap/compile.sh +++ b/scripts/bootstrap/compile.sh @@ -44,7 +44,7 @@ if [ "$ERROR_PRONE_INDEX" -lt "$GUAVA_INDEX" ]; then LIBRARY_JARS="${LIBRARY_JARS_ARRAY[*]}" fi -DIRS=$(echo src/{java_tools/singlejar/java/com/google/devtools/build/zip,main/java} tools/java/runfiles third_party/java/dd_plist/java ${OUTPUT_DIR}/src) +DIRS=$(echo src/{java_tools/singlejar/java/com/google/devtools/build/zip,main/java} tools/java/runfiles third_party/java/dd_plist/java third_party/aws-sdk-auth-lite ${OUTPUT_DIR}/src) EXCLUDE_FILES="src/main/java/com/google/devtools/build/lib/server/GrpcServerImpl.java src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/testing/*" # Exclude whole directories under the bazel src tree that bazel itself # doesn't depend on. diff --git a/site/docs/remote-caching.md b/site/docs/remote-caching.md index a2c9abb4862a7c..51e34d826da60d 100644 --- a/site/docs/remote-caching.md +++ b/site/docs/remote-caching.md @@ -18,6 +18,7 @@ make builds significantly faster. * [nginx](#nginx) * [Bazel Remote Cache](#bazel-remote-cache) * [Google Cloud Storage](#google-cloud-storage) + * [AWS S3 Storage](#aws-s3-storage) * [Other servers](#other-servers) * [Authentication](#authentication) * [HTTP Caching Protocol](#http-caching-protocol) @@ -98,6 +99,7 @@ include: * [nginx](#nginx) * [Bazel Remote Cache](#bazel-remote-cache) * [Google Cloud Storage](#google-cloud-storage) +* [AWS S3 Storage](#aws-s3-storage) ### nginx @@ -173,11 +175,59 @@ to/from your GCS bucket. 5. You can configure Cloud Storage to automatically delete old files. To do so, see [Managing Object Lifecycles](https://cloud.google.com/storage/docs/managing-lifecycles). +### AWS S3 Storage + +[AWS S3] is a fully managed object store which provides an +HTTP API that is compatible with Bazel's remote caching protocol. It requires +that you have an AWS account. + +To use S3 as the cache: + +1. [Create a bucket](https://docs.aws.amazon.com/AmazonS3/latest/gsg/CreatingABucket.html). +Ensure that you select a bucket location that's closest to you, as network bandwidth +is important for the remote cache. + +2. Generate the relevant auth token and pass it to Bazel for authentication. +Store the key securely, as anyone with the key can read and write arbitrary data +to/from your S3 bucket. + +3. Connect to S3 by adding the following flags to your Bazel command: + * Pass the following URL to Bazel by using the flag: + `--remote_http_cache=https://bucket-name.s3-website-region.amazonaws.com` + where `bucket-name` is the name of your storage bucket, and `region` is the + name of the chosen region. You will need to use the HTTP/HTTPS url that + points to your bucket. Presently `s3://` style urls are _not_ supported due + to a lack of any consistent standard. + * Pass the authentication key using the flags: + `--aws_access_key_id=$ACCESS_KEY` + `--aws_secret_access_key=$SECRET_KEY` + Alternatively, AWS credentials can be configured using standard methods + such as instance roles and environment variables. Use this flag to + authenticate with instance credentials: `--aws_default_credentials` + A specific credentials profile can be specified using `--aws_profile=$PROFILE` + +4. You can configure S3 to automatically delete old files. To do so, see +[Object Lifecycle Management](https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html) + +#### Virtual hosted and accelerated buckets + +Buckets that have been configured under a virtual host name, or are using the s3 +cloudfront acceleration proxy require some additional configuration. + +These buckets cannot determine the region that should be used from the URL alone. +to specify the region for these buckets directly use the +`--aws_region=$AWS_REGION` flag. + +For those using s3 accelerated buckets, the remote http cache url will be of the +following form: +`--remote_http_cache=https://bucket-name.s3-accelerate.amazonaws.com` where +`bucket-name` is the name of the bucket to use. + ### Other servers You can set up any HTTP/1.1 server that supports PUT and GET as the cache's backend. Users have reported success with caching backends such as [Hazelcast], -[Apache httpd], and [AWS S3]. +[AWS S3] and [Apache httpd]. ## Authentication diff --git a/src/main/java/com/google/devtools/build/lib/BUILD b/src/main/java/com/google/devtools/build/lib/BUILD index 3d7e334c2ec5e4..b0975457ccaf39 100644 --- a/src/main/java/com/google/devtools/build/lib/BUILD +++ b/src/main/java/com/google/devtools/build/lib/BUILD @@ -17,6 +17,7 @@ filegroup( "//src/main/java/com/google/devtools/build/lib/analysis/skylark/annotations:srcs", "//src/main/java/com/google/devtools/build/lib/analysis/skylark/annotations/processor:srcs", "//src/main/java/com/google/devtools/build/lib/authentication:srcs", + "//src/main/java/com/google/devtools/build/lib/authentication/aws:srcs", "//src/main/java/com/google/devtools/build/lib/authentication/google:srcs", "//src/main/java/com/google/devtools/build/lib/grpc:srcs", "//src/main/java/com/google/devtools/build/lib/bazel/repository/cache:srcs", @@ -842,6 +843,7 @@ java_library( ":bazel-rules", ":bazel/BazelRepositoryModule", ":build-base", + "//src/main/java/com/google/devtools/build/lib/authentication/aws", "//src/main/java/com/google/devtools/build/lib/authentication/google", "//src/main/java/com/google/devtools/build/lib/bazel/debug:workspace-rule-module", "//src/main/java/com/google/devtools/build/lib/buildeventservice", diff --git a/src/main/java/com/google/devtools/build/lib/authentication/aws/AwsAuthModule.java b/src/main/java/com/google/devtools/build/lib/authentication/aws/AwsAuthModule.java new file mode 100644 index 00000000000000..3fcb6fe6325f42 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/authentication/aws/AwsAuthModule.java @@ -0,0 +1,130 @@ +package com.google.devtools.build.lib.authentication.aws; + +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.AWSCredentialsProviderChain; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; +import com.amazonaws.auth.profile.ProfileCredentialsProvider; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.remote.options.RemoteOptions; +import com.google.devtools.build.lib.runtime.AuthHeaderRequest; +import com.google.devtools.build.lib.runtime.AuthHeadersProvider; +import com.google.devtools.build.lib.runtime.BlazeModule; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.runtime.CommandEnvironment; +import com.google.devtools.build.lib.runtime.ServerBuilder; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParsingResult; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import javax.annotation.Nullable; + +public class AwsAuthModule extends BlazeModule { + + private static final String SERVICE = "s3"; + private final AuthHeadersProviderDelegate delegate = new AuthHeadersProviderDelegate(); + + @Override + public void serverInit(OptionsParsingResult startupOptions, ServerBuilder builder) + throws AbruptExitException { + super.serverInit(startupOptions, builder); + builder.addAuthHeadersProvider("aws", delegate); + } + + @Override + public void beforeCommand(CommandEnvironment env) throws AbruptExitException { + delegate.setDelegate(null); + + final AwsAuthOptions opts = env.getOptions().getOptions(AwsAuthOptions.class); + final RemoteOptions remoteOpts = env.getOptions().getOptions(RemoteOptions.class); + if (remoteOpts == null || opts == null) { + return; + } + + final AwsRegion region = AwsRegion.determineRegion(opts.awsRegion, remoteOpts.remoteCache); + if (region == null) { + return; + } + + final AWSCredentialsProvider credsProvider = newCredsProvider(opts); + if (credsProvider != null) { + this.delegate.setDelegate(new AwsV4AuthHeadersProvider(region, SERVICE, credsProvider, true)); + } + } + + @Override + public Iterable> getCommandOptions(final Command command) { + return "build".equals(command.name()) + ? ImmutableList.of(AwsAuthOptions.class) + : ImmutableList.of(); + } + + @Nullable + private static AWSCredentialsProvider newCredsProvider(final AwsAuthOptions opts) + throws AbruptExitException { + final ImmutableList.Builder creds = ImmutableList.builder(); + + if (opts.awsAccessKeyId != null || opts.awsSecretAccessKey != null) { + ensure(opts.awsAccessKeyId != null, "AWS Access key provided, but missing Secret Key"); + ensure(opts.awsSecretAccessKey != null, "AWS Secret key provided, but missing Access Key"); + + final BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials( + opts.awsAccessKeyId, opts.awsSecretAccessKey); + creds.add(new AWSStaticCredentialsProvider(basicAWSCredentials)); + } + + if (opts.awsProfile != null) { + creds.add(new ProfileCredentialsProvider(opts.awsProfile)); + } + + if (opts.useAwsDefaultCredentials) { + creds.add(DefaultAWSCredentialsProviderChain.getInstance()); + } + + final List providers = creds.build(); + return providers.isEmpty() ? null : new AWSCredentialsProviderChain(providers); + } + + private static void ensure(final boolean condition, final String msg) throws AbruptExitException { + if (!condition) { + throw new AbruptExitException(msg, ExitCode.COMMAND_LINE_ERROR); + } + } + + private static class AuthHeadersProviderDelegate implements AuthHeadersProvider { + + private volatile AuthHeadersProvider delegate; + + public void setDelegate(AuthHeadersProvider delegate) { + this.delegate = delegate; + } + + @Override + public String getType() { + return delegate.getType(); + } + + @Override + public Map> getRequestHeaders(AuthHeaderRequest request) throws IOException { + Preconditions.checkState(delegate != null, "delegate has not been initialized"); + return delegate.getRequestHeaders(request); + } + + @Override + public void refresh() throws IOException { + Preconditions.checkState(delegate != null, "delegate has not been initialized"); + delegate.refresh(); + } + + @Override + public boolean isEnabled() { + return delegate != null && delegate.isEnabled(); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/authentication/aws/AwsAuthOptions.java b/src/main/java/com/google/devtools/build/lib/authentication/aws/AwsAuthOptions.java new file mode 100644 index 00000000000000..de2376d4c30f3b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/authentication/aws/AwsAuthOptions.java @@ -0,0 +1,64 @@ +package com.google.devtools.build.lib.authentication.aws; + +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionDocumentationCategory; +import com.google.devtools.common.options.OptionEffectTag; +import com.google.devtools.common.options.OptionsBase; + +public class AwsAuthOptions extends OptionsBase { + + @Option( + name = "aws_default_credentials", + defaultValue = "false", + category = "remote", + documentationCategory = OptionDocumentationCategory.UNCATEGORIZED, + effectTags = {OptionEffectTag.UNKNOWN}, + help = + "Whether to use 'AWS Default Credentials' for authentication." + + "See https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html" + + " for details. Disabled by default." + ) + public boolean useAwsDefaultCredentials; + + @Option( + name = "aws_access_key_id", + defaultValue = "null", + category = "remote", + documentationCategory = OptionDocumentationCategory.UNCATEGORIZED, + effectTags = {OptionEffectTag.UNKNOWN}, + help = "Use a specific AWS_ACCESS_KEY_ID for authentication" + ) + public String awsAccessKeyId; + + @Option( + name = "aws_secret_access_key", + defaultValue = "null", + category = "remote", + documentationCategory = OptionDocumentationCategory.UNCATEGORIZED, + effectTags = {OptionEffectTag.UNKNOWN}, + help = "Use a specific AWS_SECRET_ACCESS_KEY for authentication" + ) + public String awsSecretAccessKey; + + @Option( + name = "aws_profile", + defaultValue = "null", + category = "remote", + documentationCategory = OptionDocumentationCategory.UNCATEGORIZED, + effectTags = {OptionEffectTag.UNKNOWN}, + help = "Use a specific profile for credentials" + ) + public String awsProfile; + + @Option( + name = "aws_region", + defaultValue = "null", + category = "remote", + documentationCategory = OptionDocumentationCategory.UNCATEGORIZED, + effectTags = {OptionEffectTag.UNKNOWN}, + help = "Override AWS region detection and force a specific bucket region" + ) + public String awsRegion; + + +} diff --git a/src/main/java/com/google/devtools/build/lib/authentication/aws/AwsRegion.java b/src/main/java/com/google/devtools/build/lib/authentication/aws/AwsRegion.java new file mode 100644 index 00000000000000..1a57afb49f5b00 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/authentication/aws/AwsRegion.java @@ -0,0 +1,96 @@ +package com.google.devtools.build.lib.authentication.aws; + +import com.google.common.base.Joiner; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.ExitCode; +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +enum AwsRegion { + + US_EAST_2("us-east-2", "s3.us-east-2.amazonaws.com", "s3.dualstack.us-east-2.amazonaws.com"), + US_EAST_1("us-east-1", "s3.amazonaws.com", "s3.us-east-1.amazonaws.com", "s3-external-1.amazonaws.com", "s3.dualstack.us-east-1.amazonaws.com"), + US_WEST_1("us-west-1", "s3.us-west-1.amazonaws.com", "s3-us-west-1.amazonaws.com", "s3.dualstack.us-west-1.amazonaws.com"), + US_WEST_2("us-west-2", "s3.us-west-2.amazonaws.com", "s3-us-west-2.amazonaws.com", "s3.dualstack.us-west-2.amazonaws.com"), + CA_CENTRAL_1("ca-central-1", "s3.ca-central-1.amazonaws.com", "s3-ca-central-1.amazonaws.com", "s3.dualstack.ca-central-1.amazonaws.com"), + AP_SOUTH_1("ap-south-1", "s3.ap-south-1.amazonaws.com", "s3-ap-south-1.amazonaws.com", "s3.dualstack.ap-south-1.amazonaws.com"), + AP_NORTHEAST_2("ap-northeast-2", "s3.ap-northeast-2.amazonaws.com", "s3-ap-northeast-2.amazonaws.com", "s3.dualstack.ap-northeast-2.amazonaws.com"), + AP_NORTHEAST_3("ap-northeast-3", "s3.ap-northeast-3.amazonaws.com", "s3-ap-northeast-3.amazonaws.com", "s3.dualstack.ap-northeast-3.amazonaws.com"), + AP_SOUTHEAST_1("ap-southeast-1", "s3.ap-southeast-1.amazonaws.com", "s3-ap-southeast-1.amazonaws.com", "s3.dualstack.ap-southeast-1.amazonaws.com"), + AP_SOUTHEAST_2("ap-southeast-2", "s3.ap-southeast-2.amazonaws.com", "s3-ap-southeast-2.amazonaws.com", "s3.dualstack.ap-southeast-2.amazonaws.com"), + AP_NORTHEAST_1("ap-northeast-1", "s3.ap-northeast-1.amazonaws.com", "s3-ap-northeast-1.amazonaws.com", "s3.dualstack.ap-northeast-1.amazonaws.com"), + CN_NORTH_1("cn-north-1", "s3.cn-north-1.amazonaws.com.cn"), + CN_NORTHWEST_1("cn-northwest-1", "s3.cn-northwest-1.amazonaws.com.cn"), + EU_CENTRAL_1("eu-central-1", "s3.eu-central-1.amazonaws.com", "s3-eu-central-1.amazonaws.com", "s3.dualstack.eu-central-1.amazonaws.com"), + EU_WEST_1("eu-west-1", "s3.eu-west-1.amazonaws.com", "s3-eu-west-1.amazonaws.com", "s3.dualstack.eu-west-1.amazonaws.com"), + EU_WEST_2("eu-west-2", "s3.eu-west-2.amazonaws.com", "s3-eu-west-2.amazonaws.com", "s3.dualstack.eu-west-2.amazonaws.com"), + EU_WEST_3("eu-west-3", "s3.eu-west-3.amazonaws.com", "s3-eu-west-3.amazonaws.com", "s3.dualstack.eu-west-3.amazonaws.com"), + SA_EAST_1("sa-east-1", "s3.sa-east-1.amazonaws.com", "s3-sa-east-1.amazonaws.com", "s3.dualstack.sa-east-1.amazonaws.com"),; + + final String name; + final String[] hosts; + + AwsRegion(final String name, final String... hosts) { + this.name = name; + this.hosts = hosts; + } + + static final ImmutableMap HOST_LOOKUP; + + static { + ImmutableMap.Builder builder = ImmutableMap.builder(); + for (final AwsRegion region : AwsRegion.values()) { + for (final String host : region.hosts) { + builder.put(host, region); + } + } + HOST_LOOKUP = builder.build(); + } + + static AwsRegion forHost(final String host) { + return HOST_LOOKUP.get(host); + } + + static AwsRegion forUri(final URI uri) { + return AwsRegion.forHost(uri.getHost()); + } + + public static Collection names() { + return Arrays.stream(AwsRegion.values()) + .map(region -> region.name) + .collect(Collectors.toList()); + } + + public static AwsRegion fromName(final String s) { + return Arrays.stream(AwsRegion.values()) + .filter(x -> x.name.equalsIgnoreCase(s)) + .findFirst() + .orElse(null); + } + + public static AwsRegion determineRegion(final String knownAwsRegion, final String httpCacheUri) + throws AbruptExitException{ + + if (!Strings.isNullOrEmpty(knownAwsRegion)) { + final AwsRegion toReturn = AwsRegion.fromName(knownAwsRegion); + if (toReturn == null) { + final String msg = String.format("AWS region (%s) provided but is not a known (known=%s)", + knownAwsRegion, Joiner.on(", ").join(AwsRegion.names())); + throw new AbruptExitException(msg, ExitCode.COMMAND_LINE_ERROR); + } + return toReturn; + } + + if (!Strings.isNullOrEmpty(httpCacheUri)) { + return AwsRegion.forUri(URI.create(httpCacheUri)); + } + + return null; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/authentication/aws/AwsV4AuthHeadersProvider.java b/src/main/java/com/google/devtools/build/lib/authentication/aws/AwsV4AuthHeadersProvider.java new file mode 100644 index 00000000000000..c1d3eba68cc9a4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/authentication/aws/AwsV4AuthHeadersProvider.java @@ -0,0 +1,518 @@ +// Copyright 2019 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.authentication.aws; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.AWSSessionCredentials; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.CharMatcher; +import com.google.common.base.Charsets; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; +import com.google.common.base.Strings; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ComparisonChain; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Maps; +import com.google.common.escape.Escaper; +import com.google.common.hash.HashCode; +import com.google.common.hash.Hashing; +import com.google.common.net.HttpHeaders; +import com.google.common.net.UrlEscapers; +import com.google.devtools.build.lib.runtime.AuthHeadersProvider; +import com.google.devtools.build.lib.runtime.AuthHeaderRequest; + +import java.io.IOException; +import java.net.URI; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.Optional; +import java.util.List; +import java.util.Map; +import java.util.Collection; +import java.util.HashMap; +import java.util.Arrays; +import java.util.function.BiConsumer; +import java.util.stream.Stream; + +import static java.time.ZoneOffset.UTC; + +/** + * Provider for creating AWS credentials as Http Authentication headers + * + * This follows the specification found in + * https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html + */ +class AwsV4AuthHeadersProvider implements AuthHeadersProvider { + + private static final String AWS_SIGNATURE_VERSION = "AWS4-HMAC-SHA256"; + private static final String AWS_HDR_PREFIX = "x-amz-"; + private static final String AWS_DATE_HDR = "x-amz-date"; + private static final String AWS_STS_HEADER = "x-amz-security-token"; + private static final String AWS_CONTENT_SHA_HDR = "x-amz-content-sha256"; + private static final String UNSIGNED_PAYLOAD_SIG = "UNSIGNED-PAYLOAD"; + private static final String BARE_PAYLOAD_HASH = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + + private static final String AWS4_REQUEST = "aws4_request"; + + private static final DateTimeFormatter iso8601DateFormatter = DateTimeFormatter + .ofPattern("yyyyMMdd'T'HHmmss'Z'") + .withZone(UTC); + private static final DateTimeFormatter signDateFormatter = DateTimeFormatter + .ofPattern("yyyyMMdd") + .withZone(UTC); + + private static final Escaper QRY_PARAM_ESCAPER = UrlEscapers.urlPathSegmentEscaper(); + + private static final Splitter PARAM_SPLITTER = Splitter.on('&'); + private static final Splitter KEY_VAL_SPLITTER = Splitter.on('=').limit(1); + private static final Splitter PATH_SPLITTER = Splitter.on('/'); + private static final Joiner PARAM_JOINER = Joiner.on('&'); + private static final Joiner KEY_VAL_JOINER = Joiner.on('='); + private static final Joiner.MapJoiner HDR_JOINER = Joiner.on('\n').withKeyValueSeparator(':'); + private static final Joiner SIGN_HDR_JOINER = Joiner.on(';'); + private static final Joiner NEWLINE_JOINER = Joiner.on('\n'); + private static final Joiner PATH_JOINER = Joiner.on('/'); + private static final Joiner COMMA_JOINER = Joiner.on(','); + public static final Escaper URL_PATH_SEGMENT_ESCAPER = UrlEscapers.urlPathSegmentEscaper(); + + private final AWSCredentialsProvider awsCredsProvider; + private final AwsRegion region; + private final String service; + private final ImmutableSet headersToInclude; + private final boolean useUnsignedPayloads; + + AwsV4AuthHeadersProvider(final AwsRegion region, final String service, + final AWSCredentialsProvider awsCredsProvider, + final boolean useUnsignedPayloads, final String... additionalSignHeaders) { + this.awsCredsProvider = Preconditions.checkNotNull(awsCredsProvider); + Preconditions.checkArgument(!Strings.isNullOrEmpty(service)); + this.service = service; + this.useUnsignedPayloads = useUnsignedPayloads; + this.region = Preconditions.checkNotNull(region); + + final ImmutableSet.Builder canoicHeaderBuilder = ImmutableSet.builder(); + canoicHeaderBuilder + .add(HttpHeaders.HOST.toLowerCase()) + .add(HttpHeaders.CONTENT_LENGTH.toLowerCase()); + + Arrays.stream(additionalSignHeaders) + .map(String::toLowerCase) + .forEach(canoicHeaderBuilder::add); + this.headersToInclude = canoicHeaderBuilder.build(); + } + + @Override + public String getType() { + return "aws-sigv4"; + } + + @Override + public void refresh() throws IOException { + try { + awsCredsProvider.refresh(); + } catch (final Exception e) { + throw new IOException("Unable to refresh AWS credentials", e); + } + } + + @Override + public boolean isEnabled() { + return true; + } + + @Override + public Map> getRequestHeaders(final AuthHeaderRequest request) throws IOException { + final ZonedDateTime now = ZonedDateTime.now(UTC); + final String amzDate = now.format(iso8601DateFormatter); + final AWSCredentials creds = awsCredsProvider.getCredentials(); + + final Map> headers = new HashMap<>(); + final BiConsumer addHeader = (key, value) -> { + headers.computeIfAbsent(key, k -> new ArrayList<>()).add(value); + }; + + Optional sessionToken = Optional.empty(); + if (creds instanceof AWSSessionCredentials) { + final String stsToken = ((AWSSessionCredentials) creds).getSessionToken(); + addHeader.accept(AWS_STS_HEADER, stsToken); + sessionToken = Optional.ofNullable(stsToken); + } + + final String authHeader = authorizationHeader(creds, request, sessionToken, now, amzDate); + addHeader.accept(HttpHeaders.AUTHORIZATION, authHeader); + addHeader.accept(AWS_DATE_HDR, amzDate); + + if (useUnsignedPayloads) { + addHeader.accept(AWS_CONTENT_SHA_HDR, "UNSIGNED-PAYLOAD"); + } + + return headers; + } + + @VisibleForTesting + String authorizationHeader(final AWSCredentials creds, final AuthHeaderRequest request, + final Optional sessionToken, final ZonedDateTime date, + final String amzDate) throws IOException { + try { + final String accessKey = creds.getAWSAccessKeyId(); + final String secretKey = creds.getAWSSecretKey(); + + final String stringToSign = stringToSign(request, sessionToken, date, amzDate); + final byte[] signingKey = signature(secretKey, date, region); + final CharSequence signature = hex(hmacSHA256(signingKey, stringToSign)); + + return String.format( + "AWS4-HMAC-SHA256 Credential=%s/%s/%s/%s/%s, SignedHeaders=%s, Signature=%s", + accessKey, + date.format(signDateFormatter), + region.name, + this.service, + AWS4_REQUEST, + signedHeaders(request, sessionToken, amzDate), + signature); + } catch (NoSuchAlgorithmException nsae) { + throw new IOException("Unable to correctly initialise crypto primitives", nsae); + } + } + + @VisibleForTesting + String stringToSign(final AuthHeaderRequest request, final Optional sessionToken, + final ZonedDateTime date, final String amzDate) throws NoSuchAlgorithmException { + return NEWLINE_JOINER.join( + AWS_SIGNATURE_VERSION, + amzDate, + scope(date), + hex(sha256Hash(canonicalRequest(request, sessionToken, amzDate))) + ); + } + + @VisibleForTesting + byte[] signature(final String secretKey, final ZonedDateTime date, final AwsRegion awsRegion) + throws IOException { + final String signDate = date.format(signDateFormatter); + final byte[] dateKey = hmacSHA256("AWS4" + secretKey, signDate); + final byte[] dateRegionKey = hmacSHA256(dateKey, awsRegion.name); + final byte[] dateRegionServiceKey = hmacSHA256(dateRegionKey, this.service); + final byte[] signingKey = hmacSHA256(dateRegionServiceKey, AWS4_REQUEST); + return signingKey; + } + + @VisibleForTesting + String canonicalRequest(final AuthHeaderRequest request, final Optional sessionToken, + final String amzDate) { + final URI uri = request.uri(); + return NEWLINE_JOINER.join( + request.method().get().toUpperCase(), + canonicalURI(uri), + canonicalQueryString(uri), + canonicalHeaders(request, sessionToken, amzDate) + "\n", + signedHeaders(request, sessionToken, amzDate), + hashedPayload() + ); + } + + /** + * Binds the resulting signature to a specific date, an AWS region, and a service. + * Thus, your resulting signature will work only in the specific region and for a + * specific service. The signature is valid for seven days after the specified date. + * `date.Format() + "/" + + "/" + + "/aws4_request"` + * + * For Amazon S3, the service string is s3. For a list of region strings, see + * Regions and Endpoints in the AWS General Reference. The region column in this table + * provides the list of valid region strings. + * + * The following scope restricts the resulting signature to the us-east-1 region and Amazon S3. + * + * `20130606/us-east-1/s3/aws4_request` + * + * Note: + * Scope must use the same date that you use to compute the signing key + */ + private String scope(final ZonedDateTime date) { + return PATH_JOINER.join( + date.format(signDateFormatter), + this.region.name, + this.service, + AWS4_REQUEST + ); + } + + /** + * The URI-encoded version of the absolute path component of the URI—everything starting + * with the "/" that follows the domain name and up to the end of the string or to the + * question mark character ('?') if you have query string parameters. + * + * The URI in the following example, `/examplebucket/myphoto.jpg`, is the absolute path + * and you don't encode the "/" in the absolute path: + * + * `http://s3.amazonaws.com/examplebucket/myphoto.jpg` + * + * Note: + * You do not normalize URI paths for requests to Amazon S3. For example, you may have + * a bucket with an object named "my-object//example//photo.user". + * + * Normalizing the path changes the object name in the request to + * "my-object/example/photo.user". + * + * This is an incorrect path for that object. + */ + private CharSequence canonicalURI(final URI uri) { + return PATH_JOINER.join( + PATH_SPLITTER.splitToList(uri.getPath()) + .stream() + .map(URL_PATH_SEGMENT_ESCAPER::escape) + .iterator() + ); + } + + /** + * The URI-encoded query string parameters. + * You URI-encode name and values individually. + * You must also sort the parameters in the canonical query string alphabetically by + * key name. + * The sorting occurs after encoding. + * The query string in the following URI example is `prefix=somePrefix&marker=someMarker&max-keys=20`: + * + * `http://s3.amazonaws.com/examplebucket?prefix=somePrefix&marker=someMarker&max-keys=20` + * + * The canonical query string is as follows (line breaks are added to this example for + * readability): + * + * ```java + * UriEncode("marker")+"="+UriEncode("someMarker")+"&"+ + * UriEncode("max-keys")+"="+UriEncode("20") + "&" + + * UriEncode("prefix")+"="+UriEncode("somePrefix") + * ``` + * + * When a request targets a subresource, the corresponding query parameter value will + * be an empty string (""). + * + * For example, the following URI identifies the ACL subresource on the examplebucket + * bucket: + * `http://s3.amazonaws.com/examplebucket?acl` + * + * The CanonicalQueryString in this case is as follows: + * `UriEncode("acl") + "=" + ""` + * + * If the URI does not include a '?', there is no query string in the request, and you + * set the canonical query string to an empty string (""). You will still need to + * include the "\n". + */ + private CharSequence canonicalQueryString(final URI uri) { + final StringBuilder builder = new StringBuilder(); + + if (uri.getQuery() != null) { + final Iterator queryParams = PARAM_SPLITTER + .trimResults() + .splitToList(uri.getQuery()) + .stream() + .map((String x) -> { + if (x.contains("=")) { + // You URI-encode name and values individually. You must also sort the + // parameters in the canonical query string alphabetically by key name. + // The sorting occurs after encoding. The query string in the following URI + // example is `prefix=somePrefix&marker=someMarker&max-keys=20:` + // `http://s3.amazonaws.com/examplebucket?prefix=somePrefix&marker=someMarker&max-keys=20` + // + // The canonical query string is as follows (line breaks are added to this example for + // readability): + // ```java + // UriEncode("marker")+"="+UriEncode("someMarker")+"&"+ + // UriEncode("max-keys")+"="+UriEncode("20") + "&" + + // UriEncode("prefix")+"="+UriEncode("somePrefix") + // ``` + return KEY_VAL_JOINER.join( + KEY_VAL_SPLITTER.splitToList(x) + .stream() + .map(QRY_PARAM_ESCAPER::escape) + .iterator() + ); + } else { + // When a request targets a subresource, the corresponding query parameter + // value will be an empty string (""). For example, the following URI + // identifies the ACL subresource on the examplebucket bucket + // `http://s3.amazonaws.com/examplebucket?acl` + // The CanonicalQueryString in this case is as follows: + // `UriEncode("acl") + "=" + ""` + return KEY_VAL_JOINER.join(QRY_PARAM_ESCAPER.escape(x), ""); + } + }) + .sorted() + .iterator(); + + PARAM_JOINER.appendTo(builder, queryParams); + } else { + // If the URI does not include a '?', there is no query string in the request, and + // you set the canonical query string to an empty string (""). You will still need + // to include the newline + builder.append(""); + } + + return builder.toString(); + } + + /** + * List of request headers with their values. + * Individual header name and value pairs are separated by the newline character ("\n"). + * Header names must be in lowercase. You must sort the header names alphabetically to + * construct the string, as shown in the following example: + * + * ```java + * Lowercase()+":"+Trim()+"\n" + * Lowercase()+":"+Trim()+"\n" + * ... + * Lowercase()+":"+Trim()+"\n" + * ``` + * + * The CanonicalHeaders list must include the following: + * - HTTP host header. + * - If the Content-Type header is present in the request, you must add it to the + * CanonicalHeaders list. + * - Any x-amz-* headers that you plan to include in your request must also be added. + * For example, if you are using temporary security credentials, you need to include + * x-amz-security-token in your request. + * You must add this header in the list of CanonicalHeaders. + * + * Note: + * The x-amz-content-sha256 header is required for all AWS Signature Version 4 requests. It provides a hash of the request payload. If there is no payload, you must provide the hash of an empty string. + * + * The following is an example CanonicalHeaders string. The header names are in + * lowercase and sorted. + * + * ``` + * host:s3.amazonaws.com + * x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 + * x-amz-date:20130708T220855Z + * ``` + * + * Note: + * For the purpose of calculating an authorization signature, only the host and any + * x-amz-* headers are required; however, in order to prevent data tampering, + * you should consider including all the headers in the signature calculation. + */ + private CharSequence canonicalHeaders(final AuthHeaderRequest request, + final Optional sessionToken, final String amzDate) { + return HDR_JOINER.join(processHeaders(request, sessionToken, amzDate).iterator()); + } + + /** + * An alphabetically sorted, semicolon-separated list of lowercase request header names. + * The request headers in the list are the same headers that you included in the + * CanonicalHeaders string. For example, for the previous example, the value of + * SignedHeaders would be as follows: + * + * `host;x-amz-content-sha256;x-amz-date` + */ + private CharSequence signedHeaders(final AuthHeaderRequest request, + final Optional sessionToken, final String amzDate) { + final Stream names = processHeaders(request, sessionToken, amzDate) + .map(Map.Entry::getKey); + return SIGN_HDR_JOINER.join(names.iterator()); + } + + /** + * The hexadecimal value of the SHA256 hash of the request payload. + * `Hex(SHA256Hash()` + * + * If there is no payload in the request, you compute a hash of the empty string as follows: + * `Hex(SHA256Hash(""))` + * + * The hash returns the following value: + * `e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855` + * + * For example, when you upload an object by using a PUT request, you provide object + * data in the body. + * + * When you retrieve an object by using a GET request, you compute the empty string hash. + */ + private CharSequence hashedPayload() { + // We do not sign the payload even if we make put requests + return useUnsignedPayloads ? UNSIGNED_PAYLOAD_SIG : BARE_PAYLOAD_HASH; + } + + // Utility methods for supporting AWS signatures + //------------------------------------------------------------------------------------------ + private Stream> processHeaders(final AuthHeaderRequest request, + final Optional sessionToken, final String amzDate) { + final ListMultimap headers = ArrayListMultimap.create(request.headers().get()); + Preconditions.checkState(!headers.get("host").isEmpty()); + + if (useUnsignedPayloads && !headers.containsKey(AWS_CONTENT_SHA_HDR)) { + headers.put(AWS_CONTENT_SHA_HDR, UNSIGNED_PAYLOAD_SIG); + } + if (!headers.containsKey(AWS_DATE_HDR)) { + headers.put(AWS_DATE_HDR, amzDate); + } + sessionToken.ifPresent(token -> headers.put(AWS_STS_HEADER, token)); + + return headers + .asMap() + .entrySet() + .stream() + .filter(x -> x.getKey() != null) + .filter(x -> x.getKey().contains(AWS_HDR_PREFIX) || headersToInclude.contains(x.getKey())) + .map(this::processHeader) + .sorted((x, y) -> + ComparisonChain.start() + .compare(x.getKey(), y.getKey()) + .compare(x.getValue(), y.getValue()) + .result() + ); + } + + private Map.Entry processHeader(final Map.Entry> header) { + final String headerKey = header.getKey().toLowerCase(); + final String headerValue = processHeaderValues(header.getValue()); + return Maps.immutableEntry(headerKey, headerValue); + } + + private String processHeaderValues(final Collection values) { + return COMMA_JOINER.join(values.stream().map(this::trimSpaces).iterator()); + } + + private String trimSpaces(final String value) { + return CharMatcher.whitespace().trimAndCollapseFrom(value, ' '); + } + + private byte[] sha256Hash(final String canonicalRequest) throws NoSuchAlgorithmException { + final MessageDigest digest = MessageDigest.getInstance("SHA-256"); + return digest.digest(canonicalRequest.getBytes(Charsets.UTF_8)); + } + + private CharSequence hex(final byte[] bytes) { + return HashCode.fromBytes(bytes).toString(); + } + + byte[] hmacSHA256(final String key, final String toSign) throws IOException { + return hmacSHA256(key.getBytes(Charsets.UTF_8), toSign.getBytes(Charsets.UTF_8)); + } + + byte[] hmacSHA256(final byte[] key, final String toSign) throws IOException { + return hmacSHA256(key, toSign.getBytes(Charsets.UTF_8)); + } + + byte[] hmacSHA256(final byte[] key, final byte[] toSign) throws IOException { + return Hashing.hmacSha256(key).hashBytes(toSign).asBytes(); + } +} + + diff --git a/src/main/java/com/google/devtools/build/lib/authentication/aws/BUILD b/src/main/java/com/google/devtools/build/lib/authentication/aws/BUILD new file mode 100644 index 00000000000000..b6f214ff9ec5ea --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/authentication/aws/BUILD @@ -0,0 +1,22 @@ +package(default_visibility = ["//src:__subpackages__"]) + +filegroup( + name = "srcs", + srcs = glob(["**"]), + visibility = ["//src/main/java/com/google/devtools/build/lib:__pkg__"], +) + +java_library( + name = "aws", + srcs = glob(["*.java"]), + deps = [ + "//src/main/java/com/google/devtools/build/lib:exitcode-external", + "//src/main/java/com/google/devtools/build/lib:runtime", + "//src/main/java/com/google/devtools/build/lib:util", + "//src/main/java/com/google/devtools/build/lib/remote/options", + "//src/main/java/com/google/devtools/common/options", + "//third_party:guava", + "//third_party:jsr305", + "//third_party/aws-sdk-auth-lite", + ], +) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java b/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java index 0c04c5f2ab0e6c..2d7244d195a847 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java @@ -41,6 +41,7 @@ public final class Bazel { com.google.devtools.build.lib.runtime.BazelFileSystemModule.class, // This module needs to be loaded before any module using google authentication // As of this writing: RemoteModule and BazelBuildEventServiceModule. + com.google.devtools.build.lib.authentication.aws.AwsAuthModule.class, com.google.devtools.build.lib.authentication.google.GoogleAuthModule.class, com.google.devtools.build.lib.runtime.mobileinstall.MobileInstallModule.class, com.google.devtools.build.lib.bazel.BazelWorkspaceStatusModule.class, diff --git a/src/main/java/com/google/devtools/build/lib/remote/http/AbstractHttpHandler.java b/src/main/java/com/google/devtools/build/lib/remote/http/AbstractHttpHandler.java index 3671d14ea6d5c0..dc7700e4e836be 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/http/AbstractHttpHandler.java +++ b/src/main/java/com/google/devtools/build/lib/remote/http/AbstractHttpHandler.java @@ -79,7 +79,7 @@ protected void addCredentialHeaders(HttpRequest request, URI uri) throws IOExcep if (authHeadersProvider == null) { return; } - final AuthHeaderRequest headerRequest = new NettyAuthHeaderRequest(uri, request); + final AuthHeaderRequest headerRequest = new NettyAuthHeaderRequest(request); Map> authHeaders = authHeadersProvider.getRequestHeaders(headerRequest); if (authHeaders == null || authHeaders.isEmpty()) { return; diff --git a/src/main/java/com/google/devtools/build/lib/remote/http/NettyAuthHeaderRequest.java b/src/main/java/com/google/devtools/build/lib/remote/http/NettyAuthHeaderRequest.java index 2b2adb0f4d9a7c..4d0f58bb0e86f0 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/http/NettyAuthHeaderRequest.java +++ b/src/main/java/com/google/devtools/build/lib/remote/http/NettyAuthHeaderRequest.java @@ -12,11 +12,9 @@ */ class NettyAuthHeaderRequest implements AuthHeaderRequest { - private final URI uri; private final HttpRequest request; - NettyAuthHeaderRequest(final URI uri, final HttpRequest request) { - this.uri = uri; + NettyAuthHeaderRequest(final HttpRequest request) { this.request = request; } @@ -27,7 +25,7 @@ public boolean isHttp() { @Override public URI uri() { - return this.uri; + return URI.create(this.request.uri()); } public Optional method() { diff --git a/src/test/java/com/google/devtools/build/lib/BUILD b/src/test/java/com/google/devtools/build/lib/BUILD index 7849ac155fd597..81c2b2f4f54a05 100644 --- a/src/test/java/com/google/devtools/build/lib/BUILD +++ b/src/test/java/com/google/devtools/build/lib/BUILD @@ -41,6 +41,8 @@ filegroup( "//src/test/java/com/google/devtools/build/lib/analysis/platform:srcs", "//src/test/java/com/google/devtools/build/lib/analysis/skylark/annotations/processor:srcs", "//src/test/java/com/google/devtools/build/lib/analysis/whitelisting:srcs", + "//src/test/java/com/google/devtools/build/lib/authentication:srcs", + "//src/test/java/com/google/devtools/build/lib/authentication/aws:srcs", "//src/test/java/com/google/devtools/build/lib/bazel:srcs", "//src/test/java/com/google/devtools/build/lib/buildeventservice:srcs", "//src/test/java/com/google/devtools/build/lib/buildeventstream:srcs", diff --git a/src/test/java/com/google/devtools/build/lib/authentication/BUILD b/src/test/java/com/google/devtools/build/lib/authentication/BUILD new file mode 100644 index 00000000000000..605e6dd7380151 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/authentication/BUILD @@ -0,0 +1,11 @@ +package( + default_testonly = 1, + default_visibility = ["//src:__subpackages__"], +) + +filegroup( + name = "srcs", + testonly = 0, + srcs = glob(["**"]), + visibility = ["//src/test/java/com/google/devtools/build/lib:__pkg__"], +) diff --git a/src/test/java/com/google/devtools/build/lib/authentication/aws/AwsV4AuthHeadersProviderTest.java b/src/test/java/com/google/devtools/build/lib/authentication/aws/AwsV4AuthHeadersProviderTest.java new file mode 100644 index 00000000000000..3d51c5d09a6d8e --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/authentication/aws/AwsV4AuthHeadersProviderTest.java @@ -0,0 +1,373 @@ +// Copyright 2019 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.authentication.aws; + +import com.amazonaws.auth.*; +import com.google.common.base.CharMatcher; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableListMultimap; +import com.google.common.collect.ImmutableMultimap; +import com.google.devtools.build.lib.packages.util.ResourceLoader; +import com.google.devtools.build.lib.runtime.AuthHeaderRequest; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.IOException; +import java.net.URI; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.function.Function; + +import static com.google.common.truth.Truth.assertThat; + +@RunWith(JUnit4.class) +public class AwsV4AuthHeadersProviderTest { + + private static final String TEST_ACCESS_KEY = "AKIDEXAMPLE"; + private static final String TEST_SECRET_KEY = "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"; + private static final String TEST_SESSION_TOKEN = "AQoDYXdzEPT//////////wEXAMPLEtc764bNrC" + + "9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/" + + "qkPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HF" + + "vlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Y" + + "jc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA=="; + + private static final String TEST_REGION = "us-east-1"; + private static final String TEST_SERVICE = "service"; + + private static final String NEWLINE = "\n"; + private static final Splitter NEWLINE_SPLITTER = Splitter.on(NEWLINE).omitEmptyStrings(); + private static final Splitter REQLINE_SPLITTER = Splitter.on(CharMatcher.whitespace()); + private static final Splitter HDR_SPLITTER = Splitter.on(":"); + + private AWSCredentialsProvider basicCreds = new AWSStaticCredentialsProvider( + new BasicAWSCredentials(TEST_ACCESS_KEY, TEST_SECRET_KEY) + ); + + private AWSCredentialsProvider sessionTokenCreds = new AWSStaticCredentialsProvider( + new BasicSessionCredentials(TEST_ACCESS_KEY, TEST_SECRET_KEY, TEST_SESSION_TOKEN) + ); + + private TestVector parseTestVector(final AWSCredentialsProvider creds, final String vectorPath) throws Exception { + final String vectorName = vectorPath.substring(vectorPath.lastIndexOf('/') + 1); + + String testPath = "third_party/aws-sig-v4-test-suite/" + vectorPath; + + final String resources = ResourceLoader.readFromResources(testPath + "/" + vectorName + ".req"); + final List rawRequest = NEWLINE_SPLITTER.splitToList(resources); + final Function readTestData = (name) -> { + try { + final String testFile = testPath + "/" + vectorName + "." + name; + return ResourceLoader.readFromResources(testFile); + } catch (final IOException ioe) { + throw new RuntimeException(ioe); + } + }; + + final Iterator reqLine = REQLINE_SPLITTER.split(rawRequest.get(0)).iterator(); + final String verb = reqLine.next(); + final String resource = reqLine.next(); + @SuppressWarnings("unused") final String _version = reqLine.next(); + + final ImmutableListMultimap headers = + rawRequest.subList(1, rawRequest.size()) + .stream() + .map(HDR_SPLITTER::splitToList) + .collect(ImmutableListMultimap.toImmutableListMultimap(x -> x.get(0).toLowerCase(), x -> x.get(1))); + + final String host = headers.get("host").get(0); + final String rawUri = String.format("https://%s%s", host, resource); + final URI uri = URI.create(rawUri); + + final AuthHeaderRequest request = new AuthHeaderRequest() { + @Override + public boolean isHttp() { + return true; + } + + @Override + public Optional method() { + return Optional.of(verb); + } + + @Override + public Optional> headers() { + return Optional.of(headers); + } + + @Override + public URI uri() { + return uri; + } + }; + + final String canonicalRequest = readTestData.apply("creq"); + final String stringToSign = readTestData.apply("sts"); + final String authString = readTestData.apply("authz"); + + final ZonedDateTime date = ZonedDateTime.parse(headers.get("x-amz-date").get(0), + DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss'Z'").withZone(ZoneId.of("UTC"))); + + final AwsV4AuthHeadersProvider adapter = + new AwsV4AuthHeadersProvider(AwsRegion.fromName(TEST_REGION), TEST_SERVICE, creds, false, + "my-header1", "my-header2"); + + final Optional stsToken = Optional.empty(); + + return new TestVector(creds, adapter, request, stsToken, canonicalRequest, stringToSign, + authString, date); + } + + @Test + public void testPostVanillaVector() throws Exception { + parseTestVector(basicCreds, "post-vanilla") + .assertCanonicalRequest() + .assertStringToSign() + .assertAuthHeader(); + } + + @Test + public void getHeaderKeyDuplicate() throws Exception { + parseTestVector(basicCreds, "get-header-key-duplicate") + .assertCanonicalRequest() + .assertStringToSign() + .assertAuthHeader(); + } + + @Test + @Ignore("Right now this is dependent on the test suite to parse correctly") + public void getHeaderValueMultiline() throws Exception { + parseTestVector(basicCreds, "get-header-value-multiline") + .assertCanonicalRequest() + .assertStringToSign() + .assertAuthHeader(); + } + + @Test + public void getHeaderValueOrder() throws Exception { + parseTestVector(basicCreds, "get-header-value-order") + .assertCanonicalRequest() + .assertStringToSign() + .assertAuthHeader(); + } + + @Test + public void getHeaderValueTrim() throws Exception { + parseTestVector(basicCreds, "get-header-value-trim") + .assertCanonicalRequest() + .assertStringToSign() + .assertAuthHeader(); + } + + @Test + public void getUnreserved() throws Exception { + parseTestVector(basicCreds, "get-unreserved") + .assertCanonicalRequest() + .assertStringToSign() + .assertAuthHeader(); + } + + @Test + public void getUtf8() throws Exception { + parseTestVector(basicCreds, "get-utf8") + .assertCanonicalRequest() + .assertStringToSign() + .assertAuthHeader(); + } + + @Test + public void getVanilla() throws Exception { + parseTestVector(basicCreds, "get-vanilla") + .assertCanonicalRequest() + .assertStringToSign() + .assertAuthHeader(); + } + + @Test + public void getVanillaEmptyQueryKey() throws Exception { + parseTestVector(basicCreds, "get-vanilla-empty-query-key") + .assertCanonicalRequest() + .assertStringToSign() + .assertAuthHeader(); + } + + @Test + public void getVanillaQuery() throws Exception { + parseTestVector(basicCreds, "get-vanilla-query") + .assertCanonicalRequest() + .assertStringToSign() + .assertAuthHeader(); + } + + @Test + public void getVanillaQueryOrderKey() throws Exception { + parseTestVector(basicCreds, "get-vanilla-query-order-key") + .assertCanonicalRequest() + .assertStringToSign() + .assertAuthHeader(); + } + + @Test + public void getVanillaQueryOrderKeyCase() throws Exception { + parseTestVector(basicCreds, "get-vanilla-query-order-key-case") + .assertCanonicalRequest() + .assertStringToSign() + .assertAuthHeader(); + } + + @Test + public void getVanillaQueryOrderValue() throws Exception { + parseTestVector(basicCreds, "get-vanilla-query-order-value") + .assertCanonicalRequest() + .assertStringToSign() + .assertAuthHeader(); + } + + @Test + public void getVanillaQueryUnreserved() throws Exception { + parseTestVector(basicCreds, "get-vanilla-query-unreserved") + .assertCanonicalRequest() + .assertStringToSign() + .assertAuthHeader(); + } + + @Test + public void getVanillaUtf8Query() throws Exception { + parseTestVector(basicCreds, "get-vanilla-utf8-query") + .assertCanonicalRequest() + .assertStringToSign() + .assertAuthHeader(); + } + + @Test + public void postHeaderKeyCase() throws Exception { + parseTestVector(basicCreds, "post-header-key-case") + .assertCanonicalRequest() + .assertStringToSign() + .assertAuthHeader(); + } + + @Test + public void postHeaderKeySort() throws Exception { + parseTestVector(basicCreds, "post-header-key-sort") + .assertCanonicalRequest() + .assertStringToSign() + .assertAuthHeader(); + } + + @Test + public void postHeaderValueCase() throws Exception { + parseTestVector(basicCreds, "post-header-value-case") + .assertCanonicalRequest() + .assertStringToSign() + .assertAuthHeader(); + } + + @Test + public void postVanilla() throws Exception { + parseTestVector(basicCreds, "post-vanilla") + .assertCanonicalRequest() + .assertStringToSign() + .assertAuthHeader(); + } + + @Test + public void postVanillaEmptyQueryValue() throws Exception { + parseTestVector(basicCreds, "post-vanilla-empty-query-value") + .assertCanonicalRequest() + .assertStringToSign() + .assertAuthHeader(); + } + + @Test + public void postVanillaQuery() throws Exception { + parseTestVector(basicCreds, "post-vanilla-query") + .assertCanonicalRequest() + .assertStringToSign() + .assertAuthHeader(); + } + + @Test + public void postStsHeaderBefore() throws Exception { + parseTestVector(basicCreds, "post-sts-token/post-sts-header-before") + .assertCanonicalRequest() + .assertStringToSign() + .assertAuthHeader(); + } + + @Test + public void postStsHeaderAfter() throws Exception { + parseTestVector(sessionTokenCreds, "post-sts-token/post-sts-header-after") + .assertCanonicalRequest() + .assertStringToSign() + .assertAuthHeader(); + } + +} + +class TestVector { + private final AWSCredentialsProvider creds; + private final AwsV4AuthHeadersProvider provider; + private final AuthHeaderRequest request; + private final Optional stsToken; + private final String expectedCanonicalRequest; + private final String expectedStringToSign; + private final String expectedAuthString; + private final String amzDate; + private final ZonedDateTime date; + + TestVector(final AWSCredentialsProvider creds, final AwsV4AuthHeadersProvider provider, + final AuthHeaderRequest request, final Optional stsToken, + final String expectedCanonicalRequest, final String expectedStringToSign, + final String expectedAuthString, final ZonedDateTime date) { + this.creds = creds; + this.provider = provider; + this.request = request; + this.stsToken = stsToken; + this.expectedCanonicalRequest = expectedCanonicalRequest; + this.expectedStringToSign = expectedStringToSign; + this.expectedAuthString = expectedAuthString; + this.date = date; + this.amzDate = this.date.format(DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss'Z'")); + } + + TestVector assertCanonicalRequest() throws Exception { + final String actual = provider.canonicalRequest(request, stsToken, amzDate); + assertThat(actual) + .named("Canonical Request") + .isEqualTo(expectedCanonicalRequest); + return this; + } + + TestVector assertStringToSign() throws Exception { + final String actual = provider.stringToSign(request, stsToken, date, amzDate); + assertThat(actual) + .named("String to sign") + .isEqualTo(expectedStringToSign); + return this; + } + + TestVector assertAuthHeader() throws Exception { + final String actual = provider + .authorizationHeader(creds.getCredentials(), request, stsToken, date, amzDate); + assertThat(actual) + .named("Auth Header") + .isEqualTo(expectedAuthString); + return this; + } +} diff --git a/src/test/java/com/google/devtools/build/lib/authentication/aws/BUILD b/src/test/java/com/google/devtools/build/lib/authentication/aws/BUILD new file mode 100644 index 00000000000000..f71a6efbe60075 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/authentication/aws/BUILD @@ -0,0 +1,33 @@ +package( + default_testonly = 1, + default_visibility = ["//src:__subpackages__"], +) + +filegroup( + name = "srcs", + testonly = 0, + srcs = glob(["**"]), + visibility = ["//src/test/java/com/google/devtools/build/lib:__pkg__"], +) + +java_test( + name = "aws-authentication-tests", + srcs = ["AwsV4AuthHeadersProviderTest.java"], + resources = ["//third_party/aws-sig-v4-test-suite:testdata"], + test_class = "com.google.devtools.build.lib.AllTests", + deps = [ + "//src/main/java/com/google/devtools/build/lib:runtime", + "//src/main/java/com/google/devtools/build/lib/authentication/aws", + "//src/test/java/com/google/devtools/build/lib:packages_testutil", + "//src/test/java/com/google/devtools/build/lib:test_runner", + "//src/test/java/com/google/devtools/build/lib:testutil", + "//third_party:api_client", + "//third_party:auth", + "//third_party:guava", + "//third_party:mockito", + "//third_party:truth", + "//third_party/aws-sdk-auth-lite", + "//third_party/grpc:grpc-jar", + "//third_party/protobuf:protobuf_java", + ], +) diff --git a/third_party/BUILD b/third_party/BUILD index eab72086a8cd3c..d042f802f5e8ad 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -233,7 +233,6 @@ java_import( runtime_deps = [ ":api_client", ":guava", - "//third_party/aws-sdk-auth-lite", ], ) diff --git a/third_party/aws-sig-v4-test-suite/BUILD b/third_party/aws-sig-v4-test-suite/BUILD index 0ec0154f7d9158..b8f116d5ae1c61 100644 --- a/third_party/aws-sig-v4-test-suite/BUILD +++ b/third_party/aws-sig-v4-test-suite/BUILD @@ -9,5 +9,5 @@ filegroup( filegroup( name = "testdata", srcs = glob(["**/*"]), - visibility = ["//src/test/java/com/google/devtools/build/lib:__pkg__"], + visibility = ["//src/test/java/com/google/devtools/build/lib/authentication/aws:__pkg__"], )