-
Notifications
You must be signed in to change notification settings - Fork 4.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
WIP: Remove S3 blobstore and attempt to use Http
- Loading branch information
1 parent
3d18a08
commit 021bdd2
Showing
15 changed files
with
348 additions
and
258 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
182 changes: 182 additions & 0 deletions
182
...n/java/com/google/devtools/build/lib/remote/blobstore/http/AwsHttpCredentialsAdapter.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,182 @@ | ||
| package com.google.devtools.build.lib.remote.blobstore.http; | ||
|
|
||
| import com.amazonaws.auth.AWSCredentials; | ||
| import com.amazonaws.auth.AWSCredentialsProvider; | ||
| import com.google.common.base.Joiner; | ||
| import com.google.common.base.Preconditions; | ||
| import com.google.common.base.Splitter; | ||
| import com.google.common.collect.*; | ||
| import com.google.common.net.HttpHeaders; | ||
| import io.netty.handler.codec.http.HttpRequest; | ||
|
|
||
| import javax.crypto.Mac; | ||
| import javax.crypto.spec.SecretKeySpec; | ||
| import java.io.IOException; | ||
| import java.net.URI; | ||
| import java.text.SimpleDateFormat; | ||
| import java.util.*; | ||
|
|
||
| /** | ||
| * Adapter for converting AWS credentials into Http Authentication headers | ||
| * | ||
| * This follows the specification found in | ||
| * https://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#ConstructingTheAuthenticationHeader | ||
| * | ||
| * Which is principally the following grammar: | ||
| * | ||
| * <pre>{@code | ||
| * Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature; | ||
| * | ||
| * Signature = Base64( HMAC-SHA1( YourSecretAccessKeyID, UTF-8-Encoding-Of( StringToSign ) ) ); | ||
| * | ||
| * StringToSign = HTTP-Verb + "\n" + | ||
| * Content-MD5 + "\n" + | ||
| * Content-Type + "\n" + | ||
| * Date + "\n" + | ||
| * CanonicalizedAmzHeaders + | ||
| * CanonicalizedResource; | ||
| * | ||
| * CanonicalizedResource = [ "/" + Bucket ] + | ||
| * <HTTP-Request-URI, from the protocol name up to the query string> + | ||
| * [ subresource, if present. For example "?acl", "?location", "?logging", or "?torrent"]; | ||
| * | ||
| * CanonicalizedAmzHeaders = <described elsewhere> | ||
| * }</pre> | ||
| * | ||
| */ | ||
| public class AwsHttpCredentialsAdapter extends HttpCredentialsAdapter { | ||
|
|
||
| private static final String SIGNING_ALGO = "HmacSHA256"; | ||
| private static final String AWS_HDR_PREFIX = "x-amz-"; | ||
|
|
||
| private static final ThreadLocal<SimpleDateFormat> headerDateFormatter = | ||
| ThreadLocal.withInitial(() -> new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss z")); | ||
|
|
||
| private static final ImmutableSet<String> CANOC_PARAMETERS = ImmutableSet.of( | ||
| "acl", "torrent", "logging", "location", "policy", "requestPayment", "versioning", | ||
| "versions", "versionId", "notification", "uploadId", "uploads", "partNumber", "website", | ||
| "delete", "lifecycle", "tagging", "cors", "restore", "replication", "accelerate", | ||
| "inventory", "analytics", "metrics" | ||
| ); | ||
|
|
||
| private static final Splitter URL_SPLITTER = Splitter.on('.'); | ||
| private static final Splitter PARAM_SPLITTER = Splitter.on('&'); | ||
| private static final Joiner PARAM_JOINER = Joiner.on('&'); | ||
|
|
||
| private final AWSCredentialsProvider awsCredsProvider; | ||
|
|
||
| AwsHttpCredentialsAdapter(final AWSCredentialsProvider awsCredsProvider) { | ||
| this.awsCredsProvider = awsCredsProvider; | ||
| } | ||
|
|
||
| @Override | ||
| public void refresh() throws IOException { | ||
| awsCredsProvider.refresh(); | ||
| } | ||
|
|
||
| @Override | ||
| public Multimap<String, String> getRequestHeaders(final HttpRequest request, final URI uri) throws IOException { | ||
| final AWSCredentials creds = awsCredsProvider.getCredentials(); | ||
| final String accessKey = creds.getAWSAccessKeyId(); | ||
| final String secretKey = creds.getAWSSecretKey(); | ||
|
|
||
| final Date currentTime = new Date(System.currentTimeMillis()); | ||
| final String date = headerDateFormatter.get().format(currentTime); | ||
|
|
||
| final String stringToSign = new StringBuilder() | ||
| .append(request.method().name()) | ||
| .append('\n') | ||
| .append(canoicaliseHeaders(request, date)) | ||
| .append(canoicaliseRequest(uri)) | ||
| .toString(); | ||
|
|
||
| final String signature = generateHMac(secretKey, stringToSign); | ||
| final String authHeader = String.format("AWS %s:%s", accessKey, signature); | ||
|
|
||
| return ImmutableMultimap.<String, String>builder() | ||
| .put(HttpHeaders.AUTHORIZATION, authHeader) | ||
| .put(HttpHeaders.DATE, date) | ||
| .build(); | ||
| } | ||
|
|
||
| private String generateHMac(final String key, final String toSign) throws IOException { | ||
| try { | ||
| final Mac mac = Mac.getInstance(SIGNING_ALGO); | ||
| mac.init(new SecretKeySpec(key.getBytes(), SIGNING_ALGO)); | ||
| final byte[] signed = mac.doFinal(toSign.getBytes()); | ||
| return Base64.getEncoder().encodeToString(signed); | ||
| } catch (Exception e) { | ||
| throw new IOException("Unable to sign AWS auth header correctly", e); | ||
| } | ||
| } | ||
|
|
||
| private String canoicaliseHeaders(final HttpRequest request, final String date) { | ||
| final StringBuilder builder = new StringBuilder(); | ||
| Map<String, List<String>> signHeaders = new TreeMap<>(); | ||
|
|
||
| final io.netty.handler.codec.http.HttpHeaders headers = request.headers(); | ||
| final Iterator<Map.Entry<String, String>> headersOfInterest = headers.iteratorAsString(); | ||
| headersOfInterest.forEachRemaining(entry -> { | ||
| final String header = entry.getKey(); | ||
| Preconditions.checkNotNull(header); | ||
| if (header.toLowerCase().startsWith(AWS_HDR_PREFIX)) { | ||
| signHeaders | ||
| .computeIfAbsent(header, (_x) -> new ArrayList<>()) | ||
| .add(entry.getValue()); | ||
| } | ||
| }); | ||
|
|
||
| // Add the following positional headers as per spec, even if they are empty | ||
| builder | ||
| .append(headers.get(HttpHeaders.CONTENT_TYPE, "")) | ||
| .append('\n') | ||
| .append(headers.get(HttpHeaders.CONTENT_MD5, "")) | ||
| .append('\n') | ||
| .append(date) | ||
| .append('\n'); | ||
|
|
||
| final Joiner COMMA_JOINER = Joiner.on(','); | ||
|
|
||
| for (Map.Entry<String, List<String>> entry : signHeaders.entrySet()) { | ||
| final String header = entry.getKey().toLowerCase(); | ||
| final List<String> values = entry.getValue(); | ||
| if (header.startsWith(AWS_HDR_PREFIX)) { | ||
| builder.append(header); | ||
| builder.append(":"); | ||
| COMMA_JOINER.appendTo(builder, values); | ||
| builder.append('\n'); | ||
| } | ||
| } | ||
|
|
||
| return builder.toString(); | ||
| } | ||
|
|
||
| private String canoicaliseRequest(final URI uri) { | ||
| final StringBuilder builder = new StringBuilder(); | ||
|
|
||
| if (!uri.getHost().startsWith("s3")) { | ||
| final String vhost = Iterables.getFirst(URL_SPLITTER.split(uri.getHost()), null); | ||
| Preconditions.checkNotNull(vhost); | ||
| builder.append("/") | ||
| .append(vhost); | ||
| } | ||
|
|
||
| builder.append(uri.getPath()); | ||
|
|
||
| if (uri.getQuery() != null) { | ||
| final Iterator<String> queryParams = PARAM_SPLITTER | ||
| .trimResults() | ||
| .splitToList(uri.getQuery()) | ||
| .stream() | ||
| .filter(CANOC_PARAMETERS::contains) | ||
| .sorted() | ||
| .iterator(); | ||
|
|
||
| builder.append("?"); | ||
| PARAM_JOINER.appendTo(builder, queryParams); | ||
| } | ||
|
|
||
| return builder.toString(); | ||
| } | ||
|
|
||
| } |
Oops, something went wrong.