From d3b121c2820103b267290296e788d4975c018ee3 Mon Sep 17 00:00:00 2001 From: Jesse Lovelace Date: Wed, 28 Nov 2018 12:34:06 -0800 Subject: [PATCH 1/5] Add support for V4 signing --- .../com/google/cloud/storage/Storage.java | 14 ++++++- .../com/google/cloud/storage/StorageImpl.java | 37 +++++++++++++++++++ .../google/cloud/storage/StorageImplTest.java | 1 + 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java index 04ad43f25057..1541b7eaff97 100644 --- a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java +++ b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java @@ -892,7 +892,11 @@ class SignUrlOption implements Serializable { private final Object value; enum Option { - HTTP_METHOD, CONTENT_TYPE, MD5, EXT_HEADERS, SERVICE_ACCOUNT_CRED + HTTP_METHOD, CONTENT_TYPE, MD5, EXT_HEADERS, SERVICE_ACCOUNT_CRED, SIGNATURE_VERSION + } + + enum SignatureVersion { + V2, V4 } private SignUrlOption(Option option, Object value) { @@ -943,6 +947,14 @@ public static SignUrlOption withExtHeaders(Map extHeaders) { return new SignUrlOption(Option.EXT_HEADERS, extHeaders); } + public static SignUrlOption withV2Signature() { + return new SignUrlOption(Option.SIGNATURE_VERSION, SignatureVersion.V2); + } + + public static SignUrlOption withV4Signature() { + return new SignUrlOption(Option.SIGNATURE_VERSION, SignatureVersion.V4); + } + /** * Provides a service account signer to sign the URL. If not provided an attempt will be made to * get it from the environment. diff --git a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java index 787388006571..2f2f58ab469b 100644 --- a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java +++ b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java @@ -51,6 +51,7 @@ import com.google.cloud.storage.spi.v1.StorageRpc; import com.google.cloud.storage.spi.v1.StorageRpc.RewriteResponse; import com.google.common.base.Function; +import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -62,6 +63,9 @@ import com.google.common.io.BaseEncoding; import com.google.common.net.UrlEscapers; import com.google.common.primitives.Ints; +import org.joda.time.format.DateTimeFormatter; +import org.joda.time.format.ISODateTimeFormat; + import java.io.ByteArrayInputStream; import java.io.InputStream; import java.io.UnsupportedEncodingException; @@ -71,6 +75,7 @@ import java.net.URLEncoder; import java.util.Arrays; import java.util.Collections; +import java.util.Date; import java.util.EnumMap; import java.util.List; import java.util.Map; @@ -548,6 +553,38 @@ public URL signUrl(BlobInfo blobInfo, long duration, TimeUnit unit, SignUrlOptio throw new IllegalStateException(ex); } } + + public String constructV4QueryString(ServiceAccount account, long expiration, Map signedHeadersMap) { + long date = System.currentTimeMillis(); + + StringBuilder signedHeaders = new StringBuilder(); + + for(String header : signedHeadersMap.keySet()) { + signedHeaders.append(header.toLowerCase() + ";"); + } + signedHeaders.setLength(Math.max(signedHeaders.length() - 1, 0)); + StringBuilder queryString = new StringBuilder(); + queryString.append("X-Goog-Algorithm=GOOG4-RSA-SHA256&"); + queryString.append("X-Goog-Credential=" + UrlEscapers.urlFormParameterEscaper().escape(account.getEmail() + "/" + + ISODateTimeFormat.basicDate().print(date) + "/auto/storage/goog4_request") + "&"); + queryString.append("X-Goog-Date=" + ISODateTimeFormat.basicDateTimeNoMillis().withZoneUTC().print(date).replace("\'", "") + "&"); + queryString.append("X-Goog-Expires=" + expiration + "&"); + queryString.append("X-Goog-SignedHeaders=" + signedHeaders.toString()); + + return queryString.toString(); + } + + private String constructCanonicalRequest(HttpMethod httpMethod, URI path) { + StringBuilder canonicalRequest = new StringBuilder(); + canonicalRequest.append(httpMethod.toString() + "\n"); + canonicalRequest.append(path.toString() + "\n"); + return null; + } + + private String constructV4StringToSign() { + return null; + } + /** * Builds signature info. diff --git a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java index 574c02ea001f..a5eeba8b1811 100644 --- a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java +++ b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java @@ -1791,6 +1791,7 @@ public void testSignUrlWithExtHeaders() Map extHeaders = new HashMap(); extHeaders.put("x-goog-acl", "public-read"); extHeaders.put("x-goog-meta-owner", "myself"); + System.out.println(((StorageImpl)storage).constructV4QueryString(SERVICE_ACCOUNT, 2L + 1209600, extHeaders)); URL url = storage.signUrl( BLOB_INFO1, From d4fb2996e06dc8be3b45331b2e8d7f17c87de0a5 Mon Sep 17 00:00:00 2001 From: Jesse Lovelace Date: Mon, 18 Mar 2019 10:53:05 -0700 Subject: [PATCH 2/5] (storage) WIP: Add V4 signing support --- .../CanonicalExtensionHeadersSerializer.java | 60 +++++-- .../google/cloud/storage/SignatureInfo.java | 147 +++++++++++++++++- .../com/google/cloud/storage/StorageImpl.java | 81 +++++----- .../google/cloud/storage/StorageImplTest.java | 1 - .../cloud/storage/it/ITStorageTest.java | 67 ++++++-- .../resources/UrlSignerV4TestAccount.json | 12 ++ 6 files changed, 295 insertions(+), 73 deletions(-) create mode 100644 google-cloud-clients/google-cloud-storage/src/test/resources/UrlSignerV4TestAccount.json diff --git a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/CanonicalExtensionHeadersSerializer.java b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/CanonicalExtensionHeadersSerializer.java index 71aecbdb5962..1af05120f417 100644 --- a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/CanonicalExtensionHeadersSerializer.java +++ b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/CanonicalExtensionHeadersSerializer.java @@ -32,31 +32,17 @@ public class CanonicalExtensionHeadersSerializer { private static final char HEADER_SEPARATOR = ':'; + private static final char HEADER_NAME_SEPARATOR = ';'; public StringBuilder serialize(Map canonicalizedExtensionHeaders) { StringBuilder serializedHeaders = new StringBuilder(); if (canonicalizedExtensionHeaders == null || canonicalizedExtensionHeaders.isEmpty()) { - return serializedHeaders; } - // Make all custom header names lowercase. - Map lowercaseHeaders = new HashMap<>(); - for (String headerName : new ArrayList<>(canonicalizedExtensionHeaders.keySet())) { - - String lowercaseHeaderName = headerName.toLowerCase(); - - // If present, remove the x-goog-encryption-key and x-goog-encryption-key-sha256 headers. - if ("x-goog-encryption-key".equals(lowercaseHeaderName) - || "x-goog-encryption-key-sha256".equals(lowercaseHeaderName)) { - - continue; - } - - lowercaseHeaders.put(lowercaseHeaderName, canonicalizedExtensionHeaders.get(headerName)); - } + Map lowercaseHeaders = getLowercaseHeaders(canonicalizedExtensionHeaders); // Sort all custom headers by header name using a lexicographical sort by code point value. List sortedHeaderNames = new ArrayList<>(lowercaseHeaders.keySet()); @@ -81,4 +67,46 @@ public StringBuilder serialize(Map canonicalizedExtensionHeaders // Concatenate all custom headers return serializedHeaders; } + + public StringBuilder serializeHeaderNames(Map canonicalizedExtensionHeaders) { + StringBuilder serializedHeaders = new StringBuilder(); + + if (canonicalizedExtensionHeaders == null || canonicalizedExtensionHeaders.isEmpty()) { + return serializedHeaders; + } + + Map lowercaseHeaders = getLowercaseHeaders(canonicalizedExtensionHeaders); + + List sortedHeaderNames = new ArrayList<>(lowercaseHeaders.keySet()); + Collections.sort(sortedHeaderNames); + + for (String headerName : sortedHeaderNames) { + serializedHeaders.append(headerName).append(HEADER_NAME_SEPARATOR); + } + + serializedHeaders.setLength(serializedHeaders.length() - 1); // remove trailing semicolon + + return serializedHeaders; + } + + private Map getLowercaseHeaders( + Map canonicalizedExtensionHeaders) { + // Make all custom header names lowercase. + Map lowercaseHeaders = new HashMap<>(); + for (String headerName : new ArrayList<>(canonicalizedExtensionHeaders.keySet())) { + + String lowercaseHeaderName = headerName.toLowerCase(); + + // If present, remove the x-goog-encryption-key and x-goog-encryption-key-sha256 headers. + if ("x-goog-encryption-key".equals(lowercaseHeaderName) + || "x-goog-encryption-key-sha256".equals(lowercaseHeaderName)) { + + continue; + } + + lowercaseHeaders.put(lowercaseHeaderName, canonicalizedExtensionHeaders.get(headerName)); + } + + return lowercaseHeaders; + } } diff --git a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/SignatureInfo.java b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/SignatureInfo.java index 369776ffd9dc..7ad8d53dfe8e 100644 --- a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/SignatureInfo.java +++ b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/SignatureInfo.java @@ -16,9 +16,16 @@ package com.google.cloud.storage; +import com.google.common.collect.ImmutableMap; +import com.google.common.hash.Hashing; +import com.google.common.net.UrlEscapers; + import static com.google.common.base.Preconditions.checkArgument; import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.text.SimpleDateFormat; +import java.util.Date; import java.util.Map; /** @@ -31,6 +38,8 @@ public class SignatureInfo { public static final char COMPONENT_SEPARATOR = '\n'; + public static final String GOOG4_RSA_SHA256 = "GOOG4-RSA-SHA256"; + public static final String SCOPE = "/auto/storage/goog4_request"; private final HttpMethod httpVerb; private final String contentMd5; @@ -38,23 +47,55 @@ public class SignatureInfo { private final long expiration; private final Map canonicalizedExtensionHeaders; private final URI canonicalizedResource; + private final Storage.SignUrlOption.SignatureVersion signatureVersion; + private final String accountEmail; + private final long timestamp; + + private final String yearMonthDay; + private final String exactDate; private SignatureInfo(Builder builder) { this.httpVerb = builder.httpVerb; this.contentMd5 = builder.contentMd5; this.contentType = builder.contentType; this.expiration = builder.expiration; - this.canonicalizedExtensionHeaders = builder.canonicalizedExtensionHeaders; this.canonicalizedResource = builder.canonicalizedResource; + this.signatureVersion = builder.signatureVersion; + this.accountEmail = builder.accountEmail; + this.timestamp = builder.timestamp; + + if (Storage.SignUrlOption.SignatureVersion.V4.equals(signatureVersion) + && !builder.canonicalizedExtensionHeaders.containsKey("host")) { + canonicalizedExtensionHeaders = + new ImmutableMap.Builder() + .putAll(builder.canonicalizedExtensionHeaders) + .put("host", "storage.googleapis.com") + .build(); + } else { + canonicalizedExtensionHeaders = builder.canonicalizedExtensionHeaders; + } + + Date date = new Date(timestamp); + + yearMonthDay = new SimpleDateFormat("yyyyMMdd").format(date); + exactDate = new SimpleDateFormat("yyyyMMdd'T'hhmmss'Z'").format(date); } /** * Constructs payload to be signed. * - * @return paylod to sign + * @return payload to sign * @see Signed URLs */ public String constructUnsignedPayload() { + // TODO reverse order when V4 becomes default + if (Storage.SignUrlOption.SignatureVersion.V4.equals(signatureVersion)) { + return constructV4UnsignedPayload(); + } + return constructV2UnsignedPayload(); + } + + private String constructV2UnsignedPayload() { StringBuilder payload = new StringBuilder(); payload.append(httpVerb.name()).append(COMPONENT_SEPARATOR); @@ -80,6 +121,65 @@ public String constructUnsignedPayload() { return payload.toString(); } + private String constructV4UnsignedPayload() { + StringBuilder payload = new StringBuilder(); + + payload.append(GOOG4_RSA_SHA256).append(COMPONENT_SEPARATOR); + + payload.append(exactDate).append(COMPONENT_SEPARATOR); + + payload.append(yearMonthDay).append(SCOPE).append(COMPONENT_SEPARATOR); + + payload.append(constructV4CanonicalRequestHash()); + + return payload.toString(); + } + + private String constructV4CanonicalRequestHash() { + StringBuilder canonicalRequest = new StringBuilder(); + + CanonicalExtensionHeadersSerializer serializer = new CanonicalExtensionHeadersSerializer(); + + canonicalRequest.append(httpVerb.name()).append(COMPONENT_SEPARATOR); + + canonicalRequest.append(canonicalizedResource).append(COMPONENT_SEPARATOR); + + canonicalRequest.append(constructV4QueryString()).append(COMPONENT_SEPARATOR); + + canonicalRequest + .append(serializer.serialize(canonicalizedExtensionHeaders)) + .append(COMPONENT_SEPARATOR); + + canonicalRequest + .append(serializer.serializeHeaderNames(canonicalizedExtensionHeaders)) + .append(COMPONENT_SEPARATOR); + + canonicalRequest.append("UNSIGNED-PAYLOAD"); + + return Hashing.sha256() + .hashString(canonicalRequest.toString(), StandardCharsets.UTF_8) + .toString(); + } + + public String constructV4QueryString() { + StringBuilder signedHeaders = + new CanonicalExtensionHeadersSerializer() + .serializeHeaderNames(canonicalizedExtensionHeaders); + + StringBuilder queryString = new StringBuilder(); + queryString.append("X-Goog-Algorithm=").append(GOOG4_RSA_SHA256).append("&"); + queryString.append( + "X-Goog-Credential=" + + UrlEscapers.urlFormParameterEscaper() + .escape(accountEmail + "/" + yearMonthDay + SCOPE) + + "&"); + queryString.append("X-Goog-Date=" + exactDate + "&"); + queryString.append("X-Goog-Expires=" + expiration + "&"); + queryString.append("X-Goog-SignedHeaders=" + signedHeaders.toString()); + + return queryString.toString(); + } + public HttpMethod getHttpVerb() { return httpVerb; } @@ -104,6 +204,18 @@ public URI getCanonicalizedResource() { return canonicalizedResource; } + public Storage.SignUrlOption.SignatureVersion getSignatureVersion() { + return signatureVersion; + } + + public long getTimestamp() { + return timestamp; + } + + public String getAccountEmail() { + return accountEmail; + } + public static final class Builder { private final HttpMethod httpVerb; @@ -112,6 +224,9 @@ public static final class Builder { private final long expiration; private Map canonicalizedExtensionHeaders; private final URI canonicalizedResource; + private Storage.SignUrlOption.SignatureVersion signatureVersion; + private String accountEmail; + private long timestamp; /** * Constructs builder. @@ -134,6 +249,9 @@ public Builder(SignatureInfo signatureInfo) { this.expiration = signatureInfo.expiration; this.canonicalizedExtensionHeaders = signatureInfo.canonicalizedExtensionHeaders; this.canonicalizedResource = signatureInfo.canonicalizedResource; + this.signatureVersion = signatureInfo.signatureVersion; + this.accountEmail = signatureInfo.accountEmail; + this.timestamp = signatureInfo.timestamp; } public Builder setContentMd5(String contentMd5) { @@ -155,12 +273,37 @@ public Builder setCanonicalizedExtensionHeaders( return this; } + public Builder setSignatureVersion(Storage.SignUrlOption.SignatureVersion signatureVersion) { + this.signatureVersion = signatureVersion; + + return this; + } + + public Builder setAccountEmail(String accountEmail) { + this.accountEmail = accountEmail; + + return this; + } + + public Builder setTimestamp(long timestamp) { + this.timestamp = timestamp; + + return this; + } + /** Creates an {@code SignatureInfo} object from this builder. */ public SignatureInfo build() { checkArgument(httpVerb != null, "Required HTTP method"); checkArgument(canonicalizedResource != null, "Required canonicalized resource"); checkArgument(expiration >= 0, "Expiration must be greater than or equal to zero"); + if (Storage.SignUrlOption.SignatureVersion.V4.equals(signatureVersion)) { + checkArgument(accountEmail != null, "Account email required to use V4 signing"); + checkArgument(timestamp > 0, "Timestamp required to use V4 signing"); + checkArgument( + expiration <= 604800000, "Expiration can't be longer than 7 days to use V4 signing"); + } + return new SignatureInfo(this); } } diff --git a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java index 1e5aef8ebfb1..35b15b28fbd3 100644 --- a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java +++ b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java @@ -51,7 +51,6 @@ import com.google.cloud.storage.spi.v1.StorageRpc; import com.google.cloud.storage.spi.v1.StorageRpc.RewriteResponse; import com.google.common.base.Function; -import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -73,7 +72,6 @@ import java.net.URLEncoder; import java.util.Arrays; import java.util.Collections; -import java.util.Date; import java.util.EnumMap; import java.util.List; import java.util.Map; @@ -612,6 +610,11 @@ public URL signUrl(BlobInfo blobInfo, long duration, TimeUnit unit, SignUrlOptio for (SignUrlOption option : options) { optionMap.put(option.getOption(), option.getValue()); } + + boolean isV4 = + SignUrlOption.SignatureVersion.V4.equals( + optionMap.get(SignUrlOption.Option.SIGNATURE_VERSION)); + ServiceAccountSigner credentials = (ServiceAccountSigner) optionMap.get(SignUrlOption.Option.SERVICE_ACCOUNT_CRED); if (credentials == null) { @@ -622,8 +625,11 @@ public URL signUrl(BlobInfo blobInfo, long duration, TimeUnit unit, SignUrlOptio } long expiration = - TimeUnit.SECONDS.convert( - getOptions().getClock().millisTime() + unit.toMillis(duration), TimeUnit.MILLISECONDS); + isV4 + ? duration + : TimeUnit.SECONDS.convert( + getOptions().getClock().millisTime() + unit.toMillis(duration), + TimeUnit.MILLISECONDS); StringBuilder stPath = new StringBuilder(); if (!blobInfo.getBucket().startsWith(PATH_DELIMITER)) { @@ -643,9 +649,10 @@ public URL signUrl(BlobInfo blobInfo, long duration, TimeUnit unit, SignUrlOptio URI path = URI.create(stPath.toString()); try { - SignatureInfo signatureInfo = buildSignatureInfo(optionMap, blobInfo, expiration, path); - byte[] signatureBytes = - credentials.sign(signatureInfo.constructUnsignedPayload().getBytes(UTF_8)); + SignatureInfo signatureInfo = + buildSignatureInfo(optionMap, blobInfo, expiration, path, credentials.getAccount()); + String unsignedPayload = signatureInfo.constructUnsignedPayload(); + byte[] signatureBytes = credentials.sign(unsignedPayload.getBytes(UTF_8)); StringBuilder stBuilder = new StringBuilder(); if (optionMap.get(SignUrlOption.Option.HOST_NAME) == null) { stBuilder.append(STORAGE_XML_HOST_NAME).append(path); @@ -653,11 +660,16 @@ public URL signUrl(BlobInfo blobInfo, long duration, TimeUnit unit, SignUrlOptio stBuilder.append(optionMap.get(SignUrlOption.Option.HOST_NAME)).append(path); } - String signature = - URLEncoder.encode(BaseEncoding.base64().encode(signatureBytes), UTF_8.name()); - stBuilder.append("?GoogleAccessId=").append(credentials.getAccount()); - stBuilder.append("&Expires=").append(expiration); - stBuilder.append("&Signature=").append(signature); + BaseEncoding encoding = isV4 ? BaseEncoding.base16().lowerCase() : BaseEncoding.base64(); + String signature = URLEncoder.encode(encoding.encode(signatureBytes), UTF_8.name()); + stBuilder.append("?"); + if (isV4) { + stBuilder.append(signatureInfo.constructV4QueryString()); + } else { + stBuilder.append("GoogleAccessId=").append(credentials.getAccount()); + stBuilder.append("&Expires=").append(expiration); + } + stBuilder.append(isV4 ? "&X-Goog-Signature=" : "&Signature=").append(signature); return new URL(stBuilder.toString()); @@ -666,37 +678,6 @@ public URL signUrl(BlobInfo blobInfo, long duration, TimeUnit unit, SignUrlOptio } } - public String constructV4QueryString(ServiceAccount account, long expiration, Map signedHeadersMap) { - long date = System.currentTimeMillis(); - - StringBuilder signedHeaders = new StringBuilder(); - - for(String header : signedHeadersMap.keySet()) { - signedHeaders.append(header.toLowerCase() + ";"); - } - signedHeaders.setLength(Math.max(signedHeaders.length() - 1, 0)); - StringBuilder queryString = new StringBuilder(); - queryString.append("X-Goog-Algorithm=GOOG4-RSA-SHA256&"); - //queryString.append("X-Goog-Credential=" + UrlEscapers.urlFormParameterEscaper().escape(account.getEmail() + "/" + - // ISODateTimeFormat.basicDate().print(date) + "/auto/storage/goog4_request") + "&"); - //queryString.append("X-Goog-Date=" + ISODateTimeFormat.basicDateTimeNoMillis().withZoneUTC().print(date).replace("\'", "") + "&"); - queryString.append("X-Goog-Expires=" + expiration + "&"); - queryString.append("X-Goog-SignedHeaders=" + signedHeaders.toString()); - - return queryString.toString(); - } - - private String constructCanonicalRequest(HttpMethod httpMethod, URI path) { - StringBuilder canonicalRequest = new StringBuilder(); - canonicalRequest.append(httpMethod.toString() + "\n"); - canonicalRequest.append(path.toString() + "\n"); - return null; - } - - private String constructV4StringToSign() { - return null; - } - /** * Builds signature info. * @@ -704,10 +685,15 @@ private String constructV4StringToSign() { * @param blobInfo the blob info * @param expiration the expiration in seconds * @param path the resource URI + * @param accountEmail the account email * @return signature info */ private SignatureInfo buildSignatureInfo( - Map optionMap, BlobInfo blobInfo, long expiration, URI path) { + Map optionMap, + BlobInfo blobInfo, + long expiration, + URI path, + String accountEmail) { HttpMethod httpVerb = optionMap.containsKey(SignUrlOption.Option.HTTP_METHOD) @@ -727,6 +713,13 @@ private SignatureInfo buildSignatureInfo( signatureInfoBuilder.setContentType(blobInfo.getContentType()); } + signatureInfoBuilder.setSignatureVersion( + (SignUrlOption.SignatureVersion) optionMap.get(SignUrlOption.Option.SIGNATURE_VERSION)); + + signatureInfoBuilder.setAccountEmail(accountEmail); + + signatureInfoBuilder.setTimestamp(getOptions().getClock().millisTime()); + @SuppressWarnings("unchecked") Map extHeaders = (Map) diff --git a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java index 23d865bab5d9..0d77547e7605 100644 --- a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java +++ b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java @@ -2082,7 +2082,6 @@ public void testSignUrlWithExtHeaders() Map extHeaders = new HashMap(); extHeaders.put("x-goog-acl", "public-read"); extHeaders.put("x-goog-meta-owner", "myself"); - System.out.println(((StorageImpl)storage).constructV4QueryString(SERVICE_ACCOUNT, 2L + 1209600, extHeaders)); URL url = storage.signUrl( BLOB_INFO1, diff --git a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java index 0e9e33f177bf..1ca58f7cb0bc 100644 --- a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java +++ b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java @@ -31,10 +31,12 @@ import com.google.api.client.http.HttpTransport; import com.google.api.client.http.apache.ApacheHttpTransport; import com.google.api.client.util.DateTime; +import com.google.api.core.ApiClock; import com.google.api.gax.paging.Page; import com.google.auth.ServiceAccountSigner; import com.google.auth.http.HttpTransportFactory; import com.google.auth.oauth2.GoogleCredentials; +import com.google.auth.oauth2.ServiceAccountCredentials; import com.google.cloud.Identity; import com.google.cloud.Policy; import com.google.cloud.ReadChannel; @@ -95,6 +97,8 @@ import io.grpc.stub.MetadataUtils; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.net.URL; @@ -112,6 +116,7 @@ import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import java.util.logging.Logger; import java.util.zip.GZIPInputStream; @@ -162,18 +167,14 @@ public class ITStorageTest { public static void beforeClass() throws IOException { remoteStorageHelper = RemoteStorageHelper.create(); storage = remoteStorageHelper.getOptions().getService(); - storage.create( - BucketInfo.newBuilder(BUCKET) - .setLocation("us") - .setLifecycleRules( - ImmutableList.of( - new LifecycleRule( - LifecycleAction.newDeleteAction(), - LifecycleCondition.newBuilder().setAge(1).build()))) - .build()); + + storage.create( BucketInfo.newBuilder(BUCKET) .setLocation("us") .setLifecycleRules( + ImmutableList.of( new LifecycleRule( LifecycleAction.newDeleteAction(), + LifecycleCondition.newBuilder().setAge(1).build()))) .build()); + // Prepare KMS KeyRing for CMEK tests - prepareKmsKeys(); + prepareKmsKeys(); } @AfterClass @@ -201,6 +202,22 @@ public HttpTransport create() { } } + private static class FakeClock implements ApiClock { + private final AtomicLong currentNanoTime; + + public FakeClock(long initialNanoTime) { + this.currentNanoTime = new AtomicLong(initialNanoTime); + } + + public long nanoTime() { + return this.currentNanoTime.get(); + } + + public long millisTime() { + return TimeUnit.MILLISECONDS.convert(this.nanoTime(), TimeUnit.NANOSECONDS); + } + } + private static void prepareKmsKeys() throws IOException { String projectId = remoteStorageHelper.getOptions().getProjectId(); GoogleCredentials credentials = GoogleCredentials.getApplicationDefault(); @@ -1824,6 +1841,36 @@ public void testGetSignedUrl() throws IOException { } } + @Test + //TODO rewrite this to load JSON conformance tests instead + public void testV4UrlSigning() throws IOException { + Storage dummyAccountStorage = + remoteStorageHelper + .getOptions() + .toBuilder() + .setClock( + new FakeClock(TimeUnit.NANOSECONDS.convert(1549040400000L, TimeUnit.MILLISECONDS))) + .setCredentials( + ServiceAccountCredentials.fromStream( + new FileInputStream( + new File("src/test/resources/URLSignerV4TestAccount.json")))) + .build() + .getService(); + String bucket = "test-bucket"; + String object = "test-object"; + + BlobInfo blob = BlobInfo.newBuilder(bucket, object).build(); + + assertEquals( + "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=95e6a13d43a1d1962e667f17397f2b80ac9bdd1669210d5e08e0135df9dff4e56113485dbe429ca2266487b9d1796ebdee2d7cf682a6ef3bb9fbb4c351686fba90d7b621cf1c4eb1fdf126460dd25fa0837dfdde0a9fd98662ce60844c458448fb2b352c203d9969cb74efa4bdb742287744a4f2308afa4af0e0773f55e32e92973619249214b97283b2daa14195244444e33f938138d1e5f561088ce8011f4986dda33a556412594db7c12fc40e1ff3f1bedeb7a42f5bcda0b9567f17f65855f65071fabb88ea12371877f3f77f10e1466fff6ff6973b74a933322ff0949ce357e20abe96c3dd5cfab42c9c83e740a4d32b9e11e146f0eb3404d2e975896f74", + dummyAccountStorage.signUrl( + blob, + 10, + TimeUnit.SECONDS, + Storage.SignUrlOption.httpMethod(HttpMethod.GET), + Storage.SignUrlOption.withV4Signature()).toString()); + } + @Test public void testPostSignedUrl() throws IOException { if (storage.getOptions().getCredentials() != null) { diff --git a/google-cloud-clients/google-cloud-storage/src/test/resources/UrlSignerV4TestAccount.json b/google-cloud-clients/google-cloud-storage/src/test/resources/UrlSignerV4TestAccount.json new file mode 100644 index 000000000000..fe2701badc3d --- /dev/null +++ b/google-cloud-clients/google-cloud-storage/src/test/resources/UrlSignerV4TestAccount.json @@ -0,0 +1,12 @@ +{ + "type": "service_account", + "project_id": "dummy-project-id", + "private_key_id": "ffffffffffffffffffffffffffffffffffffffff", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCsPzMirIottfQ2\nryjQmPWocSEeGo7f7Q4/tMQXHlXFzo93AGgU2t+clEj9L5loNhLVq+vk+qmnyDz5\nQ04y8jVWyMYzzGNNrGRW/yaYqnqlKZCy1O3bmnNjV7EDbC/jE1ZLBY0U3HaSHfn6\nS9ND8MXdgD0/ulRTWwq6vU8/w6i5tYsU7n2LLlQTl1fQ7/emO9nYcCFJezHZVa0H\nmeWsdHwWsok0skwQYQNIzP3JF9BpR5gJT2gNge6KopDesJeLoLzaX7cUnDn+CAnn\nLuLDwwSsIVKyVxhBFsFXPplgpaQRwmGzwEbf/Xpt9qo26w2UMgn30jsOaKlSeAX8\ncS6ViF+tAgMBAAECggEACKRuJCP8leEOhQziUx8Nmls8wmYqO4WJJLyk5xUMUC22\nSI4CauN1e0V8aQmxnIc0CDkFT7qc9xBmsMoF+yvobbeKrFApvlyzNyM7tEa/exh8\nDGD/IzjbZ8VfWhDcUTwn5QE9DCoon9m1sG+MBNlokB3OVOt8LieAAREdEBG43kJu\nyQTOkY9BGR2AY1FnAl2VZ/jhNDyrme3tp1sW1BJrawzR7Ujo8DzlVcS2geKA9at7\n55ua5GbHz3hfzFgjVXDfnkWzId6aHypUyqHrSn1SqGEbyXTaleKTc6Pgv0PgkJjG\nhZazWWdSuf1T5Xbs0OhAK9qraoAzT6cXXvMEvvPt6QKBgQDXcZKqJAOnGEU4b9+v\nOdoh+nssdrIOBNMu1m8mYbUVYS1aakc1iDGIIWNM3qAwbG+yNEIi2xi80a2RMw2T\n9RyCNB7yqCXXVKLBiwg9FbKMai6Vpk2bWIrzahM9on7AhCax/X2AeOp+UyYhFEy6\nUFG4aHb8THscL7b515ukSuKb5QKBgQDMq+9PuaB0eHsrmL6q4vHNi3MLgijGg/zu\nAXaPygSYAwYW8KglcuLZPvWrL6OG0+CrfmaWTLsyIZO4Uhdj7MLvX6yK7IMnagvk\nL3xjgxSklEHJAwi5wFeJ8ai/1MIuCn8p2re3CbwISKpvf7Sgs/W4196P4vKvTiAz\njcTiSYFIKQKBgCjMpkS4O0TakMlGTmsFnqyOneLmu4NyIHgfPb9cA4n/9DHKLKAT\noaWxBPgatOVWs7RgtyGYsk+XubHkpC6f3X0+15mGhFwJ+CSE6tN+l2iF9zp52vqP\nQwkjzm7+pdhZbmaIpcq9m1K+9lqPWJRz/3XXuqi+5xWIZ7NaxGvRjqaNAoGAdK2b\nutZ2y48XoI3uPFsuP+A8kJX+CtWZrlE1NtmS7tnicdd19AtfmTuUL6fz0FwfW4Su\nlQZfPT/5B339CaEiq/Xd1kDor+J7rvUHM2+5p+1A54gMRGCLRv92FQ4EON0RC1o9\nm2I4SHysdO3XmjmdXmfp4BsgAKJIJzutvtbqlakCgYB+Cb10z37NJJ+WgjDt+yT2\nyUNH17EAYgWXryfRgTyi2POHuJitd64Xzuy6oBVs3wVveYFM6PIKXlj8/DahYX5I\nR2WIzoCNLL3bEZ+nC6Jofpb4kspoAeRporj29SgesK6QBYWHWX2H645RkRGYGpDo\n51gjy9m/hSNqBbH2zmh04A==\n-----END PRIVATE KEY-----\n", + "client_email": "test-iam-credentials@dummy-project-id.iam.gserviceaccount.com", + "client_id": "000000000000000000000", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com" +} \ No newline at end of file From 4e30cdc6d693b4156fae0b04c0bca5a8a1a42931 Mon Sep 17 00:00:00 2001 From: Jesse Lovelace Date: Mon, 18 Mar 2019 10:53:05 -0700 Subject: [PATCH 3/5] (storage) Add V4 signing support --- .../CanonicalExtensionHeadersSerializer.java | 72 +++++++-- .../google/cloud/storage/SignatureInfo.java | 148 +++++++++++++++++- .../com/google/cloud/storage/StorageImpl.java | 81 +++++----- .../google/cloud/storage/StorageImplTest.java | 1 - .../cloud/storage/it/ITStorageTest.java | 106 +++++++++++++ .../resources/UrlSignerV4TestAccount.json | 12 ++ .../test/resources/UrlSignerV4TestData.json | 114 ++++++++++++++ 7 files changed, 470 insertions(+), 64 deletions(-) create mode 100644 google-cloud-clients/google-cloud-storage/src/test/resources/UrlSignerV4TestAccount.json create mode 100644 google-cloud-clients/google-cloud-storage/src/test/resources/UrlSignerV4TestData.json diff --git a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/CanonicalExtensionHeadersSerializer.java b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/CanonicalExtensionHeadersSerializer.java index 71aecbdb5962..da74eb20752c 100644 --- a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/CanonicalExtensionHeadersSerializer.java +++ b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/CanonicalExtensionHeadersSerializer.java @@ -32,31 +32,17 @@ public class CanonicalExtensionHeadersSerializer { private static final char HEADER_SEPARATOR = ':'; + private static final char HEADER_NAME_SEPARATOR = ';'; - public StringBuilder serialize(Map canonicalizedExtensionHeaders) { + public StringBuilder serialize(Map canonicalizedExtensionHeaders, boolean isV4) { StringBuilder serializedHeaders = new StringBuilder(); if (canonicalizedExtensionHeaders == null || canonicalizedExtensionHeaders.isEmpty()) { - return serializedHeaders; } - // Make all custom header names lowercase. - Map lowercaseHeaders = new HashMap<>(); - for (String headerName : new ArrayList<>(canonicalizedExtensionHeaders.keySet())) { - - String lowercaseHeaderName = headerName.toLowerCase(); - - // If present, remove the x-goog-encryption-key and x-goog-encryption-key-sha256 headers. - if ("x-goog-encryption-key".equals(lowercaseHeaderName) - || "x-goog-encryption-key-sha256".equals(lowercaseHeaderName)) { - - continue; - } - - lowercaseHeaders.put(lowercaseHeaderName, canonicalizedExtensionHeaders.get(headerName)); - } + Map lowercaseHeaders = getLowercaseHeaders(canonicalizedExtensionHeaders, isV4); // Sort all custom headers by header name using a lexicographical sort by code point value. List sortedHeaderNames = new ArrayList<>(lowercaseHeaders.keySet()); @@ -81,4 +67,56 @@ public StringBuilder serialize(Map canonicalizedExtensionHeaders // Concatenate all custom headers return serializedHeaders; } + + public StringBuilder serialize(Map canonicalizedExtensionHeaders) { + return serialize(canonicalizedExtensionHeaders, false); + } + + public StringBuilder serializeHeaderNames( + Map canonicalizedExtensionHeaders, boolean isV4) { + StringBuilder serializedHeaders = new StringBuilder(); + + if (canonicalizedExtensionHeaders == null || canonicalizedExtensionHeaders.isEmpty()) { + return serializedHeaders; + } + + Map lowercaseHeaders = getLowercaseHeaders(canonicalizedExtensionHeaders, isV4); + + List sortedHeaderNames = new ArrayList<>(lowercaseHeaders.keySet()); + Collections.sort(sortedHeaderNames); + + for (String headerName : sortedHeaderNames) { + serializedHeaders.append(headerName).append(HEADER_NAME_SEPARATOR); + } + + serializedHeaders.setLength(serializedHeaders.length() - 1); // remove trailing semicolon + + return serializedHeaders; + } + + public StringBuilder serializeHeaderNames(Map canonicalizedExtentionHeaders) { + return serializeHeaderNames(canonicalizedExtentionHeaders, true); + } + + private Map getLowercaseHeaders( + Map canonicalizedExtensionHeaders, boolean isV4) { + // Make all custom header names lowercase. + Map lowercaseHeaders = new HashMap<>(); + for (String headerName : new ArrayList<>(canonicalizedExtensionHeaders.keySet())) { + + String lowercaseHeaderName = headerName.toLowerCase(); + + // If present, remove the x-goog-encryption-key and x-goog-encryption-key-sha256 headers. + if ("x-goog-encryption-key".equals(lowercaseHeaderName) + || "x-goog-encryption-key-sha256".equals(lowercaseHeaderName) + || (isV4 && "x-goog-encryption-algorithm".equals(lowercaseHeaderName))) { + + continue; + } + + lowercaseHeaders.put(lowercaseHeaderName, canonicalizedExtensionHeaders.get(headerName)); + } + + return lowercaseHeaders; + } } diff --git a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/SignatureInfo.java b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/SignatureInfo.java index 369776ffd9dc..7dbe9a0aecc9 100644 --- a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/SignatureInfo.java +++ b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/SignatureInfo.java @@ -16,9 +16,16 @@ package com.google.cloud.storage; +import com.google.common.collect.ImmutableMap; +import com.google.common.hash.Hashing; +import com.google.common.net.UrlEscapers; + import static com.google.common.base.Preconditions.checkArgument; import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.text.SimpleDateFormat; +import java.util.Date; import java.util.Map; /** @@ -31,6 +38,8 @@ public class SignatureInfo { public static final char COMPONENT_SEPARATOR = '\n'; + public static final String GOOG4_RSA_SHA256 = "GOOG4-RSA-SHA256"; + public static final String SCOPE = "/auto/storage/goog4_request"; private final HttpMethod httpVerb; private final String contentMd5; @@ -38,23 +47,55 @@ public class SignatureInfo { private final long expiration; private final Map canonicalizedExtensionHeaders; private final URI canonicalizedResource; + private final Storage.SignUrlOption.SignatureVersion signatureVersion; + private final String accountEmail; + private final long timestamp; + + private final String yearMonthDay; + private final String exactDate; private SignatureInfo(Builder builder) { this.httpVerb = builder.httpVerb; this.contentMd5 = builder.contentMd5; this.contentType = builder.contentType; this.expiration = builder.expiration; - this.canonicalizedExtensionHeaders = builder.canonicalizedExtensionHeaders; this.canonicalizedResource = builder.canonicalizedResource; + this.signatureVersion = builder.signatureVersion; + this.accountEmail = builder.accountEmail; + this.timestamp = builder.timestamp; + + if (Storage.SignUrlOption.SignatureVersion.V4.equals(signatureVersion) + && !builder.canonicalizedExtensionHeaders.containsKey("host")) { + canonicalizedExtensionHeaders = + new ImmutableMap.Builder() + .putAll(builder.canonicalizedExtensionHeaders) + .put("host", "storage.googleapis.com") + .build(); + } else { + canonicalizedExtensionHeaders = builder.canonicalizedExtensionHeaders; + } + + Date date = new Date(timestamp); + + yearMonthDay = new SimpleDateFormat("yyyyMMdd").format(date); + exactDate = new SimpleDateFormat("yyyyMMdd'T'hhmmss'Z'").format(date); } /** * Constructs payload to be signed. * - * @return paylod to sign + * @return payload to sign * @see Signed URLs */ public String constructUnsignedPayload() { + // TODO reverse order when V4 becomes default + if (Storage.SignUrlOption.SignatureVersion.V4.equals(signatureVersion)) { + return constructV4UnsignedPayload(); + } + return constructV2UnsignedPayload(); + } + + private String constructV2UnsignedPayload() { StringBuilder payload = new StringBuilder(); payload.append(httpVerb.name()).append(COMPONENT_SEPARATOR); @@ -80,6 +121,66 @@ public String constructUnsignedPayload() { return payload.toString(); } + private String constructV4UnsignedPayload() { + StringBuilder payload = new StringBuilder(); + + payload.append(GOOG4_RSA_SHA256).append(COMPONENT_SEPARATOR); + + payload.append(exactDate).append(COMPONENT_SEPARATOR); + + payload.append(yearMonthDay).append(SCOPE).append(COMPONENT_SEPARATOR); + + payload.append(constructV4CanonicalRequestHash()); + + return payload.toString(); + } + + private String constructV4CanonicalRequestHash() { + StringBuilder canonicalRequest = new StringBuilder(); + + CanonicalExtensionHeadersSerializer serializer = new CanonicalExtensionHeadersSerializer(); + + canonicalRequest.append(httpVerb.name()).append(COMPONENT_SEPARATOR); + + canonicalRequest.append(canonicalizedResource).append(COMPONENT_SEPARATOR); + + canonicalRequest.append(constructV4QueryString()).append(COMPONENT_SEPARATOR); + + canonicalRequest + .append(serializer.serialize(canonicalizedExtensionHeaders, true)) + .append(COMPONENT_SEPARATOR); + + canonicalRequest + .append(serializer.serializeHeaderNames(canonicalizedExtensionHeaders)) + .append(COMPONENT_SEPARATOR); + + canonicalRequest.append("UNSIGNED-PAYLOAD"); + return Hashing.sha256() + .hashString(canonicalRequest.toString(), StandardCharsets.UTF_8) + .toString(); + } + + public String constructV4QueryString() { + StringBuilder signedHeaders = + new CanonicalExtensionHeadersSerializer() + .serializeHeaderNames(canonicalizedExtensionHeaders); + + StringBuilder queryString = new StringBuilder(); + queryString.append("X-Goog-Algorithm=").append(GOOG4_RSA_SHA256).append("&"); + queryString.append( + "X-Goog-Credential=" + + UrlEscapers.urlFormParameterEscaper() + .escape(accountEmail + "/" + yearMonthDay + SCOPE) + + "&"); + queryString.append("X-Goog-Date=" + exactDate + "&"); + queryString.append("X-Goog-Expires=" + expiration + "&"); + queryString.append( + "X-Goog-SignedHeaders=" + + UrlEscapers.urlFormParameterEscaper().escape(signedHeaders.toString())); + + return queryString.toString(); + } + public HttpMethod getHttpVerb() { return httpVerb; } @@ -104,6 +205,18 @@ public URI getCanonicalizedResource() { return canonicalizedResource; } + public Storage.SignUrlOption.SignatureVersion getSignatureVersion() { + return signatureVersion; + } + + public long getTimestamp() { + return timestamp; + } + + public String getAccountEmail() { + return accountEmail; + } + public static final class Builder { private final HttpMethod httpVerb; @@ -112,6 +225,9 @@ public static final class Builder { private final long expiration; private Map canonicalizedExtensionHeaders; private final URI canonicalizedResource; + private Storage.SignUrlOption.SignatureVersion signatureVersion; + private String accountEmail; + private long timestamp; /** * Constructs builder. @@ -134,6 +250,9 @@ public Builder(SignatureInfo signatureInfo) { this.expiration = signatureInfo.expiration; this.canonicalizedExtensionHeaders = signatureInfo.canonicalizedExtensionHeaders; this.canonicalizedResource = signatureInfo.canonicalizedResource; + this.signatureVersion = signatureInfo.signatureVersion; + this.accountEmail = signatureInfo.accountEmail; + this.timestamp = signatureInfo.timestamp; } public Builder setContentMd5(String contentMd5) { @@ -155,12 +274,37 @@ public Builder setCanonicalizedExtensionHeaders( return this; } + public Builder setSignatureVersion(Storage.SignUrlOption.SignatureVersion signatureVersion) { + this.signatureVersion = signatureVersion; + + return this; + } + + public Builder setAccountEmail(String accountEmail) { + this.accountEmail = accountEmail; + + return this; + } + + public Builder setTimestamp(long timestamp) { + this.timestamp = timestamp; + + return this; + } + /** Creates an {@code SignatureInfo} object from this builder. */ public SignatureInfo build() { checkArgument(httpVerb != null, "Required HTTP method"); checkArgument(canonicalizedResource != null, "Required canonicalized resource"); checkArgument(expiration >= 0, "Expiration must be greater than or equal to zero"); + if (Storage.SignUrlOption.SignatureVersion.V4.equals(signatureVersion)) { + checkArgument(accountEmail != null, "Account email required to use V4 signing"); + checkArgument(timestamp > 0, "Timestamp required to use V4 signing"); + checkArgument( + expiration <= 604800000, "Expiration can't be longer than 7 days to use V4 signing"); + } + return new SignatureInfo(this); } } diff --git a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java index 1e5aef8ebfb1..afed025b1f09 100644 --- a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java +++ b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java @@ -51,7 +51,6 @@ import com.google.cloud.storage.spi.v1.StorageRpc; import com.google.cloud.storage.spi.v1.StorageRpc.RewriteResponse; import com.google.common.base.Function; -import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -73,7 +72,6 @@ import java.net.URLEncoder; import java.util.Arrays; import java.util.Collections; -import java.util.Date; import java.util.EnumMap; import java.util.List; import java.util.Map; @@ -612,6 +610,11 @@ public URL signUrl(BlobInfo blobInfo, long duration, TimeUnit unit, SignUrlOptio for (SignUrlOption option : options) { optionMap.put(option.getOption(), option.getValue()); } + + boolean isV4 = + SignUrlOption.SignatureVersion.V4.equals( + optionMap.get(SignUrlOption.Option.SIGNATURE_VERSION)); + ServiceAccountSigner credentials = (ServiceAccountSigner) optionMap.get(SignUrlOption.Option.SERVICE_ACCOUNT_CRED); if (credentials == null) { @@ -622,8 +625,11 @@ public URL signUrl(BlobInfo blobInfo, long duration, TimeUnit unit, SignUrlOptio } long expiration = - TimeUnit.SECONDS.convert( - getOptions().getClock().millisTime() + unit.toMillis(duration), TimeUnit.MILLISECONDS); + isV4 + ? unit.toMillis(duration) + : TimeUnit.SECONDS.convert( + getOptions().getClock().millisTime() + unit.toMillis(duration), + TimeUnit.MILLISECONDS); StringBuilder stPath = new StringBuilder(); if (!blobInfo.getBucket().startsWith(PATH_DELIMITER)) { @@ -643,9 +649,10 @@ public URL signUrl(BlobInfo blobInfo, long duration, TimeUnit unit, SignUrlOptio URI path = URI.create(stPath.toString()); try { - SignatureInfo signatureInfo = buildSignatureInfo(optionMap, blobInfo, expiration, path); - byte[] signatureBytes = - credentials.sign(signatureInfo.constructUnsignedPayload().getBytes(UTF_8)); + SignatureInfo signatureInfo = + buildSignatureInfo(optionMap, blobInfo, expiration, path, credentials.getAccount()); + String unsignedPayload = signatureInfo.constructUnsignedPayload(); + byte[] signatureBytes = credentials.sign(unsignedPayload.getBytes(UTF_8)); StringBuilder stBuilder = new StringBuilder(); if (optionMap.get(SignUrlOption.Option.HOST_NAME) == null) { stBuilder.append(STORAGE_XML_HOST_NAME).append(path); @@ -653,11 +660,16 @@ public URL signUrl(BlobInfo blobInfo, long duration, TimeUnit unit, SignUrlOptio stBuilder.append(optionMap.get(SignUrlOption.Option.HOST_NAME)).append(path); } - String signature = - URLEncoder.encode(BaseEncoding.base64().encode(signatureBytes), UTF_8.name()); - stBuilder.append("?GoogleAccessId=").append(credentials.getAccount()); - stBuilder.append("&Expires=").append(expiration); - stBuilder.append("&Signature=").append(signature); + BaseEncoding encoding = isV4 ? BaseEncoding.base16().lowerCase() : BaseEncoding.base64(); + String signature = URLEncoder.encode(encoding.encode(signatureBytes), UTF_8.name()); + stBuilder.append("?"); + if (isV4) { + stBuilder.append(signatureInfo.constructV4QueryString()); + } else { + stBuilder.append("GoogleAccessId=").append(credentials.getAccount()); + stBuilder.append("&Expires=").append(expiration); + } + stBuilder.append(isV4 ? "&X-Goog-Signature=" : "&Signature=").append(signature); return new URL(stBuilder.toString()); @@ -666,37 +678,6 @@ public URL signUrl(BlobInfo blobInfo, long duration, TimeUnit unit, SignUrlOptio } } - public String constructV4QueryString(ServiceAccount account, long expiration, Map signedHeadersMap) { - long date = System.currentTimeMillis(); - - StringBuilder signedHeaders = new StringBuilder(); - - for(String header : signedHeadersMap.keySet()) { - signedHeaders.append(header.toLowerCase() + ";"); - } - signedHeaders.setLength(Math.max(signedHeaders.length() - 1, 0)); - StringBuilder queryString = new StringBuilder(); - queryString.append("X-Goog-Algorithm=GOOG4-RSA-SHA256&"); - //queryString.append("X-Goog-Credential=" + UrlEscapers.urlFormParameterEscaper().escape(account.getEmail() + "/" + - // ISODateTimeFormat.basicDate().print(date) + "/auto/storage/goog4_request") + "&"); - //queryString.append("X-Goog-Date=" + ISODateTimeFormat.basicDateTimeNoMillis().withZoneUTC().print(date).replace("\'", "") + "&"); - queryString.append("X-Goog-Expires=" + expiration + "&"); - queryString.append("X-Goog-SignedHeaders=" + signedHeaders.toString()); - - return queryString.toString(); - } - - private String constructCanonicalRequest(HttpMethod httpMethod, URI path) { - StringBuilder canonicalRequest = new StringBuilder(); - canonicalRequest.append(httpMethod.toString() + "\n"); - canonicalRequest.append(path.toString() + "\n"); - return null; - } - - private String constructV4StringToSign() { - return null; - } - /** * Builds signature info. * @@ -704,10 +685,15 @@ private String constructV4StringToSign() { * @param blobInfo the blob info * @param expiration the expiration in seconds * @param path the resource URI + * @param accountEmail the account email * @return signature info */ private SignatureInfo buildSignatureInfo( - Map optionMap, BlobInfo blobInfo, long expiration, URI path) { + Map optionMap, + BlobInfo blobInfo, + long expiration, + URI path, + String accountEmail) { HttpMethod httpVerb = optionMap.containsKey(SignUrlOption.Option.HTTP_METHOD) @@ -727,6 +713,13 @@ private SignatureInfo buildSignatureInfo( signatureInfoBuilder.setContentType(blobInfo.getContentType()); } + signatureInfoBuilder.setSignatureVersion( + (SignUrlOption.SignatureVersion) optionMap.get(SignUrlOption.Option.SIGNATURE_VERSION)); + + signatureInfoBuilder.setAccountEmail(accountEmail); + + signatureInfoBuilder.setTimestamp(getOptions().getClock().millisTime()); + @SuppressWarnings("unchecked") Map extHeaders = (Map) diff --git a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java index 23d865bab5d9..0d77547e7605 100644 --- a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java +++ b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java @@ -2082,7 +2082,6 @@ public void testSignUrlWithExtHeaders() Map extHeaders = new HashMap(); extHeaders.put("x-goog-acl", "public-read"); extHeaders.put("x-goog-meta-owner", "myself"); - System.out.println(((StorageImpl)storage).constructV4QueryString(SERVICE_ACCOUNT, 2L + 1209600, extHeaders)); URL url = storage.signUrl( BLOB_INFO1, diff --git a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java index 0e9e33f177bf..f0239a5fccb0 100644 --- a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java +++ b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java @@ -31,10 +31,12 @@ import com.google.api.client.http.HttpTransport; import com.google.api.client.http.apache.ApacheHttpTransport; import com.google.api.client.util.DateTime; +import com.google.api.core.ApiClock; import com.google.api.gax.paging.Page; import com.google.auth.ServiceAccountSigner; import com.google.auth.http.HttpTransportFactory; import com.google.auth.oauth2.GoogleCredentials; +import com.google.auth.oauth2.ServiceAccountCredentials; import com.google.cloud.Identity; import com.google.cloud.Policy; import com.google.cloud.ReadChannel; @@ -83,6 +85,11 @@ import com.google.common.collect.Lists; import com.google.common.io.BaseEncoding; import com.google.common.io.ByteStreams; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; import com.google.iam.v1.Binding; import com.google.iam.v1.IAMPolicyGrpc; import com.google.iam.v1.SetIamPolicyRequest; @@ -95,12 +102,17 @@ import io.grpc.stub.MetadataUtils; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.net.URL; import java.net.URLConnection; import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Paths; import java.security.Key; +import java.text.SimpleDateFormat; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -112,6 +124,7 @@ import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import java.util.logging.Logger; import java.util.zip.GZIPInputStream; @@ -162,6 +175,7 @@ public class ITStorageTest { public static void beforeClass() throws IOException { remoteStorageHelper = RemoteStorageHelper.create(); storage = remoteStorageHelper.getOptions().getService(); + storage.create( BucketInfo.newBuilder(BUCKET) .setLocation("us") @@ -201,6 +215,22 @@ public HttpTransport create() { } } + private static class FakeClock implements ApiClock { + private final AtomicLong currentNanoTime; + + public FakeClock(long initialNanoTime) { + this.currentNanoTime = new AtomicLong(initialNanoTime); + } + + public long nanoTime() { + return this.currentNanoTime.get(); + } + + public long millisTime() { + return TimeUnit.MILLISECONDS.convert(this.nanoTime(), TimeUnit.NANOSECONDS); + } + } + private static void prepareKmsKeys() throws IOException { String projectId = remoteStorageHelper.getOptions().getProjectId(); GoogleCredentials credentials = GoogleCredentials.getApplicationDefault(); @@ -1824,6 +1854,82 @@ public void testGetSignedUrl() throws IOException { } } + public class TestCase { + String description; + String bucket; + String object; + String method; + String expiration; + String timestamp; + String expectedUrl; + JsonObject headers; + + public String toString() { + return description; + } + } + + @Test + public void testV4UrlSigning() throws Exception { + Storage dummyAccountStorage = + remoteStorageHelper + .getOptions() + .toBuilder() + .setCredentials( + ServiceAccountCredentials.fromStream( + new FileInputStream( + new File("src/test/resources/URLSignerV4TestAccount.json")))) + .build() + .getService(); + + Gson gson = new GsonBuilder().create(); + + String testCaseJson = + new String(Files.readAllBytes(Paths.get("src/test/resources/URLSignerV4TestData.json"))); + + JsonArray testCases = gson.fromJson(testCaseJson, JsonArray.class); + + for (JsonElement testCaseElement : testCases) { + TestCase testCase = gson.fromJson(testCaseElement, TestCase.class); + + dummyAccountStorage = + dummyAccountStorage + .getOptions() + .toBuilder() + .setClock( + new FakeClock( + TimeUnit.NANOSECONDS.convert( + new SimpleDateFormat("yyyyMMdd'T'hhmmss'Z'") + .parse(testCase.timestamp) + .getTime(), + TimeUnit.MILLISECONDS))) + .build() + .getService(); + + BlobInfo blob = BlobInfo.newBuilder(testCase.bucket, testCase.object).build(); + + Map headers = new HashMap<>(); + if (testCase.headers != null) { + for (Map.Entry entry : testCase.headers.entrySet()) { + JsonArray value = entry.getValue().getAsJsonArray(); + headers.put(entry.getKey(), value.get(0).getAsString()); + } + } + + assertEquals( + testCase.expectedUrl, + dummyAccountStorage + .signUrl( + blob, + Long.valueOf(testCase.expiration), + TimeUnit.MILLISECONDS, + Storage.SignUrlOption.httpMethod(HttpMethod.valueOf(testCase.method)), + Storage.SignUrlOption.withExtHeaders(headers), + Storage.SignUrlOption.withV4Signature()) + .toString()); + } + } + @Test public void testPostSignedUrl() throws IOException { if (storage.getOptions().getCredentials() != null) { diff --git a/google-cloud-clients/google-cloud-storage/src/test/resources/UrlSignerV4TestAccount.json b/google-cloud-clients/google-cloud-storage/src/test/resources/UrlSignerV4TestAccount.json new file mode 100644 index 000000000000..fe2701badc3d --- /dev/null +++ b/google-cloud-clients/google-cloud-storage/src/test/resources/UrlSignerV4TestAccount.json @@ -0,0 +1,12 @@ +{ + "type": "service_account", + "project_id": "dummy-project-id", + "private_key_id": "ffffffffffffffffffffffffffffffffffffffff", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCsPzMirIottfQ2\nryjQmPWocSEeGo7f7Q4/tMQXHlXFzo93AGgU2t+clEj9L5loNhLVq+vk+qmnyDz5\nQ04y8jVWyMYzzGNNrGRW/yaYqnqlKZCy1O3bmnNjV7EDbC/jE1ZLBY0U3HaSHfn6\nS9ND8MXdgD0/ulRTWwq6vU8/w6i5tYsU7n2LLlQTl1fQ7/emO9nYcCFJezHZVa0H\nmeWsdHwWsok0skwQYQNIzP3JF9BpR5gJT2gNge6KopDesJeLoLzaX7cUnDn+CAnn\nLuLDwwSsIVKyVxhBFsFXPplgpaQRwmGzwEbf/Xpt9qo26w2UMgn30jsOaKlSeAX8\ncS6ViF+tAgMBAAECggEACKRuJCP8leEOhQziUx8Nmls8wmYqO4WJJLyk5xUMUC22\nSI4CauN1e0V8aQmxnIc0CDkFT7qc9xBmsMoF+yvobbeKrFApvlyzNyM7tEa/exh8\nDGD/IzjbZ8VfWhDcUTwn5QE9DCoon9m1sG+MBNlokB3OVOt8LieAAREdEBG43kJu\nyQTOkY9BGR2AY1FnAl2VZ/jhNDyrme3tp1sW1BJrawzR7Ujo8DzlVcS2geKA9at7\n55ua5GbHz3hfzFgjVXDfnkWzId6aHypUyqHrSn1SqGEbyXTaleKTc6Pgv0PgkJjG\nhZazWWdSuf1T5Xbs0OhAK9qraoAzT6cXXvMEvvPt6QKBgQDXcZKqJAOnGEU4b9+v\nOdoh+nssdrIOBNMu1m8mYbUVYS1aakc1iDGIIWNM3qAwbG+yNEIi2xi80a2RMw2T\n9RyCNB7yqCXXVKLBiwg9FbKMai6Vpk2bWIrzahM9on7AhCax/X2AeOp+UyYhFEy6\nUFG4aHb8THscL7b515ukSuKb5QKBgQDMq+9PuaB0eHsrmL6q4vHNi3MLgijGg/zu\nAXaPygSYAwYW8KglcuLZPvWrL6OG0+CrfmaWTLsyIZO4Uhdj7MLvX6yK7IMnagvk\nL3xjgxSklEHJAwi5wFeJ8ai/1MIuCn8p2re3CbwISKpvf7Sgs/W4196P4vKvTiAz\njcTiSYFIKQKBgCjMpkS4O0TakMlGTmsFnqyOneLmu4NyIHgfPb9cA4n/9DHKLKAT\noaWxBPgatOVWs7RgtyGYsk+XubHkpC6f3X0+15mGhFwJ+CSE6tN+l2iF9zp52vqP\nQwkjzm7+pdhZbmaIpcq9m1K+9lqPWJRz/3XXuqi+5xWIZ7NaxGvRjqaNAoGAdK2b\nutZ2y48XoI3uPFsuP+A8kJX+CtWZrlE1NtmS7tnicdd19AtfmTuUL6fz0FwfW4Su\nlQZfPT/5B339CaEiq/Xd1kDor+J7rvUHM2+5p+1A54gMRGCLRv92FQ4EON0RC1o9\nm2I4SHysdO3XmjmdXmfp4BsgAKJIJzutvtbqlakCgYB+Cb10z37NJJ+WgjDt+yT2\nyUNH17EAYgWXryfRgTyi2POHuJitd64Xzuy6oBVs3wVveYFM6PIKXlj8/DahYX5I\nR2WIzoCNLL3bEZ+nC6Jofpb4kspoAeRporj29SgesK6QBYWHWX2H645RkRGYGpDo\n51gjy9m/hSNqBbH2zmh04A==\n-----END PRIVATE KEY-----\n", + "client_email": "test-iam-credentials@dummy-project-id.iam.gserviceaccount.com", + "client_id": "000000000000000000000", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com" +} \ No newline at end of file diff --git a/google-cloud-clients/google-cloud-storage/src/test/resources/UrlSignerV4TestData.json b/google-cloud-clients/google-cloud-storage/src/test/resources/UrlSignerV4TestData.json new file mode 100644 index 000000000000..737d1af22835 --- /dev/null +++ b/google-cloud-clients/google-cloud-storage/src/test/resources/UrlSignerV4TestData.json @@ -0,0 +1,114 @@ +// Assumed constant for all tests: +// - email: test-iam-credentials@dummy-project-id.iam.gserviceaccount.com +// - project: dummy-project-id +[ + { + "description": "Simple GET", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "20190201T090000Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=95e6a13d43a1d1962e667f17397f2b80ac9bdd1669210d5e08e0135df9dff4e56113485dbe429ca2266487b9d1796ebdee2d7cf682a6ef3bb9fbb4c351686fba90d7b621cf1c4eb1fdf126460dd25fa0837dfdde0a9fd98662ce60844c458448fb2b352c203d9969cb74efa4bdb742287744a4f2308afa4af0e0773f55e32e92973619249214b97283b2daa14195244444e33f938138d1e5f561088ce8011f4986dda33a556412594db7c12fc40e1ff3f1bedeb7a42f5bcda0b9567f17f65855f65071fabb88ea12371877f3f77f10e1466fff6ff6973b74a933322ff0949ce357e20abe96c3dd5cfab42c9c83e740a4d32b9e11e146f0eb3404d2e975896f74" + }, + + { + "description": "Simple PUT", + "bucket": "test-bucket", + "object": "test-object", + "method": "PUT", + "expiration": 10, + "timestamp": "20190201T090000Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=8adff1d4285739e31aa68e73767a46bc5511fde377497dbe08481bf5ceb34e29cc9a59921748d8ec3dd4085b7e9b7772a952afedfcdaecb3ae8352275b8b7c867f204e3db85076220a3127a8a9589302fc1181eae13b9b7fe41109ec8cdc93c1e8bac2d7a0cc32a109ca02d06957211326563ab3d3e678a0ba296e298b5fc5e14593c99d444c94724cc4be97015dbff1dca377b508fa0cb7169195de98d0e4ac96c42b918d28c8d92d33e1bd125ce0fb3cd7ad2c45dae65c22628378f6584971b8bf3945b26f2611eb651e9b6a8648970c1ecf386bb71327b082e7296c4e1ee2fc0bdd8983da80af375c817fb1ad491d0bc22c0f51dba0d66e2cffbc90803e47" + }, + + { + "description": "POST for resumable uploads", + "bucket": "test-bucket", + "object": "test-object", + "method": "POST", + "expiration": 10, + "headers": { + "x-goog-resumable": [ "start" ] + }, + "timestamp": "20190201T090000Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host%3Bx-goog-resumable&X-Goog-Signature=4a6d39b23343cedf4c30782aed4b384001828c79ffa3a080a481ea01a640dea0a0ceb58d67a12cef3b243c3f036bb3799c6ee88e8db3eaf7d0bdd4b70a228d0736e07eaa1ee076aff5c6ce09dff1f1f03a0d8ead0d2893408dd3604fdabff553aa6d7af2da67cdba6790006a70240f96717b98f1a6ccb24f00940749599be7ef72aaa5358db63ddd54b2de9e2d6d6a586eac4fe25f36d86fc6ab150418e9c6fa01b732cded226c6d62fc95b72473a4cc55a8257482583fe66d9ab6ede909eb41516a8690946c3e87b0f2052eb0e97e012a14b2f721c42e6e19b8a1cd5658ea36264f10b9b1ada66b8ed5bf7ed7d1708377ac6e5fe608ae361fb594d2e5b24c54" + }, + + { + "description": "Vary expiration and timestamp", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 20, + "timestamp": "20190301T090000Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190301%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190301T090000Z&X-Goog-Expires=20&X-Goog-SignedHeaders=host&X-Goog-Signature=9669ed5b10664dc594c758296580662912cf4bcc5a4ba0b6bf055bcbf6f34eed7bdad664f534962174a924741a0c273a4f67bc1847cef20192a6beab44223bd9d4fbbd749c407b79997598c30f82ddc269ff47ec09fa3afe74e00616d438df0d96a7d8ad0adacfad1dc3286f864d924fe919fb0dce45d3d975c5afe8e13af2db9cc37ba77835f92f7669b61e94c6d562196c1274529e76cfff1564cc2cad7d5387dc8e12f7a5dfd925685fe92c30b43709eee29fa2f66067472cee5423d1a3a4182fe8cea75c9329d181dc6acad7c393cd04f8bf5bc0515127d8ebd65d80c08e19ad03316053ea60033fd1b1fd85a69c576415da3bf0a3718d9ea6d03e0d66f0" + }, + + { + "description": "Vary bucket and object", + "bucket": "test-bucket2", + "object": "test-object2", + "method": "GET", + "expiration": 10, + "timestamp": "20190201T090000Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket2/test-object2?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=36e3d58dfd3ec1d2dd2f24b5ee372a71e811ffaa2162a2b871d26728d0354270bc116face87127532969c4a3967ed05b7309af741e19c7202f3167aa8c2ac420b61417d6451442bb91d7c822cd17be8783f01e05372769c88913561d27e6660dd8259f0081a71f831be6c50283626cbf04494ac10c394b29bb3bce74ab91548f58a37118a452693cf0483d77561fc9cac8f1765d2c724994cca46a83517a10157ee0347a233a2aaeae6e6ab5e204ff8fc5f54f90a3efdb8301d9fff5475d58cd05b181affd657f48203f4fb133c3a3d355b8eefbd10d5a0a5fd70d06e9515460ad74e22334b2cba4b29cae4f6f285cdb92d8f3126d7a1479ca3bdb69c207d860" + }, + + { + "description": "Simple headers", + "bucket": "test-bucket", + "object": "test-object", + "headers": { + "foo": [ "foo-value" ], + "BAR": [ "BAR-value" ] + }, + "method": "GET", + "expiration": 10, + "timestamp": "20190201T090000Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=bar%3Bfoo%3Bhost&X-Goog-Signature=68ecd3b008328ed30d91e2fe37444ed7b9b03f28ed4424555b5161980531ef87db1c3a5bc0265aad5640af30f96014c94fb2dba7479c41bfe1c020eb90c0c6d387d4dd09d4a5df8b60ea50eb6b01cdd786a1e37020f5f95eb8f9b6cd3f65a1f8a8a65c9fcb61ea662959efd9cd73b683f8d8804ef4d6d9b2852419b013368842731359d7f9e6d1139032ceca75d5e67cee5fd0192ea2125e5f2955d38d3d50cf116f3a52e6a62de77f6207f5b95aaa1d7d0f8a46de89ea72e7ea30f21286318d7eba0142232b0deb3a1dc9e1e812a981c66b5ffda3c6b01a8a9d113155792309fd53a3acfd054ca7776e8eec28c26480cd1e3c812f67f91d14217f39a606669d" + }, + + { + "description": "Headers should be trimmed", + "bucket": "test-bucket", + "object": "test-object", + "headers": { + "leading": [ " xyz" ], + "trailing": [ "abc " ], + "collapsed": [ "abc def" ] + }, + "method": "GET", + "expiration": 10, + "timestamp": "20190201T090000Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=collapsed%3Bhost%3Bleading%3Btrailing&X-Goog-Signature=1839511d6238d9ac2bbcbba8b23515b3757db35dfa7b8f9bc4b8b4aa270224df747c812526f1a3bcf294d67ed84cd14e074c36bc090e0a542782934a7c925af4a5ea68123e97533704ce8b08ccdf5fe6b412f89c9fc4de243e29abdb098382c5672188ee3f6fef7131413e252c78e7a35658825ad842a50609e9cc463731e17284ff7a14824c989f87cef22fb99dfec20cfeed69d8b3a08f00b43b8284eecd535e50e982b05cd74c5750cd5f986cfc21a2a05f7f3ab7fc31bd684ed1b823b64d29281e923fc6580c49005552ca19c253de087d9d2df881144e44eda40965cfdb4889bf3a35553c9809f4ed20b8355be481b92b9618952b6a04f3017b36053e15" + }, + + // Headers associated with customer-supplied encryption keys should not be included in the signature + { + "description": "Customer-supplied encryption key", + "bucket": "test-bucket", + "object": "test-object", + "headers": + { + "X-Goog-Encryption-Key": [ "ignored" ], + "X-Goog-Encryption-Key-Sha256": [ "ignored" ], + "X-Goog-Encryption-Algorithm": [ "ignored" ] + }, + "method": "GET", + "expiration": 10, + "timestamp": "20190201T090000Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=95e6a13d43a1d1962e667f17397f2b80ac9bdd1669210d5e08e0135df9dff4e56113485dbe429ca2266487b9d1796ebdee2d7cf682a6ef3bb9fbb4c351686fba90d7b621cf1c4eb1fdf126460dd25fa0837dfdde0a9fd98662ce60844c458448fb2b352c203d9969cb74efa4bdb742287744a4f2308afa4af0e0773f55e32e92973619249214b97283b2daa14195244444e33f938138d1e5f561088ce8011f4986dda33a556412594db7c12fc40e1ff3f1bedeb7a42f5bcda0b9567f17f65855f65071fabb88ea12371877f3f77f10e1466fff6ff6973b74a933322ff0949ce357e20abe96c3dd5cfab42c9c83e740a4d32b9e11e146f0eb3404d2e975896f74" + }, + + { + "description": "List Objects", + "bucket": "test-bucket", + "object": "", + "method": "GET", + "expiration": 10, + "timestamp": "20190201T090000Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=2a1d342f11ddf0c90c669b9ba89ab5099f94049a86351cacbc85845fd5a8b31e1f9c8d484926c19fbd6930da6c8d3049ca8ebcfeefb7b02e53137755d36f97baab479414528b2802f10d94541facb888edf886d91ba124e60cb3801464f61aadc575fc921c99cf8c52e281f7bc0d3e740f529201c469c8e52775b6433687e0c0dca1c6b874614c3c3d09599be1e192c40ad6827416e387bf6e88a5f501f1d8225bce498d134599d0dfe30c9c833c244d3f90cf9595b9f8175658b788ee5c4a90b575fde5e83c645772250c7098373ca754b39d0fc1ebca2f50261a015931541c9827920eba67a1c41613853a1bd23299a1f9f5d583c0feb05ea2f792ba390d27" + } + +] \ No newline at end of file From ed25ac4ea21ad6996e625042931ed40df2a8b208 Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Wed, 27 Mar 2019 12:28:58 -0700 Subject: [PATCH 4/5] Add V4 samples (#4753) --- .../storage/snippets/StorageSnippets.java | 43 +++++++++++++++++++ .../storage/snippets/ITStorageSnippets.java | 26 +++++++++++ 2 files changed, 69 insertions(+) diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/storage/snippets/StorageSnippets.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/storage/snippets/StorageSnippets.java index 015c2d785f89..a3be11591123 100644 --- a/google-cloud-examples/src/main/java/com/google/cloud/examples/storage/snippets/StorageSnippets.java +++ b/google-cloud-examples/src/main/java/com/google/cloud/examples/storage/snippets/StorageSnippets.java @@ -1474,4 +1474,47 @@ public Bucket getBucketPolicyOnly(String bucketName) throws StorageException { // [END storage_get_bucket_policy_only] return bucket; } + + /** Example of how to generate a GET V4 Signed URL */ + public URL generateV4GetObjectSignedUrl(String bucketName, String objectName) throws StorageException { + // [START storage_generate_signed_url_v4] + // Instantiate a Google Cloud Storage client + Storage storage = StorageOptions.getDefaultInstance().getService(); + + // The name of a bucket, e.g. "my-bucket" + // String bucketName = "my-bucket"; + + // The name of an object, e.g. "my-object" + // String objectName = "my-object"; + + BlobInfo blobinfo = BlobInfo.newBuilder(BlobId.of(bucketName, objectName)).build(); + URL url = storage.signUrl(blobinfo, 7, TimeUnit.DAYS, Storage.SignUrlOption.withV4Signature()); + + System.out.println("Generated GET signed URL:"); + System.out.println(url); + // [END storage_generate_signed_url_v4] + return url; + } + + /** Example of how to generate a PUT V4 Signed URL */ + public URL generateV4GPutbjectSignedUrl(String bucketName, String objectName) throws StorageException { + // [START storage_generate_upload_signed_url_v4] + // Instantiate a Google Cloud Storage client + Storage storage = StorageOptions.getDefaultInstance().getService(); + + // The name of a bucket, e.g. "my-bucket" + // String bucketName = "my-bucket"; + + // The name of a new object to upload, e.g. "my-object" + // String objectName = "my-object"; + + BlobInfo blobinfo = BlobInfo.newBuilder(BlobId.of(bucketName, objectName)).build(); + URL url = storage.signUrl(blobinfo, 7, TimeUnit.DAYS, Storage.SignUrlOption.httpMethod(HttpMethod.PUT), + Storage.SignUrlOption.withV4Signature()); + + System.out.println("Generated PUT signed URL:"); + System.out.println(url); + // [END storage_generate_upload_signed_url_v4] + return url; + } } diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/storage/snippets/ITStorageSnippets.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/storage/snippets/ITStorageSnippets.java index 3d44c94ad34a..fceba38277f6 100644 --- a/google-cloud-examples/src/test/java/com/google/cloud/examples/storage/snippets/ITStorageSnippets.java +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/storage/snippets/ITStorageSnippets.java @@ -45,8 +45,10 @@ import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; +import java.io.OutputStream; import java.net.URL; import java.net.URLConnection; +import javax.net.ssl.HttpsURLConnection; import java.nio.file.Files; import java.nio.file.Paths; import java.util.Date; @@ -581,4 +583,28 @@ public void testBucketPolicyOnly() { bucket = storageSnippets.disableBucketPolicyOnly(tempBucket); assertFalse(bucket.getIamConfiguration().isBucketPolicyOnlyEnabled()); } + + @Test + public void testV4SignedURLs() throws IOException{ + String tempBucket = RemoteStorageHelper.generateBucketName(); + Bucket bucket = storageSnippets.createBucket(tempBucket); + assertNotNull(bucket); + String tempObject = "test-upload-signed-url-object"; + URL uploadUrl = storageSnippets.generateV4GPutbjectSignedUrl(tempBucket, tempObject); + HttpsURLConnection connection = (HttpsURLConnection)uploadUrl.openConnection(); + connection.setRequestMethod("PUT"); + connection.setDoOutput(true); + byte[] write = new byte[BLOB_BYTE_CONTENT.length]; + try (OutputStream out = connection.getOutputStream()) { + out.write(BLOB_BYTE_CONTENT); + assertEquals(connection.getResponseCode(), 200); + } + URL downloadUrl = storageSnippets.generateV4GetObjectSignedUrl(tempBucket, tempObject); + connection = (HttpsURLConnection)downloadUrl.openConnection(); + byte[] readBytes = new byte[BLOB_BYTE_CONTENT.length]; + try (InputStream responseStream = connection.getInputStream()) { + assertEquals(BLOB_BYTE_CONTENT.length, responseStream.read(readBytes)); + assertArrayEquals(BLOB_BYTE_CONTENT, readBytes); + } + } } From c10481197c1c2ed0c36fda0707b0bed51d465ff0 Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Wed, 27 Mar 2019 15:16:09 -0700 Subject: [PATCH 5/5] storage: fix v4 samples (#4754) * Add V4 samples * Match C++ samples * Add missing import --- .../storage/snippets/StorageSnippets.java | 20 +++++++++++++++++-- .../storage/snippets/ITStorageSnippets.java | 1 + 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/storage/snippets/StorageSnippets.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/storage/snippets/StorageSnippets.java index a3be11591123..71357c2d61b3 100644 --- a/google-cloud-examples/src/main/java/com/google/cloud/examples/storage/snippets/StorageSnippets.java +++ b/google-cloud-examples/src/main/java/com/google/cloud/examples/storage/snippets/StorageSnippets.java @@ -39,6 +39,7 @@ import com.google.cloud.storage.Bucket; import com.google.cloud.storage.BucketInfo; import com.google.cloud.storage.CopyWriter; +import com.google.cloud.storage.HttpMethod; import com.google.cloud.storage.Storage; import com.google.cloud.storage.Storage.BlobGetOption; import com.google.cloud.storage.Storage.BlobListOption; @@ -1487,11 +1488,16 @@ public URL generateV4GetObjectSignedUrl(String bucketName, String objectName) th // The name of an object, e.g. "my-object" // String objectName = "my-object"; + // Define resource BlobInfo blobinfo = BlobInfo.newBuilder(BlobId.of(bucketName, objectName)).build(); - URL url = storage.signUrl(blobinfo, 7, TimeUnit.DAYS, Storage.SignUrlOption.withV4Signature()); + + // Generate Signed URL + URL url = storage.signUrl(blobinfo, 15, TimeUnit.MINUTES, Storage.SignUrlOption.withV4Signature()); System.out.println("Generated GET signed URL:"); System.out.println(url); + System.out.println("You can use this URL with any user agent, for example:"); + System.out.println("curl '" + url + "'"); // [END storage_generate_signed_url_v4] return url; } @@ -1508,12 +1514,22 @@ public URL generateV4GPutbjectSignedUrl(String bucketName, String objectName) th // The name of a new object to upload, e.g. "my-object" // String objectName = "my-object"; + // Define Resource BlobInfo blobinfo = BlobInfo.newBuilder(BlobId.of(bucketName, objectName)).build(); - URL url = storage.signUrl(blobinfo, 7, TimeUnit.DAYS, Storage.SignUrlOption.httpMethod(HttpMethod.PUT), + + // Generate Signed URL + Map extensionHeaders = new HashMap<>(); + extensionHeaders.put("Content-Type", "application/octet-stream"); + + URL url = storage.signUrl(blobinfo, 15, TimeUnit.MINUTES, Storage.SignUrlOption.httpMethod(HttpMethod.PUT), + Storage.SignUrlOption.withExtHeaders(extensionHeaders), + Storage.SignUrlOption.withV4Signature()); System.out.println("Generated PUT signed URL:"); System.out.println(url); + System.out.println("You can use this URL with any user agent, for example:"); + System.out.println("curl -X PUT -H 'Content-Type: application/octet-stream'--upload-file my-file '" + url + "'"); // [END storage_generate_upload_signed_url_v4] return url; } diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/storage/snippets/ITStorageSnippets.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/storage/snippets/ITStorageSnippets.java index fceba38277f6..12629b634404 100644 --- a/google-cloud-examples/src/test/java/com/google/cloud/examples/storage/snippets/ITStorageSnippets.java +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/storage/snippets/ITStorageSnippets.java @@ -594,6 +594,7 @@ public void testV4SignedURLs() throws IOException{ HttpsURLConnection connection = (HttpsURLConnection)uploadUrl.openConnection(); connection.setRequestMethod("PUT"); connection.setDoOutput(true); + connection.setRequestProperty("Content-Type", "application/octet-stream"); byte[] write = new byte[BLOB_BYTE_CONTENT.length]; try (OutputStream out = connection.getOutputStream()) { out.write(BLOB_BYTE_CONTENT);