Storage: Add V4 signing support (#4692)

* Add support for V4 signing

* (storage) WIP: Add V4 signing support

* (storage) Add V4 signing support

* Add V4 samples (#4753)

* storage: fix v4 samples (#4754)

* Add V4 samples

* Match C++ samples

* Add missing import

google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/CanonicalExtensionHeadersSerializer.java

@@ -32,31 +32,29 @@
 public class CanonicalExtensionHeadersSerializer {
 
   private static final char HEADER_SEPARATOR = ':';
+  private static final char HEADER_NAME_SEPARATOR = ';';
+
+  private final Storage.SignUrlOption.SignatureVersion signatureVersion;
+
+  public CanonicalExtensionHeadersSerializer(
+      Storage.SignUrlOption.SignatureVersion signatureVersion) {
+    this.signatureVersion = signatureVersion;
+  }
+
+  public CanonicalExtensionHeadersSerializer() {
+    // TODO switch this when V4 becomes default
+    this.signatureVersion = Storage.SignUrlOption.SignatureVersion.V2;
+  }
 
   public StringBuilder serialize(Map<String, String> canonicalizedExtensionHeaders) {
 
     StringBuilder serializedHeaders = new StringBuilder();
 
     if (canonicalizedExtensionHeaders == null || canonicalizedExtensionHeaders.isEmpty()) {
-
       return serializedHeaders;
     }
 
-    // Make all custom header names lowercase.
-    Map<String, String> lowercaseHeaders = new HashMap<>();
-    for (String headerName : new ArrayList<>(canonicalizedExtensionHeaders.keySet())) {
-
-      String lowercaseHeaderName = headerName.toLowerCase();
-
-      // If present, remove the x-goog-encryption-key and x-goog-encryption-key-sha256 headers.
-      if ("x-goog-encryption-key".equals(lowercaseHeaderName)
-          || "x-goog-encryption-key-sha256".equals(lowercaseHeaderName)) {
-
-        continue;
-      }
-
-      lowercaseHeaders.put(lowercaseHeaderName, canonicalizedExtensionHeaders.get(headerName));
-    }
+    Map<String, String> lowercaseHeaders = getLowercaseHeaders(canonicalizedExtensionHeaders);
 
     // Sort all custom headers by header name using a lexicographical sort by code point value.
     List<String> sortedHeaderNames = new ArrayList<>(lowercaseHeaders.keySet());
@@ -81,4 +79,47 @@ public StringBuilder serialize(Map<String, String> canonicalizedExtensionHeaders
     // Concatenate all custom headers
     return serializedHeaders;
   }
+
+  public StringBuilder serializeHeaderNames(Map<String, String> canonicalizedExtensionHeaders) {
+    StringBuilder serializedHeaders = new StringBuilder();
+
+    if (canonicalizedExtensionHeaders == null || canonicalizedExtensionHeaders.isEmpty()) {
+      return serializedHeaders;
+    }
+    Map<String, String> lowercaseHeaders = getLowercaseHeaders(canonicalizedExtensionHeaders);
+
+    List<String> sortedHeaderNames = new ArrayList<>(lowercaseHeaders.keySet());
+    Collections.sort(sortedHeaderNames);
+
+    for (String headerName : sortedHeaderNames) {
+      serializedHeaders.append(headerName).append(HEADER_NAME_SEPARATOR);
+    }
+
+    serializedHeaders.setLength(serializedHeaders.length() - 1); // remove trailing semicolon
+
+    return serializedHeaders;
+  }
+
+  private Map<String, String> getLowercaseHeaders(
+      Map<String, String> canonicalizedExtensionHeaders) {
+    // Make all custom header names lowercase.
+    Map<String, String> lowercaseHeaders = new HashMap<>();
+    for (String headerName : new ArrayList<>(canonicalizedExtensionHeaders.keySet())) {
+
+      String lowercaseHeaderName = headerName.toLowerCase();
+
+      // If present and we're V2, remove the x-goog-encryption-key and x-goog-encryption-key-sha256
+      // headers. (CSEK headers are allowed for V4)
+      if (Storage.SignUrlOption.SignatureVersion.V2.equals(signatureVersion)
+          && ("x-goog-encryption-key".equals(lowercaseHeaderName)
+              || "x-goog-encryption-key-sha256".equals(lowercaseHeaderName))) {
+
+        continue;
+      }
+
+      lowercaseHeaders.put(lowercaseHeaderName, canonicalizedExtensionHeaders.get(headerName));
+    }
+
+    return lowercaseHeaders;
+  }
 }

google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/SignatureInfo.java

@@ -18,8 +18,16 @@
 
 import static com.google.common.base.Preconditions.checkArgument;
 
+import com.google.common.collect.ImmutableMap;
+import com.google.common.hash.Hashing;
+import com.google.common.net.UrlEscapers;
 import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.HashMap;
 import java.util.Map;
+import java.util.TimeZone;
 
 /**
  * Signature Info holds payload components of the string that requires signing.
@@ -31,30 +39,70 @@
 public class SignatureInfo {
 
   public static final char COMPONENT_SEPARATOR = '\n';
+  public static final String GOOG4_RSA_SHA256 = "GOOG4-RSA-SHA256";
+  public static final String SCOPE = "/auto/storage/goog4_request";
 
   private final HttpMethod httpVerb;
   private final String contentMd5;
   private final String contentType;
   private final long expiration;
   private final Map<String, String> canonicalizedExtensionHeaders;
   private final URI canonicalizedResource;
+  private final Storage.SignUrlOption.SignatureVersion signatureVersion;
+  private final String accountEmail;
+  private final long timestamp;
+
+  private final String yearMonthDay;
+  private final String exactDate;
 
   private SignatureInfo(Builder builder) {
     this.httpVerb = builder.httpVerb;
     this.contentMd5 = builder.contentMd5;
     this.contentType = builder.contentType;
     this.expiration = builder.expiration;
-    this.canonicalizedExtensionHeaders = builder.canonicalizedExtensionHeaders;
     this.canonicalizedResource = builder.canonicalizedResource;
+    this.signatureVersion = builder.signatureVersion;
+    this.accountEmail = builder.accountEmail;
+    this.timestamp = builder.timestamp;
+
+    if (Storage.SignUrlOption.SignatureVersion.V4.equals(signatureVersion)
+        && (!builder.canonicalizedExtensionHeaders.containsKey("host"))) {
+      canonicalizedExtensionHeaders =
+          new ImmutableMap.Builder<String, String>()
+              .putAll(builder.canonicalizedExtensionHeaders)
+              .put("host", "storage.googleapis.com")
+              .build();
+    } else {
+      canonicalizedExtensionHeaders = builder.canonicalizedExtensionHeaders;
+    }
+
+    Date date = new Date(timestamp);
+
+    SimpleDateFormat yearMonthDayFormat = new SimpleDateFormat("yyyyMMdd");
+    SimpleDateFormat exactDateFormat = new SimpleDateFormat("yyyyMMdd'T'HHmmss'Z'");
+
+    yearMonthDayFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+    exactDateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+
+    yearMonthDay = yearMonthDayFormat.format(date);
+    exactDate = exactDateFormat.format(date);
   }
 
   /**
    * Constructs payload to be signed.
    *
-   * @return paylod to sign
+   * @return payload to sign
    * @see <a href="https://cloud.google.com/storage/docs/access-control#Signed-URLs">Signed URLs</a>
    */
   public String constructUnsignedPayload() {
+    // TODO reverse order when V4 becomes default
+    if (Storage.SignUrlOption.SignatureVersion.V4.equals(signatureVersion)) {
+      return constructV4UnsignedPayload();
+    }
+    return constructV2UnsignedPayload();
+  }
+
+  private String constructV2UnsignedPayload() {
     StringBuilder payload = new StringBuilder();
 
     payload.append(httpVerb.name()).append(COMPONENT_SEPARATOR);
@@ -67,19 +115,72 @@ public String constructUnsignedPayload() {
       payload.append(contentType);
     }
     payload.append(COMPONENT_SEPARATOR);
-
     payload.append(expiration).append(COMPONENT_SEPARATOR);
 
     if (canonicalizedExtensionHeaders != null) {
       payload.append(
-          new CanonicalExtensionHeadersSerializer().serialize(canonicalizedExtensionHeaders));
+          new CanonicalExtensionHeadersSerializer(Storage.SignUrlOption.SignatureVersion.V2)
+              .serialize(canonicalizedExtensionHeaders));
     }
 
     payload.append(canonicalizedResource);
 
     return payload.toString();
   }
 
+  private String constructV4UnsignedPayload() {
+    StringBuilder payload = new StringBuilder();
+
+    payload.append(GOOG4_RSA_SHA256).append(COMPONENT_SEPARATOR);
+    payload.append(exactDate).append(COMPONENT_SEPARATOR);
+    payload.append(yearMonthDay).append(SCOPE).append(COMPONENT_SEPARATOR);
+    payload.append(constructV4CanonicalRequestHash());
+
+    return payload.toString();
+  }
+
+  private String constructV4CanonicalRequestHash() {
+    StringBuilder canonicalRequest = new StringBuilder();
+
+    CanonicalExtensionHeadersSerializer serializer =
+        new CanonicalExtensionHeadersSerializer(Storage.SignUrlOption.SignatureVersion.V4);
+
+    canonicalRequest.append(httpVerb.name()).append(COMPONENT_SEPARATOR);
+    canonicalRequest.append(canonicalizedResource).append(COMPONENT_SEPARATOR);
+    canonicalRequest.append(constructV4QueryString()).append(COMPONENT_SEPARATOR);
+    canonicalRequest
+        .append(serializer.serialize(canonicalizedExtensionHeaders))
+        .append(COMPONENT_SEPARATOR);
+    canonicalRequest
+        .append(serializer.serializeHeaderNames(canonicalizedExtensionHeaders))
+        .append(COMPONENT_SEPARATOR);
+    canonicalRequest.append("UNSIGNED-PAYLOAD");
+
+    return Hashing.sha256()
+        .hashString(canonicalRequest.toString(), StandardCharsets.UTF_8)
+        .toString();
+  }
+
+  public String constructV4QueryString() {
+    StringBuilder signedHeaders =
+        new CanonicalExtensionHeadersSerializer(Storage.SignUrlOption.SignatureVersion.V4)
+            .serializeHeaderNames(canonicalizedExtensionHeaders);
+
+    StringBuilder queryString = new StringBuilder();
+    queryString.append("X-Goog-Algorithm=").append(GOOG4_RSA_SHA256).append("&");
+    queryString.append(
+        "X-Goog-Credential="
+            + UrlEscapers.urlFormParameterEscaper()
+                .escape(accountEmail + "/" + yearMonthDay + SCOPE)
+            + "&");
+    queryString.append("X-Goog-Date=" + exactDate + "&");
+    queryString.append("X-Goog-Expires=" + expiration + "&");
+    queryString.append(
+        "X-Goog-SignedHeaders="
+            + UrlEscapers.urlFormParameterEscaper().escape(signedHeaders.toString()));
+    return queryString.toString();
+  }
+
   public HttpMethod getHttpVerb() {
     return httpVerb;
   }
@@ -104,6 +205,18 @@ public URI getCanonicalizedResource() {
     return canonicalizedResource;
   }
 
+  public Storage.SignUrlOption.SignatureVersion getSignatureVersion() {
+    return signatureVersion;
+  }
+
+  public long getTimestamp() {
+    return timestamp;
+  }
+
+  public String getAccountEmail() {
+    return accountEmail;
+  }
+
   public static final class Builder {
 
     private final HttpMethod httpVerb;
@@ -112,6 +225,9 @@ public static final class Builder {
     private final long expiration;
     private Map<String, String> canonicalizedExtensionHeaders;
     private final URI canonicalizedResource;
+    private Storage.SignUrlOption.SignatureVersion signatureVersion;
+    private String accountEmail;
+    private long timestamp;
 
     /**
      * Constructs builder.
@@ -134,6 +250,9 @@ public Builder(SignatureInfo signatureInfo) {
       this.expiration = signatureInfo.expiration;
       this.canonicalizedExtensionHeaders = signatureInfo.canonicalizedExtensionHeaders;
       this.canonicalizedResource = signatureInfo.canonicalizedResource;
+      this.signatureVersion = signatureInfo.signatureVersion;
+      this.accountEmail = signatureInfo.accountEmail;
+      this.timestamp = signatureInfo.timestamp;
     }
 
     public Builder setContentMd5(String contentMd5) {
@@ -155,12 +274,41 @@ public Builder setCanonicalizedExtensionHeaders(
       return this;
     }
 
+    public Builder setSignatureVersion(Storage.SignUrlOption.SignatureVersion signatureVersion) {
+      this.signatureVersion = signatureVersion;
+
+      return this;
+    }
+
+    public Builder setAccountEmail(String accountEmail) {
+      this.accountEmail = accountEmail;
+
+      return this;
+    }
+
+    public Builder setTimestamp(long timestamp) {
+      this.timestamp = timestamp;
+
+      return this;
+    }
+
     /** Creates an {@code SignatureInfo} object from this builder. */
     public SignatureInfo build() {
       checkArgument(httpVerb != null, "Required HTTP method");
       checkArgument(canonicalizedResource != null, "Required canonicalized resource");
       checkArgument(expiration >= 0, "Expiration must be greater than or equal to zero");
 
+      if (Storage.SignUrlOption.SignatureVersion.V4.equals(signatureVersion)) {
+        checkArgument(accountEmail != null, "Account email required to use V4 signing");
+        checkArgument(timestamp > 0, "Timestamp required to use V4 signing");
+        checkArgument(
+            expiration <= 604800, "Expiration can't be longer than 7 days to use V4 signing");
+      }
+
+      if (canonicalizedExtensionHeaders == null) {
+        canonicalizedExtensionHeaders = new HashMap<>();
+      }
+
       return new SignatureInfo(this);
     }
   }

google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java

@@ -886,9 +886,15 @@ enum Option {
       MD5,
       EXT_HEADERS,
       SERVICE_ACCOUNT_CRED,
+      SIGNATURE_VERSION,
       HOST_NAME
     }
 
+    enum SignatureVersion {
+      V2,
+      V4
+    }
+
     private SignUrlOption(Option option, Object value) {
       this.option = option;
       this.value = value;
@@ -937,6 +943,23 @@ public static SignUrlOption withExtHeaders(Map<String, String> extHeaders) {
       return new SignUrlOption(Option.EXT_HEADERS, extHeaders);
     }
 
+    /**
+     * Use if signature version should be V2. This is the default if neither this or {@code
+     * withV4Signature()} is called.
+     */
+    public static SignUrlOption withV2Signature() {
+      return new SignUrlOption(Option.SIGNATURE_VERSION, SignatureVersion.V2);
+    }
+
+    /**
+     * Use if signature version should be V4. Note that V4 Signed URLs can't have an expiration
+     * longer than 7 days. V2 will be the default if neither this or {@code withV2Signature()} is
+     * called.
+     */
+    public static SignUrlOption withV4Signature() {
+      return new SignUrlOption(Option.SIGNATURE_VERSION, SignatureVersion.V4);
+    }
+
     /**
      * Provides a service account signer to sign the URL. If not provided an attempt will be made to
      * get it from the environment.
@@ -2101,6 +2124,16 @@ Blob create(
    *     TimeUnit.DAYS);
    * }</pre>
    *
+   * <p>Example of creating a signed URL passing the {@link SignUrlOption#withV4Signature()} option,
+   * which enables V4 signing.
+   *
+   * <pre>{@code
+   * String bucketName = "my_unique_bucket";
+   * String blobName = "my_blob_name";
+   * URL signedUrl = storage.signUrl(BlobInfo.newBuilder(bucketName, blobName).build(),
+   *     7, TimeUnit.DAYS, Storage.SignUrlOption.withV4Signature());
+   * }</pre>
+   *
    * <p>Example of creating a signed URL passing the {@link
    * SignUrlOption#signWith(ServiceAccountSigner)} option, that will be used for signing the URL.
    *