Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Storage: Added feature to change host name of signed url #3910

Merged
merged 8 commits into from
Nov 20, 2018
Merged

Storage: Added feature to change host name of signed url #3910

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
3272204
#3140 feature added to change hostname of generated Url
Nov 5, 2018
69a5a14
add test case
Nov 6, 2018
696408c
removed overload method and add method withExtHostName in SignUrlOption
Nov 12, 2018
2b54393
rename EX_HOST to SERVICE_ENDPOINT,rename withExtHostName to withHost…
Nov 13, 2018
0913306
update as per feedback
Nov 17, 2018
f3c3bf7
fix formatting
Nov 17, 2018
3194704
update comments
Nov 20, 2018
17c799f
update Signed URLs Comment
Nov 20, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion ...le-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -892,7 +892,7 @@ class SignUrlOption implements Serializable {
private final Object value;

enum Option {
HTTP_METHOD, CONTENT_TYPE, MD5, EXT_HEADERS, SERVICE_ACCOUNT_CRED
HTTP_METHOD, CONTENT_TYPE, MD5, EXT_HEADERS, SERVICE_ACCOUNT_CRED, HOST_NAME
}

private SignUrlOption(Option option, Object value) {
Expand Down Expand Up @@ -953,6 +953,13 @@ public static SignUrlOption withExtHeaders(Map<String, String> extHeaders) {
public static SignUrlOption signWith(ServiceAccountSigner signer) {
return new SignUrlOption(Option.SERVICE_ACCOUNT_CRED, signer);
}

/**
* Use a different host name than the default host name 'storage.googleapis.com'
*/
public static SignUrlOption withHostName(String hostName){
return new SignUrlOption(Option.HOST_NAME, hostName);
}
}

/**
Expand Down Expand Up @@ -2107,6 +2114,8 @@ public static Builder newBuilder() {
* granularity supported is 1 second, finer granularities will be truncated
* @param unit time unit of the {@code duration} parameter
* @param options optional URL signing options
* {@code SignUrlOption.withHostName()} option to set a custom host name instead of using
* https://storage.googleapis.com.
* @throws IllegalStateException if {@link SignUrlOption#signWith(ServiceAccountSigner)} was not
* used and no implementation of {@link ServiceAccountSigner} was provided to
* {@link StorageOptions}
Expand Down
13 changes: 12 additions & 1 deletion ...loud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,10 @@ final class StorageImpl extends BaseService<StorageOptions> implements Storage {
private static final String EMPTY_BYTE_ARRAY_MD5 = "1B2M2Y8AsgTpgAmY7PhCfg==";
private static final String EMPTY_BYTE_ARRAY_CRC32C = "AAAAAA==";
private static final String PATH_DELIMITER = "/";
/**
* Signed URLs are only supported through the GCS XML API endpoint.
*/
private static final String STORAGE_XML_HOST_NAME = "https://storage.googleapis.com";

private static final Function<Tuple<Storage, Boolean>, Boolean> DELETE_FUNCTION =
new Function<Tuple<Storage, Boolean>, Boolean>() {
Expand Down Expand Up @@ -535,7 +539,14 @@ public URL signUrl(BlobInfo blobInfo, long duration, TimeUnit unit, SignUrlOptio
SignatureInfo signatureInfo = buildSignatureInfo(optionMap, blobInfo, expiration, path);
byte[] signatureBytes =
credentials.sign(signatureInfo.constructUnsignedPayload().getBytes(UTF_8));
StringBuilder stBuilder = new StringBuilder("https://storage.googleapis.com").append(path);
StringBuilder stBuilder = new StringBuilder();
if (optionMap.get(SignUrlOption.Option.HOST_NAME) == null) {
stBuilder.append(STORAGE_XML_HOST_NAME).append(path);
}
else {
stBuilder.append(optionMap.get(SignUrlOption.Option.HOST_NAME)).append(path);
}

String signature =
URLEncoder.encode(BaseEncoding.base64().encode(signatureBytes), UTF_8.name());
stBuilder.append("?GoogleAccessId=").append(credentials.getAccount());
Expand Down
289 changes: 289 additions & 0 deletions ...-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1633,6 +1633,47 @@ public void testSignUrl()
assertTrue(
signer.verify(BaseEncoding.base64().decode(URLDecoder.decode(signature, UTF_8.name()))));
}

@Test
public void testSignUrlWithHostName()
throws NoSuchAlgorithmException, InvalidKeyException, SignatureException,
UnsupportedEncodingException {
EasyMock.replay(storageRpcMock);
ServiceAccountCredentials credentials =
new ServiceAccountCredentials(null, ACCOUNT, privateKey, null, null);
storage = options.toBuilder().setCredentials(credentials).build().getService();
URL url = storage.signUrl(BLOB_INFO1, 14, TimeUnit.DAYS, Storage.SignUrlOption.withHostName("https://example.com"));
String stringUrl = url.toString();
String expectedUrl =
new StringBuilder("https://example.com/")
.append(BUCKET_NAME1)
.append('/')
.append(BLOB_NAME1)
.append("?GoogleAccessId=")
.append(ACCOUNT)
.append("&Expires=")
.append(42L + 1209600)
.append("&Signature=")
.toString();
assertTrue(stringUrl.startsWith(expectedUrl));
String signature = stringUrl.substring(expectedUrl.length());

StringBuilder signedMessageBuilder = new StringBuilder();
signedMessageBuilder
.append(HttpMethod.GET)
.append("\n\n\n")
.append(42L + 1209600)
.append("\n/")
.append(BUCKET_NAME1)
.append('/')
.append(BLOB_NAME1);

Signature signer = Signature.getInstance("SHA256withRSA");
signer.initVerify(publicKey);
signer.update(signedMessageBuilder.toString().getBytes(UTF_8));
assertTrue(
signer.verify(BaseEncoding.base64().decode(URLDecoder.decode(signature, UTF_8.name()))));
}

@Test
public void testSignUrlLeadingSlash()
Expand Down Expand Up @@ -1675,6 +1716,48 @@ public void testSignUrlLeadingSlash()
assertTrue(
signer.verify(BaseEncoding.base64().decode(URLDecoder.decode(signature, UTF_8.name()))));
}

@Test
public void testSignUrlLeadingSlashWithHostName()
throws NoSuchAlgorithmException, InvalidKeyException, SignatureException,
UnsupportedEncodingException {
String blobName = "/b1";
EasyMock.replay(storageRpcMock);
ServiceAccountCredentials credentials =
new ServiceAccountCredentials(null, ACCOUNT, privateKey, null, null);
storage = options.toBuilder().setCredentials(credentials).build().getService();
URL url =
storage.signUrl(BlobInfo.newBuilder(BUCKET_NAME1, blobName).build(), 14, TimeUnit.DAYS, Storage.SignUrlOption.withHostName("https://example.com"));
String escapedBlobName = UrlEscapers.urlFragmentEscaper().escape(blobName);
String stringUrl = url.toString();
String expectedUrl =
new StringBuilder("https://example.com/")
.append(BUCKET_NAME1)
.append(escapedBlobName)
.append("?GoogleAccessId=")
.append(ACCOUNT)
.append("&Expires=")
.append(42L + 1209600)
.append("&Signature=")
.toString();
assertTrue(stringUrl.startsWith(expectedUrl));
String signature = stringUrl.substring(expectedUrl.length());

StringBuilder signedMessageBuilder = new StringBuilder();
signedMessageBuilder
.append(HttpMethod.GET)
.append("\n\n\n")
.append(42L + 1209600)
.append("\n/")
.append(BUCKET_NAME1)
.append(escapedBlobName);

Signature signer = Signature.getInstance("SHA256withRSA");
signer.initVerify(publicKey);
signer.update(signedMessageBuilder.toString().getBytes(UTF_8));
assertTrue(
signer.verify(BaseEncoding.base64().decode(URLDecoder.decode(signature, UTF_8.name()))));
}

@Test
public void testSignUrlWithOptions()
Expand Down Expand Up @@ -1727,6 +1810,59 @@ public void testSignUrlWithOptions()
assertTrue(
signer.verify(BaseEncoding.base64().decode(URLDecoder.decode(signature, UTF_8.name()))));
}

@Test
public void testSignUrlWithOptionsAndHostName()
throws NoSuchAlgorithmException, InvalidKeyException, SignatureException,
UnsupportedEncodingException {
EasyMock.replay(storageRpcMock);
ServiceAccountCredentials credentials =
new ServiceAccountCredentials(null, ACCOUNT, privateKey, null, null);
storage = options.toBuilder().setCredentials(credentials).build().getService();
URL url =
storage.signUrl(
BLOB_INFO1,
14,
TimeUnit.DAYS,
Storage.SignUrlOption.httpMethod(HttpMethod.POST),
Storage.SignUrlOption.withContentType(),
Storage.SignUrlOption.withMd5(),
Storage.SignUrlOption.withHostName("https://example.com"));
String stringUrl = url.toString();
String expectedUrl =
new StringBuilder("https://example.com/")
.append(BUCKET_NAME1)
.append('/')
.append(BLOB_NAME1)
.append("?GoogleAccessId=")
.append(ACCOUNT)
.append("&Expires=")
.append(42L + 1209600)
.append("&Signature=")
.toString();
assertTrue(stringUrl.startsWith(expectedUrl));
String signature = stringUrl.substring(expectedUrl.length());

StringBuilder signedMessageBuilder = new StringBuilder();
signedMessageBuilder
.append(HttpMethod.POST)
.append('\n')
.append(BLOB_INFO1.getMd5())
.append('\n')
.append(BLOB_INFO1.getContentType())
.append('\n')
.append(42L + 1209600)
.append("\n/")
.append(BUCKET_NAME1)
.append('/')
.append(BLOB_NAME1);

Signature signer = Signature.getInstance("SHA256withRSA");
signer.initVerify(publicKey);
signer.update(signedMessageBuilder.toString().getBytes(UTF_8));
assertTrue(
signer.verify(BaseEncoding.base64().decode(URLDecoder.decode(signature, UTF_8.name()))));
}

@Test
public void testSignUrlForBlobWithSpecialChars()
Expand Down Expand Up @@ -1780,6 +1916,58 @@ public void testSignUrlForBlobWithSpecialChars()
}
}

@Test
public void testSignUrlForBlobWithSpecialCharsAndHostName()
throws NoSuchAlgorithmException, InvalidKeyException, SignatureException,
UnsupportedEncodingException {
// List of chars under test were taken from
// https://en.wikipedia.org/wiki/Percent-encoding#Percent-encoding_reserved_characters
char[] specialChars =
new char[] {
'!', '#', '$', '&', '\'', '(', ')', '*', '+', ',', ':', ';', '=', '?', '@', '[', ']'
};
EasyMock.replay(storageRpcMock);
ServiceAccountCredentials credentials =
new ServiceAccountCredentials(null, ACCOUNT, privateKey, null, null);
storage = options.toBuilder().setCredentials(credentials).build().getService();

for (char specialChar : specialChars) {
String blobName = "/a" + specialChar + "b";
URL url =
storage.signUrl(BlobInfo.newBuilder(BUCKET_NAME1, blobName).build(), 14, TimeUnit.DAYS, Storage.SignUrlOption.withHostName("https://example.com"));
String escapedBlobName =
UrlEscapers.urlFragmentEscaper().escape(blobName).replace("?", "%3F");
String stringUrl = url.toString();
String expectedUrl =
new StringBuilder("https://example.com/")
.append(BUCKET_NAME1)
.append(escapedBlobName)
.append("?GoogleAccessId=")
.append(ACCOUNT)
.append("&Expires=")
.append(42L + 1209600)
.append("&Signature=")
.toString();
assertTrue(stringUrl.startsWith(expectedUrl));
String signature = stringUrl.substring(expectedUrl.length());

StringBuilder signedMessageBuilder = new StringBuilder();
signedMessageBuilder
.append(HttpMethod.GET)
.append("\n\n\n")
.append(42L + 1209600)
.append("\n/")
.append(BUCKET_NAME1)
.append(escapedBlobName);

Signature signer = Signature.getInstance("SHA256withRSA");
signer.initVerify(publicKey);
signer.update(signedMessageBuilder.toString().getBytes(UTF_8));
assertTrue(
signer.verify(BaseEncoding.base64().decode(URLDecoder.decode(signature, UTF_8.name()))));
}
}

@Test
public void testSignUrlWithExtHeaders()
throws NoSuchAlgorithmException, InvalidKeyException, SignatureException,
Expand Down Expand Up @@ -1836,7 +2024,65 @@ public void testSignUrlWithExtHeaders()
assertTrue(
signer.verify(BaseEncoding.base64().decode(URLDecoder.decode(signature, UTF_8.name()))));
}

@Test
public void testSignUrlWithExtHeadersAndHostName()
throws NoSuchAlgorithmException, InvalidKeyException, SignatureException,
UnsupportedEncodingException {
EasyMock.replay(storageRpcMock);
ServiceAccountCredentials credentials =
new ServiceAccountCredentials(null, ACCOUNT, privateKey, null, null);
storage = options.toBuilder().setCredentials(credentials).build().getService();
Map<String, String> extHeaders = new HashMap<String, String>();
extHeaders.put("x-goog-acl", "public-read");
extHeaders.put("x-goog-meta-owner", "myself");
URL url =
storage.signUrl(
BLOB_INFO1,
14,
TimeUnit.DAYS,
Storage.SignUrlOption.httpMethod(HttpMethod.PUT),
Storage.SignUrlOption.withContentType(),
Storage.SignUrlOption.withExtHeaders(extHeaders),
Storage.SignUrlOption.withHostName("https://example.com"));
String stringUrl = url.toString();
String expectedUrl =
new StringBuilder("https://example.com/")
.append(BUCKET_NAME1)
.append('/')
.append(BLOB_NAME1)
.append("?GoogleAccessId=")
.append(ACCOUNT)
.append("&Expires=")
.append(42L + 1209600)
.append("&Signature=")
.toString();
assertTrue(stringUrl.startsWith(expectedUrl));
String signature = stringUrl.substring(expectedUrl.length());

StringBuilder signedMessageBuilder = new StringBuilder();
signedMessageBuilder
.append(HttpMethod.PUT)
.append('\n')
.append('\n')
.append(BLOB_INFO1.getContentType())
.append('\n')
.append(42L + 1209600)
.append('\n')
.append("x-goog-acl:public-read\n")
.append("x-goog-meta-owner:myself\n")
.append('/')
.append(BUCKET_NAME1)
.append('/')
.append(BLOB_NAME1);

Signature signer = Signature.getInstance("SHA256withRSA");
signer.initVerify(publicKey);
signer.update(signedMessageBuilder.toString().getBytes(UTF_8));
assertTrue(
signer.verify(BaseEncoding.base64().decode(URLDecoder.decode(signature, UTF_8.name()))));
}

@Test
public void testSignUrlForBlobWithSlashes()
throws NoSuchAlgorithmException, InvalidKeyException, SignatureException,
Expand Down Expand Up @@ -1879,6 +2125,49 @@ public void testSignUrlForBlobWithSlashes()
assertTrue(
signer.verify(BaseEncoding.base64().decode(URLDecoder.decode(signature, UTF_8.name()))));
}

@Test
public void testSignUrlForBlobWithSlashesAndHostName()
throws NoSuchAlgorithmException, InvalidKeyException, SignatureException,
UnsupportedEncodingException {
EasyMock.replay(storageRpcMock);
ServiceAccountCredentials credentials =
new ServiceAccountCredentials(null, ACCOUNT, privateKey, null, null);
storage = options.toBuilder().setCredentials(credentials).build().getService();

String blobName = "/foo/bar/baz #%20other cool stuff.txt";
URL url =
storage.signUrl(BlobInfo.newBuilder(BUCKET_NAME1, blobName).build(), 14, TimeUnit.DAYS, Storage.SignUrlOption.withHostName("https://example.com"));
String escapedBlobName = UrlEscapers.urlFragmentEscaper().escape(blobName);
String stringUrl = url.toString();
String expectedUrl =
new StringBuilder("https://example.com/")
.append(BUCKET_NAME1)
.append(escapedBlobName)
.append("?GoogleAccessId=")
.append(ACCOUNT)
.append("&Expires=")
.append(42L + 1209600)
.append("&Signature=")
.toString();
assertTrue(stringUrl.startsWith(expectedUrl));
String signature = stringUrl.substring(expectedUrl.length());

StringBuilder signedMessageBuilder = new StringBuilder();
signedMessageBuilder
.append(HttpMethod.GET)
.append("\n\n\n")
.append(42L + 1209600)
.append("\n/")
.append(BUCKET_NAME1)
.append(escapedBlobName);

Signature signer = Signature.getInstance("SHA256withRSA");
signer.initVerify(publicKey);
signer.update(signedMessageBuilder.toString().getBytes(UTF_8));
assertTrue(
signer.verify(BaseEncoding.base64().decode(URLDecoder.decode(signature, UTF_8.name()))));
}

@Test
public void testGetAllArray() {
Expand Down