From 02ae002eeb75377ee6d52c4fc3fdc9f8b23f80ad Mon Sep 17 00:00:00 2001 From: Brandon Yarbrough Date: Tue, 22 Mar 2016 17:09:10 -0700 Subject: [PATCH 1/9] Add a new example for using Customer-Supplied Encryption Keys. --- storage/json-api/pom.xml | 15 +- ...CustomerSuppliedEncryptionKeysSamples.java | 315 ++++++++++++++++++ .../src/main/resources/client_secrets.json | 1 + 3 files changed, 330 insertions(+), 1 deletion(-) create mode 100644 storage/json-api/src/main/java/CustomerSuppliedEncryptionKeysSamples.java create mode 100644 storage/json-api/src/main/resources/client_secrets.json diff --git a/storage/json-api/pom.xml b/storage/json-api/pom.xml index a23ae40f1f0..ca7f59c6261 100644 --- a/storage/json-api/pom.xml +++ b/storage/json-api/pom.xml @@ -19,6 +19,14 @@ + + + ${basedir}/src/main/resources + + **/* + + + org.codehaus.mojo @@ -42,7 +50,12 @@ com.google.apis google-api-services-storage - v1-rev18-1.19.0 + v1-rev65-1.21.0 + + + com.google.oauth-client + google-oauth-client-jetty + 1.21.0 diff --git a/storage/json-api/src/main/java/CustomerSuppliedEncryptionKeysSamples.java b/storage/json-api/src/main/java/CustomerSuppliedEncryptionKeysSamples.java new file mode 100644 index 00000000000..c7603051b0c --- /dev/null +++ b/storage/json-api/src/main/java/CustomerSuppliedEncryptionKeysSamples.java @@ -0,0 +1,315 @@ +import com.google.api.client.auth.oauth2.Credential; +import com.google.api.client.extensions.java6.auth.oauth2.AuthorizationCodeInstalledApp; +import com.google.api.client.extensions.jetty.auth.oauth2.LocalServerReceiver; +import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeFlow; +import com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.googleapis.json.GoogleJsonResponseException; +import com.google.api.client.http.HttpHeaders; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.http.InputStreamContent; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.api.client.util.store.DataStoreFactory; +import com.google.api.client.util.store.FileDataStoreFactory; +import com.google.api.services.storage.Storage; +import com.google.api.services.storage.StorageScopes; +import com.google.api.services.storage.model.RewriteResponse; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.Collections; + +/** + * Demonstrates the use of GCS's CSEK features via the Java API client library + * + * This program demonstrates some quick, basic examples of using GCS's CSEK functionality. + * + *

When run, it begins by uploading an object named "encrypted_file.txt" to the specified bucket + * that will be protected with a provided CSEK.

+ * + *

Next, it will fetch that object by providing that same CSEK to GCS.

+ * + *

Finally, it will rotate that key to a new value.

+ **/ +class CustomerSuppliedEncryptionKeysSamples { + + private static final java.io.File DATA_STORE_DIR = + new java.io.File(System.getProperty("user.home"), ".store/storage_sample"); + + // You can (and should) generate your own CSEK Key! Try running this from the command line: + // python -c 'import base64; import os; print(base64.encodestring(os.urandom(32)))' + // Also, these encryption keys are included here for simplicity, but please remember that + // private keys should not be stored in source code. + private static final String CSEK_KEY = "4RzDI0TeWa9M/nAvYH05qbCskPaSU/CFV5HeCxk0IUA="; + + // You can use openssl to quickly calculate the hash of your key. Try running this: + // openssl base64 -d <<< YOUR_KEY_FROM_ABOVE | openssl dgst -sha256 -binary | openssl base64 + private static final String CSEK_KEY_HASH = "aanjNC2nwso8e2FqcWILC3/Tt1YumvIwEj34kr6PRpI="; + + // Used for the key rotation example + private static final String ANOTHER_CESK_KEY = "oevtavYZC+TfGtV86kJBKTeytXAm1s2r3xIqam+QPKM="; + private static final String ANOTHER_CSEK_KEY_HASH = + "/gd0N3k3MK0SEDxnUiaswl0FFv6+5PHpo+5KD5SBCeA="; + + private static final String OBJECT_NAME = "encrypted_file.txt"; + + /** + * Downloads a CSEK-protected object from GCS. The download may continue in the background after + * this method returns. The caller of this method is responsible for closing the input stream. + * + * @param storage A Storage object, ready for use + * @param bucketName The name of the destination bucket + * @param objectName The name of the destination object + * @param base64CSEKey An AES256 key, encoded as a base64 string. + * @param base64CSEKeyHash The SHA-256 hash of the above key, also encoded as a base64 string. + * @throws IOException if there was some error download from GCS. + * + * @return An InputStream that contains the decrypted contents of the object. + */ + public static InputStream downloadObject( + Storage storage, + String bucketName, + String objectName, + String base64CSEKey, + String base64CSEKeyHash) + throws Exception { + Storage.Objects.Get getObject = storage.objects().get(bucketName, objectName); + + // If you're using AppEngine, turn off setDirectDownloadEnabled: + // getObject.getMediaHttpDownloader().setDirectDownloadEnabled(false); + + // Now set the CSEK headers + final HttpHeaders httpHeaders = new HttpHeaders(); + httpHeaders.set("x-goog-encryption-algorithm", "AES256"); + httpHeaders.set("x-goog-encryption-key", base64CSEKey); + httpHeaders.set("x-goog-encryption-key-sha256", base64CSEKeyHash); + + // Since our request includes our private key as a header, it is a good idea to instruct caches + // and proxies not to store this request. + httpHeaders.setCacheControl("no-store"); + + getObject.setRequestHeaders(httpHeaders); + + try { + return getObject.executeMediaAsInputStream(); + } catch (GoogleJsonResponseException e) { + System.out.println("Error downloading: " + e.getContent()); + System.exit(1); + return null; + } + } + + /** + * Uploads an object to GCS, to be stored with a customer-supplied key (CSEK). The upload may + * continue in the background after this method returns. The caller of this method is responsible + * for closing the input stream. + * + * @param storage A Storage object, ready for use + * @param bucketName The name of the destination bucket + * @param objectName The name of the destination object + * @param data An InputStream containing the contents of the object to upload + * @param base64CSEKey An AES256 key, encoded as a base64 string. + * @param base64CSEKeyHash The SHA-256 hash of the above key, also encoded as a base64 string. + * @throws IOException if there was some error uploading to GCS. + */ + public static void uploadObject( + Storage storage, + String bucketName, + String objectName, + InputStream data, + String base64CSEKey, + String base64CSEKeyHash) + throws IOException { + InputStreamContent mediaContent = new InputStreamContent("text/plain", data); + Storage.Objects.Insert insertObject = + storage.objects().insert(bucketName, null, mediaContent).setName(objectName); + // The client library's default gzip setting may cause objects to be stored with gzip encoding, + // which can be desirable in some circumstances but has some disadvantages as well, such as + // making it difficult to read only a certain range of the original object. + insertObject.getMediaHttpUploader().setDisableGZipContent(true); + + // Now set the CSEK headers + final HttpHeaders httpHeaders = new HttpHeaders(); + httpHeaders.set("x-goog-encryption-algorithm", "AES256"); + httpHeaders.set("x-goog-encryption-key", base64CSEKey); + httpHeaders.set("x-goog-encryption-key-sha256", base64CSEKeyHash); + + // Since our request includes our private key as a header, it is a good idea to instruct caches + // and proxies not to store this request. + httpHeaders.setCacheControl("no-store"); + + insertObject.setRequestHeaders(httpHeaders); + + try { + insertObject.execute(); + } catch (GoogleJsonResponseException e) { + System.out.println("Error uploading: " + e.getContent()); + System.exit(1); + } + } + + /** + * Given an existing, CSEK-protected object, changes the key used to store that object. + * + * @param storage A Storage object, ready for use + * @param bucketName The name of the destination bucket + * @param objectName The name of the destination object + * @param originalBase64Key The AES256 key currently associated with this object, + * encoded as a base64 string. + * @param originalBase64KeyHash The SHA-256 hash of the above key, + * also encoded as a base64 string. + * @param newBase64Key An AES256 key which will replace the existing key, + * encoded as a base64 string. + * @param newBase64KeyHash The SHA-256 hash of the above key, also encoded as a base64 string. + * @throws IOException if there was some error download from GCS. + */ + public static void rotateKey( + Storage storage, + String bucketName, + String objectName, + String originalBase64Key, + String originalBase64KeyHash, + String newBase64Key, + String newBase64KeyHash) + throws Exception { + Storage.Objects.Rewrite rewriteObject = + storage.objects().rewrite(bucketName, objectName, bucketName, objectName, null); + + // Now set the CSEK headers + final HttpHeaders httpHeaders = new HttpHeaders(); + + // Specify the exiting object's current CSEK. + httpHeaders.set("x-goog-copy-source-encryption-algorithm", "AES256"); + httpHeaders.set("x-goog-copy-source-encryption-key", originalBase64Key); + httpHeaders.set("x-goog-copy-source-encryption-key-sha256", originalBase64KeyHash); + + // Specify the new CSEK that we would like to apply. + httpHeaders.set("x-goog-encryption-algorithm", "AES256"); + httpHeaders.set("x-goog-encryption-key", newBase64Key); + httpHeaders.set("x-goog-encryption-key-sha256", newBase64KeyHash); + + // Since our request includes our private key as a header, it is a good idea to instruct caches + // and proxies not to store this request. + httpHeaders.setCacheControl("no-store"); + + rewriteObject.setRequestHeaders(httpHeaders); + + try { + RewriteResponse rewriteResponse = rewriteObject.execute(); + + // If an object is very large, you may need to continue making successive calls to + // rewrite until the operation completes. + while (!rewriteResponse.getDone()) { + System.out.println("Rewrite did not complete. Resuming..."); + rewriteObject.setRewriteToken(rewriteResponse.getRewriteToken()); + rewriteResponse = rewriteObject.execute(); + } + } catch (GoogleJsonResponseException e) { + System.out.println("Error rotating key: " + e.getContent()); + System.exit(1); + } + } + + public static void main(String[] args) throws Exception { + if (args.length != 1) { + System.out.println("\nPlease run this with one argument: " + + "the GCS bucket into which this program should upload an object.\n\n" + + "You can create a bucket using gsutil like this:\n\n\t" + + "gsutil mb gs://name-of-bucket\n\n"); + System.exit(1); + } + String bucketName = args[0]; + // CSEK, like the JSON API, may be used only via HTTPS. + HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); + DataStoreFactory dataStoreFactory = new FileDataStoreFactory(DATA_STORE_DIR); + JsonFactory jsonFactory = JacksonFactory.getDefaultInstance(); + Credential credential = authorize(jsonFactory, httpTransport, dataStoreFactory); + Storage storage = + new Storage.Builder(httpTransport, jsonFactory, credential) + .setApplicationName("JavaCSEKApiSample") + .build(); + + InputStream dataToUpload = new ArbitrarilyLargeInputStream(10000000); + + System.out.format("Uploading object gs://%s/%s using CSEK.\n", bucketName, OBJECT_NAME); + uploadObject(storage, bucketName, OBJECT_NAME, dataToUpload, CSEK_KEY, CSEK_KEY_HASH); + System.out.format("Downloading object gs://%s/%s using CSEK.\n", bucketName, OBJECT_NAME); + InputStream objectData = + downloadObject(storage, bucketName, OBJECT_NAME, CSEK_KEY, CSEK_KEY_HASH); + readStream(objectData); + System.out.println("Rotating object to use a different CSEK."); + rotateKey(storage, bucketName, OBJECT_NAME, CSEK_KEY, CSEK_KEY_HASH, + ANOTHER_CESK_KEY, ANOTHER_CSEK_KEY_HASH); + + System.out.println(); + } + + private static Credential authorize( + JsonFactory jsonFactory, HttpTransport httpTransport, DataStoreFactory dataStoreFactory) + throws Exception { + + InputStream clientSecretStream = + CustomerSuppliedEncryptionKeysSamples.class + .getResourceAsStream("client_secrets.json"); + if (clientSecretStream == null) { + throw new RuntimeException("Could not load secrets"); + } + + // Load client secrets + GoogleClientSecrets clientSecrets = + GoogleClientSecrets.load(jsonFactory, new InputStreamReader(clientSecretStream)); + + // Set up authorization code flow + GoogleAuthorizationCodeFlow flow = + new GoogleAuthorizationCodeFlow.Builder( + httpTransport, + jsonFactory, + clientSecrets, + Collections.singleton(StorageScopes.DEVSTORAGE_FULL_CONTROL)) + .setDataStoreFactory(dataStoreFactory) + .build(); + + // Authorize + Credential credential = + new AuthorizationCodeInstalledApp(flow, new LocalServerReceiver()).authorize("user"); + + return credential; + } + + /** + * Reads the contents of an InputStream and does nothing with it. + */ + private static void readStream(InputStream is) throws IOException { + byte inputBuffer[] = new byte[256]; + while (is.read(inputBuffer) != -1) {} + // The caller is responsible for closing this InputStream. + is.close(); + } + + /** + * A helper class to provide input streams of any size. + * The input streams will be full of null bytes. + */ + static class ArbitrarilyLargeInputStream extends InputStream { + + private long bytesRead; + private final long streamSize; + + public ArbitrarilyLargeInputStream(long streamSizeInBytes) { + bytesRead = 0; + this.streamSize = streamSizeInBytes; + } + + @Override + public int read() throws IOException { + if (bytesRead >= streamSize) { + return -1; + } + bytesRead++; + return 0; + } + } + +} diff --git a/storage/json-api/src/main/resources/client_secrets.json b/storage/json-api/src/main/resources/client_secrets.json new file mode 100644 index 00000000000..0f6e14bf2fe --- /dev/null +++ b/storage/json-api/src/main/resources/client_secrets.json @@ -0,0 +1 @@ +{"installed":{"client_id":"122681785480-35mkukq7o3hh55i2uuftv0hp168hauet.apps.googleusercontent.com","project_id":"gcs-code-samples","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://accounts.google.com/o/oauth2/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_secret":"hf-15vXoQQY8OonIK7qdrR0L","redirect_uris":["urn:ietf:wg:oauth:2.0:oob","http://localhost"]}} \ No newline at end of file From bfab252cb373817bfc9c370ffbb574d7a130dec1 Mon Sep 17 00:00:00 2001 From: Brandon Yarbrough Date: Wed, 23 Mar 2016 13:56:06 -0700 Subject: [PATCH 2/9] Remove Maven "exec" config, allowing for multiple programs. Also add a README.pm, explaining how to run programs. --- storage/json-api/README.md | 31 +++++++++++++++++++++++++++++++ storage/json-api/pom.xml | 17 ----------------- 2 files changed, 31 insertions(+), 17 deletions(-) create mode 100644 storage/json-api/README.md diff --git a/storage/json-api/README.md b/storage/json-api/README.md new file mode 100644 index 00000000000..a026cb4ff40 --- /dev/null +++ b/storage/json-api/README.md @@ -0,0 +1,31 @@ +# Google Cloud Storage (GCS) and the Google Java API Client library + +Google Cloud Storage Service features a REST-based API that allows developers to store and access arbitrarily-large objects. These sample Java applications demonstrate how to access the Google Cloud Storage JSON API using the Google Java API Client Libraries. For more information, read the [Google Cloud Storage JSON API Overview][1]. + +## Quickstart + +Install [Maven](http://maven.apache.org/). + +Build your project with: + + mvn package + +You can then run a given `ClassName` via: + + mvn exec:java -Dexec.mainClass=StorageSample \ + -Dexec.args="ABucketName" + +## Products +- [Google Cloud Storage][2] + +## Language +- [Java][3] + +## Dependencies +- [Google APIs Client Library for Java][4] + +[1]: https://cloud.google.com/storage/docs/json_api +[2]: https://cloud.google.com/storage +[3]: https://java.com +[4]: http://code.google.com/p/google-api-java-client/ + diff --git a/storage/json-api/pom.xml b/storage/json-api/pom.xml index ca7f59c6261..64767c4233c 100644 --- a/storage/json-api/pom.xml +++ b/storage/json-api/pom.xml @@ -27,23 +27,6 @@ - - - org.codehaus.mojo - exec-maven-plugin - 1.1 - - - - java - - - - - StorageSample - - - ${project.artifactId}-${project.version} From dceeb5f29b588ad26c0083af18a56f36b032c2b2 Mon Sep 17 00:00:00 2001 From: Brandon Yarbrough Date: Wed, 23 Mar 2016 14:54:55 -0700 Subject: [PATCH 3/9] Switch to getApplicationDefault()-based auth. Modify docs+others accordingly. --- storage/json-api/README.md | 33 +++++- ...CustomerSuppliedEncryptionKeysSamples.java | 111 ++---------------- .../src/main/java/StorageFactory.java | 57 +++++++++ .../json-api/src/main/java/StorageSample.java | 44 +------ .../json-api/src/main/java/StorageUtils.java | 40 +++++++ 5 files changed, 140 insertions(+), 145 deletions(-) create mode 100644 storage/json-api/src/main/java/StorageFactory.java create mode 100644 storage/json-api/src/main/java/StorageUtils.java diff --git a/storage/json-api/README.md b/storage/json-api/README.md index a026cb4ff40..9d0784a5813 100644 --- a/storage/json-api/README.md +++ b/storage/json-api/README.md @@ -4,16 +4,39 @@ Google Cloud Storage Service features a REST-based API that allows developers to ## Quickstart -Install [Maven](http://maven.apache.org/). +1. Install the [Google Cloud SDK](https://cloud.google.com/sdk/), including the [gcloud tool](https://cloud.google.com/sdk/gcloud/). -Build your project with: +1. Setup the gcloud tool. + ``` + gcloud init + ``` - mvn package +1. Clone this repo. -You can then run a given `ClassName` via: + ``` + git clone https://github.com/GoogleCloudPlatform/java-docs-samples.git + ``` - mvn exec:java -Dexec.mainClass=StorageSample \ +1. Install [Maven](http://maven.apache.org/). + +1. Build this project from this directory: + + ``` + mvn package + ``` + +1. Run one of the sample apps by specifying its class name and a bucket name: + + ``` + mvn exec:java -Dexec.mainClass=StorageSample \ -Dexec.args="ABucketName" + ``` + +Note that if it's been a while, you may need to login with gcloud. + + ``` + gcloud auth login + ``` ## Products - [Google Cloud Storage][2] diff --git a/storage/json-api/src/main/java/CustomerSuppliedEncryptionKeysSamples.java b/storage/json-api/src/main/java/CustomerSuppliedEncryptionKeysSamples.java index c7603051b0c..b33e7c55ef0 100644 --- a/storage/json-api/src/main/java/CustomerSuppliedEncryptionKeysSamples.java +++ b/storage/json-api/src/main/java/CustomerSuppliedEncryptionKeysSamples.java @@ -1,25 +1,11 @@ -import com.google.api.client.auth.oauth2.Credential; -import com.google.api.client.extensions.java6.auth.oauth2.AuthorizationCodeInstalledApp; -import com.google.api.client.extensions.jetty.auth.oauth2.LocalServerReceiver; -import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeFlow; -import com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets; -import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.http.HttpHeaders; -import com.google.api.client.http.HttpTransport; import com.google.api.client.http.InputStreamContent; -import com.google.api.client.json.JsonFactory; -import com.google.api.client.json.jackson2.JacksonFactory; -import com.google.api.client.util.store.DataStoreFactory; -import com.google.api.client.util.store.FileDataStoreFactory; import com.google.api.services.storage.Storage; -import com.google.api.services.storage.StorageScopes; import com.google.api.services.storage.model.RewriteResponse; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.Collections; /** * Demonstrates the use of GCS's CSEK features via the Java API client library @@ -35,9 +21,6 @@ **/ class CustomerSuppliedEncryptionKeysSamples { - private static final java.io.File DATA_STORE_DIR = - new java.io.File(System.getProperty("user.home"), ".store/storage_sample"); - // You can (and should) generate your own CSEK Key! Try running this from the command line: // python -c 'import base64; import os; print(base64.encodestring(os.urandom(32)))' // Also, these encryption keys are included here for simplicity, but please remember that @@ -135,11 +118,11 @@ public static void uploadObject( httpHeaders.set("x-goog-encryption-algorithm", "AES256"); httpHeaders.set("x-goog-encryption-key", base64CSEKey); httpHeaders.set("x-goog-encryption-key-sha256", base64CSEKeyHash); - + // Since our request includes our private key as a header, it is a good idea to instruct caches // and proxies not to store this request. httpHeaders.setCacheControl("no-store"); - + insertObject.setRequestHeaders(httpHeaders); try { @@ -189,11 +172,11 @@ public static void rotateKey( httpHeaders.set("x-goog-encryption-algorithm", "AES256"); httpHeaders.set("x-goog-encryption-key", newBase64Key); httpHeaders.set("x-goog-encryption-key-sha256", newBase64KeyHash); - + // Since our request includes our private key as a header, it is a good idea to instruct caches // and proxies not to store this request. httpHeaders.setCacheControl("no-store"); - + rewriteObject.setRequestHeaders(httpHeaders); try { @@ -221,95 +204,23 @@ public static void main(String[] args) throws Exception { System.exit(1); } String bucketName = args[0]; - // CSEK, like the JSON API, may be used only via HTTPS. - HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); - DataStoreFactory dataStoreFactory = new FileDataStoreFactory(DATA_STORE_DIR); - JsonFactory jsonFactory = JacksonFactory.getDefaultInstance(); - Credential credential = authorize(jsonFactory, httpTransport, dataStoreFactory); - Storage storage = - new Storage.Builder(httpTransport, jsonFactory, credential) - .setApplicationName("JavaCSEKApiSample") - .build(); - - InputStream dataToUpload = new ArbitrarilyLargeInputStream(10000000); + + Storage storage = StorageFactory.getService(); + InputStream dataToUpload = new StorageUtils.ArbitrarilyLargeInputStream(10000000); System.out.format("Uploading object gs://%s/%s using CSEK.\n", bucketName, OBJECT_NAME); uploadObject(storage, bucketName, OBJECT_NAME, dataToUpload, CSEK_KEY, CSEK_KEY_HASH); + System.out.format("Downloading object gs://%s/%s using CSEK.\n", bucketName, OBJECT_NAME); InputStream objectData = downloadObject(storage, bucketName, OBJECT_NAME, CSEK_KEY, CSEK_KEY_HASH); - readStream(objectData); + StorageUtils.readStream(objectData); + System.out.println("Rotating object to use a different CSEK."); rotateKey(storage, bucketName, OBJECT_NAME, CSEK_KEY, CSEK_KEY_HASH, ANOTHER_CESK_KEY, ANOTHER_CSEK_KEY_HASH); - System.out.println(); - } - - private static Credential authorize( - JsonFactory jsonFactory, HttpTransport httpTransport, DataStoreFactory dataStoreFactory) - throws Exception { - - InputStream clientSecretStream = - CustomerSuppliedEncryptionKeysSamples.class - .getResourceAsStream("client_secrets.json"); - if (clientSecretStream == null) { - throw new RuntimeException("Could not load secrets"); - } - - // Load client secrets - GoogleClientSecrets clientSecrets = - GoogleClientSecrets.load(jsonFactory, new InputStreamReader(clientSecretStream)); - - // Set up authorization code flow - GoogleAuthorizationCodeFlow flow = - new GoogleAuthorizationCodeFlow.Builder( - httpTransport, - jsonFactory, - clientSecrets, - Collections.singleton(StorageScopes.DEVSTORAGE_FULL_CONTROL)) - .setDataStoreFactory(dataStoreFactory) - .build(); - - // Authorize - Credential credential = - new AuthorizationCodeInstalledApp(flow, new LocalServerReceiver()).authorize("user"); - - return credential; - } - - /** - * Reads the contents of an InputStream and does nothing with it. - */ - private static void readStream(InputStream is) throws IOException { - byte inputBuffer[] = new byte[256]; - while (is.read(inputBuffer) != -1) {} - // The caller is responsible for closing this InputStream. - is.close(); - } - - /** - * A helper class to provide input streams of any size. - * The input streams will be full of null bytes. - */ - static class ArbitrarilyLargeInputStream extends InputStream { - - private long bytesRead; - private final long streamSize; - - public ArbitrarilyLargeInputStream(long streamSizeInBytes) { - bytesRead = 0; - this.streamSize = streamSizeInBytes; - } - - @Override - public int read() throws IOException { - if (bytesRead >= streamSize) { - return -1; - } - bytesRead++; - return 0; - } + System.out.println("Done"); } } diff --git a/storage/json-api/src/main/java/StorageFactory.java b/storage/json-api/src/main/java/StorageFactory.java new file mode 100644 index 00000000000..f9303c26668 --- /dev/null +++ b/storage/json-api/src/main/java/StorageFactory.java @@ -0,0 +1,57 @@ +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.api.services.storage.Storage; +import com.google.api.services.storage.StorageScopes; + +import java.io.IOException; +import java.security.GeneralSecurityException; +import java.util.Collection; + +/* + * Copyright (c) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +/** + * This class manages the details of creating a Storage service, including auth. + */ +public class StorageFactory { + + private static Storage instance = null; + + public static synchronized Storage getService() throws IOException, GeneralSecurityException { + if (instance == null) { + instance = buildService(); + } + return instance; + } + + private static Storage buildService() throws IOException, GeneralSecurityException { + HttpTransport transport = GoogleNetHttpTransport.newTrustedTransport(); + JsonFactory jsonFactory = new JacksonFactory(); + GoogleCredential credential = GoogleCredential.getApplicationDefault(transport, jsonFactory); + + if (credential.createScopedRequired()) { + Collection bigqueryScopes = StorageScopes.all(); + credential = credential.createScoped(bigqueryScopes); + } + + return new Storage.Builder(transport, jsonFactory, credential) + .setApplicationName("GCS Samples") + .build(); + } +} diff --git a/storage/json-api/src/main/java/StorageSample.java b/storage/json-api/src/main/java/StorageSample.java index a09f65aea69..092271eb8c0 100644 --- a/storage/json-api/src/main/java/StorageSample.java +++ b/storage/json-api/src/main/java/StorageSample.java @@ -13,14 +13,8 @@ * the License. */ -import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; -import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; -import com.google.api.client.http.HttpTransport; import com.google.api.client.http.InputStreamContent; -import com.google.api.client.json.JsonFactory; -import com.google.api.client.json.jackson2.JacksonFactory; import com.google.api.services.storage.Storage; -import com.google.api.services.storage.StorageScopes; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.ObjectAccessControl; import com.google.api.services.storage.model.Objects; @@ -42,39 +36,9 @@ */ public class StorageSample { - /** - * Be sure to specify the name of your application. If the application name is {@code null} or - * blank, the application will log a warning. Suggested format is "MyCompany-ProductName/1.0". - */ - private static final String APPLICATION_NAME = "[[INSERT_YOUR_APP_NAME_HERE]]"; - /** Global instance of the JSON factory. */ - private static final JsonFactory JSON_FACTORY = JacksonFactory.getDefaultInstance(); private static final String TEST_FILENAME = "json-test.txt"; - // [START get_service] - private static Storage storageService; - - /** - * Returns an authenticated Storage object used to make service calls to Cloud Storage. - */ - private static Storage getService() throws IOException, GeneralSecurityException { - if (null == storageService) { - GoogleCredential credential = GoogleCredential.getApplicationDefault(); - // Depending on the environment that provides the default credentials (e.g. Compute Engine, - // App Engine), the credentials may require us to specify the scopes we need explicitly. - // Check for this case, and inject the Cloud Storage scope if required. - if (credential.createScopedRequired()) { - credential = credential.createScoped(StorageScopes.all()); - } - HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); - storageService = new Storage.Builder(httpTransport, JSON_FACTORY, credential) - .setApplicationName(APPLICATION_NAME).build(); - } - return storageService; - } - // [END get_service] - // [START list_bucket] /** * Fetch a list of the objects within the given bucket. @@ -84,7 +48,7 @@ private static Storage getService() throws IOException, GeneralSecurityException */ public static List listBucket(String bucketName) throws IOException, GeneralSecurityException { - Storage client = getService(); + Storage client = StorageFactory.getService(); Storage.Objects.List listRequest = client.objects().list(bucketName); List results = new ArrayList(); @@ -112,7 +76,7 @@ public static List listBucket(String bucketName) * @return a Bucket containing the bucket's metadata. */ public static Bucket getBucket(String bucketName) throws IOException, GeneralSecurityException { - Storage client = getService(); + Storage client = StorageFactory.getService(); Storage.Buckets.Get bucketRequest = client.buckets().get(bucketName); // Fetch the full set of the bucket's properties (e.g. include the ACLs in the response) @@ -142,7 +106,7 @@ public static void uploadStream( new ObjectAccessControl().setEntity("allUsers").setRole("READER"))); // Do the insert - Storage client = getService(); + Storage client = StorageFactory.getService(); Storage.Objects.Insert insertRequest = client.objects().insert( bucketName, objectMetadata, contentStream); @@ -159,7 +123,7 @@ public static void uploadStream( */ public static void deleteObject(String path, String bucketName) throws IOException, GeneralSecurityException { - Storage client = getService(); + Storage client = StorageFactory.getService(); client.objects().delete(bucketName, path).execute(); } // [END delete_object] diff --git a/storage/json-api/src/main/java/StorageUtils.java b/storage/json-api/src/main/java/StorageUtils.java new file mode 100644 index 00000000000..5e37b36f790 --- /dev/null +++ b/storage/json-api/src/main/java/StorageUtils.java @@ -0,0 +1,40 @@ +import java.io.IOException; +import java.io.InputStream; + +public class StorageUtils { + + /** + * Reads the contents of an InputStream and does nothing with it. + */ + public static void readStream(InputStream is) throws IOException { + byte inputBuffer[] = new byte[256]; + while (is.read(inputBuffer) != -1) {} + // The caller is responsible for closing this InputStream. + is.close(); + } + + /** + * A helper class to provide input streams of any size. + * The input streams will be full of null bytes. + */ + static class ArbitrarilyLargeInputStream extends InputStream { + + private long bytesRead; + private final long streamSize; + + public ArbitrarilyLargeInputStream(long streamSizeInBytes) { + bytesRead = 0; + this.streamSize = streamSizeInBytes; + } + + @Override + public int read() throws IOException { + if (bytesRead >= streamSize) { + return -1; + } + bytesRead++; + return 0; + } + } +} + From 2d08062c1cec5ff8ba0298055b385adf1731ddcd Mon Sep 17 00:00:00 2001 From: Brandon Yarbrough Date: Wed, 23 Mar 2016 15:02:49 -0700 Subject: [PATCH 4/9] Remove unnecessary ```s in the Markdown. --- storage/json-api/README.md | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/storage/json-api/README.md b/storage/json-api/README.md index 9d0784a5813..f7b4020b6d9 100644 --- a/storage/json-api/README.md +++ b/storage/json-api/README.md @@ -7,36 +7,27 @@ Google Cloud Storage Service features a REST-based API that allows developers to 1. Install the [Google Cloud SDK](https://cloud.google.com/sdk/), including the [gcloud tool](https://cloud.google.com/sdk/gcloud/). 1. Setup the gcloud tool. - ``` + gcloud init - ``` 1. Clone this repo. - ``` git clone https://github.com/GoogleCloudPlatform/java-docs-samples.git - ``` 1. Install [Maven](http://maven.apache.org/). 1. Build this project from this directory: - ``` mvn package - ``` 1. Run one of the sample apps by specifying its class name and a bucket name: - ``` mvn exec:java -Dexec.mainClass=StorageSample \ -Dexec.args="ABucketName" - ``` Note that if it's been a while, you may need to login with gcloud. - ``` gcloud auth login - ``` ## Products - [Google Cloud Storage][2] From cf0ec46817f74711b19449604713904ffb03352f Mon Sep 17 00:00:00 2001 From: Brandon Yarbrough Date: Wed, 23 Mar 2016 15:07:52 -0700 Subject: [PATCH 5/9] I was wrong, the backticks were necessary. --- storage/json-api/README.md | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/storage/json-api/README.md b/storage/json-api/README.md index f7b4020b6d9..a7ead2b75f1 100644 --- a/storage/json-api/README.md +++ b/storage/json-api/README.md @@ -8,26 +8,36 @@ Google Cloud Storage Service features a REST-based API that allows developers to 1. Setup the gcloud tool. - gcloud init + ``` + gcloud init + ``` 1. Clone this repo. - git clone https://github.com/GoogleCloudPlatform/java-docs-samples.git + ``` + git clone https://github.com/GoogleCloudPlatform/java-docs-samples.git + ``` 1. Install [Maven](http://maven.apache.org/). 1. Build this project from this directory: - mvn package + ``` + mvn package + ``` 1. Run one of the sample apps by specifying its class name and a bucket name: - mvn exec:java -Dexec.mainClass=StorageSample \ - -Dexec.args="ABucketName" + ``` + mvn exec:java -Dexec.mainClass=StorageSample \ + -Dexec.args="ABucketName" + ``` Note that if it's been a while, you may need to login with gcloud. - gcloud auth login + ``` + gcloud auth login + ``` ## Products - [Google Cloud Storage][2] From ed444b18cbb5f79c86d6a68a38c55745f886ab1d Mon Sep 17 00:00:00 2001 From: Brandon Yarbrough Date: Wed, 23 Mar 2016 15:34:07 -0700 Subject: [PATCH 6/9] Remove calls to setCacheControl("no-store"). --- .../java/CustomerSuppliedEncryptionKeysSamples.java | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/storage/json-api/src/main/java/CustomerSuppliedEncryptionKeysSamples.java b/storage/json-api/src/main/java/CustomerSuppliedEncryptionKeysSamples.java index b33e7c55ef0..88cb8f3b882 100644 --- a/storage/json-api/src/main/java/CustomerSuppliedEncryptionKeysSamples.java +++ b/storage/json-api/src/main/java/CustomerSuppliedEncryptionKeysSamples.java @@ -69,10 +69,6 @@ public static InputStream downloadObject( httpHeaders.set("x-goog-encryption-key", base64CSEKey); httpHeaders.set("x-goog-encryption-key-sha256", base64CSEKeyHash); - // Since our request includes our private key as a header, it is a good idea to instruct caches - // and proxies not to store this request. - httpHeaders.setCacheControl("no-store"); - getObject.setRequestHeaders(httpHeaders); try { @@ -119,10 +115,6 @@ public static void uploadObject( httpHeaders.set("x-goog-encryption-key", base64CSEKey); httpHeaders.set("x-goog-encryption-key-sha256", base64CSEKeyHash); - // Since our request includes our private key as a header, it is a good idea to instruct caches - // and proxies not to store this request. - httpHeaders.setCacheControl("no-store"); - insertObject.setRequestHeaders(httpHeaders); try { @@ -173,10 +165,6 @@ public static void rotateKey( httpHeaders.set("x-goog-encryption-key", newBase64Key); httpHeaders.set("x-goog-encryption-key-sha256", newBase64KeyHash); - // Since our request includes our private key as a header, it is a good idea to instruct caches - // and proxies not to store this request. - httpHeaders.setCacheControl("no-store"); - rewriteObject.setRequestHeaders(httpHeaders); try { From 65c3d4c449b6503e805b711d397cadd56a438c52 Mon Sep 17 00:00:00 2001 From: Brandon Yarbrough Date: Wed, 23 Mar 2016 15:35:25 -0700 Subject: [PATCH 7/9] Delete client_secrets.json file. --- storage/json-api/src/main/resources/client_secrets.json | 1 - 1 file changed, 1 deletion(-) delete mode 100644 storage/json-api/src/main/resources/client_secrets.json diff --git a/storage/json-api/src/main/resources/client_secrets.json b/storage/json-api/src/main/resources/client_secrets.json deleted file mode 100644 index 0f6e14bf2fe..00000000000 --- a/storage/json-api/src/main/resources/client_secrets.json +++ /dev/null @@ -1 +0,0 @@ -{"installed":{"client_id":"122681785480-35mkukq7o3hh55i2uuftv0hp168hauet.apps.googleusercontent.com","project_id":"gcs-code-samples","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://accounts.google.com/o/oauth2/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_secret":"hf-15vXoQQY8OonIK7qdrR0L","redirect_uris":["urn:ietf:wg:oauth:2.0:oob","http://localhost"]}} \ No newline at end of file From 47923d96a795fee748e46ae2a4071ee17e2e26d6 Mon Sep 17 00:00:00 2001 From: Brandon Yarbrough Date: Wed, 23 Mar 2016 15:38:39 -0700 Subject: [PATCH 8/9] Remove src/main/resources from pom.xml, now that there are none. --- storage/json-api/pom.xml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/storage/json-api/pom.xml b/storage/json-api/pom.xml index 64767c4233c..1f5a89fecc4 100644 --- a/storage/json-api/pom.xml +++ b/storage/json-api/pom.xml @@ -1,5 +1,6 @@ - + doc-samples com.google.cloud @@ -19,14 +20,6 @@ - - - ${basedir}/src/main/resources - - **/* - - - ${project.artifactId}-${project.version} @@ -44,7 +37,6 @@ junit junit - 4.10 test From 57b949bc40a96f84c59154844627fd8c9c7d4687 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 24 Mar 2016 09:24:48 -0700 Subject: [PATCH 9/9] Checkstyle fixes for Cloud Storage CSEK sample. Also, add license headers. --- storage/json-api/pom.xml | 19 +++++-- ...CustomerSuppliedEncryptionKeysSamples.java | 49 +++++++++++++------ .../src/main/java/StorageFactory.java | 32 ++++++------ .../json-api/src/main/java/StorageUtils.java | 20 +++++++- 4 files changed, 83 insertions(+), 37 deletions(-) diff --git a/storage/json-api/pom.xml b/storage/json-api/pom.xml index 1f5a89fecc4..b506ed928cf 100644 --- a/storage/json-api/pom.xml +++ b/storage/json-api/pom.xml @@ -1,6 +1,19 @@ - - + + doc-samples com.google.cloud diff --git a/storage/json-api/src/main/java/CustomerSuppliedEncryptionKeysSamples.java b/storage/json-api/src/main/java/CustomerSuppliedEncryptionKeysSamples.java index 88cb8f3b882..05a9cf55190 100644 --- a/storage/json-api/src/main/java/CustomerSuppliedEncryptionKeysSamples.java +++ b/storage/json-api/src/main/java/CustomerSuppliedEncryptionKeysSamples.java @@ -1,3 +1,19 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.http.HttpHeaders; import com.google.api.client.http.InputStreamContent; @@ -45,18 +61,19 @@ class CustomerSuppliedEncryptionKeysSamples { * @param storage A Storage object, ready for use * @param bucketName The name of the destination bucket * @param objectName The name of the destination object - * @param base64CSEKey An AES256 key, encoded as a base64 string. - * @param base64CSEKeyHash The SHA-256 hash of the above key, also encoded as a base64 string. - * @throws IOException if there was some error download from GCS. + * @param base64CseKey An AES256 key, encoded as a base64 string. + * @param base64CseKeyHash The SHA-256 hash of the above key, also encoded as a base64 string. * * @return An InputStream that contains the decrypted contents of the object. + * + * @throws IOException if there was some error download from GCS. */ public static InputStream downloadObject( Storage storage, String bucketName, String objectName, - String base64CSEKey, - String base64CSEKeyHash) + String base64CseKey, + String base64CseKeyHash) throws Exception { Storage.Objects.Get getObject = storage.objects().get(bucketName, objectName); @@ -66,8 +83,8 @@ public static InputStream downloadObject( // Now set the CSEK headers final HttpHeaders httpHeaders = new HttpHeaders(); httpHeaders.set("x-goog-encryption-algorithm", "AES256"); - httpHeaders.set("x-goog-encryption-key", base64CSEKey); - httpHeaders.set("x-goog-encryption-key-sha256", base64CSEKeyHash); + httpHeaders.set("x-goog-encryption-key", base64CseKey); + httpHeaders.set("x-goog-encryption-key-sha256", base64CseKeyHash); getObject.setRequestHeaders(httpHeaders); @@ -89,8 +106,8 @@ public static InputStream downloadObject( * @param bucketName The name of the destination bucket * @param objectName The name of the destination object * @param data An InputStream containing the contents of the object to upload - * @param base64CSEKey An AES256 key, encoded as a base64 string. - * @param base64CSEKeyHash The SHA-256 hash of the above key, also encoded as a base64 string. + * @param base64CseKey An AES256 key, encoded as a base64 string. + * @param base64CseKeyHash The SHA-256 hash of the above key, also encoded as a base64 string. * @throws IOException if there was some error uploading to GCS. */ public static void uploadObject( @@ -98,8 +115,8 @@ public static void uploadObject( String bucketName, String objectName, InputStream data, - String base64CSEKey, - String base64CSEKeyHash) + String base64CseKey, + String base64CseKeyHash) throws IOException { InputStreamContent mediaContent = new InputStreamContent("text/plain", data); Storage.Objects.Insert insertObject = @@ -112,8 +129,8 @@ public static void uploadObject( // Now set the CSEK headers final HttpHeaders httpHeaders = new HttpHeaders(); httpHeaders.set("x-goog-encryption-algorithm", "AES256"); - httpHeaders.set("x-goog-encryption-key", base64CSEKey); - httpHeaders.set("x-goog-encryption-key-sha256", base64CSEKeyHash); + httpHeaders.set("x-goog-encryption-key", base64CseKey); + httpHeaders.set("x-goog-encryption-key-sha256", base64CseKeyHash); insertObject.setRequestHeaders(httpHeaders); @@ -192,18 +209,18 @@ public static void main(String[] args) throws Exception { System.exit(1); } String bucketName = args[0]; - + Storage storage = StorageFactory.getService(); InputStream dataToUpload = new StorageUtils.ArbitrarilyLargeInputStream(10000000); System.out.format("Uploading object gs://%s/%s using CSEK.\n", bucketName, OBJECT_NAME); uploadObject(storage, bucketName, OBJECT_NAME, dataToUpload, CSEK_KEY, CSEK_KEY_HASH); - + System.out.format("Downloading object gs://%s/%s using CSEK.\n", bucketName, OBJECT_NAME); InputStream objectData = downloadObject(storage, bucketName, OBJECT_NAME, CSEK_KEY, CSEK_KEY_HASH); StorageUtils.readStream(objectData); - + System.out.println("Rotating object to use a different CSEK."); rotateKey(storage, bucketName, OBJECT_NAME, CSEK_KEY, CSEK_KEY_HASH, ANOTHER_CESK_KEY, ANOTHER_CSEK_KEY_HASH); diff --git a/storage/json-api/src/main/java/StorageFactory.java b/storage/json-api/src/main/java/StorageFactory.java index f9303c26668..df47b13475e 100644 --- a/storage/json-api/src/main/java/StorageFactory.java +++ b/storage/json-api/src/main/java/StorageFactory.java @@ -1,3 +1,19 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; import com.google.api.client.http.HttpTransport; @@ -10,22 +26,6 @@ import java.security.GeneralSecurityException; import java.util.Collection; -/* - * Copyright (c) 2016 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - /** * This class manages the details of creating a Storage service, including auth. */ diff --git a/storage/json-api/src/main/java/StorageUtils.java b/storage/json-api/src/main/java/StorageUtils.java index 5e37b36f790..71370a6032c 100644 --- a/storage/json-api/src/main/java/StorageUtils.java +++ b/storage/json-api/src/main/java/StorageUtils.java @@ -1,13 +1,29 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + import java.io.IOException; import java.io.InputStream; public class StorageUtils { - + /** * Reads the contents of an InputStream and does nothing with it. */ public static void readStream(InputStream is) throws IOException { - byte inputBuffer[] = new byte[256]; + byte[] inputBuffer = new byte[256]; while (is.read(inputBuffer) != -1) {} // The caller is responsible for closing this InputStream. is.close();