diff --git a/storage/json-api/README.md b/storage/json-api/README.md new file mode 100644 index 00000000000..a7ead2b75f1 --- /dev/null +++ b/storage/json-api/README.md @@ -0,0 +1,55 @@ +# Google Cloud Storage (GCS) and the Google Java API Client library + +Google Cloud Storage Service features a REST-based API that allows developers to store and access arbitrarily-large objects. These sample Java applications demonstrate how to access the Google Cloud Storage JSON API using the Google Java API Client Libraries. For more information, read the [Google Cloud Storage JSON API Overview][1]. + +## Quickstart + +1. Install the [Google Cloud SDK](https://cloud.google.com/sdk/), including the [gcloud tool](https://cloud.google.com/sdk/gcloud/). + +1. Setup the gcloud tool. + + ``` + gcloud init + ``` + +1. Clone this repo. + + ``` + git clone https://github.com/GoogleCloudPlatform/java-docs-samples.git + ``` + +1. Install [Maven](http://maven.apache.org/). + +1. Build this project from this directory: + + ``` + mvn package + ``` + +1. Run one of the sample apps by specifying its class name and a bucket name: + + ``` + mvn exec:java -Dexec.mainClass=StorageSample \ + -Dexec.args="ABucketName" + ``` + +Note that if it's been a while, you may need to login with gcloud. + + ``` + gcloud auth login + ``` + +## Products +- [Google Cloud Storage][2] + +## Language +- [Java][3] + +## Dependencies +- [Google APIs Client Library for Java][4] + +[1]: https://cloud.google.com/storage/docs/json_api +[2]: https://cloud.google.com/storage +[3]: https://java.com +[4]: http://code.google.com/p/google-api-java-client/ + diff --git a/storage/json-api/pom.xml b/storage/json-api/pom.xml index a23ae40f1f0..b506ed928cf 100644 --- a/storage/json-api/pom.xml +++ b/storage/json-api/pom.xml @@ -1,4 +1,18 @@ - + doc-samples @@ -19,36 +33,23 @@ - - - org.codehaus.mojo - exec-maven-plugin - 1.1 - - - - java - - - - - StorageSample - - - ${project.artifactId}-${project.version} com.google.apis google-api-services-storage - v1-rev18-1.19.0 + v1-rev65-1.21.0 + + + com.google.oauth-client + google-oauth-client-jetty + 1.21.0 junit junit - 4.10 test diff --git a/storage/json-api/src/main/java/CustomerSuppliedEncryptionKeysSamples.java b/storage/json-api/src/main/java/CustomerSuppliedEncryptionKeysSamples.java new file mode 100644 index 00000000000..05a9cf55190 --- /dev/null +++ b/storage/json-api/src/main/java/CustomerSuppliedEncryptionKeysSamples.java @@ -0,0 +1,231 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.google.api.client.googleapis.json.GoogleJsonResponseException; +import com.google.api.client.http.HttpHeaders; +import com.google.api.client.http.InputStreamContent; +import com.google.api.services.storage.Storage; +import com.google.api.services.storage.model.RewriteResponse; + +import java.io.IOException; +import java.io.InputStream; + +/** + * Demonstrates the use of GCS's CSEK features via the Java API client library + * + * This program demonstrates some quick, basic examples of using GCS's CSEK functionality. + * + *

When run, it begins by uploading an object named "encrypted_file.txt" to the specified bucket + * that will be protected with a provided CSEK.

+ * + *

Next, it will fetch that object by providing that same CSEK to GCS.

+ * + *

Finally, it will rotate that key to a new value.

+ **/ +class CustomerSuppliedEncryptionKeysSamples { + + // You can (and should) generate your own CSEK Key! Try running this from the command line: + // python -c 'import base64; import os; print(base64.encodestring(os.urandom(32)))' + // Also, these encryption keys are included here for simplicity, but please remember that + // private keys should not be stored in source code. + private static final String CSEK_KEY = "4RzDI0TeWa9M/nAvYH05qbCskPaSU/CFV5HeCxk0IUA="; + + // You can use openssl to quickly calculate the hash of your key. Try running this: + // openssl base64 -d <<< YOUR_KEY_FROM_ABOVE | openssl dgst -sha256 -binary | openssl base64 + private static final String CSEK_KEY_HASH = "aanjNC2nwso8e2FqcWILC3/Tt1YumvIwEj34kr6PRpI="; + + // Used for the key rotation example + private static final String ANOTHER_CESK_KEY = "oevtavYZC+TfGtV86kJBKTeytXAm1s2r3xIqam+QPKM="; + private static final String ANOTHER_CSEK_KEY_HASH = + "/gd0N3k3MK0SEDxnUiaswl0FFv6+5PHpo+5KD5SBCeA="; + + private static final String OBJECT_NAME = "encrypted_file.txt"; + + /** + * Downloads a CSEK-protected object from GCS. The download may continue in the background after + * this method returns. The caller of this method is responsible for closing the input stream. + * + * @param storage A Storage object, ready for use + * @param bucketName The name of the destination bucket + * @param objectName The name of the destination object + * @param base64CseKey An AES256 key, encoded as a base64 string. + * @param base64CseKeyHash The SHA-256 hash of the above key, also encoded as a base64 string. + * + * @return An InputStream that contains the decrypted contents of the object. + * + * @throws IOException if there was some error download from GCS. + */ + public static InputStream downloadObject( + Storage storage, + String bucketName, + String objectName, + String base64CseKey, + String base64CseKeyHash) + throws Exception { + Storage.Objects.Get getObject = storage.objects().get(bucketName, objectName); + + // If you're using AppEngine, turn off setDirectDownloadEnabled: + // getObject.getMediaHttpDownloader().setDirectDownloadEnabled(false); + + // Now set the CSEK headers + final HttpHeaders httpHeaders = new HttpHeaders(); + httpHeaders.set("x-goog-encryption-algorithm", "AES256"); + httpHeaders.set("x-goog-encryption-key", base64CseKey); + httpHeaders.set("x-goog-encryption-key-sha256", base64CseKeyHash); + + getObject.setRequestHeaders(httpHeaders); + + try { + return getObject.executeMediaAsInputStream(); + } catch (GoogleJsonResponseException e) { + System.out.println("Error downloading: " + e.getContent()); + System.exit(1); + return null; + } + } + + /** + * Uploads an object to GCS, to be stored with a customer-supplied key (CSEK). The upload may + * continue in the background after this method returns. The caller of this method is responsible + * for closing the input stream. + * + * @param storage A Storage object, ready for use + * @param bucketName The name of the destination bucket + * @param objectName The name of the destination object + * @param data An InputStream containing the contents of the object to upload + * @param base64CseKey An AES256 key, encoded as a base64 string. + * @param base64CseKeyHash The SHA-256 hash of the above key, also encoded as a base64 string. + * @throws IOException if there was some error uploading to GCS. + */ + public static void uploadObject( + Storage storage, + String bucketName, + String objectName, + InputStream data, + String base64CseKey, + String base64CseKeyHash) + throws IOException { + InputStreamContent mediaContent = new InputStreamContent("text/plain", data); + Storage.Objects.Insert insertObject = + storage.objects().insert(bucketName, null, mediaContent).setName(objectName); + // The client library's default gzip setting may cause objects to be stored with gzip encoding, + // which can be desirable in some circumstances but has some disadvantages as well, such as + // making it difficult to read only a certain range of the original object. + insertObject.getMediaHttpUploader().setDisableGZipContent(true); + + // Now set the CSEK headers + final HttpHeaders httpHeaders = new HttpHeaders(); + httpHeaders.set("x-goog-encryption-algorithm", "AES256"); + httpHeaders.set("x-goog-encryption-key", base64CseKey); + httpHeaders.set("x-goog-encryption-key-sha256", base64CseKeyHash); + + insertObject.setRequestHeaders(httpHeaders); + + try { + insertObject.execute(); + } catch (GoogleJsonResponseException e) { + System.out.println("Error uploading: " + e.getContent()); + System.exit(1); + } + } + + /** + * Given an existing, CSEK-protected object, changes the key used to store that object. + * + * @param storage A Storage object, ready for use + * @param bucketName The name of the destination bucket + * @param objectName The name of the destination object + * @param originalBase64Key The AES256 key currently associated with this object, + * encoded as a base64 string. + * @param originalBase64KeyHash The SHA-256 hash of the above key, + * also encoded as a base64 string. + * @param newBase64Key An AES256 key which will replace the existing key, + * encoded as a base64 string. + * @param newBase64KeyHash The SHA-256 hash of the above key, also encoded as a base64 string. + * @throws IOException if there was some error download from GCS. + */ + public static void rotateKey( + Storage storage, + String bucketName, + String objectName, + String originalBase64Key, + String originalBase64KeyHash, + String newBase64Key, + String newBase64KeyHash) + throws Exception { + Storage.Objects.Rewrite rewriteObject = + storage.objects().rewrite(bucketName, objectName, bucketName, objectName, null); + + // Now set the CSEK headers + final HttpHeaders httpHeaders = new HttpHeaders(); + + // Specify the exiting object's current CSEK. + httpHeaders.set("x-goog-copy-source-encryption-algorithm", "AES256"); + httpHeaders.set("x-goog-copy-source-encryption-key", originalBase64Key); + httpHeaders.set("x-goog-copy-source-encryption-key-sha256", originalBase64KeyHash); + + // Specify the new CSEK that we would like to apply. + httpHeaders.set("x-goog-encryption-algorithm", "AES256"); + httpHeaders.set("x-goog-encryption-key", newBase64Key); + httpHeaders.set("x-goog-encryption-key-sha256", newBase64KeyHash); + + rewriteObject.setRequestHeaders(httpHeaders); + + try { + RewriteResponse rewriteResponse = rewriteObject.execute(); + + // If an object is very large, you may need to continue making successive calls to + // rewrite until the operation completes. + while (!rewriteResponse.getDone()) { + System.out.println("Rewrite did not complete. Resuming..."); + rewriteObject.setRewriteToken(rewriteResponse.getRewriteToken()); + rewriteResponse = rewriteObject.execute(); + } + } catch (GoogleJsonResponseException e) { + System.out.println("Error rotating key: " + e.getContent()); + System.exit(1); + } + } + + public static void main(String[] args) throws Exception { + if (args.length != 1) { + System.out.println("\nPlease run this with one argument: " + + "the GCS bucket into which this program should upload an object.\n\n" + + "You can create a bucket using gsutil like this:\n\n\t" + + "gsutil mb gs://name-of-bucket\n\n"); + System.exit(1); + } + String bucketName = args[0]; + + Storage storage = StorageFactory.getService(); + InputStream dataToUpload = new StorageUtils.ArbitrarilyLargeInputStream(10000000); + + System.out.format("Uploading object gs://%s/%s using CSEK.\n", bucketName, OBJECT_NAME); + uploadObject(storage, bucketName, OBJECT_NAME, dataToUpload, CSEK_KEY, CSEK_KEY_HASH); + + System.out.format("Downloading object gs://%s/%s using CSEK.\n", bucketName, OBJECT_NAME); + InputStream objectData = + downloadObject(storage, bucketName, OBJECT_NAME, CSEK_KEY, CSEK_KEY_HASH); + StorageUtils.readStream(objectData); + + System.out.println("Rotating object to use a different CSEK."); + rotateKey(storage, bucketName, OBJECT_NAME, CSEK_KEY, CSEK_KEY_HASH, + ANOTHER_CESK_KEY, ANOTHER_CSEK_KEY_HASH); + + System.out.println("Done"); + } + +} diff --git a/storage/json-api/src/main/java/StorageFactory.java b/storage/json-api/src/main/java/StorageFactory.java new file mode 100644 index 00000000000..df47b13475e --- /dev/null +++ b/storage/json-api/src/main/java/StorageFactory.java @@ -0,0 +1,57 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.api.services.storage.Storage; +import com.google.api.services.storage.StorageScopes; + +import java.io.IOException; +import java.security.GeneralSecurityException; +import java.util.Collection; + +/** + * This class manages the details of creating a Storage service, including auth. + */ +public class StorageFactory { + + private static Storage instance = null; + + public static synchronized Storage getService() throws IOException, GeneralSecurityException { + if (instance == null) { + instance = buildService(); + } + return instance; + } + + private static Storage buildService() throws IOException, GeneralSecurityException { + HttpTransport transport = GoogleNetHttpTransport.newTrustedTransport(); + JsonFactory jsonFactory = new JacksonFactory(); + GoogleCredential credential = GoogleCredential.getApplicationDefault(transport, jsonFactory); + + if (credential.createScopedRequired()) { + Collection bigqueryScopes = StorageScopes.all(); + credential = credential.createScoped(bigqueryScopes); + } + + return new Storage.Builder(transport, jsonFactory, credential) + .setApplicationName("GCS Samples") + .build(); + } +} diff --git a/storage/json-api/src/main/java/StorageSample.java b/storage/json-api/src/main/java/StorageSample.java index a09f65aea69..092271eb8c0 100644 --- a/storage/json-api/src/main/java/StorageSample.java +++ b/storage/json-api/src/main/java/StorageSample.java @@ -13,14 +13,8 @@ * the License. */ -import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; -import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; -import com.google.api.client.http.HttpTransport; import com.google.api.client.http.InputStreamContent; -import com.google.api.client.json.JsonFactory; -import com.google.api.client.json.jackson2.JacksonFactory; import com.google.api.services.storage.Storage; -import com.google.api.services.storage.StorageScopes; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.ObjectAccessControl; import com.google.api.services.storage.model.Objects; @@ -42,39 +36,9 @@ */ public class StorageSample { - /** - * Be sure to specify the name of your application. If the application name is {@code null} or - * blank, the application will log a warning. Suggested format is "MyCompany-ProductName/1.0". - */ - private static final String APPLICATION_NAME = "[[INSERT_YOUR_APP_NAME_HERE]]"; - /** Global instance of the JSON factory. */ - private static final JsonFactory JSON_FACTORY = JacksonFactory.getDefaultInstance(); private static final String TEST_FILENAME = "json-test.txt"; - // [START get_service] - private static Storage storageService; - - /** - * Returns an authenticated Storage object used to make service calls to Cloud Storage. - */ - private static Storage getService() throws IOException, GeneralSecurityException { - if (null == storageService) { - GoogleCredential credential = GoogleCredential.getApplicationDefault(); - // Depending on the environment that provides the default credentials (e.g. Compute Engine, - // App Engine), the credentials may require us to specify the scopes we need explicitly. - // Check for this case, and inject the Cloud Storage scope if required. - if (credential.createScopedRequired()) { - credential = credential.createScoped(StorageScopes.all()); - } - HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); - storageService = new Storage.Builder(httpTransport, JSON_FACTORY, credential) - .setApplicationName(APPLICATION_NAME).build(); - } - return storageService; - } - // [END get_service] - // [START list_bucket] /** * Fetch a list of the objects within the given bucket. @@ -84,7 +48,7 @@ private static Storage getService() throws IOException, GeneralSecurityException */ public static List listBucket(String bucketName) throws IOException, GeneralSecurityException { - Storage client = getService(); + Storage client = StorageFactory.getService(); Storage.Objects.List listRequest = client.objects().list(bucketName); List results = new ArrayList(); @@ -112,7 +76,7 @@ public static List listBucket(String bucketName) * @return a Bucket containing the bucket's metadata. */ public static Bucket getBucket(String bucketName) throws IOException, GeneralSecurityException { - Storage client = getService(); + Storage client = StorageFactory.getService(); Storage.Buckets.Get bucketRequest = client.buckets().get(bucketName); // Fetch the full set of the bucket's properties (e.g. include the ACLs in the response) @@ -142,7 +106,7 @@ public static void uploadStream( new ObjectAccessControl().setEntity("allUsers").setRole("READER"))); // Do the insert - Storage client = getService(); + Storage client = StorageFactory.getService(); Storage.Objects.Insert insertRequest = client.objects().insert( bucketName, objectMetadata, contentStream); @@ -159,7 +123,7 @@ public static void uploadStream( */ public static void deleteObject(String path, String bucketName) throws IOException, GeneralSecurityException { - Storage client = getService(); + Storage client = StorageFactory.getService(); client.objects().delete(bucketName, path).execute(); } // [END delete_object] diff --git a/storage/json-api/src/main/java/StorageUtils.java b/storage/json-api/src/main/java/StorageUtils.java new file mode 100644 index 00000000000..71370a6032c --- /dev/null +++ b/storage/json-api/src/main/java/StorageUtils.java @@ -0,0 +1,56 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.io.InputStream; + +public class StorageUtils { + + /** + * Reads the contents of an InputStream and does nothing with it. + */ + public static void readStream(InputStream is) throws IOException { + byte[] inputBuffer = new byte[256]; + while (is.read(inputBuffer) != -1) {} + // The caller is responsible for closing this InputStream. + is.close(); + } + + /** + * A helper class to provide input streams of any size. + * The input streams will be full of null bytes. + */ + static class ArbitrarilyLargeInputStream extends InputStream { + + private long bytesRead; + private final long streamSize; + + public ArbitrarilyLargeInputStream(long streamSizeInBytes) { + bytesRead = 0; + this.streamSize = streamSizeInBytes; + } + + @Override + public int read() throws IOException { + if (bytesRead >= streamSize) { + return -1; + } + bytesRead++; + return 0; + } + } +} +