From dceeb5f29b588ad26c0083af18a56f36b032c2b2 Mon Sep 17 00:00:00 2001 From: Brandon Yarbrough Date: Wed, 23 Mar 2016 14:54:55 -0700 Subject: [PATCH] Switch to getApplicationDefault()-based auth. Modify docs+others accordingly. --- storage/json-api/README.md | 33 +++++- ...CustomerSuppliedEncryptionKeysSamples.java | 111 ++---------------- .../src/main/java/StorageFactory.java | 57 +++++++++ .../json-api/src/main/java/StorageSample.java | 44 +------ .../json-api/src/main/java/StorageUtils.java | 40 +++++++ 5 files changed, 140 insertions(+), 145 deletions(-) create mode 100644 storage/json-api/src/main/java/StorageFactory.java create mode 100644 storage/json-api/src/main/java/StorageUtils.java diff --git a/storage/json-api/README.md b/storage/json-api/README.md index a026cb4ff40..9d0784a5813 100644 --- a/storage/json-api/README.md +++ b/storage/json-api/README.md @@ -4,16 +4,39 @@ Google Cloud Storage Service features a REST-based API that allows developers to ## Quickstart -Install [Maven](http://maven.apache.org/). +1. Install the [Google Cloud SDK](https://cloud.google.com/sdk/), including the [gcloud tool](https://cloud.google.com/sdk/gcloud/). -Build your project with: +1. Setup the gcloud tool. + ``` + gcloud init + ``` - mvn package +1. Clone this repo. -You can then run a given `ClassName` via: + ``` + git clone https://github.com/GoogleCloudPlatform/java-docs-samples.git + ``` - mvn exec:java -Dexec.mainClass=StorageSample \ +1. Install [Maven](http://maven.apache.org/). + +1. Build this project from this directory: + + ``` + mvn package + ``` + +1. Run one of the sample apps by specifying its class name and a bucket name: + + ``` + mvn exec:java -Dexec.mainClass=StorageSample \ -Dexec.args="ABucketName" + ``` + +Note that if it's been a while, you may need to login with gcloud. + + ``` + gcloud auth login + ``` ## Products - [Google Cloud Storage][2] diff --git a/storage/json-api/src/main/java/CustomerSuppliedEncryptionKeysSamples.java b/storage/json-api/src/main/java/CustomerSuppliedEncryptionKeysSamples.java index c7603051b0c..b33e7c55ef0 100644 --- a/storage/json-api/src/main/java/CustomerSuppliedEncryptionKeysSamples.java +++ b/storage/json-api/src/main/java/CustomerSuppliedEncryptionKeysSamples.java @@ -1,25 +1,11 @@ -import com.google.api.client.auth.oauth2.Credential; -import com.google.api.client.extensions.java6.auth.oauth2.AuthorizationCodeInstalledApp; -import com.google.api.client.extensions.jetty.auth.oauth2.LocalServerReceiver; -import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeFlow; -import com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets; -import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.http.HttpHeaders; -import com.google.api.client.http.HttpTransport; import com.google.api.client.http.InputStreamContent; -import com.google.api.client.json.JsonFactory; -import com.google.api.client.json.jackson2.JacksonFactory; -import com.google.api.client.util.store.DataStoreFactory; -import com.google.api.client.util.store.FileDataStoreFactory; import com.google.api.services.storage.Storage; -import com.google.api.services.storage.StorageScopes; import com.google.api.services.storage.model.RewriteResponse; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.Collections; /** * Demonstrates the use of GCS's CSEK features via the Java API client library @@ -35,9 +21,6 @@ **/ class CustomerSuppliedEncryptionKeysSamples { - private static final java.io.File DATA_STORE_DIR = - new java.io.File(System.getProperty("user.home"), ".store/storage_sample"); - // You can (and should) generate your own CSEK Key! Try running this from the command line: // python -c 'import base64; import os; print(base64.encodestring(os.urandom(32)))' // Also, these encryption keys are included here for simplicity, but please remember that @@ -135,11 +118,11 @@ public static void uploadObject( httpHeaders.set("x-goog-encryption-algorithm", "AES256"); httpHeaders.set("x-goog-encryption-key", base64CSEKey); httpHeaders.set("x-goog-encryption-key-sha256", base64CSEKeyHash); - + // Since our request includes our private key as a header, it is a good idea to instruct caches // and proxies not to store this request. httpHeaders.setCacheControl("no-store"); - + insertObject.setRequestHeaders(httpHeaders); try { @@ -189,11 +172,11 @@ public static void rotateKey( httpHeaders.set("x-goog-encryption-algorithm", "AES256"); httpHeaders.set("x-goog-encryption-key", newBase64Key); httpHeaders.set("x-goog-encryption-key-sha256", newBase64KeyHash); - + // Since our request includes our private key as a header, it is a good idea to instruct caches // and proxies not to store this request. httpHeaders.setCacheControl("no-store"); - + rewriteObject.setRequestHeaders(httpHeaders); try { @@ -221,95 +204,23 @@ public static void main(String[] args) throws Exception { System.exit(1); } String bucketName = args[0]; - // CSEK, like the JSON API, may be used only via HTTPS. - HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); - DataStoreFactory dataStoreFactory = new FileDataStoreFactory(DATA_STORE_DIR); - JsonFactory jsonFactory = JacksonFactory.getDefaultInstance(); - Credential credential = authorize(jsonFactory, httpTransport, dataStoreFactory); - Storage storage = - new Storage.Builder(httpTransport, jsonFactory, credential) - .setApplicationName("JavaCSEKApiSample") - .build(); - - InputStream dataToUpload = new ArbitrarilyLargeInputStream(10000000); + + Storage storage = StorageFactory.getService(); + InputStream dataToUpload = new StorageUtils.ArbitrarilyLargeInputStream(10000000); System.out.format("Uploading object gs://%s/%s using CSEK.\n", bucketName, OBJECT_NAME); uploadObject(storage, bucketName, OBJECT_NAME, dataToUpload, CSEK_KEY, CSEK_KEY_HASH); + System.out.format("Downloading object gs://%s/%s using CSEK.\n", bucketName, OBJECT_NAME); InputStream objectData = downloadObject(storage, bucketName, OBJECT_NAME, CSEK_KEY, CSEK_KEY_HASH); - readStream(objectData); + StorageUtils.readStream(objectData); + System.out.println("Rotating object to use a different CSEK."); rotateKey(storage, bucketName, OBJECT_NAME, CSEK_KEY, CSEK_KEY_HASH, ANOTHER_CESK_KEY, ANOTHER_CSEK_KEY_HASH); - System.out.println(); - } - - private static Credential authorize( - JsonFactory jsonFactory, HttpTransport httpTransport, DataStoreFactory dataStoreFactory) - throws Exception { - - InputStream clientSecretStream = - CustomerSuppliedEncryptionKeysSamples.class - .getResourceAsStream("client_secrets.json"); - if (clientSecretStream == null) { - throw new RuntimeException("Could not load secrets"); - } - - // Load client secrets - GoogleClientSecrets clientSecrets = - GoogleClientSecrets.load(jsonFactory, new InputStreamReader(clientSecretStream)); - - // Set up authorization code flow - GoogleAuthorizationCodeFlow flow = - new GoogleAuthorizationCodeFlow.Builder( - httpTransport, - jsonFactory, - clientSecrets, - Collections.singleton(StorageScopes.DEVSTORAGE_FULL_CONTROL)) - .setDataStoreFactory(dataStoreFactory) - .build(); - - // Authorize - Credential credential = - new AuthorizationCodeInstalledApp(flow, new LocalServerReceiver()).authorize("user"); - - return credential; - } - - /** - * Reads the contents of an InputStream and does nothing with it. - */ - private static void readStream(InputStream is) throws IOException { - byte inputBuffer[] = new byte[256]; - while (is.read(inputBuffer) != -1) {} - // The caller is responsible for closing this InputStream. - is.close(); - } - - /** - * A helper class to provide input streams of any size. - * The input streams will be full of null bytes. - */ - static class ArbitrarilyLargeInputStream extends InputStream { - - private long bytesRead; - private final long streamSize; - - public ArbitrarilyLargeInputStream(long streamSizeInBytes) { - bytesRead = 0; - this.streamSize = streamSizeInBytes; - } - - @Override - public int read() throws IOException { - if (bytesRead >= streamSize) { - return -1; - } - bytesRead++; - return 0; - } + System.out.println("Done"); } } diff --git a/storage/json-api/src/main/java/StorageFactory.java b/storage/json-api/src/main/java/StorageFactory.java new file mode 100644 index 00000000000..f9303c26668 --- /dev/null +++ b/storage/json-api/src/main/java/StorageFactory.java @@ -0,0 +1,57 @@ +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.api.services.storage.Storage; +import com.google.api.services.storage.StorageScopes; + +import java.io.IOException; +import java.security.GeneralSecurityException; +import java.util.Collection; + +/* + * Copyright (c) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +/** + * This class manages the details of creating a Storage service, including auth. + */ +public class StorageFactory { + + private static Storage instance = null; + + public static synchronized Storage getService() throws IOException, GeneralSecurityException { + if (instance == null) { + instance = buildService(); + } + return instance; + } + + private static Storage buildService() throws IOException, GeneralSecurityException { + HttpTransport transport = GoogleNetHttpTransport.newTrustedTransport(); + JsonFactory jsonFactory = new JacksonFactory(); + GoogleCredential credential = GoogleCredential.getApplicationDefault(transport, jsonFactory); + + if (credential.createScopedRequired()) { + Collection bigqueryScopes = StorageScopes.all(); + credential = credential.createScoped(bigqueryScopes); + } + + return new Storage.Builder(transport, jsonFactory, credential) + .setApplicationName("GCS Samples") + .build(); + } +} diff --git a/storage/json-api/src/main/java/StorageSample.java b/storage/json-api/src/main/java/StorageSample.java index a09f65aea69..092271eb8c0 100644 --- a/storage/json-api/src/main/java/StorageSample.java +++ b/storage/json-api/src/main/java/StorageSample.java @@ -13,14 +13,8 @@ * the License. */ -import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; -import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; -import com.google.api.client.http.HttpTransport; import com.google.api.client.http.InputStreamContent; -import com.google.api.client.json.JsonFactory; -import com.google.api.client.json.jackson2.JacksonFactory; import com.google.api.services.storage.Storage; -import com.google.api.services.storage.StorageScopes; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.ObjectAccessControl; import com.google.api.services.storage.model.Objects; @@ -42,39 +36,9 @@ */ public class StorageSample { - /** - * Be sure to specify the name of your application. If the application name is {@code null} or - * blank, the application will log a warning. Suggested format is "MyCompany-ProductName/1.0". - */ - private static final String APPLICATION_NAME = "[[INSERT_YOUR_APP_NAME_HERE]]"; - /** Global instance of the JSON factory. */ - private static final JsonFactory JSON_FACTORY = JacksonFactory.getDefaultInstance(); private static final String TEST_FILENAME = "json-test.txt"; - // [START get_service] - private static Storage storageService; - - /** - * Returns an authenticated Storage object used to make service calls to Cloud Storage. - */ - private static Storage getService() throws IOException, GeneralSecurityException { - if (null == storageService) { - GoogleCredential credential = GoogleCredential.getApplicationDefault(); - // Depending on the environment that provides the default credentials (e.g. Compute Engine, - // App Engine), the credentials may require us to specify the scopes we need explicitly. - // Check for this case, and inject the Cloud Storage scope if required. - if (credential.createScopedRequired()) { - credential = credential.createScoped(StorageScopes.all()); - } - HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); - storageService = new Storage.Builder(httpTransport, JSON_FACTORY, credential) - .setApplicationName(APPLICATION_NAME).build(); - } - return storageService; - } - // [END get_service] - // [START list_bucket] /** * Fetch a list of the objects within the given bucket. @@ -84,7 +48,7 @@ private static Storage getService() throws IOException, GeneralSecurityException */ public static List listBucket(String bucketName) throws IOException, GeneralSecurityException { - Storage client = getService(); + Storage client = StorageFactory.getService(); Storage.Objects.List listRequest = client.objects().list(bucketName); List results = new ArrayList(); @@ -112,7 +76,7 @@ public static List listBucket(String bucketName) * @return a Bucket containing the bucket's metadata. */ public static Bucket getBucket(String bucketName) throws IOException, GeneralSecurityException { - Storage client = getService(); + Storage client = StorageFactory.getService(); Storage.Buckets.Get bucketRequest = client.buckets().get(bucketName); // Fetch the full set of the bucket's properties (e.g. include the ACLs in the response) @@ -142,7 +106,7 @@ public static void uploadStream( new ObjectAccessControl().setEntity("allUsers").setRole("READER"))); // Do the insert - Storage client = getService(); + Storage client = StorageFactory.getService(); Storage.Objects.Insert insertRequest = client.objects().insert( bucketName, objectMetadata, contentStream); @@ -159,7 +123,7 @@ public static void uploadStream( */ public static void deleteObject(String path, String bucketName) throws IOException, GeneralSecurityException { - Storage client = getService(); + Storage client = StorageFactory.getService(); client.objects().delete(bucketName, path).execute(); } // [END delete_object] diff --git a/storage/json-api/src/main/java/StorageUtils.java b/storage/json-api/src/main/java/StorageUtils.java new file mode 100644 index 00000000000..5e37b36f790 --- /dev/null +++ b/storage/json-api/src/main/java/StorageUtils.java @@ -0,0 +1,40 @@ +import java.io.IOException; +import java.io.InputStream; + +public class StorageUtils { + + /** + * Reads the contents of an InputStream and does nothing with it. + */ + public static void readStream(InputStream is) throws IOException { + byte inputBuffer[] = new byte[256]; + while (is.read(inputBuffer) != -1) {} + // The caller is responsible for closing this InputStream. + is.close(); + } + + /** + * A helper class to provide input streams of any size. + * The input streams will be full of null bytes. + */ + static class ArbitrarilyLargeInputStream extends InputStream { + + private long bytesRead; + private final long streamSize; + + public ArbitrarilyLargeInputStream(long streamSizeInBytes) { + bytesRead = 0; + this.streamSize = streamSizeInBytes; + } + + @Override + public int read() throws IOException { + if (bytesRead >= streamSize) { + return -1; + } + bytesRead++; + return 0; + } + } +} +