From edd0f2119a6a80d078d79498d7e6e100f85bf4a3 Mon Sep 17 00:00:00 2001 From: JP Martin Date: Tue, 5 Apr 2016 15:07:13 -0700 Subject: [PATCH] Split off the "normal case" example from the "legacy case" one. * Split off the "normal case" example from the "legacy case" one. (and create a "normal case" example, first) * Correct damage from "helpful" IDE renaming help. * Switch gcloud-java-examples to appassembler This has the following benefits: * Stat example now works * Easier to see what is available (list the bin folder) * Easier to see what running an example entails (read the script) * Simpler command line (no more -Dexec.args) * Rename example to examples and switch a Stat to ListFilesystems * Clean up pom Fix indent to 2, remove redundant groupId line that's inherited from the parent. * Reviewer comments and update_docs_version.sh * Added blank line * Sweating the small stuff Adding a \n at the end of the last line. --- gcloud-java-contrib/README.md | 6 ++ .../gcloud-java-nio-examples/README.md | 41 +++++++++ .../gcloud-java-nio-examples/pom.xml | 91 +++++++++++++++++++ .../gcloud/nio/examples/ListFilesystems.java | 47 ++++++++++ gcloud-java-contrib/gcloud-java-nio/pom.xml | 69 +++++++------- gcloud-java-contrib/pom.xml | 1 + gcloud-java-examples/README.md | 59 ++++++------ gcloud-java-examples/pom.xml | 49 ++++++++-- .../com/google/gcloud/examples/nio/Stat.java | 9 +- utilities/update_docs_version.sh | 6 +- 10 files changed, 301 insertions(+), 77 deletions(-) create mode 100644 gcloud-java-contrib/gcloud-java-nio-examples/README.md create mode 100644 gcloud-java-contrib/gcloud-java-nio-examples/pom.xml create mode 100644 gcloud-java-contrib/gcloud-java-nio-examples/src/main/java/com/google/gcloud/nio/examples/ListFilesystems.java diff --git a/gcloud-java-contrib/README.md b/gcloud-java-contrib/README.md index b8bef6eb977e..ec7bdb6e1a09 100644 --- a/gcloud-java-contrib/README.md +++ b/gcloud-java-contrib/README.md @@ -7,6 +7,12 @@ Contents -------- * [gcloud-java-nio](./gcloud-java-nio/): NIO Filesystem Provider for Google Cloud Storage. + * [gcloud-java-nio-examples](./gcloud-java-nio-examples/): How to add GCS NIO after the fact. + +See also +-------- + + * [gcloud-java-examples](../gcloud-java-examples) for an example of how to use NIO normally. Contributing ------------ diff --git a/gcloud-java-contrib/gcloud-java-nio-examples/README.md b/gcloud-java-contrib/gcloud-java-nio-examples/README.md new file mode 100644 index 000000000000..206ca93759e5 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio-examples/README.md @@ -0,0 +1,41 @@ +Example of adding the Google Cloud Storage NIO Provider to a legacy jar +======================================================================= + +This project shows how to add GCS capabilities to a jar file for a Java 7 application +that uses Java NIO without the need to recompile. + +Note that whenever possible, you instead want to recompile the app and use the normal +dependency mechanism to add a dependency to gcloud-java-nio. You can see examples of +this in the [gcloud-java-examples](../../gcloud-java-examples) project. + +To run this example: + +1. Before running the example, go to the [Google Developers Console][developers-console] to ensure that Google Cloud Storage API is enabled. + +2. Log in using gcloud SDK (`gcloud auth login` in command line) + +3. Compile the JAR with: + ``` + mvn package -DskipTests -Dmaven.javadoc.skip=true -Dmaven.source.skip=true + ``` + +4. Run the sample with: + + ``` + java -cp gcloud-java-contrib/gcloud-java-nio/target/gcloud-java-nio-0.1.6-SNAPSHOT-shaded.jar:gcloud-java-contrib/gcloud-java-nio-examples/target/gcloud-java-nio-examples-0.1.6-SNAPSHOT.jar com.google.gcloud.nio.examples.ListFilesystems + ``` + + Notice that it lists gcs, which it wouldn't if you ran it without the nio jar: + ``` + java -cp gcloud-java-contrib/gcloud-java-nio-examples/target/gcloud-java-nio-examples-0.1.6-SNAPSHOT.jar com.google.gcloud.nio.examples.ListFilesystems + ``` + +The sample doesn't have anything about GCS in it. It gets that ability from the nio jar that +we're adding to the classpath. You can use the nio "fat shaded" jar for this purpose as it also +includes the dependencies for gcloud-java-nio. +The underlying mechanism is Java's standard [ServiceLoader](https://docs.oracle.com/javase/7/docs/api/java/util/ServiceLoader.html) +facility, the [standard way](http://docs.oracle.com/javase/7/docs/technotes/guides/io/fsp/filesystemprovider.html) to plug in NIO providers like this one. + +If you have access to a project's source code you can also simply add gcloud-java-nio as +a dependency and let Maven pull in the required dependencies (this is what the nio unit tests do). +This approach is preferable as the fat jar approach may waste memory on multiple copies of dependencies. diff --git a/gcloud-java-contrib/gcloud-java-nio-examples/pom.xml b/gcloud-java-contrib/gcloud-java-nio-examples/pom.xml new file mode 100644 index 000000000000..1ae7a6c1e190 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio-examples/pom.xml @@ -0,0 +1,91 @@ + + + 4.0.0 + gcloud-java-nio-examples + jar + GCloud Java NIO example + + Demonstrates how to use the gcloud-java-nio jar to add GCS functionality to legacy code. + + + com.google.gcloud + gcloud-java-contrib + 0.1.6-SNAPSHOT + + + nio + + + + ${project.groupId} + gcloud-java-storage + ${project.version} + + + com.google.guava + guava + 19.0 + + + com.google.code.findbugs + jsr305 + 2.0.1 + + + javax.inject + javax.inject + 1 + + + com.google.auto.service + auto-service + 1.0-rc2 + provided + + + com.google.auto.value + auto-value + 1.1 + provided + + + junit + junit + 4.12 + test + + + com.google.guava + guava-testlib + 19.0 + test + + + com.google.truth + truth + 0.27 + test + + + org.mockito + mockito-core + 1.9.5 + + + org.apache.maven.plugins + maven-assembly-plugin + 2.5.4 + + + + + + org.codehaus.mojo + exec-maven-plugin + + false + + + + + diff --git a/gcloud-java-contrib/gcloud-java-nio-examples/src/main/java/com/google/gcloud/nio/examples/ListFilesystems.java b/gcloud-java-contrib/gcloud-java-nio-examples/src/main/java/com/google/gcloud/nio/examples/ListFilesystems.java new file mode 100644 index 000000000000..5e92768d1cf6 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio-examples/src/main/java/com/google/gcloud/nio/examples/ListFilesystems.java @@ -0,0 +1,47 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.nio.examples; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.spi.FileSystemProvider; + +/** + * ListFilesystems is a super-simple program that lists the available NIO filesystems. + */ +public class ListFilesystems { + + /** + * See the class documentation. + */ + public static void main(String[] args) throws IOException { + listFilesystems(); + } + + private static void listFilesystems() { + System.out.println("Installed filesystem providers:"); + for (FileSystemProvider p : FileSystemProvider.installedProviders()) { + System.out.println(" " + p.getScheme()); + } + } + +} diff --git a/gcloud-java-contrib/gcloud-java-nio/pom.xml b/gcloud-java-contrib/gcloud-java-nio/pom.xml index b3c2a99eea10..0dbfa0153d65 100644 --- a/gcloud-java-contrib/gcloud-java-nio/pom.xml +++ b/gcloud-java-contrib/gcloud-java-nio/pom.xml @@ -1,7 +1,6 @@ 4.0.0 - com.google.gcloud gcloud-java-nio jar GCloud Java NIO @@ -92,40 +91,40 @@ -Xlint:unchecked - - - org.apache.maven.plugins - maven-shade-plugin - 2.4.3 - - true - - - com - shaded.gcloud-nio.com - - com.google.gcloud.** - - - - org - shaded.gcloud-nio.org - - - google - shaded.gcloud-nio.google - - - - - - package - - shade - - - - + + + org.apache.maven.plugins + maven-shade-plugin + 2.4.3 + + true + + + com + shaded.gcloud-nio.com + + com.google.gcloud.** + + + + org + shaded.gcloud-nio.org + + + google + shaded.gcloud-nio.google + + + + + + package + + shade + + + + diff --git a/gcloud-java-contrib/pom.xml b/gcloud-java-contrib/pom.xml index 4c8eff8cab0f..e1370fb0ec8e 100644 --- a/gcloud-java-contrib/pom.xml +++ b/gcloud-java-contrib/pom.xml @@ -17,6 +17,7 @@ gcloud-java-nio + gcloud-java-nio-examples diff --git a/gcloud-java-examples/README.md b/gcloud-java-examples/README.md index 29872245a19d..3226ab1fe725 100644 --- a/gcloud-java-examples/README.md +++ b/gcloud-java-examples/README.md @@ -37,9 +37,9 @@ To run examples from your command line: 2. Set your current project using `gcloud config set project PROJECT_ID`. This step is not necessary for `ResourceManagerExample`. -3. Compile using Maven (`mvn compile` in command line from your base project directory) +3. Compile using Maven: `cd gcloud-java-examples` in command line from your base project directory and then `mvn package appassembler:assemble -DskipTests -Dmaven.javadoc.skip=true -Dmaven.source.skip=true`. -4. Run an example using Maven from command line. +4. Run an example from the command line using the Maven-generated scripts. * Here's an example run of `BigQueryExample`. @@ -55,62 +55,57 @@ To run examples from your command line: ``` Then you are ready to run the following example: ``` - mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.bigquery.BigQueryExample" -Dexec.args="create dataset new_dataset_id" - mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.bigquery.BigQueryExample" -Dexec.args="create table new_dataset_id new_table_id field_name:string" - mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.bigquery.BigQueryExample" -Dexec.args="list tables new_dataset_id" - mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.bigquery.BigQueryExample" -Dexec.args="load new_dataset_id new_table_id CSV gs://my_bucket/my_csv_file" - mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.bigquery.BigQueryExample" -Dexec.args="query 'select * from new_dataset_id.new_table_id'" + target/appassembler/bin/BigQueryExample create dataset new_dataset_id + target/appassembler/bin/BigQueryExample create table new_dataset_id new_table_id field_name:string + target/appassembler/bin/BigQueryExample list tables new_dataset_id + target/appassembler/bin/BigQueryExample load new_dataset_id new_table_id CSV gs://my_bucket/my_csv_file + target/appassembler/bin/BigQueryExample query 'select * from new_dataset_id.new_table_id' ``` * Here's an example run of `DatastoreExample`. Be sure to change the placeholder project ID "your-project-id" with your own project ID. Also note that you have to enable the Google Cloud Datastore API on the [Google Developers Console][developers-console] before running the following commands. ``` - mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.datastore.DatastoreExample" -Dexec.args="your-project-id my_name add my\ comment" - mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.datastore.DatastoreExample" -Dexec.args="your-project-id my_name display" - mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.datastore.DatastoreExample" -Dexec.args="your-project-id my_name delete" + target/appassembler/bin/DatastoreExample your-project-id my_name add my\ comment + target/appassembler/bin/DatastoreExample your-project-id my_name display + target/appassembler/bin/DatastoreExample your-project-id my_name delete ``` * Here's an example run of `ResourceManagerExample`. Be sure to change the placeholder project ID "your-project-id" with your own globally unique project ID. ``` - mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.resourcemanager.ResourceManagerExample" -Dexec.args="create your-project-id" - mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.resourcemanager.ResourceManagerExample" -Dexec.args="list" - mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.resourcemanager.ResourceManagerExample" -Dexec.args="get your-project-id" + target/appassembler/bin/ResourceManagerExample create your-project-id + target/appassembler/bin/ResourceManagerExample list + target/appassembler/bin/ResourceManagerExample get your-project-id ``` * Here's an example run of `StorageExample`. Before running the example, go to the [Google Developers Console][developers-console] to ensure that Google Cloud Storage API is enabled and that you have a bucket. Also ensure that you have a test file (`test.txt` is chosen here) to upload to Cloud Storage stored locally on your machine. ``` - mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.storage.StorageExample" -Dexec.args="upload /path/to/test.txt " - mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.storage.StorageExample" -Dexec.args="list " - mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.storage.StorageExample" -Dexec.args="download test.txt" - mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.storage.StorageExample" -Dexec.args="delete test.txt" + target/appassembler/bin/StorageExample upload /path/to/test.txt + target/appassembler/bin/StorageExample list + target/appassembler/bin/StorageExample download test.txt + target/appassembler/bin/StorageExample delete test.txt ``` - * Here's an example run of `Stat`, illustrating the use of the gcloud-java-nio jar. - Before running the example, go to the [Google Developers Console][developers-console] to ensure that Google Cloud Storage API is enabled and that you have a bucket with a file in it. - Compile the JAR with: - ``` - mvn package -DskipTests -Dmaven.javadoc.skip=true -Dmaven.source.skip=true - ``` - Then run the sample with: + * Here's an example run of `Stat`, illustrating the use of gcloud-java-nio. + + Before running the example, go to the [Google Developers Console][developers-console] to ensure that Google Cloud Storage API is enabled and that you have a bucket with a file in it. + + Run the sample with (from the gcloud-java-examples folder): ``` - java -cp gcloud-java-contrib/gcloud-java-nio/target/gcloud-java-nio-0.1.6-SNAPSHOT-shaded.jar:gcloud-java-examples/target/gcloud-java-examples-0.1.6-SNAPSHOT.jar com.google.gcloud.examples.nio.Stat --check + target/appassembler/bin/Stat --check + ``` Or, if you have a file in `gs://mybucket/myfile.txt`, you can run: ``` - java -cp gcloud-java-contrib/gcloud-java-nio/target/gcloud-java-nio-0.1.6-SNAPSHOT-shaded.jar:gcloud-java-examples/target/gcloud-java-examples-0.1.6-SNAPSHOT.jar com.google.gcloud.examples.nio.Stat gs://mybucket/myfile.txt + target/appassembler/bin/Stat gs://mybucket/myfile.txt ``` - The sample doesn't have anything about GCS in it. It gets that ability from the nio jar that - we're adding to the classpath. You can use the nio "fat shaded" jar for this purpose as it also - includes the dependencies for gcloud-java-nio. - If you have access to a project's source code you can also simply add gcloud-java-nio as - a dependency and let Maven pull in the required dependencies (this is what the nio unit tests do). - This approach is preferable as the fat jar approach may waste memory on multiple copies of dependencies. + The sample doesn't have anything special about GCS in it, it just opens files via the NIO API. + It lists gcloud-java-nio as a dependency, and that enables it to interpret `gs://` paths. Troubleshooting --------------- diff --git a/gcloud-java-examples/pom.xml b/gcloud-java-examples/pom.xml index 74cba268e2bd..fb77f1fc3541 100644 --- a/gcloud-java-examples/pom.xml +++ b/gcloud-java-examples/pom.xml @@ -21,12 +21,16 @@ gcloud-java ${project.version} - - org.apache.maven.plugins - maven-assembly-plugin - 2.5.4 - - + + ${project.groupId} + gcloud-java-nio + ${project.version} + + + org.apache.maven.plugins + maven-assembly-plugin + 2.5.4 + @@ -37,6 +41,39 @@ false + + + org.codehaus.mojo + appassembler-maven-plugin + + + + com.google.gcloud.examples.bigquery.BigQueryExample + BigQueryExample + + + com.google.gcloud.examples.datastore.DatastoreExample + DatastoreExample + + + com.google.gcloud.examples.nio.Stat + Stat + + + + com.google.gcloud.examples.resourcemanager.ResourceManagerExample + + ResourceManagerExample + + + com.google.gcloud.examples.storage.StorageExample + StorageExample + + + + diff --git a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/nio/Stat.java b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/nio/Stat.java index 6a0349701804..054657d717ef 100644 --- a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/nio/Stat.java +++ b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/nio/Stat.java @@ -20,6 +20,9 @@ * be logged in (using e.g. the gcloud auth command). * *

See the README for a command line to run this example. + * + *

In short, this version (in gcloud-java-examples) is in a package that lists gcloud-java-nio + * as a dependency, so it will work directly without having to do any special work. */ public class Stat { @@ -77,8 +80,8 @@ private static void help() { "", "This tool normally knows nothing of Google Cloud Storage. If you pass it --check", "or a GCS file name (e.g. gs://mybucket/myfile), it will show an error.", - "However, by just adding the gcloud-nio jar in your classpath, this tool is made", - "aware of gs:// paths and can access files on the cloud.", + "However, by just adding the gcloud-nio jar as a dependency and recompiling, this tool is", + "made aware of gs:// paths and can access files on the cloud.", "", "The gcloud NIO filesystem provider can similarly enable existing Java 7 programs", "to read and write cloud files, even if they have no special built-in cloud support." @@ -97,7 +100,7 @@ private static void listFilesystems() { private static void checkGcs() { FileSystem fs = FileSystems.getFileSystem(URI.create("gs://domain-registry-alpha")); - System.out.println("We seem to be able to instantiate a gs:// filesystem."); + System.out.println("Success! We can instantiate a gs:// filesystem."); System.out.println("isOpen: " + fs.isOpen()); System.out.println("isReadOnly: " + fs.isReadOnly()); } diff --git a/utilities/update_docs_version.sh b/utilities/update_docs_version.sh index 4fc0aa772963..c8f132db2a3c 100755 --- a/utilities/update_docs_version.sh +++ b/utilities/update_docs_version.sh @@ -17,7 +17,11 @@ if [ "${RELEASED_VERSION##*-}" != "SNAPSHOT" ]; then sed -ri "s/:[0-9]+\.[0-9]+\.[0-9]+'/:${RELEASED_VERSION}'/g" ${item}/README.md sed -ri "s/\"[0-9]+\.[0-9]+\.[0-9]+\"/\"${RELEASED_VERSION}\"/g" ${item}/README.md done - + + NEW_VERSION=${RELEASED_VERSION%.*}.$((${RELEASED_VERSION##*.}+1))-SNAPSHOT + echo "Changing version to $NEW_VERSION in gcloud-java-nio-example README" + sed -ri "s/gcloud-java-nio-examples-[0-9]+\.[0-9]+\.[0-9]+-SNAPSHOT/gcloud-java-nio-examples-$NEW_VERSION/g" gcloud-java-contrib/gcloud-java-nio-examples/README.md + git add README.md */README.md git config --global user.name "travis-ci" git config --global user.email "travis@travis-ci.org"