From f7b572466e5c749a537117769d156d84162b0db5 Mon Sep 17 00:00:00 2001 From: JP Martin Date: Wed, 27 Apr 2016 10:04:14 -0700 Subject: [PATCH 1/4] CountBytes --- gcloud-java-examples/README.md | 4 +- gcloud-java-examples/pom.xml | 4 + .../google/cloud/examples/nio/CountBytes.java | 78 +++++++++++++++++++ 3 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/CountBytes.java diff --git a/gcloud-java-examples/README.md b/gcloud-java-examples/README.md index 3d6890d17cea..4cc03c7ee53a 100644 --- a/gcloud-java-examples/README.md +++ b/gcloud-java-examples/README.md @@ -37,8 +37,8 @@ To run examples from your command line: 2. Set your current project using `gcloud config set project PROJECT_ID`. This step is not necessary for `ResourceManagerExample`. -3. Compile using Maven: `mvn install -DskipTests -Dmaven.javadoc.skip=true -Dmaven.source.skip=true` in command line from your base project directory - then `cd gcloud-java-examples` and finally `mvn package appassembler:assemble -DskipTests -Dmaven.javadoc.skip=true -Dmaven.source.skip=true`. +3. Compile using Maven: `mvn install -DskipTests` in command line from your base project directory + then `cd gcloud-java-examples` and finally `mvn package appassembler:assemble -DskipTests`. 4. Run an example from the command line using the Maven-generated scripts. diff --git a/gcloud-java-examples/pom.xml b/gcloud-java-examples/pom.xml index e8f00852d079..642f8fb0fff0 100644 --- a/gcloud-java-examples/pom.xml +++ b/gcloud-java-examples/pom.xml @@ -63,6 +63,10 @@ com.google.cloud.examples.nio.Stat Stat + + com.google.cloud.examples.nio.CountBytes + CountBytes + com.google.cloud.examples.resourcemanager.ResourceManagerExample diff --git a/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/CountBytes.java b/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/CountBytes.java new file mode 100644 index 000000000000..caa1a45b731f --- /dev/null +++ b/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/CountBytes.java @@ -0,0 +1,78 @@ +package com.google.cloud.examples.nio; + +import com.google.common.base.Stopwatch; + +import java.io.IOException; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.TimeUnit; + +/** + * CountBytes will read through the whole file given as input. + * + *

It's meant for testing that NIO correctly saturates your network link. + * + *

See the README for compilation instructions. Run this code with + * {@code target/appassembler/bin/CountBytes } + */ +public class CountBytes { + + /** + * See the class documentation. + */ + public static void main(String[] args) throws IOException { + if (args.length == 0 || args[0].equals("--help")) { + help(); + return; + } + for (String a : args) { + countFile(a); + } + } + + /** + * Print the length of the indicated file. + * + *

This uses the normal Java NIO Api, so it can take advantage of any installed + * NIO Filesystem provider without any extra effort. + */ + private static void countFile(String fname) { + final int bufSize = 1024 * 1024; + try { + Path path = Paths.get(new URI(fname)); + long size = Files.size(path); + System.out.println(fname + ": " + size + " bytes."); + Stopwatch sw = Stopwatch.createStarted(); + ByteBuffer buf = ByteBuffer.allocate(bufSize); + SeekableByteChannel chan = Files.newByteChannel(path); + long total = 0; + for (int read = 1; read > 0; ) { + read = chan.read(buf); + buf.flip(); + total += read; + } + long elapsed = sw.elapsed(TimeUnit.SECONDS); + System.out.println("Read all " + total + " bytes in " + elapsed + "s."); + if (total != size) { + System.out.println("Wait, this doesn't match! We saw " + total + " bytes, yet the file size is listed at " + size + " bytes."); + } + } catch (Exception ex) { + System.out.println(fname + ": " + ex.toString()); + } + } + + private static void help() { + String[] help = + {"The argument is a ", + "and we show the length of that file.", + }; + for (String s : help) { + System.out.println(s); + } + } + +} From 3943e174e5339f474931e624b46c1f5382b05413 Mon Sep 17 00:00:00 2001 From: JP Martin Date: Mon, 2 May 2016 10:16:57 -0700 Subject: [PATCH 2/4] Larger buffer improves performance --- .../java/com/google/cloud/examples/nio/CountBytes.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/CountBytes.java b/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/CountBytes.java index caa1a45b731f..81ec17fb17b4 100644 --- a/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/CountBytes.java +++ b/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/CountBytes.java @@ -41,22 +41,26 @@ public static void main(String[] args) throws IOException { * NIO Filesystem provider without any extra effort. */ private static void countFile(String fname) { - final int bufSize = 1024 * 1024; + // large buffers pay off + final int bufSize = 50 * 1024 * 1024; try { Path path = Paths.get(new URI(fname)); long size = Files.size(path); System.out.println(fname + ": " + size + " bytes."); - Stopwatch sw = Stopwatch.createStarted(); ByteBuffer buf = ByteBuffer.allocate(bufSize); + System.out.println("Reading the whole file..."); + Stopwatch sw = Stopwatch.createStarted(); SeekableByteChannel chan = Files.newByteChannel(path); long total = 0; + int readCalls = 0; for (int read = 1; read > 0; ) { + readCalls ++; read = chan.read(buf); buf.flip(); total += read; } long elapsed = sw.elapsed(TimeUnit.SECONDS); - System.out.println("Read all " + total + " bytes in " + elapsed + "s."); + System.out.println("Read all " + total + " bytes in " + elapsed + "s. (" + readCalls +" calls to chan.read)"); if (total != size) { System.out.println("Wait, this doesn't match! We saw " + total + " bytes, yet the file size is listed at " + size + " bytes."); } From a9b294cd31918dbb60ee48cebc11efdb3a804b6c Mon Sep 17 00:00:00 2001 From: JP Martin Date: Tue, 3 May 2016 11:05:57 -0700 Subject: [PATCH 3/4] Fix byte counting Added licence text, plus some esthetic changes. --- .../google/cloud/examples/nio/CountBytes.java | 35 ++++++++++++++----- .../com/google/cloud/examples/nio/Stat.java | 16 +++++++++ 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/CountBytes.java b/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/CountBytes.java index 81ec17fb17b4..3055d880c31b 100644 --- a/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/CountBytes.java +++ b/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/CountBytes.java @@ -1,3 +1,19 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.google.cloud.examples.nio; import com.google.common.base.Stopwatch; @@ -14,7 +30,7 @@ /** * CountBytes will read through the whole file given as input. * - *

It's meant for testing that NIO correctly saturates your network link. + *

It's meant for testing that NIO doesn't read too slowly. * *

See the README for compilation instructions. Run this code with * {@code target/appassembler/bin/CountBytes } @@ -53,16 +69,18 @@ private static void countFile(String fname) { SeekableByteChannel chan = Files.newByteChannel(path); long total = 0; int readCalls = 0; - for (int read = 1; read > 0; ) { - readCalls ++; - read = chan.read(buf); + while (chan.read(buf) > 0) { + readCalls++; + total += buf.position(); buf.flip(); - total += read; } + readCalls++; // We must count the last call long elapsed = sw.elapsed(TimeUnit.SECONDS); - System.out.println("Read all " + total + " bytes in " + elapsed + "s. (" + readCalls +" calls to chan.read)"); + System.out.println("Read all " + total + " bytes in " + elapsed + "s. " + + "(" + readCalls +" calls to chan.read)"); if (total != size) { - System.out.println("Wait, this doesn't match! We saw " + total + " bytes, yet the file size is listed at " + size + " bytes."); + System.out.println("Wait, this doesn't match! We saw " + total + " bytes, " + + "yet the file size is listed at " + size + " bytes."); } } catch (Exception ex) { System.out.println(fname + ": " + ex.toString()); @@ -72,11 +90,10 @@ private static void countFile(String fname) { private static void help() { String[] help = {"The argument is a ", - "and we show the length of that file.", + "and we show the length of that file." }; for (String s : help) { System.out.println(s); } } - } diff --git a/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/Stat.java b/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/Stat.java index ee9c4387a71a..b8d018f8f738 100644 --- a/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/Stat.java +++ b/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/Stat.java @@ -1,3 +1,19 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.google.cloud.examples.nio; import java.io.IOException; From 53d7b41df59cde4f6bcc343cc359f122e16335fa Mon Sep 17 00:00:00 2001 From: JP Martin Date: Wed, 4 May 2016 09:35:05 -0700 Subject: [PATCH 4/4] Comment improvement --- .../main/java/com/google/cloud/examples/nio/CountBytes.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/CountBytes.java b/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/CountBytes.java index 3055d880c31b..50ad77d72f24 100644 --- a/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/CountBytes.java +++ b/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/CountBytes.java @@ -30,7 +30,10 @@ /** * CountBytes will read through the whole file given as input. * - *

It's meant for testing that NIO doesn't read too slowly. + *

This example shows how to read a file size using NIO. + * File.size returns the size of the file as saved in Storage metadata. + * This class also shows how to read all of the file's contents using NIO, + * and reports how long it took. * *

See the README for compilation instructions. Run this code with * {@code target/appassembler/bin/CountBytes }