From 47632ab5bed28e8904b75b411c0a27402b682df8 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Tue, 15 Sep 2015 13:06:26 +0200 Subject: [PATCH 001/156] Avoid downloading gcd if gcloud is installed and gcd version matches v1beta2 --- .../gcloud/datastore/LocalGcdHelper.java | 121 +++++++++++++++--- 1 file changed, 105 insertions(+), 16 deletions(-) diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java index 2d7f5802f247..a826578104e7 100644 --- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java +++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java @@ -40,6 +40,7 @@ import java.nio.channels.ReadableByteChannel; import java.nio.file.FileVisitResult; import java.nio.file.Files; +import java.nio.file.InvalidPathException; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; @@ -47,6 +48,7 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.Locale; +import java.util.regex.Pattern; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; @@ -62,17 +64,85 @@ public class LocalGcdHelper { public static final String DEFAULT_PROJECT_ID = "projectid1"; public static final int PORT = 8080; private static final String GCD = "gcd-v1beta2-rev1-2.1.2b"; + private static final String GCD_VERSION = "v1beta2"; private static final String GCD_FILENAME = GCD + ".zip"; private static final String MD5_CHECKSUM = "d84384cdfa8658e1204f4f8be51300e8"; private static final URL GCD_URL; + private static final String GCLOUD = "gcloud"; + private static final Path INSTALLED_GCD_PATH; static { - try { - GCD_URL = new URL("http://storage.googleapis.com/gcd/tools/" + GCD_FILENAME); - } catch (MalformedURLException e) { - throw new RuntimeException(e); + INSTALLED_GCD_PATH = installedGcdPath(); + if (INSTALLED_GCD_PATH != null) { + GCD_URL = null; + } else { + try { + GCD_URL = new URL("http://storage.googleapis.com/gcd/tools/" + GCD_FILENAME); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } } } + + private static Path installedGcdPath() { + Path gcloudPath = executablePath(GCLOUD); + gcloudPath = (gcloudPath == null) ? null : gcloudPath.getParent(); + if (gcloudPath == null) { + return null; + } + Path installedGcdPath = gcloudPath.resolve("platform").resolve("gcd"); + if (Files.exists(installedGcdPath)) { + try { + String installedVersion = installedGcdVersion(); + if (installedVersion != null && installedVersion.startsWith(GCD_VERSION)) { + return installedGcdPath; + } + } catch (IOException | InterruptedException ignore) { + // ignore + } + } + return null; + } + + private static String installedGcdVersion() throws IOException, InterruptedException { + ProcessBuilder processBuilder = new ProcessBuilder() + .redirectErrorStream(true); + if (isWindows()) { + processBuilder.command("cmd", "/C", "gcloud", "version"); + } else { + processBuilder.command("bash", "gcloud", "version"); + } + Process process = processBuilder.start(); + process.waitFor(); + try (BufferedReader reader = + new BufferedReader(new InputStreamReader(process.getInputStream()))) { + for (String line = reader.readLine(); line != null; line = reader.readLine()) { + if (line.startsWith("gcd-emulator")) { + String[] lineComponents = line.split(" "); + if (lineComponents.length > 1) { + return lineComponents[1]; + } + } + } + return null; + } + + } + + private static Path executablePath(String cmd) { + String[] paths = System.getenv("PATH").split(Pattern.quote(File.pathSeparator)); + for (String pathString : paths) { + try { + Path path = Paths.get(pathString); + if (Files.exists(path.resolve(cmd))) { + return path; + } + } catch (InvalidPathException ignore) { + // ignore + } + } + return null; + } private static class ProcessStreamReader extends Thread { @@ -136,6 +206,18 @@ public void start() throws IOException, InterruptedException { File gcdFolder = gcdPath.toFile(); gcdFolder.deleteOnExit(); + Path gcdExecutablePath; + // If cloud is available we use it, otherwise we download and start gcd + if (INSTALLED_GCD_PATH == null) { + downloadGcd(); + gcdExecutablePath = gcdPath.resolve(GCD); + } else { + gcdExecutablePath = INSTALLED_GCD_PATH; + } + startGcd(gcdExecutablePath); + } + + private void downloadGcd() throws IOException { // check if we already have a local copy of the gcd utility and download it if not. File gcdZipFile = new File(System.getProperty("java.io.tmpdir"), GCD_FILENAME); if (!gcdZipFile.exists() || !MD5_CHECKSUM.equals(md5(gcdZipFile))) { @@ -148,7 +230,7 @@ public void start() throws IOException, InterruptedException { try (ZipInputStream zipIn = new ZipInputStream(new FileInputStream(gcdZipFile))) { ZipEntry entry = zipIn.getNextEntry(); while (entry != null) { - File filePath = new File(gcdFolder, entry.getName()); + File filePath = new File(gcdPath.toFile(), entry.getName()); if (!entry.isDirectory()) { extractFile(zipIn, filePath); } else { @@ -157,21 +239,26 @@ public void start() throws IOException, InterruptedException { zipIn.closeEntry(); entry = zipIn.getNextEntry(); } - } + } + } + + private void startGcd(Path executablePath) throws IOException, InterruptedException { // cleanup any possible data for the same project - File datasetFolder = new File(gcdFolder, GCD + '/' + projectId); + File datasetFolder = new File(gcdPath.toFile(), projectId); deleteRecurse(datasetFolder.toPath()); // create the datastore for the project ProcessBuilder processBuilder = new ProcessBuilder() .redirectError(ProcessBuilder.Redirect.INHERIT) - .directory(new File(gcdFolder, GCD)); + .directory(gcdPath.toFile()); if (isWindows()) { - processBuilder.command("cmd", "/C", "gcd.cmd", "create", "-p", projectId, projectId); - processBuilder.redirectOutput(new File("NULL:")); + Path gcdAbsolutePath = executablePath.toAbsolutePath().resolve("gcd.cmd"); + processBuilder.command("cmd", "/C", gcdAbsolutePath.toString(), "create", + "-p", projectId, projectId); } else { - processBuilder.redirectOutput(new File("/dev/null")); - processBuilder.command("bash", "gcd.sh", "create", "-p", projectId, projectId); + Path gcdAbsolutePath = executablePath.toAbsolutePath().resolve("gcd.sh"); + processBuilder.command("bash", gcdAbsolutePath.toString(), "create", + "-p", projectId, projectId); } Process temp = processBuilder.start(); @@ -179,14 +266,16 @@ public void start() throws IOException, InterruptedException { // start the datastore for the project processBuilder = new ProcessBuilder() - .directory(new File(gcdFolder, GCD)) + .directory(gcdPath.toFile()) .redirectErrorStream(true); if (isWindows()) { - processBuilder.command("cmd", "/C", "gcd.cmd", "start", "--testing", + Path gcdAbsolutePath = executablePath.toAbsolutePath().resolve("gcd.cmd"); + processBuilder.command("cmd", "/C", gcdAbsolutePath.toString(), "start", "--testing", "--allow_remote_shutdown", projectId); } else { - processBuilder.command("bash", "gcd.sh", "start", "--testing", "--allow_remote_shutdown", - projectId); + Path gcdAbsolutePath = executablePath.toAbsolutePath().resolve("gcd.sh"); + processBuilder.command("bash", gcdAbsolutePath.toString(), "start", "--testing", + "--allow_remote_shutdown", projectId); } temp = processBuilder.start(); processReader = ProcessStreamReader.start(temp, "Dev App Server is now running"); From 1c52730edfb6e169ea9ce8af32713f4879f07590 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Thu, 17 Sep 2015 14:10:42 +0200 Subject: [PATCH 002/156] Refactor process invocation for gcd in tests. Restored Windows support. --- .../gcloud/datastore/LocalGcdHelper.java | 160 ++++++++++++++---- 1 file changed, 123 insertions(+), 37 deletions(-) diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java index a826578104e7..2392482f9fd8 100644 --- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java +++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java @@ -47,6 +47,9 @@ import java.nio.file.attribute.BasicFileAttributes; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.Locale; import java.util.regex.Pattern; import java.util.zip.ZipEntry; @@ -56,20 +59,22 @@ * Utility to start and stop local Google Cloud Datastore process. */ public class LocalGcdHelper { - + private final String projectId; private Path gcdPath; private ProcessStreamReader processReader; public static final String DEFAULT_PROJECT_ID = "projectid1"; public static final int PORT = 8080; - private static final String GCD = "gcd-v1beta2-rev1-2.1.2b"; private static final String GCD_VERSION = "v1beta2"; - private static final String GCD_FILENAME = GCD + ".zip"; + private static final String GCD_BUILD = "rev1-2.1.2b"; + private static final String GCD_BASENAME = "gcd-" + GCD_VERSION + "-" + GCD_BUILD; + private static final String GCD_FILENAME = GCD_BASENAME + ".zip"; private static final String MD5_CHECKSUM = "d84384cdfa8658e1204f4f8be51300e8"; private static final URL GCD_URL; private static final String GCLOUD = "gcloud"; private static final Path INSTALLED_GCD_PATH; + private static final String GCD_VERSION_PREFIX = "gcd-emulator "; static { INSTALLED_GCD_PATH = installedGcdPath(); @@ -85,7 +90,13 @@ public class LocalGcdHelper { } private static Path installedGcdPath() { - Path gcloudPath = executablePath(GCLOUD); + String gcloudExecutableName; + if (isWindows()) { + gcloudExecutableName = GCLOUD + ".cmd"; + } else { + gcloudExecutableName = GCLOUD; + } + Path gcloudPath = executablePath(gcloudExecutableName); gcloudPath = (gcloudPath == null) ? null : gcloudPath.getParent(); if (gcloudPath == null) { return null; @@ -105,19 +116,18 @@ private static Path installedGcdPath() { } private static String installedGcdVersion() throws IOException, InterruptedException { - ProcessBuilder processBuilder = new ProcessBuilder() - .redirectErrorStream(true); + CommandWrapper gcloudCommand; if (isWindows()) { - processBuilder.command("cmd", "/C", "gcloud", "version"); + gcloudCommand = CommandWrapper.cmd(); } else { - processBuilder.command("bash", "gcloud", "version"); + gcloudCommand = CommandWrapper.bash(); } - Process process = processBuilder.start(); + Process process = gcloudCommand.command("gcloud", "version").redirectErrorStream().start(); process.waitFor(); try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { for (String line = reader.readLine(); line != null; line = reader.readLine()) { - if (line.startsWith("gcd-emulator")) { + if (line.startsWith(GCD_VERSION_PREFIX)) { String[] lineComponents = line.split(" "); if (lineComponents.length > 1) { return lineComponents[1]; @@ -126,7 +136,6 @@ private static String installedGcdVersion() throws IOException, InterruptedExcep } return null; } - } private static Path executablePath(String cmd) { @@ -185,6 +194,84 @@ public static ProcessStreamReader start(Process process, String blockUntil) thro return thread; } } + + private static class CommandWrapper { + + private final List prefix; + private List command; + private String nullFilename; + private boolean redirectOutputToNull; + private boolean redirectErrorStream; + private boolean redirectErrorInherit; + private Path directory; + + private CommandWrapper() { + this.prefix = new ArrayList<>(); + } + + public CommandWrapper command(String... command) { + this.command = new ArrayList<>(command.length + this.prefix.size()); + this.command.addAll(prefix); + this.command.addAll(Arrays.asList(command)); + return this; + } + + public CommandWrapper redirectOutputToNull() { + this.redirectOutputToNull = true; + return this; + } + + public CommandWrapper redirectErrorStream() { + this.redirectErrorStream = true; + return this; + } + + public CommandWrapper redirectErrorInherit() { + this.redirectErrorInherit = true; + return this; + } + + public CommandWrapper directory(Path directory) { + this.directory = directory; + return this; + } + + public ProcessBuilder builder() { + ProcessBuilder builder = new ProcessBuilder(command); + if (redirectOutputToNull) { + builder.redirectOutput(new File(nullFilename)); + } + if (directory != null) { + builder.directory(directory.toFile()); + } + if (redirectErrorStream) { + builder.redirectErrorStream(true); + } + if (redirectErrorInherit) { + builder.redirectError(ProcessBuilder.Redirect.INHERIT); + } + return builder; + } + + public Process start() throws IOException { + return builder().start(); + } + + public static CommandWrapper cmd() { + CommandWrapper wrapper = new CommandWrapper(); + wrapper.prefix.add("cmd"); + wrapper.prefix.add("/C"); + wrapper.nullFilename = "NUL:"; + return wrapper; + } + + public static CommandWrapper bash() { + CommandWrapper wrapper = new CommandWrapper(); + wrapper.prefix.add("bash"); + wrapper.nullFilename = "/dev/null"; + return wrapper; + } + } public LocalGcdHelper(String projectId) { this.projectId = projectId; @@ -210,7 +297,7 @@ public void start() throws IOException, InterruptedException { // If cloud is available we use it, otherwise we download and start gcd if (INSTALLED_GCD_PATH == null) { downloadGcd(); - gcdExecutablePath = gcdPath.resolve(GCD); + gcdExecutablePath = gcdPath.resolve(GCD_BASENAME); } else { gcdExecutablePath = INSTALLED_GCD_PATH; } @@ -226,6 +313,7 @@ private void downloadGcd() throws IOException { fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); } } + // unzip the gcd try (ZipInputStream zipIn = new ZipInputStream(new FileInputStream(gcdZipFile))) { ZipEntry entry = zipIn.getNextEntry(); @@ -246,38 +334,36 @@ private void startGcd(Path executablePath) throws IOException, InterruptedExcept // cleanup any possible data for the same project File datasetFolder = new File(gcdPath.toFile(), projectId); deleteRecurse(datasetFolder.toPath()); - - // create the datastore for the project - ProcessBuilder processBuilder = new ProcessBuilder() - .redirectError(ProcessBuilder.Redirect.INHERIT) - .directory(gcdPath.toFile()); + + // create command wrappers + CommandWrapper gcdCreateCommand; + CommandWrapper gcdStartCommand; + Path gcdAbsolutePath; if (isWindows()) { - Path gcdAbsolutePath = executablePath.toAbsolutePath().resolve("gcd.cmd"); - processBuilder.command("cmd", "/C", gcdAbsolutePath.toString(), "create", - "-p", projectId, projectId); + gcdCreateCommand = CommandWrapper.cmd(); + gcdStartCommand = CommandWrapper.cmd(); + gcdAbsolutePath = executablePath.toAbsolutePath().resolve("gcd.cmd"); } else { - Path gcdAbsolutePath = executablePath.toAbsolutePath().resolve("gcd.sh"); - processBuilder.command("bash", gcdAbsolutePath.toString(), "create", - "-p", projectId, projectId); + gcdCreateCommand = CommandWrapper.bash(); + gcdStartCommand = CommandWrapper.bash(); + gcdAbsolutePath = executablePath.toAbsolutePath().resolve("gcd.sh"); } - Process temp = processBuilder.start(); + // create the datastore for the project + Process temp = gcdCreateCommand.command(gcdAbsolutePath.toString(), "create", "-p", projectId, + projectId) + .redirectErrorInherit() + .directory(gcdPath) + .redirectOutputToNull() + .start(); temp.waitFor(); // start the datastore for the project - processBuilder = new ProcessBuilder() - .directory(gcdPath.toFile()) - .redirectErrorStream(true); - if (isWindows()) { - Path gcdAbsolutePath = executablePath.toAbsolutePath().resolve("gcd.cmd"); - processBuilder.command("cmd", "/C", gcdAbsolutePath.toString(), "start", "--testing", - "--allow_remote_shutdown", projectId); - } else { - Path gcdAbsolutePath = executablePath.toAbsolutePath().resolve("gcd.sh"); - processBuilder.command("bash", gcdAbsolutePath.toString(), "start", "--testing", - "--allow_remote_shutdown", projectId); - } - temp = processBuilder.start(); + temp = gcdStartCommand.command(gcdAbsolutePath.toString(), "start", "--testing", + "--allow_remote_shutdown", projectId) + .directory(gcdPath) + .redirectErrorStream() + .start(); processReader = ProcessStreamReader.start(temp, "Dev App Server is now running"); } From b0a88f5d512908159d5c7316f363d4a1afb97658 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Thu, 17 Sep 2015 14:32:34 +0200 Subject: [PATCH 003/156] Add fine-grained logging to LocalGcdHelper --- .../gcloud/datastore/LocalGcdHelper.java | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java index 2392482f9fd8..d0648da46be2 100644 --- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java +++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java @@ -55,11 +55,16 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; +import java.util.logging.Level; +import java.util.logging.Logger; + /** * Utility to start and stop local Google Cloud Datastore process. */ public class LocalGcdHelper { + private static final Logger log = Logger.getLogger(LocalGcdHelper.class.getName()); + private final String projectId; private Path gcdPath; private ProcessStreamReader processReader; @@ -99,14 +104,27 @@ private static Path installedGcdPath() { Path gcloudPath = executablePath(gcloudExecutableName); gcloudPath = (gcloudPath == null) ? null : gcloudPath.getParent(); if (gcloudPath == null) { + if (log.isLoggable(Level.FINE)) { + log.fine("SDK not found"); + } return null; } + if (log.isLoggable(Level.FINE)) { + log.fine("SDK found, looking for datastore emulator"); + } Path installedGcdPath = gcloudPath.resolve("platform").resolve("gcd"); if (Files.exists(installedGcdPath)) { try { String installedVersion = installedGcdVersion(); if (installedVersion != null && installedVersion.startsWith(GCD_VERSION)) { + if (log.isLoggable(Level.FINE)) { + log.fine("SDK datastore emulator found"); + } return installedGcdPath; + } else { + if (log.isLoggable(Level.FINE)) { + log.fine("SDK datastore emulator found but version mismatch"); + } } } catch (IOException | InterruptedException ignore) { // ignore @@ -308,14 +326,23 @@ private void downloadGcd() throws IOException { // check if we already have a local copy of the gcd utility and download it if not. File gcdZipFile = new File(System.getProperty("java.io.tmpdir"), GCD_FILENAME); if (!gcdZipFile.exists() || !MD5_CHECKSUM.equals(md5(gcdZipFile))) { + if (log.isLoggable(Level.FINE)) { + log.fine("Fetching datastore emulator"); + } ReadableByteChannel rbc = Channels.newChannel(GCD_URL.openStream()); try (FileOutputStream fos = new FileOutputStream(gcdZipFile)) { fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); } + } else { + if (log.isLoggable(Level.FINE)) { + log.fine("Using cached datastore emulator"); + } } - // unzip the gcd try (ZipInputStream zipIn = new ZipInputStream(new FileInputStream(gcdZipFile))) { + if (log.isLoggable(Level.FINE)) { + log.fine("Unzipping datastore emulator"); + } ZipEntry entry = zipIn.getNextEntry(); while (entry != null) { File filePath = new File(gcdPath.toFile(), entry.getName()); @@ -350,6 +377,9 @@ private void startGcd(Path executablePath) throws IOException, InterruptedExcept } // create the datastore for the project + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Creating datastore for the project: {0}", projectId); + } Process temp = gcdCreateCommand.command(gcdAbsolutePath.toString(), "create", "-p", projectId, projectId) .redirectErrorInherit() @@ -359,6 +389,9 @@ private void startGcd(Path executablePath) throws IOException, InterruptedExcept temp.waitFor(); // start the datastore for the project + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Starting datastore emulator for the project: {0}", projectId); + } temp = gcdStartCommand.command(gcdAbsolutePath.toString(), "start", "--testing", "--allow_remote_shutdown", projectId) .directory(gcdPath) From e792c2c67795572501a2bd2cc04cb7a183a41a3f Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Fri, 18 Sep 2015 09:47:57 +0200 Subject: [PATCH 004/156] Formatted LocalGcdHelper --- .../gcloud/datastore/LocalGcdHelper.java | 66 ++++++++++--------- 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java index d0648da46be2..20b44f3280d1 100644 --- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java +++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java @@ -62,7 +62,7 @@ * Utility to start and stop local Google Cloud Datastore process. */ public class LocalGcdHelper { - + private static final Logger log = Logger.getLogger(LocalGcdHelper.class.getName()); private final String projectId; @@ -93,7 +93,7 @@ public class LocalGcdHelper { } } } - + private static Path installedGcdPath() { String gcloudExecutableName; if (isWindows()) { @@ -132,7 +132,7 @@ private static Path installedGcdPath() { } return null; } - + private static String installedGcdVersion() throws IOException, InterruptedException { CommandWrapper gcloudCommand; if (isWindows()) { @@ -142,8 +142,8 @@ private static String installedGcdVersion() throws IOException, InterruptedExcep } Process process = gcloudCommand.command("gcloud", "version").redirectErrorStream().start(); process.waitFor(); - try (BufferedReader reader = - new BufferedReader(new InputStreamReader(process.getInputStream()))) { + try (BufferedReader reader = + new BufferedReader(new InputStreamReader(process.getInputStream()))) { for (String line = reader.readLine(); line != null; line = reader.readLine()) { if (line.startsWith(GCD_VERSION_PREFIX)) { String[] lineComponents = line.split(" "); @@ -155,7 +155,7 @@ private static String installedGcdVersion() throws IOException, InterruptedExcep return null; } } - + private static Path executablePath(String cmd) { String[] paths = System.getenv("PATH").split(Pattern.quote(File.pathSeparator)); for (String pathString : paths) { @@ -180,7 +180,7 @@ private static class ProcessStreamReader extends Thread { super("Local GCD InputStream reader"); setDaemon(true); this.process = process; - reader = new BufferedReader(new InputStreamReader(process.getInputStream())); + reader = new BufferedReader(new InputStreamReader(process.getInputStream())); if (!Strings.isNullOrEmpty(blockUntil)) { String line; do { @@ -212,9 +212,9 @@ public static ProcessStreamReader start(Process process, String blockUntil) thro return thread; } } - + private static class CommandWrapper { - + private final List prefix; private List command; private String nullFilename; @@ -222,38 +222,38 @@ private static class CommandWrapper { private boolean redirectErrorStream; private boolean redirectErrorInherit; private Path directory; - + private CommandWrapper() { this.prefix = new ArrayList<>(); } - + public CommandWrapper command(String... command) { this.command = new ArrayList<>(command.length + this.prefix.size()); this.command.addAll(prefix); this.command.addAll(Arrays.asList(command)); return this; } - + public CommandWrapper redirectOutputToNull() { this.redirectOutputToNull = true; return this; } - + public CommandWrapper redirectErrorStream() { this.redirectErrorStream = true; return this; } - + public CommandWrapper redirectErrorInherit() { this.redirectErrorInherit = true; return this; } - + public CommandWrapper directory(Path directory) { this.directory = directory; return this; } - + public ProcessBuilder builder() { ProcessBuilder builder = new ProcessBuilder(command); if (redirectOutputToNull) { @@ -270,11 +270,11 @@ public ProcessBuilder builder() { } return builder; } - + public Process start() throws IOException { return builder().start(); } - + public static CommandWrapper cmd() { CommandWrapper wrapper = new CommandWrapper(); wrapper.prefix.add("cmd"); @@ -282,7 +282,7 @@ public static CommandWrapper cmd() { wrapper.nullFilename = "NUL:"; return wrapper; } - + public static CommandWrapper bash() { CommandWrapper wrapper = new CommandWrapper(); wrapper.prefix.add("bash"); @@ -321,7 +321,7 @@ public void start() throws IOException, InterruptedException { } startGcd(gcdExecutablePath); } - + private void downloadGcd() throws IOException { // check if we already have a local copy of the gcd utility and download it if not. File gcdZipFile = new File(System.getProperty("java.io.tmpdir"), GCD_FILENAME); @@ -336,7 +336,7 @@ private void downloadGcd() throws IOException { } else { if (log.isLoggable(Level.FINE)) { log.fine("Using cached datastore emulator"); - } + } } // unzip the gcd try (ZipInputStream zipIn = new ZipInputStream(new FileInputStream(gcdZipFile))) { @@ -354,14 +354,14 @@ private void downloadGcd() throws IOException { zipIn.closeEntry(); entry = zipIn.getNextEntry(); } - } + } } - + private void startGcd(Path executablePath) throws IOException, InterruptedException { // cleanup any possible data for the same project File datasetFolder = new File(gcdPath.toFile(), projectId); deleteRecurse(datasetFolder.toPath()); - + // create command wrappers CommandWrapper gcdCreateCommand; CommandWrapper gcdStartCommand; @@ -380,24 +380,26 @@ private void startGcd(Path executablePath) throws IOException, InterruptedExcept if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Creating datastore for the project: {0}", projectId); } - Process temp = gcdCreateCommand.command(gcdAbsolutePath.toString(), "create", "-p", projectId, - projectId) + Process createProcess = + gcdCreateCommand.command(gcdAbsolutePath.toString(), "create", "-p", projectId, projectId) .redirectErrorInherit() .directory(gcdPath) .redirectOutputToNull() .start(); - temp.waitFor(); + createProcess.waitFor(); // start the datastore for the project if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Starting datastore emulator for the project: {0}", projectId); } - temp = gcdStartCommand.command(gcdAbsolutePath.toString(), "start", "--testing", - "--allow_remote_shutdown", projectId) + Process startProcess = + gcdStartCommand + .command(gcdAbsolutePath.toString(), "start", "--testing", "--allow_remote_shutdown", + projectId) .directory(gcdPath) .redirectErrorStream() .start(); - processReader = ProcessStreamReader.start(temp, "Dev App Server is now running"); + processReader = ProcessStreamReader.start(startProcess, "Dev App Server is now running"); } private static String md5(File gcdZipFile) throws IOException { @@ -410,7 +412,7 @@ private static String md5(File gcdZipFile) throws IOException { md5.update(bytes, 0, len); } } - return String.format("%032x",new BigInteger(1, md5.digest())); + return String.format("%032x", new BigInteger(1, md5.digest())); } catch (NoSuchAlgorithmException e) { throw new IOException(e); } @@ -520,7 +522,7 @@ public static boolean isActive(String projectId) { urlBuilder.append("/datastore/v1beta2/datasets/").append(projectId).append("/lookup"); URL url = new URL(urlBuilder.toString()); try (BufferedReader reader = - new BufferedReader(new InputStreamReader(url.openStream(), UTF_8))) { + new BufferedReader(new InputStreamReader(url.openStream(), UTF_8))) { return "Valid RPC".equals(reader.readLine()); } } catch (IOException ignore) { From f753b44d9a2f2b787b4e9fc1f51b67f47a19ba68 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Fri, 18 Sep 2015 10:11:17 +0200 Subject: [PATCH 005/156] Single factory method for CommandWrapper in LocalGcdHelper --- .../gcloud/datastore/LocalGcdHelper.java | 45 +++++++------------ 1 file changed, 16 insertions(+), 29 deletions(-) diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java index 20b44f3280d1..888fd2157d3a 100644 --- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java +++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java @@ -134,13 +134,8 @@ private static Path installedGcdPath() { } private static String installedGcdVersion() throws IOException, InterruptedException { - CommandWrapper gcloudCommand; - if (isWindows()) { - gcloudCommand = CommandWrapper.cmd(); - } else { - gcloudCommand = CommandWrapper.bash(); - } - Process process = gcloudCommand.command("gcloud", "version").redirectErrorStream().start(); + Process process = + CommandWrapper.create().command("gcloud", "version").redirectErrorStream().start(); process.waitFor(); try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { @@ -225,6 +220,14 @@ private static class CommandWrapper { private CommandWrapper() { this.prefix = new ArrayList<>(); + if (isWindows()) { + this.prefix.add("cmd"); + this.prefix.add("/C"); + this.nullFilename = "NUL:"; + } else { + this.prefix.add("bash"); + this.nullFilename = "/dev/null"; + } } public CommandWrapper command(String... command) { @@ -275,19 +278,8 @@ public Process start() throws IOException { return builder().start(); } - public static CommandWrapper cmd() { - CommandWrapper wrapper = new CommandWrapper(); - wrapper.prefix.add("cmd"); - wrapper.prefix.add("/C"); - wrapper.nullFilename = "NUL:"; - return wrapper; - } - - public static CommandWrapper bash() { - CommandWrapper wrapper = new CommandWrapper(); - wrapper.prefix.add("bash"); - wrapper.nullFilename = "/dev/null"; - return wrapper; + public static CommandWrapper create() { + return new CommandWrapper(); } } @@ -362,17 +354,11 @@ private void startGcd(Path executablePath) throws IOException, InterruptedExcept File datasetFolder = new File(gcdPath.toFile(), projectId); deleteRecurse(datasetFolder.toPath()); - // create command wrappers - CommandWrapper gcdCreateCommand; - CommandWrapper gcdStartCommand; + // Get path to cmd executable Path gcdAbsolutePath; if (isWindows()) { - gcdCreateCommand = CommandWrapper.cmd(); - gcdStartCommand = CommandWrapper.cmd(); gcdAbsolutePath = executablePath.toAbsolutePath().resolve("gcd.cmd"); } else { - gcdCreateCommand = CommandWrapper.bash(); - gcdStartCommand = CommandWrapper.bash(); gcdAbsolutePath = executablePath.toAbsolutePath().resolve("gcd.sh"); } @@ -381,7 +367,8 @@ private void startGcd(Path executablePath) throws IOException, InterruptedExcept log.log(Level.FINE, "Creating datastore for the project: {0}", projectId); } Process createProcess = - gcdCreateCommand.command(gcdAbsolutePath.toString(), "create", "-p", projectId, projectId) + CommandWrapper.create() + .command(gcdAbsolutePath.toString(), "create", "-p", projectId, projectId) .redirectErrorInherit() .directory(gcdPath) .redirectOutputToNull() @@ -393,7 +380,7 @@ private void startGcd(Path executablePath) throws IOException, InterruptedExcept log.log(Level.FINE, "Starting datastore emulator for the project: {0}", projectId); } Process startProcess = - gcdStartCommand + CommandWrapper.create() .command(gcdAbsolutePath.toString(), "start", "--testing", "--allow_remote_shutdown", projectId) .directory(gcdPath) From 3a183e4a9ebe1754f9c6c28e30e4af6846da7ac0 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Fri, 18 Sep 2015 15:22:28 +0200 Subject: [PATCH 006/156] BucketInfo: add null check for rules in builder. Add test case. --- .../src/main/java/com/google/gcloud/storage/BucketInfo.java | 2 +- .../test/java/com/google/gcloud/storage/BucketInfoTest.java | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java index 437ce1a142d5..28110e0a0292 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java @@ -453,7 +453,7 @@ public Builder notFoundPage(String notFoundPage) { } public Builder deleteRules(Iterable rules) { - this.deleteRules = ImmutableList.copyOf(rules); + this.deleteRules = rules != null ? ImmutableList.copyOf(rules) : null; return this; } diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketInfoTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketInfoTest.java index 27a5b4014f3f..c4f08fbd8b9b 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketInfoTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketInfoTest.java @@ -90,6 +90,12 @@ public void testToBuilder() { compareBuckets(BUCKET_INFO, bucketInfo); } + @Test + public void testToBuilderIncomplete() { + BucketInfo incompleteBucketInfo = BucketInfo.builder("b").build(); + assertEquals(incompleteBucketInfo.name(), incompleteBucketInfo.toBuilder().build().name()); + } + @Test public void testOf() { BucketInfo bucketInfo = BucketInfo.of("bucket"); From af171147208aeade287e2cf81794f922bf147b1a Mon Sep 17 00:00:00 2001 From: aozarov Date: Thu, 9 Jul 2015 16:09:42 -0700 Subject: [PATCH 007/156] initial work on functional blob/bucket --- .../java/com/google/gcloud/storage/Blob.java | 195 ++++++++++++++++++ .../com/google/gcloud/storage/BlobInfo.java | 58 +++--- .../com/google/gcloud/storage/Bucket.java | 89 ++++++++ .../com/google/gcloud/storage/BucketInfo.java | 157 +++++++------- .../com/google/gcloud/storage/Storage.java | 43 ++-- .../com/google/gcloud/storage/BlobTest.java | 131 ++++++++++++ 6 files changed, 539 insertions(+), 134 deletions(-) create mode 100644 gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java create mode 100644 gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java create mode 100644 gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java new file mode 100644 index 000000000000..40ffd8235ba0 --- /dev/null +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java @@ -0,0 +1,195 @@ +/* + * + * * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version + * 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless + * required by applicable law or agreed to in writing, software * distributed under the License is + * distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. * See the License for the specific language governing permissions and * limitations + * under the License. + */ + +package com.google.gcloud.storage; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.gcloud.storage.Blob.BlobSourceOption.convert; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.gcloud.spi.StorageRpc; +import com.google.gcloud.storage.Storage.BlobTargetOption; +import com.google.gcloud.storage.Storage.CopyRequest; +import com.google.gcloud.storage.Storage.SignUrlOption; + +import java.net.URL; +import java.util.Objects; + + +/** + * A Google cloud storage object. + */ +public final class Blob { + + private final Storage storage; + private BlobInfo info; + + public static class BlobSourceOption extends Option { + + private static final long serialVersionUID = 214616862061934846L; + + private BlobSourceOption(StorageRpc.Option rpcOption) { + super(rpcOption, null); + } + + private Storage.BlobSourceOption convert(BlobInfo blobInfo) { + switch (rpcOption()) { + case IF_GENERATION_MATCH: + return Storage.BlobSourceOption.generationMatch(blobInfo.generation()); + case IF_GENERATION_NOT_MATCH: + return Storage.BlobSourceOption.generationNotMatch(blobInfo.generation()); + case IF_METAGENERATION_MATCH: + return Storage.BlobSourceOption.metagenerationMatch(blobInfo.metageneration()); + case IF_METAGENERATION_NOT_MATCH: + return Storage.BlobSourceOption.metagenerationNotMatch(blobInfo.metageneration()); + default: + throw new AssertionError("Unexpected enum value"); + } + } + + public static BlobSourceOption generationMatch() { + return new BlobSourceOption(StorageRpc.Option.IF_GENERATION_MATCH); + } + + public static BlobSourceOption generationNotMatch() { + return new BlobSourceOption(StorageRpc.Option.IF_GENERATION_NOT_MATCH); + } + + public static BlobSourceOption metagenerationMatch() { + return new BlobSourceOption(StorageRpc.Option.IF_METAGENERATION_MATCH); + } + + public static BlobSourceOption metagenerationNotMatch() { + return new BlobSourceOption(StorageRpc.Option.IF_METAGENERATION_NOT_MATCH); + } + + static Storage.BlobSourceOption[] convert(BlobInfo blobInfo, BlobSourceOption... options) { + Storage.BlobSourceOption[] convertedOptions = new Storage.BlobSourceOption[options.length]; + int index = 0; + for (BlobSourceOption option : options) { + convertedOptions[index++] = option.convert(blobInfo); + } + return convertedOptions; + } + } + + public Blob(Storage storage, BlobInfo info) { + this.storage = checkNotNull(storage); + this.info = checkNotNull(info); + } + + public BlobInfo info() { + return info; + } + + /** + * Returns true if this blob exists. + * + * @throws StorageException upon failure + */ + public boolean exists() { + return storage.get(info.bucket(), info.name()) != null; + } + + /** + * Returns the blob's content. + * + * @throws StorageException upon failure + */ + public byte[] content(Storage.BlobSourceOption... options) { + return storage.readAllBytes(info.bucket(), info.name(), options); + } + + /** + * Updates the blob's information. Bucket or blob's name cannot be changed by this method. If you + * want to rename the blob or move it to a different bucket use the {@link #copyTo} and + * {@link #delete} operations. + * + * @throws StorageException upon failure + */ + public void update(BlobInfo blobInfo, BlobTargetOption... options) { + checkArgument(Objects.equals(blobInfo.bucket(), info.bucket()), "Bucket name must match"); + checkArgument(Objects.equals(blobInfo.name(), info.name()), "Blob name must match"); + info = storage.update(blobInfo, options); + } + + /** + * Deletes this blob. + * + * @return true if bucket was deleted + * @throws StorageException upon failure + */ + public boolean delete(BlobSourceOption... options) { + return storage.delete(info.bucket(), info.name(), convert(info, options)); + } + + /** + * Send a copy request. + * + * @return the copied blob. + * @throws StorageException upon failure + */ + public Blob copyTo(BlobInfo target, BlobSourceOption... options) { + return copyTo(target, ImmutableList.copyOf(options), ImmutableList.of()); + } + + /** + * Send a copy request. + * + * @return the copied blob. + * @throws StorageException upon failure + */ + public Blob copyTo(BlobInfo target, Iterable sourceOptions, + Iterable targetOptions) { + CopyRequest copyRequest = + CopyRequest.builder().source(info.bucket(), info.name()) + .sourceOptions(convert(info, Iterables.toArray(sourceOptions, BlobSourceOption.class))) + .target(target).targetOptions(targetOptions).build(); + return new Blob(storage, storage.copy(copyRequest)); + } + + /** + * Returns a channel for reading this blob's content. + * + * @throws StorageException upon failure + */ + public BlobReadChannel reader(BlobSourceOption... options) { + return storage.reader(info.bucket(), info.name(), convert(info, options)); + } + + /** + * Returns a channel for writing to this blob. + * + * @throws StorageException upon failure + */ + public BlobWriteChannel writer(BlobTargetOption... options) { + return storage.writer(info, options); + } + + /** + * Generates a signed URL for this blob. If you want to allow access to for a fixed amount of time + * for this blob, you can use this method to generate a URL that is only valid within a certain + * time period. This is particularly useful if you don't want publicly accessible blobs, but don't + * want to require users to explicitly log in. + * + * @param expirationTimeInSeconds the signed URL expiration (using epoch time) + * @see Signed-URLs + */ + public URL signUrl(long expirationTimeInSeconds, SignUrlOption... options) { + return storage.signUrl(info, expirationTimeInSeconds, options); + } + + public Storage storage() { + return storage; + } +} diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java index 3ef01af8d1f3..68255f991187 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java @@ -1,17 +1,12 @@ /* - * Copyright 2015 Google Inc. All Rights Reserved. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version + * 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless + * required by applicable law or agreed to in writing, software * distributed under the License is + * distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. * See the License for the specific language governing permissions and * limitations + * under the License. */ package com.google.gcloud.storage; @@ -37,14 +32,13 @@ import java.util.Objects; /** - * A Google Storage object. + * Google Storage object metadata. * - * @see Concepts and Terminology + * @see Concepts and + * Terminology */ public final class BlobInfo implements Serializable { - private static final long serialVersionUID = 2228487739943277159L; - static final Function FROM_PB_FUNCTION = new Function() { @Override @@ -52,7 +46,6 @@ public BlobInfo apply(StorageObject pb) { return BlobInfo.fromPb(pb); } }; - static final Function TO_PB_FUNCTION = new Function() { @Override @@ -60,7 +53,7 @@ public StorageObject apply(BlobInfo blobInfo) { return blobInfo.toPb(); } }; - + private static final long serialVersionUID = 2228487739943277159L; private final String bucket; private final String id; private final String name; @@ -378,18 +371,6 @@ public String toString() { .toString(); } - public static BlobInfo of(String bucket, String name) { - return builder(bucket, name).build(); - } - - public static Builder builder(BucketInfo bucketInfo, String name) { - return builder(bucketInfo.name(), name); - } - - public static Builder builder(String bucket, String name) { - return new Builder().bucket(bucket).name(name); - } - @Override public int hashCode() { return Objects.hash(bucket, name); @@ -397,10 +378,7 @@ public int hashCode() { @Override public boolean equals(Object obj) { - if (!(obj instanceof BlobInfo)) { - return false; - } - return Objects.equals(toPb(), ((BlobInfo) obj).toPb()); + return obj instanceof BlobInfo && Objects.equals(toPb(), ((BlobInfo) obj).toPb()); } StorageObject toPb() { @@ -445,6 +423,18 @@ public ObjectAccessControl apply(Acl acl) { return storageObject; } + public static BlobInfo of(String bucket, String name) { + return builder(bucket, name).build(); + } + + public static Builder builder(BucketInfo bucketInfo, String name) { + return builder(bucketInfo.name(), name); + } + + public static Builder builder(String bucket, String name) { + return new Builder().bucket(bucket).name(name); + } + static BlobInfo fromPb(StorageObject storageObject) { Builder builder = new Builder() .bucket(storageObject.getBucket()) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java new file mode 100644 index 000000000000..5c4d037f21c1 --- /dev/null +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java @@ -0,0 +1,89 @@ +/* + * + * * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version + * 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless + * required by applicable law or agreed to in writing, software * distributed under the License is + * distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. * See the License for the specific language governing permissions and * limitations + * under the License. + */ + +package com.google.gcloud.storage; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.gcloud.storage.Storage.BlobSourceOption; +import com.google.gcloud.storage.Storage.BucketSourceOption; +import com.google.gcloud.storage.Storage.BucketTargetOption; + +import java.util.List; +import java.util.Objects; + +/** + * A Google cloud storage bucket. + */ +public final class Bucket { + + private final Storage storage; + private BucketInfo info; + + public Bucket(Storage storage, BucketInfo info) { + this.storage = checkNotNull(storage); + this.info = checkNotNull(info); + } + + /** + * Returns true if this bucket exists. + * + * @throws StorageException upon failure + */ + public boolean exists() { + return storage.get(info.name()) != null; + } + + /** + * Update the bucket's information. Bucket's name cannot be changed. + * + * @throws StorageException upon failure + */ + public void update(BucketInfo bucketInfo, BucketTargetOption... options) { + checkArgument(Objects.equals(bucketInfo.name(), info.name()), "Bucket name must match"); + info = storage.update(bucketInfo, options); + } + + /** + * Delete this bucket. + * + * @return true if bucket was deleted + * @throws StorageException upon failure + */ + public boolean delete(BucketSourceOption... options) { + return storage.delete(info.name(), options); + } + + public ListResult list(Storage.BlobListOption... options) { + return storage.list(info.name(), options); + } + + public BlobInfo get(String blob, BlobSourceOption... options) { + return null; + } + + public List get(String... blob) { + // todo + return null; + } + + /* + * BlobInfo create(BlobInfo blobInfo, byte[] content, BlobTargetOption... options) { + * + * } + */ + + + public Storage storage() { + return storage; + } +} diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java index 28110e0a0292..b9c6b03f9dd3 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java @@ -1,17 +1,12 @@ /* - * Copyright 2015 Google Inc. All Rights Reserved. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version + * 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless + * required by applicable law or agreed to in writing, software * distributed under the License is + * distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. * See the License for the specific language governing permissions and * limitations + * under the License. */ package com.google.gcloud.storage; @@ -44,15 +39,28 @@ import java.util.Objects; /** - * A Google Storage bucket. + * Google Storage bucket metadata; * * @see Concepts and - * Terminology + * Terminology */ public final class BucketInfo implements Serializable { + static final Function FROM_PB_FUNCTION = + new Function() { + @Override + public BucketInfo apply(com.google.api.services.storage.model.Bucket pb) { + return BucketInfo.fromPb(pb); + } + }; + static final Function TO_PB_FUNCTION = + new Function() { + @Override + public com.google.api.services.storage.model.Bucket apply(BucketInfo bucketInfo) { + return bucketInfo.toPb(); + } + }; private static final long serialVersionUID = -3946094202176916586L; - private final String id; private final String name; private final Acl.Entity owner; @@ -70,27 +78,11 @@ public final class BucketInfo implements Serializable { private final Location location; private final StorageClass storageClass; - static final Function FROM_PB_FUNCTION = - new Function() { - @Override - public BucketInfo apply(com.google.api.services.storage.model.Bucket pb) { - return BucketInfo.fromPb(pb); - } - }; - - static final Function TO_PB_FUNCTION = - new Function() { - @Override - public com.google.api.services.storage.model.Bucket apply(BucketInfo bucketInfo) { - return bucketInfo.toPb(); - } - }; - public static abstract class DeleteRule implements Serializable { private static final long serialVersionUID = 3137971668395933033L; - private final Type type; private static final String SUPPORTED_ACTION = "Delete"; + private final Type type; public enum Type { AGE, CREATE_BEFORE, NUM_NEWER_VERSIONS, IS_LIVE, UNKNOWN @@ -287,18 +279,19 @@ public enum Option { } } - static { - ImmutableMap.Builder map = ImmutableMap.builder(); - for (Option option : Option.values()) { - map.put(option.name(), option); - } - STRING_TO_OPTION = map.build(); - } - private StorageClass(String value) { this.value = checkNotNull(value); } + @Override + public String toString() { + return value(); + } + + public String value() { + return value; + } + public static StorageClass standard() { return Option.STANDARD.storageClass; } @@ -312,13 +305,12 @@ public static StorageClass of(String value) { return option == null ? new StorageClass(value) : option.storageClass; } - @Override - public String toString() { - return value(); - } - - public String value() { - return value; + static { + ImmutableMap.Builder map = ImmutableMap.builder(); + for (Option option : Option.values()) { + map.put(option.name(), option); + } + STRING_TO_OPTION = map.build(); } } @@ -340,35 +332,10 @@ public enum Option { } } - static { - ImmutableMap.Builder map = ImmutableMap.builder(); - for (Option option : Option.values()) { - map.put(option.name(), option); - } - STRING_TO_OPTION = map.build(); - } - private Location(String value) { this.value = checkNotNull(value); } - public static Location us() { - return Option.US.location; - } - - public static Location eu() { - return Option.EU.location; - } - - public static Location asia() { - return Option.ASIA.location; - } - - public static Location of(String value) { - Option option = STRING_TO_OPTION.get(value.toUpperCase()); - return option == null ? new Location(value) : option.location; - } - @Override public int hashCode() { return Objects.hash(value); @@ -394,6 +361,31 @@ public String toString() { public String value() { return value; } + + public static Location us() { + return Option.US.location; + } + + public static Location eu() { + return Option.EU.location; + } + + public static Location asia() { + return Option.ASIA.location; + } + + public static Location of(String value) { + Option option = STRING_TO_OPTION.get(value.toUpperCase()); + return option == null ? new Location(value) : option.location; + } + + static { + ImmutableMap.Builder map = ImmutableMap.builder(); + for (Option option : Option.values()) { + map.put(option.name(), option); + } + STRING_TO_OPTION = map.build(); + } } public final static class Builder { @@ -613,10 +605,7 @@ public int hashCode() { @Override public boolean equals(Object obj) { - if (!(obj instanceof BucketInfo)) { - return false; - } - return Objects.equals(toPb(), ((BucketInfo) obj).toPb()); + return obj instanceof BucketInfo && Objects.equals(toPb(), ((BucketInfo) obj).toPb()); } @Override @@ -626,14 +615,6 @@ public String toString() { .toString(); } - public static BucketInfo of(String name) { - return builder(name).build(); - } - - public static Builder builder(String name) { - return new Builder().name(name); - } - com.google.api.services.storage.model.Bucket toPb() { com.google.api.services.storage.model.Bucket bucketPb = new com.google.api.services.storage.model.Bucket(); @@ -697,6 +678,14 @@ public Rule apply(DeleteRule deleteRule) { return bucketPb; } + public static BucketInfo of(String name) { + return builder(name).build(); + } + + public static Builder builder(String name) { + return new Builder().name(name); + } + static BucketInfo fromPb(com.google.api.services.storage.model.Bucket bucketPb) { Builder builder = new Builder() .name(bucketPb.getName()) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java index e3cfbc195860..5951a88aa114 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java @@ -1,17 +1,12 @@ /* - * Copyright 2015 Google Inc. All Rights Reserved. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version + * 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless + * required by applicable law or agreed to in writing, software * distributed under the License is + * distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. * See the License for the specific language governing permissions and * limitations + * under the License. */ package com.google.gcloud.storage; @@ -20,6 +15,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; import com.google.gcloud.AuthCredentials.ServiceAccountAuthCredentials; import com.google.gcloud.Service; import com.google.gcloud.spi.StorageRpc; @@ -223,7 +219,7 @@ class SignUrlOption implements Serializable { private final Object value; enum Option { - HTTP_METHOD, CONTENT_TYPE, MD5, SERVICE_ACCOUNT_CRED; + HTTP_METHOD, CONTENT_TYPE, MD5, SERVICE_ACCOUNT_CRED } private SignUrlOption(Option option, Object value) { @@ -310,8 +306,8 @@ public Long generation() { public static class Builder { private final List sourceBlobs = new LinkedList<>(); - private BlobInfo target; private final Set targetOptions = new LinkedHashSet<>(); + private BlobInfo target; public Builder addSource(Iterable blobs) { for (String blob : blobs) { @@ -342,6 +338,11 @@ public Builder targetOptions(BlobTargetOption... options) { return this; } + public Builder targetOptions(Iterable options) { + Iterables.addAll(targetOptions, options); + return this; + } + public ComposeRequest build() { checkArgument(!sourceBlobs.isEmpty()); checkNotNull(target); @@ -392,11 +393,11 @@ class CopyRequest implements Serializable { public static class Builder { + private final Set sourceOptions = new LinkedHashSet<>(); + private final Set targetOptions = new LinkedHashSet<>(); private String sourceBucket; private String sourceBlob; - private final Set sourceOptions = new LinkedHashSet<>(); private BlobInfo target; - private final Set targetOptions = new LinkedHashSet<>(); public Builder source(String bucket, String blob) { this.sourceBucket = bucket; @@ -409,6 +410,11 @@ public Builder sourceOptions(BlobSourceOption... options) { return this; } + public Builder sourceOptions(Iterable options) { + Iterables.addAll(sourceOptions, options); + return this; + } + public Builder target(BlobInfo target) { this.target = target; return this; @@ -419,6 +425,11 @@ public Builder targetOptions(BlobTargetOption... options) { return this; } + public Builder targetOptions(Iterable options) { + Iterables.addAll(targetOptions, options); + return this; + } + public CopyRequest build() { checkNotNull(sourceBucket); checkNotNull(sourceBlob); diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java new file mode 100644 index 000000000000..0b50de1682a0 --- /dev/null +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java @@ -0,0 +1,131 @@ +/* + * + * * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version + * 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless + * required by applicable law or agreed to in writing, software * distributed under the License is + * distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. * See the License for the specific language governing permissions and * limitations + * under the License. + */ + +package com.google.gcloud.storage; + +import static org.easymock.EasyMock.capture; +import static org.easymock.EasyMock.createMock; +import static org.easymock.EasyMock.createStrictMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +import com.google.gcloud.storage.Storage.CopyRequest; + +import org.easymock.Capture; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.net.URL; + +public class BlobTest { + + private Storage storage; + private Blob blob; + private BlobInfo blobInfo = BlobInfo.of("b", "n"); + + @Before + public void setUp() throws Exception { + storage = createStrictMock(Storage.class); + blob = new Blob(storage, blobInfo); + } + + @After + public void tearDown() throws Exception { + verify(storage); + } + + @Test + public void testInfo() throws Exception { + assertEquals(blobInfo, blob.info()); + replay(storage); + } + + @Test + public void testExists_True() throws Exception { + expect(storage.get(blobInfo.bucket(), blobInfo.name())).andReturn(blobInfo); + replay(storage); + assertTrue(blob.exists()); + } + + @Test + public void testExists_False() throws Exception { + expect(storage.get(blobInfo.bucket(), blobInfo.name())).andReturn(null); + replay(storage); + assertFalse(blob.exists()); + } + + @Test + public void testContent() throws Exception { + byte[] content = {1, 2}; + expect(storage.readAllBytes(blobInfo.bucket(), blobInfo.name())).andReturn(content); + replay(storage); + assertArrayEquals(content, blob.content()); + } + + @Test + public void testUpdate() throws Exception { + BlobInfo updatedInfo = blobInfo.toBuilder().cacheControl("c").build(); + expect(storage.update(updatedInfo)).andReturn(updatedInfo); + replay(storage); + blob.update(updatedInfo); + assertSame(storage, blob.storage()); + assertEquals(updatedInfo, blob.info()); + } + + @Test + public void testDelete() throws Exception { + expect(storage.delete(blobInfo.bucket(), blobInfo.name())).andReturn(true); + replay(storage); + assertTrue(blob.delete()); + } + + @Test + public void testCopyTo() throws Exception { + BlobInfo target = BlobInfo.of("bt", "nt"); + Capture capturedCopyRequest = Capture.newInstance(); + expect(storage.copy(capture(capturedCopyRequest))).andReturn(target); + replay(storage); + Blob targetBlob = blob.copyTo(target); + assertEquals(target, targetBlob.info()); + assertSame(storage, targetBlob.storage()); + } + + @Test + public void testReader() throws Exception { + BlobReadChannel channel = createMock(BlobReadChannel.class); + expect(storage.reader(blobInfo.bucket(), blobInfo.name())).andReturn(channel); + replay(storage); + assertSame(channel, blob.reader()); + } + + @Test + public void testWriter() throws Exception { + BlobWriteChannel channel = createMock(BlobWriteChannel.class); + expect(storage.writer(blobInfo)).andReturn(channel); + replay(storage); + assertSame(channel, blob.writer()); + } + + @Test + public void testSignUrl() throws Exception { + URL url = new URL("http://localhost:123/bla"); + expect(storage.signUrl(blobInfo, 100)).andReturn(url); + replay(storage); + assertEquals(url, blob.signUrl(100)); + } +} From 21fc90c0504823c779a707ba08a2cad39f66295d Mon Sep 17 00:00:00 2001 From: aozarov Date: Thu, 9 Jul 2015 16:16:42 -0700 Subject: [PATCH 008/156] update copyright --- .../java/com/google/gcloud/storage/Blob.java | 20 ++++++++++++------- .../com/google/gcloud/storage/BlobInfo.java | 19 +++++++++++------- .../com/google/gcloud/storage/Bucket.java | 19 +++++++++++------- .../com/google/gcloud/storage/BucketInfo.java | 19 +++++++++++------- .../com/google/gcloud/storage/Storage.java | 19 +++++++++++------- .../com/google/gcloud/storage/BlobTest.java | 19 +++++++++++------- 6 files changed, 73 insertions(+), 42 deletions(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java index 40ffd8235ba0..dd83ee84182a 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java @@ -1,12 +1,18 @@ + /* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 * - * * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version - * 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may - * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless - * required by applicable law or agreed to in writing, software * distributed under the License is - * distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. * See the License for the specific language governing permissions and * limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package com.google.gcloud.storage; diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java index 68255f991187..7536f96e20f8 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java @@ -1,12 +1,17 @@ /* + * Copyright 2015 Google Inc. All Rights Reserved. * - * * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version - * 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may - * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless - * required by applicable law or agreed to in writing, software * distributed under the License is - * distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. * See the License for the specific language governing permissions and * limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package com.google.gcloud.storage; diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java index 5c4d037f21c1..fd93a5bda6ea 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java @@ -1,12 +1,17 @@ /* + * Copyright 2015 Google Inc. All Rights Reserved. * - * * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version - * 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may - * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless - * required by applicable law or agreed to in writing, software * distributed under the License is - * distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. * See the License for the specific language governing permissions and * limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package com.google.gcloud.storage; diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java index b9c6b03f9dd3..3f35fbc20e3a 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java @@ -1,12 +1,17 @@ /* + * Copyright 2015 Google Inc. All Rights Reserved. * - * * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version - * 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may - * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless - * required by applicable law or agreed to in writing, software * distributed under the License is - * distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. * See the License for the specific language governing permissions and * limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package com.google.gcloud.storage; diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java index 5951a88aa114..590c8d9d74d4 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java @@ -1,12 +1,17 @@ /* + * Copyright 2015 Google Inc. All Rights Reserved. * - * * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version - * 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may - * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless - * required by applicable law or agreed to in writing, software * distributed under the License is - * distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. * See the License for the specific language governing permissions and * limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package com.google.gcloud.storage; diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java index 0b50de1682a0..e49c5fde96c6 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java @@ -1,12 +1,17 @@ /* + * Copyright 2015 Google Inc. All Rights Reserved. * - * * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version - * 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may - * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless - * required by applicable law or agreed to in writing, software * distributed under the License is - * distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. * See the License for the specific language governing permissions and * limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package com.google.gcloud.storage; From d8fa7bbdec65710681b2d1470533146f34d0595a Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Tue, 22 Sep 2015 16:58:52 +0200 Subject: [PATCH 009/156] Split ListResult into interface and implementation. Add BlobListResult for paginated list of Blobs. Add tests. --- .../google/gcloud/storage/BaseListResult.java | 77 +++++++++++++++ .../google/gcloud/storage/BlobListResult.java | 77 +++++++++++++++ .../com/google/gcloud/storage/ListResult.java | 57 ++--------- .../google/gcloud/storage/StorageImpl.java | 6 +- ...esultTest.java => BaseListResultTest.java} | 12 +-- .../gcloud/storage/BlobListResultTest.java | 94 +++++++++++++++++++ .../gcloud/storage/SerializationTest.java | 4 +- 7 files changed, 268 insertions(+), 59 deletions(-) create mode 100644 gcloud-java-storage/src/main/java/com/google/gcloud/storage/BaseListResult.java create mode 100644 gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobListResult.java rename gcloud-java-storage/src/test/java/com/google/gcloud/storage/{ListResultTest.java => BaseListResultTest.java} (75%) create mode 100644 gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobListResultTest.java diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BaseListResult.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BaseListResult.java new file mode 100644 index 000000000000..fdcd84705555 --- /dev/null +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BaseListResult.java @@ -0,0 +1,77 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.storage; + +import java.io.Serializable; +import java.util.Collections; +import java.util.Iterator; +import java.util.Objects; + +/** + * Base implementation for Google Cloud storage list result. + */ +public class BaseListResult implements ListResult, Serializable { + + private static final long serialVersionUID = -6937287874908527950L; + + private final String cursor; + private final Iterable results; + private final NextPageFetcher pageFetcher; + + public interface NextPageFetcher extends Serializable { + ListResult nextPage(); + } + + public BaseListResult(NextPageFetcher pageFetcher, String cursor, Iterable results) { + this.pageFetcher = pageFetcher; + this.cursor = cursor; + this.results = results; + } + + @Override + public String nextPageCursor() { + return cursor; + } + + @Override + public ListResult nextPage() { + if (cursor == null || pageFetcher == null) { + return null; + } + return pageFetcher.nextPage(); + } + + @Override + public Iterator iterator() { + return results == null ? Collections.emptyIterator() : results.iterator(); + } + + @Override + public int hashCode() { + return Objects.hash(cursor, results); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof BaseListResult)) { + return false; + } + BaseListResult other = (BaseListResult) obj; + return Objects.equals(cursor, other.cursor) + && Objects.equals(results, other.results); + } +} diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobListResult.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobListResult.java new file mode 100644 index 000000000000..b4f94bc4b0e7 --- /dev/null +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobListResult.java @@ -0,0 +1,77 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.storage; + +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Objects; + +/** + * Implementation of a paginated list of Google Cloud storage {@code Blob}. + */ +public class BlobListResult implements ListResult { + + private final ListResult infoList; + private final transient Storage storage; + private transient List results; + + public BlobListResult(Storage storage, ListResult infoList) { + this.storage = storage; + this.infoList = infoList; + this.results = null; + } + + @Override + public String nextPageCursor() { + return infoList.nextPageCursor(); + } + + @Override + public ListResult nextPage() { + ListResult nextPageInfoList = infoList.nextPage(); + if (nextPageInfoList == null) { + return null; + } + return new BlobListResult(storage, nextPageInfoList); + } + + @Override + public Iterator iterator() { + if (results == null) { + this.results = new LinkedList<>(); + for (Iterator it = infoList.iterator(); it.hasNext();) { + results.add(new Blob(storage, it.next())); + } + } + return results.iterator(); + } + + @Override + public int hashCode() { + return Objects.hash(infoList); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof BlobListResult)) { + return false; + } + BlobListResult other = (BlobListResult) obj; + return Objects.equals(infoList, other.infoList); + } +} diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/ListResult.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/ListResult.java index 267226c5535f..008ad5164a0a 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/ListResult.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/ListResult.java @@ -16,67 +16,28 @@ package com.google.gcloud.storage; - -import java.io.Serializable; -import java.util.Collections; import java.util.Iterator; -import java.util.Objects; /** - * Google Cloud storage list result. + * Interface for Google Cloud storage list result. */ -public final class ListResult implements Iterable, Serializable { - - private static final long serialVersionUID = -6937287874908527950L; - - private final String cursor; - private final Iterable results; - private final NextPageFetcher pageFetcher; - - public interface NextPageFetcher extends Serializable { - ListResult nextPage(); - } - - public ListResult(NextPageFetcher pageFetcher, String cursor, Iterable results) { - this.pageFetcher = pageFetcher; - this.cursor = cursor; - this.results = results; - } +public interface ListResult extends Iterable { /** * Returns the cursor for the nextPage or {@code null} if no more results. + * + * @return the string cursor for next page */ - public String nextPageCursor() { - return cursor; - } + public String nextPageCursor(); /** * Returns the results of the nextPage or {@code null} if no more result. + * + * @return the {@code ListResult} object for the next page */ - public ListResult nextPage() { - if (cursor == null || pageFetcher == null) { - return null; - } - return pageFetcher.nextPage(); - } - - @Override - public Iterator iterator() { - return results == null ? Collections.emptyIterator() : results.iterator(); - } + public ListResult nextPage(); @Override - public int hashCode() { - return Objects.hash(cursor, results); - } + public Iterator iterator(); - @Override - public boolean equals(Object obj) { - if (!(obj instanceof ListResult)) { - return false; - } - ListResult other = (ListResult) obj; - return Objects.equals(cursor, other.cursor) - && Objects.equals(results, other.results); - } } diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java index f59c6c670969..be3bf1fa5d26 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java @@ -165,7 +165,7 @@ public StorageObject call() { } private static abstract class BasePageFetcher - implements ListResult.NextPageFetcher { + implements BaseListResult.NextPageFetcher { private static final long serialVersionUID = 8236329004030295223L; protected final Map requestOptions; @@ -234,7 +234,7 @@ public Tuple> cal } }, serviceOptions.retryParams(), EXCEPTION_HANDLER); String cursor = result.x(); - return new ListResult<>(new BucketPageFetcher(serviceOptions, cursor, optionsMap), cursor, + return new BaseListResult<>(new BucketPageFetcher(serviceOptions, cursor, optionsMap), cursor, Iterables.transform(result.y(), new Function() { @Override @@ -259,7 +259,7 @@ public Tuple> call() { } }, serviceOptions.retryParams(), EXCEPTION_HANDLER); String cursor = result.x(); - return new ListResult<>(new BlobPageFetcher(bucket, serviceOptions, cursor, optionsMap), cursor, + return new BaseListResult<>(new BlobPageFetcher(bucket, serviceOptions, cursor, optionsMap), cursor, Iterables.transform(result.y(), new Function() { @Override diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ListResultTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BaseListResultTest.java similarity index 75% rename from gcloud-java-storage/src/test/java/com/google/gcloud/storage/ListResultTest.java rename to gcloud-java-storage/src/test/java/com/google/gcloud/storage/BaseListResultTest.java index c64bc0f98324..4c22edbc35c8 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ListResultTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BaseListResultTest.java @@ -24,21 +24,21 @@ import java.util.Collections; -public class ListResultTest { +public class BaseListResultTest { @Test public void testListResult() throws Exception { ImmutableList values = ImmutableList.of("1", "2"); - final ListResult nextResult = - new ListResult<>(null, "c", Collections.emptyList()); - ListResult.NextPageFetcher fetcher = new ListResult.NextPageFetcher() { + final BaseListResult nextResult = + new BaseListResult<>(null, "c", Collections.emptyList()); + BaseListResult.NextPageFetcher fetcher = new BaseListResult.NextPageFetcher() { @Override - public ListResult nextPage() { + public BaseListResult nextPage() { return nextResult; } }; - ListResult result = new ListResult<>(fetcher, "c", values); + BaseListResult result = new BaseListResult<>(fetcher, "c", values); assertEquals(nextResult, result.nextPage()); assertEquals("c", result.nextPageCursor()); assertEquals(values, ImmutableList.copyOf(result.iterator())); diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobListResultTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobListResultTest.java new file mode 100644 index 000000000000..9cbe5b3fea5a --- /dev/null +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobListResultTest.java @@ -0,0 +1,94 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.storage; + +import static org.easymock.EasyMock.createStrictMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +import com.google.common.collect.ImmutableList; +import java.util.Iterator; + +import org.junit.Before; +import org.junit.Test; + +public class BlobListResultTest { + + private static final Iterable FIRST_PAGE_RESULTS = ImmutableList.of( + BlobInfo.of("b1", "n1"), + BlobInfo.of("b2", "n2")); + + private static final Iterable SECOND_PAGE_RESULTS = ImmutableList.of( + BlobInfo.of("b1", "n1"), + BlobInfo.of("b2", "n2")); + + private BaseListResult firstPage; + private BaseListResult secondPage; + private Storage storage; + private BlobListResult blobListResult; + + @Before + public void setUp() throws Exception { + firstPage = createStrictMock(BaseListResult.class); + secondPage = createStrictMock(BaseListResult.class); + storage = createStrictMock(Storage.class); + blobListResult = new BlobListResult(storage, firstPage); + } + + @Test + public void testListResult() throws Exception { + expect(firstPage.iterator()).andReturn(FIRST_PAGE_RESULTS.iterator()); + replay(firstPage); + Iterator firstPageIterator = FIRST_PAGE_RESULTS.iterator(); + Iterator blobListIterator = blobListResult.iterator(); + while (blobListIterator.hasNext() && firstPageIterator.hasNext()) { + assertEquals(firstPageIterator.next(), blobListIterator.next().info()); + } + assertFalse(blobListIterator.hasNext()); + assertFalse(firstPageIterator.hasNext()); + verify(firstPage); + } + + @Test + public void testCursor() throws Exception { + expect(firstPage.nextPageCursor()).andReturn("c"); + replay(firstPage); + assertEquals("c", blobListResult.nextPageCursor()); + verify(firstPage); + } + + @Test + public void testNextPage() throws Exception { + expect(firstPage.nextPage()).andReturn(secondPage); + expect(secondPage.iterator()).andReturn(SECOND_PAGE_RESULTS.iterator()); + replay(firstPage); + replay(secondPage); + ListResult nextPageResult = blobListResult.nextPage(); + Iterator secondPageIterator = SECOND_PAGE_RESULTS.iterator(); + Iterator blobListIterator = nextPageResult.iterator(); + while (blobListIterator.hasNext() && secondPageIterator.hasNext()) { + assertEquals(secondPageIterator.next(), blobListIterator.next().info()); + } + assertFalse(blobListIterator.hasNext()); + assertFalse(secondPageIterator.hasNext()); + verify(firstPage); + verify(secondPage); + } +} diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java index 81342e4b5748..cd02cfbd801d 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java @@ -50,8 +50,8 @@ public class SerializationTest { Collections.singletonList(BatchResponse.Result.of(true)), Collections.>emptyList(), Collections.>emptyList()); - private static final ListResult LIST_RESULT = - new ListResult<>(null, "c", Collections.singletonList(BlobInfo.of("b", "n"))); + private static final BaseListResult LIST_RESULT = + new BaseListResult<>(null, "c", Collections.singletonList(BlobInfo.of("b", "n"))); private static final Storage.BlobListOption BLOB_LIST_OPTIONS = Storage.BlobListOption.maxResults(100); private static final Storage.BlobSourceOption BLOB_SOURCE_OPTIONS = From 44cefb06a9f1a2393f0191fe93e34b9776ced00d Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Tue, 22 Sep 2015 17:00:03 +0200 Subject: [PATCH 010/156] Blob and Bucket extended with more functionalities, added tests --- .../java/com/google/gcloud/storage/Blob.java | 73 ++++++-- .../com/google/gcloud/storage/Bucket.java | 103 +++++++++-- .../com/google/gcloud/storage/BlobTest.java | 5 +- .../com/google/gcloud/storage/BucketTest.java | 160 ++++++++++++++++++ 4 files changed, 313 insertions(+), 28 deletions(-) create mode 100644 gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketTest.java diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java index dd83ee84182a..d52b15eae8c2 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java @@ -1,4 +1,3 @@ - /* * Copyright 2015 Google Inc. All Rights Reserved. * @@ -31,7 +30,6 @@ import java.net.URL; import java.util.Objects; - /** * A Google cloud storage object. */ @@ -89,18 +87,44 @@ static Storage.BlobSourceOption[] convert(BlobInfo blobInfo, BlobSourceOption... } } + /** + * Construct a {@code Blob} object for the provided {@code BlobInfo}. The storage service is used + * to issue requests. + * + * @param storage the storage service used for issuing requests + * @param info blob's info + */ public Blob(Storage storage, BlobInfo info) { this.storage = checkNotNull(storage); this.info = checkNotNull(info); } + /** + * Construct a {@code Blob} object for the provided bucket and blob names. The storage service is + * used to issue requests. + * + * @param storage the storage service used for issuing requests + * @param bucket bucket's name + * @param blob blob's name + */ + public Blob(Storage storage, String bucket, String blob) { + this.storage = checkNotNull(storage); + this.info = BlobInfo.of(bucket, blob); + } + + /** + * Get the blobs's information. + * + * @return a {@code BlobInfo} object for this blob + */ public BlobInfo info() { return info; } /** - * Returns true if this blob exists. + * Check if this blob exists. * + * @return true if this blob exists, false otherwise * @throws StorageException upon failure */ public boolean exists() { @@ -108,8 +132,10 @@ public boolean exists() { } /** - * Returns the blob's content. + * Return this blob's content. * + * @param options blob read options + * @return the blob's content * @throws StorageException upon failure */ public byte[] content(Storage.BlobSourceOption... options) { @@ -117,10 +143,12 @@ public byte[] content(Storage.BlobSourceOption... options) { } /** - * Updates the blob's information. Bucket or blob's name cannot be changed by this method. If you + * Update the blob's information. Bucket or blob's name cannot be changed by this method. If you * want to rename the blob or move it to a different bucket use the {@link #copyTo} and * {@link #delete} operations. * + * @param blobInfo new blob's information. Bucket and blob names must match the current ones + * @param options update options * @throws StorageException upon failure */ public void update(BlobInfo blobInfo, BlobTargetOption... options) { @@ -130,9 +158,10 @@ public void update(BlobInfo blobInfo, BlobTargetOption... options) { } /** - * Deletes this blob. + * Delete this blob. * - * @return true if bucket was deleted + * @param options blob delete options + * @return true if blob was deleted * @throws StorageException upon failure */ public boolean delete(BlobSourceOption... options) { @@ -140,9 +169,11 @@ public boolean delete(BlobSourceOption... options) { } /** - * Send a copy request. + * Copy this blob. * - * @return the copied blob. + * @param target target blob + * @param options source blob options + * @return the copied blob * @throws StorageException upon failure */ public Blob copyTo(BlobInfo target, BlobSourceOption... options) { @@ -150,9 +181,12 @@ public Blob copyTo(BlobInfo target, BlobSourceOption... options) { } /** - * Send a copy request. + * Copy this blob. * - * @return the copied blob. + * @param target target blob + * @param sourceOptions source blob options + * @param targetOptions target blob options + * @return the copied blob * @throws StorageException upon failure */ public Blob copyTo(BlobInfo target, Iterable sourceOptions, @@ -165,8 +199,10 @@ public Blob copyTo(BlobInfo target, Iterable sourceOptions, } /** - * Returns a channel for reading this blob's content. + * Return a channel for reading this blob's content. * + * @param options blob read options + * @return a {@code BlobReadChannel} object to read this blob * @throws StorageException upon failure */ public BlobReadChannel reader(BlobSourceOption... options) { @@ -174,8 +210,10 @@ public BlobReadChannel reader(BlobSourceOption... options) { } /** - * Returns a channel for writing to this blob. + * Return a channel for writing to this blob. * + * @param options target blob options + * @return a {@code BlobWriteChannel} object for writing to this blob * @throws StorageException upon failure */ public BlobWriteChannel writer(BlobTargetOption... options) { @@ -183,18 +221,25 @@ public BlobWriteChannel writer(BlobTargetOption... options) { } /** - * Generates a signed URL for this blob. If you want to allow access to for a fixed amount of time + * Generate a signed URL for this blob. If you want to allow access to for a fixed amount of time * for this blob, you can use this method to generate a URL that is only valid within a certain * time period. This is particularly useful if you don't want publicly accessible blobs, but don't * want to require users to explicitly log in. * * @param expirationTimeInSeconds the signed URL expiration (using epoch time) + * @param options signed url options + * @return a signed URL for this bucket and the specified options * @see Signed-URLs */ public URL signUrl(long expirationTimeInSeconds, SignUrlOption... options) { return storage.signUrl(info, expirationTimeInSeconds, options); } + /** + * Get this blobs's {@code Storage} object. + * + * @return the storage service used by this blob to issue requests + */ public Storage storage() { return storage; } diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java index fd93a5bda6ea..56d27c3c621f 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java @@ -20,8 +20,10 @@ import static com.google.common.base.Preconditions.checkNotNull; import com.google.gcloud.storage.Storage.BlobSourceOption; +import com.google.gcloud.storage.Storage.BlobTargetOption; import com.google.gcloud.storage.Storage.BucketSourceOption; import com.google.gcloud.storage.Storage.BucketTargetOption; +import java.util.LinkedList; import java.util.List; import java.util.Objects; @@ -34,14 +36,43 @@ public final class Bucket { private final Storage storage; private BucketInfo info; + /** + * Construct a {@code Bucket} object for the provided {@code BucketInfo}. The storage service is + * used to issue requests. + * + * @param storage the storage service used for issuing requests + * @param info bucket's info + */ public Bucket(Storage storage, BucketInfo info) { this.storage = checkNotNull(storage); this.info = checkNotNull(info); } /** - * Returns true if this bucket exists. + * Construct a {@code Bucket} object for the provided name. The storage service is used to issue + * requests. + * + * @param storage the storage service used for issuing requests + * @param bucket bucket's name + */ + public Bucket(Storage storage, String bucket) { + this.storage = checkNotNull(storage); + this.info = BucketInfo.of(bucket); + } + + /** + * Get the bucket's information. + * + * @return a {@code BucketInfo} object for this bucket + */ + public BucketInfo info() { + return info; + } + + /** + * Check if this bucket exists. * + * @return true if this bucket exists, false otherwise * @throws StorageException upon failure */ public boolean exists() { @@ -51,6 +82,8 @@ public boolean exists() { /** * Update the bucket's information. Bucket's name cannot be changed. * + * @param bucketInfo new bucket's information. Name must match the one of the current bucket + * @param options update options * @throws StorageException upon failure */ public void update(BucketInfo bucketInfo, BucketTargetOption... options) { @@ -61,6 +94,7 @@ public void update(BucketInfo bucketInfo, BucketTargetOption... options) { /** * Delete this bucket. * + * @param options bucket delete options * @return true if bucket was deleted * @throws StorageException upon failure */ @@ -68,26 +102,69 @@ public boolean delete(BucketSourceOption... options) { return storage.delete(info.name(), options); } - public ListResult list(Storage.BlobListOption... options) { - return storage.list(info.name(), options); + /** + * List blobs in this bucket. + * + * @param options options for listing blobs + * @return the paginated list of {@code Blob} objects in this bucket + * @throws StorageException upon failure + */ + public ListResult list(Storage.BlobListOption... options) { + return new BlobListResult(storage, storage.list(info.name(), options)); } - public BlobInfo get(String blob, BlobSourceOption... options) { - return null; + /** + * Return the requested blob in this bucket or {@code null} if not found. + * + * @param blob name of the requested blob + * @param options blob search options + * @return the requested blob in this bucket or {@code null} if not found + * @throws StorageException upon failure + */ + public Blob get(String blob, BlobSourceOption... options) { + return new Blob(storage, storage.get(info.name(), blob, options)); } - public List get(String... blob) { - // todo - return null; + /** + * Return a list of requested blobs in this bucket. Blobs that do not exist are null. + * + * @param blobNames names of the requested blobs + * @return a list containing the requested blobs in this bucket + * @throws StorageException upon failure + */ + public List getAll(String... blobNames) { + BatchRequest.Builder batch = BatchRequest.builder(); + for (String blobName : blobNames) { + batch.get(info.name(), blobName); + } + List blobs = new LinkedList<>(); + BatchResponse response = storage.apply(batch.build()); + for (BatchResponse.Result result : response.gets()) { + BlobInfo blobInfo = result.get(); + blobs.add(blobInfo != null ? new Blob(storage, blobInfo) : null); + } + return blobs; } - /* - * BlobInfo create(BlobInfo blobInfo, byte[] content, BlobTargetOption... options) { - * - * } + /** + * Create a new blob in this bucket. + * + * @param blob a blob name + * @param content the blob content + * @param options options for blob creation + * @return a complete blob information. + * @throws StorageException upon failure */ + Blob create(String blob, byte[] content, BlobTargetOption... options) { + BlobInfo blobInfo = BlobInfo.of(info.name(), blob); + return new Blob(storage, storage.create(blobInfo, content, options)); + } - + /** + * Get this bucket's {@code Storage} object. + * + * @return the storage service used by this bucket to issue requests + */ public Storage storage() { return storage; } diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java index e49c5fde96c6..40ce864368df 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java @@ -41,7 +41,7 @@ public class BlobTest { private Storage storage; private Blob blob; - private BlobInfo blobInfo = BlobInfo.of("b", "n"); + private final BlobInfo blobInfo = BlobInfo.of("b", "n"); @Before public void setUp() throws Exception { @@ -107,6 +107,9 @@ public void testCopyTo() throws Exception { replay(storage); Blob targetBlob = blob.copyTo(target); assertEquals(target, targetBlob.info()); + assertEquals(capturedCopyRequest.getValue().sourceBlob(), blob.info().name()); + assertEquals(capturedCopyRequest.getValue().sourceBucket(), blob.info().bucket()); + assertEquals(capturedCopyRequest.getValue().target(), target); assertSame(storage, targetBlob.storage()); } diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketTest.java new file mode 100644 index 000000000000..4b23af708636 --- /dev/null +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketTest.java @@ -0,0 +1,160 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.storage; + +import static org.easymock.EasyMock.capture; +import static org.easymock.EasyMock.createStrictMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +import com.google.common.collect.ImmutableList; +import com.google.gcloud.storage.BatchResponse.Result; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import org.easymock.Capture; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class BucketTest { + + private static final Iterable BLOB_INFO_RESULTS = ImmutableList.of( + BlobInfo.of("b", "n1"), + BlobInfo.of("b", "n2"), + BlobInfo.of("b", "n3")); + + private Storage storage; + private Bucket bucket; + private final BucketInfo bucketInfo = BucketInfo.of("b"); + + @Before + public void setUp() throws Exception { + storage = createStrictMock(Storage.class); + bucket = new Bucket(storage, bucketInfo); + } + + @After + public void tearDown() throws Exception { + verify(storage); + } + + @Test + public void testInfo() throws Exception { + assertEquals(bucketInfo, bucket.info()); + replay(storage); + } + + @Test + public void testExists_True() throws Exception { + expect(storage.get(bucketInfo.name())).andReturn(bucketInfo); + replay(storage); + assertTrue(bucket.exists()); + } + + @Test + public void testExists_False() throws Exception { + expect(storage.get(bucketInfo.name())).andReturn(null); + replay(storage); + assertFalse(bucket.exists()); + } + + @Test + public void testUpdate() throws Exception { + BucketInfo updatedInfo = bucketInfo.toBuilder().notFoundPage("p").build(); + expect(storage.update(updatedInfo)).andReturn(updatedInfo); + replay(storage); + bucket.update(updatedInfo); + assertSame(storage, bucket.storage()); + assertEquals(updatedInfo, bucket.info()); + } + + @Test + public void testDelete() throws Exception { + expect(storage.delete(bucketInfo.name())).andReturn(true); + replay(storage); + assertTrue(bucket.delete()); + } + + @Test + public void testList() throws Exception { + BaseListResult blobInfoResult = new BaseListResult<>(null, "c", BLOB_INFO_RESULTS); + expect(storage.list(bucketInfo.name())).andReturn(blobInfoResult); + replay(storage); + ListResult blobResult = bucket.list(); + Iterator blobInfoIterator = blobInfoResult.iterator(); + Iterator blobIterator = blobResult.iterator(); + while (blobInfoIterator.hasNext() && blobIterator.hasNext()) { + assertEquals(blobInfoIterator.next(), blobIterator.next().info()); + } + assertFalse(blobInfoIterator.hasNext()); + assertFalse(blobIterator.hasNext()); + assertEquals(blobInfoResult.nextPageCursor(), blobResult.nextPageCursor()); + } + + @Test + public void testGet() throws Exception { + BlobInfo info = BlobInfo.of("b", "n"); + expect(storage.get(bucketInfo.name(), "n")).andReturn(info); + replay(storage); + Blob blob = bucket.get("n"); + assertEquals(info, blob.info()); + } + + @Test + public void testGetAll() throws Exception { + Capture capturedBatchRequest = Capture.newInstance(); + List> batchResultList = new LinkedList<>(); + for (BlobInfo info : BLOB_INFO_RESULTS) { + batchResultList.add(new Result<>(info)); + } + BatchResponse response = + new BatchResponse(Collections.EMPTY_LIST, Collections.EMPTY_LIST, batchResultList); + expect(storage.apply(capture(capturedBatchRequest))).andReturn(response); + replay(storage); + List blobs = bucket.getAll("n1", "n2", "n3"); + Set blobInfoSet = capturedBatchRequest.getValue().toGet().keySet(); + assertEquals(batchResultList.size(), blobInfoSet.size()); + for (BlobInfo info : BLOB_INFO_RESULTS) { + assertTrue(blobInfoSet.contains(info)); + } + Iterator blobIterator = blobs.iterator(); + Iterator> batchResultIterator = response.gets().iterator(); + while (batchResultIterator.hasNext() && blobIterator.hasNext()) { + assertEquals(batchResultIterator.next().get(), blobIterator.next().info()); + } + assertFalse(batchResultIterator.hasNext()); + assertFalse(blobIterator.hasNext()); + } + + @Test + public void testCreate() throws Exception { + BlobInfo info = BlobInfo.of("b", "n"); + byte[] content = {0xD, 0xE, 0xA, 0xD}; + expect(storage.create(info, content)).andReturn(info); + replay(storage); + Blob blob = bucket.create("n", content); + assertEquals(info, blob.info()); + } +} From c1f018c35c46b9d9e2416fc0a8c542d972d78518 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Wed, 23 Sep 2015 11:11:04 +0200 Subject: [PATCH 011/156] Avoid storing Iterable in BlobListResult --- .../google/gcloud/storage/BlobListResult.java | 44 +++++++++++++------ .../com/google/gcloud/storage/ListResult.java | 17 ++----- 2 files changed, 34 insertions(+), 27 deletions(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobListResult.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobListResult.java index b4f94bc4b0e7..9bc53fd94a9b 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobListResult.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobListResult.java @@ -16,9 +16,9 @@ package com.google.gcloud.storage; +import static com.google.common.base.Preconditions.checkNotNull; + import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; import java.util.Objects; /** @@ -27,13 +27,35 @@ public class BlobListResult implements ListResult { private final ListResult infoList; - private final transient Storage storage; - private transient List results; + private final Storage storage; + + private class BlobListIterator implements Iterator { + + private final Iterator blobInfoIterator; + + public BlobListIterator() { + this.blobInfoIterator = infoList.iterator(); + } + + @Override + public boolean hasNext() { + return blobInfoIterator.hasNext(); + } + + @Override + public Blob next() { + return new Blob(storage, blobInfoIterator.next()); + } + + @Override + public void remove() { + blobInfoIterator.remove(); + } + } public BlobListResult(Storage storage, ListResult infoList) { - this.storage = storage; - this.infoList = infoList; - this.results = null; + this.storage = checkNotNull(storage); + this.infoList = checkNotNull(infoList); } @Override @@ -52,13 +74,7 @@ public ListResult nextPage() { @Override public Iterator iterator() { - if (results == null) { - this.results = new LinkedList<>(); - for (Iterator it = infoList.iterator(); it.hasNext();) { - results.add(new Blob(storage, it.next())); - } - } - return results.iterator(); + return new BlobListIterator(); } @Override diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/ListResult.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/ListResult.java index 008ad5164a0a..a07d6edff745 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/ListResult.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/ListResult.java @@ -16,28 +16,19 @@ package com.google.gcloud.storage; -import java.util.Iterator; - /** * Interface for Google Cloud storage list result. */ public interface ListResult extends Iterable { /** - * Returns the cursor for the nextPage or {@code null} if no more results. - * - * @return the string cursor for next page + * Return the cursor for the nextPage or {@code null} if no more results. */ - public String nextPageCursor(); + String nextPageCursor(); /** - * Returns the results of the nextPage or {@code null} if no more result. - * - * @return the {@code ListResult} object for the next page + * Return the results of the nextPage or {@code null} if no more result. */ - public ListResult nextPage(); - - @Override - public Iterator iterator(); + ListResult nextPage(); } From 8591284926cec97e1d58b2d6e24b07d098fda171 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Wed, 23 Sep 2015 16:12:56 +0200 Subject: [PATCH 012/156] Remove redundant return from Blob and Bucket javadoc --- .../java/com/google/gcloud/storage/Blob.java | 17 +++++------------ .../java/com/google/gcloud/storage/Bucket.java | 15 ++++----------- 2 files changed, 9 insertions(+), 23 deletions(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java index d52b15eae8c2..ebad11150663 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java @@ -109,13 +109,11 @@ public Blob(Storage storage, BlobInfo info) { */ public Blob(Storage storage, String bucket, String blob) { this.storage = checkNotNull(storage); - this.info = BlobInfo.of(bucket, blob); + this.info = BlobInfo.of(checkNotNull(bucket), checkNotNull(blob)); } /** - * Get the blobs's information. - * - * @return a {@code BlobInfo} object for this blob + * Return the blob's information. */ public BlobInfo info() { return info; @@ -135,7 +133,6 @@ public boolean exists() { * Return this blob's content. * * @param options blob read options - * @return the blob's content * @throws StorageException upon failure */ public byte[] content(Storage.BlobSourceOption... options) { @@ -199,10 +196,9 @@ public Blob copyTo(BlobInfo target, Iterable sourceOptions, } /** - * Return a channel for reading this blob's content. + * Return a {@code BlobReadChannel} object for reading this blob's content. * * @param options blob read options - * @return a {@code BlobReadChannel} object to read this blob * @throws StorageException upon failure */ public BlobReadChannel reader(BlobSourceOption... options) { @@ -210,10 +206,9 @@ public BlobReadChannel reader(BlobSourceOption... options) { } /** - * Return a channel for writing to this blob. + * Return a {@code BlobWriteChannel} object for writing to this blob. * * @param options target blob options - * @return a {@code BlobWriteChannel} object for writing to this blob * @throws StorageException upon failure */ public BlobWriteChannel writer(BlobTargetOption... options) { @@ -236,9 +231,7 @@ public URL signUrl(long expirationTimeInSeconds, SignUrlOption... options) { } /** - * Get this blobs's {@code Storage} object. - * - * @return the storage service used by this blob to issue requests + * Return the blob's {@code Storage} object used to issue requests. */ public Storage storage() { return storage; diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java index 56d27c3c621f..77c16c106e57 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java @@ -57,13 +57,11 @@ public Bucket(Storage storage, BucketInfo info) { */ public Bucket(Storage storage, String bucket) { this.storage = checkNotNull(storage); - this.info = BucketInfo.of(bucket); + this.info = BucketInfo.of(checkNotNull(bucket)); } /** - * Get the bucket's information. - * - * @return a {@code BucketInfo} object for this bucket + * Return the bucket's information. */ public BucketInfo info() { return info; @@ -103,10 +101,9 @@ public boolean delete(BucketSourceOption... options) { } /** - * List blobs in this bucket. + * Return the paginated list of {@code Blob} in this bucket. * * @param options options for listing blobs - * @return the paginated list of {@code Blob} objects in this bucket * @throws StorageException upon failure */ public ListResult list(Storage.BlobListOption... options) { @@ -118,7 +115,6 @@ public ListResult list(Storage.BlobListOption... options) { * * @param blob name of the requested blob * @param options blob search options - * @return the requested blob in this bucket or {@code null} if not found * @throws StorageException upon failure */ public Blob get(String blob, BlobSourceOption... options) { @@ -129,7 +125,6 @@ public Blob get(String blob, BlobSourceOption... options) { * Return a list of requested blobs in this bucket. Blobs that do not exist are null. * * @param blobNames names of the requested blobs - * @return a list containing the requested blobs in this bucket * @throws StorageException upon failure */ public List getAll(String... blobNames) { @@ -161,9 +156,7 @@ Blob create(String blob, byte[] content, BlobTargetOption... options) { } /** - * Get this bucket's {@code Storage} object. - * - * @return the storage service used by this bucket to issue requests + * Return the bucket's {@code Storage} object used to issue requests. */ public Storage storage() { return storage; From a10b74b73c5f5d5a2ed337503ddd4351f95d2a64 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Thu, 24 Sep 2015 10:51:19 +0200 Subject: [PATCH 013/156] Make Blob.copyTo accept string parameters --- .../java/com/google/gcloud/storage/Blob.java | 25 +++++------ .../com/google/gcloud/storage/BlobTest.java | 43 ++++++++++++------- 2 files changed, 39 insertions(+), 29 deletions(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java index ebad11150663..afe2d1ab8375 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java @@ -20,8 +20,6 @@ import static com.google.common.base.Preconditions.checkNotNull; import static com.google.gcloud.storage.Blob.BlobSourceOption.convert; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; import com.google.gcloud.spi.StorageRpc; import com.google.gcloud.storage.Storage.BlobTargetOption; import com.google.gcloud.storage.Storage.CopyRequest; @@ -166,32 +164,31 @@ public boolean delete(BlobSourceOption... options) { } /** - * Copy this blob. + * Copy this blob to the target bucket, preserving its name. Possibly update metadata. * - * @param target target blob + * @param targetBucket target bucket's name * @param options source blob options * @return the copied blob * @throws StorageException upon failure */ - public Blob copyTo(BlobInfo target, BlobSourceOption... options) { - return copyTo(target, ImmutableList.copyOf(options), ImmutableList.of()); + public Blob copyTo(String targetBucket, BlobSourceOption... options) { + return copyTo(targetBucket, info.name(), options); } /** - * Copy this blob. + * Copy this blob to the target bucket with a new name. Possibly update metadata. * - * @param target target blob - * @param sourceOptions source blob options - * @param targetOptions target blob options + * @param targetBucket target bucket's name + * @param targetBlob target blob's name + * @param options source blob options * @return the copied blob * @throws StorageException upon failure */ - public Blob copyTo(BlobInfo target, Iterable sourceOptions, - Iterable targetOptions) { + public Blob copyTo(String targetBucket, String targetBlob, BlobSourceOption... options) { + BlobInfo updatedInfo = info.toBuilder().bucket(targetBucket).name(targetBlob).build(); CopyRequest copyRequest = CopyRequest.builder().source(info.bucket(), info.name()) - .sourceOptions(convert(info, Iterables.toArray(sourceOptions, BlobSourceOption.class))) - .target(target).targetOptions(targetOptions).build(); + .sourceOptions(convert(info, options)).target(updatedInfo).build(); return new Blob(storage, storage.copy(copyRequest)); } diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java index 40ce864368df..a46d4d92d96c 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java @@ -29,24 +29,23 @@ import static org.junit.Assert.assertTrue; import com.google.gcloud.storage.Storage.CopyRequest; - import org.easymock.Capture; import org.junit.After; import org.junit.Before; import org.junit.Test; - import java.net.URL; public class BlobTest { + private static final BlobInfo BLOB_INFO = BlobInfo.of("b", "n"); + private Storage storage; private Blob blob; - private final BlobInfo blobInfo = BlobInfo.of("b", "n"); @Before public void setUp() throws Exception { storage = createStrictMock(Storage.class); - blob = new Blob(storage, blobInfo); + blob = new Blob(storage, BLOB_INFO); } @After @@ -56,20 +55,20 @@ public void tearDown() throws Exception { @Test public void testInfo() throws Exception { - assertEquals(blobInfo, blob.info()); + assertEquals(BLOB_INFO, blob.info()); replay(storage); } @Test public void testExists_True() throws Exception { - expect(storage.get(blobInfo.bucket(), blobInfo.name())).andReturn(blobInfo); + expect(storage.get(BLOB_INFO.bucket(), BLOB_INFO.name())).andReturn(BLOB_INFO); replay(storage); assertTrue(blob.exists()); } @Test public void testExists_False() throws Exception { - expect(storage.get(blobInfo.bucket(), blobInfo.name())).andReturn(null); + expect(storage.get(BLOB_INFO.bucket(), BLOB_INFO.name())).andReturn(null); replay(storage); assertFalse(blob.exists()); } @@ -77,14 +76,14 @@ public void testExists_False() throws Exception { @Test public void testContent() throws Exception { byte[] content = {1, 2}; - expect(storage.readAllBytes(blobInfo.bucket(), blobInfo.name())).andReturn(content); + expect(storage.readAllBytes(BLOB_INFO.bucket(), BLOB_INFO.name())).andReturn(content); replay(storage); assertArrayEquals(content, blob.content()); } @Test public void testUpdate() throws Exception { - BlobInfo updatedInfo = blobInfo.toBuilder().cacheControl("c").build(); + BlobInfo updatedInfo = BLOB_INFO.toBuilder().cacheControl("c").build(); expect(storage.update(updatedInfo)).andReturn(updatedInfo); replay(storage); blob.update(updatedInfo); @@ -94,18 +93,32 @@ public void testUpdate() throws Exception { @Test public void testDelete() throws Exception { - expect(storage.delete(blobInfo.bucket(), blobInfo.name())).andReturn(true); + expect(storage.delete(BLOB_INFO.bucket(), BLOB_INFO.name())).andReturn(true); replay(storage); assertTrue(blob.delete()); } + @Test + public void testCopyToBucket() throws Exception { + BlobInfo target = BLOB_INFO.toBuilder().bucket("bt").build(); + Capture capturedCopyRequest = Capture.newInstance(); + expect(storage.copy(capture(capturedCopyRequest))).andReturn(target); + replay(storage); + Blob targetBlob = blob.copyTo("bt"); + assertEquals(target, targetBlob.info()); + assertEquals(capturedCopyRequest.getValue().sourceBlob(), blob.info().name()); + assertEquals(capturedCopyRequest.getValue().sourceBucket(), blob.info().bucket()); + assertEquals(capturedCopyRequest.getValue().target(), target); + assertSame(storage, targetBlob.storage()); + } + @Test public void testCopyTo() throws Exception { - BlobInfo target = BlobInfo.of("bt", "nt"); + BlobInfo target = BLOB_INFO.toBuilder().bucket("bt").name("nt").build(); Capture capturedCopyRequest = Capture.newInstance(); expect(storage.copy(capture(capturedCopyRequest))).andReturn(target); replay(storage); - Blob targetBlob = blob.copyTo(target); + Blob targetBlob = blob.copyTo("bt", "nt"); assertEquals(target, targetBlob.info()); assertEquals(capturedCopyRequest.getValue().sourceBlob(), blob.info().name()); assertEquals(capturedCopyRequest.getValue().sourceBucket(), blob.info().bucket()); @@ -116,7 +129,7 @@ public void testCopyTo() throws Exception { @Test public void testReader() throws Exception { BlobReadChannel channel = createMock(BlobReadChannel.class); - expect(storage.reader(blobInfo.bucket(), blobInfo.name())).andReturn(channel); + expect(storage.reader(BLOB_INFO.bucket(), BLOB_INFO.name())).andReturn(channel); replay(storage); assertSame(channel, blob.reader()); } @@ -124,7 +137,7 @@ public void testReader() throws Exception { @Test public void testWriter() throws Exception { BlobWriteChannel channel = createMock(BlobWriteChannel.class); - expect(storage.writer(blobInfo)).andReturn(channel); + expect(storage.writer(BLOB_INFO)).andReturn(channel); replay(storage); assertSame(channel, blob.writer()); } @@ -132,7 +145,7 @@ public void testWriter() throws Exception { @Test public void testSignUrl() throws Exception { URL url = new URL("http://localhost:123/bla"); - expect(storage.signUrl(blobInfo, 100)).andReturn(url); + expect(storage.signUrl(BLOB_INFO, 100)).andReturn(url); replay(storage); assertEquals(url, blob.signUrl(100)); } From ccc023045b446ac8d5e4b6e921171e09e3eac37e Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Thu, 24 Sep 2015 11:00:36 +0200 Subject: [PATCH 014/156] Minor refactoring to Bucket and BucketTest --- .../java/com/google/gcloud/storage/Bucket.java | 2 +- .../com/google/gcloud/storage/BucketTest.java | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java index 77c16c106e57..ad2aa7246547 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java @@ -23,8 +23,8 @@ import com.google.gcloud.storage.Storage.BlobTargetOption; import com.google.gcloud.storage.Storage.BucketSourceOption; import com.google.gcloud.storage.Storage.BucketTargetOption; -import java.util.LinkedList; +import java.util.LinkedList; import java.util.List; import java.util.Objects; diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketTest.java index 4b23af708636..9dd4235c16b9 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketTest.java @@ -40,6 +40,7 @@ public class BucketTest { + private static final BucketInfo BUCKET_INFO = BucketInfo.of("b"); private static final Iterable BLOB_INFO_RESULTS = ImmutableList.of( BlobInfo.of("b", "n1"), BlobInfo.of("b", "n2"), @@ -47,12 +48,11 @@ public class BucketTest { private Storage storage; private Bucket bucket; - private final BucketInfo bucketInfo = BucketInfo.of("b"); @Before public void setUp() throws Exception { storage = createStrictMock(Storage.class); - bucket = new Bucket(storage, bucketInfo); + bucket = new Bucket(storage, BUCKET_INFO); } @After @@ -62,27 +62,27 @@ public void tearDown() throws Exception { @Test public void testInfo() throws Exception { - assertEquals(bucketInfo, bucket.info()); + assertEquals(BUCKET_INFO, bucket.info()); replay(storage); } @Test public void testExists_True() throws Exception { - expect(storage.get(bucketInfo.name())).andReturn(bucketInfo); + expect(storage.get(BUCKET_INFO.name())).andReturn(BUCKET_INFO); replay(storage); assertTrue(bucket.exists()); } @Test public void testExists_False() throws Exception { - expect(storage.get(bucketInfo.name())).andReturn(null); + expect(storage.get(BUCKET_INFO.name())).andReturn(null); replay(storage); assertFalse(bucket.exists()); } @Test public void testUpdate() throws Exception { - BucketInfo updatedInfo = bucketInfo.toBuilder().notFoundPage("p").build(); + BucketInfo updatedInfo = BUCKET_INFO.toBuilder().notFoundPage("p").build(); expect(storage.update(updatedInfo)).andReturn(updatedInfo); replay(storage); bucket.update(updatedInfo); @@ -92,7 +92,7 @@ public void testUpdate() throws Exception { @Test public void testDelete() throws Exception { - expect(storage.delete(bucketInfo.name())).andReturn(true); + expect(storage.delete(BUCKET_INFO.name())).andReturn(true); replay(storage); assertTrue(bucket.delete()); } @@ -100,7 +100,7 @@ public void testDelete() throws Exception { @Test public void testList() throws Exception { BaseListResult blobInfoResult = new BaseListResult<>(null, "c", BLOB_INFO_RESULTS); - expect(storage.list(bucketInfo.name())).andReturn(blobInfoResult); + expect(storage.list(BUCKET_INFO.name())).andReturn(blobInfoResult); replay(storage); ListResult blobResult = bucket.list(); Iterator blobInfoIterator = blobInfoResult.iterator(); @@ -116,7 +116,7 @@ public void testList() throws Exception { @Test public void testGet() throws Exception { BlobInfo info = BlobInfo.of("b", "n"); - expect(storage.get(bucketInfo.name(), "n")).andReturn(info); + expect(storage.get(BUCKET_INFO.name(), "n")).andReturn(info); replay(storage); Blob blob = bucket.get("n"); assertEquals(info, blob.info()); From 929030d89b1d40036b1ec906d16505b02a830df7 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Thu, 24 Sep 2015 12:21:33 +0200 Subject: [PATCH 015/156] Fix if-match error on blob delete --- .../src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java | 1 - 1 file changed, 1 deletion(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index 89dfb4c6c414..7c2d3a7be21a 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -282,7 +282,6 @@ private Storage.Objects.Delete deleteRequest(StorageObject blob, Map .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) - .setIfGenerationMatch(100L) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } From 3cf6af39a09272ced21ba85b8c9388846581d1e2 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Thu, 24 Sep 2015 13:53:33 +0200 Subject: [PATCH 016/156] Make Bucket and Blob immutable --- .../java/com/google/gcloud/storage/Blob.java | 20 +++++++++++++---- .../com/google/gcloud/storage/Bucket.java | 22 +++++++++++++++---- .../com/google/gcloud/storage/BlobTest.java | 14 ++++++++++-- .../com/google/gcloud/storage/BucketTest.java | 14 ++++++++++-- 4 files changed, 58 insertions(+), 12 deletions(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java index afe2d1ab8375..f740a2e716ff 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java @@ -34,7 +34,7 @@ public final class Blob { private final Storage storage; - private BlobInfo info; + private final BlobInfo info; public static class BlobSourceOption extends Option { @@ -137,19 +137,31 @@ public byte[] content(Storage.BlobSourceOption... options) { return storage.readAllBytes(info.bucket(), info.name(), options); } + /** + * Fetches current blob's latest information. + * + * @param options blob read options + * @return a {@code Blob} object with latest information + * @throws StorageException upon failure + */ + public Blob reload(BlobSourceOption... options) { + return new Blob(storage, storage.get(info.bucket(), info.name(), convert(info, options))); + } + /** * Update the blob's information. Bucket or blob's name cannot be changed by this method. If you * want to rename the blob or move it to a different bucket use the {@link #copyTo} and - * {@link #delete} operations. + * {@link #delete} operations. A new {@code Blob} object is returned. * * @param blobInfo new blob's information. Bucket and blob names must match the current ones * @param options update options + * @return a {@code Blob} object with updated information * @throws StorageException upon failure */ - public void update(BlobInfo blobInfo, BlobTargetOption... options) { + public Blob update(BlobInfo blobInfo, BlobTargetOption... options) { checkArgument(Objects.equals(blobInfo.bucket(), info.bucket()), "Bucket name must match"); checkArgument(Objects.equals(blobInfo.name(), info.name()), "Blob name must match"); - info = storage.update(blobInfo, options); + return new Blob(storage, storage.update(blobInfo, options)); } /** diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java index ad2aa7246547..47c16a3646ed 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java @@ -18,6 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.gcloud.storage.Blob.BlobSourceOption.convert; import com.google.gcloud.storage.Storage.BlobSourceOption; import com.google.gcloud.storage.Storage.BlobTargetOption; @@ -34,7 +35,7 @@ public final class Bucket { private final Storage storage; - private BucketInfo info; + private final BucketInfo info; /** * Construct a {@code Bucket} object for the provided {@code BucketInfo}. The storage service is @@ -78,15 +79,28 @@ public boolean exists() { } /** - * Update the bucket's information. Bucket's name cannot be changed. + * Fetches current bucket's latest information. + * + * @param options bucket read options + * @return a {@code Bucket} object with latest information + * @throws StorageException upon failure + */ + public Bucket reload(BucketSourceOption... options) { + return new Bucket(storage, storage.get(info.name(), options)); + } + + /** + * Update the bucket's information. Bucket's name cannot be changed. A new {@code Bucket} object + * is returned. * * @param bucketInfo new bucket's information. Name must match the one of the current bucket * @param options update options + * @return a {@code Bucket} object with updated information * @throws StorageException upon failure */ - public void update(BucketInfo bucketInfo, BucketTargetOption... options) { + public Bucket update(BucketInfo bucketInfo, BucketTargetOption... options) { checkArgument(Objects.equals(bucketInfo.name(), info.name()), "Bucket name must match"); - info = storage.update(bucketInfo, options); + return new Bucket(storage, storage.update(bucketInfo, options)); } /** diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java index a46d4d92d96c..4923b8b772e8 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java @@ -81,14 +81,24 @@ public void testContent() throws Exception { assertArrayEquals(content, blob.content()); } + @Test + public void testReload() throws Exception { + BlobInfo updatedInfo = BLOB_INFO.toBuilder().cacheControl("c").build(); + expect(storage.get(BLOB_INFO.bucket(), BLOB_INFO.name())).andReturn(updatedInfo); + replay(storage); + Blob updatedBlob = blob.reload(); + assertSame(storage, blob.storage()); + assertEquals(updatedInfo, updatedBlob.info()); + } + @Test public void testUpdate() throws Exception { BlobInfo updatedInfo = BLOB_INFO.toBuilder().cacheControl("c").build(); expect(storage.update(updatedInfo)).andReturn(updatedInfo); replay(storage); - blob.update(updatedInfo); + Blob updatedBlob = blob.update(updatedInfo); assertSame(storage, blob.storage()); - assertEquals(updatedInfo, blob.info()); + assertEquals(updatedInfo, updatedBlob.info()); } @Test diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketTest.java index 9dd4235c16b9..9b8546ecc27d 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketTest.java @@ -80,14 +80,24 @@ public void testExists_False() throws Exception { assertFalse(bucket.exists()); } + @Test + public void testReload() throws Exception { + BucketInfo updatedInfo = BUCKET_INFO.toBuilder().notFoundPage("p").build(); + expect(storage.get(updatedInfo.name())).andReturn(updatedInfo); + replay(storage); + Bucket updatedBucket = bucket.reload(); + assertSame(storage, bucket.storage()); + assertEquals(updatedInfo, updatedBucket.info()); + } + @Test public void testUpdate() throws Exception { BucketInfo updatedInfo = BUCKET_INFO.toBuilder().notFoundPage("p").build(); expect(storage.update(updatedInfo)).andReturn(updatedInfo); replay(storage); - bucket.update(updatedInfo); + Bucket updatedBucket = bucket.update(updatedInfo); assertSame(storage, bucket.storage()); - assertEquals(updatedInfo, bucket.info()); + assertEquals(updatedInfo, updatedBucket.info()); } @Test From 2e90926ac9042d6285704209eee1807ae273b2a8 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Thu, 24 Sep 2015 14:13:05 +0200 Subject: [PATCH 017/156] Use transform in BlobListResult --- .../google/gcloud/storage/BlobListResult.java | 34 +++++-------------- 1 file changed, 9 insertions(+), 25 deletions(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobListResult.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobListResult.java index 9bc53fd94a9b..9e6ec9dc5655 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobListResult.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobListResult.java @@ -18,6 +18,9 @@ import static com.google.common.base.Preconditions.checkNotNull; +import com.google.common.base.Function; +import com.google.common.collect.Iterators; + import java.util.Iterator; import java.util.Objects; @@ -29,30 +32,6 @@ public class BlobListResult implements ListResult { private final ListResult infoList; private final Storage storage; - private class BlobListIterator implements Iterator { - - private final Iterator blobInfoIterator; - - public BlobListIterator() { - this.blobInfoIterator = infoList.iterator(); - } - - @Override - public boolean hasNext() { - return blobInfoIterator.hasNext(); - } - - @Override - public Blob next() { - return new Blob(storage, blobInfoIterator.next()); - } - - @Override - public void remove() { - blobInfoIterator.remove(); - } - } - public BlobListResult(Storage storage, ListResult infoList) { this.storage = checkNotNull(storage); this.infoList = checkNotNull(infoList); @@ -74,7 +53,12 @@ public ListResult nextPage() { @Override public Iterator iterator() { - return new BlobListIterator(); + return Iterators.transform(infoList.iterator(), new Function() { + @Override + public Blob apply(BlobInfo info) { + return new Blob(storage, info); + } + }); } @Override From cab0ebdc005dea7c06773c631d2acdf72fd54773 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Thu, 24 Sep 2015 17:09:23 +0200 Subject: [PATCH 018/156] Fix IllegalArgumentException when OPTIONS in cors --- .../src/main/java/com/google/gcloud/storage/HttpMethod.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/HttpMethod.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/HttpMethod.java index 9d7944140915..e2a4563316d3 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/HttpMethod.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/HttpMethod.java @@ -20,5 +20,5 @@ * Http method supported by Storage service. */ public enum HttpMethod { - GET, HEAD, PUT, POST, DELETE + GET, HEAD, PUT, POST, DELETE, OPTIONS } From 210355d0089e8740a186de0dd48d86f9f4f7fee3 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Fri, 25 Sep 2015 14:51:18 +0200 Subject: [PATCH 019/156] Add AuthCredentials.createForJson. More javadoc. --- .../com/google/gcloud/AuthCredentials.java | 40 +++++++++++++++++-- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java b/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java index 13d91836ae4c..ffc54df77a90 100644 --- a/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java +++ b/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java @@ -30,6 +30,7 @@ import com.google.auth.oauth2.GoogleCredentials; import java.io.IOException; +import java.io.InputStream; import java.io.ObjectInputStream; import java.io.ObjectStreamException; import java.io.Serializable; @@ -133,7 +134,7 @@ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundE try { computeCredential = getComputeCredential(); } catch (GeneralSecurityException e) { - throw new IOException(e); + throw new IOException(e); } } @@ -156,7 +157,7 @@ private static class ApplicationDefaultAuthCredentials extends AuthCredentials { private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { in.defaultReadObject(); - googleCredentials = GoogleCredentials.getApplicationDefault(); + googleCredentials = GoogleCredentials.getApplicationDefault(); } @Override @@ -183,8 +184,8 @@ public static AuthCredentials createForComputeEngine() * *

Returns the Application Default Credentials which are credentials that identify and * authorize the whole application. This is the built-in service account if running on - * Google Compute Engine or the credentials file from the path in the environment variable - * GOOGLE_APPLICATION_CREDENTIALS. + * Google Compute Engine or the credentials file can be read from the path in the environment + * variable GOOGLE_APPLICATION_CREDENTIALS. *

* * @return the credentials instance. @@ -194,10 +195,41 @@ public static AuthCredentials createApplicationDefaults() throws IOException { return new ApplicationDefaultAuthCredentials(); } + /** + * Creates Service Account Credentials given an account id and a private key. + * + *

For details on how to obtain Service Account Credentials see + * Service + * Account Authentication. + *

+ * + * @param account id of the Service Account + * @param privateKey private key associated to the account + * @return the credentials instance. + */ public static ServiceAccountAuthCredentials createFor(String account, PrivateKey privateKey) { return new ServiceAccountAuthCredentials(account, privateKey); } + /** + * Creates Service Account Credentials given a stream for credentials in JSON format. + * + *

For details on how to obtain Service Account Credentials in JSON format see + * Service + * Account Authentication. + *

+ * + * @param jsonCredentialStream stream for Service Account Credentials in JSON format + * @return the credentials instance. + * @throws IOException if the credentials cannot be created from the stream. + */ + public static ServiceAccountAuthCredentials createForJson(InputStream jsonCredentialStream) + throws IOException { + GoogleCredential tempCredentials = GoogleCredential.fromStream(jsonCredentialStream); + return new ServiceAccountAuthCredentials(tempCredentials.getServiceAccountId(), + tempCredentials.getServiceAccountPrivateKey()); + } + public static AuthCredentials noCredentials() { return ServiceAccountAuthCredentials.NO_CREDENTIALS; } From eb9ea81f9e2aa52558db21642cff7fc656aa4316 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Fri, 25 Sep 2015 10:27:42 +0200 Subject: [PATCH 020/156] Update Blob and Bucket javadoc --- .../src/main/java/com/google/gcloud/storage/Blob.java | 11 ++++++++++- .../main/java/com/google/gcloud/storage/Bucket.java | 11 ++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java index f740a2e716ff..d3fe181970c8 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java @@ -30,6 +30,12 @@ /** * A Google cloud storage object. + * + *

+ * Objects of this class are immutable. Operations that modify the blob like {@link #update} and + * {@link #copyTo} return a new object. To get a {@code Blob} object with the most recent + * information use {@link #reload}. + *

*/ public final class Blob { @@ -151,7 +157,10 @@ public Blob reload(BlobSourceOption... options) { /** * Update the blob's information. Bucket or blob's name cannot be changed by this method. If you * want to rename the blob or move it to a different bucket use the {@link #copyTo} and - * {@link #delete} operations. A new {@code Blob} object is returned. + * {@link #delete} operations. A new {@code Blob} object is returned. By default no checks are + * made on the metadata generation of the current blob. If you want to update the information only + * if the current blob metadata are at their latest version use the {@code metagenerationMatch} + * option: {@code blob.update(newInfo, BlobTargetOption.metagenerationMatch()}. * * @param blobInfo new blob's information. Bucket and blob names must match the current ones * @param options update options diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java index 47c16a3646ed..b56824f069f6 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java @@ -31,6 +31,12 @@ /** * A Google cloud storage bucket. + * + *

+ * Objects of this class are immutable. Operations that modify the bucket like {@link #update} + * return a new object. To get a {@code Bucket} object with the most recent information use + * {@link #reload}. + *

*/ public final class Bucket { @@ -91,7 +97,10 @@ public Bucket reload(BucketSourceOption... options) { /** * Update the bucket's information. Bucket's name cannot be changed. A new {@code Bucket} object - * is returned. + * is returned. By default no checks are made on the metadata generation of the current bucket. + * If you want to update the information only if the current bucket metadata are at their latest + * version use the {@code metagenerationMatch} option: + * {@code bucket.update(newInfo, BucketTargetOption.metagenerationMatch());} * * @param bucketInfo new bucket's information. Name must match the one of the current bucket * @param options update options From af6bbefa0191a940058f5170fdba148b425fa78d Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Fri, 25 Sep 2015 10:43:36 -0700 Subject: [PATCH 021/156] Add third RetryResult (instead of using null) to denote 'proceed' --- .../com/google/gcloud/ExceptionHandler.java | 34 +++++-------- .../google/gcloud/ExceptionHandlerTest.java | 51 ++++++++++++++++++- .../gcloud/datastore/DatastoreImpl.java | 4 +- 3 files changed, 65 insertions(+), 24 deletions(-) diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/ExceptionHandler.java b/gcloud-java-core/src/main/java/com/google/gcloud/ExceptionHandler.java index 412462ae156e..9e472965cb6e 100644 --- a/gcloud-java-core/src/main/java/com/google/gcloud/ExceptionHandler.java +++ b/gcloud-java-core/src/main/java/com/google/gcloud/ExceptionHandler.java @@ -16,7 +16,6 @@ package com.google.gcloud; -import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.base.Preconditions.checkNotNull; import com.google.common.annotations.VisibleForTesting; @@ -48,18 +47,7 @@ public final class ExceptionHandler implements Serializable { public interface Interceptor extends Serializable { enum RetryResult { - - RETRY(true), ABORT(false); - - private final boolean booleanValue; - - RetryResult(boolean booleanValue) { - this.booleanValue = booleanValue; - } - - boolean booleanValue() { - return booleanValue; - } + ABORT, RETRY, PROCEED; } /** @@ -68,7 +56,7 @@ boolean booleanValue() { * @param exception the exception that is being evaluated * @return {@link RetryResult} to indicate if the exception should be ignored ( * {@link RetryResult#RETRY}), propagated ({@link RetryResult#ABORT}), or evaluation - * should proceed ({@code null}). + * should proceed ({@link RetryResult#PROCEED}). */ RetryResult beforeEval(Exception exception); @@ -79,7 +67,7 @@ boolean booleanValue() { * @param retryResult the result of the evaluation so far. * @return {@link RetryResult} to indicate if the exception should be ignored ( * {@link RetryResult#RETRY}), propagated ({@link RetryResult#ABORT}), or evaluation - * should proceed ({@code null}). + * should proceed ({@link RetryResult#PROCEED}). */ RetryResult afterEval(Exception exception, RetryResult retryResult); } @@ -157,7 +145,7 @@ static final class RetryInfo implements Serializable { RetryInfo(Class exception, Interceptor.RetryResult retry) { this.exception = checkNotNull(exception); - this.retry = retry; + this.retry = checkNotNull(retry); } @Override @@ -253,18 +241,22 @@ public Set> getNonRetriableExceptions() { boolean shouldRetry(Exception ex) { for (Interceptor interceptor : interceptors) { - Interceptor.RetryResult retryResult = interceptor.beforeEval(ex); - if (retryResult != null) { - return retryResult.booleanValue(); + Interceptor.RetryResult retryResult = checkNotNull(interceptor.beforeEval(ex)); + if (retryResult != Interceptor.RetryResult.PROCEED) { + return interceptor.beforeEval(ex) == Interceptor.RetryResult.RETRY; } } RetryInfo retryInfo = findMostSpecificRetryInfo(this.retryInfo, ex.getClass()); Interceptor.RetryResult retryResult = retryInfo == null ? Interceptor.RetryResult.ABORT : retryInfo.retry; for (Interceptor interceptor : interceptors) { - retryResult = firstNonNull(interceptor.afterEval(ex, retryResult), retryResult); + Interceptor.RetryResult interceptorRetry = + checkNotNull(interceptor.afterEval(ex, retryResult)); + if (interceptorRetry != Interceptor.RetryResult.PROCEED) { + retryResult = interceptorRetry; + } } - return retryResult.booleanValue(); + return retryResult == Interceptor.RetryResult.RETRY; } /** diff --git a/gcloud-java-core/src/test/java/com/google/gcloud/ExceptionHandlerTest.java b/gcloud-java-core/src/test/java/com/google/gcloud/ExceptionHandlerTest.java index 3844f9de36d7..999cd06fac58 100644 --- a/gcloud-java-core/src/test/java/com/google/gcloud/ExceptionHandlerTest.java +++ b/gcloud-java-core/src/test/java/com/google/gcloud/ExceptionHandlerTest.java @@ -158,11 +158,60 @@ public RetryResult beforeEval(Exception exception) { assertTrue(handler.shouldRetry(new RuntimeException())); assertTrue(handler.shouldRetry(new NullPointerException())); - before.set(null); + before.set(RetryResult.PROCEED); assertFalse(handler.shouldRetry(new IOException())); assertTrue(handler.shouldRetry(new ClosedByInterruptException())); assertTrue(handler.shouldRetry(new InterruptedException())); assertTrue(handler.shouldRetry(new RuntimeException())); assertFalse(handler.shouldRetry(new NullPointerException())); } + + @Test + public void testNullRetryResult() { + @SuppressWarnings("serial") + Interceptor interceptor1 = new Interceptor() { + + @Override + public RetryResult beforeEval(Exception exception) { + return null; + } + + @Override + public RetryResult afterEval(Exception exception, RetryResult retryResult) { + return RetryResult.PROCEED; + } + + }; + + @SuppressWarnings("serial") + Interceptor interceptor2 = new Interceptor() { + + @Override + public RetryResult beforeEval(Exception exception) { + return RetryResult.PROCEED; + } + + @Override + public RetryResult afterEval(Exception exception, RetryResult retryResult) { + return null; + } + + }; + + ExceptionHandler handler1 = ExceptionHandler.builder().interceptor(interceptor1).build(); + try { + handler1.shouldRetry(new Exception()); + fail("Expected null pointer exception due to null RetryResult from beforeEval"); + } catch (NullPointerException e) { + // expected + } + + ExceptionHandler handler2 = ExceptionHandler.builder().interceptor(interceptor2).build(); + try { + handler2.shouldRetry(new Exception()); + fail("Expected null pointer exception due to null RetryResult from afterEval"); + } catch (NullPointerException e) { + // expected + } + } } diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreImpl.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreImpl.java index 6f2454c62167..d330c31e706e 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreImpl.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreImpl.java @@ -53,7 +53,7 @@ final class DatastoreImpl extends BaseService @Override public RetryResult afterEval(Exception exception, RetryResult retryResult) { - return null; + return Interceptor.RetryResult.PROCEED; } @Override @@ -62,7 +62,7 @@ public RetryResult beforeEval(Exception exception) { boolean retryable = ((DatastoreRpcException) exception).retryable(); return retryable ? Interceptor.RetryResult.RETRY : Interceptor.RetryResult.ABORT; } - return null; + return Interceptor.RetryResult.PROCEED; } }; private static final ExceptionHandler EXCEPTION_HANDLER = ExceptionHandler.builder() From fa7d7fc0a6cba5a30c727949ab860fefcdc92d82 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Fri, 25 Sep 2015 21:46:58 +0200 Subject: [PATCH 022/156] Minor updates javadoc Blob, Bucket, ListResult --- .../java/com/google/gcloud/storage/Blob.java | 30 ++++++++++--------- .../com/google/gcloud/storage/Bucket.java | 29 +++++++++--------- .../com/google/gcloud/storage/ListResult.java | 4 +-- 3 files changed, 32 insertions(+), 31 deletions(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java index d3fe181970c8..e0863d23a70c 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java @@ -92,7 +92,7 @@ static Storage.BlobSourceOption[] convert(BlobInfo blobInfo, BlobSourceOption... } /** - * Construct a {@code Blob} object for the provided {@code BlobInfo}. The storage service is used + * Constructs a {@code Blob} object for the provided {@code BlobInfo}. The storage service is used * to issue requests. * * @param storage the storage service used for issuing requests @@ -104,7 +104,7 @@ public Blob(Storage storage, BlobInfo info) { } /** - * Construct a {@code Blob} object for the provided bucket and blob names. The storage service is + * Constructs a {@code Blob} object for the provided bucket and blob names. The storage service is * used to issue requests. * * @param storage the storage service used for issuing requests @@ -117,14 +117,14 @@ public Blob(Storage storage, String bucket, String blob) { } /** - * Return the blob's information. + * Returns the blob's information. */ public BlobInfo info() { return info; } /** - * Check if this blob exists. + * Checks if this blob exists. * * @return true if this blob exists, false otherwise * @throws StorageException upon failure @@ -134,7 +134,7 @@ public boolean exists() { } /** - * Return this blob's content. + * Returns this blob's content. * * @param options blob read options * @throws StorageException upon failure @@ -155,12 +155,12 @@ public Blob reload(BlobSourceOption... options) { } /** - * Update the blob's information. Bucket or blob's name cannot be changed by this method. If you + * Updates the blob's information. Bucket or blob's name cannot be changed by this method. If you * want to rename the blob or move it to a different bucket use the {@link #copyTo} and * {@link #delete} operations. A new {@code Blob} object is returned. By default no checks are * made on the metadata generation of the current blob. If you want to update the information only * if the current blob metadata are at their latest version use the {@code metagenerationMatch} - * option: {@code blob.update(newInfo, BlobTargetOption.metagenerationMatch()}. + * option: {@code blob.update(newInfo, BlobTargetOption.metagenerationMatch())}. * * @param blobInfo new blob's information. Bucket and blob names must match the current ones * @param options update options @@ -174,7 +174,7 @@ public Blob update(BlobInfo blobInfo, BlobTargetOption... options) { } /** - * Delete this blob. + * Deletes this blob. * * @param options blob delete options * @return true if blob was deleted @@ -185,7 +185,8 @@ public boolean delete(BlobSourceOption... options) { } /** - * Copy this blob to the target bucket, preserving its name. Possibly update metadata. + * Copies this blob to the target bucket, preserving its name. Possibly copying also some of the + * metadata (e.g. content-type). * * @param targetBucket target bucket's name * @param options source blob options @@ -197,7 +198,8 @@ public Blob copyTo(String targetBucket, BlobSourceOption... options) { } /** - * Copy this blob to the target bucket with a new name. Possibly update metadata. + * Copies this blob to the target bucket with a new name. Possibly copying also some of the + * metadata (e.g. content-type). * * @param targetBucket target bucket's name * @param targetBlob target blob's name @@ -214,7 +216,7 @@ public Blob copyTo(String targetBucket, String targetBlob, BlobSourceOption... o } /** - * Return a {@code BlobReadChannel} object for reading this blob's content. + * Returns a {@code BlobReadChannel} object for reading this blob's content. * * @param options blob read options * @throws StorageException upon failure @@ -224,7 +226,7 @@ public BlobReadChannel reader(BlobSourceOption... options) { } /** - * Return a {@code BlobWriteChannel} object for writing to this blob. + * Returns a {@code BlobWriteChannel} object for writing to this blob. * * @param options target blob options * @throws StorageException upon failure @@ -234,7 +236,7 @@ public BlobWriteChannel writer(BlobTargetOption... options) { } /** - * Generate a signed URL for this blob. If you want to allow access to for a fixed amount of time + * Generates a signed URL for this blob. If you want to allow access to for a fixed amount of time * for this blob, you can use this method to generate a URL that is only valid within a certain * time period. This is particularly useful if you don't want publicly accessible blobs, but don't * want to require users to explicitly log in. @@ -249,7 +251,7 @@ public URL signUrl(long expirationTimeInSeconds, SignUrlOption... options) { } /** - * Return the blob's {@code Storage} object used to issue requests. + * Returns the blob's {@code Storage} object used to issue requests. */ public Storage storage() { return storage; diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java index b56824f069f6..d7cac447d051 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java @@ -18,14 +18,13 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.gcloud.storage.Blob.BlobSourceOption.convert; import com.google.gcloud.storage.Storage.BlobSourceOption; import com.google.gcloud.storage.Storage.BlobTargetOption; import com.google.gcloud.storage.Storage.BucketSourceOption; import com.google.gcloud.storage.Storage.BucketTargetOption; -import java.util.LinkedList; +import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -44,7 +43,7 @@ public final class Bucket { private final BucketInfo info; /** - * Construct a {@code Bucket} object for the provided {@code BucketInfo}. The storage service is + * Constructs a {@code Bucket} object for the provided {@code BucketInfo}. The storage service is * used to issue requests. * * @param storage the storage service used for issuing requests @@ -56,7 +55,7 @@ public Bucket(Storage storage, BucketInfo info) { } /** - * Construct a {@code Bucket} object for the provided name. The storage service is used to issue + * Constructs a {@code Bucket} object for the provided name. The storage service is used to issue * requests. * * @param storage the storage service used for issuing requests @@ -68,14 +67,14 @@ public Bucket(Storage storage, String bucket) { } /** - * Return the bucket's information. + * Returns the bucket's information. */ public BucketInfo info() { return info; } /** - * Check if this bucket exists. + * Checks if this bucket exists. * * @return true if this bucket exists, false otherwise * @throws StorageException upon failure @@ -96,11 +95,11 @@ public Bucket reload(BucketSourceOption... options) { } /** - * Update the bucket's information. Bucket's name cannot be changed. A new {@code Bucket} object + * Updates the bucket's information. Bucket's name cannot be changed. A new {@code Bucket} object * is returned. By default no checks are made on the metadata generation of the current bucket. * If you want to update the information only if the current bucket metadata are at their latest * version use the {@code metagenerationMatch} option: - * {@code bucket.update(newInfo, BucketTargetOption.metagenerationMatch());} + * {@code bucket.update(newInfo, BucketTargetOption.metagenerationMatch())} * * @param bucketInfo new bucket's information. Name must match the one of the current bucket * @param options update options @@ -113,7 +112,7 @@ public Bucket update(BucketInfo bucketInfo, BucketTargetOption... options) { } /** - * Delete this bucket. + * Deletes this bucket. * * @param options bucket delete options * @return true if bucket was deleted @@ -124,7 +123,7 @@ public boolean delete(BucketSourceOption... options) { } /** - * Return the paginated list of {@code Blob} in this bucket. + * Returns the paginated list of {@code Blob} in this bucket. * * @param options options for listing blobs * @throws StorageException upon failure @@ -134,7 +133,7 @@ public ListResult list(Storage.BlobListOption... options) { } /** - * Return the requested blob in this bucket or {@code null} if not found. + * Returns the requested blob in this bucket or {@code null} if not found. * * @param blob name of the requested blob * @param options blob search options @@ -145,7 +144,7 @@ public Blob get(String blob, BlobSourceOption... options) { } /** - * Return a list of requested blobs in this bucket. Blobs that do not exist are null. + * Returns a list of requested blobs in this bucket. Blobs that do not exist are null. * * @param blobNames names of the requested blobs * @throws StorageException upon failure @@ -155,7 +154,7 @@ public List getAll(String... blobNames) { for (String blobName : blobNames) { batch.get(info.name(), blobName); } - List blobs = new LinkedList<>(); + List blobs = new ArrayList<>(blobNames.length); BatchResponse response = storage.apply(batch.build()); for (BatchResponse.Result result : response.gets()) { BlobInfo blobInfo = result.get(); @@ -165,7 +164,7 @@ public List getAll(String... blobNames) { } /** - * Create a new blob in this bucket. + * Creates a new blob in this bucket. * * @param blob a blob name * @param content the blob content @@ -179,7 +178,7 @@ Blob create(String blob, byte[] content, BlobTargetOption... options) { } /** - * Return the bucket's {@code Storage} object used to issue requests. + * Returns the bucket's {@code Storage} object used to issue requests. */ public Storage storage() { return storage; diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/ListResult.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/ListResult.java index a07d6edff745..62b1f442310c 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/ListResult.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/ListResult.java @@ -22,12 +22,12 @@ public interface ListResult extends Iterable { /** - * Return the cursor for the nextPage or {@code null} if no more results. + * Returns the cursor for the nextPage or {@code null} if no more results. */ String nextPageCursor(); /** - * Return the results of the nextPage or {@code null} if no more result. + * Returns the results of the nextPage or {@code null} if no more result. */ ListResult nextPage(); From 800b7090409203833b61cfaf4ec227eb26e1d82a Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Fri, 25 Sep 2015 14:06:17 -0700 Subject: [PATCH 023/156] Rename RetryResult enum values, and update datastore and storage interceptor. --- .../com/google/gcloud/ExceptionHandler.java | 20 ++++---- .../google/gcloud/ExceptionHandlerTest.java | 46 +++++++++---------- .../gcloud/datastore/DatastoreImpl.java | 6 +-- .../google/gcloud/storage/StorageImpl.java | 6 +-- 4 files changed, 39 insertions(+), 39 deletions(-) diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/ExceptionHandler.java b/gcloud-java-core/src/main/java/com/google/gcloud/ExceptionHandler.java index 9e472965cb6e..a0fab3dca566 100644 --- a/gcloud-java-core/src/main/java/com/google/gcloud/ExceptionHandler.java +++ b/gcloud-java-core/src/main/java/com/google/gcloud/ExceptionHandler.java @@ -47,7 +47,7 @@ public final class ExceptionHandler implements Serializable { public interface Interceptor extends Serializable { enum RetryResult { - ABORT, RETRY, PROCEED; + NO_RETRY, RETRY, CONTINUE_EVALUATION; } /** @@ -55,8 +55,8 @@ enum RetryResult { * * @param exception the exception that is being evaluated * @return {@link RetryResult} to indicate if the exception should be ignored ( - * {@link RetryResult#RETRY}), propagated ({@link RetryResult#ABORT}), or evaluation - * should proceed ({@link RetryResult#PROCEED}). + * {@link RetryResult#RETRY}), propagated ({@link RetryResult#NO_RETRY}), or evaluation + * should proceed ({@link RetryResult#CONTINUE_EVALUATION}). */ RetryResult beforeEval(Exception exception); @@ -66,8 +66,8 @@ enum RetryResult { * @param exception the exception that is being evaluated * @param retryResult the result of the evaluation so far. * @return {@link RetryResult} to indicate if the exception should be ignored ( - * {@link RetryResult#RETRY}), propagated ({@link RetryResult#ABORT}), or evaluation - * should proceed ({@link RetryResult#PROCEED}). + * {@link RetryResult#RETRY}), propagated ({@link RetryResult#NO_RETRY}), or evaluation + * should proceed ({@link RetryResult#CONTINUE_EVALUATION}). */ RetryResult afterEval(Exception exception, RetryResult retryResult); } @@ -177,7 +177,7 @@ private ExceptionHandler(Builder builder) { addRetryInfo(new RetryInfo(exception, Interceptor.RetryResult.RETRY), retryInfo); } for (Class exception : nonRetriableExceptions) { - addRetryInfo(new RetryInfo(exception, Interceptor.RetryResult.ABORT), retryInfo); + addRetryInfo(new RetryInfo(exception, Interceptor.RetryResult.NO_RETRY), retryInfo); } } @@ -242,17 +242,17 @@ public Set> getNonRetriableExceptions() { boolean shouldRetry(Exception ex) { for (Interceptor interceptor : interceptors) { Interceptor.RetryResult retryResult = checkNotNull(interceptor.beforeEval(ex)); - if (retryResult != Interceptor.RetryResult.PROCEED) { - return interceptor.beforeEval(ex) == Interceptor.RetryResult.RETRY; + if (retryResult != Interceptor.RetryResult.CONTINUE_EVALUATION) { + return retryResult == Interceptor.RetryResult.RETRY; } } RetryInfo retryInfo = findMostSpecificRetryInfo(this.retryInfo, ex.getClass()); Interceptor.RetryResult retryResult = - retryInfo == null ? Interceptor.RetryResult.ABORT : retryInfo.retry; + retryInfo == null ? Interceptor.RetryResult.NO_RETRY : retryInfo.retry; for (Interceptor interceptor : interceptors) { Interceptor.RetryResult interceptorRetry = checkNotNull(interceptor.afterEval(ex, retryResult)); - if (interceptorRetry != Interceptor.RetryResult.PROCEED) { + if (interceptorRetry != Interceptor.RetryResult.CONTINUE_EVALUATION) { retryResult = interceptorRetry; } } diff --git a/gcloud-java-core/src/test/java/com/google/gcloud/ExceptionHandlerTest.java b/gcloud-java-core/src/test/java/com/google/gcloud/ExceptionHandlerTest.java index 999cd06fac58..5ce05ad900a8 100644 --- a/gcloud-java-core/src/test/java/com/google/gcloud/ExceptionHandlerTest.java +++ b/gcloud-java-core/src/test/java/com/google/gcloud/ExceptionHandlerTest.java @@ -23,6 +23,8 @@ import com.google.gcloud.ExceptionHandler.Interceptor; import com.google.gcloud.ExceptionHandler.Interceptor.RetryResult; +import org.junit.rules.ExpectedException; +import org.junit.Rule; import org.junit.Test; import java.io.FileNotFoundException; @@ -36,6 +38,9 @@ */ public class ExceptionHandlerTest { + @Rule + public ExpectedException thrown = ExpectedException.none(); + @Test public void testVerifyCaller() { class A implements Callable { @@ -128,13 +133,13 @@ public void testShouldTry() { assertFalse(handler.shouldRetry(new RuntimeException())); assertTrue(handler.shouldRetry(new NullPointerException())); - final AtomicReference before = new AtomicReference<>(RetryResult.ABORT); + final AtomicReference before = new AtomicReference<>(RetryResult.NO_RETRY); @SuppressWarnings("serial") Interceptor interceptor = new Interceptor() { @Override public RetryResult afterEval(Exception exception, RetryResult retryResult) { - return retryResult == RetryResult.ABORT ? RetryResult.RETRY : RetryResult.ABORT; + return retryResult == RetryResult.NO_RETRY ? RetryResult.RETRY : RetryResult.NO_RETRY; } @Override @@ -158,7 +163,7 @@ public RetryResult beforeEval(Exception exception) { assertTrue(handler.shouldRetry(new RuntimeException())); assertTrue(handler.shouldRetry(new NullPointerException())); - before.set(RetryResult.PROCEED); + before.set(RetryResult.CONTINUE_EVALUATION); assertFalse(handler.shouldRetry(new IOException())); assertTrue(handler.shouldRetry(new ClosedByInterruptException())); assertTrue(handler.shouldRetry(new InterruptedException())); @@ -167,9 +172,9 @@ public RetryResult beforeEval(Exception exception) { } @Test - public void testNullRetryResult() { + public void testNullRetryResultFromBeforeEval() { @SuppressWarnings("serial") - Interceptor interceptor1 = new Interceptor() { + Interceptor interceptor = new Interceptor() { @Override public RetryResult beforeEval(Exception exception) { @@ -178,17 +183,24 @@ public RetryResult beforeEval(Exception exception) { @Override public RetryResult afterEval(Exception exception, RetryResult retryResult) { - return RetryResult.PROCEED; + return RetryResult.CONTINUE_EVALUATION; } }; + ExceptionHandler handler = ExceptionHandler.builder().interceptor(interceptor).build(); + thrown.expect(NullPointerException.class); + handler.shouldRetry(new Exception()); + } + + @Test + public void testNullRetryResultFromAfterEval() { @SuppressWarnings("serial") - Interceptor interceptor2 = new Interceptor() { + Interceptor interceptor = new Interceptor() { @Override public RetryResult beforeEval(Exception exception) { - return RetryResult.PROCEED; + return RetryResult.CONTINUE_EVALUATION; } @Override @@ -198,20 +210,8 @@ public RetryResult afterEval(Exception exception, RetryResult retryResult) { }; - ExceptionHandler handler1 = ExceptionHandler.builder().interceptor(interceptor1).build(); - try { - handler1.shouldRetry(new Exception()); - fail("Expected null pointer exception due to null RetryResult from beforeEval"); - } catch (NullPointerException e) { - // expected - } - - ExceptionHandler handler2 = ExceptionHandler.builder().interceptor(interceptor2).build(); - try { - handler2.shouldRetry(new Exception()); - fail("Expected null pointer exception due to null RetryResult from afterEval"); - } catch (NullPointerException e) { - // expected - } + ExceptionHandler handler = ExceptionHandler.builder().interceptor(interceptor).build(); + thrown.expect(NullPointerException.class); + handler.shouldRetry(new Exception()); } } diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreImpl.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreImpl.java index d330c31e706e..c12ffc8a032d 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreImpl.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreImpl.java @@ -53,16 +53,16 @@ final class DatastoreImpl extends BaseService @Override public RetryResult afterEval(Exception exception, RetryResult retryResult) { - return Interceptor.RetryResult.PROCEED; + return Interceptor.RetryResult.CONTINUE_EVALUATION; } @Override public RetryResult beforeEval(Exception exception) { if (exception instanceof DatastoreRpcException) { boolean retryable = ((DatastoreRpcException) exception).retryable(); - return retryable ? Interceptor.RetryResult.RETRY : Interceptor.RetryResult.ABORT; + return retryable ? Interceptor.RetryResult.RETRY : Interceptor.RetryResult.NO_RETRY; } - return Interceptor.RetryResult.PROCEED; + return Interceptor.RetryResult.CONTINUE_EVALUATION; } }; private static final ExceptionHandler EXCEPTION_HANDLER = ExceptionHandler.builder() diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java index f59c6c670969..403f8a7ab78a 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java @@ -72,16 +72,16 @@ final class StorageImpl extends BaseService implements Storage { @Override public RetryResult afterEval(Exception exception, RetryResult retryResult) { - return null; + return Interceptor.RetryResult.CONTINUE_EVALUATION; } @Override public RetryResult beforeEval(Exception exception) { if (exception instanceof StorageException) { boolean retriable = ((StorageException) exception).retryable(); - return retriable ? Interceptor.RetryResult.RETRY : Interceptor.RetryResult.ABORT; + return retriable ? Interceptor.RetryResult.RETRY : Interceptor.RetryResult.NO_RETRY; } - return null; + return Interceptor.RetryResult.CONTINUE_EVALUATION; } }; static final ExceptionHandler EXCEPTION_HANDLER = ExceptionHandler.builder() From b5b43d9bd0cbf68e6ae38b282e32dc679fe8cc36 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Mon, 28 Sep 2015 09:16:53 +0200 Subject: [PATCH 024/156] Fix NPE when listing buckets in empty project and blobs in empty bucket --- .../google/gcloud/storage/StorageImpl.java | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java index 57f697f3e585..4c80c6741559 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java @@ -34,6 +34,7 @@ import com.google.api.services.storage.model.StorageObject; import com.google.common.base.Function; import com.google.common.base.Functions; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; @@ -234,14 +235,16 @@ public Tuple> cal } }, serviceOptions.retryParams(), EXCEPTION_HANDLER); String cursor = result.x(); - return new BaseListResult<>(new BucketPageFetcher(serviceOptions, cursor, optionsMap), cursor, - Iterables.transform(result.y(), + Iterable buckets = + result.y() == null ? ImmutableList.of() : Iterables.transform(result.y(), new Function() { @Override public BucketInfo apply(com.google.api.services.storage.model.Bucket bucketPb) { return BucketInfo.fromPb(bucketPb); } - })); + }); + return new BaseListResult<>(new BucketPageFetcher(serviceOptions, cursor, optionsMap), cursor, + buckets); } @Override @@ -259,14 +262,17 @@ public Tuple> call() { } }, serviceOptions.retryParams(), EXCEPTION_HANDLER); String cursor = result.x(); - return new BaseListResult<>(new BlobPageFetcher(bucket, serviceOptions, cursor, optionsMap), cursor, - Iterables.transform(result.y(), + Iterable blobs = + result.y() == null ? ImmutableList.of() : Iterables.transform(result.y(), new Function() { @Override public BlobInfo apply(StorageObject storageObject) { return BlobInfo.fromPb(storageObject); } - })); + }); + return new BaseListResult<>(new BlobPageFetcher(bucket, serviceOptions, cursor, optionsMap), + cursor, + blobs); } @Override @@ -434,7 +440,7 @@ public BlobReadChannel reader(String bucket, String blob, BlobSourceOption... op return new BlobReadChannelImpl(options(), BlobInfo.of(bucket, blob), optionsMap); } - @Override + @Override public BlobWriteChannel writer(BlobInfo blobInfo, BlobTargetOption... options) { final Map optionsMap = optionMap(blobInfo, options); return new BlobWriterChannelImpl(options(), blobInfo, optionsMap); @@ -461,12 +467,12 @@ public URL signUrl(BlobInfo blobInfo, long expiration, SignUrlOption... options) stBuilder.append(HttpMethod.GET); } stBuilder.append('\n'); - if (firstNonNull((Boolean) optionMap.get(SignUrlOption.Option.MD5) , false)) { + if (firstNonNull((Boolean) optionMap.get(SignUrlOption.Option.MD5), false)) { checkArgument(blobInfo.md5() != null, "Blob is missing a value for md5"); stBuilder.append(blobInfo.md5()); } stBuilder.append('\n'); - if (firstNonNull((Boolean) optionMap.get(SignUrlOption.Option.CONTENT_TYPE) , false)) { + if (firstNonNull((Boolean) optionMap.get(SignUrlOption.Option.CONTENT_TYPE), false)) { checkArgument(blobInfo.contentType() != null, "Blob is missing a value for content-type"); stBuilder.append(blobInfo.contentType()); } From 90dd0709ee844da91e19dca2adce75ff219fa29e Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Mon, 28 Sep 2015 10:13:20 +0200 Subject: [PATCH 025/156] Add null checks to fromPb in BucketInfo and BlobInfo, updated tests --- .../com/google/gcloud/storage/BlobInfo.java | 60 +++++++++++++------ .../com/google/gcloud/storage/BucketInfo.java | 27 ++++++--- .../google/gcloud/storage/BlobInfoTest.java | 2 + .../google/gcloud/storage/BucketInfoTest.java | 14 +++-- 4 files changed, 73 insertions(+), 30 deletions(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java index 7536f96e20f8..6c05c427ef6b 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java @@ -441,23 +441,49 @@ public static Builder builder(String bucket, String name) { } static BlobInfo fromPb(StorageObject storageObject) { - Builder builder = new Builder() - .bucket(storageObject.getBucket()) - .cacheControl(storageObject.getCacheControl()) - .contentEncoding(storageObject.getContentEncoding()) - .crc32c(storageObject.getCrc32c()) - .contentType(storageObject.getContentType()) - .generation(storageObject.getGeneration()) - .md5(storageObject.getMd5Hash()) - .mediaLink(storageObject.getMediaLink()) - .metageneration(storageObject.getMetageneration()) - .name(storageObject.getName()) - .contentDisposition(storageObject.getContentDisposition()) - .componentCount(storageObject.getComponentCount()) - .contentLanguage(storageObject.getContentLanguage()) - .etag(storageObject.getEtag()) - .id(storageObject.getId()) - .selfLink(storageObject.getSelfLink()); + Builder builder = new Builder().bucket(storageObject.getBucket()).name(storageObject.getName()); + if (storageObject.getCacheControl() != null) { + builder.cacheControl(storageObject.getCacheControl()); + } + if (storageObject.getContentEncoding() != null) { + builder.contentEncoding(storageObject.getContentEncoding()); + } + if (storageObject.getCrc32c() != null) { + builder.crc32c(storageObject.getCrc32c()); + } + if (storageObject.getContentType() != null) { + builder.contentType(storageObject.getContentType()); + } + if (storageObject.getGeneration() != null) { + builder.generation(storageObject.getGeneration()); + } + if (storageObject.getMd5Hash() != null) { + builder.md5(storageObject.getMd5Hash()); + } + if (storageObject.getMediaLink() != null) { + builder.mediaLink(storageObject.getMediaLink()); + } + if (storageObject.getMetageneration() != null) { + builder.metageneration(storageObject.getMetageneration()); + } + if (storageObject.getContentDisposition() != null) { + builder.contentDisposition(storageObject.getContentDisposition()); + } + if (storageObject.getComponentCount() != null) { + builder.componentCount(storageObject.getComponentCount()); + } + if (storageObject.getContentLanguage() != null) { + builder.contentLanguage(storageObject.getContentLanguage()); + } + if (storageObject.getEtag() != null) { + builder.etag(storageObject.getEtag()); + } + if (storageObject.getId() != null) { + builder.id(storageObject.getId()); + } + if (storageObject.getSelfLink() != null) { + builder.selfLink(storageObject.getSelfLink()); + } if (storageObject.getMetadata() != null) { builder.metadata(storageObject.getMetadata()); } diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java index 3f35fbc20e3a..ddd4665ca129 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java @@ -692,14 +692,25 @@ public static Builder builder(String name) { } static BucketInfo fromPb(com.google.api.services.storage.model.Bucket bucketPb) { - Builder builder = new Builder() - .name(bucketPb.getName()) - .id(bucketPb.getId()) - .etag(bucketPb.getEtag()) - .metageneration(bucketPb.getMetageneration()) - .createTime(bucketPb.getTimeCreated().getValue()) - .location(Location.of(bucketPb.getLocation())) - .selfLink(bucketPb.getSelfLink()); + Builder builder = new Builder().name(bucketPb.getName()); + if (bucketPb.getId() != null) { + builder.id(bucketPb.getId()); + } + if (bucketPb.getEtag() != null) { + builder.etag(bucketPb.getEtag()); + } + if (bucketPb.getMetageneration() != null) { + builder.metageneration(bucketPb.getMetageneration()); + } + if (bucketPb.getSelfLink() != null) { + builder.selfLink(bucketPb.getSelfLink()); + } + if (bucketPb.getTimeCreated() != null) { + builder.createTime(bucketPb.getTimeCreated().getValue()); + } + if (bucketPb.getLocation() != null) { + builder.location(Location.of(bucketPb.getLocation())); + } if (bucketPb.getStorageClass() != null) { builder.storageClass(StorageClass.of(bucketPb.getStorageClass())); } diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobInfoTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobInfoTest.java index 018a59c9cfc4..bd5ac476c318 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobInfoTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobInfoTest.java @@ -151,5 +151,7 @@ private void compareBlobs(BlobInfo expected, BlobInfo value) { @Test public void testToPbAndFromPb() { compareBlobs(BLOB_INFO, BlobInfo.fromPb(BLOB_INFO.toPb())); + BlobInfo blobInfo = BlobInfo.of("b", "n"); + compareBlobs(blobInfo, BlobInfo.fromPb(blobInfo.toPb())); } } diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketInfoTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketInfoTest.java index c4f08fbd8b9b..09ba0e8cda8e 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketInfoTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketInfoTest.java @@ -125,6 +125,8 @@ public void testBuilder() { @Test public void testToPbAndFromPb() { compareBuckets(BUCKET_INFO, BucketInfo.fromPb(BUCKET_INFO.toPb())); + BucketInfo bucketInfo = BucketInfo.of("b"); + compareBuckets(bucketInfo, BucketInfo.fromPb(bucketInfo.toPb())); } private void compareBuckets(BucketInfo expected, BucketInfo value) { @@ -147,21 +149,23 @@ private void compareBuckets(BucketInfo expected, BucketInfo value) { assertEquals(expected.versioningEnabled(), value.versioningEnabled()); } + @Test public void testLocation() { assertEquals("ASIA", Location.asia().value()); - assertEquals("EN", Location.eu().value()); + assertEquals("EU", Location.eu().value()); assertEquals("US", Location.us().value()); assertSame(Location.asia(), Location.of("asia")); - assertSame(Location.asia(), Location.of("EU")); - assertSame(Location.asia(), Location.of("uS")); + assertSame(Location.eu(), Location.of("EU")); + assertSame(Location.us(), Location.of("uS")); } + @Test public void testDeleteRules() { AgeDeleteRule ageRule = new AgeDeleteRule(10); assertEquals(10, ageRule.daysToLive()); assertEquals(Type.AGE, ageRule.type()); CreatedBeforeDeleteRule createBeforeRule = new CreatedBeforeDeleteRule(1); - assertEquals(10, createBeforeRule.timeMillis()); + assertEquals(1, createBeforeRule.timeMillis()); assertEquals(Type.CREATE_BEFORE, createBeforeRule.type()); NumNewerVersionsDeleteRule versionsRule = new NumNewerVersionsDeleteRule(2); assertEquals(2, versionsRule.numNewerVersions()); @@ -171,7 +175,7 @@ public void testDeleteRules() { assertEquals(Type.IS_LIVE, isLiveRule.type()); Rule rule = new Rule().set("a", "b"); RawDeleteRule rawRule = new RawDeleteRule(rule); - assertEquals(Type.UNKNOWN, isLiveRule.type()); + assertEquals(Type.UNKNOWN, rawRule.type()); ImmutableList rules = ImmutableList .of(ageRule, createBeforeRule, versionsRule, isLiveRule, rawRule); for (DeleteRule delRule : rules) { From 93b7a0c7edc39f794582bce2c01cc81b9b810b75 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Mon, 28 Sep 2015 21:47:01 +0200 Subject: [PATCH 026/156] createFromStream added to Storage, StorageRpc.create uses InputStream --- .../com/google/gcloud/spi/DefaultStorageRpc.java | 12 +++++++----- .../main/java/com/google/gcloud/spi/StorageRpc.java | 3 ++- .../main/java/com/google/gcloud/storage/Storage.java | 9 +++++++++ .../java/com/google/gcloud/storage/StorageImpl.java | 12 +++++++++++- 4 files changed, 29 insertions(+), 7 deletions(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index 7c2d3a7be21a..a3e79333085d 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -43,6 +43,7 @@ import com.google.api.client.http.HttpResponse; import com.google.api.client.http.HttpResponseException; import com.google.api.client.http.HttpTransport; +import com.google.api.client.http.InputStreamContent; import com.google.api.client.http.json.JsonHttpContent; import com.google.api.client.json.JsonFactory; import com.google.api.client.json.jackson.JacksonFactory; @@ -63,6 +64,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -119,13 +121,14 @@ public Bucket create(Bucket bucket, Map options) throws StorageExcept } @Override - public StorageObject create(StorageObject storageObject, final byte[] content, + public StorageObject create(StorageObject storageObject, final InputStream content, Map options) throws StorageException { try { - return storage.objects() + Storage.Objects.Insert insert = storage.objects() .insert(storageObject.getBucket(), storageObject, - new ByteArrayContent(storageObject.getContentType(), content)) - .setProjection(DEFAULT_PROJECTION) + new InputStreamContent(storageObject.getContentType(), content)); + insert.getMediaHttpUploader().setDirectUploadEnabled(true); + return insert.setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) @@ -521,4 +524,3 @@ public String open(StorageObject object, Map options) } } } - diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java index 7448fec9d32f..423929adff08 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableMap; import com.google.gcloud.storage.StorageException; +import java.io.InputStream; import java.util.List; import java.util.Map; @@ -128,7 +129,7 @@ public BatchResponse(Map> delete Bucket create(Bucket bucket, Map options) throws StorageException; - StorageObject create(StorageObject object, byte[] content, Map options) + StorageObject create(StorageObject object, InputStream content, Map options) throws StorageException; Tuple> list(Map options) throws StorageException; diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java index 590c8d9d74d4..900ba721b7ce 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java @@ -25,6 +25,7 @@ import com.google.gcloud.Service; import com.google.gcloud.spi.StorageRpc; +import java.io.InputStream; import java.io.Serializable; import java.net.URL; import java.util.Arrays; @@ -500,6 +501,14 @@ public static Builder builder() { */ BlobInfo create(BlobInfo blobInfo, byte[] content, BlobTargetOption... options); + /** + * Create a new blob. + * + * @return a complete blob information. + * @throws StorageException upon failure + */ + BlobInfo createFromStream(BlobInfo blobInfo, InputStream content, BlobTargetOption... options); + /** * Return the requested bucket or {@code null} if not found. * diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java index 4c80c6741559..dca844b83034 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java @@ -49,6 +49,8 @@ import com.google.gcloud.spi.StorageRpc; import com.google.gcloud.spi.StorageRpc.Tuple; +import java.io.ByteArrayInputStream; +import java.io.InputStream; import java.io.Serializable; import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; @@ -114,12 +116,20 @@ public com.google.api.services.storage.model.Bucket call() { @Override public BlobInfo create(BlobInfo blobInfo, final byte[] content, BlobTargetOption... options) { + return createFromStream(blobInfo, + new ByteArrayInputStream(firstNonNull(content, EMPTY_BYTE_ARRAY)), options); + } + + @Override + public BlobInfo createFromStream(BlobInfo blobInfo, final InputStream content, + BlobTargetOption... options) { final StorageObject blobPb = blobInfo.toPb(); final Map optionsMap = optionMap(blobInfo, options); return BlobInfo.fromPb(runWithRetries(new Callable() { @Override public StorageObject call() { - return storageRpc.create(blobPb, firstNonNull(content, EMPTY_BYTE_ARRAY), optionsMap); + return storageRpc.create(blobPb, + firstNonNull(content, new ByteArrayInputStream(EMPTY_BYTE_ARRAY)), optionsMap); } }, options().retryParams(), EXCEPTION_HANDLER)); } From 575fa1f50ee34aa68d277cd4b09a054982b871cc Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Tue, 29 Sep 2015 12:18:58 +0200 Subject: [PATCH 027/156] Add test for blob channels, remove duplicated call in BlobWriterChannelImpl --- .../gcloud/storage/BlobWriterChannelImpl.java | 2 +- .../gcloud/storage/BlobReadChannelTest.java | 194 ++++++++++++++++++ .../gcloud/storage/BlobWriterChannelTest.java | 166 +++++++++++++++ 3 files changed, 361 insertions(+), 1 deletion(-) create mode 100644 gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobReadChannelTest.java create mode 100644 gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobWriterChannelTest.java diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java index 1eefe87702e6..612e5d7551dc 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java @@ -55,7 +55,7 @@ class BlobWriterChannelImpl implements BlobWriteChannel { this.options = options; this.blobInfo = blobInfo; initTransients(); - uploadId = options.storageRpc().open(storageObject, optionsMap); + uploadId = storageRpc.open(storageObject, optionsMap); } private void writeObject(ObjectOutputStream out) throws IOException { diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobReadChannelTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobReadChannelTest.java new file mode 100644 index 000000000000..f54c02c6eb83 --- /dev/null +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobReadChannelTest.java @@ -0,0 +1,194 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.storage; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import com.google.common.collect.ImmutableMap; +import com.google.gcloud.RetryParams; +import com.google.gcloud.spi.StorageRpc; + +import org.easymock.EasyMock; +import org.junit.Test; +import org.junit.Before; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Map; +import java.util.Random; + +public class BlobReadChannelTest { + + private static final String BUCKET_NAME = "b"; + private static final String BLOB_NAME = "n"; + private static final BlobInfo BLOB_INFO = BlobInfo.of(BUCKET_NAME, BLOB_NAME); + private static final Map EMPTY_RPC_OPTIONS = ImmutableMap.of(); + private static final int DEFAULT_CHUNK_SIZE = 2 * 1024 * 1024; + private static final int CUSTOM_CHUNK_SIZE = 2 * 1024 * 1024; + private static final Random RANDOM = new Random(); + + private StorageOptions optionsMock; + private StorageRpc storageRpcMock; + private BlobReadChannelImpl reader; + + @Before + public void setUp() throws IOException, InterruptedException { + optionsMock = EasyMock.createMock(StorageOptions.class); + storageRpcMock = EasyMock.createMock(StorageRpc.class); + } + + @Test + public void testCreate() { + EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock); + EasyMock.replay(optionsMock); + reader = new BlobReadChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); + assertTrue(reader.isOpen()); + } + + @Test + public void testReadSmall() throws IOException { + EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock); + EasyMock.expect(optionsMock.retryParams()).andReturn(RetryParams.noRetries()); + EasyMock.replay(optionsMock); + reader = new BlobReadChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); + byte[] result = randomByteArray(DEFAULT_CHUNK_SIZE); + ByteBuffer readBuffer = ByteBuffer.allocate(42); + EasyMock + .expect(storageRpcMock.read(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS, 0, DEFAULT_CHUNK_SIZE)) + .andReturn(result); + EasyMock.replay(storageRpcMock); + reader.read(readBuffer); + assertArrayEquals(Arrays.copyOf(result, readBuffer.capacity()), readBuffer.array()); + } + + @Test + public void testReadBuffered() throws IOException { + EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock); + EasyMock.expect(optionsMock.retryParams()).andReturn(RetryParams.noRetries()); + EasyMock.replay(optionsMock); + reader = new BlobReadChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); + byte[] result = randomByteArray(DEFAULT_CHUNK_SIZE); + ByteBuffer firstReadBuffer = ByteBuffer.allocate(42); + ByteBuffer secondReadBuffer = ByteBuffer.allocate(42); + EasyMock + .expect(storageRpcMock.read(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS, 0, DEFAULT_CHUNK_SIZE)) + .andReturn(result); + EasyMock.replay(storageRpcMock); + reader.read(firstReadBuffer); + reader.read(secondReadBuffer); + assertArrayEquals(Arrays.copyOf(result, firstReadBuffer.capacity()), firstReadBuffer.array()); + assertArrayEquals( + Arrays.copyOfRange(result, firstReadBuffer.capacity(), firstReadBuffer.capacity() + + secondReadBuffer.capacity()), + secondReadBuffer.array()); + } + + @Test + public void testReadBig() throws IOException { + EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock); + EasyMock.expect(optionsMock.retryParams()).andReturn(RetryParams.noRetries()).times(2); + EasyMock.replay(optionsMock); + reader = new BlobReadChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); + reader.chunkSize(CUSTOM_CHUNK_SIZE); + byte[] firstResult = randomByteArray(DEFAULT_CHUNK_SIZE); + byte[] secondResult = randomByteArray(DEFAULT_CHUNK_SIZE); + ByteBuffer firstReadBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE); + ByteBuffer secondReadBuffer = ByteBuffer.allocate(42); + EasyMock + .expect(storageRpcMock.read(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS, 0, DEFAULT_CHUNK_SIZE)) + .andReturn(firstResult); + EasyMock + .expect( + storageRpcMock.read(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS, DEFAULT_CHUNK_SIZE, + CUSTOM_CHUNK_SIZE)) + .andReturn(secondResult); + EasyMock.expectLastCall(); + EasyMock.replay(storageRpcMock); + reader.read(firstReadBuffer); + reader.read(secondReadBuffer); + assertArrayEquals(firstResult, firstReadBuffer.array()); + assertArrayEquals(Arrays.copyOf(secondResult, secondReadBuffer.capacity()), + secondReadBuffer.array()); + } + + @Test + public void testReadFinish() throws IOException { + EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock); + EasyMock.expect(optionsMock.retryParams()).andReturn(RetryParams.noRetries()); + EasyMock.replay(optionsMock); + reader = new BlobReadChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); + byte[] result = {}; + ByteBuffer readBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE); + EasyMock + .expect(storageRpcMock.read(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS, 0, DEFAULT_CHUNK_SIZE)) + .andReturn(result); + EasyMock.replay(storageRpcMock); + assertEquals(-1, reader.read(readBuffer)); + } + + @Test + public void testSeek() throws IOException { + EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock); + EasyMock.expect(optionsMock.retryParams()).andReturn(RetryParams.noRetries()); + EasyMock.replay(optionsMock); + reader = new BlobReadChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); + reader.seek(42); + byte[] result = randomByteArray(DEFAULT_CHUNK_SIZE); + ByteBuffer readBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE); + EasyMock + .expect(storageRpcMock.read(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS, 42, DEFAULT_CHUNK_SIZE)) + .andReturn(result); + EasyMock.replay(storageRpcMock); + reader.read(readBuffer); + assertArrayEquals(result, readBuffer.array()); + } + + @Test + public void testClose() throws IOException { + EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock); + EasyMock.replay(optionsMock); + reader = new BlobReadChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); + assertTrue(reader.isOpen()); + reader.close(); + assertTrue(!reader.isOpen()); + } + + @Test + public void testReadClosed() { + EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock); + EasyMock.replay(optionsMock); + reader = new BlobReadChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); + reader.close(); + try { + ByteBuffer readBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE); + reader.read(readBuffer); + fail("Expected BlobReadChannel read to throw IOException"); + } catch (IOException ex) { + // expected + } + } + + private static byte[] randomByteArray(int size) { + byte[] byteArray = new byte[size]; + RANDOM.nextBytes(byteArray); + return byteArray; + } +} diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobWriterChannelTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobWriterChannelTest.java new file mode 100644 index 000000000000..65997bf86228 --- /dev/null +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobWriterChannelTest.java @@ -0,0 +1,166 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.storage; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.common.collect.ImmutableMap; +import com.google.gcloud.RetryParams; +import com.google.gcloud.spi.StorageRpc; + +import org.easymock.Capture; +import org.easymock.EasyMock; +import org.junit.Test; +import org.junit.Before; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Map; +import java.util.Random; + +public class BlobWriterChannelTest { + + private static final String BUCKET_NAME = "b"; + private static final String BLOB_NAME = "n"; + private static final String UPLOAD_ID = "uploadid"; + private static final BlobInfo BLOB_INFO = BlobInfo.of(BUCKET_NAME, BLOB_NAME); + private static final Map EMPTY_RPC_OPTIONS = ImmutableMap.of(); + private static final int MIN_CHUNK_SIZE = 256 * 1024; + private static final int DEFAULT_CHUNK_SIZE = 8 * MIN_CHUNK_SIZE; + private static final int CUSTOM_CHUNK_SIZE = 4 * MIN_CHUNK_SIZE; + private static final Random RANDOM = new Random(); + + private StorageOptions optionsMock; + private StorageRpc storageRpcMock; + private BlobWriterChannelImpl writer; + + @Before + public void setUp() throws IOException, InterruptedException { + optionsMock = EasyMock.createMock(StorageOptions.class); + storageRpcMock = EasyMock.createMock(StorageRpc.class); + } + + @Test + public void testCreate() { + EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock); + EasyMock.replay(optionsMock); + EasyMock.expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); + EasyMock.replay(storageRpcMock); + writer = new BlobWriterChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); + assertTrue(writer.isOpen()); + } + + @Test + public void testWriteWithoutFlush() throws IOException { + EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock); + EasyMock.replay(optionsMock); + EasyMock.expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); + EasyMock.replay(storageRpcMock); + writer = new BlobWriterChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); + assertEquals(MIN_CHUNK_SIZE, writer.write(ByteBuffer.allocate(MIN_CHUNK_SIZE))); + } + + @Test + public void testWriteWithFlush() throws IOException { + EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock); + EasyMock.expect(optionsMock.retryParams()).andReturn(RetryParams.noRetries()); + EasyMock.replay(optionsMock); + EasyMock.expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); + Capture capturedBuffer = Capture.newInstance(); + storageRpcMock.write(EasyMock.eq(UPLOAD_ID), EasyMock.capture(capturedBuffer), EasyMock.eq(0), + EasyMock.eq(BLOB_INFO.toPb()), EasyMock.eq(0L), EasyMock.eq(CUSTOM_CHUNK_SIZE), + EasyMock.eq(false)); + EasyMock.expectLastCall(); + EasyMock.replay(storageRpcMock); + writer = new BlobWriterChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); + writer.chunkSize(CUSTOM_CHUNK_SIZE); + ByteBuffer buffer = randomBuffer(CUSTOM_CHUNK_SIZE); + assertEquals(CUSTOM_CHUNK_SIZE, writer.write(buffer)); + assertArrayEquals(buffer.array(), capturedBuffer.getValue()); + } + + @Test + public void testCloseWithoutFlush() throws IOException { + EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock); + EasyMock.expect(optionsMock.retryParams()).andReturn(RetryParams.noRetries()); + EasyMock.replay(optionsMock); + EasyMock.expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); + Capture capturedBuffer = Capture.newInstance(); + storageRpcMock.write(EasyMock.eq(UPLOAD_ID), EasyMock.capture(capturedBuffer), EasyMock.eq(0), + EasyMock.eq(BLOB_INFO.toPb()), EasyMock.eq(0L), EasyMock.eq(0), EasyMock.eq(true)); + EasyMock.expectLastCall(); + EasyMock.replay(storageRpcMock); + writer = new BlobWriterChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); + assertTrue(writer.isOpen()); + writer.close(); + assertArrayEquals(new byte[0], capturedBuffer.getValue()); + assertTrue(!writer.isOpen()); + } + + @Test + public void testCloseWithFlush() throws IOException { + EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock); + EasyMock.expect(optionsMock.retryParams()).andReturn(RetryParams.noRetries()); + EasyMock.replay(optionsMock); + EasyMock.expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); + Capture capturedBuffer = Capture.newInstance(); + ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE); + storageRpcMock.write(EasyMock.eq(UPLOAD_ID), EasyMock.capture(capturedBuffer), EasyMock.eq(0), + EasyMock.eq(BLOB_INFO.toPb()), EasyMock.eq(0L), EasyMock.eq(MIN_CHUNK_SIZE), + EasyMock.eq(true)); + EasyMock.expectLastCall(); + EasyMock.replay(storageRpcMock); + writer = new BlobWriterChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); + assertTrue(writer.isOpen()); + writer.write(buffer); + writer.close(); + assertEquals(DEFAULT_CHUNK_SIZE, capturedBuffer.getValue().length); + assertArrayEquals(buffer.array(), Arrays.copyOf(capturedBuffer.getValue(), MIN_CHUNK_SIZE)); + assertTrue(!writer.isOpen()); + } + + @Test + public void testWriteClosed() throws IOException { + EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock); + EasyMock.expect(optionsMock.retryParams()).andReturn(RetryParams.noRetries()); + EasyMock.replay(optionsMock); + EasyMock.expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); + Capture capturedBuffer = Capture.newInstance(); + storageRpcMock.write(EasyMock.eq(UPLOAD_ID), EasyMock.capture(capturedBuffer), EasyMock.eq(0), + EasyMock.eq(BLOB_INFO.toPb()), EasyMock.eq(0L), EasyMock.eq(0), EasyMock.eq(true)); + EasyMock.expectLastCall(); + EasyMock.replay(storageRpcMock); + writer = new BlobWriterChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); + writer.close(); + try { + writer.write(ByteBuffer.allocate(MIN_CHUNK_SIZE)); + fail("Expected BlobWriteChannel write to throw IOException"); + } catch (IOException ex) { + // expected + } + } + + private static ByteBuffer randomBuffer(int size) { + byte[] byteArray = new byte[size]; + RANDOM.nextBytes(byteArray); + return ByteBuffer.wrap(byteArray); + } +} From 82cd3ea7db5e6185094bb41bc586ed6eb8d3cff1 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Tue, 29 Sep 2015 18:05:53 +0200 Subject: [PATCH 028/156] javadoc: recommend Storage.writer for large uploads --- .../src/main/java/com/google/gcloud/storage/Storage.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java index 900ba721b7ce..39bee763d23b 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java @@ -494,7 +494,8 @@ public static Builder builder() { BucketInfo create(BucketInfo bucketInfo, BucketTargetOption... options); /** - * Create a new blob. + * Create a new blob. Direct upload is used to upload {@code content}. For large content, + * {@link #writer} is recommended as it uses resumable upload. * * @return a complete blob information. * @throws StorageException upon failure @@ -502,7 +503,8 @@ public static Builder builder() { BlobInfo create(BlobInfo blobInfo, byte[] content, BlobTargetOption... options); /** - * Create a new blob. + * Create a new blob. Direct upload is used to upload {@code content}. For large content, + * {@link #writer} is recommended as it uses resumable upload. * * @return a complete blob information. * @throws StorageException upon failure From 967c4292b8320ed5d85700b080d7795a6b0370cd Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Tue, 29 Sep 2015 22:36:56 +0200 Subject: [PATCH 029/156] Remove redundant read test, add buffered write test, add EasyMock.verify --- .../gcloud/storage/BlobReadChannelTest.java | 28 ++++++--------- .../gcloud/storage/BlobWriterChannelTest.java | 34 +++++++++++++++++++ 2 files changed, 45 insertions(+), 17 deletions(-) diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobReadChannelTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobReadChannelTest.java index f54c02c6eb83..dd2426416d19 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobReadChannelTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobReadChannelTest.java @@ -34,6 +34,8 @@ import java.util.Arrays; import java.util.Map; import java.util.Random; +import static org.easymock.EasyMock.verify; +import org.junit.After; public class BlobReadChannelTest { @@ -55,28 +57,19 @@ public void setUp() throws IOException, InterruptedException { storageRpcMock = EasyMock.createMock(StorageRpc.class); } - @Test - public void testCreate() { - EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock); - EasyMock.replay(optionsMock); - reader = new BlobReadChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); - assertTrue(reader.isOpen()); + @After + public void tearDown() throws Exception { + verify(optionsMock); + verify(storageRpcMock); } @Test - public void testReadSmall() throws IOException { + public void testCreate() { EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock); - EasyMock.expect(optionsMock.retryParams()).andReturn(RetryParams.noRetries()); EasyMock.replay(optionsMock); - reader = new BlobReadChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); - byte[] result = randomByteArray(DEFAULT_CHUNK_SIZE); - ByteBuffer readBuffer = ByteBuffer.allocate(42); - EasyMock - .expect(storageRpcMock.read(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS, 0, DEFAULT_CHUNK_SIZE)) - .andReturn(result); EasyMock.replay(storageRpcMock); - reader.read(readBuffer); - assertArrayEquals(Arrays.copyOf(result, readBuffer.capacity()), readBuffer.array()); + reader = new BlobReadChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); + assertTrue(reader.isOpen()); } @Test @@ -120,7 +113,6 @@ public void testReadBig() throws IOException { storageRpcMock.read(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS, DEFAULT_CHUNK_SIZE, CUSTOM_CHUNK_SIZE)) .andReturn(secondResult); - EasyMock.expectLastCall(); EasyMock.replay(storageRpcMock); reader.read(firstReadBuffer); reader.read(secondReadBuffer); @@ -165,6 +157,7 @@ public void testSeek() throws IOException { public void testClose() throws IOException { EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock); EasyMock.replay(optionsMock); + EasyMock.replay(storageRpcMock); reader = new BlobReadChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); assertTrue(reader.isOpen()); reader.close(); @@ -175,6 +168,7 @@ public void testClose() throws IOException { public void testReadClosed() { EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock); EasyMock.replay(optionsMock); + EasyMock.replay(storageRpcMock); reader = new BlobReadChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); reader.close(); try { diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobWriterChannelTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobWriterChannelTest.java index 65997bf86228..bff9106db75b 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobWriterChannelTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobWriterChannelTest.java @@ -35,6 +35,8 @@ import java.util.Arrays; import java.util.Map; import java.util.Random; +import static org.easymock.EasyMock.verify; +import org.junit.After; public class BlobWriterChannelTest { @@ -58,6 +60,12 @@ public void setUp() throws IOException, InterruptedException { storageRpcMock = EasyMock.createMock(StorageRpc.class); } + @After + public void tearDown() throws Exception { + verify(optionsMock); + verify(storageRpcMock); + } + @Test public void testCreate() { EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock); @@ -97,6 +105,32 @@ public void testWriteWithFlush() throws IOException { assertArrayEquals(buffer.array(), capturedBuffer.getValue()); } + @Test + public void testWritesAndFlush() throws IOException { + EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock); + EasyMock.expect(optionsMock.retryParams()).andReturn(RetryParams.noRetries()); + EasyMock.replay(optionsMock); + EasyMock.expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); + Capture capturedBuffer = Capture.newInstance(); + storageRpcMock.write(EasyMock.eq(UPLOAD_ID), EasyMock.capture(capturedBuffer), EasyMock.eq(0), + EasyMock.eq(BLOB_INFO.toPb()), EasyMock.eq(0L), EasyMock.eq(DEFAULT_CHUNK_SIZE), + EasyMock.eq(false)); + EasyMock.expectLastCall(); + EasyMock.replay(storageRpcMock); + writer = new BlobWriterChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); + ByteBuffer[] buffers = new ByteBuffer[DEFAULT_CHUNK_SIZE / MIN_CHUNK_SIZE]; + for (int i = 0; i < buffers.length; i++) { + buffers[i] = randomBuffer(MIN_CHUNK_SIZE); + assertEquals(MIN_CHUNK_SIZE, writer.write(buffers[i])); + } + for (int i = 0; i < buffers.length; i++) { + assertArrayEquals( + buffers[i].array(), + Arrays.copyOfRange( + capturedBuffer.getValue(), MIN_CHUNK_SIZE * i, MIN_CHUNK_SIZE * (i + 1))); + } + } + @Test public void testCloseWithoutFlush() throws IOException { EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock); From 7145030ddb913c9ec3c9fb9418943ffe0af631e5 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Tue, 29 Sep 2015 22:40:27 +0200 Subject: [PATCH 030/156] Renaming BlobWriterChannelImpl to BlobWriteChannelImpl, renaming tests --- ...nelImpl.java => BlobWriteChannelImpl.java} | 4 ++-- .../google/gcloud/storage/StorageImpl.java | 2 +- ...Test.java => BlobReadChannelImplTest.java} | 4 ++-- ...est.java => BlobWriteChannelImplTest.java} | 20 +++++++++---------- 4 files changed, 15 insertions(+), 15 deletions(-) rename gcloud-java-storage/src/main/java/com/google/gcloud/storage/{BlobWriterChannelImpl.java => BlobWriteChannelImpl.java} (97%) rename gcloud-java-storage/src/test/java/com/google/gcloud/storage/{BlobReadChannelTest.java => BlobReadChannelImplTest.java} (99%) rename gcloud-java-storage/src/test/java/com/google/gcloud/storage/{BlobWriterChannelTest.java => BlobWriteChannelImplTest.java} (92%) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriteChannelImpl.java similarity index 97% rename from gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java rename to gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriteChannelImpl.java index 612e5d7551dc..48d60dc8e24d 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriteChannelImpl.java @@ -32,7 +32,7 @@ /** * Default implementation for BlobWriteChannel. */ -class BlobWriterChannelImpl implements BlobWriteChannel { +class BlobWriteChannelImpl implements BlobWriteChannel { private static final long serialVersionUID = 8675286882724938737L; private static final int MIN_CHUNK_SIZE = 256 * 1024; @@ -50,7 +50,7 @@ class BlobWriterChannelImpl implements BlobWriteChannel { private transient StorageRpc storageRpc; private transient StorageObject storageObject; - BlobWriterChannelImpl(StorageOptions options, BlobInfo blobInfo, + BlobWriteChannelImpl(StorageOptions options, BlobInfo blobInfo, Map optionsMap) { this.options = options; this.blobInfo = blobInfo; diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java index 4c80c6741559..5f2ee7e30d71 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java @@ -443,7 +443,7 @@ public BlobReadChannel reader(String bucket, String blob, BlobSourceOption... op @Override public BlobWriteChannel writer(BlobInfo blobInfo, BlobTargetOption... options) { final Map optionsMap = optionMap(blobInfo, options); - return new BlobWriterChannelImpl(options(), blobInfo, optionsMap); + return new BlobWriteChannelImpl(options(), blobInfo, optionsMap); } @Override diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobReadChannelTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobReadChannelImplTest.java similarity index 99% rename from gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobReadChannelTest.java rename to gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobReadChannelImplTest.java index dd2426416d19..2a43753e6178 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobReadChannelTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobReadChannelImplTest.java @@ -16,6 +16,7 @@ package com.google.gcloud.storage; +import static org.easymock.EasyMock.verify; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertEquals; @@ -34,10 +35,9 @@ import java.util.Arrays; import java.util.Map; import java.util.Random; -import static org.easymock.EasyMock.verify; import org.junit.After; -public class BlobReadChannelTest { +public class BlobReadChannelImplTest { private static final String BUCKET_NAME = "b"; private static final String BLOB_NAME = "n"; diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobWriterChannelTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobWriteChannelImplTest.java similarity index 92% rename from gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobWriterChannelTest.java rename to gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobWriteChannelImplTest.java index bff9106db75b..cc43f8f6e1f7 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobWriterChannelTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobWriteChannelImplTest.java @@ -16,6 +16,7 @@ package com.google.gcloud.storage; +import static org.easymock.EasyMock.verify; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertTrue; @@ -35,10 +36,9 @@ import java.util.Arrays; import java.util.Map; import java.util.Random; -import static org.easymock.EasyMock.verify; import org.junit.After; -public class BlobWriterChannelTest { +public class BlobWriteChannelImplTest { private static final String BUCKET_NAME = "b"; private static final String BLOB_NAME = "n"; @@ -52,7 +52,7 @@ public class BlobWriterChannelTest { private StorageOptions optionsMock; private StorageRpc storageRpcMock; - private BlobWriterChannelImpl writer; + private BlobWriteChannelImpl writer; @Before public void setUp() throws IOException, InterruptedException { @@ -72,7 +72,7 @@ public void testCreate() { EasyMock.replay(optionsMock); EasyMock.expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); EasyMock.replay(storageRpcMock); - writer = new BlobWriterChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); + writer = new BlobWriteChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); assertTrue(writer.isOpen()); } @@ -82,7 +82,7 @@ public void testWriteWithoutFlush() throws IOException { EasyMock.replay(optionsMock); EasyMock.expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); EasyMock.replay(storageRpcMock); - writer = new BlobWriterChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); + writer = new BlobWriteChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); assertEquals(MIN_CHUNK_SIZE, writer.write(ByteBuffer.allocate(MIN_CHUNK_SIZE))); } @@ -98,7 +98,7 @@ public void testWriteWithFlush() throws IOException { EasyMock.eq(false)); EasyMock.expectLastCall(); EasyMock.replay(storageRpcMock); - writer = new BlobWriterChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); + writer = new BlobWriteChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); writer.chunkSize(CUSTOM_CHUNK_SIZE); ByteBuffer buffer = randomBuffer(CUSTOM_CHUNK_SIZE); assertEquals(CUSTOM_CHUNK_SIZE, writer.write(buffer)); @@ -117,7 +117,7 @@ public void testWritesAndFlush() throws IOException { EasyMock.eq(false)); EasyMock.expectLastCall(); EasyMock.replay(storageRpcMock); - writer = new BlobWriterChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); + writer = new BlobWriteChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); ByteBuffer[] buffers = new ByteBuffer[DEFAULT_CHUNK_SIZE / MIN_CHUNK_SIZE]; for (int i = 0; i < buffers.length; i++) { buffers[i] = randomBuffer(MIN_CHUNK_SIZE); @@ -142,7 +142,7 @@ public void testCloseWithoutFlush() throws IOException { EasyMock.eq(BLOB_INFO.toPb()), EasyMock.eq(0L), EasyMock.eq(0), EasyMock.eq(true)); EasyMock.expectLastCall(); EasyMock.replay(storageRpcMock); - writer = new BlobWriterChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); + writer = new BlobWriteChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); assertTrue(writer.isOpen()); writer.close(); assertArrayEquals(new byte[0], capturedBuffer.getValue()); @@ -162,7 +162,7 @@ public void testCloseWithFlush() throws IOException { EasyMock.eq(true)); EasyMock.expectLastCall(); EasyMock.replay(storageRpcMock); - writer = new BlobWriterChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); + writer = new BlobWriteChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); assertTrue(writer.isOpen()); writer.write(buffer); writer.close(); @@ -182,7 +182,7 @@ public void testWriteClosed() throws IOException { EasyMock.eq(BLOB_INFO.toPb()), EasyMock.eq(0L), EasyMock.eq(0), EasyMock.eq(true)); EasyMock.expectLastCall(); EasyMock.replay(storageRpcMock); - writer = new BlobWriterChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); + writer = new BlobWriteChannelImpl(optionsMock, BLOB_INFO, EMPTY_RPC_OPTIONS); writer.close(); try { writer.write(ByteBuffer.allocate(MIN_CHUNK_SIZE)); From 8928c498a3c90586990c13e5496e406c064cd35a Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Tue, 29 Sep 2015 13:53:09 -0700 Subject: [PATCH 031/156] Add test for runtime exception --- .../gcloud/datastore/DatastoreTest.java | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java index 156f9684f8ba..98f30b075071 100644 --- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java +++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java @@ -41,7 +41,9 @@ import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -102,6 +104,9 @@ public class DatastoreTest { private static LocalGcdHelper gcdHelper; + @Rule + public ExpectedException thrown = ExpectedException.none(); + @BeforeClass public static void beforeClass() throws IOException, InterruptedException { if (!LocalGcdHelper.isActive(PROJECT_ID)) { @@ -635,7 +640,7 @@ public void testKeyFactory() { } @Test - public void testRetires() throws Exception { + public void testRetries() throws Exception { DatastoreV1.LookupRequest requestPb = DatastoreV1.LookupRequest.newBuilder().addKey(KEY1.toPb()).build(); DatastoreV1.LookupResponse responsePb = DatastoreV1.LookupResponse.newBuilder() @@ -657,4 +662,27 @@ public void testRetires() throws Exception { assertEquals(ENTITY1, entity); EasyMock.verify(rpcFactoryMock, rpcMock); } + + @Test + public void testRuntimeExceptionHandling() throws Exception { + DatastoreV1.LookupRequest requestPb = + DatastoreV1.LookupRequest.newBuilder().addKey(KEY1.toPb()).build(); + DatastoreRpcFactory rpcFactoryMock = EasyMock.createStrictMock(DatastoreRpcFactory.class); + DatastoreRpc rpcMock = EasyMock.createStrictMock(DatastoreRpc.class); + EasyMock.expect(rpcFactoryMock.create(EasyMock.anyObject(DatastoreOptions.class))) + .andReturn(rpcMock); + String exceptionMessage = "Artificial runtime exception"; + EasyMock.expect(rpcMock.lookup(requestPb)) + .andThrow(new RuntimeException(exceptionMessage)); + EasyMock.replay(rpcFactoryMock, rpcMock); + DatastoreOptions options = this.options.toBuilder() + .retryParams(RetryParams.getDefaultInstance()) + .serviceRpcFactory(rpcFactoryMock) + .build(); + Datastore datastore = DatastoreFactory.instance().get(options); + thrown.expect(DatastoreException.class); + thrown.expectMessage(exceptionMessage); + datastore.get(KEY1); + EasyMock.verify(rpcFactoryMock, rpcMock); + } } From 1f617b77fd77d09f327b909d2302f2f4efc6e0a5 Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Tue, 29 Sep 2015 17:14:32 -0700 Subject: [PATCH 032/156] choose port for LocalGcdHelper dynamically --- .../datastore/DatastoreOptionsTest.java | 5 +- .../gcloud/datastore/DatastoreTest.java | 7 +-- .../gcloud/datastore/LocalGcdHelper.java | 53 ++++++++++++------- 3 files changed, 40 insertions(+), 25 deletions(-) diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreOptionsTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreOptionsTest.java index e7dc71c50ff6..8a0d52aec9c8 100644 --- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreOptionsTest.java +++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreOptionsTest.java @@ -34,6 +34,7 @@ public class DatastoreOptionsTest { private static final String PROJECT_ID = "project_id"; + private static final int PORT = LocalGcdHelper.findOpenPort(); private DatastoreRpcFactory datastoreRpcFactory; private DatastoreRpc datastoreRpc; private DatastoreOptions.Builder options; @@ -46,7 +47,7 @@ public void setUp() throws IOException, InterruptedException { .normalizeDataset(false) .serviceRpcFactory(datastoreRpcFactory) .projectId(PROJECT_ID) - .host("http://localhost:" + LocalGcdHelper.PORT); + .host("http://localhost:" + PORT); EasyMock.expect(datastoreRpcFactory.create(EasyMock.anyObject(DatastoreOptions.class))) .andReturn(datastoreRpc) .anyTimes(); @@ -60,7 +61,7 @@ public void testProjectId() throws Exception { @Test public void testHost() throws Exception { - assertEquals("http://localhost:" + LocalGcdHelper.PORT, options.build().host()); + assertEquals("http://localhost:" + PORT, options.build().host()); } @Test diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java index 156f9684f8ba..2c7e8f987cb1 100644 --- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java +++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java @@ -101,11 +101,12 @@ public class DatastoreTest { private Datastore datastore; private static LocalGcdHelper gcdHelper; + private static final int PORT = LocalGcdHelper.findOpenPort(); @BeforeClass public static void beforeClass() throws IOException, InterruptedException { - if (!LocalGcdHelper.isActive(PROJECT_ID)) { - gcdHelper = LocalGcdHelper.start(PROJECT_ID); + if (!LocalGcdHelper.isActive(PROJECT_ID, PORT)) { + gcdHelper = LocalGcdHelper.start(PROJECT_ID, PORT); } } @@ -113,7 +114,7 @@ public static void beforeClass() throws IOException, InterruptedException { public void setUp() throws IOException, InterruptedException { options = DatastoreOptions.builder() .projectId(PROJECT_ID) - .host("http://localhost:" + LocalGcdHelper.PORT) + .host("http://localhost:" + PORT) .build(); datastore = DatastoreFactory.instance().get(options); StructuredQuery query = Query.keyQueryBuilder().build(); diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java index 888fd2157d3a..a78797bb88e4 100644 --- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java +++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java @@ -35,6 +35,7 @@ import java.math.BigInteger; import java.net.HttpURLConnection; import java.net.MalformedURLException; +import java.net.ServerSocket; import java.net.URL; import java.nio.channels.Channels; import java.nio.channels.ReadableByteChannel; @@ -68,9 +69,10 @@ public class LocalGcdHelper { private final String projectId; private Path gcdPath; private ProcessStreamReader processReader; + private final int port; public static final String DEFAULT_PROJECT_ID = "projectid1"; - public static final int PORT = 8080; + private static final int DEFAULT_PORT = 8080; private static final String GCD_VERSION = "v1beta2"; private static final String GCD_BUILD = "rev1-2.1.2b"; private static final String GCD_BASENAME = "gcd-" + GCD_VERSION + "-" + GCD_BUILD; @@ -94,6 +96,17 @@ public class LocalGcdHelper { } } + public static int findOpenPort() { + int port; + try (ServerSocket temp_socket = new ServerSocket(0)) { + port = temp_socket.getLocalPort(); + temp_socket.close(); + } catch (IOException e) { + port = DEFAULT_PORT; + } + return port; + } + private static Path installedGcdPath() { String gcloudExecutableName; if (isWindows()) { @@ -283,8 +296,9 @@ public static CommandWrapper create() { } } - public LocalGcdHelper(String projectId) { + public LocalGcdHelper(String projectId, int port) { this.projectId = projectId; + this.port = port; } /** @@ -297,7 +311,7 @@ public LocalGcdHelper(String projectId) { */ public void start() throws IOException, InterruptedException { // send a quick request in case we have a hanging process from a previous run - sendQuitRequest(); + sendQuitRequest(port); // Each run is associated with its own folder that is deleted once test completes. gcdPath = Files.createTempDirectory("gcd"); File gcdFolder = gcdPath.toFile(); @@ -379,13 +393,12 @@ private void startGcd(Path executablePath) throws IOException, InterruptedExcept if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Starting datastore emulator for the project: {0}", projectId); } - Process startProcess = - CommandWrapper.create() - .command(gcdAbsolutePath.toString(), "start", "--testing", "--allow_remote_shutdown", - projectId) - .directory(gcdPath) - .redirectErrorStream() - .start(); + Process startProcess = CommandWrapper.create() + .command(gcdAbsolutePath.toString(), "start", "--testing", "--allow_remote_shutdown", + "--port=" + Integer.toString(port), projectId) + .directory(gcdPath) + .redirectErrorStream() + .start(); processReader = ProcessStreamReader.start(startProcess, "Dev App Server is now running"); } @@ -419,9 +432,9 @@ private static void extractFile(ZipInputStream zipIn, File filePath) throws IOEx } } - public static void sendQuitRequest() { + public static void sendQuitRequest(int port) { try { - URL url = new URL("http", "localhost", PORT, "/_ah/admin/quit"); + URL url = new URL("http", "localhost", port, "/_ah/admin/quit"); HttpURLConnection con = (HttpURLConnection) url.openConnection(); con.setRequestMethod("POST"); con.setDoOutput(true); @@ -439,7 +452,7 @@ public static void sendQuitRequest() { } public void stop() throws IOException, InterruptedException { - sendQuitRequest(); + sendQuitRequest(port); if (processReader != null) { processReader.terminate(); } @@ -468,8 +481,8 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO }); } - public static LocalGcdHelper start(String projectId) throws IOException, InterruptedException { - LocalGcdHelper helper = new LocalGcdHelper(projectId); + public static LocalGcdHelper start(String projectId, int port) throws IOException, InterruptedException { + LocalGcdHelper helper = new LocalGcdHelper(projectId, port); helper.start(); return helper; } @@ -478,15 +491,15 @@ public static void main(String... args) throws IOException, InterruptedException if (args.length == 1) { switch (args[0]) { case "START": - if (!isActive(DEFAULT_PROJECT_ID)) { - LocalGcdHelper helper = start(DEFAULT_PROJECT_ID); + if (!isActive(DEFAULT_PROJECT_ID, DEFAULT_PORT)) { + LocalGcdHelper helper = start(DEFAULT_PROJECT_ID, DEFAULT_PORT); try (FileWriter writer = new FileWriter(".local_gcd_helper")) { writer.write(helper.gcdPath.toAbsolutePath().toString()); } } return; case "STOP": - sendQuitRequest(); + sendQuitRequest(DEFAULT_PORT); File file = new File(".local_gcd_helper"); if (file.exists()) { try (BufferedReader reader = new BufferedReader(new FileReader(file))) { @@ -503,9 +516,9 @@ public static void main(String... args) throws IOException, InterruptedException throw new RuntimeException("expecting only START | STOP"); } - public static boolean isActive(String projectId) { + public static boolean isActive(String projectId, int port) { try { - StringBuilder urlBuilder = new StringBuilder("http://localhost:").append(PORT); + StringBuilder urlBuilder = new StringBuilder("http://localhost:").append(port); urlBuilder.append("/datastore/v1beta2/datasets/").append(projectId).append("/lookup"); URL url = new URL(urlBuilder.toString()); try (BufferedReader reader = From e776320245ef694d89fbbf59195487a141aebf4d Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Tue, 29 Sep 2015 17:59:31 -0700 Subject: [PATCH 033/156] add test for nonretryable DatastoreException --- .../gcloud/datastore/DatastoreTest.java | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java index 98f30b075071..391876b3c216 100644 --- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java +++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java @@ -640,7 +640,7 @@ public void testKeyFactory() { } @Test - public void testRetries() throws Exception { + public void testRetryableException() throws Exception { DatastoreV1.LookupRequest requestPb = DatastoreV1.LookupRequest.newBuilder().addKey(KEY1.toPb()).build(); DatastoreV1.LookupResponse responsePb = DatastoreV1.LookupResponse.newBuilder() @@ -664,7 +664,31 @@ public void testRetries() throws Exception { } @Test - public void testRuntimeExceptionHandling() throws Exception { + public void testNonRetryableException() throws Exception { + DatastoreV1.LookupRequest requestPb = + DatastoreV1.LookupRequest.newBuilder().addKey(KEY1.toPb()).build(); + DatastoreRpcFactory rpcFactoryMock = EasyMock.createStrictMock(DatastoreRpcFactory.class); + DatastoreRpc rpcMock = EasyMock.createStrictMock(DatastoreRpc.class); + EasyMock.expect(rpcFactoryMock.create(EasyMock.anyObject(DatastoreOptions.class))) + .andReturn(rpcMock); + EasyMock.expect(rpcMock.lookup(requestPb)) + .andThrow(new DatastoreRpc.DatastoreRpcException(Reason.PERMISSION_DENIED)) + .times(1); + EasyMock.replay(rpcFactoryMock, rpcMock); + RetryParams retryParams = RetryParams.builder().retryMinAttempts(2).build(); + DatastoreOptions options = this.options.toBuilder() + .retryParams(retryParams) + .serviceRpcFactory(rpcFactoryMock) + .build(); + Datastore datastore = DatastoreFactory.instance().get(options); + thrown.expect(DatastoreException.class); + thrown.expectMessage(Reason.PERMISSION_DENIED.description()); + datastore.get(KEY1); + EasyMock.verify(rpcFactoryMock, rpcMock); + } + + @Test + public void testRuntimeException() throws Exception { DatastoreV1.LookupRequest requestPb = DatastoreV1.LookupRequest.newBuilder().addKey(KEY1.toPb()).build(); DatastoreRpcFactory rpcFactoryMock = EasyMock.createStrictMock(DatastoreRpcFactory.class); From 22a9e9915c1ba5fae4e1d194bf0b92686d4ba7d4 Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Tue, 29 Sep 2015 18:14:15 -0700 Subject: [PATCH 034/156] Allow a default port to be passed in as a parameter, and also fix names --- .../gcloud/datastore/DatastoreOptionsTest.java | 2 +- .../google/gcloud/datastore/DatastoreTest.java | 2 +- .../google/gcloud/datastore/LocalGcdHelper.java | 17 +++++++---------- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreOptionsTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreOptionsTest.java index 8a0d52aec9c8..b3365876773e 100644 --- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreOptionsTest.java +++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreOptionsTest.java @@ -34,7 +34,7 @@ public class DatastoreOptionsTest { private static final String PROJECT_ID = "project_id"; - private static final int PORT = LocalGcdHelper.findOpenPort(); + private static final int PORT = LocalGcdHelper.findAvailablePort(LocalGcdHelper.DEFAULT_PORT); private DatastoreRpcFactory datastoreRpcFactory; private DatastoreRpc datastoreRpc; private DatastoreOptions.Builder options; diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java index 2c7e8f987cb1..b03472a9a6f8 100644 --- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java +++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java @@ -101,7 +101,7 @@ public class DatastoreTest { private Datastore datastore; private static LocalGcdHelper gcdHelper; - private static final int PORT = LocalGcdHelper.findOpenPort(); + private static final int PORT = LocalGcdHelper.findAvailablePort(LocalGcdHelper.DEFAULT_PORT); @BeforeClass public static void beforeClass() throws IOException, InterruptedException { diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java index a78797bb88e4..ce069bd9709d 100644 --- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java +++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java @@ -72,7 +72,7 @@ public class LocalGcdHelper { private final int port; public static final String DEFAULT_PROJECT_ID = "projectid1"; - private static final int DEFAULT_PORT = 8080; + public static final int DEFAULT_PORT = 8080; private static final String GCD_VERSION = "v1beta2"; private static final String GCD_BUILD = "rev1-2.1.2b"; private static final String GCD_BASENAME = "gcd-" + GCD_VERSION + "-" + GCD_BUILD; @@ -96,15 +96,12 @@ public class LocalGcdHelper { } } - public static int findOpenPort() { - int port; - try (ServerSocket temp_socket = new ServerSocket(0)) { - port = temp_socket.getLocalPort(); - temp_socket.close(); + public static int findAvailablePort(int defaultPort) { + try (ServerSocket tempSocket = new ServerSocket(0)) { + return tempSocket.getLocalPort(); } catch (IOException e) { - port = DEFAULT_PORT; + return defaultPort; } - return port; } private static Path installedGcdPath() { @@ -491,8 +488,8 @@ public static void main(String... args) throws IOException, InterruptedException if (args.length == 1) { switch (args[0]) { case "START": - if (!isActive(DEFAULT_PROJECT_ID, DEFAULT_PORT)) { - LocalGcdHelper helper = start(DEFAULT_PROJECT_ID, DEFAULT_PORT); + if (!isActive(DEFAULT_PROJECT_ID, findAvailablePort(DEFAULT_PORT))) { + LocalGcdHelper helper = start(DEFAULT_PROJECT_ID, findAvailablePort(DEFAULT_PORT)); try (FileWriter writer = new FileWriter(".local_gcd_helper")) { writer.write(helper.gcdPath.toAbsolutePath().toString()); } From 86065a7fc7d130977a40b99b2bf77d6889d84418 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Wed, 30 Sep 2015 09:21:03 +0200 Subject: [PATCH 035/156] Rename Storage.createFromStream create, add create for empty content --- .../main/java/com/google/gcloud/storage/Storage.java | 10 +++++++++- .../java/com/google/gcloud/storage/StorageImpl.java | 9 +++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java index 39bee763d23b..2ce25a8190f3 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java @@ -493,6 +493,14 @@ public static Builder builder() { */ BucketInfo create(BucketInfo bucketInfo, BucketTargetOption... options); + /** + * Create a new blob with no content. + * + * @return a complete blob information. + * @throws StorageException upon failure + */ + BlobInfo create(BlobInfo blobInfo, BlobTargetOption... options); + /** * Create a new blob. Direct upload is used to upload {@code content}. For large content, * {@link #writer} is recommended as it uses resumable upload. @@ -509,7 +517,7 @@ public static Builder builder() { * @return a complete blob information. * @throws StorageException upon failure */ - BlobInfo createFromStream(BlobInfo blobInfo, InputStream content, BlobTargetOption... options); + BlobInfo create(BlobInfo blobInfo, InputStream content, BlobTargetOption... options); /** * Return the requested bucket or {@code null} if not found. diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java index dca844b83034..d7dbafdc8b9e 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java @@ -114,14 +114,19 @@ public com.google.api.services.storage.model.Bucket call() { }, options().retryParams(), EXCEPTION_HANDLER)); } + @Override + public BlobInfo create(BlobInfo blobInfo, BlobTargetOption... options) { + return create(blobInfo, new ByteArrayInputStream(EMPTY_BYTE_ARRAY), options); + } + @Override public BlobInfo create(BlobInfo blobInfo, final byte[] content, BlobTargetOption... options) { - return createFromStream(blobInfo, + return create(blobInfo, new ByteArrayInputStream(firstNonNull(content, EMPTY_BYTE_ARRAY)), options); } @Override - public BlobInfo createFromStream(BlobInfo blobInfo, final InputStream content, + public BlobInfo create(BlobInfo blobInfo, final InputStream content, BlobTargetOption... options) { final StorageObject blobPb = blobInfo.toPb(); final Map optionsMap = optionMap(blobInfo, options); From 1311e0f45d0ce399f2f1a061175d4941f03af3fa Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Wed, 30 Sep 2015 14:53:03 +0200 Subject: [PATCH 036/156] Tabbed view with examples in landing page --- src/site/resources/css/site.css | 57 +++++++++++++++- src/site/resources/index.html | 67 +++++++++++++++---- src/site/resources/js/main.js | 7 +- .../resources/js/vendor/highlight.pack.js | 1 + 4 files changed, 115 insertions(+), 17 deletions(-) create mode 100644 src/site/resources/js/vendor/highlight.pack.js diff --git a/src/site/resources/css/site.css b/src/site/resources/css/site.css index b1fed62bc505..e5efa44caefb 100644 --- a/src/site/resources/css/site.css +++ b/src/site/resources/css/site.css @@ -333,7 +333,50 @@ h1, h2, h3, h4, h5, h6 { What is it? */ .about pre { - font-size: 110%; + padding: 8px; + background: #fff; + line-height: 1.6em; +} + +pre, code { + font-family: Monaco, 'Droid Sans Mono', monospace !important; +} + +.about .code-sample-tabs { + margin-top: 10px; + width: 450px; +} + +.about .code-sample-tabs a.selected { + background-color: #fff; + font-weight: bold; +} + +.about .code-sample-tabs a { + display: inline-block; + padding: 2px 12px 0px 12px; + background-color: #f8f8f8; + margin-right: 8px; + color: #333; + text-decoration: none; + cursor: pointer; +} + +.about .hljs { + background-color: #fff; + font-size: .9em; + line-height: 1.6em; +} + +.hljs { + display: block; + padding: .5em; + color: #333; + background: #f8f8f8; +} + +[hljs] > pre { + margin: 0 !important; } /* @@ -507,7 +550,7 @@ h1, h2, h3, h4, h5, h6 { } .about h4 { - margin-bottom: 0; + margin: 0; font-size: 1.2em; font-weight: bold; color: #4285f4; @@ -1114,6 +1157,16 @@ h1, h2, h3, h4, h5, h6 { font-size: 2em; } + .quote-box { + width: 40%; + float: left; + } + + .quote-box--supplementary { + float: right; + width: 56%; + } + /* Hero Banner */ diff --git a/src/site/resources/index.html b/src/site/resources/index.html index e64d0ff7b28a..31ab602a3f58 100644 --- a/src/site/resources/index.html +++ b/src/site/resources/index.html @@ -15,7 +15,7 @@ - +