From 733584de36cebe29a86eaf9ceabeeba1726c65f0 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Fri, 19 Oct 2018 16:01:17 -0400 Subject: [PATCH 01/21] WIP: bigtable emulator This allows customers to easily interact with the bigtable emulator. Currently the Bigtable emulator is implemented in go. It is a fairly complex bit of code that would be very difficult to replicate in java. This makes it very difficult for users to interact with it in a java project. This PR attempts to make things easier for the user by wrapping the golang emulator in a jar and providing a JUnit rule to simplify lifecycle management of the emulator. It's implemented as 2 artifacts: - google-cloud-gcloud-maven-plugin: a maven plugin that can download artifacts from the gcloud sdk repo and add them as resources to a jar. This avoids the need to check in binaries into google-cloud-java and makes sure that the binaries are up to date. - google-cloud-bigtable-emulator: This uses the maven plugin to bundle the bigtable emulator in a jar and adds a java wrapper around the binary. The wrapper was extracted from the existing bigtable client integration tests: https://github.com/googleapis/google-cloud-java/blob/master/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java --- .../google-cloud-bigtable-emulator/pom.xml | 64 ++++ .../emulator/v2/BigtableEmulatorRule.java | 55 +++ .../cloud/bigtable/emulator/v2/Emulator.java | 217 ++++++++++++ .../google-cloud-gcloud-maven-plugin/pom.xml | 85 +++++ .../main/java/com/google/cloud/Component.java | 36 ++ .../google/cloud/DownloadComponentsMojo.java | 327 ++++++++++++++++++ google-cloud-testing/pom.xml | 2 + versions.txt | 2 + 8 files changed, 788 insertions(+) create mode 100644 google-cloud-testing/google-cloud-bigtable-emulator/pom.xml create mode 100644 google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRule.java create mode 100644 google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java create mode 100644 google-cloud-testing/google-cloud-gcloud-maven-plugin/pom.xml create mode 100644 google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/Component.java create mode 100644 google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/DownloadComponentsMojo.java diff --git a/google-cloud-testing/google-cloud-bigtable-emulator/pom.xml b/google-cloud-testing/google-cloud-bigtable-emulator/pom.xml new file mode 100644 index 000000000000..08d8d37563e1 --- /dev/null +++ b/google-cloud-testing/google-cloud-bigtable-emulator/pom.xml @@ -0,0 +1,64 @@ + + + + google-cloud-testing + com.google.cloud + 0.66.1-alpha-SNAPSHOT + + 4.0.0 + + google-cloud-bigtable-emulator + 0.66.1-alpha-SNAPSHOT + + + + + com.google.cloud + google-cloud-gcloud-maven-plugin + 0.66.1-alpha-SNAPSHOT + + + + gen-sources + generate-resources + + download + + + + bigtable-darwin-x86 + bigtable-darwin-x86_64 + bigtable-linux-x86 + bigtable-linux-x86_64 + bigtable-windows-x86 + bigtable-windows-x86_64 + + + + + + + + + + + io.grpc + grpc-core + 1.13.1 + provided + + + + com.google.api + api-common + + + + junit + junit + true + + + diff --git a/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRule.java b/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRule.java new file mode 100644 index 000000000000..ff495b7609ea --- /dev/null +++ b/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRule.java @@ -0,0 +1,55 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.emulator.v2; + +import com.google.api.core.BetaApi; +import io.grpc.ManagedChannel; +import org.junit.rules.ExternalResource; + +@BetaApi("Surface for Bigtable emulator is not yet stable") +public class BigtableEmulatorRule extends ExternalResource { + private Emulator emulator; + + public static BigtableEmulatorRule create() { + return new BigtableEmulatorRule(); + } + + protected BigtableEmulatorRule() { } + + @Override + protected void before() throws Throwable { + emulator = Emulator.createBundled(); + emulator.start(); + } + + @Override + protected void after() { + emulator.stop(); + emulator = null; + } + + public ManagedChannel getDataChannel() { + return emulator.getDataChannel(); + } + + public ManagedChannel getAdminChannel() { + return emulator.getAdminChannel(); + } + + public int getPort() { + return emulator.getPort(); + } +} diff --git a/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java b/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java new file mode 100644 index 000000000000..2795436415f3 --- /dev/null +++ b/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java @@ -0,0 +1,217 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.emulator.v2; + + +import com.google.api.core.BetaApi; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.URL; +import java.nio.file.Path; +import java.util.Locale; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.logging.Level; +import java.util.logging.Logger; + +@BetaApi("Surface for Bigtable emulator is not yet stable") +public class Emulator { + + private static final Logger LOGGER = Logger.getLogger(Emulator.class.getName()); + + private final Path executable; + private Process process; + private boolean isStopped = true; + + private int port; + private ManagedChannel dataChannel; + private ManagedChannel adminChannel; + + public static Emulator createBundled() throws IOException { + String resourcePath = String.format( + "gcloud/bigtable-%s/platform/bigtable-emulator/cbtemulator", getPlatform()); + + URL packagedEmulator = Emulator.class.getResource(resourcePath); + + File tmpEmulator = File.createTempFile("cbtemulator", ""); + tmpEmulator.deleteOnExit(); + + try (InputStream is = packagedEmulator.openStream(); + FileOutputStream os = new FileOutputStream(tmpEmulator)) { + + byte[] buff = new byte[2048]; + int length; + + while ((length = is.read(buff)) != -1) { + os.write(buff, 0, length); + } + } + tmpEmulator.setExecutable(true); + + return new Emulator(tmpEmulator.toPath()); + } + + private Emulator(Path executable) { + this.executable = executable; + } + + public void start() throws Exception { + this.port = getAvailablePort(); + + process = Runtime.getRuntime().exec(executable + " -port " + "" + port); + pipeStreamToLog(process.getInputStream(), Level.INFO); + pipeStreamToLog(process.getErrorStream(), Level.WARNING); + isStopped = false; + + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + if (!isStopped) { + isStopped = true; + process.destroy(); + } + } + }); + + waitForPort(port); + + dataChannel = newChannelBuilder(port) + .maxInboundMessageSize(256 * 1024 * 1024) + .build(); + + adminChannel = newChannelBuilder(port) + .build(); + } + + public void stop() { + try { + dataChannel.shutdownNow(); + adminChannel.shutdownNow(); + + dataChannel.awaitTermination(1, TimeUnit.MINUTES); + adminChannel.awaitTermination(1, TimeUnit.MINUTES); + } catch (InterruptedException e) { + LOGGER.warning("Interrupted while waiting for client channels to close"); + Thread.currentThread().interrupt(); + } finally { + isStopped = true; + process.destroy(); + } + } + + public int getPort() { + return port; + } + + public ManagedChannel getDataChannel() { + return dataChannel; + } + + public ManagedChannel getAdminChannel() { + return adminChannel; + } + + // + private static String getPlatform() { + String unformattedOs = System.getProperty("os.name", "unknown").toLowerCase(Locale.ENGLISH); + String os; + + if ((unformattedOs.indexOf("mac") >= 0) || (unformattedOs.indexOf("darwin") >= 0)) { + os = "darwin"; + } else if (unformattedOs.indexOf("win") >= 0) { + os = "windows"; + } else if (unformattedOs.indexOf("linux") >= 0) { + os = "linux"; + } else { + throw new UnsupportedOperationException( + "Emulator is not supported on your platform: " + unformattedOs); + } + + String unformattedArch = System.getProperty("os.arch"); + String arch; + + switch (unformattedArch) { + case "x86": + arch = "x86"; + break; + case "xmd64": + arch = "x86_64"; + break; + default: + throw new UnsupportedOperationException("Unsupport architecture: " + unformattedArch); + } + + return os + "-" + arch; + } + + private static int getAvailablePort() { + try (ServerSocket serverSocket = new ServerSocket(0)) { + return serverSocket.getLocalPort(); + } catch (IOException e) { + throw new RuntimeException("Failed to find open port"); + } + } + + private void waitForPort(int port) throws InterruptedException, TimeoutException { + for (int i = 0; i < 100; i++) { + try (Socket ignored = new Socket("localhost", port)) { + return; + } catch (IOException e) { + Thread.sleep(200); + } + } + + throw new TimeoutException("Timed out waiting for server to start"); + } + + private ManagedChannelBuilder newChannelBuilder(int port) { + // NOTE: usePlaintext is currently @ExperimentalAPI. + // See https://github.com/grpc/grpc-java/issues/1772 for discussion + return ManagedChannelBuilder.forAddress("localhost", port) + .usePlaintext(); + } + + private static void pipeStreamToLog(final InputStream stream, final Level level) { + final BufferedReader reader = new BufferedReader(new InputStreamReader(stream)); + + Thread thread = + new Thread( + new Runnable() { + @Override + public void run() { + try { + String line; + while ((line = reader.readLine()) != null) { + LOGGER.log(level, line); + } + } catch (IOException e) { + LOGGER.log(Level.WARNING, "Failed to read process stream", e); + } + } + }); + thread.setDaemon(true); + thread.start(); + } + // +} \ No newline at end of file diff --git a/google-cloud-testing/google-cloud-gcloud-maven-plugin/pom.xml b/google-cloud-testing/google-cloud-gcloud-maven-plugin/pom.xml new file mode 100644 index 000000000000..06b32dcb8f1e --- /dev/null +++ b/google-cloud-testing/google-cloud-gcloud-maven-plugin/pom.xml @@ -0,0 +1,85 @@ + + + 4.0.0 + + + google-cloud-testing + com.google.cloud + 0.66.1-alpha-SNAPSHOT + + + google-cloud-gcloud-maven-plugin + 0.66.1-alpha-SNAPSHOT + maven-plugin + Experimental Maven plugin to interact with the Google Cloud SDK + (https://cloud.google.com/sdk/). Currently this plugin is meant to be an internal implementation + detail for google-cloud-java. + + + + + + maven-plugin-plugin + 3.5 + + + mojo-descriptor + + descriptor + + + + + + + + + + org.apache.maven + maven-plugin-api + 3.0 + + + + org.apache.maven.plugin-tools + maven-plugin-annotations + 3.0 + provided + + + + com.google.auto.value + auto-value + 1.5 + provided + + + + com.google.code.findbugs + jsr305 + 3.0.2 + provided + + + + com.google.code.gson + gson + 2.7 + + + + org.apache.commons + commons-compress + 1.11 + + + + junit + junit + 3.8.1 + test + + + diff --git a/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/Component.java b/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/Component.java new file mode 100644 index 000000000000..b52b4f2bf4b7 --- /dev/null +++ b/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/Component.java @@ -0,0 +1,36 @@ +package com.google.cloud; + +import com.google.auto.value.AutoValue; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import java.io.IOException; +import java.net.URL; + +/** Representation of a gcloud component */ +@AutoValue +public abstract class Component { + public static Component create(String id, String checksum, URL source, String fileType) { + return new AutoValue_Component(id, checksum, source, fileType); + } + + public static Component fromJson(URL baseUrl, JsonObject componentObj) throws IOException { + String id = componentObj.get("id").getAsString(); + + JsonElement data = componentObj.get("data"); + if (data == null) { + throw new NullPointerException("Component " + id + " is missing a data section"); + } + + return create( + id, + data.getAsJsonObject().get("checksum").getAsString(), + new URL(baseUrl, data.getAsJsonObject().get("source").getAsString()), + data.getAsJsonObject().get("type").getAsString() + ); + } + + public abstract String getId(); + public abstract String getChecksum(); + public abstract URL getSource(); + public abstract String getFileType(); +} diff --git a/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/DownloadComponentsMojo.java b/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/DownloadComponentsMojo.java new file mode 100644 index 000000000000..73ed171f5ebc --- /dev/null +++ b/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/DownloadComponentsMojo.java @@ -0,0 +1,327 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.JsonPrimitive; +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.Writer; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.file.Files; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; +import org.apache.commons.compress.utils.IOUtils; +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; +import org.codehaus.plexus.util.FileUtils; + +/** + * Goal that downloads gcloud components and embeds them as resources in a jar. This is mainly + * intended to be used to wrapped native emulators in jars. + */ +@Mojo(name = "download") +public class DownloadComponentsMojo extends AbstractMojo { + + private static long STALE_MS = TimeUnit.HOURS.toMillis(2); + + @Parameter(defaultValue = "https://dl.google.com/dl/cloudsdk/channels/rapid/", required = true) + private URL baseUrl; + + @Parameter(required = true) + private List componentNames; + + @Parameter(defaultValue = "${project.build.outputDirectory}/gcloud", required = true) + private File destinationDir; + + @Parameter(defaultValue = "true", required = true, property = "gcloud.download.force") + private boolean forceRefresh; + + public void execute() throws MojoExecutionException { + // Ensure that the output directory exists + destinationDir.mkdirs(); + + // Update the cached manifest + try { + updateCachedManifest(); + } catch (Exception e) { + throw new MojoExecutionException("Failed to update the cached manifest", e); + } + + // Parse the relevant components + List components; + try { + components = parseManifest(); + } catch (Exception e) { + throw new MojoExecutionException("Failed to parse the manifest", e); + } + + // Get the checksums of the existing components + Map checksums; + try { + checksums = parseLocalChecksums(); + } catch (Exception e) { + getLog().warn("Failed to parse local checksums, ignoring", e); + checksums = new HashMap<>(); + } + + // Download any updated components + for (Component component : components) { + if (!forceRefresh && component.getChecksum().equals(checksums.get(component.getId()))) { + continue; + } + + try { + downloadComponent(component); + } catch (Exception e) { + throw new MojoExecutionException("Failed to download component " + component.getId(), e); + } + } + + // Write the checksums of the newly updated components. + try { + writeLocalChecksums(components); + } catch (IOException e) { + throw new MojoExecutionException("Failed to update the local checksum cache", e); + } + } + + private URL getManifestUrl() throws MalformedURLException { + return new URL(baseUrl, "components-2.json"); + } + + private File getManifestCache() { + return new File(destinationDir, "components-2.json"); + } + + private File getComponentPath(Component component) { + return new File(destinationDir, component.getId()); + } + + private File getChecksumFile() { + return new File(destinationDir, "checksums.json"); + } + + /** + * Try to update the locally cached cache manifest. This will be a noop if the cached manifest is + * fresh enough. + */ + private void updateCachedManifest() throws IOException { + URL manifestUrl = getManifestUrl(); + File localCache = getManifestCache(); + + if (!forceRefresh && localCache.exists() + && new Date().getTime() - localCache.lastModified() < STALE_MS) { + + getLog().debug("Manifest is up to date, skipping manifest download"); + return; + } + + getLog().debug("Downloading fresh manifest"); + + File tempFile = File.createTempFile(localCache.getName(), ""); + + try (BufferedInputStream in = new BufferedInputStream(manifestUrl.openStream()); + FileOutputStream fileOutputStream = new FileOutputStream(tempFile)) { + byte dataBuffer[] = new byte[1024]; + int bytesRead; + while ((bytesRead = in.read(dataBuffer, 0, 1024)) != -1) { + fileOutputStream.write(dataBuffer, 0, bytesRead); + } + } + + @SuppressWarnings("unused") + boolean ignored = localCache.delete(); + + if (!tempFile.renameTo(localCache)) { + throw new RuntimeException("Failed to move the updated cache into place"); + } + } + + /** + * Parse the locally cached manifest and extract the relevant components. + */ + private List parseManifest() throws IOException { + JsonParser parser = new JsonParser(); + JsonElement json; + try (FileReader reader = new FileReader(getManifestCache())) { + json = parser.parse(reader); + } + + JsonArray jsonComponents = json.getAsJsonObject().get("components").getAsJsonArray(); + List results = new ArrayList<>(); + + for (JsonElement jsonComponent : jsonComponents) { + JsonObject componentObj = jsonComponent.getAsJsonObject(); + String id = componentObj.get("id").getAsString(); + + if (!componentNames.contains(id)) { + continue; + } + results.add(Component.fromJson(baseUrl, componentObj)); + } + + return results; + } + + /** + * Parses a local manifest of the downloaded component checksums + */ + private Map parseLocalChecksums() throws IOException { + JsonElement json = new JsonObject(); + + if (getChecksumFile().exists()) { + JsonParser parser = new JsonParser(); + + try (FileReader reader = new FileReader(getChecksumFile())) { + json = parser.parse(reader); + } + } + + Map results = new HashMap<>(); + JsonObject checksumMap = json.getAsJsonObject(); + for (String componentName : componentNames) { + JsonElement checksumJson = checksumMap.get(componentName); + if (checksumJson != null) { + results.put(componentName, checksumJson.getAsString()); + } + } + return results; + } + + /** + * Downloads and extracts the component into the destinationDir. + */ + private void downloadComponent(Component component) throws IOException, NoSuchAlgorithmException { + getLog().info("Downloading " + component.getId()); + + if (!"tar".equals(component.getFileType())) { + throw new UnsupportedOperationException( + "Only tarballs are supported, got: " + component.getFileType()); + } + + // Download and verify the component + File tmpArchive = File.createTempFile(component.getId(), ""); + tmpArchive.deleteOnExit(); + + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + try (InputStream is = component.getSource().openStream(); + FileOutputStream os = new FileOutputStream(tmpArchive)) { + + byte dataBuffer[] = new byte[1024]; + int bytesRead; + while ((bytesRead = is.read(dataBuffer, 0, 1024)) != -1) { + digest.update(dataBuffer, 0, bytesRead); + os.write(dataBuffer, 0, bytesRead); + } + } + String checksum = byteArrayToHex(digest.digest()); + + if (!checksum.equals(component.getChecksum())) { + throw new RuntimeException(String + .format("Checksum mismatch for %s %s != %s", component.getId(), component.getChecksum(), + checksum)); + } + + // Stage the expanded archive + File tmpPath = Files.createTempDirectory(component.getId()).toFile(); + + try (TarArchiveInputStream stream = new TarArchiveInputStream( + new GzipCompressorInputStream(new FileInputStream(tmpArchive)))) { + + ArchiveEntry entry; + + while ((entry = stream.getNextEntry()) != null) { + File dest = new File(tmpPath, entry.getName()); + + if (entry.isDirectory()) { + if (!dest.mkdirs()) { + getLog().warn("Failed to expand the directory " + dest); + } + } else { + try (OutputStream outputFileStream = new FileOutputStream(dest)) { + IOUtils.copy(stream, outputFileStream); + } + } + } + } + + // Move it into place + File localPath = getComponentPath(component); + FileUtils.deleteDirectory(localPath); + + if (!tmpPath.renameTo(localPath)) { + throw new IOException("Failed to move the extracted component into place"); + } + } + + private static String byteArrayToHex(byte[] a) { + StringBuilder sb = new StringBuilder(a.length * 2); + for(byte b: a) + sb.append(String.format("%02x", b)); + return sb.toString(); + } + + + /** + * Update the checksums of the downloaded components. This will avoid the need to download them in + * the future. + */ + private void writeLocalChecksums(List components) throws IOException { + JsonObject results = new JsonObject(); + + try { + JsonParser parser = new JsonParser(); + try (FileReader reader = new FileReader(getChecksumFile())) { + results = parser.parse(reader).getAsJsonObject(); + } + } catch (FileNotFoundException e) { + // ignored + } + + for (Component component : components) { + results.add(component.getId(), new JsonPrimitive(component.getChecksum())); + } + + try (Writer writer = new FileWriter(getChecksumFile())) { + new Gson().toJson(results, writer); + } + } +} diff --git a/google-cloud-testing/pom.xml b/google-cloud-testing/pom.xml index 3b9638aae2e6..9efe51aa00ad 100644 --- a/google-cloud-testing/pom.xml +++ b/google-cloud-testing/pom.xml @@ -24,6 +24,8 @@ google-cloud-appengineflexjava google-cloud-appenginejava8 google-cloud-managedtest + google-cloud-gcloud-maven-plugin + google-cloud-bigtable-emulator diff --git a/versions.txt b/versions.txt index d3a5e637aef7..b4bd87e2306d 100644 --- a/versions.txt +++ b/versions.txt @@ -144,6 +144,8 @@ google-cloud-storage:1.48.0:1.48.1-SNAPSHOT google-cloud-tasks:0.66.0-beta:0.66.1-beta-SNAPSHOT google-cloud-texttospeech:0.66.0-beta:0.66.1-beta-SNAPSHOT google-cloud-testing:0.66.0-alpha:0.66.1-alpha-SNAPSHOT +google-cloud-gcloud-maven-plugin:0.66.0-alpha:0.66.1-alpha-SNAPSHOT +google-cloud-bigtable-emulator:0.66.0-alpha:0.66.1-alpha-SNAPSHOT google-cloud-trace:0.66.0-beta:0.66.1-beta-SNAPSHOT google-cloud-translate:1.48.0:1.48.1-SNAPSHOT google-cloud-util:0.66.0-alpha:0.66.1-alpha-SNAPSHOT From 471f16655500f53594e142b833801da218cc7f21 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Fri, 19 Oct 2018 16:30:25 -0400 Subject: [PATCH 02/21] Tests & fixes --- .../google-cloud-bigtable-emulator/pom.xml | 18 +++ .../cloud/bigtable/emulator/v2/Emulator.java | 5 +- .../emulator/v2/BigtableEmulatorRuleTest.java | 120 +++++++++++++++++ .../bigtable/emulator/v2/EmulatorTest.java | 124 ++++++++++++++++++ 4 files changed, 265 insertions(+), 2 deletions(-) create mode 100644 google-cloud-testing/google-cloud-bigtable-emulator/src/test/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRuleTest.java create mode 100644 google-cloud-testing/google-cloud-bigtable-emulator/src/test/java/com/google/cloud/bigtable/emulator/v2/EmulatorTest.java diff --git a/google-cloud-testing/google-cloud-bigtable-emulator/pom.xml b/google-cloud-testing/google-cloud-bigtable-emulator/pom.xml index 08d8d37563e1..9f9fbc453d8d 100644 --- a/google-cloud-testing/google-cloud-bigtable-emulator/pom.xml +++ b/google-cloud-testing/google-cloud-bigtable-emulator/pom.xml @@ -58,7 +58,25 @@ junit junit + compile true + + + com.google.cloud + google-cloud-bigtable-admin + test + + + + com.google.cloud + google-cloud-bigtable + test + + + + com.google.truth + truth + diff --git a/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java b/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java index 2795436415f3..1be5a93910c9 100644 --- a/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java +++ b/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java @@ -50,7 +50,7 @@ public class Emulator { public static Emulator createBundled() throws IOException { String resourcePath = String.format( - "gcloud/bigtable-%s/platform/bigtable-emulator/cbtemulator", getPlatform()); + "/gcloud/bigtable-%s/platform/bigtable-emulator/cbtemulator", getPlatform()); URL packagedEmulator = Emulator.class.getResource(resourcePath); @@ -155,7 +155,8 @@ private static String getPlatform() { case "x86": arch = "x86"; break; - case "xmd64": + case "x86_64": + case "amd64": arch = "x86_64"; break; default: diff --git a/google-cloud-testing/google-cloud-bigtable-emulator/src/test/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRuleTest.java b/google-cloud-testing/google-cloud-bigtable-emulator/src/test/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRuleTest.java new file mode 100644 index 000000000000..fd92516273c1 --- /dev/null +++ b/google-cloud-testing/google-cloud-bigtable-emulator/src/test/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRuleTest.java @@ -0,0 +1,120 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.emulator.v2; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.api.gax.core.NoCredentialsProvider; +import com.google.api.gax.grpc.GrpcTransportChannel; +import com.google.api.gax.rpc.FixedTransportChannelProvider; +import com.google.bigtable.admin.v2.InstanceName; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; +import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; +import com.google.cloud.bigtable.admin.v2.models.Table; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import com.google.protobuf.ByteString; +import java.io.IOException; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class BigtableEmulatorRuleTest { + + @Rule + public BigtableEmulatorRule bigtableRule = new BigtableEmulatorRule(); + private BigtableTableAdminClient tableAdminClient; + private BigtableDataClient dataClient; + + @Before + public void setUp() throws Exception { + tableAdminClient = createTableAdminClient(); + dataClient = createDataClient(); + } + + @After + public void tearDown() throws Exception { + tableAdminClient.close(); + dataClient.close(); + } + + private BigtableTableAdminClient createTableAdminClient() throws IOException { + BigtableTableAdminSettings.Builder settingsBuilder = BigtableTableAdminSettings.newBuilder() + .setInstanceName(InstanceName.of("fake-project", "fake-instance")); + + settingsBuilder.stubSettings() + .setTransportChannelProvider( + FixedTransportChannelProvider.create( + GrpcTransportChannel.create(bigtableRule.getAdminChannel()) + ) + ) + .setCredentialsProvider(NoCredentialsProvider.create()); + + BigtableTableAdminSettings settings = settingsBuilder.build(); + + return BigtableTableAdminClient.create(settings); + } + + private BigtableDataClient createDataClient() throws IOException { + BigtableDataSettings.Builder settingsBuilder = BigtableDataSettings.newBuilder() + .setInstanceName(com.google.cloud.bigtable.data.v2.models.InstanceName + .of("fake-project", "fake-instance")); + + settingsBuilder + .setTransportChannelProvider( + FixedTransportChannelProvider.create( + GrpcTransportChannel.create(bigtableRule.getAdminChannel()) + ) + ) + .setCredentialsProvider(NoCredentialsProvider.create()); + + BigtableDataSettings settings = settingsBuilder.build(); + + return BigtableDataClient.create(settings); + } + + @Test + public void testTableAdminClient() { + Table table = tableAdminClient.createTable( + CreateTableRequest.of("fake-table") + ); + + assertThat(table.getId()).isEqualTo("fake-table"); + } + + @Test + public void testDataClient() throws Exception { + tableAdminClient.createTable( + CreateTableRequest.of("fake-table") + .addFamily("cf") + ); + + dataClient.mutateRowCallable().call( + RowMutation.create("fake-table", "test") + .setCell("cf", "qualifier", "value") + ); + + Row row = dataClient.readRowAsync("fake-table", "test").get(); + assertThat(row.getCells().get(0).getValue()).isEqualTo(ByteString.copyFromUtf8("value")); + } +} \ No newline at end of file diff --git a/google-cloud-testing/google-cloud-bigtable-emulator/src/test/java/com/google/cloud/bigtable/emulator/v2/EmulatorTest.java b/google-cloud-testing/google-cloud-bigtable-emulator/src/test/java/com/google/cloud/bigtable/emulator/v2/EmulatorTest.java new file mode 100644 index 000000000000..b9156716a7fe --- /dev/null +++ b/google-cloud-testing/google-cloud-bigtable-emulator/src/test/java/com/google/cloud/bigtable/emulator/v2/EmulatorTest.java @@ -0,0 +1,124 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.emulator.v2; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.api.gax.core.NoCredentialsProvider; +import com.google.api.gax.grpc.GrpcTransportChannel; +import com.google.api.gax.rpc.FixedTransportChannelProvider; +import com.google.bigtable.admin.v2.InstanceName; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; +import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; +import com.google.cloud.bigtable.admin.v2.models.Table; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import com.google.protobuf.ByteString; +import java.io.IOException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class EmulatorTest { + + private Emulator emulator; + private BigtableTableAdminClient tableAdminClient; + private BigtableDataClient dataClient; + + + @Before + public void setUp() throws Exception { + emulator = Emulator.createBundled(); + emulator.start(); + tableAdminClient = createTableAdminClient(); + dataClient = createDataClient(); + } + + @After + public void tearDown() throws Exception { + tableAdminClient.close(); + dataClient.close(); + emulator.stop(); + emulator = null; + } + + private BigtableTableAdminClient createTableAdminClient() throws IOException { + BigtableTableAdminSettings.Builder settingsBuilder = BigtableTableAdminSettings.newBuilder() + .setInstanceName(InstanceName.of("fake-project", "fake-instance")); + + settingsBuilder.stubSettings() + .setTransportChannelProvider( + FixedTransportChannelProvider.create( + GrpcTransportChannel.create(emulator.getAdminChannel()) + ) + ) + .setCredentialsProvider(NoCredentialsProvider.create()); + + BigtableTableAdminSettings settings = settingsBuilder.build(); + + return BigtableTableAdminClient.create(settings); + } + + private BigtableDataClient createDataClient() throws IOException { + BigtableDataSettings.Builder settingsBuilder = BigtableDataSettings.newBuilder() + .setInstanceName(com.google.cloud.bigtable.data.v2.models.InstanceName + .of("fake-project", "fake-instance")); + + settingsBuilder + .setTransportChannelProvider( + FixedTransportChannelProvider.create( + GrpcTransportChannel.create(emulator.getAdminChannel()) + ) + ) + .setCredentialsProvider(NoCredentialsProvider.create()); + + BigtableDataSettings settings = settingsBuilder.build(); + + return BigtableDataClient.create(settings); + } + + @Test + public void testTableAdminClient() { + Table table = tableAdminClient.createTable( + CreateTableRequest.of("fake-table") + ); + + assertThat(table.getId()).isEqualTo("fake-table"); + } + + @Test + public void testDataClient() throws Exception { + tableAdminClient.createTable( + CreateTableRequest.of("fake-table") + .addFamily("cf") + ); + + dataClient.mutateRowCallable().call( + RowMutation.create("fake-table", "test") + .setCell("cf", "qualifier", "value") + ); + + Row row = dataClient.readRowAsync("fake-table", "test").get(); + assertThat(row.getCells().get(0).getValue()).isEqualTo(ByteString.copyFromUtf8("value")); + } + +} From 51c61c8cf2bd7689428bf999cc03d9e89ee5d8dd Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Fri, 19 Oct 2018 17:31:34 -0400 Subject: [PATCH 03/21] visibility tweaks --- .../bigtable/emulator/v2/BigtableEmulatorRule.java | 2 +- .../src/main/java/com/google/cloud/Component.java | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRule.java b/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRule.java index ff495b7609ea..dbe4df8099af 100644 --- a/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRule.java +++ b/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRule.java @@ -27,7 +27,7 @@ public static BigtableEmulatorRule create() { return new BigtableEmulatorRule(); } - protected BigtableEmulatorRule() { } + BigtableEmulatorRule() { } @Override protected void before() throws Throwable { diff --git a/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/Component.java b/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/Component.java index b52b4f2bf4b7..498446d51e00 100644 --- a/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/Component.java +++ b/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/Component.java @@ -8,12 +8,12 @@ /** Representation of a gcloud component */ @AutoValue -public abstract class Component { - public static Component create(String id, String checksum, URL source, String fileType) { +abstract class Component { + static Component create(String id, String checksum, URL source, String fileType) { return new AutoValue_Component(id, checksum, source, fileType); } - public static Component fromJson(URL baseUrl, JsonObject componentObj) throws IOException { + static Component fromJson(URL baseUrl, JsonObject componentObj) throws IOException { String id = componentObj.get("id").getAsString(); JsonElement data = componentObj.get("data"); @@ -29,8 +29,8 @@ public static Component fromJson(URL baseUrl, JsonObject componentObj) throws IO ); } - public abstract String getId(); - public abstract String getChecksum(); - public abstract URL getSource(); - public abstract String getFileType(); + abstract String getId(); + abstract String getChecksum(); + abstract URL getSource(); + abstract String getFileType(); } From c21a61ed139103521931ad7d28510e9683b830d7 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Wed, 31 Oct 2018 16:16:57 -0400 Subject: [PATCH 04/21] fixes --- .../google-cloud-bigtable-emulator/pom.xml | 16 ++- .../emulator/v2/BigtableEmulatorRule.java | 34 +++++ .../cloud/bigtable/emulator/v2/Emulator.java | 100 ++++++++++--- .../emulator/v2/BigtableEmulatorRuleTest.java | 131 ++++++++---------- .../bigtable/emulator/v2/EmulatorTest.java | 128 ++++++++--------- .../google-cloud-gcloud-maven-plugin/pom.xml | 8 +- .../google/cloud/DownloadComponentsMojo.java | 29 +++- 7 files changed, 278 insertions(+), 168 deletions(-) diff --git a/google-cloud-testing/google-cloud-bigtable-emulator/pom.xml b/google-cloud-testing/google-cloud-bigtable-emulator/pom.xml index 3bf0c2781dcc..a47712cdaa59 100644 --- a/google-cloud-testing/google-cloud-bigtable-emulator/pom.xml +++ b/google-cloud-testing/google-cloud-bigtable-emulator/pom.xml @@ -43,6 +43,7 @@ + io.grpc grpc-core @@ -50,6 +51,13 @@ provided + + io.grpc + grpc-netty-shaded + 1.13.1 + provided + + com.google.api api-common @@ -63,14 +71,14 @@ - com.google.cloud - google-cloud-bigtable-admin + com.google.api.grpc + grpc-google-cloud-bigtable-v2 test - com.google.cloud - google-cloud-bigtable + com.google.api.grpc + grpc-google-cloud-bigtable-admin-v2 test diff --git a/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRule.java b/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRule.java index dbe4df8099af..052dc43b6d5c 100644 --- a/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRule.java +++ b/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRule.java @@ -19,6 +19,27 @@ import io.grpc.ManagedChannel; import org.junit.rules.ExternalResource; +/** + * The BigtableEmulatorRule manages the lifecycle of the Bigtable {@link Emulator}. Before the + * start of a test, the emulator will be started on a random port and will be shutdown after the + * test finishes. + * + *

Example usage: + * + * {@code + * {@literal @RunWith(JUnit4.class)} + * public class MyTest { + * {@literal @Rule} + * public final BigtableEmulatorRule bigtableEmulator = BigtableEmulatorRule.create(); + * + * {@literal @Test} + * public void testUsingEmulator() { + * ManagedChannel adminChannel = bigtableEmulator.getAdminChannel(); + * // Do something with channel + * } + * } + * } + */ @BetaApi("Surface for Bigtable emulator is not yet stable") public class BigtableEmulatorRule extends ExternalResource { private Emulator emulator; @@ -29,26 +50,39 @@ public static BigtableEmulatorRule create() { BigtableEmulatorRule() { } + /** Initializes the Bigtable emulator before a test runs. */ @Override protected void before() throws Throwable { emulator = Emulator.createBundled(); emulator.start(); } + /** Stops the Bigtable emulator after a test finishes. */ @Override protected void after() { emulator.stop(); emulator = null; } + /** + * Gets a {@link ManagedChannel} connected to the Emulator. The channel is configured for data + * operations. + */ public ManagedChannel getDataChannel() { return emulator.getDataChannel(); } + /** + * Gets a {@link ManagedChannel} connected to the Emulator. This channel should be used for admin + * operations. + */ public ManagedChannel getAdminChannel() { return emulator.getAdminChannel(); } + /** + * Gets the port of the emulator, allowing the caller to create their own {@link ManagedChannel}. + */ public int getPort() { return emulator.getPort(); } diff --git a/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java b/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java index 1be5a93910c9..f39b8ef25944 100644 --- a/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java +++ b/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java @@ -35,6 +35,13 @@ import java.util.logging.Level; import java.util.logging.Logger; +/** + * Wraps the Bigtable emulator in a java api. + * + *

This class will use the golang binaries embedded in this jar to launch the emulator as an + * external process and redirect its output to a {@link Logger}. + * + */ @BetaApi("Surface for Bigtable emulator is not yet stable") public class Emulator { @@ -43,11 +50,17 @@ public class Emulator { private final Path executable; private Process process; private boolean isStopped = true; + private Thread shutdownHook; private int port; private ManagedChannel dataChannel; private ManagedChannel adminChannel; + /** + * Create a new instance of emulator. The emulator will use the bundled binaries in this jar. + * Please note that the emulator is created in a stopped state, please use {@link #start()} after + * creating it. + */ public static Emulator createBundled() throws IOException { String resourcePath = String.format( "/gcloud/bigtable-%s/platform/bigtable-emulator/cbtemulator", getPlatform()); @@ -76,7 +89,11 @@ private Emulator(Path executable) { this.executable = executable; } - public void start() throws Exception { + /** Starts the emulator process and waits for it to be ready. */ + public synchronized void start() throws IOException, TimeoutException, InterruptedException { + if (!isStopped) { + throw new IllegalStateException("Emulator is already started"); + } this.port = getAvailablePort(); process = Runtime.getRuntime().exec(executable + " -port " + "" + port); @@ -84,7 +101,7 @@ public void start() throws Exception { pipeStreamToLog(process.getErrorStream(), Level.WARNING); isStopped = false; - Runtime.getRuntime().addShutdownHook(new Thread() { + shutdownHook = new Thread() { @Override public void run() { if (!isStopped) { @@ -92,25 +109,40 @@ public void run() { process.destroy(); } } - }); - - waitForPort(port); + }; - dataChannel = newChannelBuilder(port) - .maxInboundMessageSize(256 * 1024 * 1024) - .build(); + Runtime.getRuntime().addShutdownHook(shutdownHook); - adminChannel = newChannelBuilder(port) - .build(); + waitForPort(port); } - public void stop() { + /** Stops the emulator process. */ + public synchronized void stop() { + if (isStopped) { + throw new IllegalStateException("Emulator already stopped"); + } + try { - dataChannel.shutdownNow(); - adminChannel.shutdownNow(); + Runtime.getRuntime().removeShutdownHook(shutdownHook); + shutdownHook = null; + + // Shutdown channels in parallel + if (dataChannel != null) { + dataChannel.shutdownNow(); + } + if (adminChannel != null) { + adminChannel.shutdownNow(); + } - dataChannel.awaitTermination(1, TimeUnit.MINUTES); - adminChannel.awaitTermination(1, TimeUnit.MINUTES); + // Then wait for actual shutdown + if (dataChannel != null) { + dataChannel.awaitTermination(1, TimeUnit.MINUTES); + dataChannel = null; + } + if (adminChannel != null) { + adminChannel.awaitTermination(1, TimeUnit.MINUTES); + adminChannel = null; + } } catch (InterruptedException e) { LOGGER.warning("Interrupted while waiting for client channels to close"); Thread.currentThread().interrupt(); @@ -120,19 +152,41 @@ public void stop() { } } - public int getPort() { + public synchronized int getPort() { + if (isStopped) { + throw new IllegalStateException("Emulator is not running"); + } return port; } - public ManagedChannel getDataChannel() { + public synchronized ManagedChannel getDataChannel() { + if (isStopped){ + throw new IllegalStateException("Emulator is not running"); + } + + if (dataChannel == null) { + dataChannel = newChannelBuilder(port) + .maxInboundMessageSize(256 * 1024 * 1024) + .build(); + } return dataChannel; } - public ManagedChannel getAdminChannel() { + public synchronized ManagedChannel getAdminChannel() { + if (isStopped) { + throw new IllegalStateException("Emulator is not running"); + } + + if (adminChannel == null) { + adminChannel = newChannelBuilder(port) + .build(); + } return adminChannel; } // + + /** Gets the current platform, which will be used to select the appropriate emulator binary. */ private static String getPlatform() { String unformattedOs = System.getProperty("os.name", "unknown").toLowerCase(Locale.ENGLISH); String os; @@ -166,6 +220,7 @@ private static String getPlatform() { return os + "-" + arch; } + /** Gets a random open port number. */ private static int getAvailablePort() { try (ServerSocket serverSocket = new ServerSocket(0)) { return serverSocket.getLocalPort(); @@ -174,6 +229,7 @@ private static int getAvailablePort() { } } + /** Waits for a port to open. It's used to wait for the emulator's gRPC server to be ready. */ private void waitForPort(int port) throws InterruptedException, TimeoutException { for (int i = 0; i < 100; i++) { try (Socket ignored = new Socket("localhost", port)) { @@ -186,6 +242,7 @@ private void waitForPort(int port) throws InterruptedException, TimeoutException throw new TimeoutException("Timed out waiting for server to start"); } + /** Creates a {@link io.grpc.ManagedChannelBuilder} preconfigured for the emulator's port. */ private ManagedChannelBuilder newChannelBuilder(int port) { // NOTE: usePlaintext is currently @ExperimentalAPI. // See https://github.com/grpc/grpc-java/issues/1772 for discussion @@ -193,6 +250,7 @@ private ManagedChannelBuilder newChannelBuilder(int port) { .usePlaintext(); } + /** Creates a thread that will pipe an {@link java.io.InputStream} to this class' Logger. */ private static void pipeStreamToLog(final InputStream stream, final Level level) { final BufferedReader reader = new BufferedReader(new InputStreamReader(stream)); @@ -207,7 +265,9 @@ public void run() { LOGGER.log(level, line); } } catch (IOException e) { - LOGGER.log(Level.WARNING, "Failed to read process stream", e); + if (!"Stream closed".equals(e.getMessage())) { + LOGGER.log(Level.WARNING, "Failed to read process stream", e); + } } } }); @@ -215,4 +275,4 @@ public void run() { thread.start(); } // -} \ No newline at end of file +} diff --git a/google-cloud-testing/google-cloud-bigtable-emulator/src/test/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRuleTest.java b/google-cloud-testing/google-cloud-bigtable-emulator/src/test/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRuleTest.java index fd92516273c1..de07230d603c 100644 --- a/google-cloud-testing/google-cloud-bigtable-emulator/src/test/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRuleTest.java +++ b/google-cloud-testing/google-cloud-bigtable-emulator/src/test/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRuleTest.java @@ -17,21 +17,20 @@ import static com.google.common.truth.Truth.assertThat; -import com.google.api.gax.core.NoCredentialsProvider; -import com.google.api.gax.grpc.GrpcTransportChannel; -import com.google.api.gax.rpc.FixedTransportChannelProvider; -import com.google.bigtable.admin.v2.InstanceName; -import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; -import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; -import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; -import com.google.cloud.bigtable.admin.v2.models.Table; -import com.google.cloud.bigtable.data.v2.BigtableDataClient; -import com.google.cloud.bigtable.data.v2.BigtableDataSettings; -import com.google.cloud.bigtable.data.v2.models.Row; -import com.google.cloud.bigtable.data.v2.models.RowMutation; +import com.google.bigtable.admin.v2.BigtableTableAdminGrpc; +import com.google.bigtable.admin.v2.BigtableTableAdminGrpc.BigtableTableAdminBlockingStub; +import com.google.bigtable.admin.v2.ColumnFamily; +import com.google.bigtable.admin.v2.CreateTableRequest; +import com.google.bigtable.admin.v2.Table; +import com.google.bigtable.v2.BigtableGrpc; +import com.google.bigtable.v2.BigtableGrpc.BigtableBlockingStub; +import com.google.bigtable.v2.MutateRowRequest; +import com.google.bigtable.v2.Mutation; +import com.google.bigtable.v2.Mutation.SetCell; +import com.google.bigtable.v2.ReadRowsRequest; +import com.google.bigtable.v2.ReadRowsResponse; import com.google.protobuf.ByteString; -import java.io.IOException; -import org.junit.After; +import java.util.Iterator; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -43,78 +42,68 @@ public class BigtableEmulatorRuleTest { @Rule public BigtableEmulatorRule bigtableRule = new BigtableEmulatorRule(); - private BigtableTableAdminClient tableAdminClient; - private BigtableDataClient dataClient; + private BigtableTableAdminBlockingStub tableAdminStub; + private BigtableBlockingStub dataStub; - @Before - public void setUp() throws Exception { - tableAdminClient = createTableAdminClient(); - dataClient = createDataClient(); - } - - @After - public void tearDown() throws Exception { - tableAdminClient.close(); - dataClient.close(); - } - - private BigtableTableAdminClient createTableAdminClient() throws IOException { - BigtableTableAdminSettings.Builder settingsBuilder = BigtableTableAdminSettings.newBuilder() - .setInstanceName(InstanceName.of("fake-project", "fake-instance")); - - settingsBuilder.stubSettings() - .setTransportChannelProvider( - FixedTransportChannelProvider.create( - GrpcTransportChannel.create(bigtableRule.getAdminChannel()) - ) - ) - .setCredentialsProvider(NoCredentialsProvider.create()); - - BigtableTableAdminSettings settings = settingsBuilder.build(); - - return BigtableTableAdminClient.create(settings); - } - private BigtableDataClient createDataClient() throws IOException { - BigtableDataSettings.Builder settingsBuilder = BigtableDataSettings.newBuilder() - .setInstanceName(com.google.cloud.bigtable.data.v2.models.InstanceName - .of("fake-project", "fake-instance")); - - settingsBuilder - .setTransportChannelProvider( - FixedTransportChannelProvider.create( - GrpcTransportChannel.create(bigtableRule.getAdminChannel()) - ) - ) - .setCredentialsProvider(NoCredentialsProvider.create()); - - BigtableDataSettings settings = settingsBuilder.build(); - - return BigtableDataClient.create(settings); + @Before + public void setUp() { + tableAdminStub = BigtableTableAdminGrpc.newBlockingStub(bigtableRule.getAdminChannel()); + dataStub = BigtableGrpc.newBlockingStub(bigtableRule.getDataChannel()); } @Test public void testTableAdminClient() { - Table table = tableAdminClient.createTable( - CreateTableRequest.of("fake-table") + Table table = tableAdminStub.createTable( + CreateTableRequest.newBuilder() + .setParent("projects/fake-project/instances/fake-instance") + .setTableId("fake-table") + .setTable( + Table.newBuilder() + .putColumnFamilies("cf", ColumnFamily.getDefaultInstance()) + ) + .build() ); - assertThat(table.getId()).isEqualTo("fake-table"); + assertThat(table.getName()) + .isEqualTo("projects/fake-project/instances/fake-instance/tables/fake-table"); } @Test public void testDataClient() throws Exception { - tableAdminClient.createTable( - CreateTableRequest.of("fake-table") - .addFamily("cf") + tableAdminStub.createTable( + CreateTableRequest.newBuilder() + .setParent("projects/fake-project/instances/fake-instance") + .setTableId("fake-table") + .setTable( + Table.newBuilder() + .putColumnFamilies("cf", ColumnFamily.getDefaultInstance()) + ) + .build() + ); + + dataStub.mutateRow( + MutateRowRequest.newBuilder() + .setTableName("projects/fake-project/instances/fake-instance/tables/fake-table") + .setRowKey(ByteString.copyFromUtf8("fake-key")) + .addMutations( + Mutation.newBuilder().setSetCell( + SetCell.newBuilder() + .setFamilyName("cf") + .setColumnQualifier(ByteString.EMPTY) + .setValue(ByteString.copyFromUtf8("value")) + ) + ) + .build() ); - dataClient.mutateRowCallable().call( - RowMutation.create("fake-table", "test") - .setCell("cf", "qualifier", "value") + Iterator results = dataStub.readRows( + ReadRowsRequest.newBuilder() + .setTableName("projects/fake-project/instances/fake-instance/tables/fake-table") + .build() ); - Row row = dataClient.readRowAsync("fake-table", "test").get(); - assertThat(row.getCells().get(0).getValue()).isEqualTo(ByteString.copyFromUtf8("value")); + ReadRowsResponse row = results.next(); + assertThat(row.getChunks(0).getValue()).isEqualTo(ByteString.copyFromUtf8("value")); } } \ No newline at end of file diff --git a/google-cloud-testing/google-cloud-bigtable-emulator/src/test/java/com/google/cloud/bigtable/emulator/v2/EmulatorTest.java b/google-cloud-testing/google-cloud-bigtable-emulator/src/test/java/com/google/cloud/bigtable/emulator/v2/EmulatorTest.java index b9156716a7fe..46139b4f0377 100644 --- a/google-cloud-testing/google-cloud-bigtable-emulator/src/test/java/com/google/cloud/bigtable/emulator/v2/EmulatorTest.java +++ b/google-cloud-testing/google-cloud-bigtable-emulator/src/test/java/com/google/cloud/bigtable/emulator/v2/EmulatorTest.java @@ -17,20 +17,20 @@ import static com.google.common.truth.Truth.assertThat; -import com.google.api.gax.core.NoCredentialsProvider; -import com.google.api.gax.grpc.GrpcTransportChannel; -import com.google.api.gax.rpc.FixedTransportChannelProvider; -import com.google.bigtable.admin.v2.InstanceName; -import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; -import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; -import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; -import com.google.cloud.bigtable.admin.v2.models.Table; -import com.google.cloud.bigtable.data.v2.BigtableDataClient; -import com.google.cloud.bigtable.data.v2.BigtableDataSettings; -import com.google.cloud.bigtable.data.v2.models.Row; -import com.google.cloud.bigtable.data.v2.models.RowMutation; +import com.google.bigtable.admin.v2.BigtableTableAdminGrpc; +import com.google.bigtable.admin.v2.BigtableTableAdminGrpc.BigtableTableAdminBlockingStub; +import com.google.bigtable.admin.v2.ColumnFamily; +import com.google.bigtable.admin.v2.CreateTableRequest; +import com.google.bigtable.admin.v2.Table; +import com.google.bigtable.v2.BigtableGrpc; +import com.google.bigtable.v2.BigtableGrpc.BigtableBlockingStub; +import com.google.bigtable.v2.MutateRowRequest; +import com.google.bigtable.v2.Mutation; +import com.google.bigtable.v2.Mutation.SetCell; +import com.google.bigtable.v2.ReadRowsRequest; +import com.google.bigtable.v2.ReadRowsResponse; import com.google.protobuf.ByteString; -import java.io.IOException; +import java.util.Iterator; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -41,84 +41,76 @@ public class EmulatorTest { private Emulator emulator; - private BigtableTableAdminClient tableAdminClient; - private BigtableDataClient dataClient; + private BigtableTableAdminBlockingStub tableAdminStub; + private BigtableBlockingStub dataStub; @Before public void setUp() throws Exception { emulator = Emulator.createBundled(); emulator.start(); - tableAdminClient = createTableAdminClient(); - dataClient = createDataClient(); + tableAdminStub = BigtableTableAdminGrpc.newBlockingStub(emulator.getAdminChannel()); + dataStub = BigtableGrpc.newBlockingStub(emulator.getDataChannel()); } @After - public void tearDown() throws Exception { - tableAdminClient.close(); - dataClient.close(); + public void tearDown() { emulator.stop(); emulator = null; } - private BigtableTableAdminClient createTableAdminClient() throws IOException { - BigtableTableAdminSettings.Builder settingsBuilder = BigtableTableAdminSettings.newBuilder() - .setInstanceName(InstanceName.of("fake-project", "fake-instance")); - - settingsBuilder.stubSettings() - .setTransportChannelProvider( - FixedTransportChannelProvider.create( - GrpcTransportChannel.create(emulator.getAdminChannel()) - ) - ) - .setCredentialsProvider(NoCredentialsProvider.create()); - - BigtableTableAdminSettings settings = settingsBuilder.build(); - - return BigtableTableAdminClient.create(settings); - } - - private BigtableDataClient createDataClient() throws IOException { - BigtableDataSettings.Builder settingsBuilder = BigtableDataSettings.newBuilder() - .setInstanceName(com.google.cloud.bigtable.data.v2.models.InstanceName - .of("fake-project", "fake-instance")); - - settingsBuilder - .setTransportChannelProvider( - FixedTransportChannelProvider.create( - GrpcTransportChannel.create(emulator.getAdminChannel()) - ) - ) - .setCredentialsProvider(NoCredentialsProvider.create()); - - BigtableDataSettings settings = settingsBuilder.build(); - - return BigtableDataClient.create(settings); - } - @Test public void testTableAdminClient() { - Table table = tableAdminClient.createTable( - CreateTableRequest.of("fake-table") + Table table = tableAdminStub.createTable( + CreateTableRequest.newBuilder() + .setParent("projects/fake-project/instances/fake-instance") + .setTableId("fake-table") + .setTable( + Table.newBuilder() + .putColumnFamilies("cf", ColumnFamily.getDefaultInstance()) + ) + .build() ); - assertThat(table.getId()).isEqualTo("fake-table"); + assertThat(table.getName()) + .isEqualTo("projects/fake-project/instances/fake-instance/tables/fake-table"); } @Test - public void testDataClient() throws Exception { - tableAdminClient.createTable( - CreateTableRequest.of("fake-table") - .addFamily("cf") + public void testDataClient() { + tableAdminStub.createTable( + CreateTableRequest.newBuilder() + .setParent("projects/fake-project/instances/fake-instance") + .setTableId("fake-table") + .setTable( + Table.newBuilder() + .putColumnFamilies("cf", ColumnFamily.getDefaultInstance()) + ) + .build() ); - dataClient.mutateRowCallable().call( - RowMutation.create("fake-table", "test") - .setCell("cf", "qualifier", "value") + dataStub.mutateRow( + MutateRowRequest.newBuilder() + .setTableName("projects/fake-project/instances/fake-instance/tables/fake-table") + .setRowKey(ByteString.copyFromUtf8("fake-key")) + .addMutations( + Mutation.newBuilder().setSetCell( + SetCell.newBuilder() + .setFamilyName("cf") + .setColumnQualifier(ByteString.EMPTY) + .setValue(ByteString.copyFromUtf8("value")) + ) + ) + .build() ); - Row row = dataClient.readRowAsync("fake-table", "test").get(); - assertThat(row.getCells().get(0).getValue()).isEqualTo(ByteString.copyFromUtf8("value")); - } + Iterator results = dataStub.readRows( + ReadRowsRequest.newBuilder() + .setTableName("projects/fake-project/instances/fake-instance/tables/fake-table") + .build() + ); + ReadRowsResponse row = results.next(); + assertThat(row.getChunks(0).getValue()).isEqualTo(ByteString.copyFromUtf8("value")); + } } diff --git a/google-cloud-testing/google-cloud-gcloud-maven-plugin/pom.xml b/google-cloud-testing/google-cloud-gcloud-maven-plugin/pom.xml index 4262b1f6164c..3a6d36e7caaa 100644 --- a/google-cloud-testing/google-cloud-gcloud-maven-plugin/pom.xml +++ b/google-cloud-testing/google-cloud-gcloud-maven-plugin/pom.xml @@ -11,7 +11,7 @@ google-cloud-gcloud-maven-plugin - 0.66.1-alpha-SNAPSHOT + 0.67.1-alpha-SNAPSHOT maven-plugin Experimental Maven plugin to interact with the Google Cloud SDK (https://cloud.google.com/sdk/). Currently this plugin is meant to be an internal implementation @@ -36,6 +36,12 @@ + + org.apache.maven + maven-core + 3.0 + + org.apache.maven maven-plugin-api diff --git a/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/DownloadComponentsMojo.java b/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/DownloadComponentsMojo.java index 73ed171f5ebc..ad1922ab7c34 100644 --- a/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/DownloadComponentsMojo.java +++ b/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/DownloadComponentsMojo.java @@ -47,6 +47,7 @@ import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; import org.apache.commons.compress.utils.IOUtils; +import org.apache.maven.execution.MavenSession; import org.apache.maven.plugin.AbstractMojo; import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugins.annotations.Mojo; @@ -55,7 +56,9 @@ /** * Goal that downloads gcloud components and embeds them as resources in a jar. This is mainly - * intended to be used to wrapped native emulators in jars. + * intended to be used to wrapped native emulators in jars. This Mojo makes extensive use of caching + * which makes it ok to include in the regular build cycle. The initial manifest fetch is cached for + * 2 hours and the actual binaries are downloaded only when their checksum changed in the manifest. */ @Mojo(name = "download") public class DownloadComponentsMojo extends AbstractMojo { @@ -71,10 +74,17 @@ public class DownloadComponentsMojo extends AbstractMojo { @Parameter(defaultValue = "${project.build.outputDirectory}/gcloud", required = true) private File destinationDir; - @Parameter(defaultValue = "true", required = true, property = "gcloud.download.force") + @Parameter(defaultValue = "false", required = true, property = "gcloud.download.force") private boolean forceRefresh; + @Parameter(defaultValue = "${session}", readonly = true) + private MavenSession session; + public void execute() throws MojoExecutionException { + if (session.isOffline() && forceRefresh) { + throw new MojoExecutionException("Can't force refresh when offline"); + } + // Ensure that the output directory exists destinationDir.mkdirs(); @@ -147,13 +157,24 @@ private void updateCachedManifest() throws IOException { URL manifestUrl = getManifestUrl(); File localCache = getManifestCache(); - if (!forceRefresh && localCache.exists() - && new Date().getTime() - localCache.lastModified() < STALE_MS) { + boolean isStale = !localCache.exists() + || new Date().getTime() - localCache.lastModified() < STALE_MS; + + if (localCache.exists() && session.isOffline()) { + if (isStale) { + getLog().info("Using stale manifest because offline mode is enabled"); + } + } + if (!forceRefresh && !isStale) { getLog().debug("Manifest is up to date, skipping manifest download"); return; } + if (session.isOffline()) { + throw new IllegalStateException("Can't download manifest in offline mode"); + } + getLog().debug("Downloading fresh manifest"); File tempFile = File.createTempFile(localCache.getName(), ""); From a294c5a4e70cb3ab90373e16cfb8886bba8f7120 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Wed, 31 Oct 2018 16:52:06 -0400 Subject: [PATCH 05/21] readme + bom --- google-cloud-bom/pom.xml | 5 + .../google-cloud-bigtable-emulator/README.md | 181 ++++++++++++++++++ 2 files changed, 186 insertions(+) create mode 100644 google-cloud-testing/google-cloud-bigtable-emulator/README.md diff --git a/google-cloud-bom/pom.xml b/google-cloud-bom/pom.xml index 5fde7e9c1de4..67ca2b441972 100644 --- a/google-cloud-bom/pom.xml +++ b/google-cloud-bom/pom.xml @@ -266,6 +266,11 @@ grpc-google-cloud-bigtable-admin-v2 0.33.1-SNAPSHOT + + com.google.cloud + google-cloud-bigtable-emulator + 0.68.1-alpha-SNAPSHOT + com.google.cloud google-cloud-bigquery diff --git a/google-cloud-testing/google-cloud-bigtable-emulator/README.md b/google-cloud-testing/google-cloud-bigtable-emulator/README.md new file mode 100644 index 000000000000..9bc5ceb6160a --- /dev/null +++ b/google-cloud-testing/google-cloud-bigtable-emulator/README.md @@ -0,0 +1,181 @@ +# Google Cloud Java Emulator for Bigtable + +A Java wrapper for the [Cloud Bigtable][cloud-bigtable] emulator. This +wrapper bundles the native Bigtable emulator and exposes a simple Java +interface to ease writing tests. Please note that this wrapper is under +heavy development and APIs may change in the future. + +[![Kokoro CI](http://storage.googleapis.com/cloud-devrel-public/java/badges/google-cloud-java/master.svg)](http://storage.googleapis.com/cloud-devrel-public/java/badges/google-cloud-java/master.html) +[![Maven](https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-bigtable-emulator.svg)](https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-bigtable-emulator.svg) +[![Codacy Badge](https://api.codacy.com/project/badge/grade/9da006ad7c3a4fe1abd142e77c003917)](https://www.codacy.com/app/mziccard/google-cloud-java) + +## Quickstart + +[//]: # ({x-version-update-start:google-cloud-bom:released}) +If you are using Maven, add this to your pom.xml file +```xml + + + + com.google.cloud + google-cloud-bom + 0.68.1-alpha-SNAPSHOT + pom + import + + + + + + + com.google.cloud + google-cloud-bigtable + + + + com.google.cloud + google-cloud-bigtable-admin + + + + com.google.cloud + google-cloud-bigtable-emulator + test + + + + junit + junit + 4.12 + +``` + +If you are using Gradle, add this to your dependencies +```Groovy +compile 'com.google.cloud:google-cloud-bigtable:0.68.0-alpha' +compile 'com.google.cloud:google-cloud-bigtable-admin:0.68.0-alpha' +testCompile 'com.google.cloud:google-cloud-bigtable-emulator:0.68.0-alpha' +testCompile 'junit:junit:4.12' +``` +If you are using SBT, add this to your dependencies +```Scala +libraryDependencies += "com.google.cloud" % "google-cloud-bigtable" % "0.68.0-alpha" +libraryDependencies += "com.google.cloud" % "google-cloud-bigtable-admin" % "0.68.0-alpha" +libraryDependencies += "com.google.cloud" % "google-cloud-bigtable-emulator" % "0.68.0-alpha" % Test +libraryDependencies += "junit" % "junit" % "4.12" % Test +``` +[//]: # ({x-version-update-end}) + +## Getting Started + +Here is a code snippet showing a simple JUnit test. Add the following imports +at the top of your file: + +```java +import com.google.api.core.ApiFuture; +import com.google.api.gax.core.NoCredentialsProvider; +import com.google.api.gax.grpc.GrpcTransportChannel; +import com.google.api.gax.rpc.FixedTransportChannelProvider; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; +import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import com.google.cloud.bigtable.emulator.v2.BigtableEmulatorRule; +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +``` + +Then, to make a query to Bigtable, use the following code: +```java +@RunWith(JUnit4.class) +public class ExampleTest { + // Initialize the emulator Rule + @Rule + public final BigtableEmulatorRule bigtableEmulator = BigtableEmulatorRule.create(); + + // Clients that will be connected to the emulator + private BigtableTableAdminClient tableAdminClient; + private BigtableDataClient dataClient; + + @Before + public void setUp() throws IOException { + // Initialize the clients to connect to the emulator + BigtableTableAdminSettings.Builder tableAdminSettings = BigtableTableAdminSettings.newBuilder() + .setInstanceName(com.google.bigtable.admin.v2.InstanceName.of("fake-project", "fake-instance")); + tableAdminSettings.stubSettings() + .setCredentialsProvider(NoCredentialsProvider.create()) + .setTransportChannelProvider( + FixedTransportChannelProvider.create( + GrpcTransportChannel.create(bigtableEmulator.getAdminChannel()) + ) + ); + tableAdminClient = BigtableTableAdminClient.create(tableAdminSettings.build()); + + BigtableDataSettings.Builder dataSettings = BigtableDataSettings.newBuilder() + .setInstanceName(com.google.cloud.bigtable.data.v2.models.InstanceName.of("fake-project", "fake-instance")) + .setCredentialsProvider(NoCredentialsProvider.create()) + .setTransportChannelProvider( + FixedTransportChannelProvider.create( + GrpcTransportChannel.create(bigtableEmulator.getDataChannel()) + ) + ); + dataClient = BigtableDataClient.create(dataSettings.build()); + + // Create a test table that can be used in tests + tableAdminClient.createTable( + CreateTableRequest.of("fake-table") + .addFamily("cf") + ); + } + + @Test + public void testWriteRead() throws ExecutionException, InterruptedException { + ApiFuture mutateFuture = dataClient.mutateRowAsync( + RowMutation.create("fake-table", "fake-key") + .setCell("cf", "col", "value") + ); + + mutateFuture.get(); + + ApiFuture rowFuture = dataClient.readRowAsync("fake-table", "fake-key"); + + Assert.assertEquals("value", rowFuture.get().getCells().get(0).getValue().toStringUtf8()); + } +} +``` + +## Java Versions + +Java 7 or above is required for using this client. + +## Versioning + +This library follows [Semantic Versioning](http://semver.org/). + +It is currently in major version zero (`0.y.z`), which means that anything may +change at any time and the public API should not be considered stable. + +## Contributing + +Contributions to this library are always welcome and highly encouraged. + +See [CONTRIBUTING] for more information on how to get started and [DEVELOPING] for a layout of the +codebase. + +## License + +Apache 2.0 - See [LICENSE] for more information. + +[CONTRIBUTING]:https://github.com/googleapis/google-cloud-java/blob/master/CONTRIBUTING.md +[LICENSE]: https://github.com/googleapis/google-cloud-java/blob/master/LICENSE +[cloud-bigtable]: https://cloud.google.com/bigtable/ + From 5ef9eca261a82a736c8545453e52abe4821482d1 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Wed, 31 Oct 2018 16:54:58 -0400 Subject: [PATCH 06/21] more readme --- .../google-cloud-gcloud-maven-plugin/README.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 google-cloud-testing/google-cloud-gcloud-maven-plugin/README.md diff --git a/google-cloud-testing/google-cloud-gcloud-maven-plugin/README.md b/google-cloud-testing/google-cloud-gcloud-maven-plugin/README.md new file mode 100644 index 000000000000..676659962a6c --- /dev/null +++ b/google-cloud-testing/google-cloud-gcloud-maven-plugin/README.md @@ -0,0 +1,9 @@ +# Google Cloud SDK tools + +This module is considered an internal implementation detail of +google-cloud-java and should not be used by other projects. It contains +maven plugins that integrate with the gcloud SDK. + +## Download plugin + +A maven plugin to download components published in the gcloud sdk repository. From 52e211bd112aa82257be04b0687a1854ccc58a8f Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Thu, 1 Nov 2018 13:22:46 -0400 Subject: [PATCH 07/21] create tmp manifest file in same dir as dest to avoid issues with cross filesystem renameTo --- .../main/java/com/google/cloud/DownloadComponentsMojo.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/DownloadComponentsMojo.java b/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/DownloadComponentsMojo.java index ad1922ab7c34..1f6803a08daa 100644 --- a/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/DownloadComponentsMojo.java +++ b/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/DownloadComponentsMojo.java @@ -177,10 +177,11 @@ private void updateCachedManifest() throws IOException { getLog().debug("Downloading fresh manifest"); - File tempFile = File.createTempFile(localCache.getName(), ""); + File tempFile = new File(localCache.toString() + ".tmp"); + tempFile.createNewFile(); try (BufferedInputStream in = new BufferedInputStream(manifestUrl.openStream()); - FileOutputStream fileOutputStream = new FileOutputStream(tempFile)) { + FileOutputStream fileOutputStream = new FileOutputStream(tempFile, false)) { byte dataBuffer[] = new byte[1024]; int bytesRead; while ((bytesRead = in.read(dataBuffer, 0, 1024)) != -1) { From 870d3bb253cc68771d116620f745d9dec1f7eac5 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Thu, 1 Nov 2018 13:48:14 -0400 Subject: [PATCH 08/21] Revert "create tmp manifest file in same dir as dest to avoid issues with cross filesystem renameTo" This reverts commit 52e211b --- .../main/java/com/google/cloud/DownloadComponentsMojo.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/DownloadComponentsMojo.java b/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/DownloadComponentsMojo.java index 1f6803a08daa..ad1922ab7c34 100644 --- a/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/DownloadComponentsMojo.java +++ b/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/DownloadComponentsMojo.java @@ -177,11 +177,10 @@ private void updateCachedManifest() throws IOException { getLog().debug("Downloading fresh manifest"); - File tempFile = new File(localCache.toString() + ".tmp"); - tempFile.createNewFile(); + File tempFile = File.createTempFile(localCache.getName(), ""); try (BufferedInputStream in = new BufferedInputStream(manifestUrl.openStream()); - FileOutputStream fileOutputStream = new FileOutputStream(tempFile, false)) { + FileOutputStream fileOutputStream = new FileOutputStream(tempFile)) { byte dataBuffer[] = new byte[1024]; int bytesRead; while ((bytesRead = in.read(dataBuffer, 0, 1024)) != -1) { From 2c7b08fe0b241cae44a68ec6902a3f4662c79b40 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Thu, 1 Nov 2018 13:55:53 -0400 Subject: [PATCH 09/21] Fix cross filesystem issues --- .../google-cloud-gcloud-maven-plugin/pom.xml | 6 ++++++ .../java/com/google/cloud/DownloadComponentsMojo.java | 11 +++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/google-cloud-testing/google-cloud-gcloud-maven-plugin/pom.xml b/google-cloud-testing/google-cloud-gcloud-maven-plugin/pom.xml index cda867eea5b7..5390d5cafc71 100644 --- a/google-cloud-testing/google-cloud-gcloud-maven-plugin/pom.xml +++ b/google-cloud-testing/google-cloud-gcloud-maven-plugin/pom.xml @@ -81,6 +81,12 @@ 1.11 + + commons-io + commons-io + 2.5 + + junit junit diff --git a/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/DownloadComponentsMojo.java b/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/DownloadComponentsMojo.java index ad1922ab7c34..34563ca0f246 100644 --- a/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/DownloadComponentsMojo.java +++ b/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/DownloadComponentsMojo.java @@ -47,12 +47,12 @@ import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; import org.apache.commons.compress.utils.IOUtils; +import org.apache.commons.io.FileUtils; import org.apache.maven.execution.MavenSession; import org.apache.maven.plugin.AbstractMojo; import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugins.annotations.Mojo; import org.apache.maven.plugins.annotations.Parameter; -import org.codehaus.plexus.util.FileUtils; /** * Goal that downloads gcloud components and embeds them as resources in a jar. This is mainly @@ -191,9 +191,7 @@ private void updateCachedManifest() throws IOException { @SuppressWarnings("unused") boolean ignored = localCache.delete(); - if (!tempFile.renameTo(localCache)) { - throw new RuntimeException("Failed to move the updated cache into place"); - } + FileUtils.moveFile(tempFile, localCache); } /** @@ -307,10 +305,7 @@ private void downloadComponent(Component component) throws IOException, NoSuchAl // Move it into place File localPath = getComponentPath(component); FileUtils.deleteDirectory(localPath); - - if (!tmpPath.renameTo(localPath)) { - throw new IOException("Failed to move the extracted component into place"); - } + FileUtils.moveDirectory(tmpPath, localPath); } private static String byteArrayToHex(byte[] a) { From fd2185e10139154278ae5a2d3d8b962dccfb4edd Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Fri, 2 Nov 2018 11:22:46 -0400 Subject: [PATCH 10/21] java 7 fixes --- .../google-cloud-gcloud-maven-plugin/pom.xml | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/google-cloud-testing/google-cloud-gcloud-maven-plugin/pom.xml b/google-cloud-testing/google-cloud-gcloud-maven-plugin/pom.xml index 5390d5cafc71..e7f1f1d87903 100644 --- a/google-cloud-testing/google-cloud-gcloud-maven-plugin/pom.xml +++ b/google-cloud-testing/google-cloud-gcloud-maven-plugin/pom.xml @@ -18,11 +18,19 @@ detail for google-cloud-java. + + 3.3.9 + + + org.apache.maven.plugins maven-plugin-plugin - 3.5 + 3.5.1 + + true + mojo-descriptor @@ -36,25 +44,27 @@ + org.apache.maven - maven-core - 3.0 + maven-plugin-api + ${maven.version} org.apache.maven - maven-plugin-api - 3.0 + maven-core + ${maven.version} org.apache.maven.plugin-tools maven-plugin-annotations - 3.0 + 3.5.1 provided + com.google.auto.value auto-value @@ -87,10 +97,10 @@ 2.5 + junit junit - 3.8.1 test From 368ce60386b0a706607ce63a1c6ceece5512d317 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Fri, 2 Nov 2018 11:27:46 -0400 Subject: [PATCH 11/21] fix bom as well --- google-cloud-bom/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-bom/pom.xml b/google-cloud-bom/pom.xml index 34529af1d519..39e6b419ba44 100644 --- a/google-cloud-bom/pom.xml +++ b/google-cloud-bom/pom.xml @@ -257,7 +257,7 @@ com.google.cloud google-cloud-bigtable-emulator - 0.68.1-alpha-SNAPSHOT + 0.69.1-alpha-SNAPSHOT com.google.cloud From 42ee7e3daa7e320f12ceab008db1b5afbfa1b8de Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Fri, 2 Nov 2018 11:29:25 -0400 Subject: [PATCH 12/21] fix readme --- .../google-cloud-bigtable-emulator/README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/google-cloud-testing/google-cloud-bigtable-emulator/README.md b/google-cloud-testing/google-cloud-bigtable-emulator/README.md index 9bc5ceb6160a..ee4bc22abcd6 100644 --- a/google-cloud-testing/google-cloud-bigtable-emulator/README.md +++ b/google-cloud-testing/google-cloud-bigtable-emulator/README.md @@ -19,7 +19,7 @@ If you are using Maven, add this to your pom.xml file com.google.cloud google-cloud-bom - 0.68.1-alpha-SNAPSHOT + 0.69.0-alpha pom import @@ -52,16 +52,16 @@ If you are using Maven, add this to your pom.xml file If you are using Gradle, add this to your dependencies ```Groovy -compile 'com.google.cloud:google-cloud-bigtable:0.68.0-alpha' -compile 'com.google.cloud:google-cloud-bigtable-admin:0.68.0-alpha' -testCompile 'com.google.cloud:google-cloud-bigtable-emulator:0.68.0-alpha' +compile 'com.google.cloud:google-cloud-bigtable:0.69.0-alpha' +compile 'com.google.cloud:google-cloud-bigtable-admin:0.69.0-alpha' +testCompile 'com.google.cloud:google-cloud-bigtable-emulator:0.69.0-alpha' testCompile 'junit:junit:4.12' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-bigtable" % "0.68.0-alpha" -libraryDependencies += "com.google.cloud" % "google-cloud-bigtable-admin" % "0.68.0-alpha" -libraryDependencies += "com.google.cloud" % "google-cloud-bigtable-emulator" % "0.68.0-alpha" % Test +libraryDependencies += "com.google.cloud" % "google-cloud-bigtable" % "0.69.0-alpha" +libraryDependencies += "com.google.cloud" % "google-cloud-bigtable-admin" % "0.69.0-alpha" +libraryDependencies += "com.google.cloud" % "google-cloud-bigtable-emulator" % "0.69.0-alpha" % Test libraryDependencies += "junit" % "junit" % "4.12" % Test ``` [//]: # ({x-version-update-end}) From 761562893184e4b65a62d6e465b64d19b0ac15ac Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Fri, 2 Nov 2018 11:49:39 -0400 Subject: [PATCH 13/21] still fixing java 7 issues --- .../google-cloud-gcloud-maven-plugin/pom.xml | 14 ----- .../main/java/com/google/cloud/Component.java | 62 ++++++++++++++++--- 2 files changed, 52 insertions(+), 24 deletions(-) diff --git a/google-cloud-testing/google-cloud-gcloud-maven-plugin/pom.xml b/google-cloud-testing/google-cloud-gcloud-maven-plugin/pom.xml index 9a7e5aaa73e2..6d5b5790b9f8 100644 --- a/google-cloud-testing/google-cloud-gcloud-maven-plugin/pom.xml +++ b/google-cloud-testing/google-cloud-gcloud-maven-plugin/pom.xml @@ -65,20 +65,6 @@ - - com.google.auto.value - auto-value - 1.5 - provided - - - - com.google.code.findbugs - jsr305 - 3.0.2 - provided - - com.google.code.gson gson diff --git a/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/Component.java b/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/Component.java index 498446d51e00..1d9fb5dd0d59 100644 --- a/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/Component.java +++ b/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/Component.java @@ -1,17 +1,17 @@ package com.google.cloud; -import com.google.auto.value.AutoValue; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import java.io.IOException; import java.net.URL; +import java.util.Objects; /** Representation of a gcloud component */ -@AutoValue -abstract class Component { - static Component create(String id, String checksum, URL source, String fileType) { - return new AutoValue_Component(id, checksum, source, fileType); - } +public class Component { + private final String id; + private final String checksum; + private final URL source; + private final String fileType; static Component fromJson(URL baseUrl, JsonObject componentObj) throws IOException { String id = componentObj.get("id").getAsString(); @@ -29,8 +29,50 @@ static Component fromJson(URL baseUrl, JsonObject componentObj) throws IOExcepti ); } - abstract String getId(); - abstract String getChecksum(); - abstract URL getSource(); - abstract String getFileType(); + static Component create(String id, String checksum, URL source, String fileType) { + return new Component(id, checksum, source, fileType); + } + + private Component(String id, String checksum, URL source, String fileType) { + this.id = id; + this.checksum = checksum; + this.source = source; + this.fileType = fileType; + } + + String getId() { + return id; + } + + String getChecksum() { + return checksum; + } + + URL getSource() { + return source; + } + + String getFileType() { + return fileType; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Component component = (Component) o; + return Objects.equals(id, component.id) && + Objects.equals(checksum, component.checksum) && + Objects.equals(source, component.source) && + Objects.equals(fileType, component.fileType); + } + + @Override + public int hashCode() { + return Objects.hash(id, checksum, source, fileType); + } } From b9f8ba553d21cf9ff7b059809c6db8ae81584300 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Fri, 2 Nov 2018 12:11:57 -0400 Subject: [PATCH 14/21] clean up --- .../emulator/v2/BigtableEmulatorRule.java | 2 +- .../cloud/bigtable/emulator/v2/Emulator.java | 10 +++++----- .../com/google/cloud/DownloadComponentsMojo.java | 16 ++++++++++------ 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRule.java b/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRule.java index 052dc43b6d5c..b02887abd9e9 100644 --- a/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRule.java +++ b/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRule.java @@ -48,7 +48,7 @@ public static BigtableEmulatorRule create() { return new BigtableEmulatorRule(); } - BigtableEmulatorRule() { } + private BigtableEmulatorRule() { } /** Initializes the Bigtable emulator before a test runs. */ @Override diff --git a/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java b/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java index f39b8ef25944..36089a467c70 100644 --- a/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java +++ b/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java @@ -96,7 +96,7 @@ public synchronized void start() throws IOException, TimeoutException, Interrupt } this.port = getAvailablePort(); - process = Runtime.getRuntime().exec(executable + " -port " + "" + port); + process = Runtime.getRuntime().exec(String.format("%s -port %d", executable, port)); pipeStreamToLog(process.getInputStream(), Level.INFO); pipeStreamToLog(process.getErrorStream(), Level.WARNING); isStopped = false; @@ -191,11 +191,11 @@ private static String getPlatform() { String unformattedOs = System.getProperty("os.name", "unknown").toLowerCase(Locale.ENGLISH); String os; - if ((unformattedOs.indexOf("mac") >= 0) || (unformattedOs.indexOf("darwin") >= 0)) { + if (unformattedOs.contains("mac") || unformattedOs.contains("darwin")) { os = "darwin"; - } else if (unformattedOs.indexOf("win") >= 0) { + } else if (unformattedOs.contains("win")) { os = "windows"; - } else if (unformattedOs.indexOf("linux") >= 0) { + } else if (unformattedOs.contains("linux")) { os = "linux"; } else { throw new UnsupportedOperationException( @@ -214,7 +214,7 @@ private static String getPlatform() { arch = "x86_64"; break; default: - throw new UnsupportedOperationException("Unsupport architecture: " + unformattedArch); + throw new UnsupportedOperationException("Unsupported architecture: " + unformattedArch); } return os + "-" + arch; diff --git a/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/DownloadComponentsMojo.java b/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/DownloadComponentsMojo.java index 34563ca0f246..491aaf284efb 100644 --- a/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/DownloadComponentsMojo.java +++ b/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/DownloadComponentsMojo.java @@ -63,7 +63,7 @@ @Mojo(name = "download") public class DownloadComponentsMojo extends AbstractMojo { - private static long STALE_MS = TimeUnit.HOURS.toMillis(2); + private static final long STALE_MS = TimeUnit.HOURS.toMillis(2); @Parameter(defaultValue = "https://dl.google.com/dl/cloudsdk/channels/rapid/", required = true) private URL baseUrl; @@ -160,10 +160,13 @@ private void updateCachedManifest() throws IOException { boolean isStale = !localCache.exists() || new Date().getTime() - localCache.lastModified() < STALE_MS; - if (localCache.exists() && session.isOffline()) { - if (isStale) { - getLog().info("Using stale manifest because offline mode is enabled"); - } + if (forceRefresh && session.isOffline()) { + throw new IllegalStateException("Can't force manifest refresh while offline"); + } + + if (session.isOffline() && localCache.exists() && isStale) { + getLog().info("Using stale manifest because offline mode is enabled"); + return; } if (!forceRefresh && !isStale) { @@ -310,8 +313,9 @@ private void downloadComponent(Component component) throws IOException, NoSuchAl private static String byteArrayToHex(byte[] a) { StringBuilder sb = new StringBuilder(a.length * 2); - for(byte b: a) + for (byte b : a) { sb.append(String.format("%02x", b)); + } return sb.toString(); } From 949b8c5032055918177a0e89867af3b2b6342926 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Fri, 2 Nov 2018 13:45:16 -0400 Subject: [PATCH 15/21] oops --- .../cloud/bigtable/emulator/v2/BigtableEmulatorRuleTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/google-cloud-testing/google-cloud-bigtable-emulator/src/test/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRuleTest.java b/google-cloud-testing/google-cloud-bigtable-emulator/src/test/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRuleTest.java index de07230d603c..03fb139c1f4c 100644 --- a/google-cloud-testing/google-cloud-bigtable-emulator/src/test/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRuleTest.java +++ b/google-cloud-testing/google-cloud-bigtable-emulator/src/test/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRuleTest.java @@ -39,9 +39,8 @@ @RunWith(JUnit4.class) public class BigtableEmulatorRuleTest { - @Rule - public BigtableEmulatorRule bigtableRule = new BigtableEmulatorRule(); + public BigtableEmulatorRule bigtableRule = BigtableEmulatorRule.create(); private BigtableTableAdminBlockingStub tableAdminStub; private BigtableBlockingStub dataStub; From 5ee1ddba7b5c5bccec5973e1a9a09ad04064d4cc Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Tue, 6 Nov 2018 11:02:47 -0500 Subject: [PATCH 16/21] fix windows build --- .../cloud/bigtable/emulator/v2/Emulator.java | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java b/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java index 36089a467c70..5648271c6c0f 100644 --- a/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java +++ b/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java @@ -21,13 +21,13 @@ import io.grpc.ManagedChannelBuilder; import java.io.BufferedReader; import java.io.File; +import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.net.ServerSocket; import java.net.Socket; -import java.net.URL; import java.nio.file.Path; import java.util.Locale; import java.util.concurrent.TimeUnit; @@ -62,17 +62,18 @@ public class Emulator { * creating it. */ public static Emulator createBundled() throws IOException { - String resourcePath = String.format( - "/gcloud/bigtable-%s/platform/bigtable-emulator/cbtemulator", getPlatform()); - - URL packagedEmulator = Emulator.class.getResource(resourcePath); + String resourcePath = getBundledResourcePath(); File tmpEmulator = File.createTempFile("cbtemulator", ""); tmpEmulator.deleteOnExit(); - try (InputStream is = packagedEmulator.openStream(); + try (InputStream is = Emulator.class.getResourceAsStream(resourcePath); FileOutputStream os = new FileOutputStream(tmpEmulator)) { + if (is == null) { + throw new FileNotFoundException("Failed to find the bundled emulator binary: " + resourcePath); + } + byte[] buff = new byte[2048]; int length; @@ -187,14 +188,16 @@ public synchronized ManagedChannel getAdminChannel() { // /** Gets the current platform, which will be used to select the appropriate emulator binary. */ - private static String getPlatform() { + private static String getBundledResourcePath() { String unformattedOs = System.getProperty("os.name", "unknown").toLowerCase(Locale.ENGLISH); String os; + String suffix = ""; if (unformattedOs.contains("mac") || unformattedOs.contains("darwin")) { os = "darwin"; } else if (unformattedOs.contains("win")) { os = "windows"; + suffix = ".exe"; } else if (unformattedOs.contains("linux")) { os = "linux"; } else { @@ -217,7 +220,8 @@ private static String getPlatform() { throw new UnsupportedOperationException("Unsupported architecture: " + unformattedArch); } - return os + "-" + arch; + return String.format( + "/gcloud/bigtable-%s-%s/platform/bigtable-emulator/cbtemulator%s", os, arch, suffix); } /** Gets a random open port number. */ From 2ec37d8c75721b205675b9fee121a801db3d60ae Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Tue, 6 Nov 2018 12:20:22 -0500 Subject: [PATCH 17/21] Minor dep tweaks --- .../google-cloud-bigtable-emulator/pom.xml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/google-cloud-testing/google-cloud-bigtable-emulator/pom.xml b/google-cloud-testing/google-cloud-bigtable-emulator/pom.xml index 3977404db955..7ebb17dbc309 100644 --- a/google-cloud-testing/google-cloud-bigtable-emulator/pom.xml +++ b/google-cloud-testing/google-cloud-bigtable-emulator/pom.xml @@ -63,6 +63,7 @@ api-common + junit junit @@ -71,20 +72,21 @@ - com.google.api.grpc - grpc-google-cloud-bigtable-v2 + com.google.truth + truth test com.google.api.grpc - grpc-google-cloud-bigtable-admin-v2 + grpc-google-cloud-bigtable-v2 test - com.google.truth - truth + com.google.api.grpc + grpc-google-cloud-bigtable-admin-v2 + test From 1afa15a09628de38e50e1a0f5f990071f0ba3cd1 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Tue, 6 Nov 2018 12:25:43 -0500 Subject: [PATCH 18/21] tweaks --- .../java/com/google/cloud/bigtable/emulator/v2/Emulator.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java b/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java index 5648271c6c0f..c69500faaeea 100644 --- a/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java +++ b/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java @@ -234,7 +234,7 @@ private static int getAvailablePort() { } /** Waits for a port to open. It's used to wait for the emulator's gRPC server to be ready. */ - private void waitForPort(int port) throws InterruptedException, TimeoutException { + private static void waitForPort(int port) throws InterruptedException, TimeoutException { for (int i = 0; i < 100; i++) { try (Socket ignored = new Socket("localhost", port)) { return; @@ -247,7 +247,7 @@ private void waitForPort(int port) throws InterruptedException, TimeoutException } /** Creates a {@link io.grpc.ManagedChannelBuilder} preconfigured for the emulator's port. */ - private ManagedChannelBuilder newChannelBuilder(int port) { + private static ManagedChannelBuilder newChannelBuilder(int port) { // NOTE: usePlaintext is currently @ExperimentalAPI. // See https://github.com/grpc/grpc-java/issues/1772 for discussion return ManagedChannelBuilder.forAddress("localhost", port) From a9e7ae39e51114c2572e444a9fcc536ef0a94e48 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Tue, 6 Nov 2018 12:25:43 -0500 Subject: [PATCH 19/21] tweaks --- google-cloud-testing/google-cloud-bigtable-emulator/README.md | 1 + .../java/com/google/cloud/bigtable/emulator/v2/Emulator.java | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/google-cloud-testing/google-cloud-bigtable-emulator/README.md b/google-cloud-testing/google-cloud-bigtable-emulator/README.md index ee4bc22abcd6..9e237a626f62 100644 --- a/google-cloud-testing/google-cloud-bigtable-emulator/README.md +++ b/google-cloud-testing/google-cloud-bigtable-emulator/README.md @@ -47,6 +47,7 @@ If you are using Maven, add this to your pom.xml file junit junit 4.12 + test ``` diff --git a/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java b/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java index 5648271c6c0f..c69500faaeea 100644 --- a/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java +++ b/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java @@ -234,7 +234,7 @@ private static int getAvailablePort() { } /** Waits for a port to open. It's used to wait for the emulator's gRPC server to be ready. */ - private void waitForPort(int port) throws InterruptedException, TimeoutException { + private static void waitForPort(int port) throws InterruptedException, TimeoutException { for (int i = 0; i < 100; i++) { try (Socket ignored = new Socket("localhost", port)) { return; @@ -247,7 +247,7 @@ private void waitForPort(int port) throws InterruptedException, TimeoutException } /** Creates a {@link io.grpc.ManagedChannelBuilder} preconfigured for the emulator's port. */ - private ManagedChannelBuilder newChannelBuilder(int port) { + private static ManagedChannelBuilder newChannelBuilder(int port) { // NOTE: usePlaintext is currently @ExperimentalAPI. // See https://github.com/grpc/grpc-java/issues/1772 for discussion return ManagedChannelBuilder.forAddress("localhost", port) From 31769e9299a0853807fc28f2e48822d8569711ec Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Thu, 8 Nov 2018 12:42:55 -0500 Subject: [PATCH 20/21] tweak --- google-cloud-testing/google-cloud-bigtable-emulator/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-testing/google-cloud-bigtable-emulator/README.md b/google-cloud-testing/google-cloud-bigtable-emulator/README.md index 511493b81898..23e63ad86c85 100644 --- a/google-cloud-testing/google-cloud-bigtable-emulator/README.md +++ b/google-cloud-testing/google-cloud-bigtable-emulator/README.md @@ -156,7 +156,7 @@ public class ExampleTest { ## Java Versions -Java 7 or above is required for using this client. +Java 7 or above is required for using this emulator. ## Versioning From 9daec05fafc7b557fae49f7dfdbadcbecf48e1ac Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Thu, 8 Nov 2018 14:03:52 -0500 Subject: [PATCH 21/21] license --- .../src/main/java/com/google/cloud/Component.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/Component.java b/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/Component.java index 1d9fb5dd0d59..65032d664efd 100644 --- a/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/Component.java +++ b/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/Component.java @@ -1,3 +1,18 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.google.cloud; import com.google.gson.JsonElement;