From eb6c4d2af57fe1786e797d6e0bfb1e503e67da15 Mon Sep 17 00:00:00 2001 From: elisheva-qlogic <43041115+elisheva-qlogic@users.noreply.github.com> Date: Thu, 3 Jan 2019 15:06:43 -0500 Subject: [PATCH] Cloud Bigtable: helloWorld sample (#4274) * helloWorld sample * test class modified * format change in ITHelloWorld --- google-cloud-examples/pom.xml | 10 +- .../cloud/examples/bigtable/HelloWorld.java | 167 ++++++++++++++++++ .../cloud/examples/bigtable/ITHelloWorld.java | 145 +++++++++++++++ 3 files changed, 321 insertions(+), 1 deletion(-) create mode 100644 google-cloud-examples/src/main/java/com/google/cloud/examples/bigtable/HelloWorld.java create mode 100644 google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITHelloWorld.java diff --git a/google-cloud-examples/pom.xml b/google-cloud-examples/pom.xml index 4aded1d268bb..d8f2bcda26cc 100644 --- a/google-cloud-examples/pom.xml +++ b/google-cloud-examples/pom.xml @@ -29,6 +29,14 @@ com.google.cloud google-cloud-bigquery + + com.google.cloud + google-cloud-bigtable-admin + + + com.google.cloud + google-cloud-bigtable + com.google.cloud google-cloud-compute @@ -188,7 +196,7 @@ 2.19.1 - **/IT*Snippets.java + **/IT*.java diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigtable/HelloWorld.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigtable/HelloWorld.java new file mode 100644 index 000000000000..7d4ebde8787b --- /dev/null +++ b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigtable/HelloWorld.java @@ -0,0 +1,167 @@ +/* + * Copyright 2018 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.examples.bigtable; + +import com.google.api.gax.rpc.NotFoundException; +import com.google.api.gax.rpc.ServerStream; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; +import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import com.google.cloud.bigtable.data.v2.models.InstanceName; +import com.google.cloud.bigtable.data.v2.models.Query; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowCell; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import java.io.IOException; + +public class HelloWorld { + + private static final String COLUMN_FAMILY = "cf1"; + private static final String COLUMN_QUALIFIER = "greeting"; + private static final String ROW_KEY_PREFIX = "rowKey"; + private final String tableId; + private final BigtableDataClient dataClient; + private final BigtableTableAdminClient adminClient; + + public static void main(String[] args) throws Exception { + + if (args.length != 2) { + System.out.println("Missing required project id or instance id"); + return; + } + String projectId = args[0]; + String instanceId = args[1]; + + HelloWorld helloWorld = new HelloWorld(projectId, instanceId, "test-table"); + helloWorld.run(); + } + + public HelloWorld(String projectId, String instanceId, String tableId) throws IOException { + this.tableId = tableId; + + // [START connecting_to_bigtable] + // Create the settings to configure a bigtable data client + BigtableDataSettings settings = + BigtableDataSettings.newBuilder() + .setInstanceName(InstanceName.of(projectId, instanceId)) + .build(); + + // Create bigtable data client + dataClient = BigtableDataClient.create(settings); + + // Create the settings to configure a bigtable table admin client + BigtableTableAdminSettings adminSettings = + BigtableTableAdminSettings.newBuilder() + .setInstanceName(com.google.bigtable.admin.v2.InstanceName.of(projectId, instanceId)) + .build(); + + // Create bigtable table admin client + adminClient = BigtableTableAdminClient.create(adminSettings); + // [END connecting_to_bigtable] + } + + public void run() throws Exception { + createTable(); + writeToTable(); + readSingleRow(); + readTable(); + deleteTable(); + dataClient.close(); + adminClient.close(); + } + + public void createTable() { + // [START creating_a_table] + // Check if table exists, create table if does not exist + if (!adminClient.exists(tableId)) { + System.out.println("Creating table: " + tableId); + CreateTableRequest createTableRequest = + CreateTableRequest.of(tableId).addFamily(COLUMN_FAMILY); + adminClient.createTable(createTableRequest); + System.out.printf("Table %s created successfully%n", tableId); + } + // [END creating_a_table] + } + + public void writeToTable() { + // [START writing_rows] + try { + System.out.println("\nWriting some greetings to the table"); + String[] greetings = {"Hello World!", "Hello Bigtable!", "Hello Java!"}; + for (int i = 0; i < greetings.length; i++) { + RowMutation rowMutation = + RowMutation.create(tableId, ROW_KEY_PREFIX + i) + .setCell(COLUMN_FAMILY, COLUMN_QUALIFIER, greetings[i]); + dataClient.mutateRow(rowMutation); + System.out.println(greetings[i]); + } + } catch (NotFoundException e) { + System.err.println("Failed to write to non-existent table: " + e.getMessage()); + } + // [END writing_rows] + } + + public void readSingleRow() { + // [START reading_a_row] + try { + System.out.println("\nReading a single row by row key"); + Row row = dataClient.readRow(tableId, ROW_KEY_PREFIX + 0); + System.out.println("Row: " + row.getKey().toStringUtf8()); + for (RowCell cell : row.getCells()) { + System.out.printf( + "Family: %s Qualifier: %s Value: %s%n", + cell.getFamily(), cell.getQualifier().toStringUtf8(), cell.getValue().toStringUtf8()); + } + } catch (NotFoundException e) { + System.err.println("Failed to read from a non-existent table: " + e.getMessage()); + } + // [END reading_a_row] + } + + public void readTable() { + // [START scanning_all_rows] + try { + System.out.println("\nReading the entire table"); + Query query = Query.create(tableId); + ServerStream rowStream = dataClient.readRows(query); + for (Row r : rowStream) { + System.out.println("Row Key: " + r.getKey().toStringUtf8()); + for (RowCell cell : r.getCells()) { + System.out.printf( + "Family: %s Qualifier: %s Value: %s%n", + cell.getFamily(), cell.getQualifier().toStringUtf8(), cell.getValue().toStringUtf8()); + } + } + } catch (NotFoundException e) { + System.err.println("Failed to read a non-existent table: " + e.getMessage()); + } + // [END scanning_all_rows] + } + + public void deleteTable() { + // [START deleting_a_table] + System.out.println("\nDeleting table: " + tableId); + try { + adminClient.deleteTable(tableId); + System.out.printf("Table %s deleted successfully%n", tableId); + } catch (NotFoundException e) { + System.err.println("Failed to delete a non-existent table: " + e.getMessage()); + } + // [END deleting_a_table] + } +} diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITHelloWorld.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITHelloWorld.java new file mode 100644 index 000000000000..ba8010046fc0 --- /dev/null +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigtable/ITHelloWorld.java @@ -0,0 +1,145 @@ +/* + * Copyright 2018 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.examples.bigtable; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; +import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import com.google.cloud.bigtable.data.v2.models.InstanceName; +import com.google.cloud.bigtable.data.v2.models.Row; +import java.io.IOException; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.AssumptionViolatedException; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class ITHelloWorld { + + private static final String INSTANCE_PROPERTY_NAME = "bigtable.instance"; + private static final String TABLE_PREFIX = "table"; + private static String tableId; + private static BigtableDataClient dataClient; + private static BigtableTableAdminClient adminClient; + private static InstanceName instanceName; + private HelloWorld helloWorld; + + @BeforeClass + public static void beforeClass() throws IOException { + String targetInstance = System.getProperty(INSTANCE_PROPERTY_NAME); + if (targetInstance == null) { + dataClient = null; + adminClient = null; + return; + } + instanceName = InstanceName.parse(targetInstance); + BigtableDataSettings settings = + BigtableDataSettings.newBuilder().setInstanceName(instanceName).build(); + dataClient = BigtableDataClient.create(settings); + BigtableTableAdminSettings adminSettings = + BigtableTableAdminSettings.newBuilder() + .setInstanceName(com.google.bigtable.admin.v2.InstanceName.parse(targetInstance)) + .build(); + adminClient = BigtableTableAdminClient.create(adminSettings); + } + + @AfterClass + public static void afterClass() throws Exception { + garbageCollect(); + dataClient.close(); + adminClient.close(); + } + + @Before + public void setup() throws IOException { + if (adminClient == null || dataClient == null) { + throw new AssumptionViolatedException( + INSTANCE_PROPERTY_NAME + " property is not set, skipping integration tests."); + } + tableId = generateTableId(); + helloWorld = new HelloWorld(instanceName.getProject(), instanceName.getInstance(), tableId); + adminClient.createTable(CreateTableRequest.of(tableId).addFamily("cf1")); + } + + @After + public void after() { + if (adminClient.exists(tableId)) { + adminClient.deleteTable(tableId); + } + } + + @Test + public void testCreateAndDeleteTable() throws IOException { + // Create table + String fakeTable = generateTableId(); + HelloWorld testHelloWorld = + new HelloWorld(instanceName.getProject(), instanceName.getInstance(), fakeTable); + testHelloWorld.createTable(); + assertTrue(adminClient.exists(fakeTable)); + + // Delete table + testHelloWorld.deleteTable(); + assertTrue(!adminClient.exists(fakeTable)); + } + + @Test + public void testWriteToTable() { + // Write to table + helloWorld.writeToTable(); + Row row = dataClient.readRow(tableId, "rowKey0"); + assertNotNull(row); + } + + // TODO: add test for helloWorld.readSingleRow() + // TODO: add test for helloWorld.readTable() + + @Test + public void testRunDoesNotFail() throws Exception { + helloWorld.run(); + } + + private String generateTableId() { + return String.format( + "%s-%016x-%x", TABLE_PREFIX, System.currentTimeMillis(), new Random().nextLong()); + } + + private static void garbageCollect() { + Pattern timestampPattern = Pattern.compile(TABLE_PREFIX + "-([0-9a-f]+)-([0-9a-f]+)"); + for (String tableId : adminClient.listTables()) { + Matcher matcher = timestampPattern.matcher(tableId); + if (!matcher.matches()) { + continue; + } + String timestampStr = matcher.group(1); + long timestamp = Long.parseLong(timestampStr, 16); + if (System.currentTimeMillis() - timestamp < TimeUnit.MINUTES.toMillis(15)) { + continue; + } + System.out.println("\nGarbage collecting orphaned table: " + tableId); + adminClient.deleteTable(tableId); + } + } +}