From 8cebdefcb990ae2e341bb1475574258cb46e1ff1 Mon Sep 17 00:00:00 2001 From: Billy Jacobson Date: Tue, 21 May 2019 13:40:53 -0400 Subject: [PATCH 01/10] Setup mvn package and copy in existing samples and tests --- bigtable/pom.xml | 82 ++++ .../com/m/examples/bigtable/HelloWorld.java | 186 +++++++++ .../bigtable/InstanceAdminExample.java | 216 ++++++++++ .../examples/bigtable/TableAdminExample.java | 386 ++++++++++++++++++ .../com/m/examples/bigtable/ITHelloWorld.java | 150 +++++++ .../bigtable/ITInstanceAdminExample.java | 152 +++++++ .../bigtable/ITTableAdminExample.java | 226 ++++++++++ 7 files changed, 1398 insertions(+) create mode 100644 bigtable/pom.xml create mode 100644 bigtable/src/main/java/com/m/examples/bigtable/HelloWorld.java create mode 100644 bigtable/src/main/java/com/m/examples/bigtable/InstanceAdminExample.java create mode 100644 bigtable/src/main/java/com/m/examples/bigtable/TableAdminExample.java create mode 100644 bigtable/src/test/java/com/m/examples/bigtable/ITHelloWorld.java create mode 100644 bigtable/src/test/java/com/m/examples/bigtable/ITInstanceAdminExample.java create mode 100644 bigtable/src/test/java/com/m/examples/bigtable/ITTableAdminExample.java diff --git a/bigtable/pom.xml b/bigtable/pom.xml new file mode 100644 index 00000000000..72317713676 --- /dev/null +++ b/bigtable/pom.xml @@ -0,0 +1,82 @@ + + + + 4.0.0 + + com.m.examples.bigtable + bigtable + 1.0-SNAPSHOT + + bigtable + + http://www.example.com + + + UTF-8 + 1.7 + 1.7 + + + + + junit + junit + 4.12 + test + + + + com.google.cloud + google-cloud-bigtable-admin + 0.81.0-alpha + + + + + + + + + maven-clean-plugin + 3.1.0 + + + + maven-resources-plugin + 3.0.2 + + + maven-compiler-plugin + 3.8.0 + + + maven-surefire-plugin + 2.22.1 + + + maven-jar-plugin + 3.0.2 + + + maven-install-plugin + 2.5.2 + + + maven-deploy-plugin + 2.8.2 + + + + maven-site-plugin + 3.7.1 + + + maven-project-info-reports-plugin + 3.0.0 + + + + + diff --git a/bigtable/src/main/java/com/m/examples/bigtable/HelloWorld.java b/bigtable/src/main/java/com/m/examples/bigtable/HelloWorld.java new file mode 100644 index 00000000000..8677e37cec9 --- /dev/null +++ b/bigtable/src/main/java/com/m/examples/bigtable/HelloWorld.java @@ -0,0 +1,186 @@ +/* + * Copyright 2019 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.m.examples.bigtable; + +// [START bigtable_hw_imports_veneer] +import com.google.api.gax.rpc.NotFoundException; +import com.google.api.gax.rpc.ServerStream; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; +import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import com.google.cloud.bigtable.data.v2.models.Query; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowCell; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import java.io.IOException; + +// [END bigtable_hw_imports_veneer] + +/** + * An example of using Google Cloud Bigtable. + * + *

This example is a very simple "hello world" application, that illustrates how to create a new + * table, write to the table, read the data back, and delete the table. + * + *

+ */ +public class HelloWorld { + + private static final String COLUMN_FAMILY = "cf1"; + private static final String COLUMN_QUALIFIER = "greeting"; + private static final String ROW_KEY_PREFIX = "rowKey"; + private final String tableId; + private final BigtableDataClient dataClient; + private final BigtableTableAdminClient adminClient; + + public static void main(String[] args) throws Exception { + + if (args.length != 2) { + System.out.println("Missing required project id or instance id"); + return; + } + String projectId = args[0]; + String instanceId = args[1]; + + HelloWorld helloWorld = new HelloWorld(projectId, instanceId, "test-table"); + helloWorld.run(); + } + + public HelloWorld(String projectId, String instanceId, String tableId) throws IOException { + this.tableId = tableId; + + // [START bigtable_hw_connect_veneer] + // Creates the settings to configure a bigtable data client. + BigtableDataSettings settings = + BigtableDataSettings.newBuilder().setProjectId(projectId).setInstanceId(instanceId).build(); + + // Creates a bigtable data client. + dataClient = BigtableDataClient.create(settings); + + // Creates the settings to configure a bigtable table admin client. + BigtableTableAdminSettings adminSettings = + BigtableTableAdminSettings.newBuilder() + .setProjectId(projectId) + .setInstanceId(instanceId) + .build(); + + // Creates a bigtable table admin client. + adminClient = BigtableTableAdminClient.create(adminSettings); + // [END bigtable_hw_connect_veneer] + } + + public void run() throws Exception { + createTable(); + writeToTable(); + readSingleRow(); + readTable(); + deleteTable(); + dataClient.close(); + adminClient.close(); + } + + /** Demonstrates how to create a table. */ + public void createTable() { + // [START bigtable_hw_create_table_veneer] + // Checks if table exists, creates table if does not exist. + if (!adminClient.exists(tableId)) { + System.out.println("Creating table: " + tableId); + CreateTableRequest createTableRequest = + CreateTableRequest.of(tableId).addFamily(COLUMN_FAMILY); + adminClient.createTable(createTableRequest); + System.out.printf("Table %s created successfully%n", tableId); + } + // [END bigtable_hw_create_table_veneer] + } + + /** Demonstrates how to write some rows to a table. */ + public void writeToTable() { + // [START bigtable_hw_write_rows_veneer] + try { + System.out.println("\nWriting some greetings to the table"); + String[] greetings = {"Hello World!", "Hello Bigtable!", "Hello Java!"}; + for (int i = 0; i < greetings.length; i++) { + RowMutation rowMutation = + RowMutation.create(tableId, ROW_KEY_PREFIX + i) + .setCell(COLUMN_FAMILY, COLUMN_QUALIFIER, greetings[i]); + dataClient.mutateRow(rowMutation); + System.out.println(greetings[i]); + } + } catch (NotFoundException e) { + System.err.println("Failed to write to non-existent table: " + e.getMessage()); + } + // [END bigtable_hw_write_rows_veneer] + } + + /** Demonstrates how to read a single row from a table. */ + public void readSingleRow() { + // [START bigtable_hw_get_by_key_veneer] + try { + System.out.println("\nReading a single row by row key"); + Row row = dataClient.readRow(tableId, ROW_KEY_PREFIX + 0); + System.out.println("Row: " + row.getKey().toStringUtf8()); + for (RowCell cell : row.getCells()) { + System.out.printf( + "Family: %s Qualifier: %s Value: %s%n", + cell.getFamily(), cell.getQualifier().toStringUtf8(), cell.getValue().toStringUtf8()); + } + } catch (NotFoundException e) { + System.err.println("Failed to read from a non-existent table: " + e.getMessage()); + } + // [END bigtable_hw_get_by_key_veneer] + } + + /** Demonstrates how to read an entire table. */ + public void readTable() { + // [START bigtable_hw_scan_all_veneer] + try { + System.out.println("\nReading the entire table"); + Query query = Query.create(tableId); + ServerStream rowStream = dataClient.readRows(query); + for (Row r : rowStream) { + System.out.println("Row Key: " + r.getKey().toStringUtf8()); + for (RowCell cell : r.getCells()) { + System.out.printf( + "Family: %s Qualifier: %s Value: %s%n", + cell.getFamily(), cell.getQualifier().toStringUtf8(), cell.getValue().toStringUtf8()); + } + } + } catch (NotFoundException e) { + System.err.println("Failed to read a non-existent table: " + e.getMessage()); + } + // [END bigtable_hw_scan_all_veneer] + } + + /** Demonstrates how to delete a table. */ + public void deleteTable() { + // [START bigtable_hw_delete_table_veneer] + System.out.println("\nDeleting table: " + tableId); + try { + adminClient.deleteTable(tableId); + System.out.printf("Table %s deleted successfully%n", tableId); + } catch (NotFoundException e) { + System.err.println("Failed to delete a non-existent table: " + e.getMessage()); + } + // [END bigtable_hw_delete_table_veneer] + } +} diff --git a/bigtable/src/main/java/com/m/examples/bigtable/InstanceAdminExample.java b/bigtable/src/main/java/com/m/examples/bigtable/InstanceAdminExample.java new file mode 100644 index 00000000000..65f98e1f4fb --- /dev/null +++ b/bigtable/src/main/java/com/m/examples/bigtable/InstanceAdminExample.java @@ -0,0 +1,216 @@ +/* + * Copyright 2019 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.m.examples.bigtable; + +import com.google.api.gax.rpc.AlreadyExistsException; +import com.google.api.gax.rpc.NotFoundException; +import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminSettings; +import com.google.cloud.bigtable.admin.v2.models.Cluster; +import com.google.cloud.bigtable.admin.v2.models.CreateClusterRequest; +import com.google.cloud.bigtable.admin.v2.models.CreateInstanceRequest; +import com.google.cloud.bigtable.admin.v2.models.Instance; +import com.google.cloud.bigtable.admin.v2.models.PartialListInstancesException; +import com.google.cloud.bigtable.admin.v2.models.StorageType; +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * An example of using Google Cloud Bigtable. + * + *

This example demonstrates the usage of BigtableInstanceAdminClient to create, configure, and + * delete Cloud Bigtable Instances and Clusters. + * + *

+ */ +public class InstanceAdminExample { + + private static final String CLUSTER = "cluster"; + private final String clusterId; + private final String instanceId; + private final BigtableInstanceAdminClient adminClient; + + public static void main(String[] args) throws IOException { + + if (args.length != 1) { + System.out.println("Missing required project id"); + return; + } + String projectId = args[0]; + + InstanceAdminExample instanceAdmin = + new InstanceAdminExample(projectId, "ssd-instance", "ssd-cluster"); + instanceAdmin.run(); + } + + public InstanceAdminExample(String projectId, String instanceId, String clusterId) + throws IOException { + this.instanceId = instanceId; + this.clusterId = clusterId; + + // [START connecting_to_bigtable] + // Creates the settings to configure a bigtable instance admin client. + BigtableInstanceAdminSettings instanceAdminSettings = + BigtableInstanceAdminSettings.newBuilder().setProjectId(projectId).build(); + + // Creates a bigtable instance admin client. + adminClient = BigtableInstanceAdminClient.create(instanceAdminSettings); + // [END connecting_to_bigtable] + } + + public void run() { + createProdInstance(); + listInstances(); + getInstance(); + listClusters(); + addCluster(); + deleteCluster(); + deleteInstance(); + adminClient.close(); + } + + /** Demonstrates how to create a Production instance within a provided project. */ + public void createProdInstance() { + // Checks if instance exists, creates instance if does not exists. + if (!adminClient.exists(instanceId)) { + System.out.println("Instance does not exist, creating a PRODUCTION instance"); + // [START bigtable_create_prod_instance] + // Creates a Production Instance with the ID "ssd-instance", + // cluster id "ssd-cluster", 3 nodes and location "us-central1-f". + CreateInstanceRequest createInstanceRequest = + CreateInstanceRequest.of(instanceId) + .addCluster(clusterId, "us-central1-f", 3, StorageType.SSD) + .setType(Instance.Type.PRODUCTION) + .addLabel("department", "accounting"); + // Creates a production instance with the given request. + try { + Instance instance = adminClient.createInstance(createInstanceRequest); + System.out.printf("PRODUCTION type instance %s created successfully%n", instance.getId()); + } catch (Exception e) { + System.err.println("Failed to create instance: " + e.getMessage()); + throw e; + } + // [END bigtable_create_prod_instance] + } + } + + /** Demonstrates how to list all instances within a project. */ + public void listInstances() { + System.out.println("\nListing Instances"); + // [START bigtable_list_instances] + try { + List instances = adminClient.listInstances(); + for (Instance instance : instances) { + System.out.println(instance.getId()); + } + } catch (PartialListInstancesException e) { + System.err.println("Failed to list instances: " + e.getMessage()); + System.err.println("The following zones are unavailable: " + e.getUnavailableZones()); + System.err.println("But the following instances are reachable: " + e.getInstances()); + } + // [END bigtable_list_instances] + } + + /** Demonstrates how to get an instance. */ + public Instance getInstance() { + System.out.println("\nGet Instance"); + // [START bigtable_get_instance] + Instance instance = null; + try { + instance = adminClient.getInstance(instanceId); + System.out.println("Instance ID: " + instance.getId()); + System.out.println("Display Name: " + instance.getDisplayName()); + System.out.print("Labels: "); + Map labels = instance.getLabels(); + for (String key : labels.keySet()) { + System.out.printf("%s - %s", key, labels.get(key)); + } + System.out.println("\nState: " + instance.getState()); + System.out.println("Type: " + instance.getType()); + } catch (NotFoundException e) { + System.err.println("Failed to get non-existent instance: " + e.getMessage()); + } + // [END bigtable_get_instance] + return instance; + } + + /** Demonstrates how to list clusters within an instance. */ + public void listClusters() { + System.out.println("\nListing Clusters"); + // [START bigtable_get_clusters] + try { + List clusters = adminClient.listClusters(instanceId); + for (Cluster cluster : clusters) { + System.out.println(cluster.getId()); + } + } catch (NotFoundException e) { + System.err.println("Failed to list clusters from a non-existent instance: " + e.getMessage()); + } + // [END bigtable_get_clusters] + } + + /** Demonstrates how to delete an instance. */ + public void deleteInstance() { + System.out.println("\nDeleting Instance"); + // [START bigtable_delete_instance] + try { + adminClient.deleteInstance(instanceId); + System.out.println("Instance deleted: " + instanceId); + } catch (NotFoundException e) { + System.err.println("Failed to delete non-existent instance: " + e.getMessage()); + } + // [END bigtable_delete_instance] + } + + /** Demonstrates how to add a cluster to an instance. */ + public void addCluster() { + System.out.printf("%nAdding cluster: %s to instance: %s%n", CLUSTER, instanceId); + // [START bigtable_create_cluster] + try { + adminClient.createCluster( + CreateClusterRequest.of(instanceId, CLUSTER) + .setZone("us-central1-c") + .setServeNodes(3) + .setStorageType(StorageType.SSD)); + System.out.printf("Cluster: %s created successfully%n", CLUSTER); + } catch (AlreadyExistsException e) { + System.err.println("Failed to add cluster, already exists: " + e.getMessage()); + } + // [END bigtable_create_cluster] + } + + /** Demonstrates how to delete a cluster from an instance. */ + public void deleteCluster() { + System.out.printf("%nDeleting cluster: %s from instance: %s%n", CLUSTER, instanceId); + // [START bigtable_delete_cluster] + try { + adminClient.deleteCluster(instanceId, CLUSTER); + System.out.printf("Cluster: %s deleted successfully%n", CLUSTER); + } catch (NotFoundException e) { + System.err.println("Failed to delete a non-existent cluster: " + e.getMessage()); + } + // [END bigtable_delete_cluster] + } +} diff --git a/bigtable/src/main/java/com/m/examples/bigtable/TableAdminExample.java b/bigtable/src/main/java/com/m/examples/bigtable/TableAdminExample.java new file mode 100644 index 00000000000..11dbeaf0422 --- /dev/null +++ b/bigtable/src/main/java/com/m/examples/bigtable/TableAdminExample.java @@ -0,0 +1,386 @@ +/* + * Copyright 2019 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.m.examples.bigtable; + +import static com.google.cloud.bigtable.admin.v2.models.GCRules.GCRULES; + +import com.google.api.gax.rpc.AlreadyExistsException; +import com.google.api.gax.rpc.NotFoundException; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; +import com.google.cloud.bigtable.admin.v2.models.ColumnFamily; +import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; +import com.google.cloud.bigtable.admin.v2.models.GCRules.DurationRule; +import com.google.cloud.bigtable.admin.v2.models.GCRules.IntersectionRule; +import com.google.cloud.bigtable.admin.v2.models.GCRules.UnionRule; +import com.google.cloud.bigtable.admin.v2.models.GCRules.VersionRule; +import com.google.cloud.bigtable.admin.v2.models.ModifyColumnFamiliesRequest; +import com.google.cloud.bigtable.admin.v2.models.Table; +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * An example of using Google Cloud Bigtable. + * + *

This example demonstrates the usage of BigtableTableAdminClient to create, configure and + * delete a Cloud Bigtable table. + * + *

    + *
  • creates table + *
  • lists all tables + *
  • gets table metadata + *
  • creates DurationRule + *
  • creates VersionRule + *
  • creates UnionRule + *
  • creates IntersectionRule + *
  • creates nested rule + *
  • lists column families + *
  • modifies column family rule + *
  • prints modified column family + *
  • deletes column family + *
  • deletes table + *
+ */ +public class TableAdminExample { + + private static final String COLUMN_FAMILY_1 = "cf1"; + private static final String COLUMN_FAMILY_2 = "cf2"; + private static final String COLUMN_FAMILY_3 = "cf3"; + private static final String COLUMN_FAMILY_4 = "cf4"; + private static final String COLUMN_FAMILY_5 = "cf5"; + private final String tableId; + private final BigtableTableAdminClient adminClient; + + public static void main(String[] args) throws IOException { + + if (args.length != 2) { + System.out.println("Missing required project id or instance id"); + return; + } + String projectId = args[0]; + String instanceId = args[1]; + + TableAdminExample tableAdmin = new TableAdminExample(projectId, instanceId, "test-table"); + tableAdmin.run(); + } + + public TableAdminExample(String projectId, String instanceId, String tableId) throws IOException { + this.tableId = tableId; + + // [START connecting_to_bigtable] + // Creates the settings to configure a bigtable table admin client. + BigtableTableAdminSettings adminSettings = + BigtableTableAdminSettings.newBuilder() + .setProjectId(projectId) + .setInstanceId(instanceId) + .build(); + + // Creates a bigtable table admin client. + adminClient = BigtableTableAdminClient.create(adminSettings); + // [END connecting_to_bigtable] + } + + public void run() { + createTable(); + listAllTables(); + getTableMeta(); + addFamilyWithMaxAgeRule(); + addFamilyWithMaxVersionsRule(); + addFamilyWithUnionRule(); + addFamilyWithIntersectionRule(); + addFamilyWithNestedRule(); + listColumnFamilies(); + modifyColumnFamilyRule(); + printModifiedColumnFamily(); + deleteColumnFamily(); + deleteTable(); + adminClient.close(); + } + + /** Demonstrates how to create a table with the specified configuration. */ + public void createTable() { + // [START bigtable_create_table] + // Checks if table exists, creates table if does not exist. + if (!adminClient.exists(tableId)) { + System.out.println("Table does not exist, creating table: " + tableId); + CreateTableRequest createTableRequest = CreateTableRequest.of(tableId).addFamily("cf"); + Table table = adminClient.createTable(createTableRequest); + System.out.printf("Table: %s created successfully%n", table.getId()); + } + // [END bigtable_create_table] + } + + /** Demonstrates how to list all tables within an instance. */ + public void listAllTables() { + System.out.println("\nListing tables in current instance"); + // [START bigtable_list_tables] + // Lists tables in the current instance. + try { + List tableIds = adminClient.listTables(); + for (String tableId : tableIds) { + System.out.println(tableId); + } + } catch (NotFoundException e) { + System.err.println("Failed to list tables from a non-existent instance: " + e.getMessage()); + } + // [END bigtable_list_tables] + } + + /** Demonstrates how to get a table's metadata. */ + public void getTableMeta() { + System.out.println("\nPrinting table metadata"); + // [START bigtable_get_table_metadata] + // Gets table metadata, and applies a view to the table fields. + try { + Table table = adminClient.getTable(tableId); + System.out.println("Table: " + table.getId()); + Collection columnFamilies = table.getColumnFamilies(); + for (ColumnFamily columnFamily : columnFamilies) { + System.out.printf( + "Column family: %s%nGC Rule: %s%n", + columnFamily.getId(), columnFamily.getGCRule().toString()); + } + } catch (NotFoundException e) { + System.err.println( + "Failed to retrieve table metadata for a non-existent table: " + e.getMessage()); + } + // [END bigtable_get_table_metadata] + } + + /** Demonstrates how to create a new instance of the DurationRule. */ + public void addFamilyWithMaxAgeRule() { + System.out.printf("%nCreating column family %s with max age GC rule%n", COLUMN_FAMILY_1); + // [START bigtable_create_family_gc_max_age] + // Creates a column family with GC policy : maximum age + // where age = current time minus cell timestamp + + // Defines the GC rule to retain data with max age of 5 days. + DurationRule maxAgeRule = GCRULES.maxAge(5, TimeUnit.DAYS); + + // Creates column family with given GC rule. + try { + // ModifyColumnFamiliesRequest can be used both for adding and modifying families, here it is + // being used to add a family + ModifyColumnFamiliesRequest columnFamiliesRequest = + ModifyColumnFamiliesRequest.of(tableId).addFamily(COLUMN_FAMILY_1, maxAgeRule); + adminClient.modifyFamilies(columnFamiliesRequest); + System.out.println("Created column family: " + COLUMN_FAMILY_1); + } catch (AlreadyExistsException e) { + System.err.println( + "Failed to create column family with rule, already exists: " + e.getMessage()); + } + // [END bigtable_create_family_gc_max_age] + } + + /** Demonstrates how to create a new instance of the VersionRule. */ + public void addFamilyWithMaxVersionsRule() { + System.out.printf("%nCreating column family %s with max versions GC rule%n", COLUMN_FAMILY_2); + // [START bigtable_create_family_gc_max_versions] + // Creates a column family with GC policy : most recent N versions + // where 1 = most recent version + + // Defines the GC policy to retain only the most recent 2 versions. + VersionRule versionRule = GCRULES.maxVersions(2); + + // Creates column family with given GC rule. + try { + // ModifyColumnFamiliesRequest can be used both for adding and modifying families, here it is + // being used to add a family + ModifyColumnFamiliesRequest columnFamiliesRequest = + ModifyColumnFamiliesRequest.of(tableId).addFamily(COLUMN_FAMILY_2, versionRule); + adminClient.modifyFamilies(columnFamiliesRequest); + System.out.println("Created column family: " + COLUMN_FAMILY_2); + } catch (AlreadyExistsException e) { + System.err.println( + "Failed to create column family with rule, already exists: " + e.getMessage()); + } + // [END bigtable_create_family_gc_max_versions] + } + + /** Demonstrates how to create a new instance of the UnionRule. */ + public void addFamilyWithUnionRule() { + System.out.printf("%nCreating column family %s with union GC rule%n", COLUMN_FAMILY_3); + // [START bigtable_create_family_gc_union] + // Creates a column family with GC policy to drop data that matches at least one condition. + + // Defines a list of GC rules to drop cells older than 5 days OR not the most recent + // version. + UnionRule unionRule = + GCRULES.union().rule(GCRULES.maxAge(5, TimeUnit.DAYS)).rule(GCRULES.maxVersions(1)); + + // Creates column family with given GC rule. + try { + // ModifyColumnFamiliesRequest can be used both for adding and modifying families, here it is + // being used to add a family + ModifyColumnFamiliesRequest columnFamiliesRequest = + ModifyColumnFamiliesRequest.of(tableId).addFamily(COLUMN_FAMILY_3, unionRule); + adminClient.modifyFamilies(columnFamiliesRequest); + System.out.println("Created column family: " + COLUMN_FAMILY_3); + } catch (AlreadyExistsException e) { + System.err.println( + "Failed to create column family with rule, already exists: " + e.getMessage()); + } + // [END bigtable_create_family_gc_union] + } + + /** Demonstrates how to create a new instance of the IntersectionRule. */ + public void addFamilyWithIntersectionRule() { + System.out.printf("%nCreating column family %s with intersection GC rule%n", COLUMN_FAMILY_4); + // [START bigtable_create_family_gc_intersection] + // Creates a column family with GC policy to drop data that matches all conditions. + + // Defines a GC rule to drop cells older than 5 days AND older than the most recent 2 versions. + DurationRule maxAgeRule = GCRULES.maxAge(5, TimeUnit.DAYS); + VersionRule versionRule = GCRULES.maxVersions(2); + IntersectionRule intersectionRule = GCRULES.intersection().rule(maxAgeRule).rule(versionRule); + + // Creates column family with given GC rule. + try { + // ModifyColumnFamiliesRequest can be used both for adding and modifying families, here it is + // being used to add a family + ModifyColumnFamiliesRequest columnFamiliesRequest = + ModifyColumnFamiliesRequest.of(tableId).addFamily(COLUMN_FAMILY_4, intersectionRule); + adminClient.modifyFamilies(columnFamiliesRequest); + System.out.println("Created column family: " + COLUMN_FAMILY_4); + } catch (AlreadyExistsException e) { + System.err.println( + "Failed to create column family with rule, already exists: " + e.getMessage()); + } + // [END bigtable_create_family_gc_intersection] + } + + /** Demonstrates how to create a nested rule using the IntersectionRule and UnionRule. */ + public void addFamilyWithNestedRule() { + System.out.printf("%nCreating column family %s with a nested GC rule%n", COLUMN_FAMILY_5); + // [START bigtable_create_family_gc_nested] + // Creates a nested GC rule: + // Drop cells that are either older than the 10 recent versions + // OR + // Drop cells that are older than a month AND older than the 2 recent versions + VersionRule versionRule1 = GCRULES.maxVersions(10); + VersionRule versionRule2 = GCRULES.maxVersions(2); + DurationRule maxAgeRule = GCRULES.maxAge(30, TimeUnit.DAYS); + IntersectionRule intersectionRule = GCRULES.intersection().rule(maxAgeRule).rule(versionRule2); + UnionRule unionRule = GCRULES.union().rule(intersectionRule).rule(versionRule1); + + // Creates column family with given GC rule. + try { + // ModifyColumnFamiliesRequest can be used both for adding and modifying families, here it is + // being used to add a family + ModifyColumnFamiliesRequest columnFamiliesRequest = + ModifyColumnFamiliesRequest.of(tableId).addFamily(COLUMN_FAMILY_5, unionRule); + adminClient.modifyFamilies(columnFamiliesRequest); + System.out.println("Created column family: " + COLUMN_FAMILY_5); + } catch (AlreadyExistsException e) { + System.err.println( + "Failed to create column family with rule, already exists: " + e.getMessage()); + } + // [END bigtable_create_family_gc_nested] + } + + /** Demonstrates how to list a table's column families. */ + public void listColumnFamilies() { + System.out.println("\nPrinting ID and GC Rule for all column families"); + // [START bigtable_list_column_families] + // Lists all families in the table with GC rules. + try { + Table table = adminClient.getTable(tableId); + Collection columnFamilies = table.getColumnFamilies(); + for (ColumnFamily columnFamily : columnFamilies) { + System.out.printf( + "Column family: %s%nGC Rule: %s%n", + columnFamily.getId(), columnFamily.getGCRule().toString()); + } + } catch (NotFoundException e) { + System.err.println( + "Failed to list column families from a non-existent table: " + e.getMessage()); + } + // [END bigtable_list_column_families] + } + + /** Demonstrates how to modify a column family's rule. */ + public void modifyColumnFamilyRule() { + System.out.printf("%nUpdating column family %s GC rule%n", COLUMN_FAMILY_1); + // [START bigtable_update_gc_rule] + // Updates the column family metadata to update the GC rule. + // Updates a column family GC rule. + VersionRule versionRule = GCRULES.maxVersions(1); + try { + // ModifyColumnFamiliesRequest can be used both for adding and modifying families, here it is + // being used to modify a family + // Updates column family with given GC rule. + ModifyColumnFamiliesRequest updateRequest = + ModifyColumnFamiliesRequest.of(tableId).updateFamily(COLUMN_FAMILY_1, versionRule); + adminClient.modifyFamilies(updateRequest); + System.out.printf("Column family %s GC rule updated%n", COLUMN_FAMILY_1); + } catch (NotFoundException e) { + System.err.println("Failed to modify a non-existent column family: " + e.getMessage()); + } + // [END bigtable_update_gc_rule] + } + + /** Demonstrates how to print the modified column family. */ + public void printModifiedColumnFamily() { + System.out.printf("%nPrint updated GC rule for column family %s%n", COLUMN_FAMILY_1); + // [START bigtable_family_get_gc_rule] + try { + Table table = adminClient.getTable(tableId); + Collection columnFamilies = table.getColumnFamilies(); + for (ColumnFamily columnFamily : columnFamilies) { + if (columnFamily.getId().equals(COLUMN_FAMILY_1)) { + System.out.printf( + "Column family: %s%nGC Rule: %s%n", + columnFamily.getId(), columnFamily.getGCRule().toString()); + } + } + } catch (NotFoundException e) { + System.err.println("Failed to print a non-existent column family: " + e.getMessage()); + } + // [END bigtable_family_get_gc_rule] + } + + /** Demonstrates how to delete a column family. */ + public void deleteColumnFamily() { + System.out.println("\nDelete column family: " + COLUMN_FAMILY_2); + // [START bigtable_delete_family] + // Deletes a column family. + try { + ModifyColumnFamiliesRequest deleted = + ModifyColumnFamiliesRequest.of(tableId).dropFamily(COLUMN_FAMILY_2); + adminClient.modifyFamilies(deleted); + System.out.printf("Column family %s deleted successfully%n", COLUMN_FAMILY_2); + } catch (NotFoundException e) { + System.err.println("Failed to delete a non-existent column family: " + e.getMessage()); + } + // [END bigtable_delete_family] + } + + /** Demonstrates how to delete a table. */ + public void deleteTable() { + // [START bigtable_delete_table] + // Deletes the entire table. + System.out.println("\nDelete table: " + tableId); + try { + adminClient.deleteTable(tableId); + System.out.printf("Table: %s deleted successfully%n", tableId); + } catch (NotFoundException e) { + System.err.println("Failed to delete a non-existent table: " + e.getMessage()); + } + // [END bigtable_delete_table] + } +} diff --git a/bigtable/src/test/java/com/m/examples/bigtable/ITHelloWorld.java b/bigtable/src/test/java/com/m/examples/bigtable/ITHelloWorld.java new file mode 100644 index 00000000000..23120fc79c4 --- /dev/null +++ b/bigtable/src/test/java/com/m/examples/bigtable/ITHelloWorld.java @@ -0,0 +1,150 @@ +/* + * Copyright 2019 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.m.examples.bigtable; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; +import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import com.google.cloud.bigtable.data.v2.models.Row; +import java.io.IOException; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.AssumptionViolatedException; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** Integration tests for {@link HelloWorld} */ +public class ITHelloWorld { + + private static final String PROJECT_PROPERTY_NAME = "bigtable.project"; + private static final String INSTANCE_PROPERTY_NAME = "bigtable.instance"; + private static final String TABLE_PREFIX = "table"; + private static String tableId; + private static BigtableDataClient dataClient; + private static BigtableTableAdminClient adminClient; + private static String projectId; + private static String instanceId; + private HelloWorld helloWorld; + + @BeforeClass + public static void beforeClass() throws IOException { + projectId = System.getProperty(PROJECT_PROPERTY_NAME); + instanceId = System.getProperty(INSTANCE_PROPERTY_NAME); + if (projectId == null || instanceId == null) { + dataClient = null; + adminClient = null; + return; + } + BigtableDataSettings settings = + BigtableDataSettings.newBuilder().setProjectId(projectId).setInstanceId(instanceId).build(); + dataClient = BigtableDataClient.create(settings); + BigtableTableAdminSettings adminSettings = + BigtableTableAdminSettings.newBuilder() + .setProjectId(projectId) + .setInstanceId(instanceId) + .build(); + adminClient = BigtableTableAdminClient.create(adminSettings); + } + + @AfterClass + public static void afterClass() throws Exception { + garbageCollect(); + dataClient.close(); + adminClient.close(); + } + + @Before + public void setup() throws IOException { + if (adminClient == null || dataClient == null) { + throw new AssumptionViolatedException( + PROJECT_PROPERTY_NAME + + " or " + + INSTANCE_PROPERTY_NAME + + " property is not set, skipping integration tests."); + } + tableId = generateTableId(); + helloWorld = new HelloWorld(projectId, instanceId, tableId); + adminClient.createTable(CreateTableRequest.of(tableId).addFamily("cf1")); + } + + @After + public void after() { + if (adminClient.exists(tableId)) { + adminClient.deleteTable(tableId); + } + } + + @Test + public void testCreateAndDeleteTable() throws IOException { + // Creates a table. + String testTable = generateTableId(); + HelloWorld testHelloWorld = new HelloWorld(projectId, instanceId, testTable); + testHelloWorld.createTable(); + assertTrue(adminClient.exists(testTable)); + + // Deletes a table. + testHelloWorld.deleteTable(); + assertTrue(!adminClient.exists(testTable)); + } + + @Test + public void testWriteToTable() { + // Writes to a table. + helloWorld.writeToTable(); + Row row = dataClient.readRow(tableId, "rowKey0"); + assertNotNull(row); + } + + // TODO: add test for helloWorld.readSingleRow() + // TODO: add test for helloWorld.readTable() + + @Test + public void testRunDoesNotFail() throws Exception { + helloWorld.run(); + } + + private String generateTableId() { + return String.format( + "%s-%016x-%x", TABLE_PREFIX, System.currentTimeMillis(), new Random().nextLong()); + } + + private static void garbageCollect() { + Pattern timestampPattern = Pattern.compile(TABLE_PREFIX + "-([0-9a-f]+)-([0-9a-f]+)"); + for (String tableId : adminClient.listTables()) { + Matcher matcher = timestampPattern.matcher(tableId); + if (!matcher.matches()) { + continue; + } + String timestampStr = matcher.group(1); + long timestamp = Long.parseLong(timestampStr, 16); + if (System.currentTimeMillis() - timestamp < TimeUnit.MINUTES.toMillis(15)) { + continue; + } + System.out.println("\nGarbage collecting orphaned table: " + tableId); + adminClient.deleteTable(tableId); + } + } +} diff --git a/bigtable/src/test/java/com/m/examples/bigtable/ITInstanceAdminExample.java b/bigtable/src/test/java/com/m/examples/bigtable/ITInstanceAdminExample.java new file mode 100644 index 00000000000..a3ad1eff6d1 --- /dev/null +++ b/bigtable/src/test/java/com/m/examples/bigtable/ITInstanceAdminExample.java @@ -0,0 +1,152 @@ +/* + * Copyright 2019 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.m.examples.bigtable; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import com.google.api.gax.rpc.NotFoundException; +import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminSettings; +import com.google.cloud.bigtable.admin.v2.models.Cluster; +import com.google.cloud.bigtable.admin.v2.models.CreateInstanceRequest; +import com.google.cloud.bigtable.admin.v2.models.Instance; +import com.google.cloud.bigtable.admin.v2.models.Instance.Type; +import com.google.cloud.bigtable.admin.v2.models.StorageType; +import java.io.IOException; +import java.util.Random; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.AssumptionViolatedException; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** Integration tests for {@link InstanceAdminExample} */ +public class ITInstanceAdminExample { + + private static final String PROJECT_PROPERTY_NAME = "bigtable.project"; + private static final String ID_PREFIX = "instanceadmin"; + private static final String CLUSTER = "cluster"; + private static String projectId; + private static BigtableInstanceAdminClient adminClient; + private String clusterId; + private String instanceId; + private InstanceAdminExample instanceAdmin; + + @BeforeClass + public static void beforeClass() throws IOException { + projectId = System.getProperty(PROJECT_PROPERTY_NAME); + if (projectId == null) { + adminClient = null; + return; + } + BigtableInstanceAdminSettings instanceAdminSettings = + BigtableInstanceAdminSettings.newBuilder().setProjectId(projectId).build(); + adminClient = BigtableInstanceAdminClient.create(instanceAdminSettings); + } + + @AfterClass + public static void afterClass() { + garbageCollect(); + adminClient.close(); + } + + @Before + public void setup() throws IOException { + if (adminClient == null) { + throw new AssumptionViolatedException( + PROJECT_PROPERTY_NAME + " property is not set, skipping integration tests."); + } + instanceId = generateId(); + clusterId = generateId(); + instanceAdmin = new InstanceAdminExample(projectId, instanceId, clusterId); + adminClient.createInstance( + CreateInstanceRequest.of(instanceId) + .addCluster(clusterId, "us-central1-f", 3, StorageType.SSD) + .setType(Type.PRODUCTION) + .addLabel("example", "instance_admin")); + } + + @After + public void after() { + if (adminClient.exists(instanceId)) { + adminClient.deleteInstance(instanceId); + } + } + + @Test + public void testCreateAndDeleteInstance() throws IOException { + // Creates an instance. + String testInstance = generateId(); + String testCluster = generateId(); + InstanceAdminExample testInstanceAdmin = + new InstanceAdminExample(projectId, testInstance, testCluster); + testInstanceAdmin.createProdInstance(); + assertTrue(adminClient.exists(testInstance)); + + // Deletes an instance. + testInstanceAdmin.deleteInstance(); + assertFalse(adminClient.exists(testInstance)); + } + + @Test + public void testGetInstance() { + // Gets an instance. + Instance instance = instanceAdmin.getInstance(); + assertNotNull(instance); + } + + @Test(expected = NotFoundException.class) + public void testAddAndDeleteCluster() { + // Adds a cluster. + instanceAdmin.addCluster(); + Cluster cluster = adminClient.getCluster(instanceId, CLUSTER); + assertNotNull(cluster); + + // Deletes a cluster. + instanceAdmin.deleteCluster(); + adminClient.getCluster(instanceId, CLUSTER); + } + + // TODO: add test for instanceAdmin.listInstances() + // TODO: and test for instanceAdmin.listClusters() + + @Test + public void testRunDoesNotFail() { + instanceAdmin.run(); + } + + private static String generateId() { + return String.format("%s-%x", ID_PREFIX, new Random().nextInt()); + } + + private static void garbageCollect() { + Pattern timestampPattern = Pattern.compile(ID_PREFIX + "-([0-9a-f]+)"); + System.out.println(); + for (Instance instance : adminClient.listInstances()) { + Matcher matcher = timestampPattern.matcher(instance.getId()); + if (!matcher.matches()) { + continue; + } + System.out.println("Garbage collecting orphaned table: " + instance); + adminClient.deleteInstance(instance.getId()); + } + } +} diff --git a/bigtable/src/test/java/com/m/examples/bigtable/ITTableAdminExample.java b/bigtable/src/test/java/com/m/examples/bigtable/ITTableAdminExample.java new file mode 100644 index 00000000000..414477cf8be --- /dev/null +++ b/bigtable/src/test/java/com/m/examples/bigtable/ITTableAdminExample.java @@ -0,0 +1,226 @@ +/* + * Copyright 2019 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.m.examples.bigtable; + +import static com.google.cloud.bigtable.admin.v2.models.GCRules.GCRULES; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; +import com.google.cloud.bigtable.admin.v2.models.ColumnFamily; +import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; +import com.google.cloud.bigtable.admin.v2.models.GCRules.DurationRule; +import com.google.cloud.bigtable.admin.v2.models.GCRules.GCRule; +import com.google.cloud.bigtable.admin.v2.models.GCRules.IntersectionRule; +import com.google.cloud.bigtable.admin.v2.models.GCRules.UnionRule; +import com.google.cloud.bigtable.admin.v2.models.GCRules.VersionRule; +import java.io.IOException; +import java.util.List; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.AssumptionViolatedException; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** Integration tests for {@link TableAdminExample} */ +public class ITTableAdminExample { + + private static final String PROJECT_PROPERTY_NAME = "bigtable.project"; + private static final String INSTANCE_PROPERTY_NAME = "bigtable.instance"; + private static final String TABLE_PREFIX = "table"; + private static BigtableTableAdminClient adminClient; + private static String instanceId; + private static String projectId; + private String tableId; + private TableAdminExample tableAdmin; + + @BeforeClass + public static void beforeClass() throws IOException { + projectId = System.getProperty(PROJECT_PROPERTY_NAME); + instanceId = System.getProperty(INSTANCE_PROPERTY_NAME); + if (projectId == null || instanceId == null) { + adminClient = null; + return; + } + BigtableTableAdminSettings adminSettings = + BigtableTableAdminSettings.newBuilder() + .setInstanceId(instanceId) + .setProjectId(projectId) + .build(); + adminClient = BigtableTableAdminClient.create(adminSettings); + } + + @AfterClass + public static void afterClass() { + garbageCollect(); + adminClient.close(); + } + + @Before + public void setup() throws IOException { + if (adminClient == null) { + throw new AssumptionViolatedException( + INSTANCE_PROPERTY_NAME + + " or " + + PROJECT_PROPERTY_NAME + + " property is not set, skipping integration tests."); + } + tableId = generateTableId(); + tableAdmin = new TableAdminExample(projectId, instanceId, tableId); + adminClient.createTable(CreateTableRequest.of(tableId).addFamily("cf")); + } + + @After + public void after() { + if (adminClient.exists(tableId)) { + adminClient.deleteTable(tableId); + } + } + + @Test + public void testCreateAndDeleteTable() throws IOException { + // Creates a table. + String testTable = generateTableId(); + TableAdminExample testTableAdmin = new TableAdminExample(projectId, instanceId, testTable); + testTableAdmin.createTable(); + assertTrue(adminClient.exists(testTable)); + + // Deletes a table. + testTableAdmin.deleteTable(); + assertFalse(adminClient.exists(testTable)); + } + + @Test + public void testCreateMaxAgeRuleAndModifyAndPrintColumnFamily() { + // Max age rule + tableAdmin.addFamilyWithMaxAgeRule(); + DurationRule maxAgeCondition = GCRULES.maxAge(5, TimeUnit.DAYS); + boolean maxAgeRule = ruleCheck(maxAgeCondition); + assertTrue(maxAgeRule); + + // Modifies cf1. + tableAdmin.modifyColumnFamilyRule(); + GCRule modifiedRule = GCRULES.maxVersions(1); + boolean maxVersionRule = ruleCheck(modifiedRule); + assertTrue(maxVersionRule); + } + + @Test + public void testCreateMaxVersionsRuleAndDeleteColumnFamily() { + // Max versions rule + tableAdmin.addFamilyWithMaxVersionsRule(); + VersionRule maxVersionCondition = GCRULES.maxVersions(2); + boolean maxVersionRule = ruleCheck(maxVersionCondition); + assertTrue(maxVersionRule); + + // Deletes cf2. + tableAdmin.deleteColumnFamily(); + boolean found = true; + List columnFamilies = adminClient.getTable(tableId).getColumnFamilies(); + for (ColumnFamily columnFamily : columnFamilies) { + if (columnFamily.equals("cf2")) { + found = false; + break; + } + } + assertTrue(found); + } + + @Test + public void testCreateUnionRule() { + // Union rule + tableAdmin.addFamilyWithUnionRule(); + DurationRule maxAgeRule = GCRULES.maxAge(5, TimeUnit.DAYS); + VersionRule versionRule = GCRULES.maxVersions(1); + UnionRule unionCondition = GCRULES.union().rule(maxAgeRule).rule(versionRule); + boolean unionRule = ruleCheck(unionCondition); + assertTrue(unionRule); + } + + @Test + public void testCreateIntersectionRule() { + // Intersection rule + tableAdmin.addFamilyWithIntersectionRule(); + DurationRule maxAgeRule = GCRULES.maxAge(5, TimeUnit.DAYS); + VersionRule versionRule = GCRULES.maxVersions(2); + IntersectionRule intersectionCondition = + GCRULES.intersection().rule(maxAgeRule).rule(versionRule); + boolean intersectionRule = ruleCheck(intersectionCondition); + assertTrue(intersectionRule); + } + + @Test + public void testCreateNestedRule() { + // Nested rule + tableAdmin.addFamilyWithNestedRule(); + VersionRule versionRule = GCRULES.maxVersions(10); + DurationRule maxAgeRule = GCRULES.maxAge(30, TimeUnit.DAYS); + VersionRule versionRule2 = GCRULES.maxVersions(2); + IntersectionRule intersectionRule = GCRULES.intersection().rule(maxAgeRule).rule(versionRule2); + UnionRule nestedCondition = GCRULES.union().rule(intersectionRule).rule(versionRule); + boolean nestedRule = ruleCheck(nestedCondition); + assertTrue(nestedRule); + } + + @Test + public void testRunDoesNotFail() { + tableAdmin.run(); + } + + // TODO: add test for tableAdmin.listAllTables() + // TODO: add test for tableAdmin.getTableMeta() + // TODO: add test for tableAdmin.listColumnFamilies() + + private boolean ruleCheck(GCRule condition) { + boolean found = false; + List columnFamilies = adminClient.getTable(tableId).getColumnFamilies(); + for (ColumnFamily columnFamily : columnFamilies) { + if (columnFamily.getGCRule().equals(condition)) { + found = true; + break; + } + } + return found; + } + + private String generateTableId() { + return String.format( + "%s-%016x-%x", TABLE_PREFIX, System.currentTimeMillis(), new Random().nextLong()); + } + + private static void garbageCollect() { + Pattern timestampPattern = Pattern.compile(TABLE_PREFIX + "-([0-9a-f]+)-([0-9a-f]+)"); + for (String tableId : adminClient.listTables()) { + Matcher matcher = timestampPattern.matcher(tableId); + if (!matcher.matches()) { + continue; + } + String timestampStr = matcher.group(1); + long timestamp = Long.parseLong(timestampStr, 16); + if (System.currentTimeMillis() - timestamp < TimeUnit.MINUTES.toMillis(10)) { + continue; + } + System.out.println("\nGarbage collecting orphaned table: " + tableId); + adminClient.deleteTable(tableId); + } + } +} From 574063d28799ea7b34cc5ed11561071303e69f6d Mon Sep 17 00:00:00 2001 From: Billy Jacobson Date: Tue, 21 May 2019 14:39:16 -0400 Subject: [PATCH 02/10] Fix style issues and ran tests --- bigtable/pom.xml | 12 ++++++++++-- .../java/com/m/examples/bigtable/HelloWorld.java | 7 ++++--- .../m/examples/bigtable/InstanceAdminExample.java | 9 ++++----- .../com/m/examples/bigtable/TableAdminExample.java | 9 ++++----- .../{ITHelloWorld.java => HelloWorldTest.java} | 9 +++++---- ...minExample.java => InstanceAdminExampleTest.java} | 9 +++++---- ...eAdminExample.java => TableAdminExampleTest.java} | 9 +++++---- 7 files changed, 37 insertions(+), 27 deletions(-) rename bigtable/src/test/java/com/m/examples/bigtable/{ITHelloWorld.java => HelloWorldTest.java} (97%) rename bigtable/src/test/java/com/m/examples/bigtable/{ITInstanceAdminExample.java => InstanceAdminExampleTest.java} (97%) rename bigtable/src/test/java/com/m/examples/bigtable/{ITTableAdminExample.java => TableAdminExampleTest.java} (98%) diff --git a/bigtable/pom.xml b/bigtable/pom.xml index 72317713676..82f4b1a9655 100644 --- a/bigtable/pom.xml +++ b/bigtable/pom.xml @@ -9,9 +9,17 @@ bigtable 1.0-SNAPSHOT + + + com.google.cloud.samples + shared-configuration + 1.0.11 + + bigtable - - http://www.example.com UTF-8 diff --git a/bigtable/src/main/java/com/m/examples/bigtable/HelloWorld.java b/bigtable/src/main/java/com/m/examples/bigtable/HelloWorld.java index 8677e37cec9..36e1fc08f9a 100644 --- a/bigtable/src/main/java/com/m/examples/bigtable/HelloWorld.java +++ b/bigtable/src/main/java/com/m/examples/bigtable/HelloWorld.java @@ -1,18 +1,19 @@ /* - * Copyright 2019 Google LLC. All Rights Reserved. + * Copyright 2019 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - */ +*/ + package com.m.examples.bigtable; // [START bigtable_hw_imports_veneer] diff --git a/bigtable/src/main/java/com/m/examples/bigtable/InstanceAdminExample.java b/bigtable/src/main/java/com/m/examples/bigtable/InstanceAdminExample.java index 65f98e1f4fb..3b317d594cd 100644 --- a/bigtable/src/main/java/com/m/examples/bigtable/InstanceAdminExample.java +++ b/bigtable/src/main/java/com/m/examples/bigtable/InstanceAdminExample.java @@ -1,18 +1,19 @@ /* - * Copyright 2019 Google LLC. All Rights Reserved. + * Copyright 2019 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - */ +*/ + package com.m.examples.bigtable; import com.google.api.gax.rpc.AlreadyExistsException; @@ -70,14 +71,12 @@ public InstanceAdminExample(String projectId, String instanceId, String clusterI this.instanceId = instanceId; this.clusterId = clusterId; - // [START connecting_to_bigtable] // Creates the settings to configure a bigtable instance admin client. BigtableInstanceAdminSettings instanceAdminSettings = BigtableInstanceAdminSettings.newBuilder().setProjectId(projectId).build(); // Creates a bigtable instance admin client. adminClient = BigtableInstanceAdminClient.create(instanceAdminSettings); - // [END connecting_to_bigtable] } public void run() { diff --git a/bigtable/src/main/java/com/m/examples/bigtable/TableAdminExample.java b/bigtable/src/main/java/com/m/examples/bigtable/TableAdminExample.java index 11dbeaf0422..56c47e19a32 100644 --- a/bigtable/src/main/java/com/m/examples/bigtable/TableAdminExample.java +++ b/bigtable/src/main/java/com/m/examples/bigtable/TableAdminExample.java @@ -1,18 +1,19 @@ /* - * Copyright 2019 Google LLC. All Rights Reserved. + * Copyright 2019 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + *http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - */ +*/ + package com.m.examples.bigtable; import static com.google.cloud.bigtable.admin.v2.models.GCRules.GCRULES; @@ -82,7 +83,6 @@ public static void main(String[] args) throws IOException { public TableAdminExample(String projectId, String instanceId, String tableId) throws IOException { this.tableId = tableId; - // [START connecting_to_bigtable] // Creates the settings to configure a bigtable table admin client. BigtableTableAdminSettings adminSettings = BigtableTableAdminSettings.newBuilder() @@ -92,7 +92,6 @@ public TableAdminExample(String projectId, String instanceId, String tableId) th // Creates a bigtable table admin client. adminClient = BigtableTableAdminClient.create(adminSettings); - // [END connecting_to_bigtable] } public void run() { diff --git a/bigtable/src/test/java/com/m/examples/bigtable/ITHelloWorld.java b/bigtable/src/test/java/com/m/examples/bigtable/HelloWorldTest.java similarity index 97% rename from bigtable/src/test/java/com/m/examples/bigtable/ITHelloWorld.java rename to bigtable/src/test/java/com/m/examples/bigtable/HelloWorldTest.java index 23120fc79c4..e19a1a17e66 100644 --- a/bigtable/src/test/java/com/m/examples/bigtable/ITHelloWorld.java +++ b/bigtable/src/test/java/com/m/examples/bigtable/HelloWorldTest.java @@ -1,18 +1,19 @@ /* - * Copyright 2019 Google LLC. All Rights Reserved. + * Copyright 2019 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - */ +*/ + package com.m.examples.bigtable; import static org.junit.Assert.assertNotNull; @@ -37,7 +38,7 @@ import org.junit.Test; /** Integration tests for {@link HelloWorld} */ -public class ITHelloWorld { +public class HelloWorldTest { private static final String PROJECT_PROPERTY_NAME = "bigtable.project"; private static final String INSTANCE_PROPERTY_NAME = "bigtable.instance"; diff --git a/bigtable/src/test/java/com/m/examples/bigtable/ITInstanceAdminExample.java b/bigtable/src/test/java/com/m/examples/bigtable/InstanceAdminExampleTest.java similarity index 97% rename from bigtable/src/test/java/com/m/examples/bigtable/ITInstanceAdminExample.java rename to bigtable/src/test/java/com/m/examples/bigtable/InstanceAdminExampleTest.java index a3ad1eff6d1..8a92449ba74 100644 --- a/bigtable/src/test/java/com/m/examples/bigtable/ITInstanceAdminExample.java +++ b/bigtable/src/test/java/com/m/examples/bigtable/InstanceAdminExampleTest.java @@ -1,18 +1,19 @@ /* - * Copyright 2019 Google LLC. All Rights Reserved. + * Copyright 2019 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - */ +*/ + package com.m.examples.bigtable; import static org.junit.Assert.assertFalse; @@ -39,7 +40,7 @@ import org.junit.Test; /** Integration tests for {@link InstanceAdminExample} */ -public class ITInstanceAdminExample { +public class InstanceAdminExampleTest { private static final String PROJECT_PROPERTY_NAME = "bigtable.project"; private static final String ID_PREFIX = "instanceadmin"; diff --git a/bigtable/src/test/java/com/m/examples/bigtable/ITTableAdminExample.java b/bigtable/src/test/java/com/m/examples/bigtable/TableAdminExampleTest.java similarity index 98% rename from bigtable/src/test/java/com/m/examples/bigtable/ITTableAdminExample.java rename to bigtable/src/test/java/com/m/examples/bigtable/TableAdminExampleTest.java index 414477cf8be..d2163bf691a 100644 --- a/bigtable/src/test/java/com/m/examples/bigtable/ITTableAdminExample.java +++ b/bigtable/src/test/java/com/m/examples/bigtable/TableAdminExampleTest.java @@ -1,18 +1,19 @@ /* - * Copyright 2019 Google LLC. All Rights Reserved. + * Copyright 2019 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - */ +*/ + package com.m.examples.bigtable; import static com.google.cloud.bigtable.admin.v2.models.GCRules.GCRULES; @@ -42,7 +43,7 @@ import org.junit.Test; /** Integration tests for {@link TableAdminExample} */ -public class ITTableAdminExample { +public class TableAdminExampleTest { private static final String PROJECT_PROPERTY_NAME = "bigtable.project"; private static final String INSTANCE_PROPERTY_NAME = "bigtable.instance"; From e1f37d1bfe1d9d17c329f1eb443e46ca56478f0b Mon Sep 17 00:00:00 2001 From: Billy Jacobson Date: Tue, 21 May 2019 17:10:28 -0400 Subject: [PATCH 03/10] Writing quickstart + test for Bigtable --- .../com/m/examples/bigtable/Quickstart.java | 53 ++++++++ .../m/examples/bigtable/QuickstartTest.java | 127 ++++++++++++++++++ 2 files changed, 180 insertions(+) create mode 100644 bigtable/src/main/java/com/m/examples/bigtable/Quickstart.java create mode 100644 bigtable/src/test/java/com/m/examples/bigtable/QuickstartTest.java diff --git a/bigtable/src/main/java/com/m/examples/bigtable/Quickstart.java b/bigtable/src/main/java/com/m/examples/bigtable/Quickstart.java new file mode 100644 index 00000000000..6cf6209848f --- /dev/null +++ b/bigtable/src/main/java/com/m/examples/bigtable/Quickstart.java @@ -0,0 +1,53 @@ +/* + * Copyright 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.m.examples.bigtable; + +// START [bigtable_quickstart_veneer] +import com.google.api.gax.rpc.NotFoundException; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowCell; + +public class Quickstart { + public static void quickstart(String projectId, String instanceId, String tableId) + throws Exception { + // String projectId = "my-project-id"; + // String instanceId = "my-instance-id"; + // String tableId = "my-table-id"; + + BigtableDataSettings settings = + BigtableDataSettings.newBuilder().setProjectId(projectId).setInstanceId(instanceId).build(); + + // Creates a bigtable data client. + BigtableDataClient dataClient = BigtableDataClient.create(settings); + + try { + System.out.println("\nReading a single row by row key"); + Row row = dataClient.readRow(tableId, "r1"); + System.out.println("Row: " + row.getKey().toStringUtf8()); + for (RowCell cell : row.getCells()) { + System.out.printf( + "Family: %s Qualifier: %s Value: %s%n", + cell.getFamily(), cell.getQualifier().toStringUtf8(), cell.getValue().toStringUtf8()); + } + } catch (NotFoundException e) { + System.err.println("Failed to read from a non-existent table: " + e.getMessage()); + } + } +} +// END [bigtable_quickstart_veneer] diff --git a/bigtable/src/test/java/com/m/examples/bigtable/QuickstartTest.java b/bigtable/src/test/java/com/m/examples/bigtable/QuickstartTest.java new file mode 100644 index 00000000000..05391524e87 --- /dev/null +++ b/bigtable/src/test/java/com/m/examples/bigtable/QuickstartTest.java @@ -0,0 +1,127 @@ +/* + * Copyright 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.m.examples.bigtable; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; + +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import org.hamcrest.CoreMatchers; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.AssumptionViolatedException; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** Integration tests for {@link Quickstart} */ +@SuppressWarnings("Duplicates") +public class QuickstartTest { + private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); + + private static void requireEnvVar(String varName) { + assertNotNull( + System.getenv(varName), + "Environment variable '%s' is required to perform these tests.".format(varName)); + } + + private static void requireSysProp(String varName) { + assertNotNull( + System.getenv(varName), + "System property '%s' is required to perform these tests.".format(varName)); + } + + @BeforeClass + public static void checkRequirements() { + requireEnvVar("GOOGLE_APPLICATION_CREDENTIALS"); + requireEnvVar("GOOGLE_CLOUD_PROJECT"); + requireSysProp(PROJECT_PROPERTY_NAME); + requireSysProp(INSTANCE_PROPERTY_NAME); + } + + private static final String PROJECT_PROPERTY_NAME = "bigtable.project"; + private static final String INSTANCE_PROPERTY_NAME = "bigtable.instance"; + private static final String TABLE_ID = "quickstart-table"; + private static BigtableDataClient dataClient; + private static BigtableTableAdminClient adminClient; + private static String projectId; + private static String instanceId; + private ByteArrayOutputStream bout; + + @BeforeClass + public static void beforeClass() throws IOException { + projectId = System.getProperty(PROJECT_PROPERTY_NAME); + instanceId = System.getProperty(INSTANCE_PROPERTY_NAME); + + BigtableDataSettings settings = + BigtableDataSettings.newBuilder().setProjectId(projectId).setInstanceId(instanceId).build(); + dataClient = BigtableDataClient.create(settings); + BigtableTableAdminSettings adminSettings = + BigtableTableAdminSettings.newBuilder() + .setProjectId(projectId) + .setInstanceId(instanceId) + .build(); + adminClient = BigtableTableAdminClient.create(adminSettings); + } + + @AfterClass + public static void afterClass() throws Exception { + dataClient.close(); + adminClient.close(); + } + + @Before + public void setup() throws IOException { + if (adminClient == null || dataClient == null) { + throw new AssumptionViolatedException( + PROJECT_PROPERTY_NAME + + " or " + + INSTANCE_PROPERTY_NAME + + " property is not set, skipping integration tests."); + } + bout = new ByteArrayOutputStream(); + System.setOut(new PrintStream(bout)); + } + + @After + public void tearDown() { + System.setOut(null); + bout.reset(); + } + + @Test + public void testQuickstart() { + try { + Quickstart.quickstart(projectId, instanceId, TABLE_ID); + } catch (Exception e) { + System.out.println("Failed to run quickstart."); + System.out.println(e); + } + + String output = bout.toString(); + assertThat(output, CoreMatchers.containsString("Reading a single row by row key")); + assertThat(output, CoreMatchers.containsString("Row: r1")); + assertThat( + output, CoreMatchers.containsString("Family: cf1 Qualifier: c1 Value: quickstart")); + } +} From c5209732f4de755834f8f0558b475740045471c9 Mon Sep 17 00:00:00 2001 From: Billy Jacobson Date: Wed, 22 May 2019 16:26:10 -0400 Subject: [PATCH 04/10] Updating pom with current versions --- bigtable/pom.xml | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/bigtable/pom.xml b/bigtable/pom.xml index 82f4b1a9655..86934be3887 100644 --- a/bigtable/pom.xml +++ b/bigtable/pom.xml @@ -1,5 +1,19 @@ + @@ -23,8 +37,8 @@ UTF-8 - 1.7 - 1.7 + 1.8 + 1.8 @@ -34,11 +48,11 @@ 4.12 test - + com.google.cloud - google-cloud-bigtable-admin - 0.81.0-alpha + google-cloud-bigtable + 0.92.0-alpha From 845c798ea3e1ebdfb39b1072786c84d4ce4c2e20 Mon Sep 17 00:00:00 2001 From: Billy Jacobson Date: Wed, 22 May 2019 22:18:42 -0400 Subject: [PATCH 05/10] =?UTF-8?q?Adding=20default=20values=20for=20the=20s?= =?UTF-8?q?ystem=20properties=CB=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bigtable/pom.xml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/bigtable/pom.xml b/bigtable/pom.xml index 86934be3887..56bc9743a1b 100644 --- a/bigtable/pom.xml +++ b/bigtable/pom.xml @@ -98,6 +98,17 @@ maven-project-info-reports-plugin 3.0.0 + + org.apache.maven.plugins + maven-failsafe-plugin + 3.0.0-M3 + + + java-docs-samples-testing + instance + + + From dd82305f6dfb8bbf4c0cc4b6275b18d2aeee6dbf Mon Sep 17 00:00:00 2001 From: Billy Jacobson Date: Wed, 22 May 2019 22:26:44 -0400 Subject: [PATCH 06/10] surefire instead of failsafe --- bigtable/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigtable/pom.xml b/bigtable/pom.xml index 56bc9743a1b..b9570dffeb9 100644 --- a/bigtable/pom.xml +++ b/bigtable/pom.xml @@ -100,7 +100,7 @@ org.apache.maven.plugins - maven-failsafe-plugin + maven-surefire-plugin 3.0.0-M3 From 91fdca33cf3026eefe27e6095b927ccfe6227315 Mon Sep 17 00:00:00 2001 From: Billy Jacobson Date: Thu, 23 May 2019 13:01:11 -0400 Subject: [PATCH 07/10] setup table for testing using cbt tool, retriggering test From 274ac80eab30498f8672ca7fe729cbeee1e59969 Mon Sep 17 00:00:00 2001 From: Billy Jacobson Date: Thu, 23 May 2019 22:54:24 -0400 Subject: [PATCH 08/10] Removing unnecessary code from quickstart test that was causing failures --- .../com/m/examples/bigtable/QuickstartTest.java | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/bigtable/src/test/java/com/m/examples/bigtable/QuickstartTest.java b/bigtable/src/test/java/com/m/examples/bigtable/QuickstartTest.java index 05391524e87..90e5dd17d82 100644 --- a/bigtable/src/test/java/com/m/examples/bigtable/QuickstartTest.java +++ b/bigtable/src/test/java/com/m/examples/bigtable/QuickstartTest.java @@ -35,16 +35,7 @@ import org.junit.Test; /** Integration tests for {@link Quickstart} */ -@SuppressWarnings("Duplicates") public class QuickstartTest { - private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); - - private static void requireEnvVar(String varName) { - assertNotNull( - System.getenv(varName), - "Environment variable '%s' is required to perform these tests.".format(varName)); - } - private static void requireSysProp(String varName) { assertNotNull( System.getenv(varName), @@ -53,8 +44,6 @@ private static void requireSysProp(String varName) { @BeforeClass public static void checkRequirements() { - requireEnvVar("GOOGLE_APPLICATION_CREDENTIALS"); - requireEnvVar("GOOGLE_CLOUD_PROJECT"); requireSysProp(PROJECT_PROPERTY_NAME); requireSysProp(INSTANCE_PROPERTY_NAME); } @@ -103,12 +92,6 @@ public void setup() throws IOException { System.setOut(new PrintStream(bout)); } - @After - public void tearDown() { - System.setOut(null); - bout.reset(); - } - @Test public void testQuickstart() { try { From ac3015acc0505df107b2b3a5c59f4f3927655494 Mon Sep 17 00:00:00 2001 From: Billy Jacobson Date: Tue, 28 May 2019 16:57:19 -0400 Subject: [PATCH 09/10] cleaning up quickstart --- .../com/m/examples/bigtable/Quickstart.java | 38 +++++++++------- .../m/examples/bigtable/QuickstartTest.java | 44 +++++-------------- 2 files changed, 32 insertions(+), 50 deletions(-) diff --git a/bigtable/src/main/java/com/m/examples/bigtable/Quickstart.java b/bigtable/src/main/java/com/m/examples/bigtable/Quickstart.java index 6cf6209848f..4d520614281 100644 --- a/bigtable/src/main/java/com/m/examples/bigtable/Quickstart.java +++ b/bigtable/src/main/java/com/m/examples/bigtable/Quickstart.java @@ -1,5 +1,5 @@ /* - * Copyright 2019 Google Inc. + * Copyright 2019 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ package com.m.examples.bigtable; // START [bigtable_quickstart_veneer] + import com.google.api.gax.rpc.NotFoundException; import com.google.cloud.bigtable.data.v2.BigtableDataClient; import com.google.cloud.bigtable.data.v2.BigtableDataSettings; @@ -24,8 +25,8 @@ import com.google.cloud.bigtable.data.v2.models.RowCell; public class Quickstart { - public static void quickstart(String projectId, String instanceId, String tableId) - throws Exception { + + public static void quickstart(String projectId, String instanceId, String tableId) { // String projectId = "my-project-id"; // String instanceId = "my-instance-id"; // String tableId = "my-table-id"; @@ -33,20 +34,25 @@ public static void quickstart(String projectId, String instanceId, String tableI BigtableDataSettings settings = BigtableDataSettings.newBuilder().setProjectId(projectId).setInstanceId(instanceId).build(); - // Creates a bigtable data client. - BigtableDataClient dataClient = BigtableDataClient.create(settings); - - try { - System.out.println("\nReading a single row by row key"); - Row row = dataClient.readRow(tableId, "r1"); - System.out.println("Row: " + row.getKey().toStringUtf8()); - for (RowCell cell : row.getCells()) { - System.out.printf( - "Family: %s Qualifier: %s Value: %s%n", - cell.getFamily(), cell.getQualifier().toStringUtf8(), cell.getValue().toStringUtf8()); + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (BigtableDataClient dataClient = BigtableDataClient.create(settings)) { + try { + System.out.println("\nReading a single row by row key"); + Row row = dataClient.readRow(tableId, "r1"); + System.out.println("Row: " + row.getKey().toStringUtf8()); + for (RowCell cell : row.getCells()) { + System.out.printf( + "Family: %s Qualifier: %s Value: %s%n", + cell.getFamily(), cell.getQualifier().toStringUtf8(), cell.getValue().toStringUtf8()); + } + } catch (NotFoundException e) { + System.err.println("Failed to read from a non-existent table: " + e.getMessage()); } - } catch (NotFoundException e) { - System.err.println("Failed to read from a non-existent table: " + e.getMessage()); + + } catch (Exception e) { + System.out.println("Error during functionName: \n" + e.toString()); } } } diff --git a/bigtable/src/test/java/com/m/examples/bigtable/QuickstartTest.java b/bigtable/src/test/java/com/m/examples/bigtable/QuickstartTest.java index 90e5dd17d82..f807e35509d 100644 --- a/bigtable/src/test/java/com/m/examples/bigtable/QuickstartTest.java +++ b/bigtable/src/test/java/com/m/examples/bigtable/QuickstartTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2019 Google Inc. + * Copyright 2019 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -36,6 +36,13 @@ /** Integration tests for {@link Quickstart} */ public class QuickstartTest { + private static final String PROJECT_PROPERTY_NAME = "bigtable.project"; + private static final String INSTANCE_PROPERTY_NAME = "bigtable.instance"; + private static final String TABLE_ID = "quickstart-table"; + private static String projectId; + private static String instanceId; + private ByteArrayOutputStream bout; + private static void requireSysProp(String varName) { assertNotNull( System.getenv(varName), @@ -48,46 +55,15 @@ public static void checkRequirements() { requireSysProp(INSTANCE_PROPERTY_NAME); } - private static final String PROJECT_PROPERTY_NAME = "bigtable.project"; - private static final String INSTANCE_PROPERTY_NAME = "bigtable.instance"; - private static final String TABLE_ID = "quickstart-table"; - private static BigtableDataClient dataClient; - private static BigtableTableAdminClient adminClient; - private static String projectId; - private static String instanceId; - private ByteArrayOutputStream bout; - @BeforeClass - public static void beforeClass() throws IOException { + public static void beforeClass() { projectId = System.getProperty(PROJECT_PROPERTY_NAME); instanceId = System.getProperty(INSTANCE_PROPERTY_NAME); - - BigtableDataSettings settings = - BigtableDataSettings.newBuilder().setProjectId(projectId).setInstanceId(instanceId).build(); - dataClient = BigtableDataClient.create(settings); - BigtableTableAdminSettings adminSettings = - BigtableTableAdminSettings.newBuilder() - .setProjectId(projectId) - .setInstanceId(instanceId) - .build(); - adminClient = BigtableTableAdminClient.create(adminSettings); } - @AfterClass - public static void afterClass() throws Exception { - dataClient.close(); - adminClient.close(); - } @Before - public void setup() throws IOException { - if (adminClient == null || dataClient == null) { - throw new AssumptionViolatedException( - PROJECT_PROPERTY_NAME - + " or " - + INSTANCE_PROPERTY_NAME - + " property is not set, skipping integration tests."); - } + public void setupStream() { bout = new ByteArrayOutputStream(); System.setOut(new PrintStream(bout)); } From d52b4a16470a28a2c0b64cfe645246cf08c8c5c4 Mon Sep 17 00:00:00 2001 From: Billy Jacobson Date: Mon, 3 Jun 2019 12:53:52 -0400 Subject: [PATCH 10/10] Changing test variables to use GOOGLE_CLOUD_PROJECT and the instance environment variable --- .kokoro/tests/run_tests.sh | 1 + bigtable/pom.xml | 11 ---------- .../m/examples/bigtable/HelloWorldTest.java | 21 +++++++++--------- .../bigtable/InstanceAdminExampleTest.java | 14 +++++++----- .../m/examples/bigtable/QuickstartTest.java | 22 ++++++++----------- .../bigtable/TableAdminExampleTest.java | 22 +++++++++---------- 6 files changed, 39 insertions(+), 52 deletions(-) diff --git a/.kokoro/tests/run_tests.sh b/.kokoro/tests/run_tests.sh index 89e660b41e7..8ef7495ac24 100755 --- a/.kokoro/tests/run_tests.sh +++ b/.kokoro/tests/run_tests.sh @@ -56,6 +56,7 @@ if [[ "$SCRIPT_DEBUG" != "true" ]]; then source "${KOKORO_GFILE_DIR}/aws-secrets.sh" source "${KOKORO_GFILE_DIR}/storage-hmac-credentials.sh" source "${KOKORO_GFILE_DIR}/dlp_secrets.txt" + source "${KOKORO_GFILE_DIR}/bigtable_secrets.txt" # Activate service account gcloud auth activate-service-account \ --key-file="$GOOGLE_APPLICATION_CREDENTIALS" \ diff --git a/bigtable/pom.xml b/bigtable/pom.xml index b9570dffeb9..86934be3887 100644 --- a/bigtable/pom.xml +++ b/bigtable/pom.xml @@ -98,17 +98,6 @@ maven-project-info-reports-plugin 3.0.0 - - org.apache.maven.plugins - maven-surefire-plugin - 3.0.0-M3 - - - java-docs-samples-testing - instance - - - diff --git a/bigtable/src/test/java/com/m/examples/bigtable/HelloWorldTest.java b/bigtable/src/test/java/com/m/examples/bigtable/HelloWorldTest.java index e19a1a17e66..522a99e0148 100644 --- a/bigtable/src/test/java/com/m/examples/bigtable/HelloWorldTest.java +++ b/bigtable/src/test/java/com/m/examples/bigtable/HelloWorldTest.java @@ -40,8 +40,7 @@ /** Integration tests for {@link HelloWorld} */ public class HelloWorldTest { - private static final String PROJECT_PROPERTY_NAME = "bigtable.project"; - private static final String INSTANCE_PROPERTY_NAME = "bigtable.instance"; + private static final String INSTANCE_PROPERTY_NAME = "BIGTABLE_TESTING_INSTANCE"; private static final String TABLE_PREFIX = "table"; private static String tableId; private static BigtableDataClient dataClient; @@ -50,10 +49,17 @@ public class HelloWorldTest { private static String instanceId; private HelloWorld helloWorld; + private static String requireEnv(String varName) { + assertNotNull( + System.getenv(varName), + "System property '%s' is required to perform these tests.".format(varName)); + return System.getenv(varName); + } + @BeforeClass public static void beforeClass() throws IOException { - projectId = System.getProperty(PROJECT_PROPERTY_NAME); - instanceId = System.getProperty(INSTANCE_PROPERTY_NAME); + projectId = requireEnv("GOOGLE_CLOUD_PROJECT"); + instanceId = requireEnv(INSTANCE_PROPERTY_NAME); if (projectId == null || instanceId == null) { dataClient = null; adminClient = null; @@ -79,13 +85,6 @@ public static void afterClass() throws Exception { @Before public void setup() throws IOException { - if (adminClient == null || dataClient == null) { - throw new AssumptionViolatedException( - PROJECT_PROPERTY_NAME - + " or " - + INSTANCE_PROPERTY_NAME - + " property is not set, skipping integration tests."); - } tableId = generateTableId(); helloWorld = new HelloWorld(projectId, instanceId, tableId); adminClient.createTable(CreateTableRequest.of(tableId).addFamily("cf1")); diff --git a/bigtable/src/test/java/com/m/examples/bigtable/InstanceAdminExampleTest.java b/bigtable/src/test/java/com/m/examples/bigtable/InstanceAdminExampleTest.java index 8a92449ba74..b7bcee7d3aa 100644 --- a/bigtable/src/test/java/com/m/examples/bigtable/InstanceAdminExampleTest.java +++ b/bigtable/src/test/java/com/m/examples/bigtable/InstanceAdminExampleTest.java @@ -42,7 +42,6 @@ /** Integration tests for {@link InstanceAdminExample} */ public class InstanceAdminExampleTest { - private static final String PROJECT_PROPERTY_NAME = "bigtable.project"; private static final String ID_PREFIX = "instanceadmin"; private static final String CLUSTER = "cluster"; private static String projectId; @@ -51,9 +50,16 @@ public class InstanceAdminExampleTest { private String instanceId; private InstanceAdminExample instanceAdmin; + private static String requireEnv(String varName) { + assertNotNull( + System.getenv(varName), + "System property '%s' is required to perform these tests.".format(varName)); + return System.getenv(varName); + } + @BeforeClass public static void beforeClass() throws IOException { - projectId = System.getProperty(PROJECT_PROPERTY_NAME); + projectId = requireEnv("GOOGLE_CLOUD_PROJECT"); if (projectId == null) { adminClient = null; return; @@ -71,10 +77,6 @@ public static void afterClass() { @Before public void setup() throws IOException { - if (adminClient == null) { - throw new AssumptionViolatedException( - PROJECT_PROPERTY_NAME + " property is not set, skipping integration tests."); - } instanceId = generateId(); clusterId = generateId(); instanceAdmin = new InstanceAdminExample(projectId, instanceId, clusterId); diff --git a/bigtable/src/test/java/com/m/examples/bigtable/QuickstartTest.java b/bigtable/src/test/java/com/m/examples/bigtable/QuickstartTest.java index f807e35509d..929506859ce 100644 --- a/bigtable/src/test/java/com/m/examples/bigtable/QuickstartTest.java +++ b/bigtable/src/test/java/com/m/examples/bigtable/QuickstartTest.java @@ -34,34 +34,30 @@ import org.junit.BeforeClass; import org.junit.Test; -/** Integration tests for {@link Quickstart} */ +/** + * Integration tests for {@link Quickstart} + */ public class QuickstartTest { - private static final String PROJECT_PROPERTY_NAME = "bigtable.project"; - private static final String INSTANCE_PROPERTY_NAME = "bigtable.instance"; + + private static final String INSTANCE_PROPERTY_NAME = "BIGTABLE_TESTING_INSTANCE"; private static final String TABLE_ID = "quickstart-table"; private static String projectId; private static String instanceId; private ByteArrayOutputStream bout; - private static void requireSysProp(String varName) { + private static String requireEnv(String varName) { assertNotNull( System.getenv(varName), "System property '%s' is required to perform these tests.".format(varName)); - } - - @BeforeClass - public static void checkRequirements() { - requireSysProp(PROJECT_PROPERTY_NAME); - requireSysProp(INSTANCE_PROPERTY_NAME); + return System.getenv(varName); } @BeforeClass public static void beforeClass() { - projectId = System.getProperty(PROJECT_PROPERTY_NAME); - instanceId = System.getProperty(INSTANCE_PROPERTY_NAME); + projectId = requireEnv("GOOGLE_CLOUD_PROJECT"); + instanceId = requireEnv(INSTANCE_PROPERTY_NAME); } - @Before public void setupStream() { bout = new ByteArrayOutputStream(); diff --git a/bigtable/src/test/java/com/m/examples/bigtable/TableAdminExampleTest.java b/bigtable/src/test/java/com/m/examples/bigtable/TableAdminExampleTest.java index d2163bf691a..51ed37cf053 100644 --- a/bigtable/src/test/java/com/m/examples/bigtable/TableAdminExampleTest.java +++ b/bigtable/src/test/java/com/m/examples/bigtable/TableAdminExampleTest.java @@ -18,6 +18,7 @@ import static com.google.cloud.bigtable.admin.v2.models.GCRules.GCRULES; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; @@ -45,8 +46,7 @@ /** Integration tests for {@link TableAdminExample} */ public class TableAdminExampleTest { - private static final String PROJECT_PROPERTY_NAME = "bigtable.project"; - private static final String INSTANCE_PROPERTY_NAME = "bigtable.instance"; + private static final String INSTANCE_PROPERTY_NAME = "BIGTABLE_TESTING_INSTANCE"; private static final String TABLE_PREFIX = "table"; private static BigtableTableAdminClient adminClient; private static String instanceId; @@ -54,10 +54,17 @@ public class TableAdminExampleTest { private String tableId; private TableAdminExample tableAdmin; + private static String requireEnv(String varName) { + assertNotNull( + System.getenv(varName), + "System property '%s' is required to perform these tests.".format(varName)); + return System.getenv(varName); + } + @BeforeClass public static void beforeClass() throws IOException { - projectId = System.getProperty(PROJECT_PROPERTY_NAME); - instanceId = System.getProperty(INSTANCE_PROPERTY_NAME); + projectId = requireEnv("GOOGLE_CLOUD_PROJECT"); + instanceId = requireEnv(INSTANCE_PROPERTY_NAME); if (projectId == null || instanceId == null) { adminClient = null; return; @@ -78,13 +85,6 @@ public static void afterClass() { @Before public void setup() throws IOException { - if (adminClient == null) { - throw new AssumptionViolatedException( - INSTANCE_PROPERTY_NAME - + " or " - + PROJECT_PROPERTY_NAME - + " property is not set, skipping integration tests."); - } tableId = generateTableId(); tableAdmin = new TableAdminExample(projectId, instanceId, tableId); adminClient.createTable(CreateTableRequest.of(tableId).addFamily("cf"));