From f04bb69cca4d1910e55895173cd68b223e411641 Mon Sep 17 00:00:00 2001 From: Marcelo Costa Date: Sat, 26 Oct 2019 12:24:28 -0300 Subject: [PATCH] Add Data Catalog createEntry samples and tests. (#1638) * Add Data Catalog createEntry samples and tests * Change second column name * Code review changes * Change region tag for the correct one * Removed tag from region tag since this sample does not tag the entry * ADD method overload to show parameter values examples * Code review changes * Code review changes * Code review and checkstyle changes * Clean up in a safer order --- datacatalog/cloud-client/build.gradle | 2 +- datacatalog/cloud-client/pom.xml | 2 +- .../example/datacatalog/CreateEntryGroup.java | 72 +++++++++++ .../datacatalog/CreateFilesetEntry.java | 118 ++++++++++++++++++ .../example/datacatalog/CreateEntryTests.java | 106 ++++++++++++++++ 5 files changed, 298 insertions(+), 2 deletions(-) create mode 100644 datacatalog/cloud-client/src/main/java/com/example/datacatalog/CreateEntryGroup.java create mode 100644 datacatalog/cloud-client/src/main/java/com/example/datacatalog/CreateFilesetEntry.java create mode 100644 datacatalog/cloud-client/src/test/java/com/example/datacatalog/CreateEntryTests.java diff --git a/datacatalog/cloud-client/build.gradle b/datacatalog/cloud-client/build.gradle index cb6066b14b0..a8b2ab32667 100644 --- a/datacatalog/cloud-client/build.gradle +++ b/datacatalog/cloud-client/build.gradle @@ -19,7 +19,7 @@ repositories { } dependencies { - compile group: 'com.google.cloud', name: 'google-cloud-datacatalog', version:'0.4.0-alpha' + compile group: 'com.google.cloud', name: 'google-cloud-datacatalog', version:'0.28.0-alpha' testCompile group: 'com.google.truth', name: 'truth', version:'0.42' testCompile group: 'junit', name: 'junit', version:'4.13-beta-2' diff --git a/datacatalog/cloud-client/pom.xml b/datacatalog/cloud-client/pom.xml index 25d04d1d386..762f119bb21 100644 --- a/datacatalog/cloud-client/pom.xml +++ b/datacatalog/cloud-client/pom.xml @@ -39,7 +39,7 @@ com.google.cloud google-cloud-datacatalog - 0.14.0-alpha + 0.28.0-alpha diff --git a/datacatalog/cloud-client/src/main/java/com/example/datacatalog/CreateEntryGroup.java b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/CreateEntryGroup.java new file mode 100644 index 00000000000..4e1e4e2519b --- /dev/null +++ b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/CreateEntryGroup.java @@ -0,0 +1,72 @@ +/* + * Copyright 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.datacatalog; + +// [START datacatalog_create_entry_group_tag] + +import com.google.api.gax.rpc.AlreadyExistsException; +import com.google.cloud.datacatalog.CreateEntryGroupRequest; +import com.google.cloud.datacatalog.EntryGroup; +import com.google.cloud.datacatalog.LocationName; +import com.google.cloud.datacatalog.v1beta1.DataCatalogClient; +import java.io.IOException; + +public class CreateEntryGroup { + + public static void createEntryGroup() { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String entryGroupId = "fileset_entry_group"; + createEntryGroup(projectId, entryGroupId); + } + + // Create Entry Group. + public static void createEntryGroup(String projectId, String entryGroupId) { + // Currently, Data Catalog stores metadata in the us-central1 region. + String location = "us-central1"; + + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) { + // Construct the EntryGroup for the EntryGroup request. + EntryGroup entryGroup = + EntryGroup.newBuilder() + .setDisplayName("My Fileset Entry Group") + .setDescription("This Entry Group consists of ....") + .build(); + + // Construct the EntryGroup request to be sent by the client. + CreateEntryGroupRequest entryGroupRequest = + CreateEntryGroupRequest.newBuilder() + .setParent(LocationName.of(projectId, location).toString()) + .setEntryGroupId(entryGroupId) + .setEntryGroup(entryGroup) + .build(); + + // Use the client to send the API request. + EntryGroup entryGroupResponse = dataCatalogClient.createEntryGroup(entryGroupRequest); + System.out.printf("\nEntry Group created with name: %s\n", entryGroupResponse.getName()); + } catch (AlreadyExistsException | IOException e) { + // AlreadyExistsException's are thrown if EntryGroup or Entry already exists. + // IOException's are thrown when unable to create the DataCatalogClient, + // for example an invalid Service Account path. + System.out.println("Error in create entry process:\n" + e.toString()); + } + } +} +// [END datacatalog_create_entry_group_tag] \ No newline at end of file diff --git a/datacatalog/cloud-client/src/main/java/com/example/datacatalog/CreateFilesetEntry.java b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/CreateFilesetEntry.java new file mode 100644 index 00000000000..b8c59214005 --- /dev/null +++ b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/CreateFilesetEntry.java @@ -0,0 +1,118 @@ +/* + * Copyright 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.datacatalog; + +// [START datacatalog_create_fileset_tag] + +import com.google.api.gax.rpc.AlreadyExistsException; +import com.google.cloud.datacatalog.ColumnSchema; +import com.google.cloud.datacatalog.CreateEntryRequest; +import com.google.cloud.datacatalog.Entry; +import com.google.cloud.datacatalog.EntryGroupName; +import com.google.cloud.datacatalog.EntryType; +import com.google.cloud.datacatalog.GcsFilesetSpec; +import com.google.cloud.datacatalog.Schema; +import com.google.cloud.datacatalog.v1beta1.DataCatalogClient; +import java.io.IOException; + +public class CreateFilesetEntry { + + public static void createEntry() { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String entryGroupId = "fileset_entry_group"; + String entryId = "fileset_entry_id"; + createEntry(projectId, entryGroupId, entryId); + } + + // Create Fileset Entry. + public static void createEntry(String projectId, String entryGroupId, String entryId) { + // Currently, Data Catalog stores metadata in the us-central1 region. + String location = "us-central1"; + + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) { + // Construct the Entry for the Entry request. + Entry entry = + Entry.newBuilder() + .setDisplayName("My Fileset") + .setDescription("This fileset consists of ....") + .setGcsFilesetSpec( + GcsFilesetSpec.newBuilder().addFilePatterns("gs://my_bucket/*").build()) + .setSchema( + Schema.newBuilder() + .addColumns( + ColumnSchema.newBuilder() + .setColumn("first_name") + .setDescription("First name") + .setMode("REQUIRED") + .setType("STRING") + .build()) + .addColumns( + ColumnSchema.newBuilder() + .setColumn("last_name") + .setDescription("Last name") + .setMode("REQUIRED") + .setType("STRING") + .build()) + .addColumns( + ColumnSchema.newBuilder() + .setColumn("addresses") + .setDescription("Addresses") + .setMode("REPEATED") + .setType("RECORD") + .addSubcolumns( + ColumnSchema.newBuilder() + .setColumn("city") + .setDescription("City") + .setMode("NULLABLE") + .setType("STRING") + .build()) + .addSubcolumns( + ColumnSchema.newBuilder() + .setColumn("state") + .setDescription("State") + .setMode("NULLABLE") + .setType("STRING") + .build()) + .build()) + .build()) + .setType(EntryType.FILESET) + .build(); + + // Construct the Entry request to be sent by the client. + CreateEntryRequest entryRequest = + CreateEntryRequest.newBuilder() + .setParent(EntryGroupName.of(projectId, location, entryGroupId).toString()) + .setEntryId(entryId) + .setEntry(entry) + .build(); + + // Use the client to send the API request. + Entry entryResponse = dataCatalogClient.createEntry(entryRequest); + System.out.printf("\nEntry created with name: %s\n", entryResponse.getName()); + } catch (AlreadyExistsException | IOException e) { + // AlreadyExistsException's are thrown if EntryGroup or Entry already exists. + // IOException's are thrown when unable to create the DataCatalogClient, + // for example an invalid Service Account path. + System.out.println("Error in create entry process:\n" + e.toString()); + } + } +} +// [END datacatalog_create_fileset_tag] \ No newline at end of file diff --git a/datacatalog/cloud-client/src/test/java/com/example/datacatalog/CreateEntryTests.java b/datacatalog/cloud-client/src/test/java/com/example/datacatalog/CreateEntryTests.java new file mode 100644 index 00000000000..2651606c89c --- /dev/null +++ b/datacatalog/cloud-client/src/test/java/com/example/datacatalog/CreateEntryTests.java @@ -0,0 +1,106 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.datacatalog; + +import static org.junit.Assert.assertThat; + +import com.google.cloud.datacatalog.EntryGroupName; +import com.google.cloud.datacatalog.EntryName; +import com.google.cloud.datacatalog.v1beta1.DataCatalogClient; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.util.UUID; +import org.hamcrest.CoreMatchers; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Integration (system) tests for {@link CreateFilesetEntry} and {@link CreateEntryGroup}. */ +@RunWith(JUnit4.class) +public class CreateEntryTests { + + private ByteArrayOutputStream bout; + + private static String ENTRY_GROUP_ID_NO_CHILDREN = + "entry_group_no_children_" + UUID.randomUUID().toString().substring(0, 8); + private static String PARENT_ENTRY_GROUP_ID = + "fileset_entry_group_parent_" + UUID.randomUUID().toString().substring(0, 8); + private static String ENTRY_ID = + "fileset_entry_id_" + UUID.randomUUID().toString().substring(0, 8); + private static String LOCATION = "us-central1"; + private static String PROJECT_ID = System.getenv().get("GOOGLE_CLOUD_PROJECT"); + + @Before + public void setUp() { + bout = new ByteArrayOutputStream(); + System.setOut(new PrintStream(bout)); + } + + @After + public void tearDown() { + System.setOut(null); + bout.reset(); + } + + @AfterClass + public static void tearDownClass() { + try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) { + dataCatalogClient.deleteEntryGroup( + EntryGroupName.of(PROJECT_ID, LOCATION, ENTRY_GROUP_ID_NO_CHILDREN).toString()); + + dataCatalogClient.deleteEntry( + EntryName.of(PROJECT_ID, LOCATION, PARENT_ENTRY_GROUP_ID, ENTRY_ID).toString()); + dataCatalogClient.deleteEntryGroup( + EntryGroupName.of(PROJECT_ID, LOCATION, PARENT_ENTRY_GROUP_ID).toString()); + } catch (Exception e) { + System.out.println("Error in cleaning up test data:\n" + e.toString()); + } + } + + @Test + public void testCreateFilesetEntry() { + // Must create a Entry Group before creating the entry. + CreateEntryGroup.createEntryGroup(PROJECT_ID, PARENT_ENTRY_GROUP_ID); + CreateFilesetEntry.createEntry(PROJECT_ID, PARENT_ENTRY_GROUP_ID, ENTRY_ID); + + String output = bout.toString(); + + String entryTemplate = + "Entry created with name: projects/%s/locations/us-central1/entryGroups/%s/entries/%s"; + assertThat( + output, + CoreMatchers.containsString( + String.format(entryTemplate, PROJECT_ID, PARENT_ENTRY_GROUP_ID, ENTRY_ID))); + } + + @Test + public void testCreateEntryGroup() { + CreateEntryGroup.createEntryGroup(PROJECT_ID, ENTRY_GROUP_ID_NO_CHILDREN); + + String output = bout.toString(); + + String entryGroupTemplate = + "Entry Group created with name: projects/%s/locations/us-central1/entryGroups/%s"; + assertThat( + output, + CoreMatchers.containsString( + String.format(entryGroupTemplate, PROJECT_ID, ENTRY_GROUP_ID_NO_CHILDREN))); + } +} \ No newline at end of file