Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Data Catalog createEntry quickstart samples and tests. #1731

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
4bc1c34
ADD sample for create fileset entry quickstart
mesmacosta Nov 13, 2019
9ced693
RAN google java format
mesmacosta Nov 13, 2019
08d40cf
CHANGE exception comment
mesmacosta Nov 13, 2019
3a1460b
FIX lint issues
mesmacosta Nov 13, 2019
a896ec3
Merge branch 'master' into task/add-dc-fileset-quickstart-sample
mesmacosta Nov 13, 2019
f6a4c02
Split quickstart into a new folder
mesmacosta Nov 13, 2019
0f15c98
Merge branch 'master' into task/add-dc-fileset-quickstart-sample
mesmacosta Nov 13, 2019
b0981ab
Rename files since class name will appear on samples
mesmacosta Nov 13, 2019
19c9a25
Ran java formatter
mesmacosta Nov 13, 2019
573074f
Merge branch 'master' into task/add-dc-fileset-quickstart-sample
mesmacosta Nov 18, 2019
0f51ac9
Merge branch 'master' into task/add-dc-fileset-quickstart-sample
mesmacosta Nov 19, 2019
63d7523
Merge branch 'master' into task/add-dc-fileset-quickstart-sample
mesmacosta Nov 20, 2019
90948e5
Merge branch 'master' into task/add-dc-fileset-quickstart-sample
mesmacosta Nov 21, 2019
1d5a089
Merge branch 'master' into task/add-dc-fileset-quickstart-sample
mesmacosta Nov 22, 2019
350f6fa
Merge branch 'master' into task/add-dc-fileset-quickstart-sample
mesmacosta Nov 24, 2019
71c4af2
Merge branch 'master' into task/add-dc-fileset-quickstart-sample
mesmacosta Nov 26, 2019
11646ea
revert region tags change
mesmacosta Nov 27, 2019
faca3b7
remove extra space
mesmacosta Nov 27, 2019
0f68bd0
Merge branch 'master' into task/add-dc-fileset-quickstart-sample
mesmacosta Nov 27, 2019
fe434f3
Merge branch 'master' into task/add-dc-fileset-quickstart-sample
mesmacosta Nov 27, 2019
8993ac4
Merge branch 'master' into task/add-dc-fileset-quickstart-sample
mesmacosta Nov 28, 2019
594821e
Merge branch 'master' into task/add-dc-fileset-quickstart-sample
mesmacosta Dec 3, 2019
9684187
Merge branch 'master' into task/add-dc-fileset-quickstart-sample
mesmacosta Dec 3, 2019
420577d
Merge branch 'master' into task/add-dc-fileset-quickstart-sample
mesmacosta Dec 4, 2019
db3c8a9
Merge branch 'master' into task/add-dc-fileset-quickstart-sample
mesmacosta Dec 4, 2019
4760778
change to real bucket name
mesmacosta Dec 5, 2019
283634d
Merge branch 'master' into task/add-dc-fileset-quickstart-sample
mesmacosta Dec 5, 2019
09d1748
Merge branch 'master' into task/add-dc-fileset-quickstart-sample
mesmacosta Dec 5, 2019
1e086cf
Merge branch 'master' into task/add-dc-fileset-quickstart-sample
mesmacosta Dec 9, 2019
c434a8c
Merge branch 'master' into task/add-dc-fileset-quickstart-sample
mesmacosta Dec 9, 2019
e986d0b
Merge branch 'master' into task/add-dc-fileset-quickstart-sample
mesmacosta Dec 10, 2019
0c13f44
Merge branch 'master' into task/add-dc-fileset-quickstart-sample
mesmacosta Dec 10, 2019
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ public static void createEntry(String projectId, String entryGroupId, String ent
.setDisplayName("My Fileset")
.setDescription("This fileset consists of ....")
.setGcsFilesetSpec(
GcsFilesetSpec.newBuilder().addFilePatterns("gs://my_bucket/*").build())
GcsFilesetSpec.newBuilder().addFilePatterns("gs://cloud-samples-data/*").build())
.setSchema(
Schema.newBuilder()
.addColumns(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,15 @@
package com.example.datacatalog;

import static org.junit.Assert.assertThat;
import static org.junit.Assert.fail;

import com.google.cloud.datacatalog.EntryGroupName;
import com.google.cloud.datacatalog.EntryName;
import com.google.cloud.datacatalog.v1beta1.DataCatalogClient;
import java.io.ByteArrayOutputStream;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import org.hamcrest.CoreMatchers;
import org.junit.After;
Expand All @@ -38,15 +41,12 @@ public class CreateEntryTests {

private ByteArrayOutputStream bout;

private static String ENTRY_GROUP_ID_NO_CHILDREN =
"entry_group_no_children_" + UUID.randomUUID().toString().substring(0, 8);
private static String PARENT_ENTRY_GROUP_ID =
"fileset_entry_group_parent_" + UUID.randomUUID().toString().substring(0, 8);
private static String ENTRY_ID =
"fileset_entry_id_" + UUID.randomUUID().toString().substring(0, 8);
private static String LOCATION = "us-central1";
private static String PROJECT_ID = System.getenv().get("GOOGLE_CLOUD_PROJECT");

private static List<String> entryGroupsPendingDeletion = new ArrayList<>();
private static List<String> entriesPendingDeletion = new ArrayList<>();

@Before
public void setUp() {
bout = new ByteArrayOutputStream();
Expand All @@ -62,45 +62,67 @@ public void tearDown() {
@AfterClass
public static void tearDownClass() {
try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) {
dataCatalogClient.deleteEntryGroup(
EntryGroupName.of(PROJECT_ID, LOCATION, ENTRY_GROUP_ID_NO_CHILDREN).toString());

dataCatalogClient.deleteEntry(
EntryName.of(PROJECT_ID, LOCATION, PARENT_ENTRY_GROUP_ID, ENTRY_ID).toString());
dataCatalogClient.deleteEntryGroup(
EntryGroupName.of(PROJECT_ID, LOCATION, PARENT_ENTRY_GROUP_ID).toString());
// Must delete Entries before deleting the Entry Group.
if (entriesPendingDeletion.isEmpty() || entryGroupsPendingDeletion.isEmpty()) {
fail("Something went wrong, no entries were generated");
}

for (String entryName : entriesPendingDeletion) {
dataCatalogClient.deleteEntry(entryName);
}

for (String entryGroupName : entryGroupsPendingDeletion) {
dataCatalogClient.deleteEntryGroup(entryGroupName);
}
} catch (Exception e) {
System.out.println("Error in cleaning up test data:\n" + e.toString());
}
}

@Test
public void testCreateFilesetEntry() {
String entryGroupId = "fileset_entry_group_parent_" + getUuid8Chars();
String entryId = "fileset_entry_id_" + getUuid8Chars();

// Must create a Entry Group before creating the entry.
CreateEntryGroup.createEntryGroup(PROJECT_ID, PARENT_ENTRY_GROUP_ID);
CreateFilesetEntry.createEntry(PROJECT_ID, PARENT_ENTRY_GROUP_ID, ENTRY_ID);
CreateEntryGroup.createEntryGroup(PROJECT_ID, entryGroupId);
CreateFilesetEntry.createEntry(PROJECT_ID, entryGroupId, entryId);

// Store names for clean up on teardown
String expectedEntryGroupName =
EntryGroupName.of(PROJECT_ID, LOCATION, entryGroupId).toString();
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FYI: I think these helpers are being deprecated. At least, they were deprecated in Python (googleapis/gapic-generator#3012) and the same reasoning applies to the Java generator (it's not possible with new proto annotations for configuring the generator).

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you know the recommended way?
I see two alternatives...

  1. Use EntryGroupName.Builder....setProject(project).setLocation(location).setEntryGroup(entryGroup).build()
  2. Build the string manually

Thanks

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

At least in Python (2) is recommended, as the resource builders are going away there. I haven't seen any discussion about it outside of Python, though.

I don't know why, as the same configuration changes affect all languages.

Copy link
Contributor Author

@mesmacosta mesmacosta Nov 28, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see, it makes sense going that way if this configuration will affect all languages...
@kurtisvg Could you point me the recommended way in Java ?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

AFAIK, they aren't going away for Java. They are generally preferred to manually formatting the string.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@tswast WDYT can we keep the helpers?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have doubts that they will actually stay in Java, based on my conversations with Luke on the ACtools team. I'll check with him if the resource helper deprecation is supposed to affect more than just Python.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, will wait for your confirmation.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can proceed with using the resource builders. I misunderstood the reasoning for removal in Python.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

so AFAIK the PR is ready to be merged, @tswast are you able to review it and merge if everything is ok?

entryGroupsPendingDeletion.add(expectedEntryGroupName);

String expectedEntryName = EntryName.of(PROJECT_ID, LOCATION, entryGroupId, entryId).toString();
entriesPendingDeletion.add(expectedEntryName);

String output = bout.toString();

String entryTemplate =
"Entry created with name: projects/%s/locations/us-central1/entryGroups/%s/entries/%s";
String entryTemplate = "Entry created with name: %s";
assertThat(
output,
CoreMatchers.containsString(
String.format(entryTemplate, PROJECT_ID, PARENT_ENTRY_GROUP_ID, ENTRY_ID)));
output, CoreMatchers.containsString(String.format(entryTemplate, expectedEntryName)));
}

@Test
public void testCreateEntryGroup() {
CreateEntryGroup.createEntryGroup(PROJECT_ID, ENTRY_GROUP_ID_NO_CHILDREN);
String entryGroupId = "entry_group_no_children_" + getUuid8Chars();

CreateEntryGroup.createEntryGroup(PROJECT_ID, entryGroupId);

// Store names for clean up on teardown
String expectedEntryGroupName =
EntryGroupName.of(PROJECT_ID, LOCATION, entryGroupId).toString();
entryGroupsPendingDeletion.add(expectedEntryGroupName);

String output = bout.toString();

String entryGroupTemplate =
"Entry Group created with name: projects/%s/locations/us-central1/entryGroups/%s";
String entryGroupTemplate = "Entry Group created with name: %s";
assertThat(
output,
CoreMatchers.containsString(
String.format(entryGroupTemplate, PROJECT_ID, ENTRY_GROUP_ID_NO_CHILDREN)));
CoreMatchers.containsString(String.format(entryGroupTemplate, expectedEntryGroupName)));
}

private String getUuid8Chars() {
return UUID.randomUUID().toString().substring(0, 8);
}
}
}
31 changes: 31 additions & 0 deletions datacatalog/quickstart/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
# Getting Started with Data Catalog and the Google Cloud Client libraries

<a href="https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/java-docs-samples&page=editor&open_in_editor=datacatalog/cloud-client/README.md">
<img alt="Open in Cloud Shell" src ="http://gstatic.com/cloudssh/images/open-btn.png"></a>

[Data Catalog][datacatalog] is a fully managed and scalable metadata management service that empowers organizations
to quickly discover, manage, and understand all their data in Google Cloud.
This sample Java application demonstrates how to access the Data Catalog API using
the [Google Cloud Client Library for Java][google-cloud-java].

[datacatalog]: https://cloud.google.com/data-catalog/
[google-cloud-java]: https://github.com/GoogleCloudPlatform/google-cloud-java

## Quickstart

#### Setup
- Install [Maven](http://maven.apache.org/).
- [Enable](https://console.cloud.google.com/apis/api/datacatalog.googleapis.com/overview) Data Catalog API.
- Set up [authentication](https://cloud.google.com/docs/authentication/getting-started).

#### Build
- Build your project with:
```
mvn clean package -DskipTests
```

#### Testing
Run the test with Maven.
```
mvn verify
```
41 changes: 41 additions & 0 deletions datacatalog/quickstart/build.gradle
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
// Copyright 2019 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

apply plugin: 'java'

repositories {
mavenCentral()
}

dependencies {
compile group: 'com.google.cloud', name: 'google-cloud-datacatalog-quickstart', version:'0.28.0-alpha'

testCompile group: 'com.google.truth', name: 'truth', version:'0.42'
testCompile group: 'junit', name: 'junit', version:'4.13-beta-2'
}

test {
useJUnit()
testLogging.showStandardStreams = true
beforeTest { descriptor ->
logger.lifecycle("test: " + descriptor + " Running")
}

onOutput { descriptor, event ->
logger.lifecycle("test: " + descriptor + ": " + event.message )
}
afterTest { descriptor, result ->
logger.lifecycle("test: " + descriptor + ": " + result )
}
}
59 changes: 59 additions & 0 deletions datacatalog/quickstart/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
<!--
Copyright 2019 Google LLC.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->

<project>
<modelVersion>4.0.0</modelVersion>
<groupId>com.example.datacatalog</groupId>
<artifactId>datacatalog-google-cloud-quickstart</artifactId>
<packaging>jar</packaging>

<!--
The parent pom defines common style checks and testing strategies for our samples.
Removing or replacing it should not affect the execution of the samples in anyway.
-->
<parent>
<groupId>com.google.cloud.samples</groupId>
<artifactId>shared-configuration</artifactId>
<version>1.0.11</version>
</parent>

<properties>
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
</properties>

<dependencies>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-datacatalog</artifactId>
<version>0.29.0-alpha</version>
</dependency>

<!-- Test dependencies -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.13-beta-3</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.truth</groupId>
<artifactId>truth</artifactId>
<version>1.0</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>
Loading