diff --git a/README.md b/README.md index 3f7fbcc9419..6d08e3e677f 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ This is a repository that contains java code snippets on [Cloud Platform Documen Technology Samples: * [Bigquery](bigquery) +* [Data Catalog](datacatalog) * [Datastore](datastore) * [Endpoints](endpoints) * [Identity-Aware Proxy](iap) diff --git a/datacatalog/cloud-client/README.md b/datacatalog/cloud-client/README.md new file mode 100644 index 00000000000..f4128e2c4b0 --- /dev/null +++ b/datacatalog/cloud-client/README.md @@ -0,0 +1,31 @@ +# Getting Started with Data Catalog and the Google Cloud Client libraries + + +Open in Cloud Shell + +[Data Catalog][datacatalog] is a fully managed and scalable metadata management service that empowers organizations +to quickly discover, manage, and understand all their data in Google Cloud. +This sample Java application demonstrates how to access the Data Catalog API using +the [Google Cloud Client Library for Java][google-cloud-java]. + +[datacatalog]: https://cloud.google.com/data-catalog/ +[google-cloud-java]: https://github.com/GoogleCloudPlatform/google-cloud-java + +## Quickstart + +#### Setup +- Install [Maven](http://maven.apache.org/). +- [Enable](https://console.cloud.google.com/apis/api/datacatalog.googleapis.com/overview) Data Catalog API. +- Set up [authentication](https://cloud.google.com/docs/authentication/getting-started). + +#### Build +- Build your project with: +``` + mvn clean package -DskipTests +``` + +#### Testing +Run the test with Maven. +``` + mvn verify +``` diff --git a/datacatalog/cloud-client/build.gradle b/datacatalog/cloud-client/build.gradle new file mode 100644 index 00000000000..cb6066b14b0 --- /dev/null +++ b/datacatalog/cloud-client/build.gradle @@ -0,0 +1,41 @@ +// Copyright 2019 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +apply plugin: 'java' + +repositories { + mavenCentral() +} + +dependencies { + compile group: 'com.google.cloud', name: 'google-cloud-datacatalog', version:'0.4.0-alpha' + + testCompile group: 'com.google.truth', name: 'truth', version:'0.42' + testCompile group: 'junit', name: 'junit', version:'4.13-beta-2' +} + +test { + useJUnit() + testLogging.showStandardStreams = true + beforeTest { descriptor -> + logger.lifecycle("test: " + descriptor + " Running") + } + + onOutput { descriptor, event -> + logger.lifecycle("test: " + descriptor + ": " + event.message ) + } + afterTest { descriptor, result -> + logger.lifecycle("test: " + descriptor + ": " + result ) + } +} diff --git a/datacatalog/cloud-client/pom.xml b/datacatalog/cloud-client/pom.xml new file mode 100644 index 00000000000..9ccd5c45756 --- /dev/null +++ b/datacatalog/cloud-client/pom.xml @@ -0,0 +1,59 @@ + + + + 4.0.0 + com.example.datacatalog + datacatalog-google-cloud-samples + jar + + + + com.google.cloud.samples + shared-configuration + 1.0.11 + + + + 1.8 + 1.8 + + + + + com.google.cloud + google-cloud-datacatalog + 0.4.0-alpha + + + + + junit + junit + 4.13-beta-2 + test + + + com.google.truth + truth + 0.42 + test + + + diff --git a/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryBigQueryDataset.java b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryBigQueryDataset.java new file mode 100644 index 00000000000..b81737645fc --- /dev/null +++ b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryBigQueryDataset.java @@ -0,0 +1,56 @@ +/* + * Copyright 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.datacatalog; + +import com.google.cloud.datacatalog.Entry; +import com.google.cloud.datacatalog.LookupEntryRequest; +import com.google.cloud.datacatalog.v1beta1.DataCatalogClient; + +public class LookupEntryBigQueryDataset { + + /** + * Lookup the Data Catalog entry referring to a BigQuery Dataset + * + * @param projectId The project ID to which the Dataset belongs, e.g. 'my-project' + * @param datasetId The dataset ID to which the Catalog Entry refers, e.g. 'my_dataset' + */ + public static void lookupEntry(String projectId, String datasetId) { + // String projectId = "my-project" + // String datasetId = "my_dataset" + + // Get an entry by the resource name from the source Google Cloud Platform service. + String linkedResource = + String.format("//bigquery.googleapis.com/projects/%s/datasets/%s", projectId, datasetId); + LookupEntryRequest request = + LookupEntryRequest.newBuilder().setLinkedResource(linkedResource).build(); + + // Alternatively, lookup by the SQL name of the entry would have the same result: + // String sqlResource = String.format("bigquery.dataset.`%s`.`%s`", projectId, datasetId); + // LookupEntryRequest request = + // LookupEntryRequest.newBuilder().setSqlResource(sqlResource).build(); + + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) { + Entry entry = dataCatalogClient.lookupEntry(request); + System.out.printf("Entry name: %s\n", entry.getName()); + } catch (Exception e) { + System.out.print("Error during lookupEntryBigQueryDataset:\n" + e.toString()); + } + } +} diff --git a/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryBigQueryTable.java b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryBigQueryTable.java new file mode 100644 index 00000000000..bae8e35c361 --- /dev/null +++ b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryBigQueryTable.java @@ -0,0 +1,61 @@ +/* + * Copyright 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.datacatalog; + +import com.google.cloud.datacatalog.Entry; +import com.google.cloud.datacatalog.LookupEntryRequest; +import com.google.cloud.datacatalog.v1beta1.DataCatalogClient; + +public class LookupEntryBigQueryTable { + + /** + * Lookup the Data Catalog entry referring to a BigQuery Table + * + * @param projectId The project ID to which the Dataset belongs, e.g. 'my-project' + * @param datasetId The dataset ID to which the Table belongs, e.g. 'my_dataset' + * @param tableId The table ID to which the Catalog Entry refers, e.g. 'my_table' + */ + public static void lookupEntry(String projectId, String datasetId, String tableId) { + // String projectId = "my-project" + // String datasetId = "my_dataset" + // String tableId = "my_table" + + // Get an entry by the resource name from the source Google Cloud Platform service. + String linkedResource = + String.format( + "//bigquery.googleapis.com/projects/%s/datasets/%s/tables/%s", + projectId, datasetId, tableId); + LookupEntryRequest request = + LookupEntryRequest.newBuilder().setLinkedResource(linkedResource).build(); + + // Alternatively, lookup by the SQL name of the entry would have the same result: + // String sqlResource = String.format("bigquery.table.`%s`.`%s`.`%s`", projectId, datasetId, + // tableId); + // LookupEntryRequest request = + // LookupEntryRequest.newBuilder().setSqlResource(sqlResource).build(); + + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) { + Entry entry = dataCatalogClient.lookupEntry(request); + System.out.printf("Entry name: %s\n", entry.getName()); + } catch (Exception e) { + System.out.print("Error during lookupEntryBigQueryTable:\n" + e.toString()); + } + } +} diff --git a/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryPubSubTopic.java b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryPubSubTopic.java new file mode 100644 index 00000000000..53195fffa9d --- /dev/null +++ b/datacatalog/cloud-client/src/main/java/com/example/datacatalog/LookupEntryPubSubTopic.java @@ -0,0 +1,56 @@ +/* + * Copyright 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.datacatalog; + +import com.google.cloud.datacatalog.Entry; +import com.google.cloud.datacatalog.LookupEntryRequest; +import com.google.cloud.datacatalog.v1beta1.DataCatalogClient; + +public class LookupEntryPubSubTopic { + + /** + * Lookup the Data Catalog entry referring to a BigQuery Dataset + * + * @param projectId The project ID to which the Dataset belongs, e.g. 'my-project' + * @param topicId The topic ID to which the Catalog Entry refers, e.g. 'my-topic' + */ + public static void lookupEntry(String projectId, String topicId) { + // String projectId = "my-project" + // String topicId = "my-topic" + + // Get an entry by the resource name from the source Google Cloud Platform service. + String linkedResource = + String.format("//pubsub.googleapis.com/projects/%s/topics/%s", projectId, topicId); + LookupEntryRequest request = + LookupEntryRequest.newBuilder().setLinkedResource(linkedResource).build(); + + // Alternatively, lookup by the SQL name of the entry would have the same result: + // String sqlResource = String.format("pubsub.topic.`%s`.`%s`", projectId, topicId); + // LookupEntryRequest request = + // LookupEntryRequest.newBuilder().setSqlResource(sqlResource).build(); + + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (DataCatalogClient dataCatalogClient = DataCatalogClient.create()) { + Entry entry = dataCatalogClient.lookupEntry(request); + System.out.printf("Entry name: %s\n", entry.getName()); + } catch (Exception e) { + System.out.print("Error during lookupEntryPubSubTopic:\n" + e.toString()); + } + } +} diff --git a/datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryTests.java b/datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryTests.java new file mode 100644 index 00000000000..bf254fec42e --- /dev/null +++ b/datacatalog/cloud-client/src/test/java/com/example/datacatalog/LookupEntryTests.java @@ -0,0 +1,81 @@ +/* + * Copyright 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.datacatalog; + +import static com.google.common.truth.Truth.assertThat; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class LookupEntryTests { + + private static final String BIGQUERY_PROJECT = "bigquery-public-data"; + private static final String BIGQUERY_DATASET = "new_york_taxi_trips"; + private static final String BIGQUERY_TABLE = "taxi_zone_geom"; + + private static final String PUBSUB_PROJECT = "pubsub-public-data"; + private static final String PUBSUB_TOPIC = "taxirides-realtime"; + + private ByteArrayOutputStream bout; + + @Before + public void setUp() { + bout = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(bout); + System.setOut(out); + } + + @After + public void tearDown() throws IOException { + bout.close(); + System.setOut(null); + } + + @Test + public void testLookupEntryBigQueryDataset() { + LookupEntryBigQueryDataset.lookupEntry(BIGQUERY_PROJECT, BIGQUERY_DATASET); + String got = bout.toString(); + assertThat(got) + .containsMatch( + "projects/" + BIGQUERY_PROJECT + "/locations/.+?/entryGroups/@bigquery/entries/.+?$"); + } + + @Test + public void testLookupEntryBigQueryTable() { + LookupEntryBigQueryTable.lookupEntry(BIGQUERY_PROJECT, BIGQUERY_DATASET, BIGQUERY_TABLE); + String got = bout.toString(); + assertThat(got) + .containsMatch( + "projects/" + BIGQUERY_PROJECT + "/locations/.+?/entryGroups/@bigquery/entries/.+?$"); + } + + @Test + public void testLookupPubSubTopic() { + LookupEntryPubSubTopic.lookupEntry(PUBSUB_PROJECT, PUBSUB_TOPIC); + String got = bout.toString(); + assertThat(got) + .containsMatch( + "projects/" + PUBSUB_PROJECT + "/locations/.+?/entryGroups/@pubsub/entries/.+?$"); + } +}