diff --git a/.github/workflows/datacatalog-snippets.yaml b/.github/workflows/datacatalog-snippets.yaml new file mode 100644 index 0000000000..0ee10b80cd --- /dev/null +++ b/.github/workflows/datacatalog-snippets.yaml @@ -0,0 +1,68 @@ +name: datacatalog-snippets +on: + push: + branches: + - main + paths: + - 'datacatalog/snippets/**' + pull_request: + paths: + - 'datacatalog/snippets/**' + pull_request_target: + types: [labeled] + paths: + - 'datacatalog/snippets/**' + schedule: + - cron: '0 0 * * 0' +jobs: + test: + if: ${{ github.event.action != 'labeled' || github.event.label.name == 'actions:force-run' }} + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + contents: 'write' + pull-requests: 'write' + id-token: 'write' + steps: + - uses: actions/checkout@v3.1.0 + with: + ref: ${{github.event.pull_request.head.sha}} + - uses: 'google-github-actions/auth@v1.0.0' + with: + workload_identity_provider: 'projects/1046198160504/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider' + service_account: 'kokoro-system-test@long-door-651.iam.gserviceaccount.com' + create_credentials_file: 'true' + access_token_lifetime: 600s + - uses: actions/setup-node@v3.5.1 + with: + node-version: 16 + - run: npm install + working-directory: datacatalog/snippets + - run: npm test + working-directory: datacatalog/snippets + env: + MOCHA_REPORTER_SUITENAME: datacatalog_snippets + MOCHA_REPORTER_OUTPUT: datacatalog_snippets_sponge_log.xml + MOCHA_REPORTER: xunit + - if: ${{ github.event.action == 'labeled' && github.event.label.name == 'actions:force-run' }} + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + try { + await github.rest.issues.removeLabel({ + name: 'actions:force-run', + owner: 'GoogleCloudPlatform', + repo: 'nodejs-docs-samples', + issue_number: context.payload.pull_request.number + }); + } catch (e) { + if (!e.message.includes('Label does not exist')) { + throw e; + } + } + - if: ${{ github.event_name == 'schedule'}} + run: | + curl https://github.com/googleapis/repo-automation-bots/releases/download/flakybot-1.1.0/flakybot -o flakybot -s -L + chmod +x ./flakybot + ./flakybot --repo GoogleCloudPlatform/nodejs-docs-samples --commit_hash ${{github.sha}} --build_url https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} diff --git a/.github/workflows/workflows.json b/.github/workflows/workflows.json index d2aebb5c0b..a97accdaa4 100644 --- a/.github/workflows/workflows.json +++ b/.github/workflows/workflows.json @@ -28,6 +28,7 @@ "containerengine/hello-world", "container/snippets", "datacatalog/cloud-client", + "datacatalog/snippets", "datalabeling", "datastore/functions", "datacatalog/quickstart", diff --git a/datacatalog/snippets/.eslintrc.yml b/datacatalog/snippets/.eslintrc.yml new file mode 100644 index 0000000000..674e33fb9f --- /dev/null +++ b/datacatalog/snippets/.eslintrc.yml @@ -0,0 +1,5 @@ +--- +rules: + no-console: off + node/no-missing-require: off + node/no-extraneous-require: off diff --git a/datacatalog/snippets/createCustomEntry.js b/datacatalog/snippets/createCustomEntry.js new file mode 100644 index 0000000000..56ef755f28 --- /dev/null +++ b/datacatalog/snippets/createCustomEntry.js @@ -0,0 +1,192 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +async function main(projectId, entryGroupId, entryId, tagTemplateId) { + // [START data_catalog_create_custom_entry] + // Import the Google Cloud client library. + const {DataCatalogClient} = require('@google-cloud/datacatalog').v1; + const datacatalog = new DataCatalogClient(); + + async function createCustomEntry() { + // Create a custom entry within an entry group. + + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // const projectId = 'my_project'; + // const entryGroupId = 'my_entry_group'; + // const entryId = 'my_entry'; + // const tagTemplateId = 'my_tag_template'; + + // Currently, Data Catalog stores metadata in the us-central1 region. + const location = 'us-central1'; + + // Delete any pre-existing Entry with the same name + // that will be used to create the new Entry. + try { + const entryName = datacatalog.entryPath( + projectId, + location, + entryGroupId, + entryId + ); + await datacatalog.deleteEntry({name: entryName}); + console.log(`Deleted Entry: ${entryName}`); + } catch (err) { + console.log('Entry does not exist.'); + } + + // Delete any pre-existing Entry Group with the same name + // that will be used to construct the new EntryGroup. + try { + const entryGroupName = datacatalog.entryGroupPath( + projectId, + location, + entryGroupId + ); + await datacatalog.deleteEntryGroup({name: entryGroupName}); + console.log(`Deleted Entry Group: ${entryGroupName}`); + } catch (err) { + console.log('Entry Group does not exist.'); + } + + // Delete any pre-existing Template with the same name + // that will be used to create a new Template. + const tagTemplateName = datacatalog.tagTemplatePath( + projectId, + location, + tagTemplateId + ); + + let tagTemplateRequest = { + name: tagTemplateName, + force: true, + }; + + try { + await datacatalog.deleteTagTemplate(tagTemplateRequest); + console.log(`Deleted template: ${tagTemplateName}`); + } catch (error) { + console.log(`Cannot delete template: ${tagTemplateName}`); + } + + // Construct the EntryGroup for the EntryGroup request. + const entryGroup = { + displayName: 'My awesome Entry Group', + description: 'This Entry Group represents an external system', + }; + + // Construct the EntryGroup request to be sent by the client. + const entryGroupRequest = { + parent: datacatalog.locationPath(projectId, location), + entryGroupId: entryGroupId, + entryGroup: entryGroup, + }; + + // Use the client to send the API request. + const [createdEntryGroup] = await datacatalog.createEntryGroup( + entryGroupRequest + ); + console.log(`Created entry group: ${createdEntryGroup.name}`); + + // Construct the Entry for the Entry request. + const entry = { + userSpecifiedSystem: 'onprem_data_system', + userSpecifiedType: 'onprem_data_asset', + displayName: 'My awesome data asset', + description: 'This data asset is managed by an external system.', + linkedResource: '//my-onprem-server.com/dataAssets/my-awesome-data-asset', + schema: { + columns: [ + { + column: 'first_column', + description: 'This columns consists of ....', + mode: 'NULLABLE', + type: 'STRING', + }, + { + column: 'second_column', + description: 'This columns consists of ....', + mode: 'NULLABLE', + type: 'DOUBLE', + }, + ], + }, + }; + + // Construct the Entry request to be sent by the client. + const entryRequest = { + parent: datacatalog.entryGroupPath(projectId, location, entryGroupId), + entryId: entryId, + entry: entry, + }; + + // Use the client to send the API request. + const [createdEntry] = await datacatalog.createEntry(entryRequest); + console.log(`Created entry: ${createdEntry.name}`); + + // Create a Tag Template. + // For more field types, including ENUM, please refer to + // https://cloud.google.com/data-catalog/docs/quickstarts/quickstart-search-tag#data-catalog-quickstart-nodejs. + const fieldSource = { + displayName: 'Source of data asset', + type: { + primitiveType: 'STRING', + }, + }; + + const tagTemplate = { + displayName: 'Demo Tag Template', + fields: { + source: fieldSource, + }, + }; + + tagTemplateRequest = { + parent: datacatalog.locationPath(projectId, location), + tagTemplateId: tagTemplateId, + tagTemplate: tagTemplate, + }; + + // Use the client to send the API request. + const [createdTagTemplate] = await datacatalog.createTagTemplate( + tagTemplateRequest + ); + console.log(`Created template: ${createdTagTemplate.name}`); + + // Attach a Tag to the custom Entry. + const tag = { + template: createdTagTemplate.name, + fields: { + source: { + stringValue: 'On-premises system name', + }, + }, + }; + + const tagRequest = { + parent: createdEntry.name, + tag: tag, + }; + + // Use the client to send the API request. + const [createdTag] = await datacatalog.createTag(tagRequest); + console.log(`Created tag: ${createdTag.name}`); + } + createCustomEntry(); + // [END data_catalog_create_custom_entry] +} +main(...process.argv.slice(2)); diff --git a/datacatalog/snippets/createFileset.js b/datacatalog/snippets/createFileset.js new file mode 100644 index 0000000000..ef528d12e5 --- /dev/null +++ b/datacatalog/snippets/createFileset.js @@ -0,0 +1,142 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +async function main(projectId, entryGroupId, entryId) { + // [START data_catalog_create_fileset] + // Import the Google Cloud client library. + const {DataCatalogClient} = require('@google-cloud/datacatalog').v1; + const datacatalog = new DataCatalogClient(); + + async function createFileset() { + // Create a fileset within an entry group. + + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // const projectId = 'my_project'; + // const entryGroupId = 'my_entry_group'; + // const entryId = 'my_entry'; + + // Currently, Data Catalog stores metadata in the us-central1 region. + const location = 'us-central1'; + + // Delete any pre-existing Entry with the same name that will be used + // when creating the new Entry. + try { + const formattedName = datacatalog.entryPath( + projectId, + location, + entryGroupId, + entryId + ); + await datacatalog.deleteEntry({name: formattedName}); + } catch (err) { + console.log('Entry does not exist.'); + } + + // Delete any pre-existing Entry Group with the same name + // that will be used to create the new Entry Group. + try { + const formattedName = datacatalog.entryGroupPath( + projectId, + location, + entryGroupId + ); + await datacatalog.deleteEntryGroup({name: formattedName}); + } catch (err) { + console.log('Entry Group does not exist.'); + } + + // Construct the Entry Group for the Entry Group request. + const entryGroup = { + displayName: 'My Fileset Entry Group', + description: 'This Entry Group consists of ....', + }; + + // Construct the Entry Group request to be sent by the client. + const entryGroupRequest = { + parent: datacatalog.locationPath(projectId, location), + entryGroupId: entryGroupId, + entryGroup: entryGroup, + }; + + // Use the client to send the API request. + await datacatalog.createEntryGroup(entryGroupRequest); + + // Construct the Entry for the Entry request. + const FILESET_TYPE = 4; + + const entry = { + displayName: 'My Fileset', + description: 'This fileset consists of ....', + gcsFilesetSpec: {filePatterns: ['gs://my_bucket/*']}, + schema: { + columns: [ + { + column: 'city', + description: 'City', + mode: 'NULLABLE', + type: 'STRING', + }, + { + column: 'state', + description: 'State', + mode: 'NULLABLE', + type: 'STRING', + }, + { + column: 'addresses', + description: 'Addresses', + mode: 'REPEATED', + subcolumns: [ + { + column: 'city', + description: 'City', + mode: 'NULLABLE', + type: 'STRING', + }, + { + column: 'state', + description: 'State', + mode: 'NULLABLE', + type: 'STRING', + }, + ], + type: 'RECORD', + }, + ], + }, + type: FILESET_TYPE, + }; + + // Construct the Entry request to be sent by the client. + const request = { + parent: datacatalog.entryGroupPath(projectId, location, entryGroupId), + entryId: entryId, + entry: entry, + }; + + // Use the client to send the API request. + const [response] = await datacatalog.createEntry(request); + + console.log(`Name: ${response.name}`); + console.log(`Display name: ${response.displayName}`); + console.log(`Type: ${response.type}`); + } + createFileset(); + // [END data_catalog_create_fileset] +} +main(...process.argv.slice(2)); diff --git a/datacatalog/snippets/grantTagTemplateUserRole.js b/datacatalog/snippets/grantTagTemplateUserRole.js new file mode 100644 index 0000000000..7b64308e19 --- /dev/null +++ b/datacatalog/snippets/grantTagTemplateUserRole.js @@ -0,0 +1,69 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +async function main(projectId, templateId, memberId) { + // [START data_catalog_grant_tag_template_user_role] + // Import the Google Cloud client library. + const {DataCatalogClient} = require('@google-cloud/datacatalog').v1; + const datacatalog = new DataCatalogClient(); + + async function grantTagTemplateUserRole() { + // Grant the tagTemplateUser role to a member of the project. + + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // const projectId = 'my_project'; // Google Cloud Platform project + // const templateId = 'my_existing_template'; + // const memberId = 'my_member_id' + + const location = 'us-central1'; + + // Format the Template name. + const templateName = datacatalog.tagTemplatePath( + projectId, + location, + templateId + ); + + // Retrieve Template's current IAM Policy. + const [getPolicyResponse] = await datacatalog.getIamPolicy({ + resource: templateName, + }); + const policy = getPolicyResponse; + + // Add Tag Template User role and member to the policy. + policy.bindings.push({ + role: 'roles/datacatalog.tagTemplateUser', + members: [memberId], + }); + + const request = { + resource: templateName, + policy: policy, + }; + + // Update Template's policy. + const [updatePolicyResponse] = await datacatalog.setIamPolicy(request); + + updatePolicyResponse.bindings.forEach(binding => { + console.log(`Role: ${binding.role}, Members: ${binding.members}`); + }); + } + grantTagTemplateUserRole(); + // [END data_catalog_grant_tag_template_user_role] +} +main(...process.argv.slice(2)); diff --git a/datacatalog/snippets/package.json b/datacatalog/snippets/package.json new file mode 100644 index 0000000000..dd47dcabe5 --- /dev/null +++ b/datacatalog/snippets/package.json @@ -0,0 +1,27 @@ +{ + "name": "@google-cloud/datacatalog-samples", + "description": "Samples for the Cloud Data Catalog client library for Node.js", + "license": "Apache-2.0", + "author": "Google LLC", + "engines": { + "node": ">=12.0.0" + }, + "repository": "googleapis/nodejs-datacatalog", + "private": true, + "files": [ + "*.js" + ], + "scripts": { + "test": "mocha --timeout 600000" + }, + "dependencies": { + "@google-cloud/bigquery": "^6.0.0", + "@google-cloud/datacatalog": "^3.1.0", + "uuid": "^9.0.0" + }, + "devDependencies": { + "chai": "^4.2.0", + "execa": "^5.0.0", + "mocha": "^8.0.0" + } +} diff --git a/datacatalog/snippets/policyTagManager/createPolicyTag.js b/datacatalog/snippets/policyTagManager/createPolicyTag.js new file mode 100644 index 0000000000..820a065798 --- /dev/null +++ b/datacatalog/snippets/policyTagManager/createPolicyTag.js @@ -0,0 +1,55 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +function main(parent) { + // [START data_catalog_ptm_create_policytag] + // Create a policy tag resource under a given parent taxonomy. + + // Import the Google Cloud client library. + const {PolicyTagManagerClient} = require('@google-cloud/datacatalog').v1; + const policyClient = new PolicyTagManagerClient(); + + async function createPolicyTag() { + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // const projectId = 'my_project'; // Google Cloud Platform project + // const location = 'us'; + // const taxonomy = 'my_existing_taxonomy'; + // const parent = `projects/${projectId}/locations/${location}/taxonomies/${taxonomy}`; + + const request = { + parent, + policyTag: { + displayName: 'nodejs_samples_tag', + // // It optionally accepts a parent ID, which can be used to create a hierarchical + // // relationship between tags. + // parentPolicyTag: `projects/${projectId}/locations/${location}/taxonomies/${taxonomy}/policyTags/my_existing_policy_tag` + }, + }; + + try { + const [metadata] = await policyClient.createPolicyTag(request); + console.log(`Created policy tag: ${metadata.name}`); + } catch (e) { + console.error(e); + process.exitCode = 1; + } + } + // [END data_catalog_ptm_create_policytag] + createPolicyTag(); +} +main(...process.argv.slice(2)); diff --git a/datacatalog/snippets/policyTagManager/createTaxonomy.js b/datacatalog/snippets/policyTagManager/createTaxonomy.js new file mode 100644 index 0000000000..542d0c9faf --- /dev/null +++ b/datacatalog/snippets/policyTagManager/createTaxonomy.js @@ -0,0 +1,56 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +function main(projectId, location, displayName) { + // [START data_catalog_ptm_create_taxonomy] + // Import the Google Cloud client library. + const {DataCatalogClient, PolicyTagManagerClient} = + require('@google-cloud/datacatalog').v1; + const dataCatalog = new DataCatalogClient(); + const policyTagManager = new PolicyTagManagerClient(); + + async function createTaxonomy() { + // const location = 'us'; + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // const projectId = 'my_project'; // Google Cloud Platform project + // const location = 'us' + // const displayName = 'my_display_name'; // Display name for new taxonomy. + + // Parent project location format is `projects/${projectId}/locations/${location}` + const parent = dataCatalog.locationPath(projectId, location); + + const request = { + parent: parent, + taxonomy: { + displayName: displayName, + activatedPolicyTypes: ['FINE_GRAINED_ACCESS_CONTROL'], + }, + }; + + try { + const [metadata] = await policyTagManager.createTaxonomy(request); + console.log(`Created taxonomy: ${metadata.name}`); + } catch (e) { + console.error(e); + process.exitCode = 1; + } + } + // [END data_catalog_ptm_create_taxonomy] + createTaxonomy(); +} +main(...process.argv.slice(2)); diff --git a/datacatalog/snippets/policyTagManager/deleteTaxonomy.js b/datacatalog/snippets/policyTagManager/deleteTaxonomy.js new file mode 100644 index 0000000000..597ca82633 --- /dev/null +++ b/datacatalog/snippets/policyTagManager/deleteTaxonomy.js @@ -0,0 +1,47 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +function main(taxonomyName) { + // [START data_catalog_ptm_delete_taxonomy] + // Import the Google Cloud client library. + const {PolicyTagManagerClient} = require('@google-cloud/datacatalog').v1; + const policyTagManager = new PolicyTagManagerClient(); + + async function deleteTaxonomy() { + /** + * TODO(developer): Uncomment the following line before running the sample. + */ + // const projectId = 'my_project'; // Google Cloud Platform project + // const location = 'us'; + // const taxonomy = 'my_existing_taxonomy'; + // const taxonomyName = `projects/${projectId}/locations/${location}/taxonomies/${taxonomy}`; + + const request = { + name: taxonomyName, + }; + + try { + await policyTagManager.deleteTaxonomy(request); + console.log(`Deleted taxonomy: ${taxonomyName}`); + } catch (e) { + console.error(e); + process.exitCode = 1; + } + } + // [END data_catalog_ptm_delete_taxonomy] + deleteTaxonomy(); +} +main(...process.argv.slice(2)); diff --git a/datacatalog/snippets/policyTagManager/getTaxonomy.js b/datacatalog/snippets/policyTagManager/getTaxonomy.js new file mode 100644 index 0000000000..dbf3a59a3c --- /dev/null +++ b/datacatalog/snippets/policyTagManager/getTaxonomy.js @@ -0,0 +1,47 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +function main(taxonomyName) { + // [START data_catalog_ptm_get_taxonomy] + // Import the Google Cloud client library. + const {PolicyTagManagerClient} = require('@google-cloud/datacatalog').v1; + const policyTagManager = new PolicyTagManagerClient(); + + async function getTaxonomy() { + /** + * TODO(developer): Uncomment the following line before running the sample. + */ + // const projectId = 'my_project'; // Google Cloud Platform project + // const location = 'us'; + // const taxonomy = 'my_existing_taxonomy'; + // const taxonomyName = `projects/${projectId}/locations/${location}/taxonomies/${taxonomy}`; + + const request = { + name: taxonomyName, + }; + + try { + const [taxonomy] = await policyTagManager.getTaxonomy(request); + console.log(`Retrieved taxonomy: ${taxonomy.name}`); + } catch (e) { + console.error(e); + process.exitCode = 1; + } + } + // [END data_catalog_ptm_get_taxonomy] + getTaxonomy(); +} +main(...process.argv.slice(2)); diff --git a/datacatalog/snippets/policyTagManager/listTaxonomies.js b/datacatalog/snippets/policyTagManager/listTaxonomies.js new file mode 100644 index 0000000000..fcfa23b8c9 --- /dev/null +++ b/datacatalog/snippets/policyTagManager/listTaxonomies.js @@ -0,0 +1,53 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +function main(projectId, location) { + // [START data_catalog_ptm_list_taxonomies] + // Import the Google Cloud client library. + const {DataCatalogClient, PolicyTagManagerClient} = + require('@google-cloud/datacatalog').v1; + const dataCatalog = new DataCatalogClient(); + const policyTagManager = new PolicyTagManagerClient(); + + async function listTaxonomies() { + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // const projectId = 'my_project'; // Google Cloud Platform project + // const location = 'us'; + + // Parent project location format is `projects/${projectId}/locations/${location}` + const parent = dataCatalog.locationPath(projectId, location); + + const request = { + parent: parent, + }; + + try { + const [taxonomies] = await policyTagManager.listTaxonomies(request); + console.log('Taxonomies:'); + taxonomies.forEach(taxonomy => { + console.log(taxonomy.name); + }); + } catch (e) { + console.error(e); + process.exitCode = 1; + } + } + // [END data_catalog_ptm_list_taxonomies] + listTaxonomies(); +} +main(...process.argv.slice(2)); diff --git a/datacatalog/snippets/quickstart.js b/datacatalog/snippets/quickstart.js new file mode 100644 index 0000000000..9e8ef19463 --- /dev/null +++ b/datacatalog/snippets/quickstart.js @@ -0,0 +1,170 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +async function main(projectId, datasetId, tableId) { + // [START data_catalog_quickstart] + // Import the Google Cloud client library and create a client. + const {DataCatalogClient} = require('@google-cloud/datacatalog').v1; + const datacatalog = new DataCatalogClient(); + + async function quickstart() { + // Common fields. + let request; + let responses; + + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // const projectId = 'my_project'; // Google Cloud Platform project + // const datasetId = 'demo_dataset'; + // const tableId = 'trips'; + + // Currently, Data Catalog stores metadata in the + // us-central1 region. + const location = 'us-central1'; + + // Create Fields. + const fieldSource = { + displayName: 'Source of data asset', + type: { + primitiveType: 'STRING', + }, + }; + + const fieldNumRows = { + displayName: 'Number of rows in data asset', + type: { + primitiveType: 'DOUBLE', + }, + }; + + const fieldHasPII = { + displayName: 'Has PII', + type: { + primitiveType: 'BOOL', + }, + }; + + const fieldPIIType = { + displayName: 'PII type', + type: { + enumType: { + allowedValues: [ + { + displayName: 'EMAIL', + }, + { + displayName: 'SOCIAL SECURITY NUMBER', + }, + { + displayName: 'NONE', + }, + ], + }, + }, + }; + + // Create Tag Template. + const tagTemplateId = 'demo_tag_template'; + + const tagTemplate = { + displayName: 'Demo Tag Template', + fields: { + source: fieldSource, + num_rows: fieldNumRows, + has_pii: fieldHasPII, + pii_type: fieldPIIType, + }, + }; + + const tagTemplatePath = datacatalog.tagTemplatePath( + projectId, + location, + tagTemplateId + ); + + // Delete any pre-existing Template with the same name. + try { + request = { + name: tagTemplatePath, + force: true, + }; + await datacatalog.deleteTagTemplate(request); + console.log(`Deleted template: ${tagTemplatePath}`); + } catch (error) { + console.log(`Cannot delete template: ${tagTemplatePath}`); + } + + // Create the Tag Template request. + const locationPath = datacatalog.locationPath(projectId, location); + + request = { + parent: locationPath, + tagTemplateId: tagTemplateId, + tagTemplate: tagTemplate, + }; + + // Execute the request. + responses = await datacatalog.createTagTemplate(request); + const createdTagTemplate = responses[0]; + console.log(`Created template: ${createdTagTemplate.name}`); + + // Lookup Data Catalog's Entry referring to the table. + responses = await datacatalog.lookupEntry({ + linkedResource: + '//bigquery.googleapis.com/projects/' + + `${projectId}/datasets/${datasetId}/tables/${tableId}`, + }); + const entry = responses[0]; + console.log(`Entry name: ${entry.name}`); + console.log(`Entry type: ${entry.type}`); + console.log(`Linked resource: ${entry.linkedResource}`); + + // Attach a Tag to the table. + const tag = { + name: entry.name, + template: createdTagTemplate.name, + fields: { + source: { + stringValue: 'copied from tlc_yellow_trips_2017', + }, + num_rows: { + doubleValue: 113496874, + }, + has_pii: { + boolValue: false, + }, + pii_type: { + enumValue: { + displayName: 'NONE', + }, + }, + }, + }; + + request = { + parent: entry.name, + tag: tag, + }; + + // Create the Tag. + await datacatalog.createTag(request); + console.log(`Tag created for entry: ${entry.name}`); + } + quickstart(); + // [END data_catalog_quickstart] +} +main(...process.argv.slice(2)); diff --git a/datacatalog/snippets/searchAssets.js b/datacatalog/snippets/searchAssets.js new file mode 100644 index 0000000000..03c91d1c3c --- /dev/null +++ b/datacatalog/snippets/searchAssets.js @@ -0,0 +1,57 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +async function main(projectId) { + // [START data_catalog_search_assets] + // Import the Google Cloud client library. + const {DataCatalogClient} = require('@google-cloud/datacatalog').v1; + const datacatalog = new DataCatalogClient(); + + async function searchAssets() { + // Search data assets. + + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // const projectId = 'my_project'; // Google Cloud Platform project + + // Set custom query. + const query = 'type=dataset'; + + // Create request. + const scope = { + includeProjectIds: [projectId], + // Alternatively, search using Google Cloud Organization scopes. + // includeOrgIds: [organizationId], + }; + + const request = { + scope: scope, + query: query, + }; + + const [result] = await datacatalog.searchCatalog(request); + + console.log(`Found ${result.length} datasets in project ${projectId}.`); + console.log('Datasets:'); + result.forEach(dataset => { + console.log(dataset.relativeResourceName); + }); + } + searchAssets(); + // [END data_catalog_search_assets] +} +main(...process.argv.slice(2)); diff --git a/datacatalog/snippets/test/.eslintrc.yml b/datacatalog/snippets/test/.eslintrc.yml new file mode 100644 index 0000000000..e1e03e9181 --- /dev/null +++ b/datacatalog/snippets/test/.eslintrc.yml @@ -0,0 +1,3 @@ +--- +rules: + node/no-extraneous-require: off \ No newline at end of file diff --git a/datacatalog/snippets/test/quickstart.test.js b/datacatalog/snippets/test/quickstart.test.js new file mode 100644 index 0000000000..61efbdaf55 --- /dev/null +++ b/datacatalog/snippets/test/quickstart.test.js @@ -0,0 +1,88 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +const {assert} = require('chai'); +const {describe, it, before} = require('mocha'); +const uuid = require('uuid'); +const cp = require('child_process'); +const {BigQuery} = require('@google-cloud/bigquery'); + +const bigquery = new BigQuery(); + +const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'}); + +const GCLOUD_TESTS_PREFIX = 'nodejs_data_catalog_samples'; +const generateUuid = () => + `${GCLOUD_TESTS_PREFIX}_${uuid.v4()}`.replace(/-/gi, '_'); + +describe('Quickstart', async () => { + const projectId = process.env.GCLOUD_PROJECT; + let datasetId; + let tableId; + + before(async () => { + // Delete any stale datasets from samples tests + await deleteDatasets(); + + // Create BigQuery dataset & table. + datasetId = generateUuid(); + tableId = generateUuid(); + await bigquery.createDataset(datasetId); + await bigquery.dataset(datasetId).createTable(tableId); + }); + + it('quickstart should attach tag to BigQuery table', async () => { + const output = execSync( + `node quickstart ${projectId} ${datasetId} ${tableId}` + ); + assert.include(output, 'Created template:'); + assert.include( + output, + `Linked resource: //bigquery.googleapis.com/projects/${projectId}/datasets/${datasetId}/tables/${tableId}` + ); + assert.include(output, 'Tag created for entry:'); + }); + + // Only delete a resource if it is older than 24 hours. That will prevent + // collisions with parallel CI test runs. + function isResourceStale(creationTime) { + const oneDayMs = 86400000; + const now = new Date(); + const created = new Date(creationTime); + return now.getTime() - created.getTime() >= oneDayMs; + } + + async function deleteDatasets() { + let [datasets] = await bigquery.getDatasets(); + datasets = datasets.filter(dataset => + dataset.id.includes(GCLOUD_TESTS_PREFIX) + ); + + for (const dataset of datasets) { + const [metadata] = await dataset.getMetadata(); + const creationTime = Number(metadata.creationTime); + + if (isResourceStale(creationTime)) { + try { + await dataset.delete({force: true}); + } catch (e) { + console.log(`dataset(${dataset.id}).delete() failed`); + console.log(e); + } + } + } + } +}); diff --git a/datacatalog/snippets/test/samples.test.js b/datacatalog/snippets/test/samples.test.js new file mode 100644 index 0000000000..e93f16b00a --- /dev/null +++ b/datacatalog/snippets/test/samples.test.js @@ -0,0 +1,230 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +const {assert} = require('chai'); +const {describe, it, before} = require('mocha'); +const uuid = require('uuid'); +const cp = require('child_process'); +const {DataCatalogClient, PolicyTagManagerClient} = + require('@google-cloud/datacatalog').v1; +const datacatalog = new DataCatalogClient(); +const policyTagManager = new PolicyTagManagerClient(); + +const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'}); + +const GCLOUD_TESTS_PREFIX = 'nodejs_samples_'; +const generateUuid = () => + `${GCLOUD_TESTS_PREFIX}_${uuid.v4()}`.replace(/-/gi, '_'); + +const TAG_TEMPLATE_ID = `${GCLOUD_TESTS_PREFIX}_test_tag_template`; +const projectId = process.env.GCLOUD_PROJECT; +const location = 'us-central1'; +let taxonomyName; + +describe('Samples', async () => { + before(async () => { + // Delete stale resources from samples tests. + await deleteEntryGroups(); + await deleteTaxonomies(); + + // Create taxonomy resource + const parent = datacatalog.locationPath(projectId, 'us'); + const taxRequest = { + parent, + taxonomy: { + displayName: generateUuid(), + activatedPolicyTypes: ['FINE_GRAINED_ACCESS_CONTROL'], + }, + }; + const [taxonomy] = await policyTagManager.createTaxonomy(taxRequest); + taxonomyName = taxonomy.name; + }); + + describe('policyTagManager', async () => { + it('should create a taxonomy', async () => { + const taxLocation = 'us'; + const displayName = generateUuid(); + const output = execSync( + `node policyTagManager/createTaxonomy ${projectId} ${taxLocation} ${displayName}` + ); + assert.include(output, 'Created taxonomy:'); + }); + + it('should get a taxonomy', async () => { + const output = execSync( + `node policyTagManager/getTaxonomy ${taxonomyName}` + ); + assert.include(output, `Retrieved taxonomy: ${taxonomyName}`); + }); + + it('should list taxonomies', async () => { + const taxLocation = 'us'; + const output = execSync( + `node policyTagManager/listTaxonomies ${projectId} ${taxLocation}` + ); + assert.include(output, 'Taxonomies:'); + }); + + it('should delete a taxonomy', async () => { + const output = execSync( + `node policyTagManager/deleteTaxonomy ${taxonomyName}` + ); + assert.include(output, 'Deleted taxonomy:'); + }); + + it('should create a policy tag', async () => { + const tagLocation = 'us'; + const displayName = generateUuid(); + const parent = datacatalog.locationPath(projectId, tagLocation); + + const request = { + parent: parent, + taxonomy: { + displayName: displayName, + activatedPolicyTypes: ['FINE_GRAINED_ACCESS_CONTROL'], + }, + }; + + const [taxonomy] = await policyTagManager.createTaxonomy(request); + + const output = execSync( + `node policyTagManager/createPolicyTag ${taxonomy.name}` + ); + assert.include(output, 'Created policy tag:'); + assert.include(output, taxonomy.name); + }); + }); + + it('should create a custom entry', async () => { + const entryGroupId = generateUuid(); + const entryId = generateUuid(); + const output = execSync( + `node createCustomEntry ${projectId} ${entryGroupId} ${entryId} ${TAG_TEMPLATE_ID}` + ); + assert.include(output, 'Created entry group:'); + assert.include(output, 'Created entry:'); + assert.include(output, 'Created template:'); + assert.include(output, 'Created tag:'); + }); + + it('should create a fileset', async () => { + const entryGroupId = generateUuid(); + const entryId = generateUuid(); + const output = execSync( + `node createFileset ${projectId} ${entryGroupId} ${entryId}` + ); + assert.include(output, `/entryGroups/${entryGroupId}/entries/${entryId}`); + assert.include(output, 'Display name:'); + assert.include(output, 'Type: FILESET'); + }); + + it('should grant tagTemplateUser role', async () => { + const resourceName = `${datacatalog.locationPath( + projectId, + location + )}/tagTemplates/${TAG_TEMPLATE_ID}`; + const [policy] = await datacatalog.getIamPolicy({resource: resourceName}); + const memberId = policy.bindings[0].members[0]; + + const output = execSync( + `node grantTagTemplateUserRole ${projectId} ${TAG_TEMPLATE_ID} ${memberId}` + ); + assert.include(output, 'roles/datacatalog.tagTemplateUser'); + assert.include(output, memberId); + }); + + it.skip('should search data assets in project', async () => { + const output = execSync(`node searchAssets ${projectId}`); + assert.match(output, /Found [0-9]+ datasets in project/); + }); + + // Only delete a resource if it is older than 24 hours. That will prevent + // collisions with parallel CI test runs. + function isResourceStale(creationTime) { + const oneDayMs = 86400000; + const now = new Date(); + const created = new Date(creationTime * 1000); + return now.getTime() - created.getTime() >= oneDayMs; + } + + async function deleteTaxonomies() { + const projectId = await policyTagManager.getProjectId(); + const location = 'us'; + + const listTaxonomiesRequest = { + parent: datacatalog.locationPath(projectId, location), + }; + + let [taxonomies] = await policyTagManager.listTaxonomies( + listTaxonomiesRequest + ); + + taxonomies = taxonomies.filter(taxonomy => { + return taxonomy.displayName.includes(GCLOUD_TESTS_PREFIX); + }); + + taxonomies.forEach(async taxonomy => { + if (isResourceStale(taxonomy.taxonomyTimestamps.createTime.seconds)) { + try { + await policyTagManager.deleteTaxonomy({name: taxonomy.name}); + } catch (e) { + console.error(e); + } + } + }); + } + + async function deleteEntries(entryGroupId) { + const [entries] = await datacatalog.listEntries({parent: entryGroupId}); + for (const entry of entries) { + try { + await datacatalog.deleteEntry({name: entry.name}); + } catch (e) { + console.log(`entry (${entry.name}).delete() failed`); + console.log(e); + } + } + } + + async function deleteEntryGroups() { + const listEntryGroupsRequest = { + parent: datacatalog.locationPath(projectId, location), + }; + + const [entryGroups] = await datacatalog.listEntryGroups( + listEntryGroupsRequest + ); + + const groups = entryGroups.filter(group => { + return group.name.includes(GCLOUD_TESTS_PREFIX); + }); + + for (const group of groups) { + const creationTime = Number( + group.dataCatalogTimestamps.createTime.seconds + ); + if (isResourceStale(creationTime)) { + try { + await deleteEntries(group.name); + await datacatalog.deleteEntryGroup({name: group.name}); + } catch (e) { + console.log(`entry group (${group.name}).delete() failed`); + console.log(e); + } + } + } + } +});