Skip to content

Commit

Permalink
add data catalog tag (#3569)
Browse files Browse the repository at this point in the history
* add data catalog tag

* add data catalog tag

* fix missing err check

* update after review comments

* fixing a bad diff

* update per code review comments

* update fields values to camel case in ruby
  • Loading branch information
megan07 authored Jun 8, 2020
1 parent 0838db4 commit 2850f2a
Show file tree
Hide file tree
Showing 11 changed files with 624 additions and 3 deletions.
106 changes: 105 additions & 1 deletion products/datacatalog/api.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -272,7 +272,6 @@ objects:
description: |
Total number of shards.
output: true

- !ruby/object:Api::Resource
name: TagTemplate
base_url: projects/{{project}}/locations/{{region}}/tagTemplates
Expand Down Expand Up @@ -382,3 +381,108 @@ objects:
The order of this field with respect to other fields in this tag template.
A higher value indicates a more important field. The value can be negative.
Multiple fields can have the same order, and field orders within a tag do not have to be sequential.
- !ruby/object:Api::Resource
name: Tag
base_url: '{{parent}}/tags'
self_link: '{{name}}'
update_url: '{{name}}'
update_verb: :PATCH
update_mask: true
self_link: '{{parent}}/tags'
delete_url: '{{name}}'
nested_query: !ruby/object:Api::Resource::NestedQuery
keys:
- tags
description: |
Tags are used to attach custom metadata to Data Catalog resources. Tags conform to the specifications within their tag template.
See [Data Catalog IAM](https://cloud.google.com/data-catalog/docs/concepts/iam) for information on the permissions needed to create or view tags.
references: !ruby/object:Api::Resource::ReferenceLinks
guides:
'Official Documentation': https://cloud.google.com/data-catalog/docs
api: https://cloud.google.com/data-catalog/docs/reference/rest/v1/projects.locations.entryGroups.tags
parameters:
- !ruby/object:Api::Type::String
name: parent
url_param_only: true
description: |
The name of the parent this tag is attached to. This can be the name of an entry or an entry group. If an entry group, the tag will be attached to
all entries in that group.
properties:
- !ruby/object:Api::Type::String
name: name
description: |
The resource name of the tag in URL format. Example:
projects/{project_id}/locations/{location}/entrygroups/{entryGroupId}/entries/{entryId}/tags/{tag_id} or
projects/{project_id}/locations/{location}/entrygroups/{entryGroupId}/tags/{tag_id}
where tag_id is a system-generated identifier. Note that this Tag may not actually be stored in the location in this name.
output: true
- !ruby/object:Api::Type::String
name: template
description: |
The resource name of the tag template that this tag uses. Example:
projects/{project_id}/locations/{location}/tagTemplates/{tagTemplateId}
This field cannot be modified after creation.
required: true
input: true
- !ruby/object:Api::Type::String
name: templateDisplayName
description: |
The display name of the tag template.
output: true
- !ruby/object:Api::Type::Map
name: fields
description: |
This maps the ID of a tag field to the value of and additional information about that field.
Valid field IDs are defined by the tag's template. A tag must have at least 1 field and at most 500 fields.
required: true
key_name: field_name
value_type: !ruby/object:Api::Type::NestedObject
name: field_value
properties:
- !ruby/object:Api::Type::String
name: display_name
description: |
The display name of this field
output: true
- !ruby/object:Api::Type::Integer
name: order
description: |
The order of this field with respect to other fields in this tag. For example, a higher value can indicate
a more important field. The value can be negative. Multiple fields can have the same order, and field orders
within a tag do not have to be sequential.
output: true
- !ruby/object:Api::Type::Double
name: doubleValue
description: |
Holds the value for a tag field with double type.
- !ruby/object:Api::Type::String
name: stringValue
description: |
Holds the value for a tag field with string type.
- !ruby/object:Api::Type::Boolean
name: boolValue
description: |
Holds the value for a tag field with boolean type.
- !ruby/object:Api::Type::String
name: timestampValue
description: |
Holds the value for a tag field with timestamp type.
- !ruby/object:Api::Type::NestedObject
name: enumValue
description: |
Holds the value for a tag field with enum type. This value must be one of the allowed values in the definition of this enum.
properties:
- !ruby/object:Api::Type::String
name: displayName
description: |
The display name of the enum value.
required: true
- !ruby/object:Api::Type::String
name: column
description: |
Resources like Entry can have schemas associated with them. This scope allows users to attach tags to an
individual column based on that schema.
For attaching a tag to a nested column, use `.` to separate the column names. Example:
`outer_column.inner_column`
59 changes: 59 additions & 0 deletions products/datacatalog/terraform.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,65 @@ overrides: !ruby/object:Overrides::ResourceOverrides
default_from_api: true
custom_code: !ruby/object:Provider::Terraform::CustomCode
custom_import: templates/terraform/custom_import/data_catalog_tag_template.go.erb
Tag: !ruby/object:Overrides::Terraform::ResourceOverride
import_format: ["{{name}}"]
id_format: "{{name}}"
examples:
- !ruby/object:Provider::Terraform::Examples
name: "data_catalog_entry_tag_basic"
primary_resource_id: "basic_tag"
vars:
entry_group_id: "my_entry_group"
entry_id: "my_entry"
tag_template_id: "my_template"
force_delete: "false"
test_vars_overrides:
force_delete: "true"
oics_vars_overrides:
force_delete: "true"
- !ruby/object:Provider::Terraform::Examples
name: "data_catalog_entry_group_tag"
primary_resource_id: "entry_group_tag"
vars:
entry_group_id: "my_entry_group"
first_entry: "first_entry"
second_entry: "second_entry"
tag_template_id: "my_template"
force_delete: "false"
test_vars_overrides:
force_delete: "true"
oics_vars_overrides:
force_delete: "true"
- !ruby/object:Provider::Terraform::Examples
name: "data_catalog_entry_tag_full"
primary_resource_id: "basic_tag"
vars:
entry_group_id: "my_entry_group"
entry_id: "my_entry"
tag_template_id: "my_template"
force_delete: "false"
test_vars_overrides:
force_delete: "true"
oics_vars_overrides:
force_delete: "true"
properties:
# Changing the name here so when mm generates methods like `flattenDataCatalogTagTemplateDisplayName`
# this doesn't conflict with tag template's display name methods
templateDisplayName: !ruby/object:Overrides::Terraform::PropertyOverride
name: template_displayname
fields.enumValue: !ruby/object:Overrides::Terraform::PropertyOverride
flatten_object: true
# because `fields` is a set, the current generated expand code can't properly retrieve
# enum_value by d.Get("enum_value") without knowing the set id, however, the value
# `v` is the correct value and passed to expand, so this custom expand will use
# that as the correct `enum_value.display_name` value
custom_expand: templates/terraform/custom_expand/data_catalog_tag.go.erb
custom_flatten: templates/terraform/custom_flatten/data_catalog_tag.go.erb
fields.enumValue.displayName: !ruby/object:Overrides::Terraform::PropertyOverride
name: enum_value
required: false
custom_code: !ruby/object:Provider::Terraform::CustomCode
custom_import: templates/terraform/custom_import/data_catalog_tag.go.erb
# This is for copying files over
files: !ruby/object:Provider::Config::Files
# These files have templating (ERB) code that will be run.
Expand Down
24 changes: 24 additions & 0 deletions templates/terraform/custom_expand/data_catalog_tag.go.erb
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
<%# # the license inside this if block pertains to this file
# Copyright 2020 Google Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#%>
func expand<%= prefix -%><%= titlelize_property(property) -%>(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
// we flattened the original["enum_value"]["display_name"] object to be just original["enum_value"] so here,
// v is the value we want from the config
transformed := make(map[string]interface{})
if val := reflect.ValueOf(v); val.IsValid() && !isEmptyValue(val) {
transformed["displayName"] = v
}

return transformed, nil
}
21 changes: 21 additions & 0 deletions templates/terraform/custom_flatten/data_catalog_tag.go.erb
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
<%# The license inside this block applies to this file.
# Copyright 2020 Google Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-%>
func flatten<%= prefix -%><%= titlelize_property(property) -%>(v interface{}, d *schema.ResourceData, config *Config) interface{} {
if v == nil {
return nil
}

return v.(map[string]interface{})["displayName"]
}
17 changes: 17 additions & 0 deletions templates/terraform/custom_import/data_catalog_tag.go.erb
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
config := meta.(*Config)

// current import_formats can't import fields with forward slashes in their value
if err := parseImportId([]string{"(?P<name>.+)"}, d, config); err != nil {
return nil, err
}

name := d.Get("name").(string)
egRegex := regexp.MustCompile("(.+)/tags")

parts := egRegex.FindStringSubmatch(name)
if len(parts) != 2 {
return nil, fmt.Errorf("entry name does not fit the format %s", egRegex)
}

d.Set("parent", parts[1])
return []*schema.ResourceData{d}, nil
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

parts := egRegex.FindStringSubmatch(name)
if len(parts) != 4 {
return nil, fmt.Errorf("entry group name does not fit the format %s", egRegex)
return nil, fmt.Errorf("tag template name does not fit the format %s", egRegex)
}
d.Set("project", parts[1])
d.Set("region", parts[2])
Expand Down
72 changes: 72 additions & 0 deletions templates/terraform/examples/data_catalog_entry_group_tag.tf.erb
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
resource "google_data_catalog_entry" "first_entry" {
entry_group = google_data_catalog_entry_group.entry_group.id
entry_id = "<%= ctx[:vars]['first_entry'] %>"

user_specified_type = "my_custom_type"
user_specified_system = "SomethingExternal"
}

resource "google_data_catalog_entry" "second_entry" {
entry_group = google_data_catalog_entry_group.entry_group.id
entry_id = "<%= ctx[:vars]['second_entry'] %>"

user_specified_type = "another_custom_type"
user_specified_system = "SomethingElseExternal"
}

resource "google_data_catalog_entry_group" "entry_group" {
entry_group_id = "<%= ctx[:vars]['entry_group_id'] %>"
}

resource "google_data_catalog_tag_template" "tag_template" {
tag_template_id = "<%= ctx[:vars]['tag_template_id'] %>"
region = "us-central1"
display_name = "Demo Tag Template"

fields {
field_id = "source"
display_name = "Source of data asset"
type {
primitive_type = "STRING"
}
is_required = true
}

fields {
field_id = "num_rows"
display_name = "Number of rows in the data asset"
type {
primitive_type = "DOUBLE"
}
}

fields {
field_id = "pii_type"
display_name = "PII type"
type {
enum_type {
allowed_values {
display_name = "EMAIL"
}
allowed_values {
display_name = "SOCIAL SECURITY NUMBER"
}
allowed_values {
display_name = "NONE"
}
}
}
}

force_delete = "<%= ctx[:vars]['force_delete'] %>"
}

resource "google_data_catalog_tag" "<%= ctx[:primary_resource_id] %>" {
parent = google_data_catalog_entry_group.entry_group.id
template = google_data_catalog_tag_template.tag_template.id

fields {
field_name = "source"
string_value = "my-string"
}
}
64 changes: 64 additions & 0 deletions templates/terraform/examples/data_catalog_entry_tag_basic.tf.erb
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
resource "google_data_catalog_entry" "entry" {
entry_group = google_data_catalog_entry_group.entry_group.id
entry_id = "<%= ctx[:vars]['entry_id'] %>"

user_specified_type = "my_custom_type"
user_specified_system = "SomethingExternal"
}

resource "google_data_catalog_entry_group" "entry_group" {
entry_group_id = "<%= ctx[:vars]['entry_group_id'] %>"
}

resource "google_data_catalog_tag_template" "tag_template" {
tag_template_id = "<%= ctx[:vars]['tag_template_id'] %>"
region = "us-central1"
display_name = "Demo Tag Template"

fields {
field_id = "source"
display_name = "Source of data asset"
type {
primitive_type = "STRING"
}
is_required = true
}

fields {
field_id = "num_rows"
display_name = "Number of rows in the data asset"
type {
primitive_type = "DOUBLE"
}
}

fields {
field_id = "pii_type"
display_name = "PII type"
type {
enum_type {
allowed_values {
display_name = "EMAIL"
}
allowed_values {
display_name = "SOCIAL SECURITY NUMBER"
}
allowed_values {
display_name = "NONE"
}
}
}
}

force_delete = "<%= ctx[:vars]['force_delete'] %>"
}

resource "google_data_catalog_tag" "<%= ctx[:primary_resource_id] %>" {
parent = google_data_catalog_entry.entry.id
template = google_data_catalog_tag_template.tag_template.id

fields {
field_name = "source"
string_value = "my-string"
}
}
Loading

0 comments on commit 2850f2a

Please sign in to comment.