From ccc8cd3e693b87fe87431beee51409bb707f9472 Mon Sep 17 00:00:00 2001 From: Cameron Thornton Date: Fri, 3 May 2024 14:25:12 -0500 Subject: [PATCH] Go rewrite - add expanders and flatteners to resource.go (#10589) --- mmv1/api/type.go | 26 +++ mmv1/products/datafusion/go_instance.yaml | 4 +- mmv1/provider/template_data.go | 20 ++- mmv1/provider/terraform.go | 14 -- .../terraform/expand_property_method.go.tmpl | 166 ++++++++++++++++++ .../terraform/flatten_property_method.go.tmpl | 154 ++++++++++++++++ mmv1/templates/terraform/resource.go.tmpl | 22 ++- .../terraform/schema_property.go.tmpl | 2 +- .../terraform/schema_subresource.go.tmpl | 2 +- 9 files changed, 379 insertions(+), 31 deletions(-) create mode 100644 mmv1/templates/terraform/expand_property_method.go.tmpl create mode 100644 mmv1/templates/terraform/flatten_property_method.go.tmpl diff --git a/mmv1/api/type.go b/mmv1/api/type.go index dc8522bcf546..9e2a8072dc18 100644 --- a/mmv1/api/type.go +++ b/mmv1/api/type.go @@ -1217,3 +1217,29 @@ func (t Type) PropertyNsPrefix() []string { "Property", } } + +// "Namespace" - prefix with product and resource - a property with +// information from the "object" variable + +func (t Type) NamespaceProperty() string { + name := google.Camelize(t.Name, "lower") + p := t + for p.Parent() != nil { + p = *p.Parent() + name = fmt.Sprintf("%s%s", google.Camelize(p.Name, "lower"), name) + } + + return fmt.Sprintf("%s%s%s", google.Camelize(t.ApiName, "lower"), t.ResourceMetadata.Name, name) +} + +// def namespace_property_from_object(property, object) +// +// name = property.name.camelize +// until property.parent.nil? +// property = property.parent +// name = property.name.camelize + name +// end +// +// "#{property.__resource.__product.api_name.camelize(:lower)}#{object.name}#{name}" +// +// end diff --git a/mmv1/products/datafusion/go_instance.yaml b/mmv1/products/datafusion/go_instance.yaml index 83c08ca70854..822843ecd264 100644 --- a/mmv1/products/datafusion/go_instance.yaml +++ b/mmv1/products/datafusion/go_instance.yaml @@ -107,8 +107,8 @@ properties: description: "The ID of the instance or a fully qualified identifier for the instance." required: true immutable: true - custom_flatten: 'templates/terraform/custom_flatten/name_from_self_link.erb' - custom_expand: 'templates/terraform/custom_expand/shortname_to_url.go.erb' + custom_flatten: 'templates/terraform/custom_flatten/go/name_from_self_link.tmpl' + custom_expand: 'templates/terraform/custom_expand/go/shortname_to_url.go.tmpl' - name: 'description' type: String description: "An optional description of the instance." diff --git a/mmv1/provider/template_data.go b/mmv1/provider/template_data.go index 4d038f93b978..ae356f34f55c 100644 --- a/mmv1/provider/template_data.go +++ b/mmv1/provider/template_data.go @@ -16,6 +16,8 @@ package provider import ( "bytes" "errors" + "fmt" + "go/format" "log" "os" "path/filepath" @@ -99,11 +101,13 @@ func NewTemplateData(outputFolder string, version product.Version) *TemplateData func (td *TemplateData) GenerateResourceFile(filePath string, resource api.Resource) { templatePath := "templates/terraform/resource.go.tmpl" templates := []string{ + templatePath, "templates/terraform/schema_property.go.tmpl", "templates/terraform/schema_subresource.go.tmpl", - templatePath, "templates/terraform/expand_resource_ref.tmpl", "templates/terraform/custom_flatten/go/bigquery_table_ref.go.tmpl", + "templates/terraform/flatten_property_method.go.tmpl", + "templates/terraform/expand_property_method.go.tmpl", } td.GenerateFile(filePath, templatePath, resource, true, templates...) } @@ -111,9 +115,9 @@ func (td *TemplateData) GenerateResourceFile(filePath string, resource api.Resou func (td *TemplateData) GenerateDocumentationFile(filePath string, resource api.Resource) { templatePath := "templates/terraform/resource.html.markdown.tmpl" templates := []string{ + templatePath, "templates/terraform/property_documentation.html.markdown.tmpl", "templates/terraform/nested_property_documentation.html.markdown.tmpl", - templatePath, } td.GenerateFile(filePath, templatePath, resource, false, templates...) } @@ -163,12 +167,12 @@ func (td *TemplateData) GenerateFile(filePath, templatePath string, input any, g sourceByte := contents.Bytes() - // if goFormat { - // sourceByte, err = format.Source(sourceByte) - // if err != nil { - // glog.Error(fmt.Errorf("error formatting %s", filePath)) - // } - // } + if goFormat { + sourceByte, err = format.Source(sourceByte) + if err != nil { + glog.Error(fmt.Errorf("error formatting %s", filePath)) + } + } err = os.WriteFile(filePath, sourceByte, 0644) if err != nil { diff --git a/mmv1/provider/terraform.go b/mmv1/provider/terraform.go index ac36423cd9f2..653bdec2478b 100644 --- a/mmv1/provider/terraform.go +++ b/mmv1/provider/terraform.go @@ -743,20 +743,6 @@ func (t Terraform) replaceImportPath(outputFolder, target string) { // // end // -// # "Namespace" - prefix with product and resource - a property with -// # information from the "object" variable -// def namespace_property_from_object(property, object) -// -// name = property.name.camelize -// until property.parent.nil? -// property = property.parent -// name = property.name.camelize + name -// end -// -// "#{property.__resource.__product.api_name.camelize(:lower)}#{object.name}#{name}" -// -// end -// // # Converts between the Magic Modules type of an object and its type in the // # TF schema // def tf_types diff --git a/mmv1/templates/terraform/expand_property_method.go.tmpl b/mmv1/templates/terraform/expand_property_method.go.tmpl new file mode 100644 index 000000000000..7a05a9d109fb --- /dev/null +++ b/mmv1/templates/terraform/expand_property_method.go.tmpl @@ -0,0 +1,166 @@ +{{/* The license inside this block applies to this file + Copyright 2024 Google LLC. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ -}} +{{- define "expandPropertyMethod" }} + {{- if $.CustomExpand }} + {{/* TODO custom flatten */}} + {{- else }}{{/* if $.CustomExpand */}} + {{- if $.IsA "Map" }} +func expand{{$.GetPrefix}}{{$.TitlelizeProperty}}(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (map[string]interface{}, error) { + if v == nil { + return map[string]interface{}{}, nil + } + m := make(map[string]interface{}) + for _, raw := range v.(*schema.Set).List() { + original := raw.(map[string]interface{}) + transformed := make(map[string]interface{}) + + {{- range $prop := $.NestedProperties }} + {{- if not (eq $prop.Name $prop.KeyName) }} + transformed{{$prop.TitlelizeProperty}}, err := expand{{$.GetPrefix}}{{$.TitlelizeProperty}}{{$prop.TitlelizeProperty}}(original["{{ underscore $prop.Name }}"], d, config) + if err != nil { + return nil, err + {{- if $prop.SendEmptyValue }} + } else { + transformed["{{$prop.ApiName}}"] = transformed{{$prop.TitlelizeProperty}} + {{- else }} + } else if val := reflect.ValueOf(transformed{{$prop.TitlelizeProperty}}); val.IsValid() && !tpgresource.IsEmptyValue(val) { + transformed["{{$prop.ApiName}}"] = transformed{{$prop.TitlelizeProperty}} + {{- end }} + } + {{- end }} + {{ end }} + + transformed{{ camelize $.KeyName "upper" }}, err := {{ $.KeyExpander }}(original["{{ underscore $.KeyName }}"], d, config) + if err != nil { + return nil, err + } + m[transformed{{ camelize $.KeyName "upper" }}] = transformed + } + return m, nil +} + {{ else if $.IsA "KeyValuePairs" }}{{/* if $.IsA "Map" */}} +func expand{{$.GetPrefix}}{{$.TitlelizeProperty}}(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (map[string]string, error) { + if v == nil { + return map[string]string{}, nil + } + m := make(map[string]string) + for k, val := range v.(map[string]interface{}) { + m[k] = val.(string) + } + return m, nil +} + {{ else if $.FlattenObject }}{{/* if $.IsA "Map" */}} +func expand{{$.GetPrefix}}{{$.TitlelizeProperty}}(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) { + transformed := make(map[string]interface{}) + {{- range $prop := $.NestedProperties }} + {{- if not (and ($prop.IsA "KeyValuePairs") $prop.IgnoreWrite) }} + transformed{{$prop.TitlelizeProperty}}, err := expand{{$.GetPrefix}}{{$.TitlelizeProperty}}{{$prop.TitlelizeProperty}}({{ if $prop.FlattenObject }}nil{{ else }}d.Get("{{ underscore $prop.Name }}"), d, config) + if err != nil { + return nil, err + {{- if $prop.SendEmptyValue }} + } else { + transformed["{{$prop.ApiName}}"] = transformed{{$prop.TitlelizeProperty}} + {{- else }} + } else if val := reflect.ValueOf(transformed{{$prop.TitlelizeProperty}}); val.IsValid() && !tpgresource.IsEmptyValue(val) { + transformed["{{$prop.ApiName}}"] = transformed{{$prop.TitlelizeProperty}} + {{- end }} + } + {{- end }} + {{- end }} + {{ end }} + return transformed, nil +} + {{ else }}{{/* if $.IsA "Map" */}} +func expand{{$.GetPrefix}}{{$.TitlelizeProperty}}(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) { + {{- if $.IsSet }} + v = v.(*schema.Set).List() + {{- end }} + {{- if $.NestedProperties }} + l := v.([]interface{}) + {{- if $.IsA "Array" }} + req := make([]interface{}, 0, len(l)) + for _, raw := range l { + if raw == nil { + continue + } + original := raw.(map[string]interface{}) + {{- else }}{{/* if $.IsA "Array */}} + {{- if $.AllowEmptyObject }} + if len(l) == 0 { + return nil, nil + } + + if l[0] == nil { + transformed := make(map[string]interface{}) + return transformed, nil + } + {{- else }} + if len(l) == 0 || l[0] == nil { + return nil, nil + } + {{- end }} + raw := l[0] + original := raw.(map[string]interface{}) + {{- end }}{{/* if $.IsA "Array */}} + transformed := make(map[string]interface{}) + {{ range $prop := $.NestedProperties }} + {{- if not (and ($prop.IsA "KeyValuePairs") $prop.IgnoreWrite) }} + transformed{{$prop.TitlelizeProperty}}, err := expand{{$.GetPrefix}}{{$.TitlelizeProperty}}{{$prop.TitlelizeProperty}}(original["{{ underscore $prop.Name }}"], d, config) + if err != nil { + return nil, err + {{- if $prop.SendEmptyValue }} + } else { + transformed["{{$prop.ApiName}}] = transformed{{$prop.TitlelizeProperty}} + {{- else }} + } else if val := reflect.ValueOf(transformed{{$prop.TitlelizeProperty}}); val.IsValid() && !tpgresource.IsEmptyValue(val) { + transformed["{{$prop.ApiName}}"] = transformed{{$prop.TitlelizeProperty}} + {{- end }} + } + {{ end }} + {{- end }} + {{- if $.IsA "Array" }} + req = append(req, transformed) + } + return req, nil + {{- else }} + return transformed, nil + {{- end }} +} + + {{ else if and ($.IsA "Array") ($.ItemType.IsA "ResourceRef")}}{{/* if $.NestedProperties */}} + l := v.([]interface{}) + req := make([]interface{}, 0, len(l)) + for _, raw := range l { + if raw == nil { + return nil, fmt.Errorf("Invalid value for {{ underscore $.Name }}: nil") + } + req = append(req, raw.(string)) + } + return req, nil +} + {{- else }} + return v, nil +} + {{- end }}{{/* if $.NestedProperties */}} + {{- end }}{{/* if $.IsA "Map" */}} + {{ if $.NestedProperties }} + {{- range $prop := $.NestedProperties }} + {{- if not (and ($prop.IsA "KeyValuePairs") $prop.IgnoreWrite) }} + {{- template "expandPropertyMethod" $prop -}} + {{- end }} + {{- end }} + {{- end }}{{/* if $.NestedProperties */}} + {{- end }}{{/* if $.CustomExpand */}} +{{- end }}{{/* define */}} diff --git a/mmv1/templates/terraform/flatten_property_method.go.tmpl b/mmv1/templates/terraform/flatten_property_method.go.tmpl new file mode 100644 index 000000000000..661d79133c32 --- /dev/null +++ b/mmv1/templates/terraform/flatten_property_method.go.tmpl @@ -0,0 +1,154 @@ +{{/* The license inside this block applies to this file + Copyright 2024 Google LLC. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ -}} +{{- define "flattenPropertyMethod" }} +{{- if $.CustomFlatten }} +{{/* TODO custom flatten */}} +{{- else }} +func flatten{{$.GetPrefix}}{{$.TitlelizeProperty}}(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} { + {{- if $.IsA "NestedObject" }} + if v == nil { + return nil + } + {{- if not $.AllowEmptyObject }} + original := v.(map[string]interface{}) + if len(original) == 0 { + return nil + } + {{- else if $.Properties }} + original := v.(map[string]interface{}) + {{- end }} + transformed := make(map[string]interface{}) + {{- range $prop := $.Properties }} + {{- if $prop.FlattenObject }} + if {{ $prop.ApiName }} := flatten{{$.GetPrefix}}{{$.TitlelizeProperty}}{{$prop.TitlelizeProperty}}(original["{{ $prop.ApiName }}"], d, config); {{ $prop.ApiName }} != nil { + obj := {{ $prop.ApiName }}.([]interface{})[0] + for k, v := range obj.(map[string]interface{}) { + transformed[k] = v + } + } + {{- else }} + transformed["{{ underscore $prop.Name }}"] = + flatten{{$.GetPrefix}}{{$.TitlelizeProperty}}{{$prop.TitlelizeProperty}}(original["{{ $prop.ApiName }}"], d, config) + {{- end }} + {{- end }} + return []interface{}{transformed} + {{- else if and ($.IsA "Array") ($.ItemType.IsA "NestedObject") }} + if v == nil { + return v + } + l := v.([]interface{}) + {{- if $.IsSet }} + {{- if $.SetHashFunc }} + transformed := schema.NewSet({{ $.SetHashFunc }}, []interface{}{}) + {{- else }} + transformed := schema.NewSet(schema.HashResource({{ $.NamespaceProperty }}Schema()), []interface{}{}) + {{- end }} + {{- else }} + transformed := make([]interface{}, 0, len(l)) + {{- end }} + for _, raw := range l { + original := raw.(map[string]interface{}) + if len(original) < 1 { + // Do not include empty json objects coming back from the api + continue + } + {{- if $.IsSet }} + transformed.Add(map[string]interface{}{ + {{- else }} + transformed = append(transformed, map[string]interface{}{ + {{- end }} + + {{- range $prop := $.ItemType.Properties }} + {{- if not $prop.IgnoreRead }} + "{{ underscore $prop.Name }}": flatten{{$.GetPrefix}}{{$.TitlelizeProperty}}{{$prop.TitlelizeProperty}}(original["{{ $prop.ApiName }}"], d, config), + {{- end }} + {{- end }} + }) + } + return transformed + {{- else if $.IsA "Map" }} + if v == nil { + return v + } + l := v.(map[string]interface{}) + transformed := make([]interface{}, 0, len(l)) + for k, raw := range l { + original := raw.(map[string]interface{}) + transformed = append(transformed, map[string]interface{}{ + "{{ $.KeyName }}": k, + {{- range $prop := $.ValueType.Properties }} + "{{ underscore $prop.Name }}": flatten{{$.GetPrefix}}{{$.TitlelizeProperty}}{{$prop.TitlelizeProperty}}(original["{{ $prop.ApiName }}"], d, config), + {{- end }} + }) + } + return transformed + {{- else if or ($.IsA "KeyValueLabels") (or ($.IsA "KeyValueAnnotations") ($.IsA "KeyValueTerraformLabels")) }} + if v == nil { + return v + } + + transformed := make(map[string]interface{}) + if l, ok := d.GetOkExists("{{ $.TerraformLineage }}"); ok { + for k := range l.(map[string]interface{}) { + transformed[k] = v.(map[string]interface{})[k] + } + } + + return transformed + {{- else if $.IsA "Integer" }} + // Handles the string fixed64 format + if strVal, ok := v.(string); ok { + if intVal, err := tpgresource.StringToFixed64(strVal); err == nil { + return intVal + } + } + + // number values are represented as float64 + if floatVal, ok := v.(float64); ok { + intVal := int(floatVal) + return intVal + } + + return v // let terraform core handle it otherwise + {{- else if and ($.IsA "Integer") ($.ItemType.IsA "ResourceRef")}} + if v == nil { + return v + } + return tpgresource.ConvertAndMapStringArr(v.([]interface{}), tpgresource.ConvertSelfLinkToV1) + {{- else if $.IsA "ResourceRef" }} + if v == nil { + return v + } + return tpgresource.ConvertSelfLinkToV1(v.(string)) + {{- else if $.IsSet }} + if v == nil { + return v + } + {{- if $.SetHashFunc }} + return schema.NewSet({{- $.SetHashFunc }}, v.([]interface{})) + {{- else if or ($.ItemType.IsA "String") ($.ItemType.IsA "Enum") }} + return schema.NewSet(schema.HashString, v.([]interface{})) + {{- end }} + {{- else }} + return v + {{- end }} +} + {{- if $.NestedProperties }} + {{- range $prop := $.NestedProperties }} + {{- template "flattenPropertyMethod" $prop -}} + {{- end }} + {{- end }} +{{- end }} +{{ end }} \ No newline at end of file diff --git a/mmv1/templates/terraform/resource.go.tmpl b/mmv1/templates/terraform/resource.go.tmpl index 7389197edec3..5019e14d33b0 100644 --- a/mmv1/templates/terraform/resource.go.tmpl +++ b/mmv1/templates/terraform/resource.go.tmpl @@ -1,4 +1,4 @@ -{{/* the license inside this block applies to this file +{{/* The license inside this block applies to this file Copyright 2024 Google LLC. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); @@ -414,7 +414,7 @@ func resource{{ $.ResourceName -}}Create(d *schema.ResourceData, meta interface{ {{ end -}} } -{{if ($.GetAsync.IsA "OpAsync")}} +{{if ($.GetAsync.IsA "PollAsync")}} func resource{{ $.ResourceName -}}PollRead(d *schema.ResourceData, meta interface{}) transport_tpg.PollReadFunc { return func() (map[string]interface{}, error) { {{if $.GetAsync.CustomPollRead -}} @@ -513,7 +513,7 @@ func resource{{ $.ResourceName -}}Read(d *schema.ResourceData, meta interface{}) return err } - url, err := tpgresource.ReplaceVars{{if $.LegacyLongFormProject -}}ForId{{ end -}}(d, config, "{{"{{"}}{{$.ProductMetadata.Name}}BasePath{{"}}"}}/{{$.SelfLinkUri}}{{$.ReadQueryParams}}") + url, err := tpgresource.ReplaceVars{{if $.LegacyLongFormProject -}}ForId{{ end -}}(d, config, "{{"{{"}}{{$.ProductMetadata.Name}}BasePath{{"}}"}}{{$.SelfLinkUri}}{{$.ReadQueryParams}}") if err != nil { return err } @@ -1170,5 +1170,17 @@ func resource{{ $.ResourceName }}Import(d *schema.ResourceData, meta interface{} {{- end }} } {{- end }} -{{/* TODO Flatteners */}} -{{/* TODO Expanders */}} \ No newline at end of file +{{- range $prop := $.GettableProperties }} + {{- if not $prop.IgnoreRead }} + {{- template "flattenPropertyMethod" $prop -}} + {{- end }} +{{- end }} +{{- range $prop := $.SettableProperties }} + {{- template "expandPropertyMethod" $prop -}} +{{- end }} +{{/* TODO custom encoder */}} +{{/* TODO custom update encoder */}} +{{/* TODO nested query */}} +{{/* TODO custom decoder */}} +{{/* TODO custom post create failure */}} +{{/* TODO state upgraders */}} \ No newline at end of file diff --git a/mmv1/templates/terraform/schema_property.go.tmpl b/mmv1/templates/terraform/schema_property.go.tmpl index f6b6d6871697..5313dda284a1 100644 --- a/mmv1/templates/terraform/schema_property.go.tmpl +++ b/mmv1/templates/terraform/schema_property.go.tmpl @@ -94,7 +94,7 @@ Possible values: ["{{ join .EnumValues "\",\"" }}"] {{ end -}} {{ if eq .ItemType.Type "NestedObject" -}} {{ if .IsSet -}} - Elem: //TODO NameSpacePropertyFromObject Schema(), + Elem: {{ .NamespaceProperty }}Schema(), {{ else -}} Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ diff --git a/mmv1/templates/terraform/schema_subresource.go.tmpl b/mmv1/templates/terraform/schema_subresource.go.tmpl index 24a8e942e829..ac96751eaa37 100644 --- a/mmv1/templates/terraform/schema_subresource.go.tmpl +++ b/mmv1/templates/terraform/schema_subresource.go.tmpl @@ -14,7 +14,7 @@ */}} {{define "SchemaSubResource"}} {{ if and (.IsSet) (eq .Type "Array") (eq .ItemType.Type "NestedObject") -}} -func ##TODO NameSpacePropertyFromObject Schema() *schema.Resource { +func{{ .NamespaceProperty }}Schema() *schema.Resource { return &schema.Resource{ Schema: map[string]*schema.Schema{ {{- range $prop := $.ItemType.Properties }}