From a0343e22860897dad34ed1ea9233333ba0188d25 Mon Sep 17 00:00:00 2001 From: Sam Levenick Date: Tue, 5 Oct 2021 10:19:44 -0700 Subject: [PATCH] Osconfig resource (alpha) (#5266) * Adding private compile * Send empty array instead of nil for non-computed arrays * Revert serialize cmd change * Remove extra logs, revert go mod and sum * Revert go sum changes * Delegate DCL version switching to function * Fix extra slash --- GNUmakefile | 9 ++++- mmv1/provider/terraform.rb | 8 ++++ ...rce_gke_hub_feature_membership_test.go.erb | 2 +- .../resource_gke_hub_feature_test.go.erb | 2 +- .../terraform/utils/provider.go.erb | 3 ++ tpgtools/main.go | 8 ++-- tpgtools/product.go | 18 ++++----- tpgtools/resource.go | 14 ++----- tpgtools/serialization_helpers.go | 13 ++++++ tpgtools/templates/resource.go.tmpl | 40 +++++++++++-------- .../templates/resource.html.markdown.tmpl | 2 +- tpgtools/templates/serialization.go.tmpl | 8 ++-- tpgtools/templates/sweeper.go.tmpl | 4 +- tpgtools/templates/test_file.go.tmpl | 4 +- 14 files changed, 85 insertions(+), 50 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index fa4c2e312857..20b98c46bbfb 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -29,6 +29,13 @@ ifneq ($(RESOURCE),) tpgtools_compile += --resource $(RESOURCE) endif +ifneq ($(OVERRIDES),) + mmv1_compile += -r $(OVERRIDES) + tpgtools_compile += --overrides $(OVERRIDES)/tpgtools/overrides +else + tpgtools_compile += --overrides "overrides" +endif + UNAME := $(shell uname) # The inplace editing semantics are different between linux and osx. @@ -53,7 +60,7 @@ mmv1: tpgtools: cd tpgtools;\ - go run . --path "api" --overrides "overrides" --output $(OUTPUT_PATH) --version $(VERSION) $(tpgtools_compile) + go run . --path "api" --output $(OUTPUT_PATH) --version $(VERSION) $(tpgtools_compile) validator: cd mmv1;\ diff --git a/mmv1/provider/terraform.rb b/mmv1/provider/terraform.rb index f31a79592044..6fcd7dcef2f2 100644 --- a/mmv1/provider/terraform.rb +++ b/mmv1/provider/terraform.rb @@ -315,5 +315,13 @@ def full_resource_name(data) "#{product_name}_#{name}" end end + + # Returns the extension for DCL packages for the given version. This is needed + # as the DCL uses "alpha" for preview resources, while we use "private" + def dcl_version(version) + return '' if version == 'ga' + return '/beta' if version == 'beta' + return '/alpha' if verison == 'private' + end end end diff --git a/mmv1/third_party/terraform/tests/resource_gke_hub_feature_membership_test.go.erb b/mmv1/third_party/terraform/tests/resource_gke_hub_feature_membership_test.go.erb index 8201e6695cb4..4ece23a2a2b9 100644 --- a/mmv1/third_party/terraform/tests/resource_gke_hub_feature_membership_test.go.erb +++ b/mmv1/third_party/terraform/tests/resource_gke_hub_feature_membership_test.go.erb @@ -6,7 +6,7 @@ import ( "testing" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" - gkehub "github.com/GoogleCloudPlatform/declarative-resource-client-library/services/google/gkehub/beta" + gkehub "github.com/GoogleCloudPlatform/declarative-resource-client-library/services/google/gkehub<%= dcl_version(version) -%>" ) func TestAccGkeHubFeatureMembership_gkehubFeatureAcmUpdate(t *testing.T) { diff --git a/mmv1/third_party/terraform/tests/resource_gke_hub_feature_test.go.erb b/mmv1/third_party/terraform/tests/resource_gke_hub_feature_test.go.erb index 1264353a00ed..7098c5a13465 100644 --- a/mmv1/third_party/terraform/tests/resource_gke_hub_feature_test.go.erb +++ b/mmv1/third_party/terraform/tests/resource_gke_hub_feature_test.go.erb @@ -6,7 +6,7 @@ import ( "testing" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" - gkehub "github.com/GoogleCloudPlatform/declarative-resource-client-library/services/google/gkehub/beta" + gkehub "github.com/GoogleCloudPlatform/declarative-resource-client-library/services/google/gkehub<%= dcl_version(version) -%>" ) func TestAccGKEHubFeature_gkehubFeatureMciUpdate(t *testing.T) { diff --git a/mmv1/third_party/terraform/utils/provider.go.erb b/mmv1/third_party/terraform/utils/provider.go.erb index 953c600a8138..863a878ca1ea 100644 --- a/mmv1/third_party/terraform/utils/provider.go.erb +++ b/mmv1/third_party/terraform/utils/provider.go.erb @@ -462,6 +462,9 @@ end # products.each do "google_monitoring_monitored_project": resourceMonitoringMonitoredProject(), <% end -%> "google_org_policy_policy": resourceOrgPolicyPolicy(), + <% if version == 'private' -%> + "google_os_config_os_policy_assignment": resourceOsConfigOsPolicyAssignment(), + <% end -%> "google_privateca_certificate_template": resourcePrivatecaCertificateTemplate(), }, // ------------------------------------ diff --git a/tpgtools/main.go b/tpgtools/main.go index fb4b8dd98006..e5e93f8bcab1 100644 --- a/tpgtools/main.go +++ b/tpgtools/main.go @@ -72,6 +72,8 @@ func main() { } if *version == GA_VERSION { terraformResourceDirectory = "google" + } else if *version == ALPHA_VERSION { + terraformResourceDirectory = "google-private" } for _, resource := range resourcesForVersion { @@ -266,10 +268,10 @@ func generateSerializationLogic(specs map[Version][]*Resource) { for _, res := range resList { var pkgName, pkgPath string pkgName = res.Package() + v.SerializationSuffix - if v == BETA_VERSION { - pkgPath = path.Join(res.Package(), v.V) - } else { + if v == GA_VERSION { pkgPath = res.Package() + } else { + pkgPath = path.Join(res.Package(), v.V) } if _, ok := packageMap[pkgPath]; !ok { diff --git a/tpgtools/product.go b/tpgtools/product.go index e5d5bdff312e..96daee838afa 100644 --- a/tpgtools/product.go +++ b/tpgtools/product.go @@ -2,7 +2,6 @@ package main import ( "log" - "regexp" "strings" "github.com/golang/glog" @@ -18,6 +17,9 @@ type ProductMetadata struct { // ProductName is the case accounted (snake case) name of the product // that the resource belongs to. ProductName string + // DCL name for the product. Used for correctly casing product name for + // references within the DCL (BasePath etc) + DCLProductName string } var productOverrides map[string]Overrides = make(map[string]Overrides, 0) @@ -28,19 +30,17 @@ func GetProductMetadataFromDocument(document *openapi.Document, packagePath stri productOverrides[packagePath] = loadOverrides(packagePath, "tpgtools_product.yaml") } title := getProductTitle(document.Info.Title, packagePath) - productMetadata := NewProductMetadata(packagePath, jsonToSnakeCase(title)) + productMetadata := NewProductMetadata(packagePath, title) return productMetadata } func NewProductMetadata(packagePath, productName string) *ProductMetadata { - if regexp.MustCompile("[A-Z]+").Match([]byte(productName)) { - log.Fatalln("error - expected product name to be snakecase") - } packageName := strings.Split(packagePath, "/")[0] return &ProductMetadata{ - PackagePath: packagePath, - PackageName: packageName, - ProductName: productName, + PackagePath: packagePath, + PackageName: packageName, + ProductName: jsonToSnakeCase(productName), + DCLProductName: productName, } } @@ -125,7 +125,7 @@ func getProductTitle(documentTitle, packagePath string) string { // ProductType is the title-cased product name of a resource. For example, // "NetworkServices". func (pm *ProductMetadata) ProductType() string { - return snakeToTitleCase(pm.ProductName) + return pm.DCLProductName } func (pm *ProductMetadata) DocsSection() string { diff --git a/tpgtools/resource.go b/tpgtools/resource.go index 5710e6e83f95..921cf69feaf5 100644 --- a/tpgtools/resource.go +++ b/tpgtools/resource.go @@ -185,12 +185,6 @@ func (r Resource) TerraformName() string { return "google_" + r.Path() } -// Type is the title-cased name of a resource, for printing information about -// the type". For example, "Instance". -func (r Resource) Type() string { - return snakeToTitleCase(r.DCLName()) -} - // PathType is the title-cased name of a resource preceded by its package, // often used to namespace functions. For example, "RedisInstance". func (r Resource) PathType() string { @@ -663,7 +657,7 @@ func (r Resource) getSamples(docs bool) []Sample { } func (r *Resource) getSampleAccessoryFolder() string { - resourceType := strings.ToLower(r.Type()) + resourceType := strings.ToLower(r.DCLTitle()) packageName := strings.ToLower(r.productMetadata.PackageName) sampleAccessoryFolder := path.Join(*tPath, packageName, "samples", resourceType) return sampleAccessoryFolder @@ -754,7 +748,7 @@ func (r *Resource) loadDCLSamples() []Sample { sampleAccessoryFolder := r.getSampleAccessoryFolder() packagePath := r.productMetadata.PackagePath version := r.versionMetadata.V - resourceType := r.Type() + resourceType := r.DCLTitle() sampleFriendlyMetaPath := path.Join(sampleAccessoryFolder, "meta.yaml") samples := []Sample{} @@ -763,7 +757,7 @@ func (r *Resource) loadDCLSamples() []Sample { } // Samples appear in the root product folder - packagePath = strings.Split(packagePath, "beta")[0] + packagePath = strings.Split(packagePath, "/")[0] samplesPath := path.Join(*fPath, packagePath, "samples") files, err := ioutil.ReadDir(samplesPath) if err != nil { @@ -810,7 +804,7 @@ func (r *Resource) loadDCLSamples() []Sample { if !versionMatch { continue - } else if primaryResourceName != resourceType { + } else if !strings.EqualFold(primaryResourceName, resourceType) { glog.Errorf("skipping %s since no match with %s.", primaryResourceName, resourceType) continue } diff --git a/tpgtools/serialization_helpers.go b/tpgtools/serialization_helpers.go index 102d2548b720..4be1b3800d45 100644 --- a/tpgtools/serialization_helpers.go +++ b/tpgtools/serialization_helpers.go @@ -8,8 +8,21 @@ import ( cloudresourcemanager "github.com/GoogleCloudPlatform/declarative-resource-client-library/services/google/cloudresourcemanager" cloudresourcemanagerBeta "github.com/GoogleCloudPlatform/declarative-resource-client-library/services/google/cloudresourcemanager/beta" + cloudresourcemanagerAlpha "github.com/GoogleCloudPlatform/declarative-resource-client-library/services/google/cloudresourcemanager/alpha" ) +func serializeAlphaProjectToHCL(r cloudresourcemanagerAlpha.Project, hasGAEquivalent bool) (string, error) { + b, err := json.Marshal(r) + if err != nil { + return "", err + } + var m map[string]interface{} + if err := json.Unmarshal(b, &m); err != nil { + return "", err + } + return serializeProjectToHCL(m, hasGAEquivalent) +} + func serializeBetaProjectToHCL(r cloudresourcemanagerBeta.Project, hasGAEquivalent bool) (string, error) { b, err := json.Marshal(r) if err != nil { diff --git a/tpgtools/templates/resource.go.tmpl b/tpgtools/templates/resource.go.tmpl index 218fbf9fd697..4aa409fb5bc5 100644 --- a/tpgtools/templates/resource.go.tmpl +++ b/tpgtools/templates/resource.go.tmpl @@ -212,7 +212,7 @@ func resource{{$.PathType}}Create(d *schema.ResourceData, meta interface{}) erro {{- end -}} {{ end }} - obj := &{{$.Package}}.{{$.Type}}{ + obj := &{{$.Package}}.{{$.DCLTitle}}{ {{- range $v := .Properties }} {{- if and ($v.Settable) ($v.StateGetter) }} {{$v.PackageName}}: {{$v.StateGetter}}, @@ -269,17 +269,17 @@ should be converted to use the DCL's ID method, so normalization can be uniform. {{- if $.AppendToBasePath }} client.Config.BasePath += "{{$.AppendToBasePath}}" {{- end }} - res, err := client.Apply{{$.Type}}(context.Background(), obj, createDirective...) + res, err := client.Apply{{$.DCLTitle}}(context.Background(), obj, createDirective...) if _, ok := err.(dcl.DiffAfterApplyError); ok { log.Printf("[DEBUG] Diff after apply returned from the DCL: %s", err) } else if err != nil { // The resource didn't actually create d.SetId("") - return fmt.Errorf("Error creating {{$.Type}}: %s", err) + return fmt.Errorf("Error creating {{$.DCLTitle}}: %s", err) } - log.Printf("[DEBUG] Finished creating {{$.Type}} %q: %#v", d.Id(), res) + log.Printf("[DEBUG] Finished creating {{$.DCLTitle}} %q: %#v", d.Id(), res) {{ if $.HasServerGeneratedName }} {{ range $v := .Properties -}} @@ -318,7 +318,7 @@ func resource{{$.PathType}}Read(d *schema.ResourceData, meta interface{}) error {{- end -}} {{ end }} - obj := &{{$.Package}}.{{$.Type}}{ + obj := &{{$.Package}}.{{$.DCLTitle}}{ {{- range $v := .Properties }} {{- if ($v.StateGetter) }} {{$v.PackageName}}: {{$v.StateGetter}}, @@ -343,7 +343,7 @@ func resource{{$.PathType}}Read(d *schema.ResourceData, meta interface{}) error {{- if $.AppendToBasePath }} client.Config.BasePath += "{{$.AppendToBasePath}}" {{- end }} - res, err := client.Get{{$.Type}}(context.Background(), obj) + res, err := client.Get{{$.DCLTitle}}(context.Background(), obj) if err != nil { resourceName := fmt.Sprintf("{{$.PathType}} %q", d.Id()) return handleNotFoundDCLError(err, d, resourceName) @@ -373,7 +373,7 @@ func resource{{$.PathType}}Update(d *schema.ResourceData, meta interface{}) erro {{- end -}} {{ end }} - obj := &{{$.Package}}.{{$.Type}}{ + obj := &{{$.Package}}.{{$.DCLTitle}}{ {{- range $v := .Properties }} {{- if ($v.StateGetter) }} {{$v.PackageName}}: {{$v.StateGetter}}, @@ -383,7 +383,7 @@ func resource{{$.PathType}}Update(d *schema.ResourceData, meta interface{}) erro {{- if $.StateHint }} // Construct state hint from old values - old := &{{$.Package}}.{{$.Type}}{ + old := &{{$.Package}}.{{$.DCLTitle}}{ {{- range $v := .Properties }} {{- if ($v.StateGetter) }} {{$v.PackageName}}: {{$v.ChangeStateGetter}}, @@ -419,17 +419,17 @@ func resource{{$.PathType}}Update(d *schema.ResourceData, meta interface{}) erro {{- if $.AppendToBasePath }} client.Config.BasePath += "{{$.AppendToBasePath}}" {{- end }} - res, err := client.Apply{{$.Type}}(context.Background(), obj, directive...) + res, err := client.Apply{{$.DCLTitle}}(context.Background(), obj, directive...) if _, ok := err.(dcl.DiffAfterApplyError); ok { log.Printf("[DEBUG] Diff after apply returned from the DCL: %s", err) } else if err != nil { // The resource didn't actually create d.SetId("") - return fmt.Errorf("Error updating {{$.Type}}: %s", err) + return fmt.Errorf("Error updating {{$.DCLTitle}}: %s", err) } - log.Printf("[DEBUG] Finished creating {{$.Type}} %q: %#v", d.Id(), res) + log.Printf("[DEBUG] Finished creating {{$.DCLTitle}} %q: %#v", d.Id(), res) return resource{{$.PathType}}Read(d, meta) } @@ -447,7 +447,7 @@ func resource{{$.PathType}}Delete(d *schema.ResourceData, meta interface{}) erro {{- end -}} {{ end }} - obj := &{{$.Package}}.{{$.Type}}{ + obj := &{{$.Package}}.{{$.DCLTitle}}{ {{- range $v := .Properties }} {{- if ($v.StateGetter) }} {{$v.PackageName}}: {{$v.StateGetter}}, @@ -480,7 +480,7 @@ func resource{{$.PathType}}Delete(d *schema.ResourceData, meta interface{}) erro } {{- end }} - log.Printf("[DEBUG] Deleting {{$.Type}} %q", d.Id()) + log.Printf("[DEBUG] Deleting {{$.DCLTitle}} %q", d.Id()) userAgent, err := generateUserAgentString(d, config.userAgent) if err != nil { return err @@ -498,11 +498,11 @@ func resource{{$.PathType}}Delete(d *schema.ResourceData, meta interface{}) erro {{- if $.AppendToBasePath }} client.Config.BasePath += "{{$.AppendToBasePath}}" {{- end }} - if err := client.Delete{{$.Type}}(context.Background(), obj); err != nil { - return fmt.Errorf("Error deleting {{$.Type}}: %s", err) + if err := client.Delete{{$.DCLTitle}}(context.Background(), obj); err != nil { + return fmt.Errorf("Error deleting {{$.DCLTitle}}: %s", err) } - log.Printf("[DEBUG] Finished deleting {{$.Type}} %q", d.Id()) + log.Printf("[DEBUG] Finished deleting {{$.DCLTitle}} %q", d.Id()) return nil } @@ -560,7 +560,11 @@ func expand{{$.PathType}}{{$v.PackagePath}}Collapsed(d *schema.ResourceData) *{{ {{ if $v.IsArray -}} func expand{{$.PathType}}{{$v.PackagePath}}Array(o interface{}) []{{$.Package}}.{{$v.ObjectType}} { if o == nil { + {{- if $v.Computed }} return nil + {{- else }} + return make([]{{$.Package}}.{{$v.ObjectType}}, 0) + {{- end }} } {{ if $v.IsSet -}} @@ -569,7 +573,11 @@ func expand{{$.PathType}}{{$v.PackagePath}}Array(o interface{}) []{{$.Package}}. objs := o.([]interface{}) if len(objs) == 0 { + {{- if $v.Computed }} return nil + {{- else }} + return make([]{{$.Package}}.{{$v.ObjectType}}, 0) + {{- end }} } items := make([]{{$.Package}}.{{$v.ObjectType}}, 0, len(objs)) diff --git a/tpgtools/templates/resource.html.markdown.tmpl b/tpgtools/templates/resource.html.markdown.tmpl index c541196f63c2..7eb7301c0284 100644 --- a/tpgtools/templates/resource.html.markdown.tmpl +++ b/tpgtools/templates/resource.html.markdown.tmpl @@ -137,7 +137,7 @@ This resource provides the following ## Import {{/* TODO excluded imports */}} -{{$.Type}} can be imported using any of these accepted formats: +{{$.DCLTitle}} can be imported using any of these accepted formats: ``` {{- range $format := $.ImportFormats }} diff --git a/tpgtools/templates/serialization.go.tmpl b/tpgtools/templates/serialization.go.tmpl index 952fd2aa3aed..2a7f95c11607 100644 --- a/tpgtools/templates/serialization.go.tmpl +++ b/tpgtools/templates/serialization.go.tmpl @@ -77,7 +77,7 @@ func DCLToTerraformSampleName(service, resource string) (string, string, error) {{- range $res := $resList }} {{- if not $res.IsAlternateLocation }} case "{{$res.DCLPackage}}{{replace $res.DCLName "_" "" -1}}": - return "{{$res.ProductType }}", "{{ $res.Type }}", nil + return "{{$res.ProductType }}", "{{ $res.DCLTitle }}", nil {{- end }} {{- end }} default: @@ -95,7 +95,7 @@ func ConvertSampleJSONToHCL(resourceType string, version string, hasGAEquivalent switch resourceType { {{- range $res := $resList }} case "{{$res.PathType}}": - r := &{{$res.Package}}{{$version.SerializationSuffix}}.{{$res.Type}}{} + r := &{{$res.Package}}{{$version.SerializationSuffix}}.{{$res.DCLTitle}}{} if err := json.Unmarshal(b, r); err != nil { return "", err } @@ -112,7 +112,7 @@ func ConvertSampleJSONToHCL(resourceType string, version string, hasGAEquivalent switch resourceType { {{- range $res := $resList }} case "{{$res.PathType}}": - r := &{{$res.Package}}{{$version.SerializationSuffix}}.{{$res.Type}}{} + r := &{{$res.Package}}{{$version.SerializationSuffix}}.{{$res.DCLTitle}}{} if err := json.Unmarshal(b, r); err != nil { return "", err } @@ -139,7 +139,7 @@ func ConvertSampleJSONToHCL(resourceType string, version string, hasGAEquivalent // the crucial point is that `terraform import; terraform apply` will not produce // any changes. We do not validate that the resource specified will pass terraform // validation unless is an object returned from the API after an Apply. -func {{ $res.PathType}}{{$version.SerializationSuffix}}AsHCL(r {{$res.Package}}{{$version.SerializationSuffix}}.{{$res.Type}}, hasGAEquivalent bool) (string, error) { +func {{ $res.PathType}}{{$version.SerializationSuffix}}AsHCL(r {{$res.Package}}{{$version.SerializationSuffix}}.{{$res.DCLTitle}}, hasGAEquivalent bool) (string, error) { outputConfig := "resource \"{{$res.TerraformName}}\" \"output\" {\n" {{- range $field := $res.Properties}} {{- if $field.Settable }} diff --git a/tpgtools/templates/sweeper.go.tmpl b/tpgtools/templates/sweeper.go.tmpl index d0d79622d247..1ae33f659cd4 100644 --- a/tpgtools/templates/sweeper.go.tmpl +++ b/tpgtools/templates/sweeper.go.tmpl @@ -72,13 +72,13 @@ func testSweep{{$.SweeperName}}(region string) error { } client := NewDCL{{$.ProductType}}Client(config, config.userAgent, "") - err = client.DeleteAll{{$.Type}}(context.Background(), {{$.SweeperFunctionArgs}} isDeletable{{$.SweeperName}}) + err = client.DeleteAll{{$.DCLTitle}}(context.Background(), {{$.SweeperFunctionArgs}} isDeletable{{$.SweeperName}}) if err != nil { return err } return nil } -func isDeletable{{$.SweeperName}}(r *{{$.Package}}.{{$.Type}}) bool { +func isDeletable{{$.SweeperName}}(r *{{$.Package}}.{{$.DCLTitle}}) bool { return isSweepableTestResource(*r.Name) } diff --git a/tpgtools/templates/test_file.go.tmpl b/tpgtools/templates/test_file.go.tmpl index 330698b0dc1c..957d56fc36b1 100644 --- a/tpgtools/templates/test_file.go.tmpl +++ b/tpgtools/templates/test_file.go.tmpl @@ -122,7 +122,7 @@ func testAccCheck{{$.PathType}}DestroyProducer(t *testing.T) func(s *terraform.S billingProject = config.BillingProject } - obj := &{{$.Package}}.{{$.Type}}{ + obj := &{{$.Package}}.{{$.DCLTitle}}{ {{- range $v := .Properties }} {{- if $v.StateGetterForDestroyTest }} {{$v.PackageName}}: {{ $v.StateGetterForDestroyTest }}, @@ -131,7 +131,7 @@ func testAccCheck{{$.PathType}}DestroyProducer(t *testing.T) func(s *terraform.S } client := NewDCL{{$.ProductType}}Client(config, config.userAgent, billingProject) - _, err := client.Get{{$.Type}}(context.Background(), obj) + _, err := client.Get{{$.DCLTitle}}(context.Background(), obj) if err == nil { return fmt.Errorf("{{$.TerraformName}} still exists %v", obj) }