feat(HNS folders): add new resources to support HNS folders (#12101)

mmv1/products/storage/Folder.yaml

@@ -0,0 +1,89 @@
+# Copyright 2024 Google Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+name: 'Folder'
+kind: 'storage#folder'
+base_url: 'b/{{bucket}}/folders'
+self_link: 'b/{{bucket}}/folders/{{%name}}'
+id_format: '{{bucket}}/{{name}}'
+delete_url: 'b/{{bucket}}/folders/{{%name}}'
+create_url: 'b/{{bucket}}/folders'
+has_self_link: true
+timeouts:
+  insert_minutes: 20
+  update_minutes: 20
+  delete_minutes: 20
+exclude_sweeper: true
+import_format:
+  - '{{bucket}}/folders/{{%name}}'
+  - '{{bucket}}/{{%name}}'
+examples:
+  - name: 'storage_folder_basic'
+    primary_resource_id: 'folder'
+    vars:
+      bucket_name: 'my-bucket'
+    ignore_read_extra:
+      - 'force_destroy'
+description: |
+  A Google Cloud Storage Folder.
+
+  The Folder resource represents a folder in a Cloud Storage bucket with hierarchical namespace enabled
+references:
+  guides:
+    'Official Documentation': 'https://cloud.google.com/storage/docs/folders-overview'
+  api: 'https://cloud.google.com/storage/docs/json_api/v1/folders'
+custom_code:
+  custom_import: templates/terraform/custom_import/storage_folder.go.tmpl
+  custom_update: templates/terraform/custom_update/storage_folder_update.go.tmpl
+  custom_delete: templates/terraform/custom_delete/storage_folder_delete.go.tmpl
+virtual_fields:
+  - name: 'force_destroy'
+    description:
+      If set to true, items within folder if any will be force destroyed.
+    type: Boolean
+    default_value: false
+parameters:
+  - name: 'bucket'
+    resource: 'Bucket'
+    imports: 'name'
+    description: 'The name of the bucket that contains the folder.'
+    required: true
+    immutable: true
+    url_param_only: true
+  - name: 'name'
+    description: |
+      The name of the folder expressed as a path. Must include
+      trailing '/'. For example, `example_dir/example_dir2/`, `example@#/`, `a-b/d-f/`.
+    required: true
+    immutable: true
+    # The API returns values with trailing slashes, even if not
+    # provided. Enforcing trailing slashes prevents diffs and ensures
+    # consistent output.
+    validation:
+      regex: '/$'
+properties:
+  - name: createTime
+    type: String
+    description: |
+      The timestamp at which this folder was created.
+    output: true
+  - name: updateTime
+    type: String
+    description: |
+      The timestamp at which this folder was most recently updated.
+    output: true
+  - name: metageneration
+    type: String
+    description: |
+      The metadata generation of the folder.
+    output: true

mmv1/templates/terraform/custom_delete/storage_folder_delete.go.tmpl

@@ -0,0 +1,113 @@
+bucket := d.Get("bucket").(string)
+name := d.Get("name").(string)
+
+var listError, deleteObjectError error
+for deleteObjectError == nil {
+    res, err := config.NewStorageClient(userAgent).Objects.List(bucket).Prefix(name).Do()
+    if err != nil {
+        log.Printf("Error listing contents of folder %s: %v", bucket, err)
+        listError = err
+        break
+    }
+
+    if len(res.Items) == 0 {
+        break // 0 items, folder empty
+    }
+
+    if !d.Get("force_destroy").(bool) {
+        deleteErr := fmt.Errorf("Error trying to delete folder %s containing objects without force_destroy set to true", bucket)
+        log.Printf("Error! %s : %s\n\n", bucket, deleteErr)
+        return deleteErr
+    }
+    // GCS requires that a folder be empty (have no objects or object
+    // versions) before it can be deleted.
+    log.Printf("[DEBUG] GCS Folder attempting to forceDestroy\n\n")
+
+    // Create a workerpool for parallel deletion of resources. In the
+    // future, it would be great to expose Terraform's global parallelism
+    // flag here, but that's currently reserved for core use. Testing
+    // shows that NumCPUs-1 is the most performant on average networks.
+    //
+    // The challenge with making this user-configurable is that the
+    // configuration would reside in the Terraform configuration file,
+    // decreasing its portability. Ideally we'd want this to connect to
+    // Terraform's top-level -parallelism flag, but that's not plumbed nor
+    // is it scheduled to be plumbed to individual providers.
+    wp := workerpool.New(runtime.NumCPU() - 1)
+
+    for _, object := range res.Items {
+        log.Printf("[DEBUG] Found %s", object.Name)
+        object := object
+
+        wp.Submit(func() {
+            log.Printf("[TRACE] Attempting to delete %s", object.Name)
+            if err := config.NewStorageClient(userAgent).Objects.Delete(bucket, object.Name).Generation(object.Generation).Do(); err != nil {
+                deleteObjectError = err
+                log.Printf("[ERR] Failed to delete storage object %s: %s", object.Name, err)
+            } else {
+                log.Printf("[TRACE] Successfully deleted %s", object.Name)
+            }
+        })
+    }
+
+    // Wait for everything to finish.
+    wp.StopWait()
+}
+
+err = retry.Retry(1*time.Minute, func() *retry.RetryError {
+    err := config.NewStorageClient(userAgent).Folders.Delete(bucket, name).Do()
+    if err == nil {
+        return nil
+    }
+    if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 429 {
+        return retry.RetryableError(gerr)
+    }
+    return retry.NonRetryableError(err)
+})
+
+if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 409 && strings.Contains(gerr.Message, "not empty") && listError != nil {
+    return fmt.Errorf("could not delete non-empty folder due to error when listing contents: %v", listError)
+}
+if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 409 && strings.Contains(gerr.Message, "not empty") && deleteObjectError != nil {
+    return fmt.Errorf("could not delete non-empty folder due to error when deleting contents: %v", deleteObjectError)
+}
+if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 409 && strings.Contains(gerr.Message, "not empty") && !d.Get("force_destroy").(bool) {
+    return fmt.Errorf("Sub folders or items may exist within folder, use force_destroy to true to delete all subfolders: %v", err)
+}
+
+if err == nil {
+    log.Printf("[DEBUG] Deleted empty folder %v\n\n", name)
+    return nil
+} else {
+    log.Printf("[ERROR] Error deleting folder %v, %v\n\n", name, err)
+}
+
+// attempts to delete any sub folders within the folder
+foldersList, err := config.NewStorageClient(userAgent).Folders.List(bucket).Prefix(name).Do()
+if err != nil {
+    return err
+}
+if d.Get("force_destroy").(bool) {
+    log.Printf("[DEBUG] folder names to delete: %#v", name)
+    items := foldersList.Items
+    for i := len(items) - 1; i >= 0; i-- {
+        err = transport_tpg.Retry(transport_tpg.RetryOptions{
+            RetryFunc: func() error {
+                err = config.NewStorageClient(userAgent).Folders.Delete(bucket, items[i].Name).Do()
+                return err
+            },
+            Timeout:              d.Timeout(schema.TimeoutDelete),
+            ErrorRetryPredicates: []transport_tpg.RetryErrorPredicateFunc{transport_tpg.Is429RetryableQuotaError},
+        })
+        if err != nil {
+            return err
+        }
+    }
+
+    log.Printf("[DEBUG] Finished deleting Folder %q: %#v", d.Id(), name)
+} else {
+    deleteErr := fmt.Errorf("Sub folders exist within folder, use force_destroy to true to delete all subfolders")
+    log.Printf("Error! %s : %s\n\n", name, deleteErr)
+    return deleteErr
+}
+return nil

mmv1/templates/terraform/custom_import/storage_folder.go.tmpl

@@ -0,0 +1,19 @@
+config := meta.(*transport_tpg.Config)
+if err := tpgresource.ParseImportId([]string{
+    "^(?P<bucket>[^/]+)/folders/(?P<name>.+)$",
+    "^(?P<bucket>[^/]+)/(?P<name>.+)$",
+}, d, config); err != nil {
+    return nil, err
+}
+
+// Replace import id for the resource id
+id, err := tpgresource.ReplaceVars(d, config, "{{"{{"}}bucket{{"}}"}}/{{"{{"}}name{{"}}"}}")
+if err != nil {
+    return nil, fmt.Errorf("Error constructing id: %s", err)
+}
+d.SetId(id)
+if err := d.Set("force_destroy", false); err != nil {
+    return nil, fmt.Errorf("Error setting force_destroy: %s", err)
+}
+
+return []*schema.ResourceData{d}, nil

mmv1/templates/terraform/custom_update/storage_folder_update.go.tmpl

@@ -0,0 +1,10 @@
+_ = config
+// we can only get here if force_destroy was updated
+if d.Get("force_destroy") != nil {
+    if err := d.Set("force_destroy", d.Get("force_destroy")); err != nil {
+        return fmt.Errorf("Error updating force_destroy: %s", err)
+    }
+}
+
+// all other fields are immutable, don't do anything else
+return nil

mmv1/templates/terraform/examples/storage_folder_basic.tf.tmpl

@@ -0,0 +1,18 @@
+resource "google_storage_bucket" "bucket" {
+  name                        = "{{index $.Vars "bucket_name"}}"
+  location                    = "EU"
+  uniform_bucket_level_access = true
+  hierarchical_namespace {
+    enabled = true
+  }
+}
+
+resource "google_storage_folder" "{{$.PrimaryResourceId}}" {
+  bucket        = google_storage_bucket.bucket.name
+  name          = "parent-folder/"
+}
+
+resource "google_storage_folder" "subfolder" {
+  bucket        = google_storage_bucket.bucket.name
+  name          = "${google_storage_folder.{{$.PrimaryResourceId}}.name}subfolder/"
+}