Migrate blueprints/data-solutions/vertex-mlops to google_workbench_instance

blueprints/apigee/bigquery-analytics/main.tf

@@ -143,7 +143,6 @@ module "bucket_export" {
     ]
   }
   notification_config = {
-    create_topic      = {}
     enabled           = true
     payload_format    = "JSON_API_V1"
     sa_email          = module.project.service_agents.storage.email

blueprints/data-solutions/vertex-mlops/README.md

@@ -72,7 +72,7 @@ module "test" {
     project_id         = "test-dev"
   }
 }
-# tftest modules=11 resources=86
+# tftest modules=11 resources=90
 ```
 <!-- BEGIN TFDOC -->
 ## Variables
@@ -128,5 +128,5 @@ module "test" {
     project_id         = "test-dev"
   }
 }
-# tftest modules=13 resources=91 e2e
+# tftest modules=13 resources=95 e2e
 ```

blueprints/data-solutions/vertex-mlops/main.tf

@@ -141,10 +141,10 @@ module "vpc-local" {
   name       = "vertex"
   subnets = [
     {
-      "name" : "subnet-${var.region}",
-      "region" : "${var.region}",
-      "ip_cidr_range" : "10.4.0.0/24",
-      "secondary_ip_range" : null
+      name                  = "subnet-${var.region}",
+      region                = var.region,
+      ip_cidr_range         = "10.5.0.0/24"
+      enable_private_access = true
     }
   ]
   psa_configs = [{
@@ -264,6 +264,7 @@ module "project" {
     "bigquery.googleapis.com",
     "bigquerystorage.googleapis.com",
     "cloudbuild.googleapis.com",
+    "containerfilesystem.googleapis.com",
     "compute.googleapis.com",
     "datacatalog.googleapis.com",
     "dataflow.googleapis.com",

blueprints/data-solutions/vertex-mlops/outputs.tf

@@ -36,7 +36,7 @@ output "notebook" {
   description = "Vertex AI notebooks ids."
   value = merge(
     { for k, v in resource.google_notebooks_runtime.runtime : k => v.id },
-    { for k, v in resource.google_notebooks_instance.playground : k => v.id }
+    { for k, v in resource.google_workbench_instance.playground : k => v.id }
   )
 }
 

blueprints/data-solutions/vertex-mlops/vertex.tf

@@ -37,6 +37,13 @@ module "service-account-notebook" {
   source     = "../../../modules/iam-service-account"
   project_id = module.project.project_id
   name       = "notebook-sa"
+  iam_project_roles = {
+    (module.project.project_id) = [
+      "roles/logging.logWriter",
+      "roles/monitoring.metricWriter",
+      "roles/serviceusage.serviceUsageConsumer",
+    ]
+  }
 }
 
 resource "google_notebooks_runtime" "runtime" {
@@ -79,54 +86,49 @@ resource "google_notebooks_runtime" "runtime" {
   }
 }
 
-resource "google_notebooks_instance" "playground" {
-  for_each     = { for k, v in var.notebooks : k => v if v.type == "USER_MANAGED" }
-  name         = "${var.prefix}-${each.key}"
-  location     = "${var.region}-b"
-  machine_type = var.notebooks[each.key].machine_type
-  project      = module.project.project_id
-
-  container_image {
-    repository = "gcr.io/deeplearning-platform-release/base-cpu"
-    tag        = "latest"
-  }
-
-  install_gpu_driver = true
-  boot_disk_type     = "PD_SSD"
-  boot_disk_size_gb  = 110
-  disk_encryption    = var.service_encryption_keys.notebooks != null ? "CMEK" : null
-  kms_key            = var.service_encryption_keys.notebooks
-
-  no_public_ip    = var.notebooks[each.key].internal_ip_only
-  no_proxy_access = false
-
-  network = local.vpc
-  subnet  = local.subnet
+resource "google_workbench_instance" "playground" {
+  for_each = { for k, v in var.notebooks : k => v if v.type == "USER_MANAGED" }
+  project  = module.project.project_id
+  name     = "${var.prefix}-${each.key}"
+  location = "${var.region}-b"
 
-  instance_owners = try(tolist(var.notebooks[each.key].owner), null)
-  service_account = module.service-account-notebook.email
-  service_account_scopes = [
-    "https://www.googleapis.com/auth/cloud-platform",
-    "https://www.googleapis.com/auth/userinfo.email",
-  ]
+  gce_setup {
+    machine_type = var.notebooks[each.key].machine_type
+    container_image {
+      repository = "gcr.io/deeplearning-platform-release/workbench-container"
+      tag        = "latest"
+    }
+    boot_disk {
+      disk_size_gb    = 150
+      disk_type       = "PD_SSD"
+      disk_encryption = var.service_encryption_keys.notebooks != null ? "CMEK" : null
+      kms_key         = var.service_encryption_keys.notebooks
+    }
 
+    disable_public_ip = var.notebooks[each.key].internal_ip_only
 
-  metadata = {
-    notebook-disable-nbconvert = "false"
-    notebook-disable-downloads = "false"
-    notebook-disable-terminal  = "false"
-    notebook-disable-root      = "true"
+    network_interfaces {
+      network = local.vpc
+      subnet  = local.subnet
+    }
+    service_accounts {
+      email = module.service-account-notebook.email
+    }
+    # full list of supported metadata keys:
+    # https://cloud.google.com/vertex-ai/docs/workbench/instances/manage-metadata
+    metadata = {
+      notebook-disable-nbconvert = "false"
+      notebook-disable-downloads = "false"
+      notebook-disable-terminal  = "false"
+      notebook-disable-root      = "true"
+    }
+    tags = ["ssh"]
   }
+  disable_proxy_access = true
+  instance_owners      = try(tolist(var.notebooks[each.key].owner), null)
 
-  # Remove once terraform-provider-google/issues/9164 is fixed
-  lifecycle {
-    ignore_changes = [disk_encryption, kms_key]
-  }
 
-  #TODO Uncomment once terraform-provider-google/issues/9273 is fixed
-  # tags = ["ssh"]
   depends_on = [
     google_project_iam_member.shared_vpc,
   ]
 }
-

modules/gcs/README.md

@@ -326,7 +326,7 @@ module "bucket" {
 | [lifecycle_rules](variables.tf#L107) | Bucket lifecycle rule. | <code title="map&#40;object&#40;&#123;&#10;  action &#61; object&#40;&#123;&#10;    type          &#61; string&#10;    storage_class &#61; optional&#40;string&#41;&#10;  &#125;&#41;&#10;  condition &#61; object&#40;&#123;&#10;    age                        &#61; optional&#40;number&#41;&#10;    created_before             &#61; optional&#40;string&#41;&#10;    custom_time_before         &#61; optional&#40;string&#41;&#10;    days_since_custom_time     &#61; optional&#40;number&#41;&#10;    days_since_noncurrent_time &#61; optional&#40;number&#41;&#10;    matches_prefix             &#61; optional&#40;list&#40;string&#41;&#41;&#10;    matches_storage_class      &#61; optional&#40;list&#40;string&#41;&#41; &#35; STANDARD, MULTI_REGIONAL, REGIONAL, NEARLINE, COLDLINE, ARCHIVE, DURABLE_REDUCED_AVAILABILITY&#10;    matches_suffix             &#61; optional&#40;list&#40;string&#41;&#41;&#10;    noncurrent_time_before     &#61; optional&#40;string&#41;&#10;    num_newer_versions         &#61; optional&#40;number&#41;&#10;    with_state                 &#61; optional&#40;string&#41; &#35; &#34;LIVE&#34;, &#34;ARCHIVED&#34;, &#34;ANY&#34;&#10;  &#125;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
 | [logging_config](variables.tf#L162) | Bucket logging configuration. | <code title="object&#40;&#123;&#10;  log_bucket        &#61; string&#10;  log_object_prefix &#61; optional&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
 | [managed_folders](variables.tf#L171) | Managed folders to create within the bucket in {PATH => CONFIG} format. | <code title="map&#40;object&#40;&#123;&#10;  force_destroy &#61; optional&#40;bool, false&#41;&#10;  iam           &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10;  iam_bindings &#61; optional&#40;map&#40;object&#40;&#123;&#10;    members &#61; list&#40;string&#41;&#10;    role    &#61; string&#10;    condition &#61; optional&#40;object&#40;&#123;&#10;      expression  &#61; string&#10;      title       &#61; string&#10;      description &#61; optional&#40;string&#41;&#10;    &#125;&#41;&#41;&#10;  &#125;&#41;&#41;, &#123;&#125;&#41;&#10;  iam_bindings_additive &#61; optional&#40;map&#40;object&#40;&#123;&#10;    member &#61; string&#10;    role   &#61; string&#10;    condition &#61; optional&#40;object&#40;&#123;&#10;      expression  &#61; string&#10;      title       &#61; string&#10;      description &#61; optional&#40;string&#41;&#10;    &#125;&#41;&#41;&#10;  &#125;&#41;&#41;, &#123;&#125;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
-| [notification_config](variables.tf#L204) | GCS Notification configuration. | <code title="object&#40;&#123;&#10;  enabled        &#61; bool&#10;  payload_format &#61; string&#10;  sa_email       &#61; string&#10;  topic_name     &#61; string&#10;  create_topic &#61; optional&#40;object&#40;&#123;&#10;    kms_key_id &#61; optional&#40;string&#41;&#10;  &#125;&#41;&#41;&#10;  event_types        &#61; optional&#40;list&#40;string&#41;&#41;&#10;  custom_attributes  &#61; optional&#40;map&#40;string&#41;&#41;&#10;  object_name_prefix &#61; optional&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
+| [notification_config](variables.tf#L204) | GCS Notification configuration. | <code title="object&#40;&#123;&#10;  enabled        &#61; bool&#10;  payload_format &#61; string&#10;  sa_email       &#61; string&#10;  topic_name     &#61; string&#10;  create_topic &#61; optional&#40;object&#40;&#123;&#10;    kms_key_id &#61; optional&#40;string&#41;&#10;  &#125;&#41;, &#123;&#125;&#41;&#10;  event_types        &#61; optional&#40;list&#40;string&#41;&#41;&#10;  custom_attributes  &#61; optional&#40;map&#40;string&#41;&#41;&#10;  object_name_prefix &#61; optional&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
 | [objects_to_upload](variables.tf#L221) | Objects to be uploaded to bucket. | <code title="map&#40;object&#40;&#123;&#10;  name                &#61; string&#10;  metadata            &#61; optional&#40;map&#40;string&#41;&#41;&#10;  content             &#61; optional&#40;string&#41;&#10;  source              &#61; optional&#40;string&#41;&#10;  cache_control       &#61; optional&#40;string&#41;&#10;  content_disposition &#61; optional&#40;string&#41;&#10;  content_encoding    &#61; optional&#40;string&#41;&#10;  content_language    &#61; optional&#40;string&#41;&#10;  content_type        &#61; optional&#40;string&#41;&#10;  event_based_hold    &#61; optional&#40;bool&#41;&#10;  temporary_hold      &#61; optional&#40;bool&#41;&#10;  detect_md5hash      &#61; optional&#40;string&#41;&#10;  storage_class       &#61; optional&#40;string&#41;&#10;  kms_key_name        &#61; optional&#40;string&#41;&#10;  customer_encryption &#61; optional&#40;object&#40;&#123;&#10;    encryption_algorithm &#61; optional&#40;string&#41;&#10;    encryption_key       &#61; string&#10;  &#125;&#41;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
 | [prefix](variables.tf#L247) | Optional prefix used to generate the bucket name. | <code>string</code> |  | <code>null</code> |
 | [public_access_prevention](variables.tf#L262) | Prevents public access to the bucket. | <code>string</code> |  | <code>null</code> |

modules/gcs/variables.tf

@@ -210,7 +210,7 @@ variable "notification_config" {
     topic_name     = string
     create_topic = optional(object({
       kms_key_id = optional(string)
-    }))
+    }), {})
     event_types        = optional(list(string))
     custom_attributes  = optional(map(string))
     object_name_prefix = optional(string)