Add cloud dataplex module (#1308)

* Add dataplex module

* fix dataplex test

* resolve comments

* python test removed

* Change variable desc

* refactor variables

* fix typos

* fix assets & zones resources

* fix linting error

* fix tests

* fix typo

modules/cloud-dataplex/README.md

@@ -0,0 +1,74 @@
+# Cloud Dataplex instance with lake, zone & assests
+
+This module manages the creation of Cloud Dataplex instance along with lake, zone & assets in single regions. 
+
+
+## Simple example
+
+This example shows how to setup a Cloud Dataplex instance, lake, zone & asset creation in GCP project.
+
+```hcl
+
+module "dataplex" {
+  source     = "./fabric/modules/cloud-dataplex"
+  name       = "terraform-lake"
+  prefix     = "test"
+  project_id = "myproject"
+  region     = "europe-west2"
+  zones = {
+    zone_1 = {
+      type      = "RAW"
+      discovery = true
+      assets = {
+        asset_1 = {
+          bucket_name            = "asset_1"
+          cron_schedule          = "15 15 * * *"
+          discovery_spec_enabled = true
+          resource_spec_type     = "STORAGE_BUCKET"
+        }
+      }
+    },
+    zone_2 = {
+      type      = "CURATED"
+      discovery = true
+      assets = {
+        asset_2 = {
+          bucket_name            = "asset_2"
+          cron_schedule          = "15 15 * * *"
+          discovery_spec_enabled = true
+          resource_spec_type     = "STORAGE_BUCKET"
+        }
+      }
+    }
+  }
+}
+
+# tftest modules=1 resources=5
+```
+## TODO
+
+- [ ] Add IAM support
+- [ ] support different type of assets
+- [ ] support multi-regions
+<!-- BEGIN TFDOC -->
+
+## Variables
+
+| name | description | type | required | default |
+|---|---|:---:|:---:|:---:|
+| [name](variables.tf#L23) | Name of Dataplex Lake. | <code>string</code> | ✓ |  |
+| [prefix](variables.tf#L28) | Optional prefix used to generate Dataplex Lake. | <code>string</code> | ✓ |  |
+| [project_id](variables.tf#L33) | The ID of the project where this Dataplex Lake will be created. | <code>string</code> | ✓ |  |
+| [region](variables.tf#L38) | Region of the Dataplax Lake. | <code>string</code> | ✓ |  |
+| [zones](variables.tf#L43) | Dataplex lake zones, such as `RAW` and `CURATED`. | <code title="map&#40;object&#40;&#123;&#10;  type      &#61; string&#10;  discovery &#61; optional&#40;bool, true&#41;&#10;  assets &#61; map&#40;object&#40;&#123;&#10;    bucket_name            &#61; string&#10;    cron_schedule          &#61; optional&#40;string, &#34;15 15 &#42; &#42; &#42;&#34;&#41;&#10;    discovery_spec_enabled &#61; optional&#40;bool, true&#41;&#10;    resource_spec_type     &#61; optional&#40;string, &#34;STORAGE_BUCKET&#34;&#41;&#10;  &#125;&#41;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | ✓ |  |
+| [location_type](variables.tf#L17) | The location type of the Dataplax Lake. | <code>string</code> |  | <code>&#34;SINGLE_REGION&#34;</code> |
+
+## Outputs
+
+| name | description | sensitive |
+|---|---|:---:|
+| [assets](outputs.tf#L17) | Assets attached to the lake of Dataplex Lake. |  |
+| [lake](outputs.tf#L22) | The lake name of Dataplex Lake. |  |
+| [zones](outputs.tf#L27) | The zone name of Dataplex Lake. |  |
+
+<!-- END TFDOC -->

modules/cloud-dataplex/main.tf

@@ -0,0 +1,80 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+  prefix = var.prefix == null ? "" : "${var.prefix}-"
+  zone_assets = flatten([
+    for zone, zones_info in var.zones : [
+      for asset, asset_data in zones_info.assets : {
+        zone_name              = zone
+        asset_name             = asset
+        bucket_name            = asset_data.bucket_name
+        cron_schedule          = asset_data.cron_schedule
+        discovery_spec_enabled = asset_data.discovery_spec_enabled
+        resource_spec_type     = asset_data.resource_spec_type
+      }
+    ]
+  ])
+}
+
+resource "google_dataplex_lake" "basic_lake" {
+  name     = "${local.prefix}${var.name}"
+  location = var.region
+  provider = google-beta
+  project  = var.project_id
+}
+
+resource "google_dataplex_zone" "basic_zone" {
+  for_each = var.zones
+  name     = each.key
+  location = var.region
+  provider = google-beta
+  lake     = google_dataplex_lake.basic_lake.name
+  type     = each.value.type
+
+  discovery_spec {
+    enabled = each.value.discovery
+  }
+
+  resource_spec {
+    location_type = var.location_type
+  }
+
+  project = var.project_id
+}
+
+resource "google_dataplex_asset" "primary" {
+  for_each = {
+    for tm in local.zone_assets : "${tm.zone_name}-${tm.asset_name}" => tm
+  }
+  name     = each.value.asset_name
+  location = var.region
+  provider = google-beta
+
+  lake          = google_dataplex_lake.basic_lake.name
+  dataplex_zone = google_dataplex_zone.basic_zone[each.value.zone_name].name
+
+  discovery_spec {
+    enabled  = each.value.discovery_spec_enabled
+    schedule = each.value.cron_schedule
+  }
+
+  resource_spec {
+    name = "projects/${var.project_id}/buckets/${each.value.bucket_name}"
+    type = each.value.resource_spec_type
+  }
+  project = var.project_id
+}

modules/cloud-dataplex/outputs.tf

@@ -0,0 +1,31 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "assets" {
+  description = "Assets attached to the lake of Dataplex Lake."
+  value       = local.zone_assets[*]["asset_name"]
+}
+
+output "lake" {
+  description = "The lake name of Dataplex Lake."
+  value       = google_dataplex_lake.basic_lake.name
+}
+
+output "zones" {
+  description = "The zone name of Dataplex Lake."
+  value       = local.zone_assets[*]["zone_name"]
+}
+

modules/cloud-dataplex/variables.tf

@@ -0,0 +1,55 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "location_type" {
+  description = "The location type of the Dataplax Lake."
+  type        = string
+  default     = "SINGLE_REGION"
+}
+
+variable "name" {
+  description = "Name of Dataplex Lake."
+  type        = string
+}
+
+variable "prefix" {
+  description = "Optional prefix used to generate Dataplex Lake."
+  type        = string
+}
+
+variable "project_id" {
+  description = "The ID of the project where this Dataplex Lake will be created."
+  type        = string
+}
+
+variable "region" {
+  description = "Region of the Dataplax Lake."
+  type        = string
+}
+
+variable "zones" {
+  description = "Dataplex lake zones, such as `RAW` and `CURATED`."
+  type = map(object({
+    type      = string
+    discovery = optional(bool, true)
+    assets = map(object({
+      bucket_name            = string
+      cron_schedule          = optional(string, "15 15 * * *")
+      discovery_spec_enabled = optional(bool, true)
+      resource_spec_type     = optional(string, "STORAGE_BUCKET")
+    }))
+  }))
+}

modules/cloud-dataplex/versions.tf

@@ -0,0 +1,29 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+terraform {
+  required_version = ">= 1.0.0"
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = ">= 4.47.0" # tftest
+    }
+    google-beta = {
+      source  = "hashicorp/google-beta"
+      version = ">= 4.47.0" # tftest
+    }
+  }
+}
+
+

tests/modules/cloud_dataplex/examples/dataplex_lake.tfvars

@@ -0,0 +1,30 @@
+name       = "terraform-lake"
+prefix     = "test"
+project_id = "myproject"
+region     = "europe-west2"
+zones = {
+  zone_1 = {
+    type      = "RAW"
+    discovery = true
+    assets = {
+      asset_1 = {
+        bucket_name            = "asset_1"
+        cron_schedule          = "15 15 * * *"
+        discovery_spec_enabled = true
+        resource_spec_type     = "STORAGE_BUCKET"
+      }
+    }
+  },
+  zone_2 = {
+    type      = "CURATED"
+    discovery = true
+    assets = {
+      asset_2 = {
+        bucket_name            = "asset_2"
+        cron_schedule          = "15 15 * * *"
+        discovery_spec_enabled = true
+        resource_spec_type     = "STORAGE_BUCKET"
+      }
+    }
+  }
+}