-
Notifications
You must be signed in to change notification settings - Fork 914
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* First commit. * Implement fixes. * fix google_dataplex_zone_iam_binding
- Loading branch information
Showing
6 changed files
with
151 additions
and
33 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,7 +1,6 @@ | ||
| # Cloud Dataplex instance with lake, zone & assests | ||
|
|
||
| This module manages the creation of Cloud Dataplex instance along with lake, zone & assets in single regions. | ||
|
|
||
| This module manages the creation of Cloud Dataplex instance along with lake, zone & assets in single regions. | ||
|
|
||
| ## Simple example | ||
|
|
||
|
|
@@ -16,52 +15,111 @@ module "dataplex" { | |
| project_id = "myproject" | ||
| region = "europe-west2" | ||
| zones = { | ||
| zone_1 = { | ||
| landing = { | ||
| type = "RAW" | ||
| discovery = true | ||
| assets = { | ||
| asset_1 = { | ||
| bucket_name = "asset_1" | ||
| gcs_1 = { | ||
| resource_name = "gcs_bucket" | ||
| cron_schedule = "15 15 * * *" | ||
| discovery_spec_enabled = true | ||
| resource_spec_type = "STORAGE_BUCKET" | ||
| } | ||
| } | ||
| }, | ||
| zone_2 = { | ||
| curated = { | ||
| type = "CURATED" | ||
| discovery = false | ||
| assets = { | ||
| bq_1 = { | ||
| resource_name = "bq_dataset" | ||
| cron_schedule = null | ||
| discovery_spec_enabled = false | ||
| resource_spec_type = "BIGQUERY_DATASET" | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| # tftest modules=1 resources=5 | ||
| ``` | ||
|
|
||
| ## IAM | ||
|
|
||
| This example shows how to setup a Cloud Dataplex instance, lake, zone & asset creation in GCP project assigning IAM roles at lake and zone level. | ||
|
|
||
| ```hcl | ||
| module "dataplex" { | ||
| source = "./fabric/modules/cloud-dataplex" | ||
| name = "lake" | ||
| prefix = "test" | ||
| project_id = "myproject" | ||
| region = "europe-west2" | ||
| iam = { | ||
| "roles/dataplex.viewer" = [ | ||
| "group:[email protected]", | ||
| "group:[email protected]" | ||
| ] | ||
| } | ||
| zones = { | ||
| landing = { | ||
| type = "RAW" | ||
| discovery = true | ||
| assets = { | ||
| asset_2 = { | ||
| bucket_name = "asset_2" | ||
| gcs_1 = { | ||
| resource_name = "gcs_bucket" | ||
| cron_schedule = "15 15 * * *" | ||
| discovery_spec_enabled = true | ||
| resource_spec_type = "STORAGE_BUCKET" | ||
| } | ||
| } | ||
| }, | ||
| curated = { | ||
| type = "CURATED" | ||
| discovery = false | ||
| iam = { | ||
| "roles/viewer" = [ | ||
| "group:[email protected]", | ||
| "group:[email protected]" | ||
| ] | ||
| "roles/dataplex.dataReader" = [ | ||
| "group:[email protected]", | ||
| "group:[email protected]" | ||
| ] | ||
| } | ||
| assets = { | ||
| bq_1 = { | ||
| resource_name = "bq_dataset" | ||
| cron_schedule = null | ||
| discovery_spec_enabled = false | ||
| resource_spec_type = "BIGQUERY_DATASET" | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| # tftest modules=1 resources=5 | ||
| # tftest modules=1 resources=8 | ||
| ``` | ||
|
|
||
| ## TODO | ||
|
|
||
| - [ ] Add IAM support | ||
| - [ ] support different type of assets | ||
| - [ ] support multi-regions | ||
| <!-- BEGIN TFDOC --> | ||
|
|
||
| ## Variables | ||
|
|
||
| | name | description | type | required | default | | ||
| |---|---|:---:|:---:|:---:| | ||
| | [name](variables.tf#L23) | Name of Dataplex Lake. | <code>string</code> | ✓ | | | ||
| | [prefix](variables.tf#L28) | Optional prefix used to generate Dataplex Lake. | <code>string</code> | ✓ | | | ||
| | [project_id](variables.tf#L33) | The ID of the project where this Dataplex Lake will be created. | <code>string</code> | ✓ | | | ||
| | [region](variables.tf#L38) | Region of the Dataplax Lake. | <code>string</code> | ✓ | | | ||
| | [zones](variables.tf#L43) | Dataplex lake zones, such as `RAW` and `CURATED`. | <code title="map(object({ type = string discovery = optional(bool, true) assets = map(object({ bucket_name = string cron_schedule = optional(string, "15 15 * * *") discovery_spec_enabled = optional(bool, true) resource_spec_type = optional(string, "STORAGE_BUCKET") })) }))">map(object({…}))</code> | ✓ | | | ||
| | [location_type](variables.tf#L17) | The location type of the Dataplax Lake. | <code>string</code> | | <code>"SINGLE_REGION"</code> | | ||
| | [name](variables.tf#L30) | Name of Dataplex Lake. | <code>string</code> | ✓ | | | ||
| | [project_id](variables.tf#L41) | The ID of the project where this Dataplex Lake will be created. | <code>string</code> | ✓ | | | ||
| | [region](variables.tf#L46) | Region of the Dataplax Lake. | <code>string</code> | ✓ | | | ||
| | [zones](variables.tf#L51) | Dataplex lake zones, such as `RAW` and `CURATED`. | <code title="map(object({ type = string discovery = optional(bool, true) iam = optional(map(list(string)), null) assets = map(object({ resource_name = string resource_project = optional(string) cron_schedule = optional(string, "15 15 * * *") discovery_spec_enabled = optional(bool, true) resource_spec_type = optional(string, "STORAGE_BUCKET") })) }))">map(object({…}))</code> | ✓ | | | ||
| | [iam](variables.tf#L17) | Dataplex lake IAM bindings in {ROLE => [MEMBERS]} format. | <code>map(list(string))</code> | | <code>{}</code> | | ||
| | [location_type](variables.tf#L24) | The location type of the Dataplax Lake. | <code>string</code> | | <code>"SINGLE_REGION"</code> | | ||
| | [prefix](variables.tf#L35) | Optional prefix used to generate Dataplex Lake. | <code>string</code> | | <code>null</code> | | ||
|
|
||
| ## Outputs | ||
|
|
||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -155,6 +155,7 @@ | |
| service_agent: "service-%[email protected]" | ||
| - name: "dataplex" | ||
| service_agent: "service-%[email protected]" | ||
| jit: true # roles/dataplex.serviceAgent | ||
| - name: "dataproc" | ||
| service_agent: "service-%[email protected]" | ||
| - name: "datastream" | ||
|
|
||