From 26d8f0cf65bf2a1ee5c1ba5bd2deb0152504822d Mon Sep 17 00:00:00 2001 From: Lorenzo Caggioni Date: Mon, 20 Jun 2022 09:11:09 +0200 Subject: [PATCH 1/4] Fix stage-02 tfvars links --- fast/stages/02-networking-nva/README.md | 4 ++-- fast/stages/02-networking-peering/README.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fast/stages/02-networking-nva/README.md b/fast/stages/02-networking-nva/README.md index fd9b2831f5..ed3c8309ac 100644 --- a/fast/stages/02-networking-nva/README.md +++ b/fast/stages/02-networking-nva/README.md @@ -225,8 +225,8 @@ If you have set a valid value for `outputs_location` in the bootstrap and in the ```bash # `outputs_location` is set to `~/fast-config` -ln -s ../../configs/example/02-networking/terraform-bootstrap.auto.tfvars.json -ln -s ../../configs/example/02-networking/terraform-resman.auto.tfvars.json +ln -s ~/fast-config/tfvars/00-bootstrap.auto.tfvars.json . +ln -s ~/fast-config/tfvars/01-resman.auto.tfvars.json . # also copy the tfvars file used for the bootstrap stage cp ../00-bootstrap/terraform.tfvars . ``` diff --git a/fast/stages/02-networking-peering/README.md b/fast/stages/02-networking-peering/README.md index 316caf7e99..a4beab1ad5 100644 --- a/fast/stages/02-networking-peering/README.md +++ b/fast/stages/02-networking-peering/README.md @@ -155,8 +155,8 @@ If you have set a valid value for `outputs_location` in the bootstrap and in the ```bash # `outputs_location` is set to `~/fast-config` -ln -s ../../configs/example/02-networking/terraform-bootstrap.auto.tfvars.json -ln -s ../../configs/example/02-networking/terraform-resman.auto.tfvars.json +ln -s ~/fast-config/tfvars/00-bootstrap.auto.tfvars.json . +ln -s ~/fast-config/tfvars/01-resman.auto.tfvars.json . # also copy the tfvars file used for the bootstrap stage cp ../00-bootstrap/terraform.tfvars . ``` From d4eef718879bda88d93d2dcb7ef7edc9d28b0669 Mon Sep 17 00:00:00 2001 From: Lorenzo Caggioni Date: Mon, 20 Jun 2022 09:52:46 +0200 Subject: [PATCH 2/4] Fix links --- fast/stages/03-data-platform/dev/README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/fast/stages/03-data-platform/dev/README.md b/fast/stages/03-data-platform/dev/README.md index 7bc41760f5..af462a3989 100644 --- a/fast/stages/03-data-platform/dev/README.md +++ b/fast/stages/03-data-platform/dev/README.md @@ -108,6 +108,14 @@ If you're running this on top of Fast, you should run the following commands to ln -s ~/fast-config/providers/03-data-platform-dev-providers.tf . ``` +If you have not configured `outputs_location` in bootstrap, you can derive the providers file from that stage's outputs: + +```bash +cd ../../01-resman +terraform output -json providers | jq -r '.["03-data-platform-dev"]' \ + > ../03-data-platform/dev/providers.tf +``` + ### Variable configuration There are two broad sets of variables that can be configured: @@ -124,6 +132,8 @@ If you configured a valid path for `outputs_location` in the bootstrap security ln -s ~/fast-config/tfvars/00-bootstrap.auto.tfvars.json . ln -s ~/fast-config/tfvars/01-resman.auto.tfvars.json . ln -s ~/fast-config/tfvars/02-networking.auto.tfvars.json . +# also copy the tfvars file used for the bootstrap stage +cp ../../00-bootstrap/terraform.tfvars . ``` If you're not using FAST or its output files, refer to the [Variables](#variables) table at the bottom of this document for a full list of variables, their origin (e.g., a stage or specific to this one), and descriptions explaining their meaning. From 0b6f98db781d6f4f9904488d4dd2d0bc445ed260 Mon Sep 17 00:00:00 2001 From: Lorenzo Caggioni Date: Mon, 20 Jun 2022 17:48:39 +0200 Subject: [PATCH 3/4] Add output logic --- fast/stages/03-data-platform/dev/README.md | 57 ++++++++++--------- fast/stages/03-data-platform/dev/outputs.tf | 25 ++++++++ fast/stages/03-data-platform/dev/variables.tf | 8 +++ 3 files changed, 62 insertions(+), 28 deletions(-) diff --git a/fast/stages/03-data-platform/dev/README.md b/fast/stages/03-data-platform/dev/README.md index af462a3989..9174ae6e92 100644 --- a/fast/stages/03-data-platform/dev/README.md +++ b/fast/stages/03-data-platform/dev/README.md @@ -156,43 +156,44 @@ You can find examples in the `[demo](../../../../examples/data-solutions/data-pl ## Files -| name | description | modules | -|---|---|---| -| [main.tf](./main.tf) | Data Platformy. | data-platform-foundations | -| [outputs.tf](./outputs.tf) | Output variables. | | -| [variables.tf](./variables.tf) | Terraform Variables. | | +| name | description | modules | resources | +|---|---|---|---| +| [main.tf](./main.tf) | Data Platformy. | data-platform-foundations | | +| [outputs.tf](./outputs.tf) | Output variables. | | google_storage_bucket_object · local_file | +| [variables.tf](./variables.tf) | Terraform Variables. | | | ## Variables | name | description | type | required | default | producer | |---|---|:---:|:---:|:---:|:---:| -| [billing_account](variables.tf#L17) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | ✓ | | 00-globals | -| [folder_ids](variables.tf#L56) | Folder to be used for the networking resources in folders/nnnn format. | object({…}) | ✓ | | 01-resman | -| [host_project_ids](variables.tf#L74) | Shared VPC project ids. | object({…}) | ✓ | | 02-networking | -| [organization](variables.tf#L100) | Organization details. | object({…}) | ✓ | | 00-globals | -| [prefix](variables.tf#L116) | Unique prefix used for resource names. Not used for projects if 'project_create' is null. | string | ✓ | | 00-globals | -| [composer_config](variables.tf#L26) | | object({…}) | | {…} | | -| [data_catalog_tags](variables.tf#L39) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {…} | | -| [data_force_destroy](variables.tf#L50) | Flag to set 'force_destroy' on data services like BigQery or Cloud Storage. | bool | | false | | -| [groups](variables.tf#L64) | Groups. | map(string) | | {…} | | -| [network_config_composer](variables.tf#L82) | Network configurations to use for Composer. | object({…}) | | {…} | | -| [outputs_location](variables.tf#L110) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | -| [project_services](variables.tf#L122) | List of core services enabled on all projects. | list(string) | | […] | | -| [region](variables.tf#L133) | Region used for regional resources. | string | | "europe-west1" | | -| [service_encryption_keys](variables.tf#L139) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null | | -| [subnet_self_links](variables.tf#L151) | Shared VPC subnet self links. | object({…}) | | null | 02-networking | -| [vpc_self_links](variables.tf#L160) | Shared VPC self links. | object({…}) | | null | 02-networking | +| [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 00-bootstrap | +| [billing_account](variables.tf#L25) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | ✓ | | 00-globals | +| [folder_ids](variables.tf#L64) | Folder to be used for the networking resources in folders/nnnn format. | object({…}) | ✓ | | 01-resman | +| [host_project_ids](variables.tf#L82) | Shared VPC project ids. | object({…}) | ✓ | | 02-networking | +| [organization](variables.tf#L108) | Organization details. | object({…}) | ✓ | | 00-globals | +| [prefix](variables.tf#L124) | Unique prefix used for resource names. Not used for projects if 'project_create' is null. | string | ✓ | | 00-globals | +| [composer_config](variables.tf#L34) | | object({…}) | | {…} | | +| [data_catalog_tags](variables.tf#L47) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {…} | | +| [data_force_destroy](variables.tf#L58) | Flag to set 'force_destroy' on data services like BigQery or Cloud Storage. | bool | | false | | +| [groups](variables.tf#L72) | Groups. | map(string) | | {…} | | +| [network_config_composer](variables.tf#L90) | Network configurations to use for Composer. | object({…}) | | {…} | | +| [outputs_location](variables.tf#L118) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | +| [project_services](variables.tf#L130) | List of core services enabled on all projects. | list(string) | | […] | | +| [region](variables.tf#L141) | Region used for regional resources. | string | | "europe-west1" | | +| [service_encryption_keys](variables.tf#L147) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null | | +| [subnet_self_links](variables.tf#L159) | Shared VPC subnet self links. | object({…}) | | null | 02-networking | +| [vpc_self_links](variables.tf#L168) | Shared VPC self links. | object({…}) | | null | 02-networking | ## Outputs | name | description | sensitive | consumers | |---|---|:---:|---| -| [bigquery_datasets](outputs.tf#L17) | BigQuery datasets. | | | -| [demo_commands](outputs.tf#L47) | Demo commands. | | | -| [gcs_buckets](outputs.tf#L22) | GCS buckets. | | | -| [kms_keys](outputs.tf#L27) | Cloud MKS keys. | | | -| [projects](outputs.tf#L32) | GCP Projects informations. | | | -| [vpc_network](outputs.tf#L37) | VPC network. | | | -| [vpc_subnet](outputs.tf#L42) | VPC subnetworks. | | | +| [bigquery_datasets](outputs.tf#L42) | BigQuery datasets. | | | +| [demo_commands](outputs.tf#L72) | Demo commands. | | | +| [gcs_buckets](outputs.tf#L47) | GCS buckets. | | | +| [kms_keys](outputs.tf#L52) | Cloud MKS keys. | | | +| [projects](outputs.tf#L57) | GCP Projects informations. | | | +| [vpc_network](outputs.tf#L62) | VPC network. | | | +| [vpc_subnet](outputs.tf#L67) | VPC subnetworks. | | | diff --git a/fast/stages/03-data-platform/dev/outputs.tf b/fast/stages/03-data-platform/dev/outputs.tf index 0820b64f64..346bf16b02 100644 --- a/fast/stages/03-data-platform/dev/outputs.tf +++ b/fast/stages/03-data-platform/dev/outputs.tf @@ -14,6 +14,31 @@ # tfdoc:file:description Output variables. +locals { + tfvars = { + bigquery_dataset = module.data-platform.bigquery-datasets + gcs_buckets = module.data-platform.gcs-buckets + projects = module.data-platform.projects + } +} + +# generate tfvars file for subsequent stages + +resource "local_file" "tfvars" { + for_each = var.outputs_location == null ? {} : { 1 = 1 } + file_permission = "0644" + filename = "${pathexpand(var.outputs_location)}/tfvars/03-data-platform-dev.auto.tfvars.json" + content = jsonencode(local.tfvars) +} + +resource "google_storage_bucket_object" "tfvars" { + bucket = var.automation.outputs_bucket + name = "tfvars/03-data-platform-dev.auto.tfvars.json" + content = jsonencode(local.tfvars) +} + +# outputs + output "bigquery_datasets" { description = "BigQuery datasets." value = module.data-platform.bigquery-datasets diff --git a/fast/stages/03-data-platform/dev/variables.tf b/fast/stages/03-data-platform/dev/variables.tf index 3b1645e411..9939d19400 100644 --- a/fast/stages/03-data-platform/dev/variables.tf +++ b/fast/stages/03-data-platform/dev/variables.tf @@ -14,6 +14,14 @@ # tfdoc:file:description Terraform Variables. +variable "automation" { + # tfdoc:variable:source 00-bootstrap + description = "Automation resources created by the bootstrap stage." + type = object({ + outputs_bucket = string + }) +} + variable "billing_account" { # tfdoc:variable:source 00-globals description = "Billing account id and organization id ('nnnnnnnn' or null)." From 41d8847a60a8100f0108c3424242fd7cc0689c9d Mon Sep 17 00:00:00 2001 From: Lorenzo Caggioni Date: Mon, 20 Jun 2022 17:56:36 +0200 Subject: [PATCH 4/4] Fix test --- tests/fast/stages/s03_data_platform/fixture/main.tf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/fast/stages/s03_data_platform/fixture/main.tf b/tests/fast/stages/s03_data_platform/fixture/main.tf index e7fd8d4d43..bf9be2fd20 100644 --- a/tests/fast/stages/s03_data_platform/fixture/main.tf +++ b/tests/fast/stages/s03_data_platform/fixture/main.tf @@ -18,6 +18,9 @@ module "stage" { source = "../../../../../fast/stages/03-data-platform/dev/" + automation = { + outputs_bucket = "test" + } billing_account = { id = "012345-67890A-BCDEF0", organization_id = 123456