GoogleCloudPlatform · Francesco-cloud24 · Nov 16, 2023 · Nov 16, 2023 · Nov 16, 2023 · Nov 16, 2023
diff --git a/modules/composer/README.md b/modules/composer/README.md
@@ -0,0 +1,95 @@
+# Google Cloud Composer
+This module Manages a Google Cloud [Composer](https://cloud.google.com/composer) environment.
+
+<!-- BEGIN TOC -->
+- [Examples](#examples)
+  - [Simple](#simple)
+  - [Node confiugration](#node-confiugration)
+  - [Private environment confiugration](#private-environment-confiugration)
+- [Variables](#variables)
+- [Outputs](#outputs)
+<!-- END TOC -->
+
+## Examples
+
+### Simple
+```hcl
+module "composer-environment" {
+  source     = "./fabric/modules/composer"
+  project_id = "my-project"
+  name       = "my-env"
+  region     = "europe-west1"
+}
+# tftest modules=1 resources=1
+```
+### Node confiugration
+It is possible to configure nodes using the variable [node_config](variables.tf#L36).
+```hcl
+module "composer-environment" {
+  source     = "./fabric/modules/composer"
+  project_id = "my-project"
+  name       = "my-env"
+  region     = "europe-west1"
+  node_config = {
+    zone            = "europe-west1"
+    machine_type    = "n2-standard-4"
+    network         = ""
+    subnetwork      = "projects/PROJECT/regions/europe-west1/subnetworks/SUBNET"
+    disk_size_gb    = 50
+    ip_allocation_policy = [{
+      use_ip_aliases = true
+      cluster_secondary_range_name  = "composer-pod"
+      services_secondary_range_name = "composer-services"
+    }]
+  }
+}
+# tftest modules=1 resources=1
+```
+
+### Private environment confiugration
+To set private environment configuration set [private_environment_config](variables.tf#L85)`.enable_private_endpoint` at `true`, configure the master, sql and web server cidr blocks.
+```hcl
+module "composer-environment" {
+  source     = "./fabric/modules/composer"
+  project_id = "my-project"
+  name       = "my-env"
+  region     = "europe-west1"
+  private_environment_config = {
+    enable_private_endpoint    = true
+    master_ipv4_cidr_block     = "10.10.10.0/28"
+    cloud_sql_ipv4_cidr_block  = "10.10.11.0/24"
+    web_server_ipv4_cidr_block = "10.10.12.0/28"
+  }
+}
+# tftest modules=1 resources=1
+```
+<!-- BEGIN TFDOC -->
+## Variables
+
+| name | description | type | required | default |
+|---|---|:---:|:---:|:---:|
+| [name](variables.tf#L70) | Name of the environment. | <code>string</code> | ✓ |  |
+| [project_id](variables.tf#L117) | The ID of the project in which the resource belongs. | <code>string</code> | ✓ |  |
+| [region](variables.tf#L135) | The location or Compute Engine region for the environment. | <code>string</code> | ✓ |  |
+| [allowed_ip_range](variables.tf#L17) | The network-level access control policy for the Airflow web server. If unspecified, no network-level access restrictions are applied. For Cloud Composer 1 only. | <code title="object&#40;&#123;&#10;  value       &#61; string&#10;  description &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
+| [database_config](variables.tf#L26) | The configuration settings for Cloud SQL instance used internally by Apache Airflow software. For Cloud Composer 1 only. | <code title="object&#40;&#123;&#10;  machine_type &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
+| [encryption_config](variables.tf#L34) | The encryption options for the Cloud Composer environment and its dependencies. | <code title="object&#40;&#123;&#10;  kms_key_name &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
+| [environment_size](variables.tf#L42) | The environment size controls the performance parameters of the managed Cloud Composer infrastructure that includes the Airflow database. Values for environment size are ENVIRONMENT_SIZE_SMALL, ENVIRONMENT_SIZE_MEDIUM, and ENVIRONMENT_SIZE_LARGE. | <code>string</code> |  | <code>&#34;ENVIRONMENT_SIZE_SMALL&#34;</code> |
+| [maintenance_window](variables.tf#L48) | The configuration settings for Cloud Composer maintenance windows. BETA FOR Cloud Composer 1. | <code title="object&#40;&#123;&#10;  start_time &#61; string&#10;  end_time   &#61; string&#10;  recurrence &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
+| [master_authorized_networks_config](variables.tf#L58) | Configuration options for the master authorized networks feature. Enabled master authorized networks will disallow all external traffic to access Kubernetes master through HTTPS except traffic from the given CIDR blocks, Google Compute Engine Public IPs and Google Prod IPs. | <code title="object&#40;&#123;&#10;  display_name &#61; bool&#10;  cidr_blocks &#61; object&#40;&#123;&#10;    display_name &#61; optional&#40;string&#41;&#10;    cidr_block   &#61; string&#10;  &#125;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
+| [node_config](variables.tf#L75) | The configuration used for the Kubernetes Engine cluster. | <code title="object&#40;&#123;&#10;  zone            &#61; optional&#40;string&#41;&#10;  machine_type    &#61; optional&#40;string&#41;&#10;  network         &#61; optional&#40;string&#41;&#10;  subnetwork      &#61; optional&#40;string&#41;&#10;  disk_size_gb    &#61; optional&#40;number&#41;&#10;  oauth_scopes    &#61; optional&#40;list&#40;string&#41;&#41;&#10;  service_account &#61; optional&#40;string&#41;&#10;  tags            &#61; optional&#40;list&#40;string&#41;&#41;&#10;  ip_allocation_policy &#61; optional&#40;list&#40;object&#40;&#123;&#10;    use_ip_aliases                &#61; bool&#10;    cluster_secondary_range_name  &#61; optional&#40;string&#41;&#10;    services_secondary_range_name &#61; optional&#40;string&#41;&#10;    cluster_ipv4_cidr_block       &#61; optional&#40;string&#41;&#10;    services_ipv4_cidr_block      &#61; optional&#40;string&#41;&#10;  &#125;&#41;&#41;&#41;&#10;  enable_ip_masq_agent &#61; optional&#40;bool&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
+| [node_count](variables.tf#L98) | The number of nodes in the Kubernetes Engine cluster of the environment. For Cloud Composer 1 only. | <code>number</code> |  | <code>3</code> |
+| [private_environment_config](variables.tf#L104) | The configuration used for the Private IP Cloud Composer environment. | <code title="object&#40;&#123;&#10;  connection_type                  &#61; optional&#40;string&#41;&#10;  enable_private_endpoint          &#61; optional&#40;bool&#41;&#10;  master_ipv4_cidr_block           &#61; optional&#40;string&#41;&#10;  cloud_sql_ipv4_cidr_block        &#61; optional&#40;string&#41;&#10;  web_server_ipv4_cidr_block       &#61; optional&#40;string&#41;&#10;  enable_privately_used_public_ips &#61; optional&#40;bool&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
+| [recovery_config](variables.tf#L122) | The configuration settings for recovery, for Cloud Composer 2 only. | <code title="object&#40;&#123;&#10;  scheduled_snapshots_config &#61; object&#40;&#123;&#10;    enabled                    &#61; bool&#10;    snapshot_location          &#61; optional&#40;string&#41;&#10;    snapshot_creation_schedule &#61; optional&#40;string&#41;&#10;    time_zone                  &#61; optional&#40;string&#41;&#10;  &#125;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
+| [resilience_mode](variables.tf#L140) | The resilience mode states whether high resilience is enabled for the environment or not. Values for resilience mode are HIGH_RESILIENCE for high resilience and STANDARD_RESILIENCE for standard resilience. For Cloud Composer 2.1.15 or newer only. | <code>string</code> |  | <code>null</code> |
+| [software_config](variables.tf#L146) | The configuration settings for software inside the environment. | <code title="object&#40;&#123;&#10;  airflow_config_overrides &#61; optional&#40;map&#40;string&#41;&#41;&#10;  pypi_packages            &#61; optional&#40;map&#40;string&#41;&#41;&#10;  env_variables            &#61; optional&#40;map&#40;string&#41;&#41;&#10;  image_version            &#61; optional&#40;string&#41;&#10;  python_version           &#61; optional&#40;string&#41;&#10;  scheduler_count          &#61; optional&#40;number&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
+| [web_server_config](variables.tf#L159) | The configuration settings for the Airflow web server App Engine instance. For Cloud Composer 1 only. | <code title="object&#40;&#123;&#10;  machine_type &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
+| [workloads_config](variables.tf#L167) | The Kubernetes workloads configuration for GKE cluster associated with the Cloud Composer environment. For Cloud Composer 2 only. | <code title="object&#40;&#123;&#10;  scheduler &#61; optional&#40;object&#40;&#123;&#10;    cpu        &#61; optional&#40;number&#41;&#10;    count      &#61; optional&#40;number&#41;&#10;    memory_gb  &#61; optional&#40;number&#41;&#10;    storage_gb &#61; optional&#40;number&#41;&#10;  &#125;&#41;&#41;&#10;  triggerer &#61; optional&#40;object&#40;&#123;&#10;    cpu       &#61; optional&#40;number&#41;&#10;    count     &#61; optional&#40;number&#41;&#10;    memory_gb &#61; optional&#40;number&#41;&#10;  &#125;&#41;&#41;&#10;  web_server &#61; optional&#40;object&#40;&#123;&#10;    cpu        &#61; optional&#40;number&#41;&#10;    storage_gb &#61; optional&#40;number&#41;&#10;    memory_gb  &#61; optional&#40;number&#41;&#10;  &#125;&#41;&#41;&#10;  worker &#61; optional&#40;object&#40;&#123;&#10;    cpu        &#61; optional&#40;number&#41;&#10;    storage_gb &#61; optional&#40;number&#41;&#10;    memory_gb  &#61; optional&#40;number&#41;&#10;    min_count  &#61; optional&#40;number&#41;&#10;    max_count  &#61; optional&#40;number&#41;&#10;  &#125;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
+
+## Outputs
+
+| name | description | sensitive |
+|---|---|:---:|
+| [composer-gke-cluster](outputs.tf#L17) | The Kubernetes Engine cluster used to run this environment. |  |
+| [composer-id](outputs.tf#L22) | An identifier for the resource with format projects/{{project}}/locations/{{region}}/environments/{{name}}. |  |
+<!-- END TFDOC -->
diff --git a/modules/composer/main.tf b/modules/composer/main.tf
@@ -0,0 +1,198 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+  image         = try(split("-", var.software_config.image_version), "")
+  version       = try(split(".", local.image.1), null)
+  major_version = try(local.version.0, 2)
+  mid_version   = try(local.version.1, 1)
+  min_version   = try(local.version.2, 15)
+
+  node_count       = local.major_version == 1 ? var.node_count : null
+  environment_size = local.major_version == 2 ? var.environment_size : null
+  resilience_mode  = (local.major_version == 2 && local.mid_version >= 1 && local.min_version >= 15) ? var.resilience_mode : null
+
+}
+
+
+resource "google_composer_environment" "composer-env" {
+  name    = var.name
+  region  = var.region
+  project = var.project_id
+
+  config {
+    node_count       = local.node_count       # ONLY COMPOSER 1
+    environment_size = local.environment_size # ONLY COMPOSER 2
+    resilience_mode  = local.resilience_mode  # ONLY COMPOSER >= 2.1.15
+
+    dynamic "node_config" {
+      for_each = var.node_config != null ? [""] : []
+      content {
+        zone                 = var.node_config.zone
+        machine_type         = var.node_config.machine_type
+        network              = var.node_config.network
+        subnetwork           = var.node_config.subnetwork
+        disk_size_gb         = var.node_config.disk_size_gb
+        oauth_scopes         = var.node_config.oauth_scopes
+        service_account      = var.node_config.service_account
+        tags                 = var.node_config.tags
+        ip_allocation_policy = var.node_config.ip_allocation_policy
+        # max_pods_per_node = "" # BETA FEATURES - DISABLED
+        enable_ip_masq_agent = var.node_config.enable_ip_masq_agent
+      }
+    }
+
+    dynamic "recovery_config" {                                                        # ONLY COMPOSER 2
+      for_each = (var.recovery_config != null && local.major_version == 2) ? [""] : [] # max 1
+      content {
+        dynamic "scheduled_snapshots_config" {
+          for_each = var.recovery_config.scheduled_snapshots_config != null ? [""] : [] # max 1
+          content {
+            enabled                    = var.recovery_config.scheduled_snapshots_config.enabled
+            snapshot_location          = var.recovery_config.scheduled_snapshots_config.snapshot_location
+            snapshot_creation_schedule = var.recovery_config.scheduled_snapshots_config.snapshot_creation_schedule
+            time_zone                  = var.recovery_config.scheduled_snapshots_config.time_zone
+          }
+        }
+      }
+    }
+
+    dynamic "software_config" {
+      for_each = var.software_config != null ? [""] : [] # max 1
+      content {
+        airflow_config_overrides = var.software_config.airflow_config_overrides
+        pypi_packages            = var.software_config.pypi_packages
+        env_variables            = var.software_config.env_variables
+        image_version            = var.software_config.image_version
+        python_version           = var.software_config.python_version
+        scheduler_count          = var.software_config.scheduler_count
+      }
+    }
+
+    dynamic "private_environment_config" {
+      for_each = var.private_environment_config != null ? [""] : [] # max 1
+      content {
+        connection_type                  = var.private_environment_config.connection_type
+        enable_private_endpoint          = var.private_environment_config.enable_private_endpoint
+        master_ipv4_cidr_block           = var.private_environment_config.master_ipv4_cidr_block
+        cloud_sql_ipv4_cidr_block        = var.private_environment_config.cloud_sql_ipv4_cidr_block
+        web_server_ipv4_cidr_block       = var.private_environment_config.web_server_ipv4_cidr_block
+        enable_privately_used_public_ips = var.private_environment_config.enable_privately_used_public_ips
+      }
+    }
+
+    dynamic "web_server_network_access_control" {                                       # ONLY COMPOSER 1
+      for_each = (var.allowed_ip_range != null && local.major_version == 1) ? [""] : [] # max 1
+      content {
+        dynamic "allowed_ip_range" {
+          for_each = var.allowed_ip_range
+          content {
+            value       = allowed_ip_range.value
+            description = allowed_ip_range.description
+          }
+
+        }
+      }
+    }
+
+    dynamic "database_config" {                                                        # ONLY COMPOSER 1
+      for_each = (var.database_config != null && local.major_version == 1) ? [""] : [] # max 1
+      content {
+        machine_type = var.database_config.machine_type
+      }
+    }
+
+    dynamic "web_server_config" {
+      for_each = (var.web_server_config != null && local.major_version == 1) ? [""] : [] # max 1
+      content {
+        machine_type = var.web_server_config.machine_type
+      }
+    }
+
+    dynamic "encryption_config" {
+      for_each = var.encryption_config != null ? [""] : [] # max 1
+      content {
+        kms_key_name = var.encryption_config.kms_key_name
+      }
+    }
+
+    dynamic "maintenance_window" {                                                        # ONLY COMPOSER 2, BETA FOR COMPOSER 1
+      for_each = (var.maintenance_window != null && local.major_version == 2) ? [""] : [] # max 1
+      content {
+        start_time = var.maintenance_window.start_time
+        end_time   = var.maintenance_window.end_time
+        recurrence = var.maintenance_window.recurrence
+      }
+    }
+
+    dynamic "master_authorized_networks_config" {
+      for_each = var.master_authorized_networks_config != null ? [""] : [] # max 1
+      content {
+        enabled = true
+        dynamic "cidr_blocks" {
+          for_each = var.master_authorized_networks_config.cidr_blocks
+          content {
+            display_name = cidr_blocks.display_name
+            cidr_block   = cidr_blocks.cidr_block
+          }
+        }
+      }
+    }
+
+    dynamic "workloads_config" {                                                        # ONLY COMPOSER 2
+      for_each = (var.workloads_config != null && local.major_version == 2) ? [""] : [] # max 1
+      content {
+        dynamic "scheduler" {
+          for_each = var.workloads_config.scheduler != null ? [""] : [] # max 1
+          content {
+            cpu        = var.workloads_config.scheduler.cpu
+            count      = var.workloads_config.scheduler.count
+            memory_gb  = var.workloads_config.scheduler.memory_gb
+            storage_gb = var.workloads_config.scheduler.storage_gb
+          }
+        }
+        dynamic "triggerer" {
+          for_each = var.workloads_config.triggerer != null ? [""] : [] # max 1
+          content {
+            cpu       = var.workloads_config.triggerer.cpu
+            memory_gb = var.workloads_config.triggerer.memory_gb
+            count     = var.workloads_config.triggerer.count
+          }
+        }
+        dynamic "web_server" {
+          for_each = var.workloads_config.web_server != null ? [""] : [] # max 1
+          content {
+            cpu        = var.workloads_config.web_server.cpu
+            memory_gb  = var.workloads_config.web_server.memory_gb
+            storage_gb = var.workloads_config.web_server.storage_gb
+          }
+        }
+        dynamic "worker" {
+          for_each = var.workloads_config.web_server != null ? [""] : [] # max 1
+          content {
+            cpu        = var.workloads_config.worker.cpu
+            memory_gb  = var.workloads_config.worker.memory_gb
+            storage_gb = var.workloads_config.worker.storage_gb
+            min_count  = var.workloads_config.worker.min_count
+            max_count  = var.workloads_config.worker.max_count
+          }
+        }
+      }
+    }
+  }
+}
+
+
diff --git a/modules/composer/outputs.tf b/modules/composer/outputs.tf
@@ -0,0 +1,25 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "composer-gke-cluster" {
+  description = "The Kubernetes Engine cluster used to run this environment."
+  value       = google_composer_environment.composer-env.config.0.gke_cluster
+}
+
+output "composer-id" {
+  description = "An identifier for the resource with format projects/{{project}}/locations/{{region}}/environments/{{name}}."
+  value       = google_composer_environment.composer-env.id
+}