From e2cceabf6d5ad6dd819483e68d458dbfe3711947 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 24 Nov 2019 16:19:33 +0100 Subject: [PATCH 001/106] gke-cluster --- modules/gke-cluster/README.md | 0 modules/gke-cluster/main.tf | 197 +++++++++++++++++++++++++ modules/gke-cluster/outputs.tf | 38 +++++ modules/gke-cluster/variables.tf | 246 +++++++++++++++++++++++++++++++ 4 files changed, 481 insertions(+) create mode 100644 modules/gke-cluster/README.md create mode 100644 modules/gke-cluster/main.tf create mode 100644 modules/gke-cluster/outputs.tf create mode 100644 modules/gke-cluster/variables.tf diff --git a/modules/gke-cluster/README.md b/modules/gke-cluster/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/modules/gke-cluster/main.tf b/modules/gke-cluster/main.tf new file mode 100644 index 0000000000..9c57d67303 --- /dev/null +++ b/modules/gke-cluster/main.tf @@ -0,0 +1,197 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_container_cluster" "cluster" { + provider = google-beta + project = var.project_id + name = var.name + description = var.description + location = var.location + node_locations = length(var.node_locations) == 0 ? null : var.node_locations + min_master_version = var.min_master_version + network = var.network + subnetwork = var.subnetwork + logging_service = var.logging_service + monitoring_service = var.monitoring_service + resource_labels = var.labels + default_max_pods_per_node = var.default_max_pods_per_node + enable_binary_authorization = var.enable_binary_authorization + enable_intranode_visibility = var.enable_intranode_visibility + enable_shielded_nodes = var.enable_shielded_nodes + enable_tpu = var.enable_tpu + initial_node_count = 1 + remove_default_node_pool = true + + # node_config + + addons_config { + http_load_balancing { + disabled = ! var.addons.http_load_balancing + } + horizontal_pod_autoscaling { + disabled = ! var.addons.horizontal_pod_autoscaling + } + network_policy_config { + disabled = ! var.addons.network_policy_config + } + # beta addons + # cloudrun is dynamic as it tends to trigger cluster recreation on change + dynamic cloudrun_config { + for_each = var.addons.istio_config.enabled && var.addons.cloudrun_config ? [""] : [] + content { + disabled = false + } + } + istio_config { + disabled = ! var.addons.istio_config.enabled + auth = var.addons.istio_config.tls ? "AUTH_MUTUAL_TLS" : "AUTH_NONE" + } + } + + # TODO(ludomagno): support setting address ranges instead of range names + # https://www.terraform.io/docs/providers/google/r/container_cluster.html#cluster_ipv4_cidr_block + ip_allocation_policy { + cluster_secondary_range_name = var.secondary_range_pods + services_secondary_range_name = var.secondary_range_services + } + + # TODO(ludomagno): make optional, and support beta feature + # https://www.terraform.io/docs/providers/google/r/container_cluster.html#daily_maintenance_window + maintenance_policy { + daily_maintenance_window { + start_time = var.maintenance_start_time + } + } + + master_auth { + client_certificate_config { + issue_client_certificate = false + } + } + + dynamic master_authorized_networks_config { + for_each = length(var.master_authorized_ranges) == 0 ? [] : list(var.master_authorized_ranges) + iterator = ranges + content { + dynamic cidr_blocks { + for_each = ranges.value + iterator = range + content { + cidr_block = range.value + display_name = range.key + } + } + } + } + + dynamic network_policy { + for_each = var.addons.network_policy_config ? [""] : [] + content { + enabled = true + provider = "CALICO" + } + } + + dynamic private_cluster_config { + for_each = var.private_cluster ? list(var.private_cluster_config) : [] + iterator = config + content { + enable_private_nodes = config.value.enable_private_nodes + enable_private_endpoint = config.value.enable_private_endpoint + master_ipv4_cidr_block = config.value.master_ipv4_cidr_block + } + } + + # beta features + + dynamic authenticator_groups_config { + for_each = var.authenticator_security_group == null ? [] : [""] + content { + security_group = var.authenticator_security_group + } + } + + dynamic cluster_autoscaling { + for_each = var.cluster_autoscaling.enabled ? [var.cluster_autoscaling] : [] + iterator = config + content { + enabled = true + resource_limits { + resource_type = "cpu" + minimum = config.cpu_min + maximum = config.cpu_max + } + resource_limits { + resource_type = "memory" + minimum = config.memory_min + maximum = config.memory_max + } + } + } + + dynamic database_encryption { + for_each = var.database_encryption.enabled ? [var.database_encryption] : [] + iterator = config + content { + state = config.value.state + key_name = config.value.key_name + } + } + + dynamic pod_security_policy_config { + for_each = var.pod_security_policy != null ? [""] : [] + content { + enabled = var.pod_security_policy + } + } + + dynamic release_channel { + for_each = var.release_channel != null ? [""] : [] + content { + channel = var.release_channel + } + } + + dynamic resource_usage_export_config { + for_each = ( + var.resource_usage_export_config.enabled != null + && + var.resource_usage_export_config.dataset != null + ? [""] : [] + ) + content { + enable_network_egress_metering = var.resource_usage_export_config.enabled + bigquery_destination { + dataset_id = var.resource_usage_export_config.dataset + } + } + } + + dynamic vertical_pod_autoscaling { + for_each = var.vertical_pod_autoscaling == null ? [] : [""] + content { + enabled = var.vertical_pod_autoscaling + } + } + + dynamic workload_identity_config { + for_each = var.workload_identity ? [""] : [] + content { + identity_namespace = "${var.project_id}.svc.id.goog" + } + } + +} diff --git a/modules/gke-cluster/outputs.tf b/modules/gke-cluster/outputs.tf new file mode 100644 index 0000000000..f21b680955 --- /dev/null +++ b/modules/gke-cluster/outputs.tf @@ -0,0 +1,38 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the +); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an + BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cluster" { + description = "Cluster resource." + sensitive = true + value = google_container_cluster.cluster +} + +output "endpoint" { + description = "Cluster endpoint." + value = google_container_cluster.cluster.endpoint +} + +output "master_version" { + description = "Master version." + value = google_container_cluster.cluster.master_version +} + +output "name" { + description = "Cluster name." + value = google_container_cluster.cluster.name +} diff --git a/modules/gke-cluster/variables.tf b/modules/gke-cluster/variables.tf new file mode 100644 index 0000000000..c66e60fa3c --- /dev/null +++ b/modules/gke-cluster/variables.tf @@ -0,0 +1,246 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "addons" { + description = "Addons enabled in the cluster (true means enabled)." + type = object({ + horizontal_pod_autoscaling = bool + http_load_balancing = bool + network_policy_config = bool + cloudrun_config = bool + istio_config = object({ + enabled = bool + tls = bool + }) + }) + default = { + horizontal_pod_autoscaling = true + http_load_balancing = true + network_policy_config = false + cloudrun_config = false + istio_config = { + enabled = false + tls = false + } + } +} + +variable "authenticator_security_group" { + description = "RBAC security group for Google Groups for GKE, format is gke-security-groups@yourdomain.com." + type = string + default = null +} + +variable "cluster_autoscaling" { + description = "Enable and configure limits for Node Auto-Provisioning with Cluster Autoscaler." + type = object({ + enabled = bool + cpu_min = number + cpu_max = number + memory_min = number + memory_max = number + }) + default = { + enabled = false + cpu_min = 0 + cpu_max = 0 + memory_min = 0 + memory_max = 0 + } +} + +variable "database_encryption" { + description = "Enable and configure GKE application-layer secrets encryption." + type = object({ + enabled = bool + state = string + key_name = string + }) + default = { + enabled = false + state = "DECRYPTED" + key_name = null + } +} + +variable "default_max_pods_per_node" { + description = "Maximum number of pods per node in this cluster." + type = number + default = 110 +} + +variable "description" { + description = "Cluster description." + type = string + default = null +} + +variable "enable_binary_authorization" { + description = "Enable Google Binary Authorization." + type = bool + default = null +} + +variable "enable_intranode_visibility" { + description = "Enable intra-node visibility to make same node pod to pod traffic visible." + type = bool + default = null +} + +variable "enable_shielded_nodes" { + description = "Enable Shielded Nodes features on all nodes in this cluster." + type = bool + default = null +} + +variable "enable_tpu" { + description = "Enable Cloud TPU resources in this cluster." + type = bool + default = null +} + +variable "labels" { + description = "Cluster resource labels." + type = map(string) + default = null +} + +variable "location" { + description = "Cluster zone or region." + type = string +} + +variable "logging_service" { + description = "Logging service (disable with an empty string)." + type = string + default = "logging.googleapis.com/kubernetes" +} + +variable "maintenance_start_time" { + description = "Maintenance start time in RFC3339 format 'HH:MM', where HH is [00-23] and MM is [00-59] GMT." + type = string + default = "03:00" +} + +variable "master_authorized_ranges" { + description = "External Ip address ranges that can access the Kubernetes cluster master through HTTPS." + type = map(string) + default = {} +} + +variable "min_master_version" { + description = "Minimum version of the master, defaults to the version of the most recent official release." + type = string + default = null +} + +variable "monitoring_service" { + description = "Monitoring service (disable with an empty string)." + type = string + default = "monitoring.googleapis.com/kubernetes" +} + +variable "name" { + description = "Cluster name." + type = string +} + +variable "network" { + description = "Name or self link of the VPC used for the cluster. Use the self link for Shared VPC." + type = string +} + +variable "node_locations" { + description = "Zones in which the cluster's nodes are located." + type = list(string) + default = [] +} + +variable "pod_security_policy" { + description = "Enable the PodSecurityPolicy feature." + type = bool + default = null +} + +variable "private_cluster" { + description = "Enable private cluster." + type = bool + default = false +} + +variable "private_cluster_config" { + description = "Private cluster configuration." + type = object({ + enable_private_nodes = bool + enable_private_endpoint = bool + master_ipv4_cidr_block = string + }) + default = { + enable_private_nodes = true + enable_private_endpoint = false + master_ipv4_cidr_block = null + } +} + +variable "project_id" { + description = "Cluster project id." + type = string +} + +variable "release_channel" { + description = "Release channel for GKE upgrades." + type = string + default = null +} + +variable "resource_usage_export_config" { + description = "Configure the ResourceUsageExportConfig feature." + type = object({ + enabled = bool + dataset = string + }) + default = { + enabled = null + dataset = null + } +} + +variable "secondary_range_pods" { + description = "Subnet secondary range name used for pods." + type = string +} + +variable "secondary_range_services" { + description = "Subnet secondary range name used for services." + type = string +} + +variable "subnetwork" { + description = "VPC subnetwork name or self link." + type = string +} + +variable "vertical_pod_autoscaling" { + description = "Enable the Vertical Pod Autoscaling feature." + type = bool + default = null +} + +variable "workload_identity" { + description = "Enable the Workload Identity feature." + type = bool + default = true +} From 690fc8982fd4391d8137c11bf9cf2eae4d0ec6ac Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Mon, 25 Nov 2019 02:36:19 +0100 Subject: [PATCH 002/106] net-vpc module and tests --- modules/net-vpc/main.tf | 91 +++++++++++++++++ modules/net-vpc/outputs.tf | 69 +++++++++++++ modules/net-vpc/variables.tf | 94 ++++++++++++++++++ modules/net-vpc/versions.tf | 19 ++++ tests/conftest.py | 57 +++++++++++ tests/modules/net_vpc/__init__.py | 14 +++ tests/modules/net_vpc/conftest.py | 29 ++++++ .../net_vpc/fixtures/vpc-iam-bindings/main.tf | 99 +++++++++++++++++++ .../net_vpc/fixtures/vpc-standalone/main.tf | 47 +++++++++ .../net_vpc/fixtures/vpc-subnets/main.tf | 97 ++++++++++++++++++ .../modules/net_vpc/test_vpc_iam_bindings.py | 81 +++++++++++++++ tests/modules/net_vpc/test_vpc_standalone.py | 29 ++++++ tests/modules/net_vpc/test_vpc_subnets.py | 63 ++++++++++++ tests/requirements.txt | 2 +- 14 files changed, 790 insertions(+), 1 deletion(-) create mode 100644 modules/net-vpc/main.tf create mode 100644 modules/net-vpc/outputs.tf create mode 100644 modules/net-vpc/variables.tf create mode 100644 modules/net-vpc/versions.tf create mode 100644 tests/modules/net_vpc/__init__.py create mode 100644 tests/modules/net_vpc/conftest.py create mode 100644 tests/modules/net_vpc/fixtures/vpc-iam-bindings/main.tf create mode 100644 tests/modules/net_vpc/fixtures/vpc-standalone/main.tf create mode 100644 tests/modules/net_vpc/fixtures/vpc-subnets/main.tf create mode 100644 tests/modules/net_vpc/test_vpc_iam_bindings.py create mode 100644 tests/modules/net_vpc/test_vpc_standalone.py create mode 100644 tests/modules/net_vpc/test_vpc_subnets.py diff --git a/modules/net-vpc/main.tf b/modules/net-vpc/main.tf new file mode 100644 index 0000000000..f687adca02 --- /dev/null +++ b/modules/net-vpc/main.tf @@ -0,0 +1,91 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # map of log configs for all subnets to reduce the number of lookups needed + subnet_log_configs = { + for name, attrs in var.subnets : name => lookup(var.log_configs, name, {}) + } + # map of log config that reflects enable_flow_logs and sets defaults + log_configs = { + for name, attrs in var.subnets : name => ( + attrs.enable_flow_logs + ? [{ + for key, value in var.log_config_defaults : key => lookup( + local.subnet_log_configs[name], key, value + ) + }] + : [] + ) + } + # distinct is needed to make the expanding function argument work + iam_pairs = concat([], distinct([ + for subnet, roles in var.iam_roles : + [for role in roles : { subnet = subnet, role = role }] + ])...) + iam_keypairs = { + for pair in local.iam_pairs : + "${pair.subnet}-${pair.role}" => pair + } +} + +resource "google_compute_network" "network" { + project = var.project_id + name = var.name + description = var.description + auto_create_subnetworks = var.auto_create_subnetworks + routing_mode = var.routing_mode +} + +resource "google_compute_shared_vpc_host_project" "shared_vpc_host" { + count = var.shared_vpc_host ? 1 : 0 + project = var.project_id + depends_on = [google_compute_network.network] +} + +resource "google_compute_subnetwork" "subnetwork" { + for_each = var.subnets + project = var.project_id + network = google_compute_network.network.name + name = each.key + description = each.value.description + ip_cidr_range = each.value.ip_cidr_range + region = each.value.region + private_ip_google_access = each.value.private_ip_google_access + secondary_ip_range = [ + for name, range in each.value.secondary_ip_range : + { range_name = name, ip_cidr_range = range } + ] + dynamic "log_config" { + for_each = local.log_configs[each.key] + content { + aggregation_interval = log_config.value.aggregation_interval + flow_sampling = log_config.value.flow_sampling + metadata = log_config.value.metadata + } + } +} + +resource "google_compute_subnetwork_iam_binding" "binding" { + for_each = local.iam_keypairs + project = var.project_id + subnetwork = google_compute_subnetwork.subnetwork[each.value.subnet].name + region = google_compute_subnetwork.subnetwork[each.value.subnet].region + role = each.value.role + members = lookup( + lookup(var.iam_members, each.value.subnet, {}), each.value.role, [] + ) +} diff --git a/modules/net-vpc/outputs.tf b/modules/net-vpc/outputs.tf new file mode 100644 index 0000000000..a6866c3365 --- /dev/null +++ b/modules/net-vpc/outputs.tf @@ -0,0 +1,69 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "network" { + description = "Network resource." + value = google_compute_network.network +} + +output "name" { + description = "The name of the VPC being created." + value = google_compute_network.network.name +} + +output "self_link" { + description = "The URI of the VPC being created." + value = google_compute_network.network.self_link +} + +output "project_id" { + description = "Shared VPC host project id." + value = ( + var.shared_vpc_host + ? google_compute_shared_vpc_host_project.shared_vpc_host[*].project + : null + ) +} + +output "subnets" { + description = "Subnet resources." + value = google_compute_subnetwork.subnetwork +} + +output "subnet_ips" { + description = "Map of subnet address ranges keyed by name." + value = { for k, v in google_compute_subnetwork.subnetwork : k => v.ip_cidr_range } +} + +output "subnet_self_links" { + description = "Map of subnet self links keyed by name." + value = { for k, v in google_compute_subnetwork.subnetwork : k => v.self_link } +} + +output "subnet_regions" { + description = "Map of subnet regions keyed by name." + value = { for k, v in google_compute_subnetwork.subnetwork : k => v.region } +} + +output "subnet_secondary_ranges" { + description = "Map of subnet secondary ranges keyed by name." + value = { for k, v in google_compute_subnetwork.subnetwork : k => v.secondary_ip_range } +} + +output "bindings" { + description = "Subnet IAM bindings." + value = { for k, v in google_compute_subnetwork_iam_binding.binding : k => v } +} diff --git a/modules/net-vpc/variables.tf b/modules/net-vpc/variables.tf new file mode 100644 index 0000000000..c227c10564 --- /dev/null +++ b/modules/net-vpc/variables.tf @@ -0,0 +1,94 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "auto_create_subnetworks" { + description = "When set to true, the network is created in 'auto subnet mode' and it will create a subnet for each region automatically across the 10.128.0.0/9 address range. When set to false, the network is created in 'custom subnet mode' so the user can explicitly connect subnetwork resources." + type = bool + default = false +} + +variable "description" { + description = "An optional description of this resource. The resource must be recreated to modify this field." + type = string + default = "Terraform-managed." +} + +variable "iam_roles" { + description = "List of IAM roles keyed by subnet." + type = map(list(string)) + default = {} +} + +variable "iam_members" { + description = "List of IAM members keyed by subnet and role." + type = map(map(list(string))) + default = {} +} + +variable "log_configs" { + description = "Map of per-subnet optional configurations for flow logs when enabled." + type = map(map(string)) + default = {} +} + +variable "log_config_defaults" { + description = "Default configuration for flow logs when enabled." + type = object({ + aggregation_interval = string + flow_sampling = number + metadata = string + }) + default = { + aggregation_interval = "INTERVAL_5_SEC" + flow_sampling = 0.5 + metadata = "INCLUDE_ALL_METADATA" + } +} + +variable "name" { + description = "The name of the network being created" + type = string +} + +variable "project_id" { + description = "The ID of the project where this VPC will be created" + type = string +} + +variable "routing_mode" { + description = "The network routing mode (default 'GLOBAL')" + type = string + default = "GLOBAL" +} + +variable "shared_vpc_host" { + description = "Makes this project a Shared VPC host if 'true' (default 'false')" + type = bool + default = false +} + +variable "subnets" { + description = "The list of subnets being created" + type = map(object({ + ip_cidr_range = string + region = string + description = string + private_ip_google_access = bool + enable_flow_logs = bool + secondary_ip_range = map(string) + })) + default = {} +} diff --git a/modules/net-vpc/versions.tf b/modules/net-vpc/versions.tf new file mode 100644 index 0000000000..1fe4caaac6 --- /dev/null +++ b/modules/net-vpc/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = "~> 0.12.0" +} diff --git a/tests/conftest.py b/tests/conftest.py index 3dea14be84..81402b8ed2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,14 +14,20 @@ "Shared fixtures." +import collections import os import pytest import tftest +# top-level repository folder _BASEDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +# fixture result +Result = collections.namedtuple( + 'Result', 'terraform plan output destroy') + @pytest.fixture(scope='session') def plan(): @@ -34,3 +40,54 @@ def run_plan(testdir): return tf.plan(output=True) return run_plan + + +@pytest.fixture(scope='session') +def run_fixture(): + "Returns a function to run Terraform on a fixture." + + def run(fixture_path, extra_files=None, run_plan=True, run_apply=True, + run_destroy=not os.environ.get('TFTEST_INCREMENTAL')): + """Runs Terraform on fixture and return result. + + Convenience method to wrap a tftest instance for a single fixture. Runs + init on the tftest instance and optionally runs plan, aply and destroy, + returning outputs. + + Args: + fixture_path: the relative path from root to the fixture folder + extra_files: extra files that are passed in to tftest for injection + run_plan: run plan on the tftest instance + run_apply: run apply on the tftest instance + run_destroy: run destroy on the tftest instance, skips destroy by + default if the TFTEST_INCREMENTAL environment variable is set + + Returns: + A Result named tuple with the tftest instance and outputs for plan, + output and destroy. + """ + tf = tftest.TerraformTest(fixture_path, _BASEDIR, + os.environ.get('TERRAFORM', 'terraform')) + tf.setup(extra_files=extra_files, cleanup_on_exit=run_destroy) + plan = output = destroy = None + if run_plan: + plan = tf.plan(output=True) + if run_apply: + tf.apply() + output = tf.output(json_format=True) + if run_destroy: + tf.destroy() + return Result(tf, plan, output, destroy) + + return run + + +@pytest.fixture +def pretty_print(): + "Returns a fuction that pretty prints a data structure." + + def pretty_printer(data): + import json + print(json.dumps(data, indent=2)) + + return pretty_printer diff --git a/tests/modules/net_vpc/__init__.py b/tests/modules/net_vpc/__init__.py new file mode 100644 index 0000000000..35dff2e86e --- /dev/null +++ b/tests/modules/net_vpc/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/tests/modules/net_vpc/conftest.py b/tests/modules/net_vpc/conftest.py new file mode 100644 index 0000000000..9c9906e421 --- /dev/null +++ b/tests/modules/net_vpc/conftest.py @@ -0,0 +1,29 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"Apply fixture." + +import os + +import pytest + + +# path of this folder relative to root +_PATH = os.path.sep.join(os.path.abspath(__file__).split(os.path.sep)[-3:-1]) + + +@pytest.fixture(scope='module') +def fix_path(): + "Returns a function that prepends the test module path." + return lambda p: os.path.join(_PATH, p) diff --git a/tests/modules/net_vpc/fixtures/vpc-iam-bindings/main.tf b/tests/modules/net_vpc/fixtures/vpc-iam-bindings/main.tf new file mode 100644 index 0000000000..a897fc5910 --- /dev/null +++ b/tests/modules/net_vpc/fixtures/vpc-iam-bindings/main.tf @@ -0,0 +1,99 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project id used for this fixture." + type = string +} + +variable "subnets" { + description = "Subnet definitions." + default = { + subnet-a = { + ip_cidr_range = "192.168.0.0/24" + region = "europe-west1" + description = "First subnet." + private_ip_google_access = false + enable_flow_logs = false + secondary_ip_range = {} + }, + subnet-b = { + ip_cidr_range = "192.168.1.0/24" + region = "europe-west1" + description = "Second subnet." + private_ip_google_access = false + enable_flow_logs = false + secondary_ip_range = {} + }, + subnet-c = { + ip_cidr_range = "192.168.2.0/24" + region = "europe-west1" + description = "Third subnet." + private_ip_google_access = false + enable_flow_logs = false + secondary_ip_range = {} + }, + } +} + +locals { + members = [ + for s in google_service_account.binding_members : + "serviceAccount:${s.email}" + ] +} + +resource "google_service_account" "binding_members" { + for_each = toset(split(" ", "a b c d e")) + project = var.project_id + account_id = "user-${each.value}" +} + +module "vpc" { + source = "../../../../net-vpc" + project_id = var.project_id + name = "vpc-iam-bindings" + description = "Created by the vpc-iam-bindings fixture." + subnets = var.subnets + iam_roles = { + subnet-b = ["roles/compute.networkUser", "roles/compute.networkViewer"] + subnet-c = ["roles/compute.networkViewer"] + } + iam_members = { + subnet-b = { + "roles/compute.networkUser" = slice(local.members, 0, 2) + "roles/compute.networkViewer" = slice(local.members, 3, 4) + } + subnet-c = { + "roles/compute.networkViewer" = slice(local.members, 3, 5) + } + } +} + +output "network" { + description = "Network resource." + value = module.vpc.network +} + +output "subnets" { + description = "Subnet resources." + value = module.vpc.subnets +} + +output "bindings" { + description = "Subnet IAM bindings." + value = module.vpc.bindings +} diff --git a/tests/modules/net_vpc/fixtures/vpc-standalone/main.tf b/tests/modules/net_vpc/fixtures/vpc-standalone/main.tf new file mode 100644 index 0000000000..5c7a018e2a --- /dev/null +++ b/tests/modules/net_vpc/fixtures/vpc-standalone/main.tf @@ -0,0 +1,47 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "name" { + description = "Network name." + type = string + default = "net-vpc-standalone" +} + +variable "project_id" { + description = "Project id used for this fixture." + type = string +} + +module "vpc-simple" { + source = "../../../../net-vpc" + project_id = var.project_id + name = var.name +} + +output "name" { + description = "Network name." + value = module.vpc-simple.name +} + +output "self_link" { + description = "Network self link." + value = module.vpc-simple.self_link +} + +output "subnets" { + description = "Subnet resources." + value = module.vpc-simple.subnets +} diff --git a/tests/modules/net_vpc/fixtures/vpc-subnets/main.tf b/tests/modules/net_vpc/fixtures/vpc-subnets/main.tf new file mode 100644 index 0000000000..01c924df38 --- /dev/null +++ b/tests/modules/net_vpc/fixtures/vpc-subnets/main.tf @@ -0,0 +1,97 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project id used for this fixture." + type = string +} + +variable "subnets" { + description = "Subnet definitions." + default = { + subnet-simple = { + ip_cidr_range = "192.168.0.0/24" + region = "europe-west1" + description = "Simple subnet." + private_ip_google_access = false + enable_flow_logs = false + secondary_ip_range = {} + }, + subnet-options = { + ip_cidr_range = "192.168.1.0/24" + region = "europe-west2" + description = "Simple subnet with options." + private_ip_google_access = true + enable_flow_logs = true + secondary_ip_range = {} + }, + subnet-alias-ranges = { + ip_cidr_range = "192.168.2.0/24" + region = "europe-west1" + description = "Simple subnet with alias ranges." + private_ip_google_access = true + enable_flow_logs = true + secondary_ip_range = { + alias-1 = "172.16.10.0/24" + alias-2 = "172.16.20.0/24" + } + } + } +} + +variable "log_configs" { + description = "Logging configurations." + default = { + subnet-alias-ranges = { + flow_sampling = 0.75 + } + } +} + +module "vpc" { + source = "../../../../net-vpc" + project_id = var.project_id + name = "vpc-subnets" + description = "Created by the vpc-subnets fixture." + routing_mode = "REGIONAL" + subnets = var.subnets + log_configs = var.log_configs +} + +output "network" { + description = "Network resource." + value = module.vpc.network +} + +output "subnets" { + description = "Subnet resources." + value = module.vpc.subnets +} + +output "subnet_ips" { + description = "Map of subnet address ranges keyed by name." + value = module.vpc.subnet_ips +} + +output "subnet_regions" { + description = "Map of subnet regions keyed by name." + value = module.vpc.subnet_regions +} + +output "subnet_secondary_ranges" { + description = "Map of subnet secondary ranges keyed by name." + value = module.vpc.subnet_secondary_ranges +} diff --git a/tests/modules/net_vpc/test_vpc_iam_bindings.py b/tests/modules/net_vpc/test_vpc_iam_bindings.py new file mode 100644 index 0000000000..a936fe60cc --- /dev/null +++ b/tests/modules/net_vpc/test_vpc_iam_bindings.py @@ -0,0 +1,81 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections +import re + +import pytest + + +Binding = collections.namedtuple('Binding', 'subnet role members') + + +@pytest.fixture(scope='module') +def result(run_fixture, fix_path): + "Runs fixture interpolating current path, and returns result." + return run_fixture(fix_path('fixtures/vpc-iam-bindings')) + + +@pytest.fixture(scope='module') +def bindings(result): + "Returns a streamlined list of bindings." + return [ + Binding(b['subnetwork'].split('/')[-1], b['role'], b['members']) + for b in result.output['bindings'].values() + ] + + +@pytest.fixture(scope='module') +def subnet_names(result): + "Returns the list of subnet names." + return [s['name'] for s in result.output['subnets'].values()] + + +def test_vpc_attributes(result): + "Test network attributes." + network = result.output['network'] + assert network['routing_mode'] == 'GLOBAL' + assert network['description'] == 'Created by the vpc-iam-bindings fixture.' + + +def test_subnet_names(result, subnet_names): + "Test subnet names output." + resource_names = sorted( + [s['name'] for s in result.output['subnets'].values()]) + assert resource_names == sorted(subnet_names) + + +def test_binding_roles(result, bindings, subnet_names): + "Test that the correct roles from IAM bindings are set." + assert len([b for b in bindings if b.subnet == 'subnet-a']) == 0 + assert set([b.role for b in bindings if b.subnet == 'subnet-b']) == set([ + 'roles/compute.networkUser', 'roles/compute.networkViewer' + ]) + assert [b.role for b in bindings if b.subnet == 'subnet-c'] == [ + 'roles/compute.networkViewer' + ] + + +def test_binding_members(result, bindings, subnet_names): + "Test that the correct members from IAM bindings are set." + r = re.compile(r'^serviceAccount:([^@]+)@.*$') + for b in bindings: + members = [r.sub(r'\1', m) for m in b.members] + if b.subnet == 'subnet-b': + if b.role == 'roles/compute.networkUser': + assert members == ['user-a', 'user-b'] + else: + assert members == ['user-d'] + else: + assert members == ['user-d', 'user-e'] diff --git a/tests/modules/net_vpc/test_vpc_standalone.py b/tests/modules/net_vpc/test_vpc_standalone.py new file mode 100644 index 0000000000..1fa77394e6 --- /dev/null +++ b/tests/modules/net_vpc/test_vpc_standalone.py @@ -0,0 +1,29 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + + +@pytest.fixture(scope='module') +def result(run_fixture, fix_path): + "Runs fixture interpolating current path, and returns result." + return run_fixture(fix_path('fixtures/vpc-standalone')) + + +def test_vpc_creation(result): + "Test that VPC is created with the correct attributes." + assert result.output['name'] == result.plan.variables['name'] + assert result.output['self_link'].endswith(result.plan.variables['name']) + assert result.plan.variables['project_id'] in result.output['self_link'] + assert result.output['subnets'] == {} diff --git a/tests/modules/net_vpc/test_vpc_subnets.py b/tests/modules/net_vpc/test_vpc_subnets.py new file mode 100644 index 0000000000..008b77f8f0 --- /dev/null +++ b/tests/modules/net_vpc/test_vpc_subnets.py @@ -0,0 +1,63 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + + +@pytest.fixture(scope='module') +def result(run_fixture, fix_path): + "Runs fixture interpolating current path, and returns result." + return run_fixture(fix_path('fixtures/vpc-subnets')) + + +def test_vpc_attributes(result): + "Test network attributes." + network = result.output['network'] + assert network['routing_mode'] == 'REGIONAL' + assert network['description'] == 'Created by the vpc-subnets fixture.' + + +def test_subnet_names(result): + "Test subnet names output." + resource_names = sorted([s['name'] + for s in result.output['subnets'].values()]) + assert resource_names == sorted( + [k for k in result.plan.variables['subnets']]) + + +def test_subnet_ips(result): + "Test subnet IPs output." + for name, attrs in result.plan.variables['subnets'].items(): + assert result.output['subnet_ips'][name] == attrs['ip_cidr_range'] + + +def test_subnet_regions(result): + "Test subnet regions output." + assert result.output['subnet_regions'] == dict( + (k, v['region']) for k, v in result.plan.variables['subnets'].items()) + + +def test_secondary_ip_ranges(result): + "Test subnet secondary ranges output." + for name, attrs in result.plan.variables['subnets'].items(): + ranges = [{'range_name': k, 'ip_cidr_range': v} + for k, v in attrs['secondary_ip_range'].items()] + assert ranges == result.output['subnet_secondary_ranges'][name] + + +def test_flow_logs(result): + "Test that log config is set using the enable flow logs variable." + for name, attrs in result.plan.variables['subnets'].items(): + log_config = 1 if attrs['enable_flow_logs'] else 0 + assert len(result.output['subnets'][name]['log_config']) == log_config diff --git a/tests/requirements.txt b/tests/requirements.txt index a5b3fbc2d0..6f5740dffd 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,3 +1,3 @@ pytest>=4.3.1 pytest-tldr>=0.2.1 -tftest>=1.2.0 +tftest>=1.3.0 From 544d023c4d6a46ea659e5ad9d4045bf02507c6f0 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Mon, 25 Nov 2019 02:39:20 +0100 Subject: [PATCH 003/106] add TODO to net-vpc module --- modules/net-vpc/main.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/net-vpc/main.tf b/modules/net-vpc/main.tf index f687adca02..859343dc00 100644 --- a/modules/net-vpc/main.tf +++ b/modules/net-vpc/main.tf @@ -56,6 +56,8 @@ resource "google_compute_shared_vpc_host_project" "shared_vpc_host" { depends_on = [google_compute_network.network] } +# TODO(ludoo): add shared vpc service project registration + resource "google_compute_subnetwork" "subnetwork" { for_each = var.subnets project = var.project_id From c964738c024c5d85d3bff7ac92d051deb8a639c5 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Tue, 26 Nov 2019 16:37:42 +0100 Subject: [PATCH 004/106] add minimal README files with input/output variables to gke and net-vpc modules --- modules/gke-cluster/README.md | 52 +++++++++++++++++++++++++++++++++++ modules/net-vpc/README.md | 38 +++++++++++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 modules/net-vpc/README.md diff --git a/modules/gke-cluster/README.md b/modules/gke-cluster/README.md index e69de29bb2..86b152263b 100644 --- a/modules/gke-cluster/README.md +++ b/modules/gke-cluster/README.md @@ -0,0 +1,52 @@ +# Minimalistic GKE module + +TODO(ludoo): add description. + +## Example usage + +TODO(ludoo): add example + +## Variables + +| name | description | type | required | +|---|---|:---: |:---:| +| location | Cluster zone or region. | string | ✓ +| name | Cluster name. | string | ✓ +| network | Name or self link of the VPC used for the cluster. Use the self link for Shared VPC. | string | ✓ +| project_id | Cluster project id. | string | ✓ +| secondary_range_pods | Subnet secondary range name used for pods. | string | ✓ +| secondary_range_services | Subnet secondary range name used for services. | string | ✓ +| subnetwork | VPC subnetwork name or self link. | string | ✓ +| *addons* | Addons enabled in the cluster (true means enabled). | object | +| *authenticator_security_group* | RBAC security group for Google Groups for GKE, format is gke-security-groups@yourdomain.com. | string | +| *cluster_autoscaling* | Enable and configure limits for Node Auto-Provisioning with Cluster Autoscaler. | object | +| *database_encryption* | Enable and configure GKE application-layer secrets encryption. | object | +| *default_max_pods_per_node* | Maximum number of pods per node in this cluster. | number | +| *description* | Cluster description. | string | +| *enable_binary_authorization* | Enable Google Binary Authorization. | bool | +| *enable_intranode_visibility* | Enable intra-node visibility to make same node pod to pod traffic visible. | bool | +| *enable_shielded_nodes* | Enable Shielded Nodes features on all nodes in this cluster. | bool | +| *enable_tpu* | Enable Cloud TPU resources in this cluster. | bool | +| *labels* | Cluster resource labels. | map | +| *logging_service* | Logging service (disable with an empty string). | string | +| *maintenance_start_time* | Maintenance start time in RFC3339 format 'HH:MM', where HH is [00-23] and MM is [00-59] GMT. | string | +| *master_authorized_ranges* | External Ip address ranges that can access the Kubernetes cluster master through HTTPS. | map | +| *min_master_version* | Minimum version of the master, defaults to the version of the most recent official release. | string | +| *monitoring_service* | Monitoring service (disable with an empty string). | string | +| *node_locations* | Zones in which the cluster's nodes are located. | list | +| *pod_security_policy* | Enable the PodSecurityPolicy feature. | bool | +| *private_cluster* | Enable private cluster. | bool | +| *private_cluster_config* | Private cluster configuration. | object | +| *release_channel* | Release channel for GKE upgrades. | string | +| *resource_usage_export_config* | Configure the ResourceUsageExportConfig feature. | object | +| *vertical_pod_autoscaling* | Enable the Vertical Pod Autoscaling feature. | bool | +| *workload_identity* | Enable the Workload Identity feature. | bool | + +## Outputs + +| name | description | +|---|---| +| cluster | Cluster resource. | +| endpoint | Cluster endpoint. | +| master_version | Master version. | +| name | Cluster name. | diff --git a/modules/net-vpc/README.md b/modules/net-vpc/README.md new file mode 100644 index 0000000000..e7765e2bdd --- /dev/null +++ b/modules/net-vpc/README.md @@ -0,0 +1,38 @@ +# Minimalistic VPC module + +TODO(ludoo): add description. + +## Example usage + +TODO(ludoo): add example + +## Variables + +| name | description | type | required | +|---|---|:---: |:---:| +| name | The name of the network being created | string | ✓ +| project_id | The ID of the project where this VPC will be created | string | ✓ +| *auto_create_subnetworks* | When set to true, the network is created in 'auto subnet mode' and it will create a subnet for each region automatically across the 10.128.0.0/9 address range. When set to false, the network is created in 'custom subnet mode' so the user can explicitly connect subnetwork resources. | bool | +| *description* | An optional description of this resource. The resource must be recreated to modify this field. | string | +| *iam_members* | List of IAM members keyed by subnet and role. | map | +| *iam_roles* | List of IAM roles keyed by subnet. | map | +| *log_config_defaults* | Default configuration for flow logs when enabled. | object | +| *log_configs* | Map of per-subnet optional configurations for flow logs when enabled. | map | +| *routing_mode* | The network routing mode (default 'GLOBAL') | string | +| *shared_vpc_host* | Makes this project a Shared VPC host if 'true' (default 'false') | bool | +| *subnets* | The list of subnets being created | map | + +## Outputs + +| name | description | +|---|---| +| bindings | Subnet IAM bindings. | +| name | The name of the VPC being created. | +| network | Network resource. | +| project_id | Shared VPC host project id. | +| self_link | The URI of the VPC being created. | +| subnet_ips | Map of subnet address ranges keyed by name. | +| subnet_regions | Map of subnet regions keyed by name. | +| subnet_secondary_ranges | Map of subnet secondary ranges keyed by name. | +| subnet_self_links | Map of subnet self links keyed by name. | +| subnets | Subnet resources. | From be68e1ec9cbf8737f22452e65f0b296fe878ca1e Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Thu, 28 Nov 2019 17:25:02 +0100 Subject: [PATCH 005/106] BigQuery Module (#24) * Bigquery Module * Added README file * Added type hints --- modules/bigquery/README.md | 141 ++++++++++++++++++++++++++++++++++++ modules/bigquery/main.tf | 74 +++++++++++++++++++ modules/bigquery/outputs.tf | 27 +++++++ modules/bigquery/vars.tf | 70 ++++++++++++++++++ 4 files changed, 312 insertions(+) create mode 100644 modules/bigquery/README.md create mode 100644 modules/bigquery/main.tf create mode 100644 modules/bigquery/outputs.tf create mode 100644 modules/bigquery/vars.tf diff --git a/modules/bigquery/README.md b/modules/bigquery/README.md new file mode 100644 index 0000000000..55a7a09ef2 --- /dev/null +++ b/modules/bigquery/README.md @@ -0,0 +1,141 @@ +# Opinionated BigQuery module + +TODO(jccb): add description. + +## Example usage + +```hcl +module "bq" { + source = "../modules/bigquery/" + project_id = "my-project" + datasets = [ + { + id = "dataset1" + name = "dataset1" + description = "dataset1 description" + location = "EU" + labels = { + environment = "dev" + } + # default_partition_expiration_ms = 10000 + # default_table_expiration_ms - 10000 + } ] + + dataset_access = { + dataset1 = { + "me@myorg.com" = "roles/bigquery.dataViewer" + } + } + + tables = [ + { + table_id = "table1" + dataset_id = "dataset1" + description = "dataset1 description" + labels = { + team = "sales" + } + schema = [ + { + type = "DATE" + name = "date" + }, + { + type = "STRING" + name = "name" + } + ] + time_partitioning = { + field = "date" + type = "DAY" + } + } + ] + + views = [ + { + dataset = "dataset1" + table = "view1" + query = "select date from `my-project.dataset1.table1`" + } + ] +} +``` + +## Variables + +| name | description | type | required | +|----------------|-------------------------------------------------------------|:------:|:--------:| +| project_id | The ID of the project where the datasets, tables, and views | string | ✓ | +| datasets | List of datasets to be created (see structure below) | string | | +| tables | List of tables to be created (see structure below) | string | | +| views | List of views to be created (see structure below) | string | | +| dataset_access | Dataset-level permissions to be granted | string | | + +### dataset structure +The datasets list should contains objects with the following structure: + +``` hcl + { + id = "dataset id" + name = "dataset name" + description = "dataset description" + location = "dataset location" + labels = { # optional + label1 = "value1" + } + default_partition_expiration_ms = 10000 # optional + default_table_expiration_ms - 10000 # optional + } + +``` + +### table structure + +``` hcl + { + table_id = "table name" + dataset_id = "table dataset" + description = "table description" + labels = { + label1 = "value1" + } + schema = [ + { + type = "DATE" + name = "date" + } + { + type = "STRING" + name = "name" + } + ] + } + +``` + +You can optionally specify ```expiration_time```, +```time_partitioning``` and ```clustering``` for any table. + + +### views structure + +The structure for the views field is just a dataset (where the view +will be created), the table (view name), and the query for the view. Keep in mind that the query should use the complete name for any table it references. + +``` hcl + { + dataset = "dataset1" + table = "view1" + query = "select date from `my-project.dataset.table`" + } +``` + + +## Outputs + +| name | description | +|----------|----------------------------------------| +| datasets | Map of dataset objects keyed by name | +| tables | List of table objects | +| views | List of table objects created as views | diff --git a/modules/bigquery/main.tf b/modules/bigquery/main.tf new file mode 100644 index 0000000000..7421d22f01 --- /dev/null +++ b/modules/bigquery/main.tf @@ -0,0 +1,74 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_bigquery_dataset" "datasets" { + count = length(var.datasets) + project = var.project_id + dataset_id = var.datasets[count.index]["id"] + friendly_name = var.datasets[count.index]["name"] + description = var.datasets[count.index]["description"] + location = var.datasets[count.index]["location"] + labels = lookup(var.datasets[count.index], "labels", null) + default_table_expiration_ms = lookup(var.datasets[count.index], "default_table_expiration_ms", null) + default_partition_expiration_ms = lookup(var.datasets[count.index], "default_partition_expiration_ms", null) + + dynamic "access" { + for_each = lookup(var.dataset_access, var.datasets[count.index]["id"], tomap({})) + content { + user_by_email = access.key + role = access.value + } + } + + access { + role = "OWNER" + special_group = "projectOwners" + } +} + +resource "google_bigquery_table" "tables" { + count = length(var.tables) + project = var.project_id + dataset_id = var.tables[count.index]["dataset_id"] + table_id = var.tables[count.index]["table_id"] + labels = var.tables[count.index]["labels"] + expiration_time = lookup(var.tables[count.index], "expiration_time", null) + clustering = lookup(var.tables[count.index], "clustering", null) + schema = jsonencode(var.tables[count.index]["schema"]) + + dynamic "time_partitioning" { + for_each = lookup(var.tables[count.index], "time_partitioning", false) == false ? [] : [1] + content { + field = var.tables[count.index]["time_partitioning"]["field"] + type = var.tables[count.index]["time_partitioning"]["type"] + expiration_ms = lookup(var.tables[count.index], "expiration_ms", null) + } + } + + depends_on = [google_bigquery_dataset.datasets] +} + +resource "google_bigquery_table" "views" { + count = length(var.views) + project = var.project_id + dataset_id = var.views[count.index]["dataset"] + table_id = var.views[count.index]["table"] + view { + query = var.views[count.index]["query"] + use_legacy_sql = false + } + depends_on = [google_bigquery_table.tables] +} diff --git a/modules/bigquery/outputs.tf b/modules/bigquery/outputs.tf new file mode 100644 index 0000000000..6eabd78f13 --- /dev/null +++ b/modules/bigquery/outputs.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "datasets" { + value = zipmap(google_bigquery_dataset.datasets[*].dataset_id, google_bigquery_dataset.datasets) +} + +output "tables" { + value = google_bigquery_table.tables +} + +output "views" { + value = google_bigquery_table.views +} diff --git a/modules/bigquery/vars.tf b/modules/bigquery/vars.tf new file mode 100644 index 0000000000..077507be44 --- /dev/null +++ b/modules/bigquery/vars.tf @@ -0,0 +1,70 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project ID" + type = string +} + +variable "datasets" { + description = "Datatase IDs" + default = [] + type = list + # type = list(object({ + # id = string + # name = string + # description = string + # location = string + # # labels = map(string) # optional + # # default_partition_expiration_ms = number # optional + # # default_table_expiration_ms = number # optional + # })) +} + +variable "tables" { + description = "Tables" + default = [] + type = list + # type = list(object({ + # table_id = string + # dataset_id = string + # labels = map(string) + # schema = string + # # expiration_time = number # optional + # # clustering = string # optional + # # time_partitioning = object({ # optional + # # field = string + # # type = string + # # expiration_ms = number # optional + # # } + # })) +} + +variable "views" { + description = "Views" + default = [] + type = list(object({ + dataset = string + table = string + query = string + })) +} + +variable "dataset_access" { + description = "Dataset permissions" + default = {} + type = map(map(string)) +} From 653dd701eeba00233f26d009f3de6a3be445706f Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 24 Nov 2019 16:19:33 +0100 Subject: [PATCH 006/106] gke-cluster --- modules/gke-cluster/README.md | 0 modules/gke-cluster/main.tf | 197 +++++++++++++++++++++++++ modules/gke-cluster/outputs.tf | 38 +++++ modules/gke-cluster/variables.tf | 246 +++++++++++++++++++++++++++++++ 4 files changed, 481 insertions(+) create mode 100644 modules/gke-cluster/README.md create mode 100644 modules/gke-cluster/main.tf create mode 100644 modules/gke-cluster/outputs.tf create mode 100644 modules/gke-cluster/variables.tf diff --git a/modules/gke-cluster/README.md b/modules/gke-cluster/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/modules/gke-cluster/main.tf b/modules/gke-cluster/main.tf new file mode 100644 index 0000000000..9c57d67303 --- /dev/null +++ b/modules/gke-cluster/main.tf @@ -0,0 +1,197 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_container_cluster" "cluster" { + provider = google-beta + project = var.project_id + name = var.name + description = var.description + location = var.location + node_locations = length(var.node_locations) == 0 ? null : var.node_locations + min_master_version = var.min_master_version + network = var.network + subnetwork = var.subnetwork + logging_service = var.logging_service + monitoring_service = var.monitoring_service + resource_labels = var.labels + default_max_pods_per_node = var.default_max_pods_per_node + enable_binary_authorization = var.enable_binary_authorization + enable_intranode_visibility = var.enable_intranode_visibility + enable_shielded_nodes = var.enable_shielded_nodes + enable_tpu = var.enable_tpu + initial_node_count = 1 + remove_default_node_pool = true + + # node_config + + addons_config { + http_load_balancing { + disabled = ! var.addons.http_load_balancing + } + horizontal_pod_autoscaling { + disabled = ! var.addons.horizontal_pod_autoscaling + } + network_policy_config { + disabled = ! var.addons.network_policy_config + } + # beta addons + # cloudrun is dynamic as it tends to trigger cluster recreation on change + dynamic cloudrun_config { + for_each = var.addons.istio_config.enabled && var.addons.cloudrun_config ? [""] : [] + content { + disabled = false + } + } + istio_config { + disabled = ! var.addons.istio_config.enabled + auth = var.addons.istio_config.tls ? "AUTH_MUTUAL_TLS" : "AUTH_NONE" + } + } + + # TODO(ludomagno): support setting address ranges instead of range names + # https://www.terraform.io/docs/providers/google/r/container_cluster.html#cluster_ipv4_cidr_block + ip_allocation_policy { + cluster_secondary_range_name = var.secondary_range_pods + services_secondary_range_name = var.secondary_range_services + } + + # TODO(ludomagno): make optional, and support beta feature + # https://www.terraform.io/docs/providers/google/r/container_cluster.html#daily_maintenance_window + maintenance_policy { + daily_maintenance_window { + start_time = var.maintenance_start_time + } + } + + master_auth { + client_certificate_config { + issue_client_certificate = false + } + } + + dynamic master_authorized_networks_config { + for_each = length(var.master_authorized_ranges) == 0 ? [] : list(var.master_authorized_ranges) + iterator = ranges + content { + dynamic cidr_blocks { + for_each = ranges.value + iterator = range + content { + cidr_block = range.value + display_name = range.key + } + } + } + } + + dynamic network_policy { + for_each = var.addons.network_policy_config ? [""] : [] + content { + enabled = true + provider = "CALICO" + } + } + + dynamic private_cluster_config { + for_each = var.private_cluster ? list(var.private_cluster_config) : [] + iterator = config + content { + enable_private_nodes = config.value.enable_private_nodes + enable_private_endpoint = config.value.enable_private_endpoint + master_ipv4_cidr_block = config.value.master_ipv4_cidr_block + } + } + + # beta features + + dynamic authenticator_groups_config { + for_each = var.authenticator_security_group == null ? [] : [""] + content { + security_group = var.authenticator_security_group + } + } + + dynamic cluster_autoscaling { + for_each = var.cluster_autoscaling.enabled ? [var.cluster_autoscaling] : [] + iterator = config + content { + enabled = true + resource_limits { + resource_type = "cpu" + minimum = config.cpu_min + maximum = config.cpu_max + } + resource_limits { + resource_type = "memory" + minimum = config.memory_min + maximum = config.memory_max + } + } + } + + dynamic database_encryption { + for_each = var.database_encryption.enabled ? [var.database_encryption] : [] + iterator = config + content { + state = config.value.state + key_name = config.value.key_name + } + } + + dynamic pod_security_policy_config { + for_each = var.pod_security_policy != null ? [""] : [] + content { + enabled = var.pod_security_policy + } + } + + dynamic release_channel { + for_each = var.release_channel != null ? [""] : [] + content { + channel = var.release_channel + } + } + + dynamic resource_usage_export_config { + for_each = ( + var.resource_usage_export_config.enabled != null + && + var.resource_usage_export_config.dataset != null + ? [""] : [] + ) + content { + enable_network_egress_metering = var.resource_usage_export_config.enabled + bigquery_destination { + dataset_id = var.resource_usage_export_config.dataset + } + } + } + + dynamic vertical_pod_autoscaling { + for_each = var.vertical_pod_autoscaling == null ? [] : [""] + content { + enabled = var.vertical_pod_autoscaling + } + } + + dynamic workload_identity_config { + for_each = var.workload_identity ? [""] : [] + content { + identity_namespace = "${var.project_id}.svc.id.goog" + } + } + +} diff --git a/modules/gke-cluster/outputs.tf b/modules/gke-cluster/outputs.tf new file mode 100644 index 0000000000..f21b680955 --- /dev/null +++ b/modules/gke-cluster/outputs.tf @@ -0,0 +1,38 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the +); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an + BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cluster" { + description = "Cluster resource." + sensitive = true + value = google_container_cluster.cluster +} + +output "endpoint" { + description = "Cluster endpoint." + value = google_container_cluster.cluster.endpoint +} + +output "master_version" { + description = "Master version." + value = google_container_cluster.cluster.master_version +} + +output "name" { + description = "Cluster name." + value = google_container_cluster.cluster.name +} diff --git a/modules/gke-cluster/variables.tf b/modules/gke-cluster/variables.tf new file mode 100644 index 0000000000..c66e60fa3c --- /dev/null +++ b/modules/gke-cluster/variables.tf @@ -0,0 +1,246 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "addons" { + description = "Addons enabled in the cluster (true means enabled)." + type = object({ + horizontal_pod_autoscaling = bool + http_load_balancing = bool + network_policy_config = bool + cloudrun_config = bool + istio_config = object({ + enabled = bool + tls = bool + }) + }) + default = { + horizontal_pod_autoscaling = true + http_load_balancing = true + network_policy_config = false + cloudrun_config = false + istio_config = { + enabled = false + tls = false + } + } +} + +variable "authenticator_security_group" { + description = "RBAC security group for Google Groups for GKE, format is gke-security-groups@yourdomain.com." + type = string + default = null +} + +variable "cluster_autoscaling" { + description = "Enable and configure limits for Node Auto-Provisioning with Cluster Autoscaler." + type = object({ + enabled = bool + cpu_min = number + cpu_max = number + memory_min = number + memory_max = number + }) + default = { + enabled = false + cpu_min = 0 + cpu_max = 0 + memory_min = 0 + memory_max = 0 + } +} + +variable "database_encryption" { + description = "Enable and configure GKE application-layer secrets encryption." + type = object({ + enabled = bool + state = string + key_name = string + }) + default = { + enabled = false + state = "DECRYPTED" + key_name = null + } +} + +variable "default_max_pods_per_node" { + description = "Maximum number of pods per node in this cluster." + type = number + default = 110 +} + +variable "description" { + description = "Cluster description." + type = string + default = null +} + +variable "enable_binary_authorization" { + description = "Enable Google Binary Authorization." + type = bool + default = null +} + +variable "enable_intranode_visibility" { + description = "Enable intra-node visibility to make same node pod to pod traffic visible." + type = bool + default = null +} + +variable "enable_shielded_nodes" { + description = "Enable Shielded Nodes features on all nodes in this cluster." + type = bool + default = null +} + +variable "enable_tpu" { + description = "Enable Cloud TPU resources in this cluster." + type = bool + default = null +} + +variable "labels" { + description = "Cluster resource labels." + type = map(string) + default = null +} + +variable "location" { + description = "Cluster zone or region." + type = string +} + +variable "logging_service" { + description = "Logging service (disable with an empty string)." + type = string + default = "logging.googleapis.com/kubernetes" +} + +variable "maintenance_start_time" { + description = "Maintenance start time in RFC3339 format 'HH:MM', where HH is [00-23] and MM is [00-59] GMT." + type = string + default = "03:00" +} + +variable "master_authorized_ranges" { + description = "External Ip address ranges that can access the Kubernetes cluster master through HTTPS." + type = map(string) + default = {} +} + +variable "min_master_version" { + description = "Minimum version of the master, defaults to the version of the most recent official release." + type = string + default = null +} + +variable "monitoring_service" { + description = "Monitoring service (disable with an empty string)." + type = string + default = "monitoring.googleapis.com/kubernetes" +} + +variable "name" { + description = "Cluster name." + type = string +} + +variable "network" { + description = "Name or self link of the VPC used for the cluster. Use the self link for Shared VPC." + type = string +} + +variable "node_locations" { + description = "Zones in which the cluster's nodes are located." + type = list(string) + default = [] +} + +variable "pod_security_policy" { + description = "Enable the PodSecurityPolicy feature." + type = bool + default = null +} + +variable "private_cluster" { + description = "Enable private cluster." + type = bool + default = false +} + +variable "private_cluster_config" { + description = "Private cluster configuration." + type = object({ + enable_private_nodes = bool + enable_private_endpoint = bool + master_ipv4_cidr_block = string + }) + default = { + enable_private_nodes = true + enable_private_endpoint = false + master_ipv4_cidr_block = null + } +} + +variable "project_id" { + description = "Cluster project id." + type = string +} + +variable "release_channel" { + description = "Release channel for GKE upgrades." + type = string + default = null +} + +variable "resource_usage_export_config" { + description = "Configure the ResourceUsageExportConfig feature." + type = object({ + enabled = bool + dataset = string + }) + default = { + enabled = null + dataset = null + } +} + +variable "secondary_range_pods" { + description = "Subnet secondary range name used for pods." + type = string +} + +variable "secondary_range_services" { + description = "Subnet secondary range name used for services." + type = string +} + +variable "subnetwork" { + description = "VPC subnetwork name or self link." + type = string +} + +variable "vertical_pod_autoscaling" { + description = "Enable the Vertical Pod Autoscaling feature." + type = bool + default = null +} + +variable "workload_identity" { + description = "Enable the Workload Identity feature." + type = bool + default = true +} From 814915d22b86f380754c03deaaeedc78f5ad6057 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Mon, 25 Nov 2019 02:36:19 +0100 Subject: [PATCH 007/106] net-vpc module and tests --- modules/net-vpc/main.tf | 91 +++++++++++++++++ modules/net-vpc/outputs.tf | 69 +++++++++++++ modules/net-vpc/variables.tf | 94 ++++++++++++++++++ modules/net-vpc/versions.tf | 19 ++++ tests/conftest.py | 57 +++++++++++ tests/modules/net_vpc/__init__.py | 14 +++ tests/modules/net_vpc/conftest.py | 29 ++++++ .../net_vpc/fixtures/vpc-iam-bindings/main.tf | 99 +++++++++++++++++++ .../net_vpc/fixtures/vpc-standalone/main.tf | 47 +++++++++ .../net_vpc/fixtures/vpc-subnets/main.tf | 97 ++++++++++++++++++ .../modules/net_vpc/test_vpc_iam_bindings.py | 81 +++++++++++++++ tests/modules/net_vpc/test_vpc_standalone.py | 29 ++++++ tests/modules/net_vpc/test_vpc_subnets.py | 63 ++++++++++++ tests/requirements.txt | 2 +- 14 files changed, 790 insertions(+), 1 deletion(-) create mode 100644 modules/net-vpc/main.tf create mode 100644 modules/net-vpc/outputs.tf create mode 100644 modules/net-vpc/variables.tf create mode 100644 modules/net-vpc/versions.tf create mode 100644 tests/modules/net_vpc/__init__.py create mode 100644 tests/modules/net_vpc/conftest.py create mode 100644 tests/modules/net_vpc/fixtures/vpc-iam-bindings/main.tf create mode 100644 tests/modules/net_vpc/fixtures/vpc-standalone/main.tf create mode 100644 tests/modules/net_vpc/fixtures/vpc-subnets/main.tf create mode 100644 tests/modules/net_vpc/test_vpc_iam_bindings.py create mode 100644 tests/modules/net_vpc/test_vpc_standalone.py create mode 100644 tests/modules/net_vpc/test_vpc_subnets.py diff --git a/modules/net-vpc/main.tf b/modules/net-vpc/main.tf new file mode 100644 index 0000000000..f687adca02 --- /dev/null +++ b/modules/net-vpc/main.tf @@ -0,0 +1,91 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # map of log configs for all subnets to reduce the number of lookups needed + subnet_log_configs = { + for name, attrs in var.subnets : name => lookup(var.log_configs, name, {}) + } + # map of log config that reflects enable_flow_logs and sets defaults + log_configs = { + for name, attrs in var.subnets : name => ( + attrs.enable_flow_logs + ? [{ + for key, value in var.log_config_defaults : key => lookup( + local.subnet_log_configs[name], key, value + ) + }] + : [] + ) + } + # distinct is needed to make the expanding function argument work + iam_pairs = concat([], distinct([ + for subnet, roles in var.iam_roles : + [for role in roles : { subnet = subnet, role = role }] + ])...) + iam_keypairs = { + for pair in local.iam_pairs : + "${pair.subnet}-${pair.role}" => pair + } +} + +resource "google_compute_network" "network" { + project = var.project_id + name = var.name + description = var.description + auto_create_subnetworks = var.auto_create_subnetworks + routing_mode = var.routing_mode +} + +resource "google_compute_shared_vpc_host_project" "shared_vpc_host" { + count = var.shared_vpc_host ? 1 : 0 + project = var.project_id + depends_on = [google_compute_network.network] +} + +resource "google_compute_subnetwork" "subnetwork" { + for_each = var.subnets + project = var.project_id + network = google_compute_network.network.name + name = each.key + description = each.value.description + ip_cidr_range = each.value.ip_cidr_range + region = each.value.region + private_ip_google_access = each.value.private_ip_google_access + secondary_ip_range = [ + for name, range in each.value.secondary_ip_range : + { range_name = name, ip_cidr_range = range } + ] + dynamic "log_config" { + for_each = local.log_configs[each.key] + content { + aggregation_interval = log_config.value.aggregation_interval + flow_sampling = log_config.value.flow_sampling + metadata = log_config.value.metadata + } + } +} + +resource "google_compute_subnetwork_iam_binding" "binding" { + for_each = local.iam_keypairs + project = var.project_id + subnetwork = google_compute_subnetwork.subnetwork[each.value.subnet].name + region = google_compute_subnetwork.subnetwork[each.value.subnet].region + role = each.value.role + members = lookup( + lookup(var.iam_members, each.value.subnet, {}), each.value.role, [] + ) +} diff --git a/modules/net-vpc/outputs.tf b/modules/net-vpc/outputs.tf new file mode 100644 index 0000000000..a6866c3365 --- /dev/null +++ b/modules/net-vpc/outputs.tf @@ -0,0 +1,69 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "network" { + description = "Network resource." + value = google_compute_network.network +} + +output "name" { + description = "The name of the VPC being created." + value = google_compute_network.network.name +} + +output "self_link" { + description = "The URI of the VPC being created." + value = google_compute_network.network.self_link +} + +output "project_id" { + description = "Shared VPC host project id." + value = ( + var.shared_vpc_host + ? google_compute_shared_vpc_host_project.shared_vpc_host[*].project + : null + ) +} + +output "subnets" { + description = "Subnet resources." + value = google_compute_subnetwork.subnetwork +} + +output "subnet_ips" { + description = "Map of subnet address ranges keyed by name." + value = { for k, v in google_compute_subnetwork.subnetwork : k => v.ip_cidr_range } +} + +output "subnet_self_links" { + description = "Map of subnet self links keyed by name." + value = { for k, v in google_compute_subnetwork.subnetwork : k => v.self_link } +} + +output "subnet_regions" { + description = "Map of subnet regions keyed by name." + value = { for k, v in google_compute_subnetwork.subnetwork : k => v.region } +} + +output "subnet_secondary_ranges" { + description = "Map of subnet secondary ranges keyed by name." + value = { for k, v in google_compute_subnetwork.subnetwork : k => v.secondary_ip_range } +} + +output "bindings" { + description = "Subnet IAM bindings." + value = { for k, v in google_compute_subnetwork_iam_binding.binding : k => v } +} diff --git a/modules/net-vpc/variables.tf b/modules/net-vpc/variables.tf new file mode 100644 index 0000000000..c227c10564 --- /dev/null +++ b/modules/net-vpc/variables.tf @@ -0,0 +1,94 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "auto_create_subnetworks" { + description = "When set to true, the network is created in 'auto subnet mode' and it will create a subnet for each region automatically across the 10.128.0.0/9 address range. When set to false, the network is created in 'custom subnet mode' so the user can explicitly connect subnetwork resources." + type = bool + default = false +} + +variable "description" { + description = "An optional description of this resource. The resource must be recreated to modify this field." + type = string + default = "Terraform-managed." +} + +variable "iam_roles" { + description = "List of IAM roles keyed by subnet." + type = map(list(string)) + default = {} +} + +variable "iam_members" { + description = "List of IAM members keyed by subnet and role." + type = map(map(list(string))) + default = {} +} + +variable "log_configs" { + description = "Map of per-subnet optional configurations for flow logs when enabled." + type = map(map(string)) + default = {} +} + +variable "log_config_defaults" { + description = "Default configuration for flow logs when enabled." + type = object({ + aggregation_interval = string + flow_sampling = number + metadata = string + }) + default = { + aggregation_interval = "INTERVAL_5_SEC" + flow_sampling = 0.5 + metadata = "INCLUDE_ALL_METADATA" + } +} + +variable "name" { + description = "The name of the network being created" + type = string +} + +variable "project_id" { + description = "The ID of the project where this VPC will be created" + type = string +} + +variable "routing_mode" { + description = "The network routing mode (default 'GLOBAL')" + type = string + default = "GLOBAL" +} + +variable "shared_vpc_host" { + description = "Makes this project a Shared VPC host if 'true' (default 'false')" + type = bool + default = false +} + +variable "subnets" { + description = "The list of subnets being created" + type = map(object({ + ip_cidr_range = string + region = string + description = string + private_ip_google_access = bool + enable_flow_logs = bool + secondary_ip_range = map(string) + })) + default = {} +} diff --git a/modules/net-vpc/versions.tf b/modules/net-vpc/versions.tf new file mode 100644 index 0000000000..1fe4caaac6 --- /dev/null +++ b/modules/net-vpc/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = "~> 0.12.0" +} diff --git a/tests/conftest.py b/tests/conftest.py index 3dea14be84..81402b8ed2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,14 +14,20 @@ "Shared fixtures." +import collections import os import pytest import tftest +# top-level repository folder _BASEDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +# fixture result +Result = collections.namedtuple( + 'Result', 'terraform plan output destroy') + @pytest.fixture(scope='session') def plan(): @@ -34,3 +40,54 @@ def run_plan(testdir): return tf.plan(output=True) return run_plan + + +@pytest.fixture(scope='session') +def run_fixture(): + "Returns a function to run Terraform on a fixture." + + def run(fixture_path, extra_files=None, run_plan=True, run_apply=True, + run_destroy=not os.environ.get('TFTEST_INCREMENTAL')): + """Runs Terraform on fixture and return result. + + Convenience method to wrap a tftest instance for a single fixture. Runs + init on the tftest instance and optionally runs plan, aply and destroy, + returning outputs. + + Args: + fixture_path: the relative path from root to the fixture folder + extra_files: extra files that are passed in to tftest for injection + run_plan: run plan on the tftest instance + run_apply: run apply on the tftest instance + run_destroy: run destroy on the tftest instance, skips destroy by + default if the TFTEST_INCREMENTAL environment variable is set + + Returns: + A Result named tuple with the tftest instance and outputs for plan, + output and destroy. + """ + tf = tftest.TerraformTest(fixture_path, _BASEDIR, + os.environ.get('TERRAFORM', 'terraform')) + tf.setup(extra_files=extra_files, cleanup_on_exit=run_destroy) + plan = output = destroy = None + if run_plan: + plan = tf.plan(output=True) + if run_apply: + tf.apply() + output = tf.output(json_format=True) + if run_destroy: + tf.destroy() + return Result(tf, plan, output, destroy) + + return run + + +@pytest.fixture +def pretty_print(): + "Returns a fuction that pretty prints a data structure." + + def pretty_printer(data): + import json + print(json.dumps(data, indent=2)) + + return pretty_printer diff --git a/tests/modules/net_vpc/__init__.py b/tests/modules/net_vpc/__init__.py new file mode 100644 index 0000000000..35dff2e86e --- /dev/null +++ b/tests/modules/net_vpc/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/tests/modules/net_vpc/conftest.py b/tests/modules/net_vpc/conftest.py new file mode 100644 index 0000000000..9c9906e421 --- /dev/null +++ b/tests/modules/net_vpc/conftest.py @@ -0,0 +1,29 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"Apply fixture." + +import os + +import pytest + + +# path of this folder relative to root +_PATH = os.path.sep.join(os.path.abspath(__file__).split(os.path.sep)[-3:-1]) + + +@pytest.fixture(scope='module') +def fix_path(): + "Returns a function that prepends the test module path." + return lambda p: os.path.join(_PATH, p) diff --git a/tests/modules/net_vpc/fixtures/vpc-iam-bindings/main.tf b/tests/modules/net_vpc/fixtures/vpc-iam-bindings/main.tf new file mode 100644 index 0000000000..a897fc5910 --- /dev/null +++ b/tests/modules/net_vpc/fixtures/vpc-iam-bindings/main.tf @@ -0,0 +1,99 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project id used for this fixture." + type = string +} + +variable "subnets" { + description = "Subnet definitions." + default = { + subnet-a = { + ip_cidr_range = "192.168.0.0/24" + region = "europe-west1" + description = "First subnet." + private_ip_google_access = false + enable_flow_logs = false + secondary_ip_range = {} + }, + subnet-b = { + ip_cidr_range = "192.168.1.0/24" + region = "europe-west1" + description = "Second subnet." + private_ip_google_access = false + enable_flow_logs = false + secondary_ip_range = {} + }, + subnet-c = { + ip_cidr_range = "192.168.2.0/24" + region = "europe-west1" + description = "Third subnet." + private_ip_google_access = false + enable_flow_logs = false + secondary_ip_range = {} + }, + } +} + +locals { + members = [ + for s in google_service_account.binding_members : + "serviceAccount:${s.email}" + ] +} + +resource "google_service_account" "binding_members" { + for_each = toset(split(" ", "a b c d e")) + project = var.project_id + account_id = "user-${each.value}" +} + +module "vpc" { + source = "../../../../net-vpc" + project_id = var.project_id + name = "vpc-iam-bindings" + description = "Created by the vpc-iam-bindings fixture." + subnets = var.subnets + iam_roles = { + subnet-b = ["roles/compute.networkUser", "roles/compute.networkViewer"] + subnet-c = ["roles/compute.networkViewer"] + } + iam_members = { + subnet-b = { + "roles/compute.networkUser" = slice(local.members, 0, 2) + "roles/compute.networkViewer" = slice(local.members, 3, 4) + } + subnet-c = { + "roles/compute.networkViewer" = slice(local.members, 3, 5) + } + } +} + +output "network" { + description = "Network resource." + value = module.vpc.network +} + +output "subnets" { + description = "Subnet resources." + value = module.vpc.subnets +} + +output "bindings" { + description = "Subnet IAM bindings." + value = module.vpc.bindings +} diff --git a/tests/modules/net_vpc/fixtures/vpc-standalone/main.tf b/tests/modules/net_vpc/fixtures/vpc-standalone/main.tf new file mode 100644 index 0000000000..5c7a018e2a --- /dev/null +++ b/tests/modules/net_vpc/fixtures/vpc-standalone/main.tf @@ -0,0 +1,47 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "name" { + description = "Network name." + type = string + default = "net-vpc-standalone" +} + +variable "project_id" { + description = "Project id used for this fixture." + type = string +} + +module "vpc-simple" { + source = "../../../../net-vpc" + project_id = var.project_id + name = var.name +} + +output "name" { + description = "Network name." + value = module.vpc-simple.name +} + +output "self_link" { + description = "Network self link." + value = module.vpc-simple.self_link +} + +output "subnets" { + description = "Subnet resources." + value = module.vpc-simple.subnets +} diff --git a/tests/modules/net_vpc/fixtures/vpc-subnets/main.tf b/tests/modules/net_vpc/fixtures/vpc-subnets/main.tf new file mode 100644 index 0000000000..01c924df38 --- /dev/null +++ b/tests/modules/net_vpc/fixtures/vpc-subnets/main.tf @@ -0,0 +1,97 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project id used for this fixture." + type = string +} + +variable "subnets" { + description = "Subnet definitions." + default = { + subnet-simple = { + ip_cidr_range = "192.168.0.0/24" + region = "europe-west1" + description = "Simple subnet." + private_ip_google_access = false + enable_flow_logs = false + secondary_ip_range = {} + }, + subnet-options = { + ip_cidr_range = "192.168.1.0/24" + region = "europe-west2" + description = "Simple subnet with options." + private_ip_google_access = true + enable_flow_logs = true + secondary_ip_range = {} + }, + subnet-alias-ranges = { + ip_cidr_range = "192.168.2.0/24" + region = "europe-west1" + description = "Simple subnet with alias ranges." + private_ip_google_access = true + enable_flow_logs = true + secondary_ip_range = { + alias-1 = "172.16.10.0/24" + alias-2 = "172.16.20.0/24" + } + } + } +} + +variable "log_configs" { + description = "Logging configurations." + default = { + subnet-alias-ranges = { + flow_sampling = 0.75 + } + } +} + +module "vpc" { + source = "../../../../net-vpc" + project_id = var.project_id + name = "vpc-subnets" + description = "Created by the vpc-subnets fixture." + routing_mode = "REGIONAL" + subnets = var.subnets + log_configs = var.log_configs +} + +output "network" { + description = "Network resource." + value = module.vpc.network +} + +output "subnets" { + description = "Subnet resources." + value = module.vpc.subnets +} + +output "subnet_ips" { + description = "Map of subnet address ranges keyed by name." + value = module.vpc.subnet_ips +} + +output "subnet_regions" { + description = "Map of subnet regions keyed by name." + value = module.vpc.subnet_regions +} + +output "subnet_secondary_ranges" { + description = "Map of subnet secondary ranges keyed by name." + value = module.vpc.subnet_secondary_ranges +} diff --git a/tests/modules/net_vpc/test_vpc_iam_bindings.py b/tests/modules/net_vpc/test_vpc_iam_bindings.py new file mode 100644 index 0000000000..a936fe60cc --- /dev/null +++ b/tests/modules/net_vpc/test_vpc_iam_bindings.py @@ -0,0 +1,81 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections +import re + +import pytest + + +Binding = collections.namedtuple('Binding', 'subnet role members') + + +@pytest.fixture(scope='module') +def result(run_fixture, fix_path): + "Runs fixture interpolating current path, and returns result." + return run_fixture(fix_path('fixtures/vpc-iam-bindings')) + + +@pytest.fixture(scope='module') +def bindings(result): + "Returns a streamlined list of bindings." + return [ + Binding(b['subnetwork'].split('/')[-1], b['role'], b['members']) + for b in result.output['bindings'].values() + ] + + +@pytest.fixture(scope='module') +def subnet_names(result): + "Returns the list of subnet names." + return [s['name'] for s in result.output['subnets'].values()] + + +def test_vpc_attributes(result): + "Test network attributes." + network = result.output['network'] + assert network['routing_mode'] == 'GLOBAL' + assert network['description'] == 'Created by the vpc-iam-bindings fixture.' + + +def test_subnet_names(result, subnet_names): + "Test subnet names output." + resource_names = sorted( + [s['name'] for s in result.output['subnets'].values()]) + assert resource_names == sorted(subnet_names) + + +def test_binding_roles(result, bindings, subnet_names): + "Test that the correct roles from IAM bindings are set." + assert len([b for b in bindings if b.subnet == 'subnet-a']) == 0 + assert set([b.role for b in bindings if b.subnet == 'subnet-b']) == set([ + 'roles/compute.networkUser', 'roles/compute.networkViewer' + ]) + assert [b.role for b in bindings if b.subnet == 'subnet-c'] == [ + 'roles/compute.networkViewer' + ] + + +def test_binding_members(result, bindings, subnet_names): + "Test that the correct members from IAM bindings are set." + r = re.compile(r'^serviceAccount:([^@]+)@.*$') + for b in bindings: + members = [r.sub(r'\1', m) for m in b.members] + if b.subnet == 'subnet-b': + if b.role == 'roles/compute.networkUser': + assert members == ['user-a', 'user-b'] + else: + assert members == ['user-d'] + else: + assert members == ['user-d', 'user-e'] diff --git a/tests/modules/net_vpc/test_vpc_standalone.py b/tests/modules/net_vpc/test_vpc_standalone.py new file mode 100644 index 0000000000..1fa77394e6 --- /dev/null +++ b/tests/modules/net_vpc/test_vpc_standalone.py @@ -0,0 +1,29 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + + +@pytest.fixture(scope='module') +def result(run_fixture, fix_path): + "Runs fixture interpolating current path, and returns result." + return run_fixture(fix_path('fixtures/vpc-standalone')) + + +def test_vpc_creation(result): + "Test that VPC is created with the correct attributes." + assert result.output['name'] == result.plan.variables['name'] + assert result.output['self_link'].endswith(result.plan.variables['name']) + assert result.plan.variables['project_id'] in result.output['self_link'] + assert result.output['subnets'] == {} diff --git a/tests/modules/net_vpc/test_vpc_subnets.py b/tests/modules/net_vpc/test_vpc_subnets.py new file mode 100644 index 0000000000..008b77f8f0 --- /dev/null +++ b/tests/modules/net_vpc/test_vpc_subnets.py @@ -0,0 +1,63 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + + +@pytest.fixture(scope='module') +def result(run_fixture, fix_path): + "Runs fixture interpolating current path, and returns result." + return run_fixture(fix_path('fixtures/vpc-subnets')) + + +def test_vpc_attributes(result): + "Test network attributes." + network = result.output['network'] + assert network['routing_mode'] == 'REGIONAL' + assert network['description'] == 'Created by the vpc-subnets fixture.' + + +def test_subnet_names(result): + "Test subnet names output." + resource_names = sorted([s['name'] + for s in result.output['subnets'].values()]) + assert resource_names == sorted( + [k for k in result.plan.variables['subnets']]) + + +def test_subnet_ips(result): + "Test subnet IPs output." + for name, attrs in result.plan.variables['subnets'].items(): + assert result.output['subnet_ips'][name] == attrs['ip_cidr_range'] + + +def test_subnet_regions(result): + "Test subnet regions output." + assert result.output['subnet_regions'] == dict( + (k, v['region']) for k, v in result.plan.variables['subnets'].items()) + + +def test_secondary_ip_ranges(result): + "Test subnet secondary ranges output." + for name, attrs in result.plan.variables['subnets'].items(): + ranges = [{'range_name': k, 'ip_cidr_range': v} + for k, v in attrs['secondary_ip_range'].items()] + assert ranges == result.output['subnet_secondary_ranges'][name] + + +def test_flow_logs(result): + "Test that log config is set using the enable flow logs variable." + for name, attrs in result.plan.variables['subnets'].items(): + log_config = 1 if attrs['enable_flow_logs'] else 0 + assert len(result.output['subnets'][name]['log_config']) == log_config diff --git a/tests/requirements.txt b/tests/requirements.txt index a5b3fbc2d0..6f5740dffd 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,3 +1,3 @@ pytest>=4.3.1 pytest-tldr>=0.2.1 -tftest>=1.2.0 +tftest>=1.3.0 From db5005e3340981a736db6b1265ded7d714113ec6 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Mon, 25 Nov 2019 02:39:20 +0100 Subject: [PATCH 008/106] add TODO to net-vpc module --- modules/net-vpc/main.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/net-vpc/main.tf b/modules/net-vpc/main.tf index f687adca02..859343dc00 100644 --- a/modules/net-vpc/main.tf +++ b/modules/net-vpc/main.tf @@ -56,6 +56,8 @@ resource "google_compute_shared_vpc_host_project" "shared_vpc_host" { depends_on = [google_compute_network.network] } +# TODO(ludoo): add shared vpc service project registration + resource "google_compute_subnetwork" "subnetwork" { for_each = var.subnets project = var.project_id From 370c5fed2f04c91f33a7780275400eb62dff6351 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Tue, 26 Nov 2019 16:37:42 +0100 Subject: [PATCH 009/106] add minimal README files with input/output variables to gke and net-vpc modules --- modules/gke-cluster/README.md | 52 +++++++++++++++++++++++++++++++++++ modules/net-vpc/README.md | 38 +++++++++++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 modules/net-vpc/README.md diff --git a/modules/gke-cluster/README.md b/modules/gke-cluster/README.md index e69de29bb2..86b152263b 100644 --- a/modules/gke-cluster/README.md +++ b/modules/gke-cluster/README.md @@ -0,0 +1,52 @@ +# Minimalistic GKE module + +TODO(ludoo): add description. + +## Example usage + +TODO(ludoo): add example + +## Variables + +| name | description | type | required | +|---|---|:---: |:---:| +| location | Cluster zone or region. | string | ✓ +| name | Cluster name. | string | ✓ +| network | Name or self link of the VPC used for the cluster. Use the self link for Shared VPC. | string | ✓ +| project_id | Cluster project id. | string | ✓ +| secondary_range_pods | Subnet secondary range name used for pods. | string | ✓ +| secondary_range_services | Subnet secondary range name used for services. | string | ✓ +| subnetwork | VPC subnetwork name or self link. | string | ✓ +| *addons* | Addons enabled in the cluster (true means enabled). | object | +| *authenticator_security_group* | RBAC security group for Google Groups for GKE, format is gke-security-groups@yourdomain.com. | string | +| *cluster_autoscaling* | Enable and configure limits for Node Auto-Provisioning with Cluster Autoscaler. | object | +| *database_encryption* | Enable and configure GKE application-layer secrets encryption. | object | +| *default_max_pods_per_node* | Maximum number of pods per node in this cluster. | number | +| *description* | Cluster description. | string | +| *enable_binary_authorization* | Enable Google Binary Authorization. | bool | +| *enable_intranode_visibility* | Enable intra-node visibility to make same node pod to pod traffic visible. | bool | +| *enable_shielded_nodes* | Enable Shielded Nodes features on all nodes in this cluster. | bool | +| *enable_tpu* | Enable Cloud TPU resources in this cluster. | bool | +| *labels* | Cluster resource labels. | map | +| *logging_service* | Logging service (disable with an empty string). | string | +| *maintenance_start_time* | Maintenance start time in RFC3339 format 'HH:MM', where HH is [00-23] and MM is [00-59] GMT. | string | +| *master_authorized_ranges* | External Ip address ranges that can access the Kubernetes cluster master through HTTPS. | map | +| *min_master_version* | Minimum version of the master, defaults to the version of the most recent official release. | string | +| *monitoring_service* | Monitoring service (disable with an empty string). | string | +| *node_locations* | Zones in which the cluster's nodes are located. | list | +| *pod_security_policy* | Enable the PodSecurityPolicy feature. | bool | +| *private_cluster* | Enable private cluster. | bool | +| *private_cluster_config* | Private cluster configuration. | object | +| *release_channel* | Release channel for GKE upgrades. | string | +| *resource_usage_export_config* | Configure the ResourceUsageExportConfig feature. | object | +| *vertical_pod_autoscaling* | Enable the Vertical Pod Autoscaling feature. | bool | +| *workload_identity* | Enable the Workload Identity feature. | bool | + +## Outputs + +| name | description | +|---|---| +| cluster | Cluster resource. | +| endpoint | Cluster endpoint. | +| master_version | Master version. | +| name | Cluster name. | diff --git a/modules/net-vpc/README.md b/modules/net-vpc/README.md new file mode 100644 index 0000000000..e7765e2bdd --- /dev/null +++ b/modules/net-vpc/README.md @@ -0,0 +1,38 @@ +# Minimalistic VPC module + +TODO(ludoo): add description. + +## Example usage + +TODO(ludoo): add example + +## Variables + +| name | description | type | required | +|---|---|:---: |:---:| +| name | The name of the network being created | string | ✓ +| project_id | The ID of the project where this VPC will be created | string | ✓ +| *auto_create_subnetworks* | When set to true, the network is created in 'auto subnet mode' and it will create a subnet for each region automatically across the 10.128.0.0/9 address range. When set to false, the network is created in 'custom subnet mode' so the user can explicitly connect subnetwork resources. | bool | +| *description* | An optional description of this resource. The resource must be recreated to modify this field. | string | +| *iam_members* | List of IAM members keyed by subnet and role. | map | +| *iam_roles* | List of IAM roles keyed by subnet. | map | +| *log_config_defaults* | Default configuration for flow logs when enabled. | object | +| *log_configs* | Map of per-subnet optional configurations for flow logs when enabled. | map | +| *routing_mode* | The network routing mode (default 'GLOBAL') | string | +| *shared_vpc_host* | Makes this project a Shared VPC host if 'true' (default 'false') | bool | +| *subnets* | The list of subnets being created | map | + +## Outputs + +| name | description | +|---|---| +| bindings | Subnet IAM bindings. | +| name | The name of the VPC being created. | +| network | Network resource. | +| project_id | Shared VPC host project id. | +| self_link | The URI of the VPC being created. | +| subnet_ips | Map of subnet address ranges keyed by name. | +| subnet_regions | Map of subnet regions keyed by name. | +| subnet_secondary_ranges | Map of subnet secondary ranges keyed by name. | +| subnet_self_links | Map of subnet self links keyed by name. | +| subnets | Subnet resources. | From 8930b94998f9ffba377c6d9308ae300a0960d492 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Thu, 28 Nov 2019 17:25:02 +0100 Subject: [PATCH 010/106] BigQuery Module (#24) * Bigquery Module * Added README file * Added type hints --- modules/bigquery/README.md | 141 ++++++++++++++++++++++++++++++++++++ modules/bigquery/main.tf | 74 +++++++++++++++++++ modules/bigquery/outputs.tf | 27 +++++++ modules/bigquery/vars.tf | 70 ++++++++++++++++++ 4 files changed, 312 insertions(+) create mode 100644 modules/bigquery/README.md create mode 100644 modules/bigquery/main.tf create mode 100644 modules/bigquery/outputs.tf create mode 100644 modules/bigquery/vars.tf diff --git a/modules/bigquery/README.md b/modules/bigquery/README.md new file mode 100644 index 0000000000..55a7a09ef2 --- /dev/null +++ b/modules/bigquery/README.md @@ -0,0 +1,141 @@ +# Opinionated BigQuery module + +TODO(jccb): add description. + +## Example usage + +```hcl +module "bq" { + source = "../modules/bigquery/" + project_id = "my-project" + datasets = [ + { + id = "dataset1" + name = "dataset1" + description = "dataset1 description" + location = "EU" + labels = { + environment = "dev" + } + # default_partition_expiration_ms = 10000 + # default_table_expiration_ms - 10000 + } ] + + dataset_access = { + dataset1 = { + "me@myorg.com" = "roles/bigquery.dataViewer" + } + } + + tables = [ + { + table_id = "table1" + dataset_id = "dataset1" + description = "dataset1 description" + labels = { + team = "sales" + } + schema = [ + { + type = "DATE" + name = "date" + }, + { + type = "STRING" + name = "name" + } + ] + time_partitioning = { + field = "date" + type = "DAY" + } + } + ] + + views = [ + { + dataset = "dataset1" + table = "view1" + query = "select date from `my-project.dataset1.table1`" + } + ] +} +``` + +## Variables + +| name | description | type | required | +|----------------|-------------------------------------------------------------|:------:|:--------:| +| project_id | The ID of the project where the datasets, tables, and views | string | ✓ | +| datasets | List of datasets to be created (see structure below) | string | | +| tables | List of tables to be created (see structure below) | string | | +| views | List of views to be created (see structure below) | string | | +| dataset_access | Dataset-level permissions to be granted | string | | + +### dataset structure +The datasets list should contains objects with the following structure: + +``` hcl + { + id = "dataset id" + name = "dataset name" + description = "dataset description" + location = "dataset location" + labels = { # optional + label1 = "value1" + } + default_partition_expiration_ms = 10000 # optional + default_table_expiration_ms - 10000 # optional + } + +``` + +### table structure + +``` hcl + { + table_id = "table name" + dataset_id = "table dataset" + description = "table description" + labels = { + label1 = "value1" + } + schema = [ + { + type = "DATE" + name = "date" + } + { + type = "STRING" + name = "name" + } + ] + } + +``` + +You can optionally specify ```expiration_time```, +```time_partitioning``` and ```clustering``` for any table. + + +### views structure + +The structure for the views field is just a dataset (where the view +will be created), the table (view name), and the query for the view. Keep in mind that the query should use the complete name for any table it references. + +``` hcl + { + dataset = "dataset1" + table = "view1" + query = "select date from `my-project.dataset.table`" + } +``` + + +## Outputs + +| name | description | +|----------|----------------------------------------| +| datasets | Map of dataset objects keyed by name | +| tables | List of table objects | +| views | List of table objects created as views | diff --git a/modules/bigquery/main.tf b/modules/bigquery/main.tf new file mode 100644 index 0000000000..7421d22f01 --- /dev/null +++ b/modules/bigquery/main.tf @@ -0,0 +1,74 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_bigquery_dataset" "datasets" { + count = length(var.datasets) + project = var.project_id + dataset_id = var.datasets[count.index]["id"] + friendly_name = var.datasets[count.index]["name"] + description = var.datasets[count.index]["description"] + location = var.datasets[count.index]["location"] + labels = lookup(var.datasets[count.index], "labels", null) + default_table_expiration_ms = lookup(var.datasets[count.index], "default_table_expiration_ms", null) + default_partition_expiration_ms = lookup(var.datasets[count.index], "default_partition_expiration_ms", null) + + dynamic "access" { + for_each = lookup(var.dataset_access, var.datasets[count.index]["id"], tomap({})) + content { + user_by_email = access.key + role = access.value + } + } + + access { + role = "OWNER" + special_group = "projectOwners" + } +} + +resource "google_bigquery_table" "tables" { + count = length(var.tables) + project = var.project_id + dataset_id = var.tables[count.index]["dataset_id"] + table_id = var.tables[count.index]["table_id"] + labels = var.tables[count.index]["labels"] + expiration_time = lookup(var.tables[count.index], "expiration_time", null) + clustering = lookup(var.tables[count.index], "clustering", null) + schema = jsonencode(var.tables[count.index]["schema"]) + + dynamic "time_partitioning" { + for_each = lookup(var.tables[count.index], "time_partitioning", false) == false ? [] : [1] + content { + field = var.tables[count.index]["time_partitioning"]["field"] + type = var.tables[count.index]["time_partitioning"]["type"] + expiration_ms = lookup(var.tables[count.index], "expiration_ms", null) + } + } + + depends_on = [google_bigquery_dataset.datasets] +} + +resource "google_bigquery_table" "views" { + count = length(var.views) + project = var.project_id + dataset_id = var.views[count.index]["dataset"] + table_id = var.views[count.index]["table"] + view { + query = var.views[count.index]["query"] + use_legacy_sql = false + } + depends_on = [google_bigquery_table.tables] +} diff --git a/modules/bigquery/outputs.tf b/modules/bigquery/outputs.tf new file mode 100644 index 0000000000..6eabd78f13 --- /dev/null +++ b/modules/bigquery/outputs.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "datasets" { + value = zipmap(google_bigquery_dataset.datasets[*].dataset_id, google_bigquery_dataset.datasets) +} + +output "tables" { + value = google_bigquery_table.tables +} + +output "views" { + value = google_bigquery_table.views +} diff --git a/modules/bigquery/vars.tf b/modules/bigquery/vars.tf new file mode 100644 index 0000000000..077507be44 --- /dev/null +++ b/modules/bigquery/vars.tf @@ -0,0 +1,70 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project ID" + type = string +} + +variable "datasets" { + description = "Datatase IDs" + default = [] + type = list + # type = list(object({ + # id = string + # name = string + # description = string + # location = string + # # labels = map(string) # optional + # # default_partition_expiration_ms = number # optional + # # default_table_expiration_ms = number # optional + # })) +} + +variable "tables" { + description = "Tables" + default = [] + type = list + # type = list(object({ + # table_id = string + # dataset_id = string + # labels = map(string) + # schema = string + # # expiration_time = number # optional + # # clustering = string # optional + # # time_partitioning = object({ # optional + # # field = string + # # type = string + # # expiration_ms = number # optional + # # } + # })) +} + +variable "views" { + description = "Views" + default = [] + type = list(object({ + dataset = string + table = string + query = string + })) +} + +variable "dataset_access" { + description = "Dataset permissions" + default = {} + type = map(map(string)) +} From 8eb8fc29d693e35b542be816b50deee6571c6df3 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 8 Dec 2019 12:47:11 +0100 Subject: [PATCH 011/106] GCS module --- modules/gcs/README.md | 43 +++++++++++ modules/gcs/main.tf | 87 +++++++++++++++++++++ modules/gcs/outputs.tf | 55 ++++++++++++++ modules/gcs/variables.tf | 158 +++++++++++++++++++++++++++++++++++++++ modules/gcs/versions.tf | 19 +++++ 5 files changed, 362 insertions(+) create mode 100644 modules/gcs/README.md create mode 100644 modules/gcs/main.tf create mode 100644 modules/gcs/outputs.tf create mode 100644 modules/gcs/variables.tf create mode 100644 modules/gcs/versions.tf diff --git a/modules/gcs/README.md b/modules/gcs/README.md new file mode 100644 index 0000000000..30aa427ae3 --- /dev/null +++ b/modules/gcs/README.md @@ -0,0 +1,43 @@ +# Google Cloud Storage Module + +## Variables + +| name | description | type | required | +|---|---|:---: |:---:| +| names | Bucket name suffixes. | list | ✓ +| prefix | Prefix used to generate the bucket name. | string | ✓ +| project_id | Bucket project id. | string | ✓ +| *admins* | IAM-style members who will be granted roles/storage.admin on all buckets. | list | +| *bucket_admins* | Map of lowercase unprefixed name => comma-delimited IAM-style bucket storage admins. | map | +| *bucket_creators* | Map of lowercase unprefixed name => comma-delimited IAM-style per-bucket object creators. | map | +| *bucket_hmackey_admins* | Map of lowercase unprefixed name => comma-delimited IAM-style per-bucket hmacKey admins. | map | +| *bucket_object_admins* | Map of lowercase unprefixed name => comma-delimited IAM-style per-bucket object admins. | map | +| *bucket_policy_only* | Disable ad-hoc ACLs on specified buckets. Defaults to true. Map of lowercase unprefixed name => boolean | map | +| *bucket_viewers* | Map of lowercase unprefixed name => comma-delimited IAM-style per-bucket object viewers. | map | +| *creators* | IAM-style members who will be granted roles/storage.objectCreators on all buckets. | list | +| *force_destroy* | Optional map of lowercase unprefixed name => boolean, defaults to false. | map | +| *hmackey_admins* | IAM-style members who will be granted roles/storage.hmacKeyAdmin on all buckets. | list | +| *labels* | Labels to be attached to the buckets | map | +| *location* | Bucket location. | string | +| *object_admins* | IAM-style members who will be granted roles/storage.objectAdmin on all buckets. | list | +| *set_admin_roles* | Grant roles/storage.admin role to storage_admins and bucket_storage_admins. | bool | +| *set_creator_roles* | Grant roles/storage.objectCreator role to creators and bucket_creators. | bool | +| *set_hmackey_admin_roles* | Grant roles/storage.hmacKeyAdmin role to storage_admins and bucket_storage_admins. | bool | +| *set_object_admin_roles* | Grant roles/storage.objectAdmin role to admins and bucket_admins. | bool | +| *set_viewer_roles* | Grant roles/storage.objectViewer role to viewers and bucket_viewers. | bool | +| *storage_class* | Bucket storage class. | string | +| *versioning* | Optional map of lowercase unprefixed name => boolean, defaults to false. | map | +| *viewers* | IAM-style members who will be granted roles/storage.objectViewer on all buckets. | list | + +## Outputs + +| name | description | +|---|---| +| bucket | Bucket resource (for single use). | +| buckets | Bucket resources. | +| name | Bucket name (for single use). | +| names | Bucket names. | +| names_list | List of bucket names. | +| url | Bucket URL (for single use). | +| urls | Bucket URLs. | +| urls_list | List of bucket URLs. | diff --git a/modules/gcs/main.tf b/modules/gcs/main.tf new file mode 100644 index 0000000000..0e986d146f --- /dev/null +++ b/modules/gcs/main.tf @@ -0,0 +1,87 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + prefix = var.prefix == "" ? "" : join("-", list(var.prefix, lower(var.location), "")) + names = toset(var.names) + buckets_list = [for name in var.names : google_storage_bucket.buckets[name]] +} + +resource "google_storage_bucket" "buckets" { + for_each = local.names + name = "${local.prefix}${lower(each.key)}" + project = var.project_id + location = var.location + storage_class = var.storage_class + force_destroy = lookup(var.force_destroy, each.key, false) + bucket_policy_only = lookup(var.bucket_policy_only, each.key, true) + versioning { + enabled = lookup(var.versioning, each.key, false) + } + labels = merge(var.labels, { + name = "${local.prefix}${lower(each.key)}" + }) +} + +resource "google_storage_bucket_iam_binding" "object_admins" { + for_each = var.set_object_admin_roles ? local.names : toset([]) + bucket = google_storage_bucket.buckets[each.key].name + role = "roles/storage.objectAdmin" + members = compact(concat( + var.object_admins, + split(",", lookup(var.bucket_object_admins, each.key, "")) + )) +} + +resource "google_storage_bucket_iam_binding" "creators" { + for_each = var.set_creator_roles ? local.names : toset([]) + bucket = google_storage_bucket.buckets[each.key].name + role = "roles/storage.objectCreator" + members = compact(concat( + var.creators, + split(",", lookup(var.bucket_creators, each.key, "")) + )) +} + +resource "google_storage_bucket_iam_binding" "viewers" { + for_each = var.set_viewer_roles ? local.names : toset([]) + bucket = google_storage_bucket.buckets[each.key].name + role = "roles/storage.objectViewer" + members = compact(concat( + var.viewers, + split(",", lookup(var.bucket_viewers, each.key, "")) + )) +} + +resource "google_storage_bucket_iam_binding" "hmackey_admins" { + for_each = var.set_hmackey_admin_roles ? local.names : toset([]) + bucket = google_storage_bucket.buckets[each.key].name + role = "roles/storage.hmacKeyAdmin" + members = compact(concat( + var.hmackey_admins, + split(",", lookup(var.bucket_hmackey_admins, each.key, "")) + )) +} + +resource "google_storage_bucket_iam_binding" "admins" { + for_each = var.set_admin_roles ? local.names : toset([]) + bucket = google_storage_bucket.buckets[each.key].name + role = "roles/storage.admin" + members = compact(concat( + var.admins, + split(",", lookup(var.bucket_admins, each.key, "")) + )) +} diff --git a/modules/gcs/outputs.tf b/modules/gcs/outputs.tf new file mode 100644 index 0000000000..b6b6422b13 --- /dev/null +++ b/modules/gcs/outputs.tf @@ -0,0 +1,55 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "bucket" { + description = "Bucket resource (for single use)." + value = length(var.names) > 0 ? google_storage_bucket.buckets[var.names[0]] : null +} + +output "name" { + description = "Bucket name (for single use)." + value = length(var.names) > 0 ? google_storage_bucket.buckets[var.names[0]].name : null +} + +output "url" { + description = "Bucket URL (for single use)." + value = length(var.names) > 0 ? google_storage_bucket.buckets[var.names[0]].url : null +} + +output "buckets" { + description = "Bucket resources." + value = google_storage_bucket.buckets +} + +output "names" { + description = "Bucket names." + value = { for name in var.names : name => google_storage_bucket.buckets[name].name } +} + +output "urls" { + description = "Bucket URLs." + value = { for name in var.names : name => google_storage_bucket.buckets[name].url } +} + +output "names_list" { + description = "List of bucket names." + value = [for bucket in local.buckets_list : bucket.name] +} + +output "urls_list" { + description = "List of bucket URLs." + value = [for bucket in local.buckets_list : bucket.url] +} diff --git a/modules/gcs/variables.tf b/modules/gcs/variables.tf new file mode 100644 index 0000000000..7f52faad56 --- /dev/null +++ b/modules/gcs/variables.tf @@ -0,0 +1,158 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Bucket project id." + type = string +} + +variable "prefix" { + description = "Prefix used to generate the bucket name." + type = string +} + +variable "names" { + description = "Bucket name suffixes." + type = list(string) +} + +variable "location" { + description = "Bucket location." + type = string + default = "EU" +} + +variable "storage_class" { + description = "Bucket storage class." + type = string + default = "MULTI_REGIONAL" +} + +variable "force_destroy" { + description = "Optional map of lowercase unprefixed name => boolean, defaults to false." + type = map(bool) + default = {} +} + +variable "versioning" { + description = "Optional map of lowercase unprefixed name => boolean, defaults to false." + type = map(bool) + default = {} +} + +variable "bucket_policy_only" { + description = "Disable ad-hoc ACLs on specified buckets. Defaults to true. Map of lowercase unprefixed name => boolean" + type = map(bool) + default = {} +} + +variable "object_admins" { + description = "IAM-style members who will be granted roles/storage.objectAdmin on all buckets." + type = list(string) + default = [] +} + +variable "creators" { + description = "IAM-style members who will be granted roles/storage.objectCreators on all buckets." + type = list(string) + default = [] +} + +variable "viewers" { + description = "IAM-style members who will be granted roles/storage.objectViewer on all buckets." + type = list(string) + default = [] +} + +variable "hmackey_admins" { + description = "IAM-style members who will be granted roles/storage.hmacKeyAdmin on all buckets." + type = list(string) + default = [] +} + +variable "admins" { + description = "IAM-style members who will be granted roles/storage.admin on all buckets." + type = list(string) + default = [] +} + +variable "bucket_object_admins" { + description = "Map of lowercase unprefixed name => comma-delimited IAM-style per-bucket object admins." + type = map(string) + default = {} +} + +variable "bucket_creators" { + description = "Map of lowercase unprefixed name => comma-delimited IAM-style per-bucket object creators." + type = map(string) + default = {} +} + +variable "bucket_viewers" { + description = "Map of lowercase unprefixed name => comma-delimited IAM-style per-bucket object viewers." + type = map(string) + default = {} +} + +variable "bucket_hmackey_admins" { + description = "Map of lowercase unprefixed name => comma-delimited IAM-style per-bucket hmacKey admins." + type = map(string) + default = {} +} + +variable "bucket_admins" { + description = "Map of lowercase unprefixed name => comma-delimited IAM-style bucket storage admins." + type = map(string) + default = {} +} + +variable "labels" { + description = "Labels to be attached to the buckets" + type = map(string) + default = {} +} + +# we need flags to allow member lists to contain dynamic elements + +variable "set_object_admin_roles" { + description = "Grant roles/storage.objectAdmin role to admins and bucket_admins." + type = bool + default = false +} + +variable "set_creator_roles" { + description = "Grant roles/storage.objectCreator role to creators and bucket_creators." + type = bool + default = false +} + +variable "set_viewer_roles" { + description = "Grant roles/storage.objectViewer role to viewers and bucket_viewers." + type = bool + default = false +} + +variable "set_hmackey_admin_roles" { + description = "Grant roles/storage.hmacKeyAdmin role to storage_admins and bucket_storage_admins." + type = bool + default = false +} + +variable "set_admin_roles" { + description = "Grant roles/storage.admin role to storage_admins and bucket_storage_admins." + type = bool + default = false +} diff --git a/modules/gcs/versions.tf b/modules/gcs/versions.tf new file mode 100644 index 0000000000..84432a3b20 --- /dev/null +++ b/modules/gcs/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12.9" +} From aa3a184100c62e8cfd9ac11c85c5aea1b2e288fc Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Mon, 9 Dec 2019 09:15:43 +0100 Subject: [PATCH 012/106] net vpc module: improve secondary range outputs --- modules/net-vpc/outputs.tf | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/modules/net-vpc/outputs.tf b/modules/net-vpc/outputs.tf index a6866c3365..b7f88f052e 100644 --- a/modules/net-vpc/outputs.tf +++ b/modules/net-vpc/outputs.tf @@ -60,7 +60,13 @@ output "subnet_regions" { output "subnet_secondary_ranges" { description = "Map of subnet secondary ranges keyed by name." - value = { for k, v in google_compute_subnetwork.subnetwork : k => v.secondary_ip_range } + value = { + for k, v in google_compute_subnetwork.subnetwork : + k => { + for range in v.secondary_ip_range : + range.range_name => range.ip_cidr_range + } + } } output "bindings" { From 61cd6575a4fca582b58e00ffe702da3999f7ea47 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Mon, 9 Dec 2019 09:23:17 +0100 Subject: [PATCH 013/106] net vpc module: add serve project registration --- modules/net-vpc/main.tf | 7 ++++++- modules/net-vpc/outputs.tf | 4 ++++ modules/net-vpc/variables.tf | 6 ++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/modules/net-vpc/main.tf b/modules/net-vpc/main.tf index 859343dc00..102b35afef 100644 --- a/modules/net-vpc/main.tf +++ b/modules/net-vpc/main.tf @@ -56,7 +56,12 @@ resource "google_compute_shared_vpc_host_project" "shared_vpc_host" { depends_on = [google_compute_network.network] } -# TODO(ludoo): add shared vpc service project registration +resource "google_compute_shared_vpc_service_project" "service_projects" { + for_each = var.shared_vpc_host ? toset(var.shared_vpc_service_projects) : toset([]) + host_project = var.project_id + service_project = each.value + depends_on = [google_compute_shared_vpc_host_project.shared_vpc_host] +} resource "google_compute_subnetwork" "subnetwork" { for_each = var.subnets diff --git a/modules/net-vpc/outputs.tf b/modules/net-vpc/outputs.tf index b7f88f052e..6886286b56 100644 --- a/modules/net-vpc/outputs.tf +++ b/modules/net-vpc/outputs.tf @@ -36,6 +36,10 @@ output "project_id" { ? google_compute_shared_vpc_host_project.shared_vpc_host[*].project : null ) + depends_on = [ + google_compute_shared_vpc_host_project.shared_vpc_host, + google_compute_shared_vpc_service_project.service_projects + ] } output "subnets" { diff --git a/modules/net-vpc/variables.tf b/modules/net-vpc/variables.tf index c227c10564..636b243964 100644 --- a/modules/net-vpc/variables.tf +++ b/modules/net-vpc/variables.tf @@ -80,6 +80,12 @@ variable "shared_vpc_host" { default = false } +variable "shared_vpc_service_projects" { + description = "Shared VPC service projects to register with this host" + type = list(string) + default = [] +} + variable "subnets" { description = "The list of subnets being created" type = map(object({ From 8a1c78393540e03071bcd6cdc50d4e46213ea935 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Mon, 9 Dec 2019 16:00:45 +0100 Subject: [PATCH 014/106] project module --- modules/project/README.md | 47 +++++++++++++ modules/project/main.tf | 124 +++++++++++++++++++++++++++++++++++ modules/project/outputs.tf | 56 ++++++++++++++++ modules/project/variables.tf | 114 ++++++++++++++++++++++++++++++++ 4 files changed, 341 insertions(+) create mode 100644 modules/project/README.md create mode 100644 modules/project/main.tf create mode 100644 modules/project/outputs.tf create mode 100644 modules/project/variables.tf diff --git a/modules/project/README.md b/modules/project/README.md new file mode 100644 index 0000000000..2228edc408 --- /dev/null +++ b/modules/project/README.md @@ -0,0 +1,47 @@ +# Google Cloud Simple Project Creation + +This module allows simple Google Cloud Platform project creation, with minimal service and project-level IAM binding management. It's designed to be used for architectural design and rapid prototyping, as part of the [Cloud Foundation Fabric](https://github.com/terraform-google-modules/cloud-foundation-fabric) environments. + +The resources/services/activations/deletions that this module will create/trigger are: + +- one project +- zero or one project metadata items for OSLogin activation +- zero or more project service activations +- zero or more project-level IAM bindings +- zero or more project-level custom roles +- zero or one project liens + +## Variables + +| name | description | type | required | +|---|---|:---: |:---:| +| name | Project name and id suffix. | string | ✓ +| parent | The resource name of the parent Folder or Organization. Must be of the form folders/folder_id or organizations/org_id. | string | ✓ +| prefix | Prefix used to generate project id and name. | string | ✓ +| *auto_create_network* | Whether to create the default network for the project | bool | +| *billing_account* | Billing account id. | string | +| *custom_roles* | Map of role name => list of permissions to create in this project. | map | +| *iam_authoritative_members* | Map of member lists used to set authoritative bindings, keyed by role. | map | +| *iam_authoritative_roles* | List of roles used to set authoritative bindings. | list | +| *iam_non_authoritative_members* | Map of member lists used to set non authoritative bindings, keyed by role. | map | +| *iam_non_authoritative_roles* | List of roles used to set non authoritative bindings. | list | +| *labels* | Resource labels. | map | +| *lien_reason* | If non-empty, creates a project lien with this description. | string | +| *oslogin* | Enable OS Login. | bool | +| *oslogin_admins* | List of IAM-style identities that will be granted roles necessary for OS Login administrators. | list | +| *oslogin_users* | List of IAM-style identities that will be granted roles necessary for OS Login users. | list | +| *owners* | IAM-style identities that will be granted non-authoritative viewer role. | list | +| *services* | Service APIs to enable. | list | + +## Outputs + +| name | description | +|---|---| +| cloudsvc_service_account | Cloud services service account (depends on services). | +| custom_roles | Ids of the created custom roles. | +| gce_service_account | Default GCE service account (depends on services). | +| gke_service_account | Default GKE service account (depends on services). | +| name | Name (depends on services). | +| number | Project number (depends on services). | +| project_id | Project id (depends on services). | + diff --git a/modules/project/main.tf b/modules/project/main.tf new file mode 100644 index 0000000000..d1a444b868 --- /dev/null +++ b/modules/project/main.tf @@ -0,0 +1,124 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cloudsvc_service_account = "${google_project.project.number}@cloudservices.gserviceaccount.com" + gce_service_account = "${google_project.project.number}-compute@developer.gserviceaccount.com" + gke_service_account = "service-${google_project.project.number}@container-engine-robot.iam.gserviceaccount.com" + iam_non_authoritative_pairs = flatten([ + for role in var.iam_non_authoritative_roles : [ + for member in lookup(var.iam_non_authoritative_members, role, []) : + { role = role, member = member } + ] + ]) + iam_non_authoritative = { + for pair in local.iam_non_authoritative_pairs : + "${pair.role}-${pair.member}" => pair + } + parent_type = split("/", var.parent)[0] + parent_id = split("/", var.parent)[1] +} + +resource "google_project" "project" { + org_id = local.parent_type == "organizations" ? local.parent_id : "" + folder_id = local.parent_type == "folders" ? local.parent_id : "" + project_id = "${var.prefix}-${var.name}" + name = "${var.prefix}-${var.name}" + billing_account = var.billing_account + auto_create_network = var.auto_create_network + labels = var.labels +} + +resource "google_project_iam_custom_role" "roles" { + for_each = var.custom_roles + project = google_project.project.project_id + role_id = each.key + title = "Custom role ${each.key}" + description = "Terraform-managed" + permissions = each.value +} + +resource "google_compute_project_metadata_item" "oslogin_meta" { + count = var.oslogin ? 1 : 0 + project = google_project.project.project_id + key = "enable-oslogin" + value = "TRUE" + # depend on services or it will fail on destroy + depends_on = [google_project_service.project_services] +} + +resource "google_resource_manager_lien" "lien" { + count = var.lien_reason != "" ? 1 : 0 + parent = "projects/${google_project.project.number}" + restrictions = ["resourcemanager.projects.delete"] + origin = "created-by-terraform" + reason = var.lien_reason +} + +resource "google_project_service" "project_services" { + for_each = toset(var.services) + project = google_project.project.project_id + service = each.value + disable_on_destroy = true + disable_dependent_services = true +} + +# IAM notes: +# - external users need to have accepted the invitation email to join +# - oslogin roles also require role to list instances +# - non-authoritative roles might fail due to dynamic values + +resource "google_project_iam_binding" "authoritative" { + for_each = toset(var.iam_authoritative_roles) + project = google_project.project.project_id + role = each.value + members = lookup(var.iam_authoritative_members, each.value, []) +} + +resource "google_project_iam_member" "non_authoritative" { + for_each = length(var.iam_non_authoritative_roles) > 0 ? local.iam_non_authoritative : {} + project = google_project.project.project_id + role = each.value.role + member = each.value.member +} + +resource "google_project_iam_member" "oslogin_iam_serviceaccountuser" { + for_each = var.oslogin ? toset(distinct(concat(var.oslogin_admins, var.oslogin_users))) : toset([]) + project = google_project.project.project_id + role = "roles/iam.serviceAccountUser" + member = each.value +} + +resource "google_project_iam_member" "oslogin_compute_viewer" { + for_each = var.oslogin ? toset(distinct(concat(var.oslogin_admins, var.oslogin_users))) : toset([]) + project = google_project.project.project_id + role = "roles/compute.viewer" + member = each.value +} + +resource "google_project_iam_member" "oslogin_admins" { + for_each = var.oslogin ? toset(var.oslogin_admins) : toset([]) + project = google_project.project.project_id + role = "roles/compute.osAdminLogin" + member = each.value +} + +resource "google_project_iam_member" "oslogin_users" { + for_each = var.oslogin ? toset(var.oslogin_users) : toset([]) + project = google_project.project.project_id + role = "roles/compute.osLogin" + member = each.value +} diff --git a/modules/project/outputs.tf b/modules/project/outputs.tf new file mode 100644 index 0000000000..4bdb17fab6 --- /dev/null +++ b/modules/project/outputs.tf @@ -0,0 +1,56 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "project_id" { + description = "Project id (depends on services)." + value = google_project.project.project_id + depends_on = [google_project_service.project_services] +} + +output "name" { + description = "Name (depends on services)." + value = google_project.project.name + depends_on = [google_project_service.project_services] +} + +output "number" { + description = "Project number (depends on services)." + value = google_project.project.number + depends_on = [google_project_service.project_services] +} + +output "cloudsvc_service_account" { + description = "Cloud services service account (depends on services)." + value = "${local.cloudsvc_service_account}" + depends_on = [google_project_service.project_services] +} + +output "gce_service_account" { + description = "Default GCE service account (depends on services)." + value = local.gce_service_account + depends_on = [google_project_service.project_services] +} + +output "gke_service_account" { + description = "Default GKE service account (depends on services)." + value = local.gke_service_account + depends_on = [google_project_service.project_services] +} + +output "custom_roles" { + description = "Ids of the created custom roles." + value = [for role in google_project_iam_custom_role.roles : role.role_id] +} diff --git a/modules/project/variables.tf b/modules/project/variables.tf new file mode 100644 index 0000000000..f2dfcb948b --- /dev/null +++ b/modules/project/variables.tf @@ -0,0 +1,114 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "auto_create_network" { + description = "Whether to create the default network for the project" + type = bool + default = false +} + +variable "billing_account" { + description = "Billing account id." + type = string + default = "" +} + +variable "custom_roles" { + description = "Map of role name => list of permissions to create in this project." + type = map(list(string)) + default = {} +} + +variable "iam_authoritative_members" { + description = "Map of member lists used to set authoritative bindings, keyed by role." + type = map(list(string)) + default = {} +} + +variable "iam_authoritative_roles" { + description = "List of roles used to set authoritative bindings." + type = list(string) + default = [] +} + +variable "iam_non_authoritative_members" { + description = "Map of member lists used to set non authoritative bindings, keyed by role." + type = map(list(string)) + default = {} +} + +variable "iam_non_authoritative_roles" { + description = "List of roles used to set non authoritative bindings." + type = list(string) + default = [] +} + +variable "labels" { + description = "Resource labels." + type = map(string) + default = {} +} + +variable "lien_reason" { + description = "If non-empty, creates a project lien with this description." + type = string + default = "" +} + +variable "name" { + description = "Project name and id suffix." + type = string +} + +variable "oslogin" { + description = "Enable OS Login." + type = bool + default = false +} + +variable "oslogin_admins" { + description = "List of IAM-style identities that will be granted roles necessary for OS Login administrators." + type = list(string) + default = [] +} + +variable "oslogin_users" { + description = "List of IAM-style identities that will be granted roles necessary for OS Login users." + type = list(string) + default = [] +} + +variable "owners" { + description = "IAM-style identities that will be granted non-authoritative viewer role." + type = list(string) + default = [] +} + +variable "parent" { + description = "The resource name of the parent Folder or Organization. Must be of the form folders/folder_id or organizations/org_id." + type = string +} + +variable "prefix" { + description = "Prefix used to generate project id and name." + type = string +} + +variable "services" { + description = "Service APIs to enable." + type = list(string) + default = [] +} From 9e20918f907720d21f441cb157477344c9d1e046 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sat, 14 Dec 2019 18:33:02 +0100 Subject: [PATCH 015/106] move bigquery module to not-ready folder --- modules/{ => not-ready}/bigquery/README.md | 0 modules/{ => not-ready}/bigquery/main.tf | 0 modules/{ => not-ready}/bigquery/outputs.tf | 0 modules/{ => not-ready}/bigquery/vars.tf | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename modules/{ => not-ready}/bigquery/README.md (100%) rename modules/{ => not-ready}/bigquery/main.tf (100%) rename modules/{ => not-ready}/bigquery/outputs.tf (100%) rename modules/{ => not-ready}/bigquery/vars.tf (100%) diff --git a/modules/bigquery/README.md b/modules/not-ready/bigquery/README.md similarity index 100% rename from modules/bigquery/README.md rename to modules/not-ready/bigquery/README.md diff --git a/modules/bigquery/main.tf b/modules/not-ready/bigquery/main.tf similarity index 100% rename from modules/bigquery/main.tf rename to modules/not-ready/bigquery/main.tf diff --git a/modules/bigquery/outputs.tf b/modules/not-ready/bigquery/outputs.tf similarity index 100% rename from modules/bigquery/outputs.tf rename to modules/not-ready/bigquery/outputs.tf diff --git a/modules/bigquery/vars.tf b/modules/not-ready/bigquery/vars.tf similarity index 100% rename from modules/bigquery/vars.tf rename to modules/not-ready/bigquery/vars.tf From 26ab786bb0f6f055b489e85269f84b4808fc748a Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sat, 14 Dec 2019 18:33:13 +0100 Subject: [PATCH 016/106] folders module --- modules/folder/main.tf | 59 +++++++++++++++++++++++++++++++++++++ modules/folder/outputs.tf | 55 ++++++++++++++++++++++++++++++++++ modules/folder/variables.tf | 38 ++++++++++++++++++++++++ modules/folder/versions.tf | 19 ++++++++++++ 4 files changed, 171 insertions(+) create mode 100644 modules/folder/main.tf create mode 100644 modules/folder/outputs.tf create mode 100644 modules/folder/variables.tf create mode 100644 modules/folder/versions.tf diff --git a/modules/folder/main.tf b/modules/folder/main.tf new file mode 100644 index 0000000000..e988681d88 --- /dev/null +++ b/modules/folder/main.tf @@ -0,0 +1,59 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + folders = [for name in var.names : google_folder.folders[name]] + # distinct is needed to make the expanding function argument work + iam_pairs = concat([], distinct([ + for name, roles in var.iam_roles : + [for role in roles : { name = name, role = role }] + ])...) + iam_keypairs = { + for pair in local.iam_pairs : + "${pair.name}-${pair.role}" => pair + } +} + +resource "google_folder" "folders" { + for_each = toset(var.names) + display_name = each.value + parent = var.parent +} + +# give project creation access to service accounts +# https://cloud.google.com/resource-manager/docs/access-control-folders#granting_folder-specific_roles_to_enable_project_creation +# - external users need to have accepted the invitation email to join +# "roles/owner", +# "roles/resourcemanager.folderViewer", +# "roles/resourcemanager.projectCreator", +# "roles/compute.networkAdmin", + + +resource "google_folder_iam_binding" "authoritative" { + for_each = local.iam_keypairs + folder = google_folder.folders[each.value.name].name + role = each.value.role + members = lookup( + lookup(var.iam_members, each.value.name, {}), each.value.role, [] + ) +} + +# resource "google_folder_iam_member" "non_authoritative" { +# for_each = length(var.iam_non_authoritative_roles) > 0 ? local.iam_non_authoritative : {} +# folder = google_project.project.project_id +# role = each.value.role +# member = each.value.member +# } diff --git a/modules/folder/outputs.tf b/modules/folder/outputs.tf new file mode 100644 index 0000000000..18ac60c6fa --- /dev/null +++ b/modules/folder/outputs.tf @@ -0,0 +1,55 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "folder" { + description = "Folder resource (for single use)." + value = local.folders[0] +} + +output "id" { + description = "Folder id (for single use)." + value = local.folders[0].name +} + +output "name" { + description = "Folder name (for single use)." + value = local.folders[0].display_name +} + +output "folders" { + description = "Folder resources." + value = local.folders +} + +output "ids" { + description = "Folder ids." + value = zipmap(var.names, [for f in local.folders : f.name]) +} + +output "names" { + description = "Folder names." + value = zipmap(var.names, [for f in local.folders : f.display_name]) +} + +output "ids_list" { + description = "List of folder ids." + value = [for f in local.folders : f.name] +} + +output "names_list" { + description = "List of folder names." + value = [for f in local.folders : f.display_name] +} diff --git a/modules/folder/variables.tf b/modules/folder/variables.tf new file mode 100644 index 0000000000..9489bdacb5 --- /dev/null +++ b/modules/folder/variables.tf @@ -0,0 +1,38 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "iam_members" { + description = "List of IAM members keyed by folder name and role." + type = map(map(list(string))) + default = {} +} + +variable "iam_roles" { + description = "List of IAM roles keyed by folder name." + type = map(list(string)) + default = {} +} + +variable "names" { + description = "Folder names." + type = list(string) + default = [] +} + +variable "parent" { + description = "Parent in folders/folder_id or organizations/org_id format." + type = string +} diff --git a/modules/folder/versions.tf b/modules/folder/versions.tf new file mode 100644 index 0000000000..832ec1df39 --- /dev/null +++ b/modules/folder/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12" +} From fc92d305b6781a64777f3b64a2f86afa32bfee6c Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sat, 14 Dec 2019 18:37:58 +0100 Subject: [PATCH 017/106] rename project module's iam variables --- modules/project/main.tf | 16 ++++++++-------- modules/project/variables.tf | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/modules/project/main.tf b/modules/project/main.tf index d1a444b868..579b9c091a 100644 --- a/modules/project/main.tf +++ b/modules/project/main.tf @@ -18,14 +18,14 @@ locals { cloudsvc_service_account = "${google_project.project.number}@cloudservices.gserviceaccount.com" gce_service_account = "${google_project.project.number}-compute@developer.gserviceaccount.com" gke_service_account = "service-${google_project.project.number}@container-engine-robot.iam.gserviceaccount.com" - iam_non_authoritative_pairs = flatten([ - for role in var.iam_non_authoritative_roles : [ - for member in lookup(var.iam_non_authoritative_members, role, []) : + iam_nonauth_pairs = flatten([ + for role in var.iam_nonauth_roles : [ + for member in lookup(var.iam_nonauth_members, role, []) : { role = role, member = member } ] ]) - iam_non_authoritative = { - for pair in local.iam_non_authoritative_pairs : + iam_nonauth = { + for pair in local.iam_nonauth_pairs : "${pair.role}-${pair.member}" => pair } parent_type = split("/", var.parent)[0] @@ -82,14 +82,14 @@ resource "google_project_service" "project_services" { # - non-authoritative roles might fail due to dynamic values resource "google_project_iam_binding" "authoritative" { - for_each = toset(var.iam_authoritative_roles) + for_each = toset(var.iam_roles) project = google_project.project.project_id role = each.value - members = lookup(var.iam_authoritative_members, each.value, []) + members = lookup(var.iam_members, each.value, []) } resource "google_project_iam_member" "non_authoritative" { - for_each = length(var.iam_non_authoritative_roles) > 0 ? local.iam_non_authoritative : {} + for_each = length(var.iam_nonauth_roles) > 0 ? local.iam_nonauth : {} project = google_project.project.project_id role = each.value.role member = each.value.member diff --git a/modules/project/variables.tf b/modules/project/variables.tf index f2dfcb948b..bc8459c58f 100644 --- a/modules/project/variables.tf +++ b/modules/project/variables.tf @@ -32,25 +32,25 @@ variable "custom_roles" { default = {} } -variable "iam_authoritative_members" { +variable "iam_members" { description = "Map of member lists used to set authoritative bindings, keyed by role." type = map(list(string)) default = {} } -variable "iam_authoritative_roles" { +variable "iam_roles" { description = "List of roles used to set authoritative bindings." type = list(string) default = [] } -variable "iam_non_authoritative_members" { +variable "iam_nonauth_members" { description = "Map of member lists used to set non authoritative bindings, keyed by role." type = map(list(string)) default = {} } -variable "iam_non_authoritative_roles" { +variable "iam_nonauth_roles" { description = "List of roles used to set non authoritative bindings." type = list(string) default = [] From 54d634d9110f2b10bf3311fb404d33f2aa487bac Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sat, 14 Dec 2019 19:01:54 +0100 Subject: [PATCH 018/106] slight tweak to folder module outputs --- modules/folder/outputs.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/folder/outputs.tf b/modules/folder/outputs.tf index 18ac60c6fa..56964d5ea3 100644 --- a/modules/folder/outputs.tf +++ b/modules/folder/outputs.tf @@ -16,17 +16,17 @@ output "folder" { description = "Folder resource (for single use)." - value = local.folders[0] + value = length(var.names) > 0 ? local.folders[0] : null } output "id" { description = "Folder id (for single use)." - value = local.folders[0].name + value = length(var.names) > 0 ? local.folders[0].name : null } output "name" { description = "Folder name (for single use)." - value = local.folders[0].display_name + value = length(var.names) > 0 ? local.folders[0].display_name : null } output "folders" { From d29b77198f0a8220eb1f53afbfd23e427c77cee0 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sat, 14 Dec 2019 19:02:03 +0100 Subject: [PATCH 019/106] gcs module --- modules/gcs/main.tf | 75 ++++++-------------- modules/gcs/outputs.tf | 16 ++--- modules/gcs/variables.tf | 147 +++++++++------------------------------ 3 files changed, 64 insertions(+), 174 deletions(-) diff --git a/modules/gcs/main.tf b/modules/gcs/main.tf index 0e986d146f..05067e6c4e 100644 --- a/modules/gcs/main.tf +++ b/modules/gcs/main.tf @@ -15,13 +15,21 @@ */ locals { - prefix = var.prefix == "" ? "" : join("-", list(var.prefix, lower(var.location), "")) - names = toset(var.names) - buckets_list = [for name in var.names : google_storage_bucket.buckets[name]] + buckets = [for name in var.names : google_storage_bucket.buckets[name]] + # distinct is needed to make the expanding function argument work + iam_pairs = concat([], distinct([ + for name, roles in var.iam_roles : + [for role in roles : { name = name, role = role }] + ])...) + iam_keypairs = { + for pair in local.iam_pairs : + "${pair.name}-${pair.role}" => pair + } + prefix = var.prefix == "" ? "" : join("-", [var.prefix, lower(var.location), ""]) } resource "google_storage_bucket" "buckets" { - for_each = local.names + for_each = toset(var.names) name = "${local.prefix}${lower(each.key)}" project = var.project_id location = var.location @@ -32,56 +40,17 @@ resource "google_storage_bucket" "buckets" { enabled = lookup(var.versioning, each.key, false) } labels = merge(var.labels, { - name = "${local.prefix}${lower(each.key)}" + location = lower(var.location) + name = lower(each.key) + storage_class = lower(var.storage_class) }) } -resource "google_storage_bucket_iam_binding" "object_admins" { - for_each = var.set_object_admin_roles ? local.names : toset([]) - bucket = google_storage_bucket.buckets[each.key].name - role = "roles/storage.objectAdmin" - members = compact(concat( - var.object_admins, - split(",", lookup(var.bucket_object_admins, each.key, "")) - )) -} - -resource "google_storage_bucket_iam_binding" "creators" { - for_each = var.set_creator_roles ? local.names : toset([]) - bucket = google_storage_bucket.buckets[each.key].name - role = "roles/storage.objectCreator" - members = compact(concat( - var.creators, - split(",", lookup(var.bucket_creators, each.key, "")) - )) -} - -resource "google_storage_bucket_iam_binding" "viewers" { - for_each = var.set_viewer_roles ? local.names : toset([]) - bucket = google_storage_bucket.buckets[each.key].name - role = "roles/storage.objectViewer" - members = compact(concat( - var.viewers, - split(",", lookup(var.bucket_viewers, each.key, "")) - )) -} - -resource "google_storage_bucket_iam_binding" "hmackey_admins" { - for_each = var.set_hmackey_admin_roles ? local.names : toset([]) - bucket = google_storage_bucket.buckets[each.key].name - role = "roles/storage.hmacKeyAdmin" - members = compact(concat( - var.hmackey_admins, - split(",", lookup(var.bucket_hmackey_admins, each.key, "")) - )) -} - -resource "google_storage_bucket_iam_binding" "admins" { - for_each = var.set_admin_roles ? local.names : toset([]) - bucket = google_storage_bucket.buckets[each.key].name - role = "roles/storage.admin" - members = compact(concat( - var.admins, - split(",", lookup(var.bucket_admins, each.key, "")) - )) +resource "google_storage_bucket_iam_binding" "bindings" { + for_each = local.iam_keypairs + bucket = google_storage_bucket.buckets[each.value.name].name + role = each.value.role + members = lookup( + lookup(var.iam_members, each.value.name, {}), each.value.role, [] + ) } diff --git a/modules/gcs/outputs.tf b/modules/gcs/outputs.tf index b6b6422b13..4458c146d0 100644 --- a/modules/gcs/outputs.tf +++ b/modules/gcs/outputs.tf @@ -16,40 +16,40 @@ output "bucket" { description = "Bucket resource (for single use)." - value = length(var.names) > 0 ? google_storage_bucket.buckets[var.names[0]] : null + value = length(var.names) > 0 ? local.buckets[0] : null } output "name" { description = "Bucket name (for single use)." - value = length(var.names) > 0 ? google_storage_bucket.buckets[var.names[0]].name : null + value = length(var.names) > 0 ? local.buckets[0].name : null } output "url" { description = "Bucket URL (for single use)." - value = length(var.names) > 0 ? google_storage_bucket.buckets[var.names[0]].url : null + value = length(var.names) > 0 ? local.buckets[0].url : null } output "buckets" { description = "Bucket resources." - value = google_storage_bucket.buckets + value = local.buckets } output "names" { description = "Bucket names." - value = { for name in var.names : name => google_storage_bucket.buckets[name].name } + value = zipmap(var.names, [for b in local.buckets : b.name]) } output "urls" { description = "Bucket URLs." - value = { for name in var.names : name => google_storage_bucket.buckets[name].url } + value = zipmap(var.names, [for b in local.buckets : b.url]) } output "names_list" { description = "List of bucket names." - value = [for bucket in local.buckets_list : bucket.name] + value = [for b in local.buckets : b.name] } output "urls_list" { description = "List of bucket URLs." - value = [for bucket in local.buckets_list : bucket.url] + value = [for b in local.buckets : b.name] } diff --git a/modules/gcs/variables.tf b/modules/gcs/variables.tf index 7f52faad56..d6a72eae54 100644 --- a/modules/gcs/variables.tf +++ b/modules/gcs/variables.tf @@ -14,145 +14,66 @@ * limitations under the License. */ -variable "project_id" { - description = "Bucket project id." - type = string -} - -variable "prefix" { - description = "Prefix used to generate the bucket name." - type = string -} - -variable "names" { - description = "Bucket name suffixes." - type = list(string) -} - -variable "location" { - description = "Bucket location." - type = string - default = "EU" -} - -variable "storage_class" { - description = "Bucket storage class." - type = string - default = "MULTI_REGIONAL" -} - -variable "force_destroy" { - description = "Optional map of lowercase unprefixed name => boolean, defaults to false." - type = map(bool) - default = {} -} - -variable "versioning" { - description = "Optional map of lowercase unprefixed name => boolean, defaults to false." - type = map(bool) - default = {} -} - variable "bucket_policy_only" { - description = "Disable ad-hoc ACLs on specified buckets. Defaults to true. Map of lowercase unprefixed name => boolean" + description = "Optional map to disable object ACLS keyed by name, defaults to true." type = map(bool) default = {} } -variable "object_admins" { - description = "IAM-style members who will be granted roles/storage.objectAdmin on all buckets." - type = list(string) - default = [] -} - -variable "creators" { - description = "IAM-style members who will be granted roles/storage.objectCreators on all buckets." - type = list(string) - default = [] -} - -variable "viewers" { - description = "IAM-style members who will be granted roles/storage.objectViewer on all buckets." - type = list(string) - default = [] -} - -variable "hmackey_admins" { - description = "IAM-style members who will be granted roles/storage.hmacKeyAdmin on all buckets." - type = list(string) - default = [] -} - -variable "admins" { - description = "IAM-style members who will be granted roles/storage.admin on all buckets." - type = list(string) - default = [] -} - -variable "bucket_object_admins" { - description = "Map of lowercase unprefixed name => comma-delimited IAM-style per-bucket object admins." - type = map(string) - default = {} -} - -variable "bucket_creators" { - description = "Map of lowercase unprefixed name => comma-delimited IAM-style per-bucket object creators." - type = map(string) - default = {} -} - -variable "bucket_viewers" { - description = "Map of lowercase unprefixed name => comma-delimited IAM-style per-bucket object viewers." - type = map(string) +variable "force_destroy" { + description = "Optional map to set force destroy keyed by name, defaults to false." + type = map(bool) default = {} } -variable "bucket_hmackey_admins" { - description = "Map of lowercase unprefixed name => comma-delimited IAM-style per-bucket hmacKey admins." - type = map(string) +variable "iam_members" { + description = "List of IAM members keyed by name and role." + type = map(map(list(string))) default = {} } -variable "bucket_admins" { - description = "Map of lowercase unprefixed name => comma-delimited IAM-style bucket storage admins." - type = map(string) +variable "iam_roles" { + description = "List of IAM roles keyed by name." + type = map(list(string)) default = {} } variable "labels" { - description = "Labels to be attached to the buckets" + description = "Labels to be attached to all buckets." type = map(string) default = {} } -# we need flags to allow member lists to contain dynamic elements +variable "location" { + description = "Bucket location." + type = string + default = "EU" +} -variable "set_object_admin_roles" { - description = "Grant roles/storage.objectAdmin role to admins and bucket_admins." - type = bool - default = false +variable "names" { + description = "Bucket name suffixes." + type = list(string) } -variable "set_creator_roles" { - description = "Grant roles/storage.objectCreator role to creators and bucket_creators." - type = bool - default = false +variable "prefix" { + description = "Prefix used to generate the bucket name." + type = string + default = "" } -variable "set_viewer_roles" { - description = "Grant roles/storage.objectViewer role to viewers and bucket_viewers." - type = bool - default = false +variable "project_id" { + description = "Bucket project id." + type = string } -variable "set_hmackey_admin_roles" { - description = "Grant roles/storage.hmacKeyAdmin role to storage_admins and bucket_storage_admins." - type = bool - default = false +variable "storage_class" { + description = "Bucket storage class." + type = string + default = "MULTI_REGIONAL" } -variable "set_admin_roles" { - description = "Grant roles/storage.admin role to storage_admins and bucket_storage_admins." - type = bool - default = false +variable "versioning" { + description = "Optional map to set versioning keyed by name, defaults to false." + type = map(bool) + default = {} } From 4f80c61383d9526393e819453100cdb243dff5f8 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 15 Dec 2019 17:27:13 +0100 Subject: [PATCH 020/106] simplify net-vpc module variables --- modules/net-vpc/main.tf | 25 ++++++++++--------------- modules/net-vpc/variables.tf | 27 +++++++++++++++++++++------ 2 files changed, 31 insertions(+), 21 deletions(-) diff --git a/modules/net-vpc/main.tf b/modules/net-vpc/main.tf index 102b35afef..d865113091 100644 --- a/modules/net-vpc/main.tf +++ b/modules/net-vpc/main.tf @@ -15,17 +15,12 @@ */ locals { - # map of log configs for all subnets to reduce the number of lookups needed - subnet_log_configs = { - for name, attrs in var.subnets : name => lookup(var.log_configs, name, {}) - } - # map of log config that reflects enable_flow_logs and sets defaults log_configs = { for name, attrs in var.subnets : name => ( - attrs.enable_flow_logs + lookup(var.subnet_flow_logs, name, false) ? [{ for key, value in var.log_config_defaults : key => lookup( - local.subnet_log_configs[name], key, value + lookup(var.log_configs, name, {}), key, value ) }] : [] @@ -64,18 +59,18 @@ resource "google_compute_shared_vpc_service_project" "service_projects" { } resource "google_compute_subnetwork" "subnetwork" { - for_each = var.subnets - project = var.project_id - network = google_compute_network.network.name - name = each.key - description = each.value.description - ip_cidr_range = each.value.ip_cidr_range - region = each.value.region - private_ip_google_access = each.value.private_ip_google_access + for_each = var.subnets + project = var.project_id + network = google_compute_network.network.name + region = each.value.region + name = each.key + ip_cidr_range = each.value.ip_cidr_range secondary_ip_range = [ for name, range in each.value.secondary_ip_range : { range_name = name, ip_cidr_range = range } ] + description = lookup(var.subnet_descriptions, each.key, "Terraform-managed.") + private_ip_google_access = lookup(var.subnet_private_access, each.key, true) dynamic "log_config" { for_each = local.log_configs[each.key] content { diff --git a/modules/net-vpc/variables.tf b/modules/net-vpc/variables.tf index 636b243964..b1eaeaaed4 100644 --- a/modules/net-vpc/variables.tf +++ b/modules/net-vpc/variables.tf @@ -89,12 +89,27 @@ variable "shared_vpc_service_projects" { variable "subnets" { description = "The list of subnets being created" type = map(object({ - ip_cidr_range = string - region = string - description = string - private_ip_google_access = bool - enable_flow_logs = bool - secondary_ip_range = map(string) + ip_cidr_range = string + region = string + secondary_ip_range = map(string) })) default = {} } + +variable "subnet_descriptions" { + description = "Optional map of subnet descriptions, keyed by subnet name." + type = map(string) + default = {} +} + +variable "subnet_flow_logs" { + description = "Optional map of boolean to control flow logs (default is disabled), keyed by subnet name." + type = map(bool) + default = {} +} + +variable "subnet_private_access" { + description = "Optional map of boolean to control private Google access (default is enabled), keyed by subnet name." + type = map(bool) + default = {} +} From d0e51c7d7ccd009785a43919d8cc395bf0be43d2 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 15 Dec 2019 17:27:43 +0100 Subject: [PATCH 021/106] fix module tests configurations, fix net-vpc module tests --- tests/modules/net_vpc/conftest.py | 2 +- .../net_vpc/fixtures/vpc-iam-bindings/main.tf | 29 +++------ .../net_vpc/fixtures/vpc-standalone/main.tf | 2 +- .../net_vpc/fixtures/vpc-subnets/main.tf | 63 ++++++++++++------- tests/modules/net_vpc/test_vpc_subnets.py | 7 +-- 5 files changed, 54 insertions(+), 49 deletions(-) diff --git a/tests/modules/net_vpc/conftest.py b/tests/modules/net_vpc/conftest.py index 9c9906e421..b7c13e6ff3 100644 --- a/tests/modules/net_vpc/conftest.py +++ b/tests/modules/net_vpc/conftest.py @@ -20,7 +20,7 @@ # path of this folder relative to root -_PATH = os.path.sep.join(os.path.abspath(__file__).split(os.path.sep)[-3:-1]) +_PATH = os.path.sep.join(os.path.abspath(__file__).split(os.path.sep)[-4:-1]) @pytest.fixture(scope='module') diff --git a/tests/modules/net_vpc/fixtures/vpc-iam-bindings/main.tf b/tests/modules/net_vpc/fixtures/vpc-iam-bindings/main.tf index a897fc5910..7c8dbd8290 100644 --- a/tests/modules/net_vpc/fixtures/vpc-iam-bindings/main.tf +++ b/tests/modules/net_vpc/fixtures/vpc-iam-bindings/main.tf @@ -23,28 +23,19 @@ variable "subnets" { description = "Subnet definitions." default = { subnet-a = { - ip_cidr_range = "192.168.0.0/24" - region = "europe-west1" - description = "First subnet." - private_ip_google_access = false - enable_flow_logs = false - secondary_ip_range = {} + ip_cidr_range = "192.168.0.0/24" + region = "europe-west1" + secondary_ip_range = {} }, subnet-b = { - ip_cidr_range = "192.168.1.0/24" - region = "europe-west1" - description = "Second subnet." - private_ip_google_access = false - enable_flow_logs = false - secondary_ip_range = {} + ip_cidr_range = "192.168.1.0/24" + region = "europe-west1" + secondary_ip_range = {} }, subnet-c = { - ip_cidr_range = "192.168.2.0/24" - region = "europe-west1" - description = "Third subnet." - private_ip_google_access = false - enable_flow_logs = false - secondary_ip_range = {} + ip_cidr_range = "192.168.2.0/24" + region = "europe-west1" + secondary_ip_range = {} }, } } @@ -63,7 +54,7 @@ resource "google_service_account" "binding_members" { } module "vpc" { - source = "../../../../net-vpc" + source = "../../../../../modules/net-vpc" project_id = var.project_id name = "vpc-iam-bindings" description = "Created by the vpc-iam-bindings fixture." diff --git a/tests/modules/net_vpc/fixtures/vpc-standalone/main.tf b/tests/modules/net_vpc/fixtures/vpc-standalone/main.tf index 5c7a018e2a..ec33dc1cbd 100644 --- a/tests/modules/net_vpc/fixtures/vpc-standalone/main.tf +++ b/tests/modules/net_vpc/fixtures/vpc-standalone/main.tf @@ -26,7 +26,7 @@ variable "project_id" { } module "vpc-simple" { - source = "../../../../net-vpc" + source = "../../../../../modules/net-vpc" project_id = var.project_id name = var.name } diff --git a/tests/modules/net_vpc/fixtures/vpc-subnets/main.tf b/tests/modules/net_vpc/fixtures/vpc-subnets/main.tf index 01c924df38..c3db05e4fa 100644 --- a/tests/modules/net_vpc/fixtures/vpc-subnets/main.tf +++ b/tests/modules/net_vpc/fixtures/vpc-subnets/main.tf @@ -23,27 +23,18 @@ variable "subnets" { description = "Subnet definitions." default = { subnet-simple = { - ip_cidr_range = "192.168.0.0/24" - region = "europe-west1" - description = "Simple subnet." - private_ip_google_access = false - enable_flow_logs = false - secondary_ip_range = {} + ip_cidr_range = "192.168.0.0/24" + region = "europe-west1" + secondary_ip_range = {} }, subnet-options = { - ip_cidr_range = "192.168.1.0/24" - region = "europe-west2" - description = "Simple subnet with options." - private_ip_google_access = true - enable_flow_logs = true - secondary_ip_range = {} + ip_cidr_range = "192.168.1.0/24" + region = "europe-west2" + secondary_ip_range = {} }, subnet-alias-ranges = { - ip_cidr_range = "192.168.2.0/24" - region = "europe-west1" - description = "Simple subnet with alias ranges." - private_ip_google_access = true - enable_flow_logs = true + ip_cidr_range = "192.168.2.0/24" + region = "europe-west1" secondary_ip_range = { alias-1 = "172.16.10.0/24" alias-2 = "172.16.20.0/24" @@ -52,6 +43,27 @@ variable "subnets" { } } +variable "subnet_descriptions" { + default = { + subnet-options = "Simple subnet with options." + subnet-alias-ranges = "Simple subnet with alias ranges." + } +} + +variable "subnet_flow_logs" { + default = { + subnet-options = true + subnet-alias-ranges = true + } +} + +variable "subnet_private_access" { + default = { + subnet-simple = false + subnet-options = true + } +} + variable "log_configs" { description = "Logging configurations." default = { @@ -62,13 +74,16 @@ variable "log_configs" { } module "vpc" { - source = "../../../../net-vpc" - project_id = var.project_id - name = "vpc-subnets" - description = "Created by the vpc-subnets fixture." - routing_mode = "REGIONAL" - subnets = var.subnets - log_configs = var.log_configs + source = "../../../../../modules/net-vpc" + project_id = var.project_id + name = "vpc-subnets" + description = "Created by the vpc-subnets fixture." + routing_mode = "REGIONAL" + subnets = var.subnets + subnet_descriptions = var.subnet_descriptions + subnet_flow_logs = var.subnet_flow_logs + subnet_private_access = var.subnet_private_access + log_configs = var.log_configs } output "network" { diff --git a/tests/modules/net_vpc/test_vpc_subnets.py b/tests/modules/net_vpc/test_vpc_subnets.py index 008b77f8f0..36aa421b40 100644 --- a/tests/modules/net_vpc/test_vpc_subnets.py +++ b/tests/modules/net_vpc/test_vpc_subnets.py @@ -51,13 +51,12 @@ def test_subnet_regions(result): def test_secondary_ip_ranges(result): "Test subnet secondary ranges output." for name, attrs in result.plan.variables['subnets'].items(): - ranges = [{'range_name': k, 'ip_cidr_range': v} - for k, v in attrs['secondary_ip_range'].items()] - assert ranges == result.output['subnet_secondary_ranges'][name] + assert attrs['secondary_ip_range'] == result.output['subnet_secondary_ranges'][name] def test_flow_logs(result): "Test that log config is set using the enable flow logs variable." + enable_flow_logs = result.plan.variables['subnet_flow_logs'] for name, attrs in result.plan.variables['subnets'].items(): - log_config = 1 if attrs['enable_flow_logs'] else 0 + log_config = enable_flow_logs.get(name, False) assert len(result.output['subnets'][name]['log_config']) == log_config From dded3a66492576b26c533982c71320dd605b4abb Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 15 Dec 2019 17:32:52 +0100 Subject: [PATCH 022/106] add pydoc utility --- tools/tfdoc/REQUIREMENTS.txt | 1 + tools/tfdoc/tfdoc.py | 192 +++++++++++++++++++++++++++++++++++ 2 files changed, 193 insertions(+) create mode 100644 tools/tfdoc/REQUIREMENTS.txt create mode 100755 tools/tfdoc/tfdoc.py diff --git a/tools/tfdoc/REQUIREMENTS.txt b/tools/tfdoc/REQUIREMENTS.txt new file mode 100644 index 0000000000..b98f6609d1 --- /dev/null +++ b/tools/tfdoc/REQUIREMENTS.txt @@ -0,0 +1 @@ +click \ No newline at end of file diff --git a/tools/tfdoc/tfdoc.py b/tools/tfdoc/tfdoc.py new file mode 100755 index 0000000000..e97af6a6e2 --- /dev/null +++ b/tools/tfdoc/tfdoc.py @@ -0,0 +1,192 @@ +#! /usr/bin/env python3 + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections +import enum +import os +import re + +import click + + +RE_TYPE_FORMAT = re.compile(r'(?s)^\s*([a-z]+).*?$') +RE_OUTPUTS = re.compile(r'''(?smx) + (?:^\s*output\s*"([^"]+)"\s*\{$) | + (?:^\s*description\s*=\s*"([^"]+)"\s*$) +''') +RE_VARIABLES = re.compile(r'''(?smx) + # empty lines + (^\s*$) | + # block comment + (^\s*/\*.*?\*/$) | + # line comment + (^\s*\#.*?$) | + # variable declaration start + (?:^\s*variable\s*"([^"]+)"\s*\{$) | + # variable description start + (?:^\s*description\s*=\s*"([^"]+)"\s*$) | + # variable type start + (?:^\s*type\s*=\s*(.*?)$) | + # variable default start + (?:^\s*default\s*=\s*"?(.*?)"?\s*$) | + # variable body + (?:^\s*(\S.*?)$) +''') + +OutputData = collections.namedtuple('Output', 'name description') +OutputToken = enum.Enum('OutputToken', 'NAME DESCRIPTION') +VariableData = collections.namedtuple( + 'Variable', 'name description type default required') +VariableToken = enum.Enum( + 'VariableToken', + 'EMPTY BLOCK_COMMENT LINE_COMMENT NAME DESCRIPTION TYPE DEFAULT REST') + + +class ItemParsed(Exception): + pass + + +class Output(object): + "Output parsing helper class." + + def __init__(self): + self.in_progress = False + self.name = self.description = None + + def parse_token(self, token_type, token_data): + if token_type == 'NAME': + if self.in_progress: + raise ItemParsed(self.close()) + self.in_progress = True + self.name = token_data + elif token_type == 'DESCRIPTION': + setattr(self, token_type.lower(), token_data) + + def close(self): + return OutputData(self.name, self.description) + + +class Variable(object): + "Variable parsing helper class." + + def __init__(self): + self.in_progress = False + self.name = self.description = self.type = self.default = None + self._data = [] + self._data_context = None + + def parse_token(self, token_type, token_data): + if token_type == 'NAME': + if self.in_progress: + raise ItemParsed(self.close()) + self.in_progress = True + self.name = token_data + elif token_type == 'DESCRIPTION': + setattr(self, token_type.lower(), token_data) + elif token_type in ('DEFAULT', 'TYPE'): + self._start(token_type.lower(), token_data) + elif token_type == 'REST': + self._data.append(token_data) + + def _close(self, strip=False): + if self._data_context: + data = self._data + if strip and '}' in data[-1]: + data = data[:-1] + setattr(self, self._data_context, ('\n'.join(data)).strip()) + + def _start(self, context, data): + self._close() + self._data = [data] + self._data_context = context + + def close(self): + self._close(strip=True) + return VariableData(self.name, self.description, self.type, self.default, + self.default is None) + + +def parse_items(content, item_re, item_enum, item_class, item_data_class): + "Parse variable or output items in data." + item = item_class() + for m in item_re.finditer(content): + try: + item.parse_token(item_enum(m.lastindex).name, m.group(m.lastindex)) + except ItemParsed as e: + item = item_class() + item.parse_token(item_enum(m.lastindex).name, m.group(m.lastindex)) + yield e.args[0] + if item.in_progress: + yield item.close() + + +def format_output(output): + "Format output." + return + + +def format_variables(variables, required_first=True): + "Format variables." + if not variables: + return + variables.sort(key=lambda v: v.name) + variables.sort(key=lambda v: v.required, reverse=True) + yield '| name | description | type | required |' + yield '|---|---|:---: |:---:|' + for v in variables: + yield '| {name} | {description} | {type} | {required}'.format( + name=v.name if v.required else '*%s*' % v.name, + description=v.description, type=RE_TYPE_FORMAT.sub(r'\1', v.type), + required='✓' if v.required else '' + ) + + +def format_outputs(outputs): + "Format variables." + if not outputs: + return + outputs.sort(key=lambda v: v.name) + yield '| name | description |' + yield '|---|---|' + for o in outputs: + yield '| {name} | {description} |'.format( + name=o.name, description=o.description) + + +@click.command() +@click.argument('module', type=click.Path(exists=True)) +def main(module=None): + "Program entry point." + try: + with open(os.path.join(module, 'variables.tf')) as file: + variables = [v for v in parse_items( + file.read(), RE_VARIABLES, VariableToken, Variable, VariableData)] + with open(os.path.join(module, 'outputs.tf')) as file: + outputs = [o for o in parse_items( + file.read(), RE_OUTPUTS, OutputToken, Output, OutputData)] + except (IOError, OSError) as e: + raise SystemExit(e) + print('## Variables\n') + for line in format_variables(variables): + print(line) + print() + print('## Outputs\n') + for line in format_outputs(outputs): + print(line) + + +if __name__ == '__main__': + main() From 433b4df5ebe5781b1f2a9ad42295d666d9f16201 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 15 Dec 2019 17:34:34 +0100 Subject: [PATCH 023/106] add/update module READMEs --- modules/folder/README.md | 23 +++++++++++++++++++++++ modules/net-vpc/README.md | 7 +++++++ 2 files changed, 30 insertions(+) create mode 100644 modules/folder/README.md diff --git a/modules/folder/README.md b/modules/folder/README.md new file mode 100644 index 0000000000..21de85158e --- /dev/null +++ b/modules/folder/README.md @@ -0,0 +1,23 @@ +# Google Cloud Folder module + +## Variables + +| name | description | type | required | +|---|---|:---: |:---:| +| parent | Parent in folders/folder_id or organizations/org_id format. | string | ✓ +| *iam_members* | List of IAM members keyed by folder name and role. | map | +| *iam_roles* | List of IAM roles keyed by folder name. | map | +| *names* | Folder names. | list | + +## Outputs + +| name | description | +|---|---| +| folder | Folder resource (for single use). | +| folders | Folder resources. | +| id | Folder id (for single use). | +| ids | Folder ids. | +| ids_list | List of folder ids. | +| name | Folder name (for single use). | +| names | Folder names. | +| names_list | List of folder names. | diff --git a/modules/net-vpc/README.md b/modules/net-vpc/README.md index e7765e2bdd..ee9a778566 100644 --- a/modules/net-vpc/README.md +++ b/modules/net-vpc/README.md @@ -8,6 +8,8 @@ TODO(ludoo): add example ## Variables +## Variables + | name | description | type | required | |---|---|:---: |:---:| | name | The name of the network being created | string | ✓ @@ -20,6 +22,10 @@ TODO(ludoo): add example | *log_configs* | Map of per-subnet optional configurations for flow logs when enabled. | map | | *routing_mode* | The network routing mode (default 'GLOBAL') | string | | *shared_vpc_host* | Makes this project a Shared VPC host if 'true' (default 'false') | bool | +| *shared_vpc_service_projects* | Shared VPC service projects to register with this host | list | +| *subnet_descriptions* | Optional map of subnet descriptions, keyed by subnet name. | map | +| *subnet_flow_logs* | Optional map of boolean to control flow logs (default is disabled), keyed by subnet name. | map | +| *subnet_private_access* | Optional map of boolean to control private Google access (default is enabled), keyed by subnet name. | map | | *subnets* | The list of subnets being created | map | ## Outputs @@ -36,3 +42,4 @@ TODO(ludoo): add example | subnet_secondary_ranges | Map of subnet secondary ranges keyed by name. | | subnet_self_links | Map of subnet self links keyed by name. | | subnets | Subnet resources. | + From 07e1920d6b1f751b66a5c56efd8e71db83ad6818 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 15 Dec 2019 17:36:14 +0100 Subject: [PATCH 024/106] add/update module READMEs --- modules/folder/README.md | 2 +- modules/gcs/README.md | 34 ++++++++++++---------------------- modules/project/README.md | 30 ++++++++++++++++-------------- 3 files changed, 29 insertions(+), 37 deletions(-) diff --git a/modules/folder/README.md b/modules/folder/README.md index 21de85158e..f1cf4d9a0e 100644 --- a/modules/folder/README.md +++ b/modules/folder/README.md @@ -1,4 +1,4 @@ -# Google Cloud Folder module +# Google Cloud Folder Module ## Variables diff --git a/modules/gcs/README.md b/modules/gcs/README.md index 30aa427ae3..6695542a09 100644 --- a/modules/gcs/README.md +++ b/modules/gcs/README.md @@ -2,32 +2,21 @@ ## Variables +## Variables + | name | description | type | required | |---|---|:---: |:---:| | names | Bucket name suffixes. | list | ✓ -| prefix | Prefix used to generate the bucket name. | string | ✓ | project_id | Bucket project id. | string | ✓ -| *admins* | IAM-style members who will be granted roles/storage.admin on all buckets. | list | -| *bucket_admins* | Map of lowercase unprefixed name => comma-delimited IAM-style bucket storage admins. | map | -| *bucket_creators* | Map of lowercase unprefixed name => comma-delimited IAM-style per-bucket object creators. | map | -| *bucket_hmackey_admins* | Map of lowercase unprefixed name => comma-delimited IAM-style per-bucket hmacKey admins. | map | -| *bucket_object_admins* | Map of lowercase unprefixed name => comma-delimited IAM-style per-bucket object admins. | map | -| *bucket_policy_only* | Disable ad-hoc ACLs on specified buckets. Defaults to true. Map of lowercase unprefixed name => boolean | map | -| *bucket_viewers* | Map of lowercase unprefixed name => comma-delimited IAM-style per-bucket object viewers. | map | -| *creators* | IAM-style members who will be granted roles/storage.objectCreators on all buckets. | list | -| *force_destroy* | Optional map of lowercase unprefixed name => boolean, defaults to false. | map | -| *hmackey_admins* | IAM-style members who will be granted roles/storage.hmacKeyAdmin on all buckets. | list | -| *labels* | Labels to be attached to the buckets | map | -| *location* | Bucket location. | string | -| *object_admins* | IAM-style members who will be granted roles/storage.objectAdmin on all buckets. | list | -| *set_admin_roles* | Grant roles/storage.admin role to storage_admins and bucket_storage_admins. | bool | -| *set_creator_roles* | Grant roles/storage.objectCreator role to creators and bucket_creators. | bool | -| *set_hmackey_admin_roles* | Grant roles/storage.hmacKeyAdmin role to storage_admins and bucket_storage_admins. | bool | -| *set_object_admin_roles* | Grant roles/storage.objectAdmin role to admins and bucket_admins. | bool | -| *set_viewer_roles* | Grant roles/storage.objectViewer role to viewers and bucket_viewers. | bool | -| *storage_class* | Bucket storage class. | string | -| *versioning* | Optional map of lowercase unprefixed name => boolean, defaults to false. | map | -| *viewers* | IAM-style members who will be granted roles/storage.objectViewer on all buckets. | list | +| *bucket_policy_only* | Optional map to disable object ACLS keyed by name, defaults to true. | map | +| *force_destroy* | Optional map to set force destroy keyed by name, defaults to false. | map | +| *iam_members* | List of IAM members keyed by name and role. | map | +| *iam_roles* | List of IAM roles keyed by name. | map | +| *labels* | Labels to be attached to all buckets. | map | +| *location* | Bucket location. | string | +| *prefix* | Prefix used to generate the bucket name. | string | +| *storage_class* | Bucket storage class. | string | +| *versioning* | Optional map to set versioning keyed by name, defaults to false. | map | ## Outputs @@ -41,3 +30,4 @@ | url | Bucket URL (for single use). | | urls | Bucket URLs. | | urls_list | List of bucket URLs. | + diff --git a/modules/project/README.md b/modules/project/README.md index 2228edc408..620d254105 100644 --- a/modules/project/README.md +++ b/modules/project/README.md @@ -13,25 +13,27 @@ The resources/services/activations/deletions that this module will create/trigge ## Variables +## Variables + | name | description | type | required | |---|---|:---: |:---:| | name | Project name and id suffix. | string | ✓ | parent | The resource name of the parent Folder or Organization. Must be of the form folders/folder_id or organizations/org_id. | string | ✓ | prefix | Prefix used to generate project id and name. | string | ✓ -| *auto_create_network* | Whether to create the default network for the project | bool | -| *billing_account* | Billing account id. | string | -| *custom_roles* | Map of role name => list of permissions to create in this project. | map | -| *iam_authoritative_members* | Map of member lists used to set authoritative bindings, keyed by role. | map | -| *iam_authoritative_roles* | List of roles used to set authoritative bindings. | list | -| *iam_non_authoritative_members* | Map of member lists used to set non authoritative bindings, keyed by role. | map | -| *iam_non_authoritative_roles* | List of roles used to set non authoritative bindings. | list | -| *labels* | Resource labels. | map | -| *lien_reason* | If non-empty, creates a project lien with this description. | string | -| *oslogin* | Enable OS Login. | bool | -| *oslogin_admins* | List of IAM-style identities that will be granted roles necessary for OS Login administrators. | list | -| *oslogin_users* | List of IAM-style identities that will be granted roles necessary for OS Login users. | list | -| *owners* | IAM-style identities that will be granted non-authoritative viewer role. | list | -| *services* | Service APIs to enable. | list | +| *auto_create_network* | Whether to create the default network for the project | bool | +| *billing_account* | Billing account id. | string | +| *custom_roles* | Map of role name => list of permissions to create in this project. | map | +| *iam_members* | Map of member lists used to set authoritative bindings, keyed by role. | map | +| *iam_nonauth_members* | Map of member lists used to set non authoritative bindings, keyed by role. | map | +| *iam_nonauth_roles* | List of roles used to set non authoritative bindings. | list | +| *iam_roles* | List of roles used to set authoritative bindings. | list | +| *labels* | Resource labels. | map | +| *lien_reason* | If non-empty, creates a project lien with this description. | string | +| *oslogin* | Enable OS Login. | bool | +| *oslogin_admins* | List of IAM-style identities that will be granted roles necessary for OS Login administrators. | list | +| *oslogin_users* | List of IAM-style identities that will be granted roles necessary for OS Login users. | list | +| *owners* | IAM-style identities that will be granted non-authoritative viewer role. | list | +| *services* | Service APIs to enable. | list | ## Outputs From b1f02737774b05406601bc957677cb8fb25615ac Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 15 Dec 2019 17:42:04 +0100 Subject: [PATCH 025/106] add/update module READMEs --- modules/net-vpc/README.md | 28 +++++++++---------- tools/tfdoc/tfdoc.py | 57 ++++++++++++++++++++++----------------- 2 files changed, 45 insertions(+), 40 deletions(-) diff --git a/modules/net-vpc/README.md b/modules/net-vpc/README.md index ee9a778566..4a99f0097c 100644 --- a/modules/net-vpc/README.md +++ b/modules/net-vpc/README.md @@ -8,25 +8,23 @@ TODO(ludoo): add example ## Variables -## Variables - | name | description | type | required | |---|---|:---: |:---:| | name | The name of the network being created | string | ✓ | project_id | The ID of the project where this VPC will be created | string | ✓ -| *auto_create_subnetworks* | When set to true, the network is created in 'auto subnet mode' and it will create a subnet for each region automatically across the 10.128.0.0/9 address range. When set to false, the network is created in 'custom subnet mode' so the user can explicitly connect subnetwork resources. | bool | -| *description* | An optional description of this resource. The resource must be recreated to modify this field. | string | -| *iam_members* | List of IAM members keyed by subnet and role. | map | -| *iam_roles* | List of IAM roles keyed by subnet. | map | -| *log_config_defaults* | Default configuration for flow logs when enabled. | object | -| *log_configs* | Map of per-subnet optional configurations for flow logs when enabled. | map | -| *routing_mode* | The network routing mode (default 'GLOBAL') | string | -| *shared_vpc_host* | Makes this project a Shared VPC host if 'true' (default 'false') | bool | -| *shared_vpc_service_projects* | Shared VPC service projects to register with this host | list | -| *subnet_descriptions* | Optional map of subnet descriptions, keyed by subnet name. | map | -| *subnet_flow_logs* | Optional map of boolean to control flow logs (default is disabled), keyed by subnet name. | map | -| *subnet_private_access* | Optional map of boolean to control private Google access (default is enabled), keyed by subnet name. | map | -| *subnets* | The list of subnets being created | map | +| *auto_create_subnetworks* | When set to true, the network is created in 'auto subnet mode' and it will create a subnet for each region automatically across the 10.128.0.0/9 address range. When set to false, the network is created in 'custom subnet mode' so the user can explicitly connect subnetwork resources. | bool | +| *description* | An optional description of this resource. The resource must be recreated to modify this field. | string | +| *iam_members* | List of IAM members keyed by subnet and role. | map | +| *iam_roles* | List of IAM roles keyed by subnet. | map | +| *log_config_defaults* | Default configuration for flow logs when enabled. | object | +| *log_configs* | Map of per-subnet optional configurations for flow logs when enabled. | map | +| *routing_mode* | The network routing mode (default 'GLOBAL') | string | +| *shared_vpc_host* | Makes this project a Shared VPC host if 'true' (default 'false') | bool | +| *shared_vpc_service_projects* | Shared VPC service projects to register with this host | list | +| *subnet_descriptions* | Optional map of subnet descriptions, keyed by subnet name. | map | +| *subnet_flow_logs* | Optional map of boolean to control flow logs (default is disabled), keyed by subnet name. | map | +| *subnet_private_access* | Optional map of boolean to control private Google access (default is enabled), keyed by subnet name. | map | +| *subnets* | The list of subnets being created | object map | ## Outputs diff --git a/tools/tfdoc/tfdoc.py b/tools/tfdoc/tfdoc.py index e97af6a6e2..71ab15df35 100755 --- a/tools/tfdoc/tfdoc.py +++ b/tools/tfdoc/tfdoc.py @@ -119,25 +119,30 @@ def close(self): self.default is None) -def parse_items(content, item_re, item_enum, item_class, item_data_class): - "Parse variable or output items in data." - item = item_class() - for m in item_re.finditer(content): - try: - item.parse_token(item_enum(m.lastindex).name, m.group(m.lastindex)) - except ItemParsed as e: - item = item_class() - item.parse_token(item_enum(m.lastindex).name, m.group(m.lastindex)) - yield e.args[0] - if item.in_progress: - yield item.close() - - def format_output(output): "Format output." return +def format_outputs(outputs): + "Format variables." + if not outputs: + return + outputs.sort(key=lambda v: v.name) + yield '| name | description |' + yield '|---|---|' + for o in outputs: + yield '| {name} | {description} |'.format( + name=o.name, description=o.description) + + +def format_type(type_spec): + "Format variable type." + if type_spec.startswith('map(object'): + return 'object map' + return RE_TYPE_FORMAT.sub(r'\1', type_spec) + + def format_variables(variables, required_first=True): "Format variables." if not variables: @@ -149,21 +154,23 @@ def format_variables(variables, required_first=True): for v in variables: yield '| {name} | {description} | {type} | {required}'.format( name=v.name if v.required else '*%s*' % v.name, - description=v.description, type=RE_TYPE_FORMAT.sub(r'\1', v.type), + description=v.description, type=format_type(v.type), required='✓' if v.required else '' ) -def format_outputs(outputs): - "Format variables." - if not outputs: - return - outputs.sort(key=lambda v: v.name) - yield '| name | description |' - yield '|---|---|' - for o in outputs: - yield '| {name} | {description} |'.format( - name=o.name, description=o.description) +def parse_items(content, item_re, item_enum, item_class, item_data_class): + "Parse variable or output items in data." + item = item_class() + for m in item_re.finditer(content): + try: + item.parse_token(item_enum(m.lastindex).name, m.group(m.lastindex)) + except ItemParsed as e: + item = item_class() + item.parse_token(item_enum(m.lastindex).name, m.group(m.lastindex)) + yield e.args[0] + if item.in_progress: + yield item.close() @click.command() From 82b20abf1d36dc7ccd4e1c8d7f7f98dee8ba3a2d Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 15 Dec 2019 21:27:04 +0100 Subject: [PATCH 026/106] improve variable type summary generation in tfdoc --- modules/net-vpc/README.md | 22 +++++++++++----------- modules/net-vpc/variables.tf | 4 ++-- tools/tfdoc/tfdoc.py | 18 ++++++++++++++---- 3 files changed, 27 insertions(+), 17 deletions(-) diff --git a/modules/net-vpc/README.md b/modules/net-vpc/README.md index 4a99f0097c..4e80884557 100644 --- a/modules/net-vpc/README.md +++ b/modules/net-vpc/README.md @@ -12,19 +12,19 @@ TODO(ludoo): add example |---|---|:---: |:---:| | name | The name of the network being created | string | ✓ | project_id | The ID of the project where this VPC will be created | string | ✓ -| *auto_create_subnetworks* | When set to true, the network is created in 'auto subnet mode' and it will create a subnet for each region automatically across the 10.128.0.0/9 address range. When set to false, the network is created in 'custom subnet mode' so the user can explicitly connect subnetwork resources. | bool | -| *description* | An optional description of this resource. The resource must be recreated to modify this field. | string | -| *iam_members* | List of IAM members keyed by subnet and role. | map | -| *iam_roles* | List of IAM roles keyed by subnet. | map | -| *log_config_defaults* | Default configuration for flow logs when enabled. | object | -| *log_configs* | Map of per-subnet optional configurations for flow logs when enabled. | map | +| *auto_create_subnetworks* | Set to true to create an auto mode subnet, defaults to custom mode. | bool | +| *description* | An optional description of this resource (triggers recreation on change). | string | +| *iam_members* | List of IAM members keyed by subnet and role. | map(map(list(string))) | +| *iam_roles* | List of IAM roles keyed by subnet. | map(list(string)) | +| *log_config_defaults* | Default configuration for flow logs when enabled. | object({...}) | +| *log_configs* | Map of per-subnet optional configurations for flow logs when enabled. | map(map(string)) | | *routing_mode* | The network routing mode (default 'GLOBAL') | string | | *shared_vpc_host* | Makes this project a Shared VPC host if 'true' (default 'false') | bool | -| *shared_vpc_service_projects* | Shared VPC service projects to register with this host | list | -| *subnet_descriptions* | Optional map of subnet descriptions, keyed by subnet name. | map | -| *subnet_flow_logs* | Optional map of boolean to control flow logs (default is disabled), keyed by subnet name. | map | -| *subnet_private_access* | Optional map of boolean to control private Google access (default is enabled), keyed by subnet name. | map | -| *subnets* | The list of subnets being created | object map | +| *shared_vpc_service_projects* | Shared VPC service projects to register with this host | list(string) | +| *subnet_descriptions* | Optional map of subnet descriptions, keyed by subnet name. | map(string) | +| *subnet_flow_logs* | Optional map of boolean to control flow logs (default is disabled), keyed by subnet name. | map(bool) | +| *subnet_private_access* | Optional map of boolean to control private Google access (default is enabled), keyed by subnet name. | map(bool) | +| *subnets* | The list of subnets being created | map(object({...})) | ## Outputs diff --git a/modules/net-vpc/variables.tf b/modules/net-vpc/variables.tf index b1eaeaaed4..ce7be1f0a2 100644 --- a/modules/net-vpc/variables.tf +++ b/modules/net-vpc/variables.tf @@ -15,13 +15,13 @@ */ variable "auto_create_subnetworks" { - description = "When set to true, the network is created in 'auto subnet mode' and it will create a subnet for each region automatically across the 10.128.0.0/9 address range. When set to false, the network is created in 'custom subnet mode' so the user can explicitly connect subnetwork resources." + description = "Set to true to create an auto mode subnet, defaults to custom mode." type = bool default = false } variable "description" { - description = "An optional description of this resource. The resource must be recreated to modify this field." + description = "An optional description of this resource (triggers recreation on change)." type = string default = "Terraform-managed." } diff --git a/tools/tfdoc/tfdoc.py b/tools/tfdoc/tfdoc.py index 71ab15df35..1bb44c0c62 100755 --- a/tools/tfdoc/tfdoc.py +++ b/tools/tfdoc/tfdoc.py @@ -22,7 +22,7 @@ import click -RE_TYPE_FORMAT = re.compile(r'(?s)^\s*([a-z]+).*?$') +RE_TYPE = re.compile(r'([\(\{\}\)])') RE_OUTPUTS = re.compile(r'''(?smx) (?:^\s*output\s*"([^"]+)"\s*\{$) | (?:^\s*description\s*=\s*"([^"]+)"\s*$) @@ -138,9 +138,19 @@ def format_outputs(outputs): def format_type(type_spec): "Format variable type." - if type_spec.startswith('map(object'): - return 'object map' - return RE_TYPE_FORMAT.sub(r'\1', type_spec) + buffer = [] + stack = [] + for t in RE_TYPE.split(type_spec.split("\n")[0]): + if not t: + continue + if t in '({': + stack.append(t) + elif t in '})': + stack.pop() + buffer.append(t) + for t in reversed(stack): + buffer.append(')' if t == '(' else '}') + return ''.join(buffer).replace('object({})', 'object({...})') def format_variables(variables, required_first=True): From 62f8ddbe8b8aaec6e8ad247122f1f14b5b551db2 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 15 Dec 2019 22:03:58 +0100 Subject: [PATCH 027/106] tfdoc: add support for replacing doc in README.md files --- tools/tfdoc/tfdoc.py | 45 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/tools/tfdoc/tfdoc.py b/tools/tfdoc/tfdoc.py index 1bb44c0c62..3adc861e2e 100755 --- a/tools/tfdoc/tfdoc.py +++ b/tools/tfdoc/tfdoc.py @@ -22,11 +22,13 @@ import click -RE_TYPE = re.compile(r'([\(\{\}\)])') +MARK_BEGIN = '' +MARK_END = '' RE_OUTPUTS = re.compile(r'''(?smx) (?:^\s*output\s*"([^"]+)"\s*\{$) | (?:^\s*description\s*=\s*"([^"]+)"\s*$) ''') +RE_TYPE = re.compile(r'([\(\{\}\)])') RE_VARIABLES = re.compile(r'''(?smx) # empty lines (^\s*$) | @@ -169,6 +171,17 @@ def format_variables(variables, required_first=True): ) +def get_doc(variables, outputs): + "Return formatted documentation." + buffer = ['## Variables\n'] + for line in format_variables(variables): + buffer.append(line) + buffer.append('\n## Outputs\n') + for line in format_outputs(outputs): + buffer.append(line) + return '\n'.join(buffer) + + def parse_items(content, item_re, item_enum, item_class, item_data_class): "Parse variable or output items in data." item = item_class() @@ -183,9 +196,25 @@ def parse_items(content, item_re, item_enum, item_class, item_data_class): yield item.close() +def replace_doc(module, doc): + "Replace document in module's README.md file." + try: + readme = open(os.path.join(module, 'README.md')).read() + m = re.search('(?sm)%s.*%s' % (MARK_BEGIN, MARK_END), readme) + if not m: + raise SystemExit('Pattern not found in README file.') + replacement = "{pre}{begin}\n{doc}\n{end}{post}".format( + pre=readme[:m.start()], begin=MARK_BEGIN, doc=doc, + end=MARK_END, post=readme[m.end():]) + open(os.path.join(module, 'README.md'), 'w').write(replacement) + except (IOError, OSError) as e: + raise SystemExit('Error replacing in README: %s' % e) + + @click.command() @click.argument('module', type=click.Path(exists=True)) -def main(module=None): +@click.option('--replace/--no-replace', default=True) +def main(module=None, replace=True): "Program entry point." try: with open(os.path.join(module, 'variables.tf')) as file: @@ -196,13 +225,11 @@ def main(module=None): file.read(), RE_OUTPUTS, OutputToken, Output, OutputData)] except (IOError, OSError) as e: raise SystemExit(e) - print('## Variables\n') - for line in format_variables(variables): - print(line) - print() - print('## Outputs\n') - for line in format_outputs(outputs): - print(line) + doc = get_doc(variables, outputs) + if replace: + replace_doc(module, doc) + else: + print(doc) if __name__ == '__main__': From 6ad99ee8d57bc0484827c0b8a43f9eccd95c2703 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 15 Dec 2019 22:04:11 +0100 Subject: [PATCH 028/106] improve module READMEs --- modules/folder/README.md | 8 +++--- modules/gcs/README.md | 25 +++++++++--------- modules/gke-cluster/README.md | 50 ++++++++++++++++++----------------- modules/net-vpc/README.md | 2 ++ modules/project/README.md | 25 +++++++++--------- 5 files changed, 57 insertions(+), 53 deletions(-) diff --git a/modules/folder/README.md b/modules/folder/README.md index f1cf4d9a0e..8ab8213eae 100644 --- a/modules/folder/README.md +++ b/modules/folder/README.md @@ -1,13 +1,14 @@ # Google Cloud Folder Module + ## Variables | name | description | type | required | |---|---|:---: |:---:| | parent | Parent in folders/folder_id or organizations/org_id format. | string | ✓ -| *iam_members* | List of IAM members keyed by folder name and role. | map | -| *iam_roles* | List of IAM roles keyed by folder name. | map | -| *names* | Folder names. | list | +| *iam_members* | List of IAM members keyed by folder name and role. | map(map(list(string))) | +| *iam_roles* | List of IAM roles keyed by folder name. | map(list(string)) | +| *names* | Folder names. | list(string) | ## Outputs @@ -21,3 +22,4 @@ | name | Folder name (for single use). | | names | Folder names. | | names_list | List of folder names. | + diff --git a/modules/gcs/README.md b/modules/gcs/README.md index 6695542a09..b492392411 100644 --- a/modules/gcs/README.md +++ b/modules/gcs/README.md @@ -1,22 +1,21 @@ # Google Cloud Storage Module -## Variables - + ## Variables | name | description | type | required | |---|---|:---: |:---:| -| names | Bucket name suffixes. | list | ✓ +| names | Bucket name suffixes. | list(string) | ✓ | project_id | Bucket project id. | string | ✓ -| *bucket_policy_only* | Optional map to disable object ACLS keyed by name, defaults to true. | map | -| *force_destroy* | Optional map to set force destroy keyed by name, defaults to false. | map | -| *iam_members* | List of IAM members keyed by name and role. | map | -| *iam_roles* | List of IAM roles keyed by name. | map | -| *labels* | Labels to be attached to all buckets. | map | -| *location* | Bucket location. | string | -| *prefix* | Prefix used to generate the bucket name. | string | -| *storage_class* | Bucket storage class. | string | -| *versioning* | Optional map to set versioning keyed by name, defaults to false. | map | +| *bucket_policy_only* | Optional map to disable object ACLS keyed by name, defaults to true. | map(bool) | +| *force_destroy* | Optional map to set force destroy keyed by name, defaults to false. | map(bool) | +| *iam_members* | List of IAM members keyed by name and role. | map(map(list(string))) | +| *iam_roles* | List of IAM roles keyed by name. | map(list(string)) | +| *labels* | Labels to be attached to all buckets. | map(string) | +| *location* | Bucket location. | string | +| *prefix* | Prefix used to generate the bucket name. | string | +| *storage_class* | Bucket storage class. | string | +| *versioning* | Optional map to set versioning keyed by name, defaults to false. | map(bool) | ## Outputs @@ -30,4 +29,4 @@ | url | Bucket URL (for single use). | | urls | Bucket URLs. | | urls_list | List of bucket URLs. | - + diff --git a/modules/gke-cluster/README.md b/modules/gke-cluster/README.md index 86b152263b..daf17070b2 100644 --- a/modules/gke-cluster/README.md +++ b/modules/gke-cluster/README.md @@ -6,6 +6,7 @@ TODO(ludoo): add description. TODO(ludoo): add example + ## Variables | name | description | type | required | @@ -17,30 +18,30 @@ TODO(ludoo): add example | secondary_range_pods | Subnet secondary range name used for pods. | string | ✓ | secondary_range_services | Subnet secondary range name used for services. | string | ✓ | subnetwork | VPC subnetwork name or self link. | string | ✓ -| *addons* | Addons enabled in the cluster (true means enabled). | object | -| *authenticator_security_group* | RBAC security group for Google Groups for GKE, format is gke-security-groups@yourdomain.com. | string | -| *cluster_autoscaling* | Enable and configure limits for Node Auto-Provisioning with Cluster Autoscaler. | object | -| *database_encryption* | Enable and configure GKE application-layer secrets encryption. | object | -| *default_max_pods_per_node* | Maximum number of pods per node in this cluster. | number | -| *description* | Cluster description. | string | -| *enable_binary_authorization* | Enable Google Binary Authorization. | bool | -| *enable_intranode_visibility* | Enable intra-node visibility to make same node pod to pod traffic visible. | bool | -| *enable_shielded_nodes* | Enable Shielded Nodes features on all nodes in this cluster. | bool | -| *enable_tpu* | Enable Cloud TPU resources in this cluster. | bool | -| *labels* | Cluster resource labels. | map | -| *logging_service* | Logging service (disable with an empty string). | string | -| *maintenance_start_time* | Maintenance start time in RFC3339 format 'HH:MM', where HH is [00-23] and MM is [00-59] GMT. | string | -| *master_authorized_ranges* | External Ip address ranges that can access the Kubernetes cluster master through HTTPS. | map | -| *min_master_version* | Minimum version of the master, defaults to the version of the most recent official release. | string | -| *monitoring_service* | Monitoring service (disable with an empty string). | string | -| *node_locations* | Zones in which the cluster's nodes are located. | list | -| *pod_security_policy* | Enable the PodSecurityPolicy feature. | bool | -| *private_cluster* | Enable private cluster. | bool | -| *private_cluster_config* | Private cluster configuration. | object | -| *release_channel* | Release channel for GKE upgrades. | string | -| *resource_usage_export_config* | Configure the ResourceUsageExportConfig feature. | object | -| *vertical_pod_autoscaling* | Enable the Vertical Pod Autoscaling feature. | bool | -| *workload_identity* | Enable the Workload Identity feature. | bool | +| *addons* | Addons enabled in the cluster (true means enabled). | object({...}) | +| *authenticator_security_group* | RBAC security group for Google Groups for GKE, format is gke-security-groups@yourdomain.com. | string | +| *cluster_autoscaling* | Enable and configure limits for Node Auto-Provisioning with Cluster Autoscaler. | object({...}) | +| *database_encryption* | Enable and configure GKE application-layer secrets encryption. | object({...}) | +| *default_max_pods_per_node* | Maximum number of pods per node in this cluster. | number | +| *description* | Cluster description. | string | +| *enable_binary_authorization* | Enable Google Binary Authorization. | bool | +| *enable_intranode_visibility* | Enable intra-node visibility to make same node pod to pod traffic visible. | bool | +| *enable_shielded_nodes* | Enable Shielded Nodes features on all nodes in this cluster. | bool | +| *enable_tpu* | Enable Cloud TPU resources in this cluster. | bool | +| *labels* | Cluster resource labels. | map(string) | +| *logging_service* | Logging service (disable with an empty string). | string | +| *maintenance_start_time* | Maintenance start time in RFC3339 format 'HH:MM', where HH is [00-23] and MM is [00-59] GMT. | string | +| *master_authorized_ranges* | External Ip address ranges that can access the Kubernetes cluster master through HTTPS. | map(string) | +| *min_master_version* | Minimum version of the master, defaults to the version of the most recent official release. | string | +| *monitoring_service* | Monitoring service (disable with an empty string). | string | +| *node_locations* | Zones in which the cluster's nodes are located. | list(string) | +| *pod_security_policy* | Enable the PodSecurityPolicy feature. | bool | +| *private_cluster* | Enable private cluster. | bool | +| *private_cluster_config* | Private cluster configuration. | object({...}) | +| *release_channel* | Release channel for GKE upgrades. | string | +| *resource_usage_export_config* | Configure the ResourceUsageExportConfig feature. | object({...}) | +| *vertical_pod_autoscaling* | Enable the Vertical Pod Autoscaling feature. | bool | +| *workload_identity* | Enable the Workload Identity feature. | bool | ## Outputs @@ -50,3 +51,4 @@ TODO(ludoo): add example | endpoint | Cluster endpoint. | | master_version | Master version. | | name | Cluster name. | + diff --git a/modules/net-vpc/README.md b/modules/net-vpc/README.md index 4e80884557..aef69713df 100644 --- a/modules/net-vpc/README.md +++ b/modules/net-vpc/README.md @@ -6,6 +6,7 @@ TODO(ludoo): add description. TODO(ludoo): add example + ## Variables | name | description | type | required | @@ -40,4 +41,5 @@ TODO(ludoo): add example | subnet_secondary_ranges | Map of subnet secondary ranges keyed by name. | | subnet_self_links | Map of subnet self links keyed by name. | | subnets | Subnet resources. | + diff --git a/modules/project/README.md b/modules/project/README.md index 620d254105..0f57230b06 100644 --- a/modules/project/README.md +++ b/modules/project/README.md @@ -11,8 +11,7 @@ The resources/services/activations/deletions that this module will create/trigge - zero or more project-level custom roles - zero or one project liens -## Variables - + ## Variables | name | description | type | required | @@ -22,18 +21,18 @@ The resources/services/activations/deletions that this module will create/trigge | prefix | Prefix used to generate project id and name. | string | ✓ | *auto_create_network* | Whether to create the default network for the project | bool | | *billing_account* | Billing account id. | string | -| *custom_roles* | Map of role name => list of permissions to create in this project. | map | -| *iam_members* | Map of member lists used to set authoritative bindings, keyed by role. | map | -| *iam_nonauth_members* | Map of member lists used to set non authoritative bindings, keyed by role. | map | -| *iam_nonauth_roles* | List of roles used to set non authoritative bindings. | list | -| *iam_roles* | List of roles used to set authoritative bindings. | list | -| *labels* | Resource labels. | map | +| *custom_roles* | Map of role name => list of permissions to create in this project. | map(list(string)) | +| *iam_members* | Map of member lists used to set authoritative bindings, keyed by role. | map(list(string)) | +| *iam_nonauth_members* | Map of member lists used to set non authoritative bindings, keyed by role. | map(list(string)) | +| *iam_nonauth_roles* | List of roles used to set non authoritative bindings. | list(string) | +| *iam_roles* | List of roles used to set authoritative bindings. | list(string) | +| *labels* | Resource labels. | map(string) | | *lien_reason* | If non-empty, creates a project lien with this description. | string | | *oslogin* | Enable OS Login. | bool | -| *oslogin_admins* | List of IAM-style identities that will be granted roles necessary for OS Login administrators. | list | -| *oslogin_users* | List of IAM-style identities that will be granted roles necessary for OS Login users. | list | -| *owners* | IAM-style identities that will be granted non-authoritative viewer role. | list | -| *services* | Service APIs to enable. | list | +| *oslogin_admins* | List of IAM-style identities that will be granted roles necessary for OS Login administrators. | list(string) | +| *oslogin_users* | List of IAM-style identities that will be granted roles necessary for OS Login users. | list(string) | +| *owners* | IAM-style identities that will be granted non-authoritative viewer role. | list(string) | +| *services* | Service APIs to enable. | list(string) | ## Outputs @@ -46,4 +45,4 @@ The resources/services/activations/deletions that this module will create/trigge | name | Name (depends on services). | | number | Project number (depends on services). | | project_id | Project id (depends on services). | - + From 7973a9bc82b4be950121461de6404bd8c1c05736 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Mon, 16 Dec 2019 16:02:36 +0100 Subject: [PATCH 029/106] net-vpc-firewall module --- modules/net-vpc-firewall/README.md | 93 +++++++++++++++++ modules/net-vpc-firewall/main.tf | 145 ++++++++++++++++++++++++++ modules/net-vpc-firewall/outputs.tf | 56 ++++++++++ modules/net-vpc-firewall/variables.tf | 74 +++++++++++++ modules/net-vpc-firewall/versions.tf | 22 ++++ 5 files changed, 390 insertions(+) create mode 100644 modules/net-vpc-firewall/README.md create mode 100644 modules/net-vpc-firewall/main.tf create mode 100644 modules/net-vpc-firewall/outputs.tf create mode 100644 modules/net-vpc-firewall/variables.tf create mode 100644 modules/net-vpc-firewall/versions.tf diff --git a/modules/net-vpc-firewall/README.md b/modules/net-vpc-firewall/README.md new file mode 100644 index 0000000000..6658ba8cc3 --- /dev/null +++ b/modules/net-vpc-firewall/README.md @@ -0,0 +1,93 @@ +# Google Cloud VPC Firewall + +This module allows creation of a minimal VPC firewall, supporting basic configurable rules for IP range-based intra-VPC and administrator ingress, tag-based SSH/HTTP/HTTPS ingress, and custom rule definitions. + +The HTTP and HTTPS rules use the same network tags that are assigned to instances when the "Allow HTTP[S] traffic" checkbox is flagged in the Cloud Console. The SSH rule uses a generic `ssh` tag. + +All IP source ranges are configurable through variables, and are set by default to `0.0.0.0/0` for tag-based rules. Allowed protocols and/or ports for the intra-VPC rule are also configurable through a variable. + +Custom rules are set through a map where keys are rule names, and values use this custom type: + +```hcl +map(object({ + description = string + direction = string # (INGRESS|EGRESS) + action = string # (allow|deny) + ranges = list(string) # list of IP CIDR ranges + sources = list(string) # tags or SAs (ignored for EGRESS) + targets = list(string) # tags or SAs + use_service_accounts = bool # use tags or SAs in sources/targets + rules = list(object({ + protocol = string + ports = list(string) + })) + extra_attributes = map(string) # map, optional keys disabled or priority +})) +``` + +The resources created/managed by this module are: + +- one optional ingress rule from internal CIDR ranges, only allowing ICMP by default +- one optional ingress rule from admin CIDR ranges, allowing all protocols on all ports +- one optional ingress rule for SSH on network tag `ssh` +- one optional ingress rule for HTTP on network tag `http-server` +- one optional ingress rule for HTTPS on network tag `https-server` +- one or more optional custom rules + + +## Usage + +Basic usage of this module is as follows: + +```hcl +module "net-firewall" { + source = "terraform-google-modules/network/google//modules/fabric-net-firewall" + project_id = "my-project" + network = "my-vpc" + internal_ranges_enabled = true + internal_ranges = ["10.0.0.0/0"] + custom_rules = { + ingress-sample = { + description = "Dummy sample ingress rule, tag-based." + direction = "INGRESS" + action = "allow" + ranges = ["192.168.0.0"] + sources = ["spam-tag"] + targets = ["foo-tag", "egg-tag"] + use_service_accounts = false + rules = [ + { + protocol = "tcp" + ports = [] + } + ] + extra_attributes = {} + } + } +} +``` + + +## Variables + +| name | description | type | required | +|---|---|:---: |:---:| +| network | Name of the network this set of firewall rules applies to. | string | ✓ +| project_id | Project id of the project that holds the network. | string | ✓ +| *admin_ranges* | IP CIDR ranges that have complete access to all subnets. | list(string) | +| *admin_ranges_enabled* | Enable admin ranges-based rules. | bool | +| *custom_rules* | List of custom rule definitions (refer to variables file for syntax). | map(object({...})) | +| *http_source_ranges* | List of IP CIDR ranges for tag-based HTTP rule, defaults to 0.0.0.0/0. | list(string) | +| *https_source_ranges* | List of IP CIDR ranges for tag-based HTTPS rule, defaults to 0.0.0.0/0. | list(string) | +| *ssh_source_ranges* | List of IP CIDR ranges for tag-based SSH rule, defaults to 0.0.0.0/0. | list(string) | + +## Outputs + +| name | description | +|---|---| +| admin_ranges | Admin ranges data. | +| custom_egress_allow_rules | Custom egress rules with allow blocks. | +| custom_egress_deny_rules | Custom egress rules with allow blocks. | +| custom_ingress_allow_rules | Custom ingress rules with allow blocks. | +| custom_ingress_deny_rules | Custom ingress rules with deny blocks. | + diff --git a/modules/net-vpc-firewall/main.tf b/modules/net-vpc-firewall/main.tf new file mode 100644 index 0000000000..7252e3a8a8 --- /dev/null +++ b/modules/net-vpc-firewall/main.tf @@ -0,0 +1,145 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + rules-allow = { + for name, attrs in var.custom_rules : name => attrs if attrs.action == "allow" + } + rules-deny = { + for name, attrs in var.custom_rules : name => attrs if attrs.action == "deny" + } +} + +############################################################################### +# rules based on IP ranges +############################################################################### + +resource "google_compute_firewall" "allow-admins" { + count = var.admin_ranges_enabled == true ? 1 : 0 + name = "${var.network}-ingress-admins" + description = "Access from the admin subnet to all subnets" + network = var.network + project = var.project_id + source_ranges = var.admin_ranges + allow { protocol = "icmp" } + allow { protocol = "tcp" } + allow { protocol = "udp" } +} + +############################################################################### +# rules based on tags +############################################################################### + +resource "google_compute_firewall" "allow-tag-ssh" { + count = length(var.ssh_source_ranges) > 0 ? 1 : 0 + name = "${var.network}-ingress-tag-ssh" + description = "Allow SSH to machines with the 'ssh' tag" + network = var.network + project = var.project_id + source_ranges = var.ssh_source_ranges + target_tags = ["ssh"] + allow { + protocol = "tcp" + ports = ["22"] + } +} + +resource "google_compute_firewall" "allow-tag-http" { + count = length(var.http_source_ranges) > 0 ? 1 : 0 + name = "${var.network}-ingress-tag-http" + description = "Allow HTTP to machines with the 'http-server' tag" + network = var.network + project = var.project_id + source_ranges = var.http_source_ranges + target_tags = ["http-server"] + allow { + protocol = "tcp" + ports = ["80"] + } +} + +resource "google_compute_firewall" "allow-tag-https" { + count = length(var.https_source_ranges) > 0 ? 1 : 0 + name = "${var.network}-ingress-tag-https" + description = "Allow HTTPS to machines with the 'https' tag" + network = var.network + project = var.project_id + source_ranges = var.https_source_ranges + target_tags = ["https-server"] + allow { + protocol = "tcp" + ports = ["443"] + } +} + +################################################################################ +# dynamic rules # +################################################################################ + +resource "google_compute_firewall" "custom_allow" { + # provider = "google-beta" + for_each = local.rules-allow + name = each.key + description = each.value.description + direction = each.value.direction + network = var.network + project = var.project_id + source_ranges = each.value.direction == "INGRESS" ? each.value.ranges : null + destination_ranges = each.value.direction == "EGRESS" ? each.value.ranges : null + source_tags = each.value.use_service_accounts || each.value.direction == "EGRESS" ? null : each.value.sources + source_service_accounts = each.value.use_service_accounts && each.value.direction == "INGRESS" ? each.value.sources : null + target_tags = each.value.use_service_accounts ? null : each.value.targets + target_service_accounts = each.value.use_service_accounts ? each.value.targets : null + disabled = lookup(each.value.extra_attributes, "disabled", false) + priority = lookup(each.value.extra_attributes, "priority", 1000) + # enable_logging = lookup(each.value.extra_attributes, "enable_logging", false) + dynamic "allow" { + for_each = each.value.rules + iterator = rule + content { + protocol = rule.value.protocol + ports = rule.value.ports + } + } +} + +resource "google_compute_firewall" "custom_deny" { + # provider = "google-beta" + for_each = local.rules-deny + name = each.key + description = each.value.description + direction = each.value.direction + network = var.network + project = var.project_id + source_ranges = each.value.direction == "INGRESS" ? each.value.ranges : null + destination_ranges = each.value.direction == "EGRESS" ? each.value.ranges : null + source_tags = each.value.use_service_accounts || each.value.direction == "EGRESS" ? null : each.value.sources + source_service_accounts = each.value.use_service_accounts && each.value.direction == "INGRESS" ? each.value.sources : null + target_tags = each.value.use_service_accounts ? null : each.value.targets + target_service_accounts = each.value.use_service_accounts ? each.value.targets : null + disabled = lookup(each.value.extra_attributes, "disabled", false) + priority = lookup(each.value.extra_attributes, "priority", 1000) + # enable_logging = lookup(each.value.extra_attributes, "enable_logging", false) + + dynamic "deny" { + for_each = each.value.rules + iterator = rule + content { + protocol = rule.value.protocol + ports = rule.value.ports + } + } +} diff --git a/modules/net-vpc-firewall/outputs.tf b/modules/net-vpc-firewall/outputs.tf new file mode 100644 index 0000000000..0819e9d118 --- /dev/null +++ b/modules/net-vpc-firewall/outputs.tf @@ -0,0 +1,56 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "admin_ranges" { + description = "Admin ranges data." + + value = { + enabled = var.admin_ranges_enabled + ranges = var.admin_ranges_enabled ? join(",", var.admin_ranges) : "" + } +} + +output "custom_ingress_allow_rules" { + description = "Custom ingress rules with allow blocks." + value = [ + for rule in google_compute_firewall.custom_allow : + rule.name if rule.direction == "INGRESS" + ] +} + +output "custom_ingress_deny_rules" { + description = "Custom ingress rules with deny blocks." + value = [ + for rule in google_compute_firewall.custom_deny : + rule.name if rule.direction == "INGRESS" + ] +} + +output "custom_egress_allow_rules" { + description = "Custom egress rules with allow blocks." + value = [ + for rule in google_compute_firewall.custom_allow : + rule.name if rule.direction == "EGRESS" + ] +} + +output "custom_egress_deny_rules" { + description = "Custom egress rules with allow blocks." + value = [ + for rule in google_compute_firewall.custom_deny : + rule.name if rule.direction == "EGRESS" + ] +} diff --git a/modules/net-vpc-firewall/variables.tf b/modules/net-vpc-firewall/variables.tf new file mode 100644 index 0000000000..c250ac5199 --- /dev/null +++ b/modules/net-vpc-firewall/variables.tf @@ -0,0 +1,74 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "network" { + description = "Name of the network this set of firewall rules applies to." + type = string +} + +variable "project_id" { + description = "Project id of the project that holds the network." + type = string +} + +variable "admin_ranges_enabled" { + description = "Enable admin ranges-based rules." + type = bool + default = false +} + +variable "admin_ranges" { + description = "IP CIDR ranges that have complete access to all subnets." + type = list(string) + default = [] +} + +variable "ssh_source_ranges" { + description = "List of IP CIDR ranges for tag-based SSH rule, defaults to 0.0.0.0/0." + type = list(string) + default = ["0.0.0.0/0"] +} + +variable "http_source_ranges" { + description = "List of IP CIDR ranges for tag-based HTTP rule, defaults to 0.0.0.0/0." + type = list(string) + default = ["0.0.0.0/0"] +} + +variable "https_source_ranges" { + description = "List of IP CIDR ranges for tag-based HTTPS rule, defaults to 0.0.0.0/0." + type = list(string) + default = ["0.0.0.0/0"] +} + +variable "custom_rules" { + description = "List of custom rule definitions (refer to variables file for syntax)." + type = map(object({ + description = string + direction = string + action = string # (allow|deny) + ranges = list(string) + sources = list(string) + targets = list(string) + use_service_accounts = bool + rules = list(object({ + protocol = string + ports = list(string) + })) + extra_attributes = map(string) + })) + default = {} +} diff --git a/modules/net-vpc-firewall/versions.tf b/modules/net-vpc-firewall/versions.tf new file mode 100644 index 0000000000..46ce80b846 --- /dev/null +++ b/modules/net-vpc-firewall/versions.tf @@ -0,0 +1,22 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = "~> 0.12.6" + required_providers { + google = "~> 2.19" + } +} From 162fd23055e265024f4cef52734afc94767777e3 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Mon, 16 Dec 2019 16:13:27 +0100 Subject: [PATCH 030/106] add support for sensitive output attribute in tfdoc --- tools/tfdoc/tfdoc.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/tools/tfdoc/tfdoc.py b/tools/tfdoc/tfdoc.py index 3adc861e2e..73fb05f58a 100755 --- a/tools/tfdoc/tfdoc.py +++ b/tools/tfdoc/tfdoc.py @@ -26,7 +26,8 @@ MARK_END = '' RE_OUTPUTS = re.compile(r'''(?smx) (?:^\s*output\s*"([^"]+)"\s*\{$) | - (?:^\s*description\s*=\s*"([^"]+)"\s*$) + (?:^\s*description\s*=\s*"([^"]+)"\s*$) | + (?:^\s*sensitive\s*=\s*(\S+)\s*$) ''') RE_TYPE = re.compile(r'([\(\{\}\)])') RE_VARIABLES = re.compile(r'''(?smx) @@ -48,8 +49,8 @@ (?:^\s*(\S.*?)$) ''') -OutputData = collections.namedtuple('Output', 'name description') -OutputToken = enum.Enum('OutputToken', 'NAME DESCRIPTION') +OutputData = collections.namedtuple('Output', 'name description sensitive') +OutputToken = enum.Enum('OutputToken', 'NAME DESCRIPTION SENSITIVE') VariableData = collections.namedtuple( 'Variable', 'name description type default required') VariableToken = enum.Enum( @@ -66,7 +67,7 @@ class Output(object): def __init__(self): self.in_progress = False - self.name = self.description = None + self.name = self.description = self.sensitive = None def parse_token(self, token_type, token_data): if token_type == 'NAME': @@ -74,11 +75,11 @@ def parse_token(self, token_type, token_data): raise ItemParsed(self.close()) self.in_progress = True self.name = token_data - elif token_type == 'DESCRIPTION': + else: setattr(self, token_type.lower(), token_data) def close(self): - return OutputData(self.name, self.description) + return OutputData(self.name, self.description, self.sensitive) class Variable(object): @@ -131,11 +132,12 @@ def format_outputs(outputs): if not outputs: return outputs.sort(key=lambda v: v.name) - yield '| name | description |' - yield '|---|---|' + yield '| name | description | sensitive |' + yield '|---|---|:---:|' for o in outputs: - yield '| {name} | {description} |'.format( - name=o.name, description=o.description) + yield '| {name} | {description} | {sensitive} |'.format( + name=o.name, description=o.description, + sensitive='✓' if o.sensitive else '') def format_type(type_spec): From 1b574f80f08155edf3dc55686708aeac10036cd2 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Mon, 16 Dec 2019 16:14:03 +0100 Subject: [PATCH 031/106] remove empty function from tfdoc --- tools/tfdoc/tfdoc.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tools/tfdoc/tfdoc.py b/tools/tfdoc/tfdoc.py index 73fb05f58a..3902a4f563 100755 --- a/tools/tfdoc/tfdoc.py +++ b/tools/tfdoc/tfdoc.py @@ -122,11 +122,6 @@ def close(self): self.default is None) -def format_output(output): - "Format output." - return - - def format_outputs(outputs): "Format variables." if not outputs: From e09229361d65fafc8a0993841150ff93a3d200e2 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Mon, 16 Dec 2019 16:17:25 +0100 Subject: [PATCH 032/106] render variable type as code in tfdoc --- tools/tfdoc/tfdoc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/tfdoc/tfdoc.py b/tools/tfdoc/tfdoc.py index 3902a4f563..af38b1de4f 100755 --- a/tools/tfdoc/tfdoc.py +++ b/tools/tfdoc/tfdoc.py @@ -123,7 +123,7 @@ def close(self): def format_outputs(outputs): - "Format variables." + "Format outputs." if not outputs: return outputs.sort(key=lambda v: v.name) @@ -161,7 +161,7 @@ def format_variables(variables, required_first=True): yield '| name | description | type | required |' yield '|---|---|:---: |:---:|' for v in variables: - yield '| {name} | {description} | {type} | {required}'.format( + yield '| {name} | {description} | `{type}` | {required}'.format( name=v.name if v.required else '*%s*' % v.name, description=v.description, type=format_type(v.type), required='✓' if v.required else '' From 5e02abee4df9b467272bcdffeb6feaf28c602a3f Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Mon, 16 Dec 2019 16:17:36 +0100 Subject: [PATCH 033/106] update module READMEs --- modules/folder/README.md | 28 +++++------ modules/gcs/README.md | 42 ++++++++--------- modules/gke-cluster/README.md | 74 +++++++++++++++--------------- modules/net-address/README.md | 0 modules/net-address/main.tf | 0 modules/net-address/outputs.tf | 0 modules/net-address/variables.tf | 0 modules/net-vpc-firewall/README.md | 30 ++++++------ modules/net-vpc/README.md | 54 +++++++++++----------- modules/project/README.md | 52 ++++++++++----------- 10 files changed, 140 insertions(+), 140 deletions(-) create mode 100644 modules/net-address/README.md create mode 100644 modules/net-address/main.tf create mode 100644 modules/net-address/outputs.tf create mode 100644 modules/net-address/variables.tf diff --git a/modules/folder/README.md b/modules/folder/README.md index 8ab8213eae..4df53a1e34 100644 --- a/modules/folder/README.md +++ b/modules/folder/README.md @@ -5,21 +5,21 @@ | name | description | type | required | |---|---|:---: |:---:| -| parent | Parent in folders/folder_id or organizations/org_id format. | string | ✓ -| *iam_members* | List of IAM members keyed by folder name and role. | map(map(list(string))) | -| *iam_roles* | List of IAM roles keyed by folder name. | map(list(string)) | -| *names* | Folder names. | list(string) | +| parent | Parent in folders/folder_id or organizations/org_id format. | `string` | ✓ +| *iam_members* | List of IAM members keyed by folder name and role. | `map(map(list(string)))` | +| *iam_roles* | List of IAM roles keyed by folder name. | `map(list(string))` | +| *names* | Folder names. | `list(string)` | ## Outputs -| name | description | -|---|---| -| folder | Folder resource (for single use). | -| folders | Folder resources. | -| id | Folder id (for single use). | -| ids | Folder ids. | -| ids_list | List of folder ids. | -| name | Folder name (for single use). | -| names | Folder names. | -| names_list | List of folder names. | +| name | description | sensitive | +|---|---|:---:| +| folder | Folder resource (for single use). | | +| folders | Folder resources. | | +| id | Folder id (for single use). | | +| ids | Folder ids. | | +| ids_list | List of folder ids. | | +| name | Folder name (for single use). | | +| names | Folder names. | | +| names_list | List of folder names. | | diff --git a/modules/gcs/README.md b/modules/gcs/README.md index b492392411..4304a531e9 100644 --- a/modules/gcs/README.md +++ b/modules/gcs/README.md @@ -5,28 +5,28 @@ | name | description | type | required | |---|---|:---: |:---:| -| names | Bucket name suffixes. | list(string) | ✓ -| project_id | Bucket project id. | string | ✓ -| *bucket_policy_only* | Optional map to disable object ACLS keyed by name, defaults to true. | map(bool) | -| *force_destroy* | Optional map to set force destroy keyed by name, defaults to false. | map(bool) | -| *iam_members* | List of IAM members keyed by name and role. | map(map(list(string))) | -| *iam_roles* | List of IAM roles keyed by name. | map(list(string)) | -| *labels* | Labels to be attached to all buckets. | map(string) | -| *location* | Bucket location. | string | -| *prefix* | Prefix used to generate the bucket name. | string | -| *storage_class* | Bucket storage class. | string | -| *versioning* | Optional map to set versioning keyed by name, defaults to false. | map(bool) | +| names | Bucket name suffixes. | `list(string)` | ✓ +| project_id | Bucket project id. | `string` | ✓ +| *bucket_policy_only* | Optional map to disable object ACLS keyed by name, defaults to true. | `map(bool)` | +| *force_destroy* | Optional map to set force destroy keyed by name, defaults to false. | `map(bool)` | +| *iam_members* | List of IAM members keyed by name and role. | `map(map(list(string)))` | +| *iam_roles* | List of IAM roles keyed by name. | `map(list(string))` | +| *labels* | Labels to be attached to all buckets. | `map(string)` | +| *location* | Bucket location. | `string` | +| *prefix* | Prefix used to generate the bucket name. | `string` | +| *storage_class* | Bucket storage class. | `string` | +| *versioning* | Optional map to set versioning keyed by name, defaults to false. | `map(bool)` | ## Outputs -| name | description | -|---|---| -| bucket | Bucket resource (for single use). | -| buckets | Bucket resources. | -| name | Bucket name (for single use). | -| names | Bucket names. | -| names_list | List of bucket names. | -| url | Bucket URL (for single use). | -| urls | Bucket URLs. | -| urls_list | List of bucket URLs. | +| name | description | sensitive | +|---|---|:---:| +| bucket | Bucket resource (for single use). | | +| buckets | Bucket resources. | | +| name | Bucket name (for single use). | | +| names | Bucket names. | | +| names_list | List of bucket names. | | +| url | Bucket URL (for single use). | | +| urls | Bucket URLs. | | +| urls_list | List of bucket URLs. | | diff --git a/modules/gke-cluster/README.md b/modules/gke-cluster/README.md index daf17070b2..47b620f372 100644 --- a/modules/gke-cluster/README.md +++ b/modules/gke-cluster/README.md @@ -11,44 +11,44 @@ TODO(ludoo): add example | name | description | type | required | |---|---|:---: |:---:| -| location | Cluster zone or region. | string | ✓ -| name | Cluster name. | string | ✓ -| network | Name or self link of the VPC used for the cluster. Use the self link for Shared VPC. | string | ✓ -| project_id | Cluster project id. | string | ✓ -| secondary_range_pods | Subnet secondary range name used for pods. | string | ✓ -| secondary_range_services | Subnet secondary range name used for services. | string | ✓ -| subnetwork | VPC subnetwork name or self link. | string | ✓ -| *addons* | Addons enabled in the cluster (true means enabled). | object({...}) | -| *authenticator_security_group* | RBAC security group for Google Groups for GKE, format is gke-security-groups@yourdomain.com. | string | -| *cluster_autoscaling* | Enable and configure limits for Node Auto-Provisioning with Cluster Autoscaler. | object({...}) | -| *database_encryption* | Enable and configure GKE application-layer secrets encryption. | object({...}) | -| *default_max_pods_per_node* | Maximum number of pods per node in this cluster. | number | -| *description* | Cluster description. | string | -| *enable_binary_authorization* | Enable Google Binary Authorization. | bool | -| *enable_intranode_visibility* | Enable intra-node visibility to make same node pod to pod traffic visible. | bool | -| *enable_shielded_nodes* | Enable Shielded Nodes features on all nodes in this cluster. | bool | -| *enable_tpu* | Enable Cloud TPU resources in this cluster. | bool | -| *labels* | Cluster resource labels. | map(string) | -| *logging_service* | Logging service (disable with an empty string). | string | -| *maintenance_start_time* | Maintenance start time in RFC3339 format 'HH:MM', where HH is [00-23] and MM is [00-59] GMT. | string | -| *master_authorized_ranges* | External Ip address ranges that can access the Kubernetes cluster master through HTTPS. | map(string) | -| *min_master_version* | Minimum version of the master, defaults to the version of the most recent official release. | string | -| *monitoring_service* | Monitoring service (disable with an empty string). | string | -| *node_locations* | Zones in which the cluster's nodes are located. | list(string) | -| *pod_security_policy* | Enable the PodSecurityPolicy feature. | bool | -| *private_cluster* | Enable private cluster. | bool | -| *private_cluster_config* | Private cluster configuration. | object({...}) | -| *release_channel* | Release channel for GKE upgrades. | string | -| *resource_usage_export_config* | Configure the ResourceUsageExportConfig feature. | object({...}) | -| *vertical_pod_autoscaling* | Enable the Vertical Pod Autoscaling feature. | bool | -| *workload_identity* | Enable the Workload Identity feature. | bool | +| location | Cluster zone or region. | `string` | ✓ +| name | Cluster name. | `string` | ✓ +| network | Name or self link of the VPC used for the cluster. Use the self link for Shared VPC. | `string` | ✓ +| project_id | Cluster project id. | `string` | ✓ +| secondary_range_pods | Subnet secondary range name used for pods. | `string` | ✓ +| secondary_range_services | Subnet secondary range name used for services. | `string` | ✓ +| subnetwork | VPC subnetwork name or self link. | `string` | ✓ +| *addons* | Addons enabled in the cluster (true means enabled). | `object({...})` | +| *authenticator_security_group* | RBAC security group for Google Groups for GKE, format is gke-security-groups@yourdomain.com. | `string` | +| *cluster_autoscaling* | Enable and configure limits for Node Auto-Provisioning with Cluster Autoscaler. | `object({...})` | +| *database_encryption* | Enable and configure GKE application-layer secrets encryption. | `object({...})` | +| *default_max_pods_per_node* | Maximum number of pods per node in this cluster. | `number` | +| *description* | Cluster description. | `string` | +| *enable_binary_authorization* | Enable Google Binary Authorization. | `bool` | +| *enable_intranode_visibility* | Enable intra-node visibility to make same node pod to pod traffic visible. | `bool` | +| *enable_shielded_nodes* | Enable Shielded Nodes features on all nodes in this cluster. | `bool` | +| *enable_tpu* | Enable Cloud TPU resources in this cluster. | `bool` | +| *labels* | Cluster resource labels. | `map(string)` | +| *logging_service* | Logging service (disable with an empty string). | `string` | +| *maintenance_start_time* | Maintenance start time in RFC3339 format 'HH:MM', where HH is [00-23] and MM is [00-59] GMT. | `string` | +| *master_authorized_ranges* | External Ip address ranges that can access the Kubernetes cluster master through HTTPS. | `map(string)` | +| *min_master_version* | Minimum version of the master, defaults to the version of the most recent official release. | `string` | +| *monitoring_service* | Monitoring service (disable with an empty string). | `string` | +| *node_locations* | Zones in which the cluster's nodes are located. | `list(string)` | +| *pod_security_policy* | Enable the PodSecurityPolicy feature. | `bool` | +| *private_cluster* | Enable private cluster. | `bool` | +| *private_cluster_config* | Private cluster configuration. | `object({...})` | +| *release_channel* | Release channel for GKE upgrades. | `string` | +| *resource_usage_export_config* | Configure the ResourceUsageExportConfig feature. | `object({...})` | +| *vertical_pod_autoscaling* | Enable the Vertical Pod Autoscaling feature. | `bool` | +| *workload_identity* | Enable the Workload Identity feature. | `bool` | ## Outputs -| name | description | -|---|---| -| cluster | Cluster resource. | -| endpoint | Cluster endpoint. | -| master_version | Master version. | -| name | Cluster name. | +| name | description | sensitive | +|---|---|:---:| +| cluster | Cluster resource. | ✓ | +| endpoint | Cluster endpoint. | | +| master_version | Master version. | | +| name | Cluster name. | | diff --git a/modules/net-address/README.md b/modules/net-address/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/modules/net-address/main.tf b/modules/net-address/main.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/modules/net-address/outputs.tf b/modules/net-address/outputs.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/modules/net-address/variables.tf b/modules/net-address/variables.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/modules/net-vpc-firewall/README.md b/modules/net-vpc-firewall/README.md index 6658ba8cc3..e70e6f4f08 100644 --- a/modules/net-vpc-firewall/README.md +++ b/modules/net-vpc-firewall/README.md @@ -72,22 +72,22 @@ module "net-firewall" { | name | description | type | required | |---|---|:---: |:---:| -| network | Name of the network this set of firewall rules applies to. | string | ✓ -| project_id | Project id of the project that holds the network. | string | ✓ -| *admin_ranges* | IP CIDR ranges that have complete access to all subnets. | list(string) | -| *admin_ranges_enabled* | Enable admin ranges-based rules. | bool | -| *custom_rules* | List of custom rule definitions (refer to variables file for syntax). | map(object({...})) | -| *http_source_ranges* | List of IP CIDR ranges for tag-based HTTP rule, defaults to 0.0.0.0/0. | list(string) | -| *https_source_ranges* | List of IP CIDR ranges for tag-based HTTPS rule, defaults to 0.0.0.0/0. | list(string) | -| *ssh_source_ranges* | List of IP CIDR ranges for tag-based SSH rule, defaults to 0.0.0.0/0. | list(string) | +| network | Name of the network this set of firewall rules applies to. | `string` | ✓ +| project_id | Project id of the project that holds the network. | `string` | ✓ +| *admin_ranges* | IP CIDR ranges that have complete access to all subnets. | `list(string)` | +| *admin_ranges_enabled* | Enable admin ranges-based rules. | `bool` | +| *custom_rules* | List of custom rule definitions (refer to variables file for syntax). | `map(object({...}))` | +| *http_source_ranges* | List of IP CIDR ranges for tag-based HTTP rule, defaults to 0.0.0.0/0. | `list(string)` | +| *https_source_ranges* | List of IP CIDR ranges for tag-based HTTPS rule, defaults to 0.0.0.0/0. | `list(string)` | +| *ssh_source_ranges* | List of IP CIDR ranges for tag-based SSH rule, defaults to 0.0.0.0/0. | `list(string)` | ## Outputs -| name | description | -|---|---| -| admin_ranges | Admin ranges data. | -| custom_egress_allow_rules | Custom egress rules with allow blocks. | -| custom_egress_deny_rules | Custom egress rules with allow blocks. | -| custom_ingress_allow_rules | Custom ingress rules with allow blocks. | -| custom_ingress_deny_rules | Custom ingress rules with deny blocks. | +| name | description | sensitive | +|---|---|:---:| +| admin_ranges | Admin ranges data. | | +| custom_egress_allow_rules | Custom egress rules with allow blocks. | | +| custom_egress_deny_rules | Custom egress rules with allow blocks. | | +| custom_ingress_allow_rules | Custom ingress rules with allow blocks. | | +| custom_ingress_deny_rules | Custom ingress rules with deny blocks. | | diff --git a/modules/net-vpc/README.md b/modules/net-vpc/README.md index aef69713df..c3e57b30d7 100644 --- a/modules/net-vpc/README.md +++ b/modules/net-vpc/README.md @@ -11,35 +11,35 @@ TODO(ludoo): add example | name | description | type | required | |---|---|:---: |:---:| -| name | The name of the network being created | string | ✓ -| project_id | The ID of the project where this VPC will be created | string | ✓ -| *auto_create_subnetworks* | Set to true to create an auto mode subnet, defaults to custom mode. | bool | -| *description* | An optional description of this resource (triggers recreation on change). | string | -| *iam_members* | List of IAM members keyed by subnet and role. | map(map(list(string))) | -| *iam_roles* | List of IAM roles keyed by subnet. | map(list(string)) | -| *log_config_defaults* | Default configuration for flow logs when enabled. | object({...}) | -| *log_configs* | Map of per-subnet optional configurations for flow logs when enabled. | map(map(string)) | -| *routing_mode* | The network routing mode (default 'GLOBAL') | string | -| *shared_vpc_host* | Makes this project a Shared VPC host if 'true' (default 'false') | bool | -| *shared_vpc_service_projects* | Shared VPC service projects to register with this host | list(string) | -| *subnet_descriptions* | Optional map of subnet descriptions, keyed by subnet name. | map(string) | -| *subnet_flow_logs* | Optional map of boolean to control flow logs (default is disabled), keyed by subnet name. | map(bool) | -| *subnet_private_access* | Optional map of boolean to control private Google access (default is enabled), keyed by subnet name. | map(bool) | -| *subnets* | The list of subnets being created | map(object({...})) | +| name | The name of the network being created | `string` | ✓ +| project_id | The ID of the project where this VPC will be created | `string` | ✓ +| *auto_create_subnetworks* | Set to true to create an auto mode subnet, defaults to custom mode. | `bool` | +| *description* | An optional description of this resource (triggers recreation on change). | `string` | +| *iam_members* | List of IAM members keyed by subnet and role. | `map(map(list(string)))` | +| *iam_roles* | List of IAM roles keyed by subnet. | `map(list(string))` | +| *log_config_defaults* | Default configuration for flow logs when enabled. | `object({...})` | +| *log_configs* | Map of per-subnet optional configurations for flow logs when enabled. | `map(map(string))` | +| *routing_mode* | The network routing mode (default 'GLOBAL') | `string` | +| *shared_vpc_host* | Makes this project a Shared VPC host if 'true' (default 'false') | `bool` | +| *shared_vpc_service_projects* | Shared VPC service projects to register with this host | `list(string)` | +| *subnet_descriptions* | Optional map of subnet descriptions, keyed by subnet name. | `map(string)` | +| *subnet_flow_logs* | Optional map of boolean to control flow logs (default is disabled), keyed by subnet name. | `map(bool)` | +| *subnet_private_access* | Optional map of boolean to control private Google access (default is enabled), keyed by subnet name. | `map(bool)` | +| *subnets* | The list of subnets being created | `map(object({...}))` | ## Outputs -| name | description | -|---|---| -| bindings | Subnet IAM bindings. | -| name | The name of the VPC being created. | -| network | Network resource. | -| project_id | Shared VPC host project id. | -| self_link | The URI of the VPC being created. | -| subnet_ips | Map of subnet address ranges keyed by name. | -| subnet_regions | Map of subnet regions keyed by name. | -| subnet_secondary_ranges | Map of subnet secondary ranges keyed by name. | -| subnet_self_links | Map of subnet self links keyed by name. | -| subnets | Subnet resources. | +| name | description | sensitive | +|---|---|:---:| +| bindings | Subnet IAM bindings. | | +| name | The name of the VPC being created. | | +| network | Network resource. | | +| project_id | Shared VPC host project id. | | +| self_link | The URI of the VPC being created. | | +| subnet_ips | Map of subnet address ranges keyed by name. | | +| subnet_regions | Map of subnet regions keyed by name. | | +| subnet_secondary_ranges | Map of subnet secondary ranges keyed by name. | | +| subnet_self_links | Map of subnet self links keyed by name. | | +| subnets | Subnet resources. | | diff --git a/modules/project/README.md b/modules/project/README.md index 0f57230b06..83f652ce6b 100644 --- a/modules/project/README.md +++ b/modules/project/README.md @@ -16,33 +16,33 @@ The resources/services/activations/deletions that this module will create/trigge | name | description | type | required | |---|---|:---: |:---:| -| name | Project name and id suffix. | string | ✓ -| parent | The resource name of the parent Folder or Organization. Must be of the form folders/folder_id or organizations/org_id. | string | ✓ -| prefix | Prefix used to generate project id and name. | string | ✓ -| *auto_create_network* | Whether to create the default network for the project | bool | -| *billing_account* | Billing account id. | string | -| *custom_roles* | Map of role name => list of permissions to create in this project. | map(list(string)) | -| *iam_members* | Map of member lists used to set authoritative bindings, keyed by role. | map(list(string)) | -| *iam_nonauth_members* | Map of member lists used to set non authoritative bindings, keyed by role. | map(list(string)) | -| *iam_nonauth_roles* | List of roles used to set non authoritative bindings. | list(string) | -| *iam_roles* | List of roles used to set authoritative bindings. | list(string) | -| *labels* | Resource labels. | map(string) | -| *lien_reason* | If non-empty, creates a project lien with this description. | string | -| *oslogin* | Enable OS Login. | bool | -| *oslogin_admins* | List of IAM-style identities that will be granted roles necessary for OS Login administrators. | list(string) | -| *oslogin_users* | List of IAM-style identities that will be granted roles necessary for OS Login users. | list(string) | -| *owners* | IAM-style identities that will be granted non-authoritative viewer role. | list(string) | -| *services* | Service APIs to enable. | list(string) | +| name | Project name and id suffix. | `string` | ✓ +| parent | The resource name of the parent Folder or Organization. Must be of the form folders/folder_id or organizations/org_id. | `string` | ✓ +| prefix | Prefix used to generate project id and name. | `string` | ✓ +| *auto_create_network* | Whether to create the default network for the project | `bool` | +| *billing_account* | Billing account id. | `string` | +| *custom_roles* | Map of role name => list of permissions to create in this project. | `map(list(string))` | +| *iam_members* | Map of member lists used to set authoritative bindings, keyed by role. | `map(list(string))` | +| *iam_nonauth_members* | Map of member lists used to set non authoritative bindings, keyed by role. | `map(list(string))` | +| *iam_nonauth_roles* | List of roles used to set non authoritative bindings. | `list(string)` | +| *iam_roles* | List of roles used to set authoritative bindings. | `list(string)` | +| *labels* | Resource labels. | `map(string)` | +| *lien_reason* | If non-empty, creates a project lien with this description. | `string` | +| *oslogin* | Enable OS Login. | `bool` | +| *oslogin_admins* | List of IAM-style identities that will be granted roles necessary for OS Login administrators. | `list(string)` | +| *oslogin_users* | List of IAM-style identities that will be granted roles necessary for OS Login users. | `list(string)` | +| *owners* | IAM-style identities that will be granted non-authoritative viewer role. | `list(string)` | +| *services* | Service APIs to enable. | `list(string)` | ## Outputs -| name | description | -|---|---| -| cloudsvc_service_account | Cloud services service account (depends on services). | -| custom_roles | Ids of the created custom roles. | -| gce_service_account | Default GCE service account (depends on services). | -| gke_service_account | Default GKE service account (depends on services). | -| name | Name (depends on services). | -| number | Project number (depends on services). | -| project_id | Project id (depends on services). | +| name | description | sensitive | +|---|---|:---:| +| cloudsvc_service_account | Cloud services service account (depends on services). | | +| custom_roles | Ids of the created custom roles. | | +| gce_service_account | Default GCE service account (depends on services). | | +| gke_service_account | Default GKE service account (depends on services). | | +| name | Name (depends on services). | | +| number | Project number (depends on services). | | +| project_id | Project id (depends on services). | | From 65916367d1651014918e83444468eacd9f3869e0 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Mon, 16 Dec 2019 17:22:38 +0100 Subject: [PATCH 034/106] net address module --- modules/net-address/README.md | 22 +++++++++++ modules/net-address/main.tf | 44 +++++++++++++++++++++ modules/net-address/outputs.tf | 48 +++++++++++++++++++++++ modules/net-address/variables.tf | 65 ++++++++++++++++++++++++++++++++ 4 files changed, 179 insertions(+) diff --git a/modules/net-address/README.md b/modules/net-address/README.md index e69de29bb2..4310d0b703 100644 --- a/modules/net-address/README.md +++ b/modules/net-address/README.md @@ -0,0 +1,22 @@ +# Net Address Reservation Module + + +## Variables + +| name | description | type | required | +|---|---|:---: |:---:| +| project_id | Project where the addresses will be created. | `string` | ✓ +| *external_addresses* | Map of external address regions, keyed by name. | `map(string)` | +| *global_addresses* | List of global addresses to create. | `list(string)` | +| *internal_address_addresses* | Optional explicit addresses for internal addresses, keyed by name. | `map(string)` | +| *internal_address_tiers* | Optional network tiers for internal addresses, keyed by name. | `map(string)` | +| *internal_addresses* | Map of internal addresses to create, keyed by name. | `map(object({...}))` | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| external_addresses | None | | +| global_addresses | None | | +| internal_addresses | None | | + \ No newline at end of file diff --git a/modules/net-address/main.tf b/modules/net-address/main.tf index e69de29bb2..b752f2aa90 100644 --- a/modules/net-address/main.tf +++ b/modules/net-address/main.tf @@ -0,0 +1,44 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_compute_global_address" "global" { + for_each = toset(var.global_addresses) + project = var.project_id + name = each.value +} + +resource "google_compute_address" "external" { + for_each = var.external_addresses + project = var.project_id + name = each.key + description = "Terraform managed." + address_type = "EXTERNAL" + region = each.value + # labels = lookup(var.external_address_labels, each.key, {}) +} + +resource "google_compute_address" "internal" { + for_each = var.internal_addresses + project = var.project_id + name = each.key + description = "Terraform managed." + address_type = "INTERNAL" + region = each.value.region + subnetwork = each.value.subnetwork + address = lookup(var.internal_address_addresses, each.key, null) + network_tier = lookup(var.internal_address_tiers, each.key, null) + # labels = lookup(var.internal_address_labels, each.key, {}) +} diff --git a/modules/net-address/outputs.tf b/modules/net-address/outputs.tf index e69de29bb2..7d26158a62 100644 --- a/modules/net-address/outputs.tf +++ b/modules/net-address/outputs.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "external_addresses" { + value = { + for address in google_compute_address.external : + address.name => { + address = address.address + self_link = address.self_link + users = address.users + } + } +} + +output "global_addresses" { + value = { + for address in google_compute_global_address.global : + address.name => { + address = address.address + self_link = address.self_link + status = address.status + } + } +} + +output "internal_addresses" { + value = { + for address in google_compute_address.internal : + address.name => { + address = address.address + self_link = address.self_link + users = address.users + } + } +} diff --git a/modules/net-address/variables.tf b/modules/net-address/variables.tf index e69de29bb2..02b85f68b4 100644 --- a/modules/net-address/variables.tf +++ b/modules/net-address/variables.tf @@ -0,0 +1,65 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "external_addresses" { + description = "Map of external address regions, keyed by name." + type = map(string) + default = {} +} + +# variable "external_address_labels" { +# description = "Optional labels for external addresses, keyed by address name." +# type = map(map(string)) +# default = {} +# } + +variable "global_addresses" { + description = "List of global addresses to create." + type = list(string) + default = [] +} + +variable "internal_addresses" { + description = "Map of internal addresses to create, keyed by name." + type = map(object({ + region = string + subnetwork = string + })) + default = {} +} + +variable "internal_address_addresses" { + description = "Optional explicit addresses for internal addresses, keyed by name." + type = map(string) + default = {} +} + +variable "internal_address_tiers" { + description = "Optional network tiers for internal addresses, keyed by name." + type = map(string) + default = {} +} + +# variable "internal_address_labels" { +# description = "Optional labels for internal addresses, keyed by address name." +# type = map(map(string)) +# default = {} +# } + +variable "project_id" { + description = "Project where the addresses will be created." + type = string +} From 0edb9da9771eb334824926a23ebb0b38922da6a2 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Mon, 16 Dec 2019 20:09:09 +0100 Subject: [PATCH 035/106] net cloudnat module --- modules/net-cloudnat/README.md | 32 +++++++++ modules/net-cloudnat/main.tf | 60 ++++++++++++++++ modules/net-cloudnat/outputs.tf | 40 +++++++++++ modules/net-cloudnat/variables.tf | 111 ++++++++++++++++++++++++++++++ modules/net-cloudnat/versions.tf | 19 +++++ 5 files changed, 262 insertions(+) create mode 100644 modules/net-cloudnat/README.md create mode 100644 modules/net-cloudnat/main.tf create mode 100644 modules/net-cloudnat/outputs.tf create mode 100644 modules/net-cloudnat/variables.tf create mode 100644 modules/net-cloudnat/versions.tf diff --git a/modules/net-cloudnat/README.md b/modules/net-cloudnat/README.md new file mode 100644 index 0000000000..0690992c7f --- /dev/null +++ b/modules/net-cloudnat/README.md @@ -0,0 +1,32 @@ +# Google Cloud NAT Module + + +## Variables + +| name | description | type | required | +|---|---|:---: |:---:| +| project_id | Project where resources will be created. | `string` | ✓ +| region | Region where resources will be created. | `string` | ✓ +| *addresses* | Optional list of external address self links. | `list(string)` | +| *config_min_ports_per_vm* | Minimum number of ports allocated to a VM from this NAT config. | `number` | +| *config_source_subnets* | Subnetwork configuration, valid values are ALL_SUBNETWORKS_ALL_IP_RANGES, ALL_SUBNETWORKS_ALL_PRIMARY_IP_RANGES, LIST_OF_SUBNETWORKS. | `string` | +| *config_timeouts* | Timeout configurations. | `object({...})` | +| *create_router* | Create router for Cloud NAT instead of using existing one. | `bool` | +| *name* | Name of the Cloud NAT resource. | `string` | +| *network* | Name of the VPC where optional router will be created. | `string` | +| *prefix* | Optional prefix that will be prepended to resource names. | `string` | +| *router_asn* | Router ASN used for auto-created router. | `number` | +| *router_name* | Name of the existing or auto-created router. | `string` | +| *router_network* | Name of the VPC used for auto-created router. | `string` | +| *subnetworks* | Subnetworks to NAT, only used when config_source_subnets equals LIST_OF_SUBNETWORKS. | `list(object({...}))` | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| name | Name of the Cloud NAT. | | +| nat_ip_allocate_option | NAT IP allocation mode. | | +| region | Cloud NAT region. | | +| router | Cloud NAT router resources (if auto created). | | +| router_name | Cloud NAT router name. | | + diff --git a/modules/net-cloudnat/main.tf b/modules/net-cloudnat/main.tf new file mode 100644 index 0000000000..de7f540c25 --- /dev/null +++ b/modules/net-cloudnat/main.tf @@ -0,0 +1,60 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + prefix = var.prefix == "" ? "" : "${var.prefix}-" + router = ( + var.create_router + ? google_compute_router.router[0].name + : var.router_name + ) +} + +resource "google_compute_router" "router" { + count = var.create_router ? 1 : 0 + name = "${local.prefix}${var.router_name}" + project = var.project_id + region = var.region + network = var.router_network + bgp { + asn = var.router_asn + } +} + +resource "google_compute_router_nat" "nat" { + project = var.project_id + region = var.region + name = "${local.prefix}${var.name}" + router = local.router + nat_ips = var.addresses + nat_ip_allocate_option = length(var.addresses) > 0 ? "MANUAL_ONLY" : "AUTO_ONLY" + source_subnetwork_ip_ranges_to_nat = var.config_source_subnets + min_ports_per_vm = var.config_min_ports_per_vm + icmp_idle_timeout_sec = var.config_timeouts.icmp + udp_idle_timeout_sec = var.config_timeouts.udp + tcp_established_idle_timeout_sec = var.config_timeouts.tcp_established + tcp_transitory_idle_timeout_sec = var.config_timeouts.tcp_transitory + + dynamic "subnetwork" { + for_each = var.subnetworks + content { + name = subnetwork.value.self_link + source_ip_ranges_to_nat = subnetwork.value.config_source_ranges + secondary_ip_range_names = subnetwork.value.secondary_ranges + } + } +} + diff --git a/modules/net-cloudnat/outputs.tf b/modules/net-cloudnat/outputs.tf new file mode 100644 index 0000000000..69a4972cb3 --- /dev/null +++ b/modules/net-cloudnat/outputs.tf @@ -0,0 +1,40 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "name" { + description = "Name of the Cloud NAT." + value = google_compute_router_nat.nat.name +} + +output "nat_ip_allocate_option" { + description = "NAT IP allocation mode." + value = google_compute_router_nat.nat.nat_ip_allocate_option +} + +output "region" { + description = "Cloud NAT region." + value = google_compute_router_nat.nat.region +} + +output "router" { + description = "Cloud NAT router resources (if auto created)." + value = var.create_router ? google_compute_router.router[0] : null +} + +output "router_name" { + description = "Cloud NAT router name." + value = local.router +} diff --git a/modules/net-cloudnat/variables.tf b/modules/net-cloudnat/variables.tf new file mode 100644 index 0000000000..8c967b8e11 --- /dev/null +++ b/modules/net-cloudnat/variables.tf @@ -0,0 +1,111 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "addresses" { + description = "Optional list of external address self links." + type = list(string) + default = [] +} + +variable "config_min_ports_per_vm" { + description = "Minimum number of ports allocated to a VM from this NAT config." + type = number + default = 64 +} + +variable "config_source_subnets" { + description = "Subnetwork configuration, valid values are ALL_SUBNETWORKS_ALL_IP_RANGES, ALL_SUBNETWORKS_ALL_PRIMARY_IP_RANGES, LIST_OF_SUBNETWORKS." + type = string + default = "ALL_SUBNETWORKS_ALL_IP_RANGES" +} + +variable "config_timeouts" { + description = "Timeout configurations." + type = object({ + icmp = number + tcp_established = number + tcp_transitory = number + udp = number + }) + default = { + icmp = 30 + tcp_established = 1200 + tcp_transitory = 30 + udp = 30 + } +} + +variable "create_router" { + description = "Create router for Cloud NAT instead of using existing one." + type = bool + default = true +} + +variable "name" { + description = "Name of the Cloud NAT resource." + type = string + default = "cloud-nat" +} + +variable "network" { + description = "Name of the VPC where optional router will be created." + type = string + default = "" +} + +variable "prefix" { + description = "Optional prefix that will be prepended to resource names." + type = string + default = "" +} + +variable "project_id" { + description = "Project where resources will be created." + type = string +} + +variable "region" { + description = "Region where resources will be created." + type = string +} + +variable "router_name" { + description = "Name of the existing or auto-created router." + type = string + default = "cloud-nat" +} + +variable "router_asn" { + description = "Router ASN used for auto-created router." + type = number + default = 64514 +} + +variable "router_network" { + description = "Name of the VPC used for auto-created router." + type = string + default = "" +} + +variable "subnetworks" { + description = "Subnetworks to NAT, only used when config_source_subnets equals LIST_OF_SUBNETWORKS." + type = list(object({ + self_link = string, + config_source_ranges = list(string) + secondary_ranges = list(string) + })) + default = [] +} diff --git a/modules/net-cloudnat/versions.tf b/modules/net-cloudnat/versions.tf new file mode 100644 index 0000000000..832ec1df39 --- /dev/null +++ b/modules/net-cloudnat/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12" +} From 21a0d070925714a6f1f41db646bb1afcef3988fe Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Tue, 17 Dec 2019 07:41:17 +0100 Subject: [PATCH 036/106] remove redundant variable from net-cloudnat module --- modules/net-cloudnat/README.md | 1 - modules/net-cloudnat/variables.tf | 6 ------ 2 files changed, 7 deletions(-) diff --git a/modules/net-cloudnat/README.md b/modules/net-cloudnat/README.md index 0690992c7f..8548ab6351 100644 --- a/modules/net-cloudnat/README.md +++ b/modules/net-cloudnat/README.md @@ -13,7 +13,6 @@ | *config_timeouts* | Timeout configurations. | `object({...})` | | *create_router* | Create router for Cloud NAT instead of using existing one. | `bool` | | *name* | Name of the Cloud NAT resource. | `string` | -| *network* | Name of the VPC where optional router will be created. | `string` | | *prefix* | Optional prefix that will be prepended to resource names. | `string` | | *router_asn* | Router ASN used for auto-created router. | `number` | | *router_name* | Name of the existing or auto-created router. | `string` | diff --git a/modules/net-cloudnat/variables.tf b/modules/net-cloudnat/variables.tf index 8c967b8e11..452bc24cad 100644 --- a/modules/net-cloudnat/variables.tf +++ b/modules/net-cloudnat/variables.tf @@ -60,12 +60,6 @@ variable "name" { default = "cloud-nat" } -variable "network" { - description = "Name of the VPC where optional router will be created." - type = string - default = "" -} - variable "prefix" { description = "Optional prefix that will be prepended to resource names." type = string From c40edfa0576b58ece2f9e133d23e4f90b4b13804 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Tue, 17 Dec 2019 10:51:24 +0100 Subject: [PATCH 037/106] vpc module: add support for peering, use network name as subnet name prefix --- modules/net-vpc/main.tf | 24 +++++++++++++++++++++++- modules/net-vpc/variables.tf | 10 ++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/modules/net-vpc/main.tf b/modules/net-vpc/main.tf index d865113091..a01fdc3b35 100644 --- a/modules/net-vpc/main.tf +++ b/modules/net-vpc/main.tf @@ -35,6 +35,7 @@ locals { for pair in local.iam_pairs : "${pair.subnet}-${pair.role}" => pair } + peer_network = var.peering_config == null ? null : element(reverse(split("/", var.peering_config.peer_vpc_self_link)), 0) } resource "google_compute_network" "network" { @@ -45,6 +46,27 @@ resource "google_compute_network" "network" { routing_mode = var.routing_mode } +resource "google_compute_network_peering" "local" { + provider = google-beta + count = var.peering_config == null ? 0 : 1 + name = "${google_compute_network.network.name}-${local.peer_network}" + network = google_compute_network.network.self_link + peer_network = var.peering_config.peer_vpc_self_link + export_custom_routes = var.peering_config.export_routes + import_custom_routes = var.peering_config.import_routes +} + +resource "google_compute_network_peering" "remote" { + provider = google-beta + count = var.peering_config == null ? 0 : 1 + name = "${local.peer_network}-${google_compute_network.network.name}" + network = var.peering_config.peer_vpc_self_link + peer_network = google_compute_network.network.self_link + export_custom_routes = var.peering_config.import_routes + import_custom_routes = var.peering_config.export_routes + depends_on = [google_compute_network_peering.local] +} + resource "google_compute_shared_vpc_host_project" "shared_vpc_host" { count = var.shared_vpc_host ? 1 : 0 project = var.project_id @@ -63,7 +85,7 @@ resource "google_compute_subnetwork" "subnetwork" { project = var.project_id network = google_compute_network.network.name region = each.value.region - name = each.key + name = "${var.name}-${each.key}" ip_cidr_range = each.value.ip_cidr_range secondary_ip_range = [ for name, range in each.value.secondary_ip_range : diff --git a/modules/net-vpc/variables.tf b/modules/net-vpc/variables.tf index ce7be1f0a2..8863e9957c 100644 --- a/modules/net-vpc/variables.tf +++ b/modules/net-vpc/variables.tf @@ -63,6 +63,16 @@ variable "name" { type = string } +variable "peering_config" { + description = "VPC peering configuration." + type = object({ + peer_vpc_self_link = string + export_routes = bool + import_routes = bool + }) + default = null +} + variable "project_id" { description = "The ID of the project where this VPC will be created" type = string From b6a295f1b002df042337f37550f3893e71db02d7 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Tue, 17 Dec 2019 14:22:56 +0100 Subject: [PATCH 038/106] net-vpn-static module --- modules/net-vpn-static/main.tf | 100 ++++++++++++++++++++++++++++ modules/net-vpn-static/outputs.tf | 65 ++++++++++++++++++ modules/net-vpn-static/variables.tf | 73 ++++++++++++++++++++ 3 files changed, 238 insertions(+) create mode 100644 modules/net-vpn-static/main.tf create mode 100644 modules/net-vpn-static/outputs.tf create mode 100644 modules/net-vpn-static/variables.tf diff --git a/modules/net-vpn-static/main.tf b/modules/net-vpn-static/main.tf new file mode 100644 index 0000000000..a8197db079 --- /dev/null +++ b/modules/net-vpn-static/main.tf @@ -0,0 +1,100 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + gateway_address = ( + var.create_address + ? google_compute_address.gateway[0].address + : var.gateway_address + ) + route_pairs = { + for pair in setproduct(keys(var.tunnels), var.remote_ranges) : + "${pair[0]}-${join("-", regexall("[0-9]+", pair[1]))}" => { + tunnel = pair[0], range = pair[1] + } + } + secret = random_id.secret.b64_url +} + +resource "google_compute_address" "gateway" { + count = var.create_address ? 1 : 0 + name = "vpn-${var.name}" + project = var.project_id + region = var.region +} + +resource "google_compute_forwarding_rule" "esp" { + name = "vpn-${var.name}-esp" + project = var.project_id + region = var.region + target = google_compute_vpn_gateway.gateway.self_link + ip_address = local.gateway_address + ip_protocol = "ESP" +} + +resource "google_compute_forwarding_rule" "udp-500" { + name = "vpn-${var.name}-udp-500" + project = var.project_id + region = var.region + target = google_compute_vpn_gateway.gateway.self_link + ip_address = local.gateway_address + ip_protocol = "UDP" + port_range = "500" +} + +resource "google_compute_forwarding_rule" "udp-4500" { + name = "vpn-${var.name}-udp-4500" + project = var.project_id + region = var.region + target = google_compute_vpn_gateway.gateway.self_link + ip_address = local.gateway_address + ip_protocol = "UDP" + port_range = "4500" +} + +resource "google_compute_route" "route" { + for_each = local.route_pairs + name = "vpn-${each.key}" + project = var.project_id + network = var.network + dest_range = each.value.range + priority = var.route_priority + next_hop_vpn_tunnel = google_compute_vpn_tunnel.tunnels[each.value.tunnel].self_link +} + +resource "google_compute_vpn_gateway" "gateway" { + name = var.name + project = var.project_id + region = var.region + network = var.network +} + +resource "google_compute_vpn_tunnel" "tunnels" { + for_each = var.tunnels + name = "${var.name}-${each.key}" + project = var.project_id + region = var.region + peer_ip = each.value.peer_ip + local_traffic_selector = each.value.traffic_selectors.local + remote_traffic_selector = each.value.traffic_selectors.remote + ike_version = each.value.ike_version + shared_secret = each.value.shared_secret == "" ? local.secret : each.value.shared_secret + target_vpn_gateway = google_compute_vpn_gateway.gateway.self_link +} + +resource "random_id" "secret" { + byte_length = 8 +} diff --git a/modules/net-vpn-static/outputs.tf b/modules/net-vpn-static/outputs.tf new file mode 100644 index 0000000000..09ecc5ec0b --- /dev/null +++ b/modules/net-vpn-static/outputs.tf @@ -0,0 +1,65 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "address" { + description = "VPN gateway address." + value = local.gateway_address +} + +output "gateway" { + description = "VPN gateway resource." + value = google_compute_vpn_gateway.gateway +} + +output "name" { + description = "VPN gateway name." + value = google_compute_vpn_gateway.gateway.name +} + +output "self_link" { + description = "VPN gateway self link." + value = google_compute_vpn_gateway.gateway.self_link +} + +output "tunnels" { + description = "VPN tunnel resources." + value = { + for name in keys(var.tunnels) : + name => google_compute_vpn_tunnel.tunnels[name] + } +} + +output "tunnel_names" { + description = "VPN tunnel names." + value = { + for name in keys(var.tunnels) : + name => google_compute_vpn_tunnel.tunnels[name].name + } +} + +output "tunnel_self_links" { + description = "VPN tunnel self links." + value = { + for name in keys(var.tunnels) : + name => google_compute_vpn_tunnel.tunnels[name].self_link + } +} + +output "random_secret" { + description = "Generated secret." + sensitive = true + value = local.secret +} diff --git a/modules/net-vpn-static/variables.tf b/modules/net-vpn-static/variables.tf new file mode 100644 index 0000000000..ee7209df7c --- /dev/null +++ b/modules/net-vpn-static/variables.tf @@ -0,0 +1,73 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "create_address" { + description = "Create gateway address resource instead of using external one, defaults to true." + type = bool + default = true +} + +variable "gateway_address" { + description = "Optional address assigned to the VPN, used if create_address is false." + type = string + default = "" +} + +variable "name" { + description = "VPN gateway name, and prefix used for dependent resources." + type = string +} + +variable "network" { + description = "VPC used for the gateway and routes." + type = string +} + +variable "project_id" { + description = "Project where resources will be created." + type = string +} + +variable "region" { + description = "Region used for resources." + type = string +} + +variable "remote_ranges" { + description = "Remote IP CIDR ranges." + type = list(string) + default = [] +} + +variable "route_priority" { + description = "Route priority, defaults to 1000." + type = number + default = 1000 +} + +variable "tunnels" { + description = "VPN tunnel configurations." + type = map(object({ + ike_version = number + peer_ip = string + shared_secret = string + traffic_selectors = object({ + local = list(string) + remote = list(string) + }) + })) + default = {} +} From 3890691bcaae14b8c099a9784e5416a5eca85bd7 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Tue, 17 Dec 2019 14:23:57 +0100 Subject: [PATCH 039/106] net-vpn-static module README --- modules/net-vpn-static/README.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 modules/net-vpn-static/README.md diff --git a/modules/net-vpn-static/README.md b/modules/net-vpn-static/README.md new file mode 100644 index 0000000000..602d8cae02 --- /dev/null +++ b/modules/net-vpn-static/README.md @@ -0,0 +1,4 @@ +# Cloud VPN Route-based Module + + + From 5d713828e700a8bc941ef82173e61cead440b21f Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Tue, 17 Dec 2019 14:24:35 +0100 Subject: [PATCH 040/106] net-vpn-static module README --- modules/net-vpn-static/README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/modules/net-vpn-static/README.md b/modules/net-vpn-static/README.md index 602d8cae02..9fa038b383 100644 --- a/modules/net-vpn-static/README.md +++ b/modules/net-vpn-static/README.md @@ -1,4 +1,30 @@ # Cloud VPN Route-based Module +## Variables + +| name | description | type | required | +|---|---|:---: |:---:| +| name | VPN gateway name, and prefix used for dependent resources. | `string` | ✓ +| network | VPC used for the gateway and routes. | `string` | ✓ +| project_id | Project where resources will be created. | `string` | ✓ +| region | Region used for resources. | `string` | ✓ +| *create_address* | Create gateway address resource instead of using external one, defaults to true. | `bool` | +| *gateway_address* | Optional address assigned to the VPN, used if create_address is false. | `string` | +| *remote_ranges* | Remote IP CIDR ranges. | `list(string)` | +| *route_priority* | Route priority, defaults to 1000. | `number` | +| *tunnels* | VPN tunnel configurations. | `map(object({...}))` | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| address | VPN gateway address. | | +| gateway | VPN gateway resource. | | +| name | VPN gateway name. | | +| random_secret | Generated secret. | ✓ | +| self_link | VPN gateway self link. | | +| tunnel_names | VPN tunnel names. | | +| tunnel_self_links | VPN tunnel self links. | | +| tunnels | VPN tunnel resources. | | From 5340bbdb8c4152bb147b34df744eff29e8982118 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Tue, 17 Dec 2019 15:29:10 +0100 Subject: [PATCH 041/106] tfdoc: fix error on undeclared variable type --- tools/tfdoc/tfdoc.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/tfdoc/tfdoc.py b/tools/tfdoc/tfdoc.py index af38b1de4f..d6bf3c0015 100755 --- a/tools/tfdoc/tfdoc.py +++ b/tools/tfdoc/tfdoc.py @@ -137,6 +137,8 @@ def format_outputs(outputs): def format_type(type_spec): "Format variable type." + if not type_spec: + return '' buffer = [] stack = [] for t in RE_TYPE.split(type_spec.split("\n")[0]): From 5cb2f4db9016edb5ad1a02d68df5be86b5fe18f7 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Tue, 17 Dec 2019 15:29:44 +0100 Subject: [PATCH 042/106] dns module --- modules/dns/README.md | 76 +++++++++++++++++++++++++ modules/dns/main.tf | 117 +++++++++++++++++++++++++++++++++++++++ modules/dns/outputs.tf | 40 +++++++++++++ modules/dns/variables.tf | 96 ++++++++++++++++++++++++++++++++ modules/dns/versions.tf | 19 +++++++ 5 files changed, 348 insertions(+) create mode 100644 modules/dns/README.md create mode 100644 modules/dns/main.tf create mode 100644 modules/dns/outputs.tf create mode 100644 modules/dns/variables.tf create mode 100644 modules/dns/versions.tf diff --git a/modules/dns/README.md b/modules/dns/README.md new file mode 100644 index 0000000000..a1bdb43bc0 --- /dev/null +++ b/modules/dns/README.md @@ -0,0 +1,76 @@ +# Terraform Google Cloud DNS Module + +This module makes it easy to create Google Cloud DNS zones of different types, and manage their records. It supports creating public, private, forwarding, and peering zones. + +The resources/services/activations/deletions that this module will create/trigger are: + +- One `google_dns_managed_zone` for the zone +- Zero or more `google_dns_record_set` for the zone records + +## Usage + +Basic usage of this module for a private zone is as follows: + +```hcl +module "dns-private-zone" { + source = "./modules/dns + project_id = "my-project" + type = "private" + name = "example-com" + domain = "example.com." + client_networks = [var.vpc_self_link] + recordsets = [ + {name = "", type = "NS", ttl = 300, records = ["127.0.0.1"]}, + {name = "localhost", type = "A", ttl = 300, records = ["127.0.0.1"]}, + ] +} + +``` + + +## Variables + +| name | description | type | required | +|---|---|:---: |:---:| +| domain | Zone domain, must end with a period. | `string` | ✓ +| name | Zone name, must be unique within the project. | `string` | ✓ +| project_id | Project id for the zone. | `string` | ✓ +| *client_networks* | List of VPC self links that can see this zone. | `list(string)` | +| *default_key_specs_key* | DNSSEC default key signing specifications: algorithm, key_length, key_type, kind. | `any` | +| *default_key_specs_zone* | DNSSEC default zone signing specifications: algorithm, key_length, key_type, kind. | `any` | +| *description* | Domain description. | `string` | +| *dnssec_config* | DNSSEC configuration: kind, non_existence, state. | `any` | +| *forwarders* | List of target name servers, only valid for 'forwarding' zone types. | `list(string)` | +| *peer_network* | Peering network self link, only valid for 'peering' zone types. | `string` | +| *recordsets* | List of DNS record objects to manage. | `string` | +| *type* | Type of zone to create, valid values are 'public', 'private', 'forwarding', 'peering'. | `string` | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| domain | The DNS zone domain. | | +| name | The DNS zone name. | | +| name_servers | The DNS zone name servers. | | +| type | The DNS zone type. | | +| zone | DNS zone resource. | | + + +## Requirements + +### IAM Roles + +The following roles must be used to provision the resources in this module: + +- Storage Admin: `roles/dns.admin` + +### APIs + +A project with the following APIs enabled must be used to host the +resources of this module: + +- Google Cloud DNS API: `dns.googleapis.com` + +## DNSSEC + +For DNSSEC configuration, refer to the [`dns_managed_zone` documentation](https://www.terraform.io/docs/providers/google/r/dns_managed_zone.html#dnssec_config). \ No newline at end of file diff --git a/modules/dns/main.tf b/modules/dns/main.tf new file mode 100644 index 0000000000..1f332ff3ac --- /dev/null +++ b/modules/dns/main.tf @@ -0,0 +1,117 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + is_static_zone = var.type == "public" || var.type == "private" + recordsets = { + for record in var.recordsets : + join("/", [record.name, record.type]) => record + } + zone = element(concat( + google_dns_managed_zone.non-public, google_dns_managed_zone.public + ), 0) +} + +resource "google_dns_managed_zone" "non-public" { + count = var.type != "public" ? 1 : 0 + provider = google-beta + project = var.project_id + name = var.name + dns_name = var.domain + description = "Terraform-managed zone." + visibility = "private" + + dynamic forwarding_config { + for_each = var.type == "forwarding" ? { config = var.forwarders } : {} + iterator = config + content { + dynamic "target_name_servers" { + for_each = config.value + iterator = address + content { + ipv4_address = address.value + } + } + } + } + + dynamic peering_config { + for_each = var.type == "peering" ? { config = var.peer_network } : {} + iterator = config + content { + target_network { + network_url = config.value + } + } + } + + private_visibility_config { + dynamic "networks" { + for_each = var.client_networks + iterator = network + content { + network_url = network.value + } + } + } + +} + +resource "google_dns_managed_zone" "public" { + count = var.type == "public" ? 1 : 0 + project = var.project_id + name = var.name + dns_name = var.domain + description = var.description + visibility = "public" + + dynamic "dnssec_config" { + for_each = var.dnssec_config == {} ? [] : list(var.dnssec_config) + iterator = config + content { + kind = lookup(config, "kind", "dns#managedZoneDnsSecConfig") + non_existence = lookup(config, "non_existence", "nsec3") + state = lookup(config, "state", "off") + + default_key_specs { + algorithm = lookup(var.default_key_specs_key, "algorithm", "rsasha256") + key_length = lookup(var.default_key_specs_key, "key_length", 2048) + key_type = lookup(var.default_key_specs_key, "key_type", "keySigning") + kind = lookup(var.default_key_specs_key, "kind", "dns#dnsKeySpec") + } + default_key_specs { + algorithm = lookup(var.default_key_specs_zone, "algorithm", "rsasha256") + key_length = lookup(var.default_key_specs_zone, "key_length", 1024) + key_type = lookup(var.default_key_specs_zone, "key_type", "zoneSigning") + kind = lookup(var.default_key_specs_zone, "kind", "dns#dnsKeySpec") + } + } + } + +} + +resource "google_dns_record_set" "cloud-static-records" { + for_each = local.recordsets + project = var.project_id + managed_zone = var.name + name = each.value.name != "" ? "${each.value.name}.${var.domain}" : var.domain + type = each.value.type + ttl = each.value.ttl + rrdatas = each.value.records + depends_on = [ + google_dns_managed_zone.non-public, google_dns_managed_zone.public + ] +} diff --git a/modules/dns/outputs.tf b/modules/dns/outputs.tf new file mode 100644 index 0000000000..df35628a21 --- /dev/null +++ b/modules/dns/outputs.tf @@ -0,0 +1,40 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "type" { + description = "The DNS zone type." + value = var.type +} + +output "zone" { + description = "DNS zone resource." + value = local.zone +} + +output "name" { + description = "The DNS zone name." + value = local.zone.name +} + +output "domain" { + description = "The DNS zone domain." + value = local.zone.dns_name +} + +output "name_servers" { + description = "The DNS zone name servers." + value = local.zone.name_servers +} diff --git a/modules/dns/variables.tf b/modules/dns/variables.tf new file mode 100644 index 0000000000..c15e81b372 --- /dev/null +++ b/modules/dns/variables.tf @@ -0,0 +1,96 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +############################################################################### +# zone variables # +############################################################################### + +variable "client_networks" { + description = "List of VPC self links that can see this zone." + type = list(string) + default = [] +} + +variable "description" { + description = "Domain description." + type = string + default = "Terraform managed." +} + +# TODO(ludoo): add link to DNSSEC documentation in README +# https://www.terraform.io/docs/providers/google/r/dns_managed_zone.html#dnssec_config + +variable "default_key_specs_key" { + description = "DNSSEC default key signing specifications: algorithm, key_length, key_type, kind." + type = any + default = {} +} + +variable "default_key_specs_zone" { + description = "DNSSEC default zone signing specifications: algorithm, key_length, key_type, kind." + type = any + default = {} +} + +variable "dnssec_config" { + description = "DNSSEC configuration: kind, non_existence, state." + type = any + default = {} +} + +variable "domain" { + description = "Zone domain, must end with a period." + type = string +} + +variable "forwarders" { + description = "List of target name servers, only valid for 'forwarding' zone types." + type = list(string) + default = [] +} + +variable "name" { + description = "Zone name, must be unique within the project." + type = string +} + +variable "peer_network" { + description = "Peering network self link, only valid for 'peering' zone types." + type = string + default = "" +} + +variable "project_id" { + description = "Project id for the zone." + type = string +} + +variable "recordsets" { + type = list(object({ + name = string + type = string + ttl = number + records = list(string) + })) + description = "List of DNS record objects to manage." + default = [] +} + +variable "type" { + description = "Type of zone to create, valid values are 'public', 'private', 'forwarding', 'peering'." + type = string + default = "private" +} diff --git a/modules/dns/versions.tf b/modules/dns/versions.tf new file mode 100644 index 0000000000..ce6918e09d --- /dev/null +++ b/modules/dns/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12.6" +} From 9760af26185f73327525b3851f17e24270918de9 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Tue, 17 Dec 2019 15:29:57 +0100 Subject: [PATCH 043/106] set version for all modules --- modules/folder/versions.tf | 4 ++-- modules/gcs/versions.tf | 2 +- modules/gke-cluster/versions.tf | 19 +++++++++++++++++++ modules/net-address/versions.tf | 19 +++++++++++++++++++ modules/net-cloudnat/versions.tf | 4 ++-- modules/net-vpc-firewall/versions.tf | 5 +---- modules/net-vpc/versions.tf | 2 +- modules/net-vpn-static/versions.tf | 19 +++++++++++++++++++ modules/not-ready/versions.tf | 19 +++++++++++++++++++ modules/project/versions.tf | 19 +++++++++++++++++++ 10 files changed, 102 insertions(+), 10 deletions(-) create mode 100644 modules/gke-cluster/versions.tf create mode 100644 modules/net-address/versions.tf create mode 100644 modules/net-vpn-static/versions.tf create mode 100644 modules/not-ready/versions.tf create mode 100644 modules/project/versions.tf diff --git a/modules/folder/versions.tf b/modules/folder/versions.tf index 832ec1df39..ce6918e09d 100644 --- a/modules/folder/versions.tf +++ b/modules/folder/versions.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2019 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,5 +15,5 @@ */ terraform { - required_version = ">= 0.12" + required_version = ">= 0.12.6" } diff --git a/modules/gcs/versions.tf b/modules/gcs/versions.tf index 84432a3b20..ce6918e09d 100644 --- a/modules/gcs/versions.tf +++ b/modules/gcs/versions.tf @@ -15,5 +15,5 @@ */ terraform { - required_version = ">= 0.12.9" + required_version = ">= 0.12.6" } diff --git a/modules/gke-cluster/versions.tf b/modules/gke-cluster/versions.tf new file mode 100644 index 0000000000..ce6918e09d --- /dev/null +++ b/modules/gke-cluster/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12.6" +} diff --git a/modules/net-address/versions.tf b/modules/net-address/versions.tf new file mode 100644 index 0000000000..ce6918e09d --- /dev/null +++ b/modules/net-address/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12.6" +} diff --git a/modules/net-cloudnat/versions.tf b/modules/net-cloudnat/versions.tf index 832ec1df39..ce6918e09d 100644 --- a/modules/net-cloudnat/versions.tf +++ b/modules/net-cloudnat/versions.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2019 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,5 +15,5 @@ */ terraform { - required_version = ">= 0.12" + required_version = ">= 0.12.6" } diff --git a/modules/net-vpc-firewall/versions.tf b/modules/net-vpc-firewall/versions.tf index 46ce80b846..ce6918e09d 100644 --- a/modules/net-vpc-firewall/versions.tf +++ b/modules/net-vpc-firewall/versions.tf @@ -15,8 +15,5 @@ */ terraform { - required_version = "~> 0.12.6" - required_providers { - google = "~> 2.19" - } + required_version = ">= 0.12.6" } diff --git a/modules/net-vpc/versions.tf b/modules/net-vpc/versions.tf index 1fe4caaac6..ce6918e09d 100644 --- a/modules/net-vpc/versions.tf +++ b/modules/net-vpc/versions.tf @@ -15,5 +15,5 @@ */ terraform { - required_version = "~> 0.12.0" + required_version = ">= 0.12.6" } diff --git a/modules/net-vpn-static/versions.tf b/modules/net-vpn-static/versions.tf new file mode 100644 index 0000000000..ce6918e09d --- /dev/null +++ b/modules/net-vpn-static/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12.6" +} diff --git a/modules/not-ready/versions.tf b/modules/not-ready/versions.tf new file mode 100644 index 0000000000..ce6918e09d --- /dev/null +++ b/modules/not-ready/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12.6" +} diff --git a/modules/project/versions.tf b/modules/project/versions.tf new file mode 100644 index 0000000000..ce6918e09d --- /dev/null +++ b/modules/project/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12.6" +} From 438cfd7d56acd23805cc250b7a28c88578107101 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Tue, 17 Dec 2019 22:14:43 +0100 Subject: [PATCH 044/106] kms module (untested) --- modules/kms/README.md | 52 +++++++++++++++++++++++++++++ modules/kms/main.tf | 60 ++++++++++++++++++++++++++++++++++ modules/kms/outputs.tf | 40 +++++++++++++++++++++++ modules/kms/variables.tf | 70 ++++++++++++++++++++++++++++++++++++++++ modules/kms/versions.tf | 19 +++++++++++ 5 files changed, 241 insertions(+) create mode 100644 modules/kms/README.md create mode 100644 modules/kms/main.tf create mode 100644 modules/kms/outputs.tf create mode 100644 modules/kms/variables.tf create mode 100644 modules/kms/versions.tf diff --git a/modules/kms/README.md b/modules/kms/README.md new file mode 100644 index 0000000000..353f82bedf --- /dev/null +++ b/modules/kms/README.md @@ -0,0 +1,52 @@ +# Google KMS Module + +Simple Cloud KMS module that allows managing a keyring, zero or more keys in the keyring, and IAM role bindings on individual keys. + +The resources/services/activations/deletions that this module will create/trigger are: + +- Create a KMS keyring in the provided project +- Create zero or more keys in the keyring +- Create IAM role bindings for owners, encrypters, decrypters + + +## Variables + +| name | description | type | required | +|---|---|:---: |:---:| +| keyring | Keyring name. | `string` | ✓ +| location | Location for the keyring. | `string` | ✓ +| project_id | Project id where the keyring will be created. | `string` | ✓ +| *iam_members* | List of IAM members keyed by name and role. | `map(map(list(string)))` | +| *iam_roles* | List of IAM roles keyed by name. | `map(list(string))` | +| *key_attributes* | Optional key attributes per key. | `map(object({...}))` | +| *key_defaults* | Key attribute defaults. | `object({...})` | +| *keys* | Key names. | `list(string)` | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| key_self_links | Key self links. | | +| keyring | Keyring resource. | | +| keys | Key resources. | | +| name | Keyring self link. | | +| self_link | Keyring self link. | | + + +## Requirements + +These sections describe requirements for using this module. + +### IAM + +The following roles must be used to provision the resources of this module: + +- Cloud KMS Admin: `roles/cloudkms.admin` or +- Owner: `roles/owner` + +### APIs + +A project with the following APIs enabled must be used to host the +resources of this module: + +- Google Cloud Key Management Service: `cloudkms.googleapis.com` diff --git a/modules/kms/main.tf b/modules/kms/main.tf new file mode 100644 index 0000000000..26f5b8b8b3 --- /dev/null +++ b/modules/kms/main.tf @@ -0,0 +1,60 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # distinct is needed to make the expanding function argument work + iam_pairs = concat([], distinct([ + for name, roles in var.iam_roles : + [for role in roles : { name = name, role = role }] + ])...) + iam_keypairs = { + for pair in local.iam_pairs : + "${pair.name}-${pair.role}" => pair + } + key_attributes = { + for name in var.keys : + name => merge(lookup(var.key_attributes, name, {}), var.key_defaults) + } +} + +resource "google_kms_key_ring" "key_ring" { + name = var.keyring + project = var.project_id + location = var.location +} + +resource "google_kms_crypto_key" "keys" { + for_each = toset(var.keys) + name = each.value + key_ring = google_kms_key_ring.key_ring.self_link + rotation_period = local.key_attributes[each.value].rotation_period + + dynamic lifecycle { + for_each = local.key_attributes[each.value].protected ? [""] : [] + content { + prevent_destroy = true + } + } +} + +resource "google_kms_crypto_key_iam_binding" "bindings" { + for_each = local.iam_keypairs + role = each.value.role + crypto_key_id = google_kms_crypto_key.keys[each.value.name].self_link + members = lookup( + lookup(var.iam_members, each.value.name, {}), each.value.role, [] + ) +} diff --git a/modules/kms/outputs.tf b/modules/kms/outputs.tf new file mode 100644 index 0000000000..cb8d2dba8d --- /dev/null +++ b/modules/kms/outputs.tf @@ -0,0 +1,40 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "keyring" { + description = "Keyring resource." + value = google_kms_key_ring.key_ring +} + +output "name" { + description = "Keyring self link." + value = google_kms_key_ring.key_ring.self_link +} + +output "self_link" { + description = "Keyring self link." + value = google_kms_key_ring.key_ring.self_link +} + +output "keys" { + description = "Key resources." + value = [for name in var.keys : google_kms_crypto_key.keys[name]] +} + +output "key_self_links" { + description = "Key self links." + value = [for name in var.keys : google_kms_crypto_key.keys[name].self_link] +} diff --git a/modules/kms/variables.tf b/modules/kms/variables.tf new file mode 100644 index 0000000000..aad52c58f5 --- /dev/null +++ b/modules/kms/variables.tf @@ -0,0 +1,70 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "iam_members" { + description = "List of IAM members keyed by name and role." + type = map(map(list(string))) + default = {} +} + +variable "iam_roles" { + description = "List of IAM roles keyed by name." + type = map(list(string)) + default = {} +} + +variable "keyring" { + description = "Keyring name." + type = string +} + +variable "key_attributes" { + description = "Optional key attributes per key." + type = map(object({ + protected = bool + rotation_period = string + })) + default = {} +} + +variable "key_defaults" { + description = "Key attribute defaults." + type = object({ + protected = bool + rotation_period = string + }) + default = { + protected = true + rotation_period = "100000s" + } +} + +variable "keys" { + description = "Key names." + type = list(string) + default = [] +} + +# cf https://cloud.google.com/kms/docs/locations +variable "location" { + description = "Location for the keyring." + type = string +} + +variable "project_id" { + description = "Project id where the keyring will be created." + type = string +} diff --git a/modules/kms/versions.tf b/modules/kms/versions.tf new file mode 100644 index 0000000000..ce6918e09d --- /dev/null +++ b/modules/kms/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12.6" +} From dfcdc2607d5c24f58dc34410c935bc7e5bfd1ad7 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Tue, 17 Dec 2019 22:20:40 +0100 Subject: [PATCH 045/106] change kms key self links output to map, fix gcs and kms iam variable descriptions --- modules/gcs/variables.tf | 4 ++-- modules/kms/outputs.tf | 5 ++++- modules/kms/variables.tf | 4 ++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/modules/gcs/variables.tf b/modules/gcs/variables.tf index d6a72eae54..9d7eb60e88 100644 --- a/modules/gcs/variables.tf +++ b/modules/gcs/variables.tf @@ -27,13 +27,13 @@ variable "force_destroy" { } variable "iam_members" { - description = "List of IAM members keyed by name and role." + description = "IAM members keyed by bucket name and role." type = map(map(list(string))) default = {} } variable "iam_roles" { - description = "List of IAM roles keyed by name." + description = "IAM roles keyed by bucket name." type = map(list(string)) default = {} } diff --git a/modules/kms/outputs.tf b/modules/kms/outputs.tf index cb8d2dba8d..00d772bb2a 100644 --- a/modules/kms/outputs.tf +++ b/modules/kms/outputs.tf @@ -36,5 +36,8 @@ output "keys" { output "key_self_links" { description = "Key self links." - value = [for name in var.keys : google_kms_crypto_key.keys[name].self_link] + value = { + for name in var.keys : + name => google_kms_crypto_key.keys[name].self_link + } } diff --git a/modules/kms/variables.tf b/modules/kms/variables.tf index aad52c58f5..e976b72b21 100644 --- a/modules/kms/variables.tf +++ b/modules/kms/variables.tf @@ -15,13 +15,13 @@ */ variable "iam_members" { - description = "List of IAM members keyed by name and role." + description = "IAM members keyed by key name and role." type = map(map(list(string))) default = {} } variable "iam_roles" { - description = "List of IAM roles keyed by name." + description = "IAM roles keyed by key name." type = map(list(string)) default = {} } From 172cf9fe83ea135466639946ceeb5a9065920dcb Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Wed, 18 Dec 2019 07:46:56 +0100 Subject: [PATCH 046/106] fix kms module --- modules/kms/main.tf | 33 +++++++++++++++++++++++---------- modules/kms/outputs.tf | 14 ++++++++------ 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/modules/kms/main.tf b/modules/kms/main.tf index 26f5b8b8b3..8e76805163 100644 --- a/modules/kms/main.tf +++ b/modules/kms/main.tf @@ -26,8 +26,12 @@ locals { } key_attributes = { for name in var.keys : - name => merge(lookup(var.key_attributes, name, {}), var.key_defaults) + name => lookup(var.key_attributes, name, var.key_defaults) } + keys = merge( + { for name, resource in google_kms_crypto_key.keys : name => resource }, + { for name, resource in google_kms_crypto_key.keys-ephemeral : name => resource } + ) } resource "google_kms_key_ring" "key_ring" { @@ -37,23 +41,32 @@ resource "google_kms_key_ring" "key_ring" { } resource "google_kms_crypto_key" "keys" { - for_each = toset(var.keys) - name = each.value + for_each = { + for name, attrs in local.key_attributes : + name => attrs if attrs.protected + } + name = each.key key_ring = google_kms_key_ring.key_ring.self_link - rotation_period = local.key_attributes[each.value].rotation_period + rotation_period = each.value.rotation_period + lifecycle { + prevent_destroy = true + } +} - dynamic lifecycle { - for_each = local.key_attributes[each.value].protected ? [""] : [] - content { - prevent_destroy = true - } +resource "google_kms_crypto_key" "keys-ephemeral" { + for_each = { + for name, attrs in local.key_attributes : + name => attrs if ! attrs.protected } + name = each.value + key_ring = google_kms_key_ring.key_ring.self_link + rotation_period = each.value.rotation_period } resource "google_kms_crypto_key_iam_binding" "bindings" { for_each = local.iam_keypairs role = each.value.role - crypto_key_id = google_kms_crypto_key.keys[each.value.name].self_link + crypto_key_id = local.keys[each.value.name].self_link members = lookup( lookup(var.iam_members, each.value.name, {}), each.value.role, [] ) diff --git a/modules/kms/outputs.tf b/modules/kms/outputs.tf index 00d772bb2a..de30ea172c 100644 --- a/modules/kms/outputs.tf +++ b/modules/kms/outputs.tf @@ -19,9 +19,14 @@ output "keyring" { value = google_kms_key_ring.key_ring } +output "location" { + description = "Keyring self link." + value = google_kms_key_ring.key_ring.location +} + output "name" { description = "Keyring self link." - value = google_kms_key_ring.key_ring.self_link + value = google_kms_key_ring.key_ring.name } output "self_link" { @@ -31,13 +36,10 @@ output "self_link" { output "keys" { description = "Key resources." - value = [for name in var.keys : google_kms_crypto_key.keys[name]] + value = local.keys } output "key_self_links" { description = "Key self links." - value = { - for name in var.keys : - name => google_kms_crypto_key.keys[name].self_link - } + value = { for name, resource in local.keys : name => resource.self_link } } From c3e6001b4cb5daa29b3725b205eb21beb866d7d7 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Wed, 18 Dec 2019 07:54:34 +0100 Subject: [PATCH 047/106] update kms module readme --- modules/kms/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/kms/README.md b/modules/kms/README.md index 353f82bedf..46a8807e04 100644 --- a/modules/kms/README.md +++ b/modules/kms/README.md @@ -16,8 +16,8 @@ The resources/services/activations/deletions that this module will create/trigge | keyring | Keyring name. | `string` | ✓ | location | Location for the keyring. | `string` | ✓ | project_id | Project id where the keyring will be created. | `string` | ✓ -| *iam_members* | List of IAM members keyed by name and role. | `map(map(list(string)))` | -| *iam_roles* | List of IAM roles keyed by name. | `map(list(string))` | +| *iam_members* | IAM members keyed by key name and role. | `map(map(list(string)))` | +| *iam_roles* | IAM roles keyed by key name. | `map(list(string))` | | *key_attributes* | Optional key attributes per key. | `map(object({...}))` | | *key_defaults* | Key attribute defaults. | `object({...})` | | *keys* | Key names. | `list(string)` | @@ -29,6 +29,7 @@ The resources/services/activations/deletions that this module will create/trigge | key_self_links | Key self links. | | | keyring | Keyring resource. | | | keys | Key resources. | | +| location | Keyring self link. | | | name | Keyring self link. | | | self_link | Keyring self link. | | From cbc276ecf2f8a7728f2e70a82bbce2fc8870d8a3 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Wed, 18 Dec 2019 08:24:45 +0100 Subject: [PATCH 048/106] simplify local iam pairs in modules --- modules/folder/main.tf | 5 ++--- modules/gcs/main.tf | 5 ++--- modules/kms/main.tf | 4 ++-- modules/net-vpc/main.tf | 5 ++--- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/modules/folder/main.tf b/modules/folder/main.tf index e988681d88..920f712d29 100644 --- a/modules/folder/main.tf +++ b/modules/folder/main.tf @@ -16,11 +16,10 @@ locals { folders = [for name in var.names : google_folder.folders[name]] - # distinct is needed to make the expanding function argument work - iam_pairs = concat([], distinct([ + iam_pairs = flatten([ for name, roles in var.iam_roles : [for role in roles : { name = name, role = role }] - ])...) + ]) iam_keypairs = { for pair in local.iam_pairs : "${pair.name}-${pair.role}" => pair diff --git a/modules/gcs/main.tf b/modules/gcs/main.tf index 05067e6c4e..91839332ad 100644 --- a/modules/gcs/main.tf +++ b/modules/gcs/main.tf @@ -16,11 +16,10 @@ locals { buckets = [for name in var.names : google_storage_bucket.buckets[name]] - # distinct is needed to make the expanding function argument work - iam_pairs = concat([], distinct([ + iam_pairs = flatten([ for name, roles in var.iam_roles : [for role in roles : { name = name, role = role }] - ])...) + ]) iam_keypairs = { for pair in local.iam_pairs : "${pair.name}-${pair.role}" => pair diff --git a/modules/kms/main.tf b/modules/kms/main.tf index 8e76805163..fe85f5d200 100644 --- a/modules/kms/main.tf +++ b/modules/kms/main.tf @@ -16,10 +16,10 @@ locals { # distinct is needed to make the expanding function argument work - iam_pairs = concat([], distinct([ + iam_pairs = flatten([ for name, roles in var.iam_roles : [for role in roles : { name = name, role = role }] - ])...) + ]) iam_keypairs = { for pair in local.iam_pairs : "${pair.name}-${pair.role}" => pair diff --git a/modules/net-vpc/main.tf b/modules/net-vpc/main.tf index a01fdc3b35..0d680ab140 100644 --- a/modules/net-vpc/main.tf +++ b/modules/net-vpc/main.tf @@ -26,11 +26,10 @@ locals { : [] ) } - # distinct is needed to make the expanding function argument work - iam_pairs = concat([], distinct([ + iam_pairs = flatten([ for subnet, roles in var.iam_roles : [for role in roles : { subnet = subnet, role = role }] - ])...) + ]) iam_keypairs = { for pair in local.iam_pairs : "${pair.subnet}-${pair.role}" => pair From 9ac87b9bf28edd5a457762e8c20771772c16628f Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Wed, 18 Dec 2019 08:44:02 +0100 Subject: [PATCH 049/106] service accounts module (unfinished) --- modules/iam-service-accounts/README.md | 99 ++++++++++++++++++++ modules/iam-service-accounts/main.tf | 109 ++++++++++++++++++++++ modules/iam-service-accounts/outputs.tf | 78 ++++++++++++++++ modules/iam-service-accounts/variables.tf | 62 ++++++++++++ modules/iam-service-accounts/versions.tf | 19 ++++ 5 files changed, 367 insertions(+) create mode 100644 modules/iam-service-accounts/README.md create mode 100644 modules/iam-service-accounts/main.tf create mode 100644 modules/iam-service-accounts/outputs.tf create mode 100644 modules/iam-service-accounts/variables.tf create mode 100644 modules/iam-service-accounts/versions.tf diff --git a/modules/iam-service-accounts/README.md b/modules/iam-service-accounts/README.md new file mode 100644 index 0000000000..4735858aaf --- /dev/null +++ b/modules/iam-service-accounts/README.md @@ -0,0 +1,99 @@ +# Terraform Service Accounts Module + +This module allows easy creation of one or more service accounts, and granting them basic roles. + +The resources/services/activations/deletions that this module will create/trigger are: + +- one or more service accounts +- optional project-level IAM role bindings for each service account +- one optional billing IAM role binding per service account, at the organization or billing account level +- two optional organization-level IAM bindings per service account, to enable the service accounts to create and manage Shared VPC networks +- one optional service account key per service account + +## Compatibility + + This module is meant for use with Terraform 0.12. If you haven't [upgraded](https://www.terraform.io/upgrade-guides/0-12.html) + and need a Terraform 0.11.x-compatible version of this module, the last released version intended for + Terraform 0.11.x is [0.1.1](https://registry.terraform.io/modules/terraform-google-modules/service-accounts/google/0.1.1). + +## Usage + +Basic usage of this module is as follows: + +```hcl +module "service_accounts" { + source = "terraform-google-modules/service-accounts/google" + version = "~> 2.0" + project_id = "" + prefix = "test-sa" + names = ["first", "second"] + project_roles = [ + "project-foo=>roles/viewer", + "project-spam=>roles/storage.objectViewer", + ] +} +``` + +Functional examples are included in the +[examples](./examples/) directory. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| billing\_account\_id | If assigning billing role, specificy a billing account (default is to assign at the organizational level). | string | `""` | no | +| generate\_keys | Generate keys for service accounts. | bool | `"false"` | no | +| grant\_billing\_role | Grant billing user role. | bool | `"false"` | no | +| grant\_xpn\_roles | Grant roles for shared VPC management. | bool | `"true"` | no | +| names | Names of the service accounts to create. | list(string) | `` | no | +| org\_id | Id of the organization for org-level roles. | string | `""` | no | +| prefix | Prefix applied to service account names. | string | `""` | no | +| project\_id | Project id where service account will be created. | string | n/a | yes | +| project\_roles | Common roles to apply to all service accounts, project=>role as elements. | list(string) | `` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| email | Service account email (for single use). | +| emails | Service account emails. | +| emails\_list | Service account emails. | +| iam\_email | IAM-format service account email (for single use). | +| iam\_emails | IAM-format service account emails. | +| iam\_emails\_list | IAM-format service account emails. | +| key | Service account key (for single use). | +| keys | Map of service account keys. | +| service\_account | Service account resource (for single use). | +| service\_accounts | Service account resources. | + + + +## Requirements + +These sections describe requirements for using this module. + +### Software + +The following dependencies must be available: + +- [Terraform][terraform] v0.12 +- [Terraform Provider for GCP][terraform-provider-gcp] plugin >= v2.0 + +### IAM + +Service account or user credentials with the following roles must be used to provision the resources of this module: + +- Service Account Admin: `roles/iam.serviceAccountAdmin` +- (optional) Service Account Key Admin: `roles/iam.serviceAccountAdmin` when `generate_keys` is set to `true` +- (optional) roles needed to grant optional IAM roles at the project or organizational level + +## Contributing + +Refer to the [contribution guidelines](./CONTRIBUTING.md) for +information on contributing to this module. + +[iam-module]: https://registry.terraform.io/modules/terraform-google-modules/iam/google +[project-factory-module]: https://registry.terraform.io/modules/terraform-google-modules/project-factory/google +[terraform-provider-gcp]: https://www.terraform.io/docs/providers/google/index.html +[terraform]: https://www.terraform.io/downloads.html diff --git a/modules/iam-service-accounts/main.tf b/modules/iam-service-accounts/main.tf new file mode 100644 index 0000000000..e4155c5bdd --- /dev/null +++ b/modules/iam-service-accounts/main.tf @@ -0,0 +1,109 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + iam_billing_pairs = flatten([ + for entity, roles in var.iam_billing_roles : [ + for role in roles : [ + for name in var.names : { entity = entity, role = role, name = name } + ] + ] + ]) + iam_folder_pairs = flatten([ + for entity, roles in var.iam_folder_roles : [ + for role in roles : [ + for name in var.names : { entity = entity, role = role, name = name } + ] + ] + ]) + iam_organization_pairs = flatten([ + for entity, roles in var.iam_organization_roles : [ + for role in roles : [ + for name in var.names : { entity = entity, role = role, name = name } + ] + ] + ]) + iam_project_pairs = flatten([ + for entity, roles in var.iam_project_roles : [ + for role in roles : [ + for name in var.names : { entity = entity, role = role, name = name } + ] + ] + ]) + prefix = var.prefix != "" ? "${var.prefix}-" : "" + resource = local.resources[var.names[0]] + resource_iam_emails = { + for name, resource in local.resources : name => "serviceAccount: ${resource.email}" + } + resources = { + for name in var.names : google_service_account.service_accounts[name] + } +} + +resource "google_service_account" "service_accounts" { + for_each = toset(var.names) + project = var.project_id + account_id = "${local.prefix}${lower(each.value)}" + display_name = "Terraform-managed." +} + +resource "google_service_account_key" "keys" { + for_each = var.generate_keys ? toset(var.names) : toset([]) + service_account_id = google_service_account.service_accounts[each.value].email +} + +resource "google_billing_account_iam_member" "roles" { + for_each = { + for pair in local.iam_billing_pairs : + "${pair.name}-${pair.entity}-${pair.role}" => pair + } + billing_account_id = each.value.entity + role = each.value.role + member = local.resource_iam_emails[each.value.name] +} + +resource "google_folder_iam_member" "roles" { + for_each = { + for pair in local.iam_folder_pairs : + "${pair.name}-${pair.entity}-${pair.role}" => pair + } + folder = each.value.entity + role = each.value.role + member = local.resource_iam_emails[each.value.name] +} + +resource "google_organization_iam_member" "roles" { + for_each = { + for pair in local.iam_organization_pairs : + "${pair.name}-${pair.entity}-${pair.role}" => pair + } + org_id = each.value.entity + role = each.value.role + member = local.resource_iam_emails[each.value.name] +} + +resource "google_project_iam_member" "project-roles" { + for_each = { + for pair in local.iam_project_pairs : + "${pair.name}-${pair.entity}-${pair.role}" => pair + } + project = each.value.entity + role = each.value.role + member = local.resource_iam_emails[each.value.name] +} + +# TODO(ludoo): link from README +# ref: https://cloud.google.com/vpc/docs/shared-vpc diff --git a/modules/iam-service-accounts/outputs.tf b/modules/iam-service-accounts/outputs.tf new file mode 100644 index 0000000000..bd33d96706 --- /dev/null +++ b/modules/iam-service-accounts/outputs.tf @@ -0,0 +1,78 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "service_account" { + description = "Service account resource (for single use)." + value = local.resource +} + +output "service_accounts" { + description = "Service account resources." + value = local.resources +} + +output "email" { + description = "Service account email (for single use)." + value = local.resource.email +} + +output "iam_email" { + description = "IAM-format service account email (for single use)." + value = "serviceAccount:${local.resource.email}" +} + +output "key" { + description = "Service account key (for single use)." + value = data.template_file.keys[0].rendered +} + +output "emails" { + description = "Service account emails." + value = zipmap(var.names, slice(local.emails, 0, length(var.names))) +} + +output "iam_emails" { + description = "IAM-format service account emails." + value = zipmap(var.names, slice(local.iam_emails, 0, length(var.names))) +} + +output "emails_list" { + description = "Service account emails." + value = local.emails +} + +output "iam_emails_list" { + description = "IAM-format service account emails." + value = local.iam_emails +} + +data "template_file" "keys" { + count = length(var.names) + template = "$${key}" + + vars = { + key = var.generate_keys ? base64decode(google_service_account_key.keys[count.index].private_key) : "" + } +} + +output "keys" { + description = "Map of service account keys." + sensitive = true + value = zipmap( + var.names, + slice(data.template_file.keys[*].rendered, 0, length(var.names)) + ) +} diff --git a/modules/iam-service-accounts/variables.tf b/modules/iam-service-accounts/variables.tf new file mode 100644 index 0000000000..c1ac74966c --- /dev/null +++ b/modules/iam-service-accounts/variables.tf @@ -0,0 +1,62 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "generate_keys" { + description = "Generate keys for service accounts." + type = bool + default = false +} + +variable "names" { + description = "Names of the service accounts to create." + type = list(string) + default = [] +} + +variable "prefix" { + description = "Prefix applied to service account names." + type = string + default = "" +} + +variable "project_id" { + description = "Project id where service account will be created." + type = string +} + +variable "iam_billing_roles" { + description = "Project role lists by billing account id." + type = map(list(string)) + default = {} +} + +variable "iam_folder_roles" { + description = "Project role lists by folder id." + type = map(list(string)) + default = {} +} + +variable "iam_organization_roles" { + description = "Project role lists by organization id." + type = map(list(string)) + default = {} +} + +variable "iam_project_roles" { + description = "Project role lists by project id." + type = map(list(string)) + default = {} +} diff --git a/modules/iam-service-accounts/versions.tf b/modules/iam-service-accounts/versions.tf new file mode 100644 index 0000000000..ce6918e09d --- /dev/null +++ b/modules/iam-service-accounts/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12.6" +} From b9ed136e7525e4ffe1c33dbc472b588d0fece47f Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Wed, 18 Dec 2019 19:33:56 +0100 Subject: [PATCH 050/106] work on service accounts module --- modules/iam-service-accounts/main.tf | 6 ++++- modules/iam-service-accounts/outputs.tf | 28 +++++++---------------- modules/iam-service-accounts/variables.tf | 8 +++---- 3 files changed, 17 insertions(+), 25 deletions(-) diff --git a/modules/iam-service-accounts/main.tf b/modules/iam-service-accounts/main.tf index e4155c5bdd..975a6473a5 100644 --- a/modules/iam-service-accounts/main.tf +++ b/modules/iam-service-accounts/main.tf @@ -43,8 +43,12 @@ locals { ] ] ]) + keys = { + for name in var.names : + name => base64encode(google_service_account_key.keys[name].private_key) + } prefix = var.prefix != "" ? "${var.prefix}-" : "" - resource = local.resources[var.names[0]] + resource = lookup(local.resources, var.names[0], {}) resource_iam_emails = { for name, resource in local.resources : name => "serviceAccount: ${resource.email}" } diff --git a/modules/iam-service-accounts/outputs.tf b/modules/iam-service-accounts/outputs.tf index bd33d96706..cbf0252e49 100644 --- a/modules/iam-service-accounts/outputs.tf +++ b/modules/iam-service-accounts/outputs.tf @@ -26,53 +26,41 @@ output "service_accounts" { output "email" { description = "Service account email (for single use)." - value = local.resource.email + value = local.resource == null ? null : local.resource.email } output "iam_email" { description = "IAM-format service account email (for single use)." - value = "serviceAccount:${local.resource.email}" + value = local.resource == null ? null : "serviceAccount:${local.resource.email}" } output "key" { description = "Service account key (for single use)." - value = data.template_file.keys[0].rendered + value = lookup(local.keys, var.names[0], null) } output "emails" { description = "Service account emails." - value = zipmap(var.names, slice(local.emails, 0, length(var.names))) + value = { for name, resource in local.resources : name => resource.email } } output "iam_emails" { description = "IAM-format service account emails." - value = zipmap(var.names, slice(local.iam_emails, 0, length(var.names))) + value = { for name, resource in local.resources : name => "serviceAccount:${resource.email}" } } output "emails_list" { description = "Service account emails." - value = local.emails + value = [for name, resource in local.resources : resource.email] } output "iam_emails_list" { description = "IAM-format service account emails." - value = local.iam_emails -} - -data "template_file" "keys" { - count = length(var.names) - template = "$${key}" - - vars = { - key = var.generate_keys ? base64decode(google_service_account_key.keys[count.index].private_key) : "" - } + value = [for name, resource in local.resources : "serviceAccount:${resource.email}"] } output "keys" { description = "Map of service account keys." sensitive = true - value = zipmap( - var.names, - slice(data.template_file.keys[*].rendered, 0, length(var.names)) - ) + value = local.keys } diff --git a/modules/iam-service-accounts/variables.tf b/modules/iam-service-accounts/variables.tf index c1ac74966c..357dbf5aee 100644 --- a/modules/iam-service-accounts/variables.tf +++ b/modules/iam-service-accounts/variables.tf @@ -38,25 +38,25 @@ variable "project_id" { } variable "iam_billing_roles" { - description = "Project role lists by billing account id." + description = "Project roles applied to all service accounts, by billing account id." type = map(list(string)) default = {} } variable "iam_folder_roles" { - description = "Project role lists by folder id." + description = "Project roles applied to all service accounts, by folder id." type = map(list(string)) default = {} } variable "iam_organization_roles" { - description = "Project role lists by organization id." + description = "Project roles applied to all service accounts, by organization id." type = map(list(string)) default = {} } variable "iam_project_roles" { - description = "Project role lists by project id." + description = "Project roles applied to all service accounts, by project id." type = map(list(string)) default = {} } From 339c53f2779c64f4b0193e32937cf58686bfb906 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Wed, 18 Dec 2019 19:34:35 +0100 Subject: [PATCH 051/106] project module: add gcr service account --- modules/project/main.tf | 1 + modules/project/outputs.tf | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/modules/project/main.tf b/modules/project/main.tf index 579b9c091a..345d92c68a 100644 --- a/modules/project/main.tf +++ b/modules/project/main.tf @@ -17,6 +17,7 @@ locals { cloudsvc_service_account = "${google_project.project.number}@cloudservices.gserviceaccount.com" gce_service_account = "${google_project.project.number}-compute@developer.gserviceaccount.com" + gcr_service_account = "service-${google_project.project.number}@containerregistry.iam.gserviceaccount.com" gke_service_account = "service-${google_project.project.number}@container-engine-robot.iam.gserviceaccount.com" iam_nonauth_pairs = flatten([ for role in var.iam_nonauth_roles : [ diff --git a/modules/project/outputs.tf b/modules/project/outputs.tf index 4bdb17fab6..04d040c865 100644 --- a/modules/project/outputs.tf +++ b/modules/project/outputs.tf @@ -44,6 +44,12 @@ output "gce_service_account" { depends_on = [google_project_service.project_services] } +output "gcr_service_account" { + description = "Default GCR service account (depends on services)." + value = local.gcr_service_account + depends_on = [google_project_service.project_services] +} + output "gke_service_account" { description = "Default GKE service account (depends on services)." value = local.gke_service_account From 664a06c0882db76adc4b2c9acac4fe945c902e2b Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Wed, 18 Dec 2019 20:31:11 +0100 Subject: [PATCH 052/106] project module: update outputs in README --- modules/project/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/project/README.md b/modules/project/README.md index 83f652ce6b..62903a1b1c 100644 --- a/modules/project/README.md +++ b/modules/project/README.md @@ -41,6 +41,7 @@ The resources/services/activations/deletions that this module will create/trigge | cloudsvc_service_account | Cloud services service account (depends on services). | | | custom_roles | Ids of the created custom roles. | | | gce_service_account | Default GCE service account (depends on services). | | +| gcr_service_account | Default GCR service account (depends on services). | | | gke_service_account | Default GKE service account (depends on services). | | | name | Name (depends on services). | | | number | Project number (depends on services). | | From ea97a9358eeab140d566b4ae2bd545835b66bd81 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Wed, 18 Dec 2019 20:31:28 +0100 Subject: [PATCH 053/106] first working version of the iam service accounts module --- modules/iam-service-accounts/README.md | 106 +++++++----------------- modules/iam-service-accounts/main.tf | 16 ++-- modules/iam-service-accounts/outputs.tf | 2 +- 3 files changed, 40 insertions(+), 84 deletions(-) diff --git a/modules/iam-service-accounts/README.md b/modules/iam-service-accounts/README.md index 4735858aaf..372398f991 100644 --- a/modules/iam-service-accounts/README.md +++ b/modules/iam-service-accounts/README.md @@ -5,81 +5,45 @@ This module allows easy creation of one or more service accounts, and granting t The resources/services/activations/deletions that this module will create/trigger are: - one or more service accounts -- optional project-level IAM role bindings for each service account -- one optional billing IAM role binding per service account, at the organization or billing account level -- two optional organization-level IAM bindings per service account, to enable the service accounts to create and manage Shared VPC networks +- optional non-autoritative IAM role bindings for each service account for the following resource types + - organization + - billing account + - folder + - project - one optional service account key per service account -## Compatibility + +## Variables - This module is meant for use with Terraform 0.12. If you haven't [upgraded](https://www.terraform.io/upgrade-guides/0-12.html) - and need a Terraform 0.11.x-compatible version of this module, the last released version intended for - Terraform 0.11.x is [0.1.1](https://registry.terraform.io/modules/terraform-google-modules/service-accounts/google/0.1.1). - -## Usage - -Basic usage of this module is as follows: - -```hcl -module "service_accounts" { - source = "terraform-google-modules/service-accounts/google" - version = "~> 2.0" - project_id = "" - prefix = "test-sa" - names = ["first", "second"] - project_roles = [ - "project-foo=>roles/viewer", - "project-spam=>roles/storage.objectViewer", - ] -} -``` - -Functional examples are included in the -[examples](./examples/) directory. - - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|:----:|:-----:|:-----:| -| billing\_account\_id | If assigning billing role, specificy a billing account (default is to assign at the organizational level). | string | `""` | no | -| generate\_keys | Generate keys for service accounts. | bool | `"false"` | no | -| grant\_billing\_role | Grant billing user role. | bool | `"false"` | no | -| grant\_xpn\_roles | Grant roles for shared VPC management. | bool | `"true"` | no | -| names | Names of the service accounts to create. | list(string) | `` | no | -| org\_id | Id of the organization for org-level roles. | string | `""` | no | -| prefix | Prefix applied to service account names. | string | `""` | no | -| project\_id | Project id where service account will be created. | string | n/a | yes | -| project\_roles | Common roles to apply to all service accounts, project=>role as elements. | list(string) | `` | no | +| name | description | type | required | +|---|---|:---: |:---:| +| project_id | Project id where service account will be created. | `string` | ✓ +| *generate_keys* | Generate keys for service accounts. | `bool` | +| *iam_billing_roles* | Project roles applied to all service accounts, by billing account id. | `map(list(string))` | +| *iam_folder_roles* | Project roles applied to all service accounts, by folder id. | `map(list(string))` | +| *iam_organization_roles* | Project roles applied to all service accounts, by organization id. | `map(list(string))` | +| *iam_project_roles* | Project roles applied to all service accounts, by project id. | `map(list(string))` | +| *names* | Names of the service accounts to create. | `list(string)` | +| *prefix* | Prefix applied to service account names. | `string` | ## Outputs -| Name | Description | -|------|-------------| -| email | Service account email (for single use). | -| emails | Service account emails. | -| emails\_list | Service account emails. | -| iam\_email | IAM-format service account email (for single use). | -| iam\_emails | IAM-format service account emails. | -| iam\_emails\_list | IAM-format service account emails. | -| key | Service account key (for single use). | -| keys | Map of service account keys. | -| service\_account | Service account resource (for single use). | -| service\_accounts | Service account resources. | - - +| name | description | sensitive | +|---|---|:---:| +| email | Service account email (for single use). | | +| emails | Service account emails. | | +| emails_list | Service account emails. | | +| iam_email | IAM-format service account email (for single use). | | +| iam_emails | IAM-format service account emails. | | +| iam_emails_list | IAM-format service account emails. | | +| key | Service account key (for single use). | | +| keys | Map of service account keys. | ✓ | +| service_account | Service account resource (for single use). | | +| service_accounts | Service account resources. | | + ## Requirements -These sections describe requirements for using this module. - -### Software - -The following dependencies must be available: - -- [Terraform][terraform] v0.12 -- [Terraform Provider for GCP][terraform-provider-gcp] plugin >= v2.0 - ### IAM Service account or user credentials with the following roles must be used to provision the resources of this module: @@ -87,13 +51,3 @@ Service account or user credentials with the following roles must be used to pro - Service Account Admin: `roles/iam.serviceAccountAdmin` - (optional) Service Account Key Admin: `roles/iam.serviceAccountAdmin` when `generate_keys` is set to `true` - (optional) roles needed to grant optional IAM roles at the project or organizational level - -## Contributing - -Refer to the [contribution guidelines](./CONTRIBUTING.md) for -information on contributing to this module. - -[iam-module]: https://registry.terraform.io/modules/terraform-google-modules/iam/google -[project-factory-module]: https://registry.terraform.io/modules/terraform-google-modules/project-factory/google -[terraform-provider-gcp]: https://www.terraform.io/docs/providers/google/index.html -[terraform]: https://www.terraform.io/downloads.html diff --git a/modules/iam-service-accounts/main.tf b/modules/iam-service-accounts/main.tf index 975a6473a5..996a21a1c3 100644 --- a/modules/iam-service-accounts/main.tf +++ b/modules/iam-service-accounts/main.tf @@ -43,17 +43,19 @@ locals { ] ] ]) - keys = { + keys = var.generate_keys ? { for name in var.names : - name => base64encode(google_service_account_key.keys[name].private_key) - } - prefix = var.prefix != "" ? "${var.prefix}-" : "" - resource = lookup(local.resources, var.names[0], {}) + name => lookup(google_service_account_key.keys, name, null) + } : {} + prefix = var.prefix != "" ? "${var.prefix}-" : "" + resource = lookup(local.resources, var.names[0], null) resource_iam_emails = { - for name, resource in local.resources : name => "serviceAccount: ${resource.email}" + for name, resource in local.resources : + name => "serviceAccount:${resource.email}" } resources = { - for name in var.names : google_service_account.service_accounts[name] + for name in var.names : + name => lookup(google_service_account.service_accounts, name, null) } } diff --git a/modules/iam-service-accounts/outputs.tf b/modules/iam-service-accounts/outputs.tf index cbf0252e49..9901675ef9 100644 --- a/modules/iam-service-accounts/outputs.tf +++ b/modules/iam-service-accounts/outputs.tf @@ -46,7 +46,7 @@ output "emails" { output "iam_emails" { description = "IAM-format service account emails." - value = { for name, resource in local.resources : name => "serviceAccount:${resource.email}" } + value = local.resource_iam_emails } output "emails_list" { From 9ea16b2bd6d09bd83739e4d9beae9a0542a41b25 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Thu, 19 Dec 2019 12:16:37 +0100 Subject: [PATCH 054/106] iam service accounts module: extra checks in locals --- modules/iam-service-accounts/main.tf | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/modules/iam-service-accounts/main.tf b/modules/iam-service-accounts/main.tf index 996a21a1c3..701da3bdcc 100644 --- a/modules/iam-service-accounts/main.tf +++ b/modules/iam-service-accounts/main.tf @@ -43,12 +43,24 @@ locals { ] ] ]) - keys = var.generate_keys ? { - for name in var.names : - name => lookup(google_service_account_key.keys, name, null) - } : {} - prefix = var.prefix != "" ? "${var.prefix}-" : "" - resource = lookup(local.resources, var.names[0], null) + keys = ( + var.generate_keys + ? { + for name in var.names : + name => lookup(google_service_account_key.keys, name, null) + } + : {} + ) + prefix = ( + var.prefix != "" + ? "${var.prefix}-" + : "" + ) + resource = ( + length(var.names) > 0 + ? lookup(local.resources, var.names[0], null) + : null + ) resource_iam_emails = { for name, resource in local.resources : name => "serviceAccount:${resource.email}" From 9fb15ce7eec34286218f891ffbf17bc833b25a2f Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Thu, 19 Dec 2019 17:37:13 +0100 Subject: [PATCH 055/106] modules/net-cloudnat: reorder variables --- modules/net-cloudnat/variables.tf | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/net-cloudnat/variables.tf b/modules/net-cloudnat/variables.tf index 452bc24cad..baa51d6483 100644 --- a/modules/net-cloudnat/variables.tf +++ b/modules/net-cloudnat/variables.tf @@ -76,18 +76,18 @@ variable "region" { type = string } -variable "router_name" { - description = "Name of the existing or auto-created router." - type = string - default = "cloud-nat" -} - variable "router_asn" { description = "Router ASN used for auto-created router." type = number default = 64514 } +variable "router_name" { + description = "Name of the existing or auto-created router." + type = string + default = "cloud-nat" +} + variable "router_network" { description = "Name of the VPC used for auto-created router." type = string From 1b6dea70e67d62bd285a2f1567a015248e3b60de Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Thu, 19 Dec 2019 17:37:37 +0100 Subject: [PATCH 056/106] modules/net-vpn-dynamic: initial import (untested) --- modules/net-vpn-dynamic/README.md | 30 +++++++ modules/net-vpn-dynamic/main.tf | 123 +++++++++++++++++++++++++++ modules/net-vpn-dynamic/outputs.tf | 65 ++++++++++++++ modules/net-vpn-dynamic/variables.tf | 94 ++++++++++++++++++++ modules/net-vpn-dynamic/versions.tf | 19 +++++ 5 files changed, 331 insertions(+) create mode 100644 modules/net-vpn-dynamic/README.md create mode 100644 modules/net-vpn-dynamic/main.tf create mode 100644 modules/net-vpn-dynamic/outputs.tf create mode 100644 modules/net-vpn-dynamic/variables.tf create mode 100644 modules/net-vpn-dynamic/versions.tf diff --git a/modules/net-vpn-dynamic/README.md b/modules/net-vpn-dynamic/README.md new file mode 100644 index 0000000000..9fa038b383 --- /dev/null +++ b/modules/net-vpn-dynamic/README.md @@ -0,0 +1,30 @@ +# Cloud VPN Route-based Module + + +## Variables + +| name | description | type | required | +|---|---|:---: |:---:| +| name | VPN gateway name, and prefix used for dependent resources. | `string` | ✓ +| network | VPC used for the gateway and routes. | `string` | ✓ +| project_id | Project where resources will be created. | `string` | ✓ +| region | Region used for resources. | `string` | ✓ +| *create_address* | Create gateway address resource instead of using external one, defaults to true. | `bool` | +| *gateway_address* | Optional address assigned to the VPN, used if create_address is false. | `string` | +| *remote_ranges* | Remote IP CIDR ranges. | `list(string)` | +| *route_priority* | Route priority, defaults to 1000. | `number` | +| *tunnels* | VPN tunnel configurations. | `map(object({...}))` | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| address | VPN gateway address. | | +| gateway | VPN gateway resource. | | +| name | VPN gateway name. | | +| random_secret | Generated secret. | ✓ | +| self_link | VPN gateway self link. | | +| tunnel_names | VPN tunnel names. | | +| tunnel_self_links | VPN tunnel self links. | | +| tunnels | VPN tunnel resources. | | + diff --git a/modules/net-vpn-dynamic/main.tf b/modules/net-vpn-dynamic/main.tf new file mode 100644 index 0000000000..3e9674d6a5 --- /dev/null +++ b/modules/net-vpn-dynamic/main.tf @@ -0,0 +1,123 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + gateway_address = ( + var.create_address + ? google_compute_address.gateway[0].address + : var.gateway_address + ) + router = ( + var.create_router + ? google_compute_router.router[0].name + : var.router_name + ) + secret = random_id.secret.b64_url +} + +resource "google_compute_address" "gateway" { + count = var.create_address ? 1 : 0 + name = "vpn-${var.name}" + project = var.project_id + region = var.region +} + +resource "google_compute_forwarding_rule" "esp" { + name = "vpn-${var.name}-esp" + project = var.project_id + region = var.region + target = google_compute_vpn_gateway.gateway.self_link + ip_address = local.gateway_address + ip_protocol = "ESP" +} + +resource "google_compute_forwarding_rule" "udp-500" { + name = "vpn-${var.name}-udp-500" + project = var.project_id + region = var.region + target = google_compute_vpn_gateway.gateway.self_link + ip_address = local.gateway_address + ip_protocol = "UDP" + port_range = "500" +} + +resource "google_compute_forwarding_rule" "udp-4500" { + name = "vpn-${var.name}-udp-4500" + project = var.project_id + region = var.region + target = google_compute_vpn_gateway.gateway.self_link + ip_address = local.gateway_address + ip_protocol = "UDP" + port_range = "4500" +} + +resource "google_compute_router" "router" { + count = var.create_router ? 1 : 0 + name = "${local.prefix}${var.router_name}" + project = var.project_id + region = var.region + network = var.router_network + bgp { + asn = var.router_asn + } +} + +resource "google_compute_router_peer" "bgp_peer" { + for_each = var.tunnels + region = var.region + project = var.project_id + name = "${var.name}-${each.key}" + router = local.router + peer_ip_address = each.value.bgp_peer_address + peer_asn = each.value.bgp_peer_asn + advertised_route_priority = var.advertised_route_priority + interface = google_compute_router_interface.router_interface[each.key].name +} + +resource "google_compute_router_interface" "router_interface" { + for_each = var.tunnels + project = var.project_id + region = var.region + name = "${var.name}-${each.key}" + router = local.router + ip_range = length(each.value.bgp_session_range) == 0 ? null : each.value.bgp_session_range + vpn_tunnel = google_compute_vpn_tunnel.tunnels[each.key].name +} + +resource "google_compute_vpn_gateway" "gateway" { + name = var.name + project = var.project_id + region = var.region + network = var.network +} + +resource "google_compute_vpn_tunnel" "tunnels" { + for_each = var.tunnels + project = var.project_id + region = var.region + name = "${var.name}-${each.key}" + router = local.router + peer_ip = each.value.peer_ip + local_traffic_selector = each.value.traffic_selectors.local + remote_traffic_selector = each.value.traffic_selectors.remote + ike_version = each.value.ike_version + shared_secret = each.value.shared_secret == "" ? local.secret : each.value.shared_secret + target_vpn_gateway = google_compute_vpn_gateway.gateway.self_link +} + +resource "random_id" "secret" { + byte_length = 8 +} diff --git a/modules/net-vpn-dynamic/outputs.tf b/modules/net-vpn-dynamic/outputs.tf new file mode 100644 index 0000000000..09ecc5ec0b --- /dev/null +++ b/modules/net-vpn-dynamic/outputs.tf @@ -0,0 +1,65 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "address" { + description = "VPN gateway address." + value = local.gateway_address +} + +output "gateway" { + description = "VPN gateway resource." + value = google_compute_vpn_gateway.gateway +} + +output "name" { + description = "VPN gateway name." + value = google_compute_vpn_gateway.gateway.name +} + +output "self_link" { + description = "VPN gateway self link." + value = google_compute_vpn_gateway.gateway.self_link +} + +output "tunnels" { + description = "VPN tunnel resources." + value = { + for name in keys(var.tunnels) : + name => google_compute_vpn_tunnel.tunnels[name] + } +} + +output "tunnel_names" { + description = "VPN tunnel names." + value = { + for name in keys(var.tunnels) : + name => google_compute_vpn_tunnel.tunnels[name].name + } +} + +output "tunnel_self_links" { + description = "VPN tunnel self links." + value = { + for name in keys(var.tunnels) : + name => google_compute_vpn_tunnel.tunnels[name].self_link + } +} + +output "random_secret" { + description = "Generated secret." + sensitive = true + value = local.secret +} diff --git a/modules/net-vpn-dynamic/variables.tf b/modules/net-vpn-dynamic/variables.tf new file mode 100644 index 0000000000..00e62a5882 --- /dev/null +++ b/modules/net-vpn-dynamic/variables.tf @@ -0,0 +1,94 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "create_address" { + description = "Create gateway address resource instead of using external one, defaults to true." + type = bool + default = true +} + +variable "create_router" { + description = "Create router for Cloud NAT instead of using existing one." + type = bool + default = true +} + +variable "gateway_address" { + description = "Optional address assigned to the VPN, used if create_address is false." + type = string + default = "" +} + +variable "name" { + description = "VPN gateway name, and prefix used for dependent resources." + type = string +} + +variable "network" { + description = "VPC used for the gateway and routes." + type = string +} + +variable "project_id" { + description = "Project where resources will be created." + type = string +} + +variable "region" { + description = "Region used for resources." + type = string +} + +variable "remote_ranges" { + description = "Remote IP CIDR ranges." + type = list(string) + default = [] +} + +variable "route_priority" { + description = "Route priority, defaults to 1000." + type = number + default = 1000 +} + +variable "router_asn" { + description = "Router ASN used for auto-created router." + type = number + default = 64514 +} + +variable "router_name" { + description = "Name of the existing or auto-created router." + type = string + default = "vpn" +} + +variable "tunnels" { + description = "VPN tunnel configurations." + type = map(object({ + bgp_peer_address = string + bgp_peer_asn = number + bgp_session_range = list(string) + ike_version = number + peer_ip = string + shared_secret = string + traffic_selectors = object({ + local = list(string) + remote = list(string) + }) + })) + default = {} +} diff --git a/modules/net-vpn-dynamic/versions.tf b/modules/net-vpn-dynamic/versions.tf new file mode 100644 index 0000000000..ce6918e09d --- /dev/null +++ b/modules/net-vpn-dynamic/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12.6" +} From fac51b55fc6791d51efbeac92e3706a551d7c94c Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Thu, 19 Dec 2019 19:00:24 +0100 Subject: [PATCH 057/106] modules/net-vpn-dynamic: first working version --- modules/net-vpn-dynamic/main.tf | 43 ++++++++++++++++++---------- modules/net-vpn-dynamic/variables.tf | 29 ++++++++++--------- modules/net-vpn-static/main.tf | 1 + 3 files changed, 44 insertions(+), 29 deletions(-) diff --git a/modules/net-vpn-dynamic/main.tf b/modules/net-vpn-dynamic/main.tf index 3e9674d6a5..07a529e35d 100644 --- a/modules/net-vpn-dynamic/main.tf +++ b/modules/net-vpn-dynamic/main.tf @@ -66,11 +66,25 @@ resource "google_compute_forwarding_rule" "udp-4500" { resource "google_compute_router" "router" { count = var.create_router ? 1 : 0 - name = "${local.prefix}${var.router_name}" + name = var.router_name != "" ? var.router_name : "vpn-${var.name}" project = var.project_id region = var.region - network = var.router_network + network = var.network bgp { + advertise_mode = ( + var.router_advertise_config == null ? "DEFAULT" : "CUSTOM" + ) + advertised_groups = (var.router_advertise_config == null ? null : ( + var.router_advertise_config.all_subnets ? ["ALL_SUBNETS"] : [] + )) + dynamic advertised_ip_ranges { + for_each = var.router_advertise_config == null ? {} : var.router_advertise_config.ip_ranges + iterator = range + content { + range = range.key + description = range.value + } + } asn = var.router_asn } } @@ -83,7 +97,7 @@ resource "google_compute_router_peer" "bgp_peer" { router = local.router peer_ip_address = each.value.bgp_peer_address peer_asn = each.value.bgp_peer_asn - advertised_route_priority = var.advertised_route_priority + advertised_route_priority = var.route_priority interface = google_compute_router_interface.router_interface[each.key].name } @@ -93,7 +107,7 @@ resource "google_compute_router_interface" "router_interface" { region = var.region name = "${var.name}-${each.key}" router = local.router - ip_range = length(each.value.bgp_session_range) == 0 ? null : each.value.bgp_session_range + ip_range = each.value.bgp_session_range == "" ? null : each.value.bgp_session_range vpn_tunnel = google_compute_vpn_tunnel.tunnels[each.key].name } @@ -105,17 +119,16 @@ resource "google_compute_vpn_gateway" "gateway" { } resource "google_compute_vpn_tunnel" "tunnels" { - for_each = var.tunnels - project = var.project_id - region = var.region - name = "${var.name}-${each.key}" - router = local.router - peer_ip = each.value.peer_ip - local_traffic_selector = each.value.traffic_selectors.local - remote_traffic_selector = each.value.traffic_selectors.remote - ike_version = each.value.ike_version - shared_secret = each.value.shared_secret == "" ? local.secret : each.value.shared_secret - target_vpn_gateway = google_compute_vpn_gateway.gateway.self_link + for_each = var.tunnels + project = var.project_id + region = var.region + name = "${var.name}-${each.key}" + router = local.router + peer_ip = each.value.peer_ip + ike_version = each.value.ike_version + shared_secret = each.value.shared_secret == "" ? local.secret : each.value.shared_secret + target_vpn_gateway = google_compute_vpn_gateway.gateway.self_link + depends_on = [google_compute_forwarding_rule.esp] } resource "random_id" "secret" { diff --git a/modules/net-vpn-dynamic/variables.tf b/modules/net-vpn-dynamic/variables.tf index 00e62a5882..64f2060c65 100644 --- a/modules/net-vpn-dynamic/variables.tf +++ b/modules/net-vpn-dynamic/variables.tf @@ -52,18 +52,21 @@ variable "region" { type = string } -variable "remote_ranges" { - description = "Remote IP CIDR ranges." - type = list(string) - default = [] -} - variable "route_priority" { description = "Route priority, defaults to 1000." type = number default = 1000 } +variable "router_advertise_config" { + description = "Router custom advertisement configuration, ip_ranges is a map of range as key and description as value." + type = object({ + all_subnets = bool + ip_ranges = map(string) + }) + default = null +} + variable "router_asn" { description = "Router ASN used for auto-created router." type = number @@ -73,22 +76,20 @@ variable "router_asn" { variable "router_name" { description = "Name of the existing or auto-created router." type = string - default = "vpn" + default = "" } variable "tunnels" { description = "VPN tunnel configurations." type = map(object({ - bgp_peer_address = string - bgp_peer_asn = number - bgp_session_range = list(string) + bgp_peer_address = string + bgp_peer_asn = number + # each BGP session on the same Cloud Router must use a unique /30 CIDR + # from the 169.254.0.0/16 block. + bgp_session_range = string ike_version = number peer_ip = string shared_secret = string - traffic_selectors = object({ - local = list(string) - remote = list(string) - }) })) default = {} } diff --git a/modules/net-vpn-static/main.tf b/modules/net-vpn-static/main.tf index a8197db079..6edfddf3fc 100644 --- a/modules/net-vpn-static/main.tf +++ b/modules/net-vpn-static/main.tf @@ -93,6 +93,7 @@ resource "google_compute_vpn_tunnel" "tunnels" { ike_version = each.value.ike_version shared_secret = each.value.shared_secret == "" ? local.secret : each.value.shared_secret target_vpn_gateway = google_compute_vpn_gateway.gateway.self_link + depends_on = [google_compute_forwarding_rule.esp] } resource "random_id" "secret" { From bb0df9f5c65e694d8684b2415bddf41eb2831343 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Thu, 19 Dec 2019 19:07:20 +0100 Subject: [PATCH 058/106] modules/net-vpn-dynamic: add outputs for auto-created router --- modules/net-vpn-dynamic/outputs.tf | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/modules/net-vpn-dynamic/outputs.tf b/modules/net-vpn-dynamic/outputs.tf index 09ecc5ec0b..a65a4495ee 100644 --- a/modules/net-vpn-dynamic/outputs.tf +++ b/modules/net-vpn-dynamic/outputs.tf @@ -29,6 +29,20 @@ output "name" { value = google_compute_vpn_gateway.gateway.name } +output "router" { + description = "Router resource (only if auto-created)." + value = ( + var.create_router && length(google_compute_router.router) > 0 + ? google_compute_router.router[0] + : null + ) +} + +output "router_name" { + description = "Router name." + value = local.router +} + output "self_link" { description = "VPN gateway self link." value = google_compute_vpn_gateway.gateway.self_link From 529f8cc33389b8d2216e422e5a587064ee2fbb73 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Thu, 19 Dec 2019 19:12:14 +0100 Subject: [PATCH 059/106] modules/net-vpn-dynamic: update README --- modules/net-vpn-dynamic/README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/modules/net-vpn-dynamic/README.md b/modules/net-vpn-dynamic/README.md index 9fa038b383..bbaa6a02ff 100644 --- a/modules/net-vpn-dynamic/README.md +++ b/modules/net-vpn-dynamic/README.md @@ -1,4 +1,4 @@ -# Cloud VPN Route-based Module +# Cloud VPN Dynamic Module ## Variables @@ -10,9 +10,12 @@ | project_id | Project where resources will be created. | `string` | ✓ | region | Region used for resources. | `string` | ✓ | *create_address* | Create gateway address resource instead of using external one, defaults to true. | `bool` | +| *create_router* | Create router for Cloud NAT instead of using existing one. | `bool` | | *gateway_address* | Optional address assigned to the VPN, used if create_address is false. | `string` | -| *remote_ranges* | Remote IP CIDR ranges. | `list(string)` | | *route_priority* | Route priority, defaults to 1000. | `number` | +| *router_advertise_config* | Router custom advertisement configuration, ip_ranges is a map of range as key and description as value. | `object({...})` | +| *router_asn* | Router ASN used for auto-created router. | `number` | +| *router_name* | Name of the existing or auto-created router. | `string` | | *tunnels* | VPN tunnel configurations. | `map(object({...}))` | ## Outputs @@ -23,6 +26,8 @@ | gateway | VPN gateway resource. | | | name | VPN gateway name. | | | random_secret | Generated secret. | ✓ | +| router | Router resource (only if auto-created). | | +| router_name | Router name. | | | self_link | VPN gateway self link. | | | tunnel_names | VPN tunnel names. | | | tunnel_self_links | VPN tunnel self links. | | From 9c07882e727b369c0c2384c0d897a0334db08dea Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Fri, 20 Dec 2019 01:21:34 +0100 Subject: [PATCH 060/106] modules/net-[vpn,cloudnat]: clean up variable,s remove prefix --- modules/net-cloudnat/main.tf | 9 ++++----- modules/net-cloudnat/outputs.tf | 2 +- modules/net-cloudnat/variables.tf | 19 +++---------------- modules/net-vpn-dynamic/main.tf | 10 +++++----- modules/net-vpn-dynamic/outputs.tf | 6 +----- modules/net-vpn-dynamic/variables.tf | 18 +++--------------- modules/net-vpn-static/main.tf | 4 ++-- modules/net-vpn-static/variables.tf | 8 +------- 8 files changed, 20 insertions(+), 56 deletions(-) diff --git a/modules/net-cloudnat/main.tf b/modules/net-cloudnat/main.tf index de7f540c25..70944a55a8 100644 --- a/modules/net-cloudnat/main.tf +++ b/modules/net-cloudnat/main.tf @@ -15,17 +15,16 @@ */ locals { - prefix = var.prefix == "" ? "" : "${var.prefix}-" router = ( - var.create_router + var.router_name == "" ? google_compute_router.router[0].name : var.router_name ) } resource "google_compute_router" "router" { - count = var.create_router ? 1 : 0 - name = "${local.prefix}${var.router_name}" + count = var.router_name == "" ? 1 : 0 + name = var.router_name project = var.project_id region = var.region network = var.router_network @@ -37,7 +36,7 @@ resource "google_compute_router" "router" { resource "google_compute_router_nat" "nat" { project = var.project_id region = var.region - name = "${local.prefix}${var.name}" + name = var.name router = local.router nat_ips = var.addresses nat_ip_allocate_option = length(var.addresses) > 0 ? "MANUAL_ONLY" : "AUTO_ONLY" diff --git a/modules/net-cloudnat/outputs.tf b/modules/net-cloudnat/outputs.tf index 69a4972cb3..5aed8d794a 100644 --- a/modules/net-cloudnat/outputs.tf +++ b/modules/net-cloudnat/outputs.tf @@ -31,7 +31,7 @@ output "region" { output "router" { description = "Cloud NAT router resources (if auto created)." - value = var.create_router ? google_compute_router.router[0] : null + value = var.router_name == "" ? google_compute_router.router[0] : null } output "router_name" { diff --git a/modules/net-cloudnat/variables.tf b/modules/net-cloudnat/variables.tf index baa51d6483..acc86ec660 100644 --- a/modules/net-cloudnat/variables.tf +++ b/modules/net-cloudnat/variables.tf @@ -27,7 +27,7 @@ variable "config_min_ports_per_vm" { } variable "config_source_subnets" { - description = "Subnetwork configuration, valid values are ALL_SUBNETWORKS_ALL_IP_RANGES, ALL_SUBNETWORKS_ALL_PRIMARY_IP_RANGES, LIST_OF_SUBNETWORKS." + description = "Subnetwork configuration (ALL_SUBNETWORKS_ALL_IP_RANGES, ALL_SUBNETWORKS_ALL_PRIMARY_IP_RANGES, LIST_OF_SUBNETWORKS)." type = string default = "ALL_SUBNETWORKS_ALL_IP_RANGES" } @@ -48,22 +48,9 @@ variable "config_timeouts" { } } -variable "create_router" { - description = "Create router for Cloud NAT instead of using existing one." - type = bool - default = true -} - variable "name" { description = "Name of the Cloud NAT resource." type = string - default = "cloud-nat" -} - -variable "prefix" { - description = "Optional prefix that will be prepended to resource names." - type = string - default = "" } variable "project_id" { @@ -83,9 +70,9 @@ variable "router_asn" { } variable "router_name" { - description = "Name of the existing or auto-created router." + description = "Router name, leave blank to create." type = string - default = "cloud-nat" + default = "" } variable "router_network" { diff --git a/modules/net-vpn-dynamic/main.tf b/modules/net-vpn-dynamic/main.tf index 07a529e35d..1bb69faaf3 100644 --- a/modules/net-vpn-dynamic/main.tf +++ b/modules/net-vpn-dynamic/main.tf @@ -16,12 +16,12 @@ locals { gateway_address = ( - var.create_address + var.gateway_address == "" ? google_compute_address.gateway[0].address : var.gateway_address ) router = ( - var.create_router + var.router_name == "" ? google_compute_router.router[0].name : var.router_name ) @@ -29,7 +29,7 @@ locals { } resource "google_compute_address" "gateway" { - count = var.create_address ? 1 : 0 + count = var.gateway_address == "" ? 1 : 0 name = "vpn-${var.name}" project = var.project_id region = var.region @@ -65,8 +65,8 @@ resource "google_compute_forwarding_rule" "udp-4500" { } resource "google_compute_router" "router" { - count = var.create_router ? 1 : 0 - name = var.router_name != "" ? var.router_name : "vpn-${var.name}" + count = var.router_name == "" ? 1 : 0 + name = "vpn-${var.name}" project = var.project_id region = var.region network = var.network diff --git a/modules/net-vpn-dynamic/outputs.tf b/modules/net-vpn-dynamic/outputs.tf index a65a4495ee..93106a9a6d 100644 --- a/modules/net-vpn-dynamic/outputs.tf +++ b/modules/net-vpn-dynamic/outputs.tf @@ -31,11 +31,7 @@ output "name" { output "router" { description = "Router resource (only if auto-created)." - value = ( - var.create_router && length(google_compute_router.router) > 0 - ? google_compute_router.router[0] - : null - ) + value = var.router_name == "" ? google_compute_router.router[0] : null } output "router_name" { diff --git a/modules/net-vpn-dynamic/variables.tf b/modules/net-vpn-dynamic/variables.tf index 64f2060c65..9d6eb6022b 100644 --- a/modules/net-vpn-dynamic/variables.tf +++ b/modules/net-vpn-dynamic/variables.tf @@ -14,20 +14,8 @@ * limitations under the License. */ -variable "create_address" { - description = "Create gateway address resource instead of using external one, defaults to true." - type = bool - default = true -} - -variable "create_router" { - description = "Create router for Cloud NAT instead of using existing one." - type = bool - default = true -} - variable "gateway_address" { - description = "Optional address assigned to the VPN, used if create_address is false." + description = "Optional address assigned to the VPN, leave blank to create one." type = string default = "" } @@ -59,7 +47,7 @@ variable "route_priority" { } variable "router_advertise_config" { - description = "Router custom advertisement configuration, ip_ranges is a map of range as key and description as value." + description = "Router custom advertisement configuration, ip_ranges is a map of address ranges and descriptions." type = object({ all_subnets = bool ip_ranges = map(string) @@ -74,7 +62,7 @@ variable "router_asn" { } variable "router_name" { - description = "Name of the existing or auto-created router." + description = "Name of router, leave blank to create one." type = string default = "" } diff --git a/modules/net-vpn-static/main.tf b/modules/net-vpn-static/main.tf index 6edfddf3fc..6eadd8db6e 100644 --- a/modules/net-vpn-static/main.tf +++ b/modules/net-vpn-static/main.tf @@ -16,7 +16,7 @@ locals { gateway_address = ( - var.create_address + var.gateway_address == "" ? google_compute_address.gateway[0].address : var.gateway_address ) @@ -30,7 +30,7 @@ locals { } resource "google_compute_address" "gateway" { - count = var.create_address ? 1 : 0 + count = var.gateway_address == "" ? 1 : 0 name = "vpn-${var.name}" project = var.project_id region = var.region diff --git a/modules/net-vpn-static/variables.tf b/modules/net-vpn-static/variables.tf index ee7209df7c..a3ba01d896 100644 --- a/modules/net-vpn-static/variables.tf +++ b/modules/net-vpn-static/variables.tf @@ -14,14 +14,8 @@ * limitations under the License. */ -variable "create_address" { - description = "Create gateway address resource instead of using external one, defaults to true." - type = bool - default = true -} - variable "gateway_address" { - description = "Optional address assigned to the VPN, used if create_address is false." + description = "Optional address assigned to the VPN, leave blank to create one." type = string default = "" } From 73fa5befb31751c078eb878a867135e97b8fd508 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Fri, 20 Dec 2019 12:24:06 +0100 Subject: [PATCH 061/106] modules/net-vpn-dynamic: add advertisement configuration to tunnel bgp peer, refactor variables --- modules/net-vpn-dynamic/README.md | 10 ++-- modules/net-vpn-dynamic/main.tf | 70 ++++++++++++++++++++++------ modules/net-vpn-dynamic/variables.tf | 19 ++++++-- 3 files changed, 74 insertions(+), 25 deletions(-) diff --git a/modules/net-vpn-dynamic/README.md b/modules/net-vpn-dynamic/README.md index bbaa6a02ff..f8373b7db1 100644 --- a/modules/net-vpn-dynamic/README.md +++ b/modules/net-vpn-dynamic/README.md @@ -9,14 +9,12 @@ | network | VPC used for the gateway and routes. | `string` | ✓ | project_id | Project where resources will be created. | `string` | ✓ | region | Region used for resources. | `string` | ✓ -| *create_address* | Create gateway address resource instead of using external one, defaults to true. | `bool` | -| *create_router* | Create router for Cloud NAT instead of using existing one. | `bool` | -| *gateway_address* | Optional address assigned to the VPN, used if create_address is false. | `string` | +| *gateway_address* | Optional address assigned to the VPN, leave blank to create one. | `string` | | *route_priority* | Route priority, defaults to 1000. | `number` | -| *router_advertise_config* | Router custom advertisement configuration, ip_ranges is a map of range as key and description as value. | `object({...})` | +| *router_advertise_config* | Router custom advertisement configuration, ip_ranges is a map of address ranges and descriptions. | `object({...})` | | *router_asn* | Router ASN used for auto-created router. | `number` | -| *router_name* | Name of the existing or auto-created router. | `string` | -| *tunnels* | VPN tunnel configurations. | `map(object({...}))` | +| *router_name* | Name of router, leave blank to create one. | `string` | +| *tunnels* | VPN tunnel configurations, bgp_peer_options is usually null. | `map(object({...}))` | ## Outputs diff --git a/modules/net-vpn-dynamic/main.tf b/modules/net-vpn-dynamic/main.tf index 1bb69faaf3..bfab01a4fc 100644 --- a/modules/net-vpn-dynamic/main.tf +++ b/modules/net-vpn-dynamic/main.tf @@ -72,13 +72,25 @@ resource "google_compute_router" "router" { network = var.network bgp { advertise_mode = ( - var.router_advertise_config == null ? "DEFAULT" : "CUSTOM" + var.router_advertise_config == null + ? null + : var.router_advertise_config.mode + ) + advertised_groups = ( + var.router_advertise_config == null ? null : ( + var.router_advertise_config.mode != "CUSTOM" + ? null + : var.router_advertise_config.groups + ) ) - advertised_groups = (var.router_advertise_config == null ? null : ( - var.router_advertise_config.all_subnets ? ["ALL_SUBNETS"] : [] - )) dynamic advertised_ip_ranges { - for_each = var.router_advertise_config == null ? {} : var.router_advertise_config.ip_ranges + for_each = ( + var.router_advertise_config == null ? {} : ( + var.router_advertise_config.mode != "CUSTOM" + ? null + : var.router_advertise_config.ip_ranges + ) + ) iterator = range content { range = range.key @@ -90,15 +102,45 @@ resource "google_compute_router" "router" { } resource "google_compute_router_peer" "bgp_peer" { - for_each = var.tunnels - region = var.region - project = var.project_id - name = "${var.name}-${each.key}" - router = local.router - peer_ip_address = each.value.bgp_peer_address - peer_asn = each.value.bgp_peer_asn - advertised_route_priority = var.route_priority - interface = google_compute_router_interface.router_interface[each.key].name + for_each = var.tunnels + region = var.region + project = var.project_id + name = "${var.name}-${each.key}" + router = local.router + peer_ip_address = each.value.bgp_peer.address + peer_asn = each.value.bgp_peer.asn + advertised_route_priority = ( + each.value.bgp_peer_options == null ? var.route_priority : ( + each.value.bgp_peer_options.route_priority == null + ? var.route_priority + : each.value.bgp_peer_options.route_priority + ) + ) + advertise_mode = ( + each.value.bgp_peer_options == null ? null : each.value.bgp_peer_options.advertise_mode + ) + advertised_groups = ( + each.value.bgp_peer_options == null ? null : ( + each.value.bgp_peer_options.advertise_mode != "CUSTOM" + ? null + : each.value.bgp_peer_options.advertise_groups + ) + ) + dynamic advertised_ip_ranges { + for_each = ( + each.value.bgp_peer_options == null ? {} : ( + each.value.bgp_peer_options.advertise_mode != "CUSTOM" + ? {} + : each.value.bgp_peer_options.advertise_ip_ranges + ) + ) + iterator = range + content { + range = range.key + description = range.value + } + } + interface = google_compute_router_interface.router_interface[each.key].name } resource "google_compute_router_interface" "router_interface" { diff --git a/modules/net-vpn-dynamic/variables.tf b/modules/net-vpn-dynamic/variables.tf index 9d6eb6022b..9ae408908e 100644 --- a/modules/net-vpn-dynamic/variables.tf +++ b/modules/net-vpn-dynamic/variables.tf @@ -49,8 +49,9 @@ variable "route_priority" { variable "router_advertise_config" { description = "Router custom advertisement configuration, ip_ranges is a map of address ranges and descriptions." type = object({ - all_subnets = bool - ip_ranges = map(string) + groups = list(string) + ip_ranges = map(string) + mode = string }) default = null } @@ -68,10 +69,18 @@ variable "router_name" { } variable "tunnels" { - description = "VPN tunnel configurations." + description = "VPN tunnel configurations, bgp_peer_options is usually null." type = map(object({ - bgp_peer_address = string - bgp_peer_asn = number + bgp_peer = object({ + address = string + asn = number + }) + bgp_peer_options = object({ + advertise_groups = list(string) + advertise_ip_ranges = map(string) + advertise_mode = string + route_priority = number + }) # each BGP session on the same Cloud Router must use a unique /30 CIDR # from the 169.254.0.0/16 block. bgp_session_range = string From cd17ec6475b05b3b47533d4dceb55885758efda9 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Fri, 20 Dec 2019 15:19:03 +0100 Subject: [PATCH 062/106] tfdoc: add tooltips for variable types and defaults --- tools/tfdoc/tfdoc.py | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/tools/tfdoc/tfdoc.py b/tools/tfdoc/tfdoc.py index d6bf3c0015..06f17d344e 100755 --- a/tools/tfdoc/tfdoc.py +++ b/tools/tfdoc/tfdoc.py @@ -18,6 +18,7 @@ import enum import os import re +import string import click @@ -48,6 +49,8 @@ # variable body (?:^\s*(\S.*?)$) ''') +REPL_VALID = string.digits + string.ascii_letters + ' .,;:_-' + OutputData = collections.namedtuple('Output', 'name description sensitive') OutputToken = enum.Enum('OutputToken', 'NAME DESCRIPTION SENSITIVE') @@ -122,6 +125,11 @@ def close(self): self.default is None) +def _escape(s): + "Basic, minimal HTML escaping" + return ''.join(c if c in REPL_VALID else ('&#%s;' % ord(c)) for c in s) + + def format_outputs(outputs): "Format outputs." if not outputs: @@ -160,13 +168,27 @@ def format_variables(variables, required_first=True): return variables.sort(key=lambda v: v.name) variables.sort(key=lambda v: v.required, reverse=True) - yield '| name | description | type | required |' - yield '|---|---|:---: |:---:|' + yield '| name | description | type | required | default |' + yield '|---|---|:---: |:---:|:---:|' + row = ( + '| {name} | {description} | {type} ' + '| {required} | {default} |' + ) for v in variables: - yield '| {name} | {description} | `{type}` | {required}'.format( + default = default_spec = type_spec = '' + if not v.required: + if '\n' in v.default: + default = '...' + default_spec = _escape(v.default) + else: + default = v.default or '' + if v.type and '(' in v.type: + type_spec = _escape(v.type) + yield row.format( name=v.name if v.required else '*%s*' % v.name, - description=v.description, type=format_type(v.type), - required='✓' if v.required else '' + description=v.description, required='✓' if v.required else '', + type=format_type(v.type), type_spec=type_spec, + default=default, default_spec=default_spec ) From 8fde9a8c7bba72f0bb7e0b15c5e22c6ba3372f78 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Fri, 20 Dec 2019 15:19:27 +0100 Subject: [PATCH 063/106] modules: update README variables and outputs --- modules/dns/README.md | 28 +++++------ modules/folder/README.md | 12 ++--- modules/gcs/README.md | 26 +++++----- modules/gke-cluster/README.md | 66 +++++++++++++------------- modules/iam-service-accounts/README.md | 20 ++++---- modules/kms/README.md | 20 ++++---- modules/net-address/README.md | 16 +++---- modules/net-cloudnat/README.md | 28 +++++------ modules/net-vpc-firewall/README.md | 20 ++++---- modules/net-vpc/README.md | 35 +++++++------- modules/net-vpn-dynamic/README.md | 24 +++++----- modules/net-vpn-static/README.md | 21 ++++---- modules/project/README.md | 38 +++++++-------- 13 files changed, 176 insertions(+), 178 deletions(-) diff --git a/modules/dns/README.md b/modules/dns/README.md index a1bdb43bc0..6658190223 100644 --- a/modules/dns/README.md +++ b/modules/dns/README.md @@ -30,20 +30,20 @@ module "dns-private-zone" { ## Variables -| name | description | type | required | -|---|---|:---: |:---:| -| domain | Zone domain, must end with a period. | `string` | ✓ -| name | Zone name, must be unique within the project. | `string` | ✓ -| project_id | Project id for the zone. | `string` | ✓ -| *client_networks* | List of VPC self links that can see this zone. | `list(string)` | -| *default_key_specs_key* | DNSSEC default key signing specifications: algorithm, key_length, key_type, kind. | `any` | -| *default_key_specs_zone* | DNSSEC default zone signing specifications: algorithm, key_length, key_type, kind. | `any` | -| *description* | Domain description. | `string` | -| *dnssec_config* | DNSSEC configuration: kind, non_existence, state. | `any` | -| *forwarders* | List of target name servers, only valid for 'forwarding' zone types. | `list(string)` | -| *peer_network* | Peering network self link, only valid for 'peering' zone types. | `string` | -| *recordsets* | List of DNS record objects to manage. | `string` | -| *type* | Type of zone to create, valid values are 'public', 'private', 'forwarding', 'peering'. | `string` | +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| domain | Zone domain, must end with a period. | string | ✓ | | +| name | Zone name, must be unique within the project. | string | ✓ | | +| project_id | Project id for the zone. | string | ✓ | | +| *client_networks* | List of VPC self links that can see this zone. | list(string) | | [] | +| *default_key_specs_key* | DNSSEC default key signing specifications: algorithm, key_length, key_type, kind. | any | | {} | +| *default_key_specs_zone* | DNSSEC default zone signing specifications: algorithm, key_length, key_type, kind. | any | | {} | +| *description* | Domain description. | string | | Terraform managed. | +| *dnssec_config* | DNSSEC configuration: kind, non_existence, state. | any | | {} | +| *forwarders* | List of target name servers, only valid for 'forwarding' zone types. | list(string) | | [] | +| *peer_network* | Peering network self link, only valid for 'peering' zone types. | string | | | +| *recordsets* | List of DNS record objects to manage. | string | | [] | +| *type* | Type of zone to create, valid values are 'public', 'private', 'forwarding', 'peering'. | string | | private | ## Outputs diff --git a/modules/folder/README.md b/modules/folder/README.md index 4df53a1e34..c46c27e5ad 100644 --- a/modules/folder/README.md +++ b/modules/folder/README.md @@ -3,12 +3,12 @@ ## Variables -| name | description | type | required | -|---|---|:---: |:---:| -| parent | Parent in folders/folder_id or organizations/org_id format. | `string` | ✓ -| *iam_members* | List of IAM members keyed by folder name and role. | `map(map(list(string)))` | -| *iam_roles* | List of IAM roles keyed by folder name. | `map(list(string))` | -| *names* | Folder names. | `list(string)` | +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| parent | Parent in folders/folder_id or organizations/org_id format. | string | ✓ | | +| *iam_members* | List of IAM members keyed by folder name and role. | map(map(list(string))) | | {} | +| *iam_roles* | List of IAM roles keyed by folder name. | map(list(string)) | | {} | +| *names* | Folder names. | list(string) | | [] | ## Outputs diff --git a/modules/gcs/README.md b/modules/gcs/README.md index 4304a531e9..40c3727713 100644 --- a/modules/gcs/README.md +++ b/modules/gcs/README.md @@ -3,19 +3,19 @@ ## Variables -| name | description | type | required | -|---|---|:---: |:---:| -| names | Bucket name suffixes. | `list(string)` | ✓ -| project_id | Bucket project id. | `string` | ✓ -| *bucket_policy_only* | Optional map to disable object ACLS keyed by name, defaults to true. | `map(bool)` | -| *force_destroy* | Optional map to set force destroy keyed by name, defaults to false. | `map(bool)` | -| *iam_members* | List of IAM members keyed by name and role. | `map(map(list(string)))` | -| *iam_roles* | List of IAM roles keyed by name. | `map(list(string))` | -| *labels* | Labels to be attached to all buckets. | `map(string)` | -| *location* | Bucket location. | `string` | -| *prefix* | Prefix used to generate the bucket name. | `string` | -| *storage_class* | Bucket storage class. | `string` | -| *versioning* | Optional map to set versioning keyed by name, defaults to false. | `map(bool)` | +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| names | Bucket name suffixes. | list(string) | ✓ | | +| project_id | Bucket project id. | string | ✓ | | +| *bucket_policy_only* | Optional map to disable object ACLS keyed by name, defaults to true. | map(bool) | | {} | +| *force_destroy* | Optional map to set force destroy keyed by name, defaults to false. | map(bool) | | {} | +| *iam_members* | IAM members keyed by bucket name and role. | map(map(list(string))) | | {} | +| *iam_roles* | IAM roles keyed by bucket name. | map(list(string)) | | {} | +| *labels* | Labels to be attached to all buckets. | map(string) | | {} | +| *location* | Bucket location. | string | | EU | +| *prefix* | Prefix used to generate the bucket name. | string | | | +| *storage_class* | Bucket storage class. | string | | MULTI_REGIONAL | +| *versioning* | Optional map to set versioning keyed by name, defaults to false. | map(bool) | | {} | ## Outputs diff --git a/modules/gke-cluster/README.md b/modules/gke-cluster/README.md index 47b620f372..5bb7a9b0c5 100644 --- a/modules/gke-cluster/README.md +++ b/modules/gke-cluster/README.md @@ -9,39 +9,39 @@ TODO(ludoo): add example ## Variables -| name | description | type | required | -|---|---|:---: |:---:| -| location | Cluster zone or region. | `string` | ✓ -| name | Cluster name. | `string` | ✓ -| network | Name or self link of the VPC used for the cluster. Use the self link for Shared VPC. | `string` | ✓ -| project_id | Cluster project id. | `string` | ✓ -| secondary_range_pods | Subnet secondary range name used for pods. | `string` | ✓ -| secondary_range_services | Subnet secondary range name used for services. | `string` | ✓ -| subnetwork | VPC subnetwork name or self link. | `string` | ✓ -| *addons* | Addons enabled in the cluster (true means enabled). | `object({...})` | -| *authenticator_security_group* | RBAC security group for Google Groups for GKE, format is gke-security-groups@yourdomain.com. | `string` | -| *cluster_autoscaling* | Enable and configure limits for Node Auto-Provisioning with Cluster Autoscaler. | `object({...})` | -| *database_encryption* | Enable and configure GKE application-layer secrets encryption. | `object({...})` | -| *default_max_pods_per_node* | Maximum number of pods per node in this cluster. | `number` | -| *description* | Cluster description. | `string` | -| *enable_binary_authorization* | Enable Google Binary Authorization. | `bool` | -| *enable_intranode_visibility* | Enable intra-node visibility to make same node pod to pod traffic visible. | `bool` | -| *enable_shielded_nodes* | Enable Shielded Nodes features on all nodes in this cluster. | `bool` | -| *enable_tpu* | Enable Cloud TPU resources in this cluster. | `bool` | -| *labels* | Cluster resource labels. | `map(string)` | -| *logging_service* | Logging service (disable with an empty string). | `string` | -| *maintenance_start_time* | Maintenance start time in RFC3339 format 'HH:MM', where HH is [00-23] and MM is [00-59] GMT. | `string` | -| *master_authorized_ranges* | External Ip address ranges that can access the Kubernetes cluster master through HTTPS. | `map(string)` | -| *min_master_version* | Minimum version of the master, defaults to the version of the most recent official release. | `string` | -| *monitoring_service* | Monitoring service (disable with an empty string). | `string` | -| *node_locations* | Zones in which the cluster's nodes are located. | `list(string)` | -| *pod_security_policy* | Enable the PodSecurityPolicy feature. | `bool` | -| *private_cluster* | Enable private cluster. | `bool` | -| *private_cluster_config* | Private cluster configuration. | `object({...})` | -| *release_channel* | Release channel for GKE upgrades. | `string` | -| *resource_usage_export_config* | Configure the ResourceUsageExportConfig feature. | `object({...})` | -| *vertical_pod_autoscaling* | Enable the Vertical Pod Autoscaling feature. | `bool` | -| *workload_identity* | Enable the Workload Identity feature. | `bool` | +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| location | Cluster zone or region. | string | ✓ | | +| name | Cluster name. | string | ✓ | | +| network | Name or self link of the VPC used for the cluster. Use the self link for Shared VPC. | string | ✓ | | +| project_id | Cluster project id. | string | ✓ | | +| secondary_range_pods | Subnet secondary range name used for pods. | string | ✓ | | +| secondary_range_services | Subnet secondary range name used for services. | string | ✓ | | +| subnetwork | VPC subnetwork name or self link. | string | ✓ | | +| *addons* | Addons enabled in the cluster (true means enabled). | object({...}) | | ... | +| *authenticator_security_group* | RBAC security group for Google Groups for GKE, format is gke-security-groups@yourdomain.com. | string | | null | +| *cluster_autoscaling* | Enable and configure limits for Node Auto-Provisioning with Cluster Autoscaler. | object({...}) | | ... | +| *database_encryption* | Enable and configure GKE application-layer secrets encryption. | object({...}) | | ... | +| *default_max_pods_per_node* | Maximum number of pods per node in this cluster. | number | | 110 | +| *description* | Cluster description. | string | | null | +| *enable_binary_authorization* | Enable Google Binary Authorization. | bool | | null | +| *enable_intranode_visibility* | Enable intra-node visibility to make same node pod to pod traffic visible. | bool | | null | +| *enable_shielded_nodes* | Enable Shielded Nodes features on all nodes in this cluster. | bool | | null | +| *enable_tpu* | Enable Cloud TPU resources in this cluster. | bool | | null | +| *labels* | Cluster resource labels. | map(string) | | null | +| *logging_service* | Logging service (disable with an empty string). | string | | logging.googleapis.com/kubernetes | +| *maintenance_start_time* | Maintenance start time in RFC3339 format 'HH:MM', where HH is [00-23] and MM is [00-59] GMT. | string | | 03:00 | +| *master_authorized_ranges* | External Ip address ranges that can access the Kubernetes cluster master through HTTPS. | map(string) | | {} | +| *min_master_version* | Minimum version of the master, defaults to the version of the most recent official release. | string | | null | +| *monitoring_service* | Monitoring service (disable with an empty string). | string | | monitoring.googleapis.com/kubernetes | +| *node_locations* | Zones in which the cluster's nodes are located. | list(string) | | [] | +| *pod_security_policy* | Enable the PodSecurityPolicy feature. | bool | | null | +| *private_cluster* | Enable private cluster. | bool | | false | +| *private_cluster_config* | Private cluster configuration. | object({...}) | | ... | +| *release_channel* | Release channel for GKE upgrades. | string | | null | +| *resource_usage_export_config* | Configure the ResourceUsageExportConfig feature. | object({...}) | | ... | +| *vertical_pod_autoscaling* | Enable the Vertical Pod Autoscaling feature. | bool | | null | +| *workload_identity* | Enable the Workload Identity feature. | bool | | true | ## Outputs diff --git a/modules/iam-service-accounts/README.md b/modules/iam-service-accounts/README.md index 372398f991..bd8b834593 100644 --- a/modules/iam-service-accounts/README.md +++ b/modules/iam-service-accounts/README.md @@ -15,16 +15,16 @@ The resources/services/activations/deletions that this module will create/trigge ## Variables -| name | description | type | required | -|---|---|:---: |:---:| -| project_id | Project id where service account will be created. | `string` | ✓ -| *generate_keys* | Generate keys for service accounts. | `bool` | -| *iam_billing_roles* | Project roles applied to all service accounts, by billing account id. | `map(list(string))` | -| *iam_folder_roles* | Project roles applied to all service accounts, by folder id. | `map(list(string))` | -| *iam_organization_roles* | Project roles applied to all service accounts, by organization id. | `map(list(string))` | -| *iam_project_roles* | Project roles applied to all service accounts, by project id. | `map(list(string))` | -| *names* | Names of the service accounts to create. | `list(string)` | -| *prefix* | Prefix applied to service account names. | `string` | +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| project_id | Project id where service account will be created. | string | ✓ | | +| *generate_keys* | Generate keys for service accounts. | bool | | false | +| *iam_billing_roles* | Project roles applied to all service accounts, by billing account id. | map(list(string)) | | {} | +| *iam_folder_roles* | Project roles applied to all service accounts, by folder id. | map(list(string)) | | {} | +| *iam_organization_roles* | Project roles applied to all service accounts, by organization id. | map(list(string)) | | {} | +| *iam_project_roles* | Project roles applied to all service accounts, by project id. | map(list(string)) | | {} | +| *names* | Names of the service accounts to create. | list(string) | | [] | +| *prefix* | Prefix applied to service account names. | string | | | ## Outputs diff --git a/modules/kms/README.md b/modules/kms/README.md index 46a8807e04..6530c4da1f 100644 --- a/modules/kms/README.md +++ b/modules/kms/README.md @@ -11,16 +11,16 @@ The resources/services/activations/deletions that this module will create/trigge ## Variables -| name | description | type | required | -|---|---|:---: |:---:| -| keyring | Keyring name. | `string` | ✓ -| location | Location for the keyring. | `string` | ✓ -| project_id | Project id where the keyring will be created. | `string` | ✓ -| *iam_members* | IAM members keyed by key name and role. | `map(map(list(string)))` | -| *iam_roles* | IAM roles keyed by key name. | `map(list(string))` | -| *key_attributes* | Optional key attributes per key. | `map(object({...}))` | -| *key_defaults* | Key attribute defaults. | `object({...})` | -| *keys* | Key names. | `list(string)` | +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| keyring | Keyring name. | string | ✓ | | +| location | Location for the keyring. | string | ✓ | | +| project_id | Project id where the keyring will be created. | string | ✓ | | +| *iam_members* | IAM members keyed by key name and role. | map(map(list(string))) | | {} | +| *iam_roles* | IAM roles keyed by key name. | map(list(string)) | | {} | +| *key_attributes* | Optional key attributes per key. | map(object({...})) | | {} | +| *key_defaults* | Key attribute defaults. | object({...}) | | ... | +| *keys* | Key names. | list(string) | | [] | ## Outputs diff --git a/modules/net-address/README.md b/modules/net-address/README.md index 4310d0b703..8da8f8c3bf 100644 --- a/modules/net-address/README.md +++ b/modules/net-address/README.md @@ -3,14 +3,14 @@ ## Variables -| name | description | type | required | -|---|---|:---: |:---:| -| project_id | Project where the addresses will be created. | `string` | ✓ -| *external_addresses* | Map of external address regions, keyed by name. | `map(string)` | -| *global_addresses* | List of global addresses to create. | `list(string)` | -| *internal_address_addresses* | Optional explicit addresses for internal addresses, keyed by name. | `map(string)` | -| *internal_address_tiers* | Optional network tiers for internal addresses, keyed by name. | `map(string)` | -| *internal_addresses* | Map of internal addresses to create, keyed by name. | `map(object({...}))` | +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| project_id | Project where the addresses will be created. | string | ✓ | | +| *external_addresses* | Map of external address regions, keyed by name. | map(string) | | {} | +| *global_addresses* | List of global addresses to create. | list(string) | | [] | +| *internal_address_addresses* | Optional explicit addresses for internal addresses, keyed by name. | map(string) | | {} | +| *internal_address_tiers* | Optional network tiers for internal addresses, keyed by name. | map(string) | | {} | +| *internal_addresses* | Map of internal addresses to create, keyed by name. | map(object({...})) | | {} | ## Outputs diff --git a/modules/net-cloudnat/README.md b/modules/net-cloudnat/README.md index 8548ab6351..564b187034 100644 --- a/modules/net-cloudnat/README.md +++ b/modules/net-cloudnat/README.md @@ -3,21 +3,19 @@ ## Variables -| name | description | type | required | -|---|---|:---: |:---:| -| project_id | Project where resources will be created. | `string` | ✓ -| region | Region where resources will be created. | `string` | ✓ -| *addresses* | Optional list of external address self links. | `list(string)` | -| *config_min_ports_per_vm* | Minimum number of ports allocated to a VM from this NAT config. | `number` | -| *config_source_subnets* | Subnetwork configuration, valid values are ALL_SUBNETWORKS_ALL_IP_RANGES, ALL_SUBNETWORKS_ALL_PRIMARY_IP_RANGES, LIST_OF_SUBNETWORKS. | `string` | -| *config_timeouts* | Timeout configurations. | `object({...})` | -| *create_router* | Create router for Cloud NAT instead of using existing one. | `bool` | -| *name* | Name of the Cloud NAT resource. | `string` | -| *prefix* | Optional prefix that will be prepended to resource names. | `string` | -| *router_asn* | Router ASN used for auto-created router. | `number` | -| *router_name* | Name of the existing or auto-created router. | `string` | -| *router_network* | Name of the VPC used for auto-created router. | `string` | -| *subnetworks* | Subnetworks to NAT, only used when config_source_subnets equals LIST_OF_SUBNETWORKS. | `list(object({...}))` | +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| name | Name of the Cloud NAT resource. | string | ✓ | | +| project_id | Project where resources will be created. | string | ✓ | | +| region | Region where resources will be created. | string | ✓ | | +| *addresses* | Optional list of external address self links. | list(string) | | [] | +| *config_min_ports_per_vm* | Minimum number of ports allocated to a VM from this NAT config. | number | | 64 | +| *config_source_subnets* | Subnetwork configuration (ALL_SUBNETWORKS_ALL_IP_RANGES, ALL_SUBNETWORKS_ALL_PRIMARY_IP_RANGES, LIST_OF_SUBNETWORKS). | string | | ALL_SUBNETWORKS_ALL_IP_RANGES | +| *config_timeouts* | Timeout configurations. | object({...}) | | ... | +| *router_asn* | Router ASN used for auto-created router. | number | | 64514 | +| *router_name* | Router name, leave blank to create. | string | | | +| *router_network* | Name of the VPC used for auto-created router. | string | | | +| *subnetworks* | Subnetworks to NAT, only used when config_source_subnets equals LIST_OF_SUBNETWORKS. | list(object({...})) | | [] | ## Outputs diff --git a/modules/net-vpc-firewall/README.md b/modules/net-vpc-firewall/README.md index e70e6f4f08..fb307a75b4 100644 --- a/modules/net-vpc-firewall/README.md +++ b/modules/net-vpc-firewall/README.md @@ -70,16 +70,16 @@ module "net-firewall" { ## Variables -| name | description | type | required | -|---|---|:---: |:---:| -| network | Name of the network this set of firewall rules applies to. | `string` | ✓ -| project_id | Project id of the project that holds the network. | `string` | ✓ -| *admin_ranges* | IP CIDR ranges that have complete access to all subnets. | `list(string)` | -| *admin_ranges_enabled* | Enable admin ranges-based rules. | `bool` | -| *custom_rules* | List of custom rule definitions (refer to variables file for syntax). | `map(object({...}))` | -| *http_source_ranges* | List of IP CIDR ranges for tag-based HTTP rule, defaults to 0.0.0.0/0. | `list(string)` | -| *https_source_ranges* | List of IP CIDR ranges for tag-based HTTPS rule, defaults to 0.0.0.0/0. | `list(string)` | -| *ssh_source_ranges* | List of IP CIDR ranges for tag-based SSH rule, defaults to 0.0.0.0/0. | `list(string)` | +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| network | Name of the network this set of firewall rules applies to. | string | ✓ | | +| project_id | Project id of the project that holds the network. | string | ✓ | | +| *admin_ranges* | IP CIDR ranges that have complete access to all subnets. | list(string) | | [] | +| *admin_ranges_enabled* | Enable admin ranges-based rules. | bool | | false | +| *custom_rules* | List of custom rule definitions (refer to variables file for syntax). | map(object({...})) | | {} | +| *http_source_ranges* | List of IP CIDR ranges for tag-based HTTP rule, defaults to 0.0.0.0/0. | list(string) | | ["0.0.0.0/0"] | +| *https_source_ranges* | List of IP CIDR ranges for tag-based HTTPS rule, defaults to 0.0.0.0/0. | list(string) | | ["0.0.0.0/0"] | +| *ssh_source_ranges* | List of IP CIDR ranges for tag-based SSH rule, defaults to 0.0.0.0/0. | list(string) | | ["0.0.0.0/0"] | ## Outputs diff --git a/modules/net-vpc/README.md b/modules/net-vpc/README.md index c3e57b30d7..9faeb53b2b 100644 --- a/modules/net-vpc/README.md +++ b/modules/net-vpc/README.md @@ -9,23 +9,24 @@ TODO(ludoo): add example ## Variables -| name | description | type | required | -|---|---|:---: |:---:| -| name | The name of the network being created | `string` | ✓ -| project_id | The ID of the project where this VPC will be created | `string` | ✓ -| *auto_create_subnetworks* | Set to true to create an auto mode subnet, defaults to custom mode. | `bool` | -| *description* | An optional description of this resource (triggers recreation on change). | `string` | -| *iam_members* | List of IAM members keyed by subnet and role. | `map(map(list(string)))` | -| *iam_roles* | List of IAM roles keyed by subnet. | `map(list(string))` | -| *log_config_defaults* | Default configuration for flow logs when enabled. | `object({...})` | -| *log_configs* | Map of per-subnet optional configurations for flow logs when enabled. | `map(map(string))` | -| *routing_mode* | The network routing mode (default 'GLOBAL') | `string` | -| *shared_vpc_host* | Makes this project a Shared VPC host if 'true' (default 'false') | `bool` | -| *shared_vpc_service_projects* | Shared VPC service projects to register with this host | `list(string)` | -| *subnet_descriptions* | Optional map of subnet descriptions, keyed by subnet name. | `map(string)` | -| *subnet_flow_logs* | Optional map of boolean to control flow logs (default is disabled), keyed by subnet name. | `map(bool)` | -| *subnet_private_access* | Optional map of boolean to control private Google access (default is enabled), keyed by subnet name. | `map(bool)` | -| *subnets* | The list of subnets being created | `map(object({...}))` | +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| name | The name of the network being created | string | ✓ | | +| project_id | The ID of the project where this VPC will be created | string | ✓ | | +| *auto_create_subnetworks* | Set to true to create an auto mode subnet, defaults to custom mode. | bool | | false | +| *description* | An optional description of this resource (triggers recreation on change). | string | | Terraform-managed. | +| *iam_members* | List of IAM members keyed by subnet and role. | map(map(list(string))) | | {} | +| *iam_roles* | List of IAM roles keyed by subnet. | map(list(string)) | | {} | +| *log_config_defaults* | Default configuration for flow logs when enabled. | object({...}) | | ... | +| *log_configs* | Map of per-subnet optional configurations for flow logs when enabled. | map(map(string)) | | {} | +| *peering_config* | VPC peering configuration. | object({...}) | | null | +| *routing_mode* | The network routing mode (default 'GLOBAL') | string | | GLOBAL | +| *shared_vpc_host* | Makes this project a Shared VPC host if 'true' (default 'false') | bool | | false | +| *shared_vpc_service_projects* | Shared VPC service projects to register with this host | list(string) | | [] | +| *subnet_descriptions* | Optional map of subnet descriptions, keyed by subnet name. | map(string) | | {} | +| *subnet_flow_logs* | Optional map of boolean to control flow logs (default is disabled), keyed by subnet name. | map(bool) | | {} | +| *subnet_private_access* | Optional map of boolean to control private Google access (default is enabled), keyed by subnet name. | map(bool) | | {} | +| *subnets* | The list of subnets being created | map(object({...})) | | {} | ## Outputs diff --git a/modules/net-vpn-dynamic/README.md b/modules/net-vpn-dynamic/README.md index f8373b7db1..508c819780 100644 --- a/modules/net-vpn-dynamic/README.md +++ b/modules/net-vpn-dynamic/README.md @@ -3,18 +3,18 @@ ## Variables -| name | description | type | required | -|---|---|:---: |:---:| -| name | VPN gateway name, and prefix used for dependent resources. | `string` | ✓ -| network | VPC used for the gateway and routes. | `string` | ✓ -| project_id | Project where resources will be created. | `string` | ✓ -| region | Region used for resources. | `string` | ✓ -| *gateway_address* | Optional address assigned to the VPN, leave blank to create one. | `string` | -| *route_priority* | Route priority, defaults to 1000. | `number` | -| *router_advertise_config* | Router custom advertisement configuration, ip_ranges is a map of address ranges and descriptions. | `object({...})` | -| *router_asn* | Router ASN used for auto-created router. | `number` | -| *router_name* | Name of router, leave blank to create one. | `string` | -| *tunnels* | VPN tunnel configurations, bgp_peer_options is usually null. | `map(object({...}))` | +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| name | VPN gateway name, and prefix used for dependent resources. | string | ✓ | | +| network | VPC used for the gateway and routes. | string | ✓ | | +| project_id | Project where resources will be created. | string | ✓ | | +| region | Region used for resources. | string | ✓ | | +| *gateway_address* | Optional address assigned to the VPN, leave blank to create one. | string | | | +| *route_priority* | Route priority, defaults to 1000. | number | | 1000 | +| *router_advertise_config* | Router custom advertisement configuration, ip_ranges is a map of address ranges and descriptions. | object({...}) | | null | +| *router_asn* | Router ASN used for auto-created router. | number | | 64514 | +| *router_name* | Name of router, leave blank to create one. | string | | | +| *tunnels* | VPN tunnel configurations, bgp_peer_options is usually null. | map(object({...})) | | {} | ## Outputs diff --git a/modules/net-vpn-static/README.md b/modules/net-vpn-static/README.md index 9fa038b383..78ef521858 100644 --- a/modules/net-vpn-static/README.md +++ b/modules/net-vpn-static/README.md @@ -3,17 +3,16 @@ ## Variables -| name | description | type | required | -|---|---|:---: |:---:| -| name | VPN gateway name, and prefix used for dependent resources. | `string` | ✓ -| network | VPC used for the gateway and routes. | `string` | ✓ -| project_id | Project where resources will be created. | `string` | ✓ -| region | Region used for resources. | `string` | ✓ -| *create_address* | Create gateway address resource instead of using external one, defaults to true. | `bool` | -| *gateway_address* | Optional address assigned to the VPN, used if create_address is false. | `string` | -| *remote_ranges* | Remote IP CIDR ranges. | `list(string)` | -| *route_priority* | Route priority, defaults to 1000. | `number` | -| *tunnels* | VPN tunnel configurations. | `map(object({...}))` | +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| name | VPN gateway name, and prefix used for dependent resources. | string | ✓ | | +| network | VPC used for the gateway and routes. | string | ✓ | | +| project_id | Project where resources will be created. | string | ✓ | | +| region | Region used for resources. | string | ✓ | | +| *gateway_address* | Optional address assigned to the VPN, leave blank to create one. | string | | | +| *remote_ranges* | Remote IP CIDR ranges. | list(string) | | [] | +| *route_priority* | Route priority, defaults to 1000. | number | | 1000 | +| *tunnels* | VPN tunnel configurations. | map(object({...})) | | {} | ## Outputs diff --git a/modules/project/README.md b/modules/project/README.md index 62903a1b1c..9d6724d09c 100644 --- a/modules/project/README.md +++ b/modules/project/README.md @@ -14,25 +14,25 @@ The resources/services/activations/deletions that this module will create/trigge ## Variables -| name | description | type | required | -|---|---|:---: |:---:| -| name | Project name and id suffix. | `string` | ✓ -| parent | The resource name of the parent Folder or Organization. Must be of the form folders/folder_id or organizations/org_id. | `string` | ✓ -| prefix | Prefix used to generate project id and name. | `string` | ✓ -| *auto_create_network* | Whether to create the default network for the project | `bool` | -| *billing_account* | Billing account id. | `string` | -| *custom_roles* | Map of role name => list of permissions to create in this project. | `map(list(string))` | -| *iam_members* | Map of member lists used to set authoritative bindings, keyed by role. | `map(list(string))` | -| *iam_nonauth_members* | Map of member lists used to set non authoritative bindings, keyed by role. | `map(list(string))` | -| *iam_nonauth_roles* | List of roles used to set non authoritative bindings. | `list(string)` | -| *iam_roles* | List of roles used to set authoritative bindings. | `list(string)` | -| *labels* | Resource labels. | `map(string)` | -| *lien_reason* | If non-empty, creates a project lien with this description. | `string` | -| *oslogin* | Enable OS Login. | `bool` | -| *oslogin_admins* | List of IAM-style identities that will be granted roles necessary for OS Login administrators. | `list(string)` | -| *oslogin_users* | List of IAM-style identities that will be granted roles necessary for OS Login users. | `list(string)` | -| *owners* | IAM-style identities that will be granted non-authoritative viewer role. | `list(string)` | -| *services* | Service APIs to enable. | `list(string)` | +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| name | Project name and id suffix. | string | ✓ | | +| parent | The resource name of the parent Folder or Organization. Must be of the form folders/folder_id or organizations/org_id. | string | ✓ | | +| prefix | Prefix used to generate project id and name. | string | ✓ | | +| *auto_create_network* | Whether to create the default network for the project | bool | | false | +| *billing_account* | Billing account id. | string | | | +| *custom_roles* | Map of role name => list of permissions to create in this project. | map(list(string)) | | {} | +| *iam_members* | Map of member lists used to set authoritative bindings, keyed by role. | map(list(string)) | | {} | +| *iam_nonauth_members* | Map of member lists used to set non authoritative bindings, keyed by role. | map(list(string)) | | {} | +| *iam_nonauth_roles* | List of roles used to set non authoritative bindings. | list(string) | | [] | +| *iam_roles* | List of roles used to set authoritative bindings. | list(string) | | [] | +| *labels* | Resource labels. | map(string) | | {} | +| *lien_reason* | If non-empty, creates a project lien with this description. | string | | | +| *oslogin* | Enable OS Login. | bool | | false | +| *oslogin_admins* | List of IAM-style identities that will be granted roles necessary for OS Login administrators. | list(string) | | [] | +| *oslogin_users* | List of IAM-style identities that will be granted roles necessary for OS Login users. | list(string) | | [] | +| *owners* | IAM-style identities that will be granted non-authoritative viewer role. | list(string) | | [] | +| *services* | Service APIs to enable. | list(string) | | [] | ## Outputs From b38f086e3ffa4e154ee9d9d7089230a0154745c7 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Fri, 20 Dec 2019 15:22:45 +0100 Subject: [PATCH 064/106] tfdoc: improve variable default rendering --- tools/tfdoc/tfdoc.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/tfdoc/tfdoc.py b/tools/tfdoc/tfdoc.py index 06f17d344e..2b95c47f97 100755 --- a/tools/tfdoc/tfdoc.py +++ b/tools/tfdoc/tfdoc.py @@ -172,23 +172,23 @@ def format_variables(variables, required_first=True): yield '|---|---|:---: |:---:|:---:|' row = ( '| {name} | {description} | {type} ' - '| {required} | {default} |' + '| {required} | {default} |' ) for v in variables: default = default_spec = type_spec = '' if not v.required: + default = '{}' if '\n' in v.default: - default = '...' - default_spec = _escape(v.default) + default.format(_escape(v.default), '...') else: - default = v.default or '' + default = default.format('', v.default or '') if v.type and '(' in v.type: type_spec = _escape(v.type) yield row.format( name=v.name if v.required else '*%s*' % v.name, description=v.description, required='✓' if v.required else '', type=format_type(v.type), type_spec=type_spec, - default=default, default_spec=default_spec + default=default ) From cce1e761e353db0e0fcf2acaa36af1eca39bfba6 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Fri, 20 Dec 2019 15:22:50 +0100 Subject: [PATCH 065/106] modules: update README variables and outputs --- modules/dns/README.md | 6 +++--- modules/folder/README.md | 2 +- modules/gcs/README.md | 4 ++-- modules/gke-cluster/README.md | 24 ++++++++++++------------ modules/iam-service-accounts/README.md | 2 +- modules/kms/README.md | 8 ++++---- modules/net-address/README.md | 2 +- modules/net-cloudnat/README.md | 8 ++++---- modules/net-vpc-firewall/README.md | 4 ++-- modules/net-vpc/README.md | 6 +++--- modules/net-vpn-dynamic/README.md | 8 ++++---- modules/net-vpn-static/README.md | 8 ++++---- modules/project/README.md | 6 +++--- 13 files changed, 44 insertions(+), 44 deletions(-) diff --git a/modules/dns/README.md b/modules/dns/README.md index 6658190223..c2bf5597ce 100644 --- a/modules/dns/README.md +++ b/modules/dns/README.md @@ -32,9 +32,9 @@ module "dns-private-zone" { | name | description | type | required | default | |---|---|:---: |:---:|:---:| -| domain | Zone domain, must end with a period. | string | ✓ | | -| name | Zone name, must be unique within the project. | string | ✓ | | -| project_id | Project id for the zone. | string | ✓ | | +| domain | Zone domain, must end with a period. | string | ✓ | | +| name | Zone name, must be unique within the project. | string | ✓ | | +| project_id | Project id for the zone. | string | ✓ | | | *client_networks* | List of VPC self links that can see this zone. | list(string) | | [] | | *default_key_specs_key* | DNSSEC default key signing specifications: algorithm, key_length, key_type, kind. | any | | {} | | *default_key_specs_zone* | DNSSEC default zone signing specifications: algorithm, key_length, key_type, kind. | any | | {} | diff --git a/modules/folder/README.md b/modules/folder/README.md index c46c27e5ad..0f61024e10 100644 --- a/modules/folder/README.md +++ b/modules/folder/README.md @@ -5,7 +5,7 @@ | name | description | type | required | default | |---|---|:---: |:---:|:---:| -| parent | Parent in folders/folder_id or organizations/org_id format. | string | ✓ | | +| parent | Parent in folders/folder_id or organizations/org_id format. | string | ✓ | | | *iam_members* | List of IAM members keyed by folder name and role. | map(map(list(string))) | | {} | | *iam_roles* | List of IAM roles keyed by folder name. | map(list(string)) | | {} | | *names* | Folder names. | list(string) | | [] | diff --git a/modules/gcs/README.md b/modules/gcs/README.md index 40c3727713..484b5bdaa0 100644 --- a/modules/gcs/README.md +++ b/modules/gcs/README.md @@ -5,8 +5,8 @@ | name | description | type | required | default | |---|---|:---: |:---:|:---:| -| names | Bucket name suffixes. | list(string) | ✓ | | -| project_id | Bucket project id. | string | ✓ | | +| names | Bucket name suffixes. | list(string) | ✓ | | +| project_id | Bucket project id. | string | ✓ | | | *bucket_policy_only* | Optional map to disable object ACLS keyed by name, defaults to true. | map(bool) | | {} | | *force_destroy* | Optional map to set force destroy keyed by name, defaults to false. | map(bool) | | {} | | *iam_members* | IAM members keyed by bucket name and role. | map(map(list(string))) | | {} | diff --git a/modules/gke-cluster/README.md b/modules/gke-cluster/README.md index 5bb7a9b0c5..e9d4b6d639 100644 --- a/modules/gke-cluster/README.md +++ b/modules/gke-cluster/README.md @@ -11,17 +11,17 @@ TODO(ludoo): add example | name | description | type | required | default | |---|---|:---: |:---:|:---:| -| location | Cluster zone or region. | string | ✓ | | -| name | Cluster name. | string | ✓ | | -| network | Name or self link of the VPC used for the cluster. Use the self link for Shared VPC. | string | ✓ | | -| project_id | Cluster project id. | string | ✓ | | -| secondary_range_pods | Subnet secondary range name used for pods. | string | ✓ | | -| secondary_range_services | Subnet secondary range name used for services. | string | ✓ | | -| subnetwork | VPC subnetwork name or self link. | string | ✓ | | -| *addons* | Addons enabled in the cluster (true means enabled). | object({...}) | | ... | +| location | Cluster zone or region. | string | ✓ | | +| name | Cluster name. | string | ✓ | | +| network | Name or self link of the VPC used for the cluster. Use the self link for Shared VPC. | string | ✓ | | +| project_id | Cluster project id. | string | ✓ | | +| secondary_range_pods | Subnet secondary range name used for pods. | string | ✓ | | +| secondary_range_services | Subnet secondary range name used for services. | string | ✓ | | +| subnetwork | VPC subnetwork name or self link. | string | ✓ | | +| *addons* | Addons enabled in the cluster (true means enabled). | object({...}) | | {} | | *authenticator_security_group* | RBAC security group for Google Groups for GKE, format is gke-security-groups@yourdomain.com. | string | | null | -| *cluster_autoscaling* | Enable and configure limits for Node Auto-Provisioning with Cluster Autoscaler. | object({...}) | | ... | -| *database_encryption* | Enable and configure GKE application-layer secrets encryption. | object({...}) | | ... | +| *cluster_autoscaling* | Enable and configure limits for Node Auto-Provisioning with Cluster Autoscaler. | object({...}) | | {} | +| *database_encryption* | Enable and configure GKE application-layer secrets encryption. | object({...}) | | {} | | *default_max_pods_per_node* | Maximum number of pods per node in this cluster. | number | | 110 | | *description* | Cluster description. | string | | null | | *enable_binary_authorization* | Enable Google Binary Authorization. | bool | | null | @@ -37,9 +37,9 @@ TODO(ludoo): add example | *node_locations* | Zones in which the cluster's nodes are located. | list(string) | | [] | | *pod_security_policy* | Enable the PodSecurityPolicy feature. | bool | | null | | *private_cluster* | Enable private cluster. | bool | | false | -| *private_cluster_config* | Private cluster configuration. | object({...}) | | ... | +| *private_cluster_config* | Private cluster configuration. | object({...}) | | {} | | *release_channel* | Release channel for GKE upgrades. | string | | null | -| *resource_usage_export_config* | Configure the ResourceUsageExportConfig feature. | object({...}) | | ... | +| *resource_usage_export_config* | Configure the ResourceUsageExportConfig feature. | object({...}) | | {} | | *vertical_pod_autoscaling* | Enable the Vertical Pod Autoscaling feature. | bool | | null | | *workload_identity* | Enable the Workload Identity feature. | bool | | true | diff --git a/modules/iam-service-accounts/README.md b/modules/iam-service-accounts/README.md index bd8b834593..0d9a024d62 100644 --- a/modules/iam-service-accounts/README.md +++ b/modules/iam-service-accounts/README.md @@ -17,7 +17,7 @@ The resources/services/activations/deletions that this module will create/trigge | name | description | type | required | default | |---|---|:---: |:---:|:---:| -| project_id | Project id where service account will be created. | string | ✓ | | +| project_id | Project id where service account will be created. | string | ✓ | | | *generate_keys* | Generate keys for service accounts. | bool | | false | | *iam_billing_roles* | Project roles applied to all service accounts, by billing account id. | map(list(string)) | | {} | | *iam_folder_roles* | Project roles applied to all service accounts, by folder id. | map(list(string)) | | {} | diff --git a/modules/kms/README.md b/modules/kms/README.md index 6530c4da1f..3f78989872 100644 --- a/modules/kms/README.md +++ b/modules/kms/README.md @@ -13,13 +13,13 @@ The resources/services/activations/deletions that this module will create/trigge | name | description | type | required | default | |---|---|:---: |:---:|:---:| -| keyring | Keyring name. | string | ✓ | | -| location | Location for the keyring. | string | ✓ | | -| project_id | Project id where the keyring will be created. | string | ✓ | | +| keyring | Keyring name. | string | ✓ | | +| location | Location for the keyring. | string | ✓ | | +| project_id | Project id where the keyring will be created. | string | ✓ | | | *iam_members* | IAM members keyed by key name and role. | map(map(list(string))) | | {} | | *iam_roles* | IAM roles keyed by key name. | map(list(string)) | | {} | | *key_attributes* | Optional key attributes per key. | map(object({...})) | | {} | -| *key_defaults* | Key attribute defaults. | object({...}) | | ... | +| *key_defaults* | Key attribute defaults. | object({...}) | | {} | | *keys* | Key names. | list(string) | | [] | ## Outputs diff --git a/modules/net-address/README.md b/modules/net-address/README.md index 8da8f8c3bf..0d35b759e3 100644 --- a/modules/net-address/README.md +++ b/modules/net-address/README.md @@ -5,7 +5,7 @@ | name | description | type | required | default | |---|---|:---: |:---:|:---:| -| project_id | Project where the addresses will be created. | string | ✓ | | +| project_id | Project where the addresses will be created. | string | ✓ | | | *external_addresses* | Map of external address regions, keyed by name. | map(string) | | {} | | *global_addresses* | List of global addresses to create. | list(string) | | [] | | *internal_address_addresses* | Optional explicit addresses for internal addresses, keyed by name. | map(string) | | {} | diff --git a/modules/net-cloudnat/README.md b/modules/net-cloudnat/README.md index 564b187034..d639f4d729 100644 --- a/modules/net-cloudnat/README.md +++ b/modules/net-cloudnat/README.md @@ -5,13 +5,13 @@ | name | description | type | required | default | |---|---|:---: |:---:|:---:| -| name | Name of the Cloud NAT resource. | string | ✓ | | -| project_id | Project where resources will be created. | string | ✓ | | -| region | Region where resources will be created. | string | ✓ | | +| name | Name of the Cloud NAT resource. | string | ✓ | | +| project_id | Project where resources will be created. | string | ✓ | | +| region | Region where resources will be created. | string | ✓ | | | *addresses* | Optional list of external address self links. | list(string) | | [] | | *config_min_ports_per_vm* | Minimum number of ports allocated to a VM from this NAT config. | number | | 64 | | *config_source_subnets* | Subnetwork configuration (ALL_SUBNETWORKS_ALL_IP_RANGES, ALL_SUBNETWORKS_ALL_PRIMARY_IP_RANGES, LIST_OF_SUBNETWORKS). | string | | ALL_SUBNETWORKS_ALL_IP_RANGES | -| *config_timeouts* | Timeout configurations. | object({...}) | | ... | +| *config_timeouts* | Timeout configurations. | object({...}) | | {} | | *router_asn* | Router ASN used for auto-created router. | number | | 64514 | | *router_name* | Router name, leave blank to create. | string | | | | *router_network* | Name of the VPC used for auto-created router. | string | | | diff --git a/modules/net-vpc-firewall/README.md b/modules/net-vpc-firewall/README.md index fb307a75b4..c5bce167c4 100644 --- a/modules/net-vpc-firewall/README.md +++ b/modules/net-vpc-firewall/README.md @@ -72,8 +72,8 @@ module "net-firewall" { | name | description | type | required | default | |---|---|:---: |:---:|:---:| -| network | Name of the network this set of firewall rules applies to. | string | ✓ | | -| project_id | Project id of the project that holds the network. | string | ✓ | | +| network | Name of the network this set of firewall rules applies to. | string | ✓ | | +| project_id | Project id of the project that holds the network. | string | ✓ | | | *admin_ranges* | IP CIDR ranges that have complete access to all subnets. | list(string) | | [] | | *admin_ranges_enabled* | Enable admin ranges-based rules. | bool | | false | | *custom_rules* | List of custom rule definitions (refer to variables file for syntax). | map(object({...})) | | {} | diff --git a/modules/net-vpc/README.md b/modules/net-vpc/README.md index 9faeb53b2b..6306bd21fb 100644 --- a/modules/net-vpc/README.md +++ b/modules/net-vpc/README.md @@ -11,13 +11,13 @@ TODO(ludoo): add example | name | description | type | required | default | |---|---|:---: |:---:|:---:| -| name | The name of the network being created | string | ✓ | | -| project_id | The ID of the project where this VPC will be created | string | ✓ | | +| name | The name of the network being created | string | ✓ | | +| project_id | The ID of the project where this VPC will be created | string | ✓ | | | *auto_create_subnetworks* | Set to true to create an auto mode subnet, defaults to custom mode. | bool | | false | | *description* | An optional description of this resource (triggers recreation on change). | string | | Terraform-managed. | | *iam_members* | List of IAM members keyed by subnet and role. | map(map(list(string))) | | {} | | *iam_roles* | List of IAM roles keyed by subnet. | map(list(string)) | | {} | -| *log_config_defaults* | Default configuration for flow logs when enabled. | object({...}) | | ... | +| *log_config_defaults* | Default configuration for flow logs when enabled. | object({...}) | | {} | | *log_configs* | Map of per-subnet optional configurations for flow logs when enabled. | map(map(string)) | | {} | | *peering_config* | VPC peering configuration. | object({...}) | | null | | *routing_mode* | The network routing mode (default 'GLOBAL') | string | | GLOBAL | diff --git a/modules/net-vpn-dynamic/README.md b/modules/net-vpn-dynamic/README.md index 508c819780..73541592a1 100644 --- a/modules/net-vpn-dynamic/README.md +++ b/modules/net-vpn-dynamic/README.md @@ -5,10 +5,10 @@ | name | description | type | required | default | |---|---|:---: |:---:|:---:| -| name | VPN gateway name, and prefix used for dependent resources. | string | ✓ | | -| network | VPC used for the gateway and routes. | string | ✓ | | -| project_id | Project where resources will be created. | string | ✓ | | -| region | Region used for resources. | string | ✓ | | +| name | VPN gateway name, and prefix used for dependent resources. | string | ✓ | | +| network | VPC used for the gateway and routes. | string | ✓ | | +| project_id | Project where resources will be created. | string | ✓ | | +| region | Region used for resources. | string | ✓ | | | *gateway_address* | Optional address assigned to the VPN, leave blank to create one. | string | | | | *route_priority* | Route priority, defaults to 1000. | number | | 1000 | | *router_advertise_config* | Router custom advertisement configuration, ip_ranges is a map of address ranges and descriptions. | object({...}) | | null | diff --git a/modules/net-vpn-static/README.md b/modules/net-vpn-static/README.md index 78ef521858..12df1aec6c 100644 --- a/modules/net-vpn-static/README.md +++ b/modules/net-vpn-static/README.md @@ -5,10 +5,10 @@ | name | description | type | required | default | |---|---|:---: |:---:|:---:| -| name | VPN gateway name, and prefix used for dependent resources. | string | ✓ | | -| network | VPC used for the gateway and routes. | string | ✓ | | -| project_id | Project where resources will be created. | string | ✓ | | -| region | Region used for resources. | string | ✓ | | +| name | VPN gateway name, and prefix used for dependent resources. | string | ✓ | | +| network | VPC used for the gateway and routes. | string | ✓ | | +| project_id | Project where resources will be created. | string | ✓ | | +| region | Region used for resources. | string | ✓ | | | *gateway_address* | Optional address assigned to the VPN, leave blank to create one. | string | | | | *remote_ranges* | Remote IP CIDR ranges. | list(string) | | [] | | *route_priority* | Route priority, defaults to 1000. | number | | 1000 | diff --git a/modules/project/README.md b/modules/project/README.md index 9d6724d09c..4864bb7d40 100644 --- a/modules/project/README.md +++ b/modules/project/README.md @@ -16,9 +16,9 @@ The resources/services/activations/deletions that this module will create/trigge | name | description | type | required | default | |---|---|:---: |:---:|:---:| -| name | Project name and id suffix. | string | ✓ | | -| parent | The resource name of the parent Folder or Organization. Must be of the form folders/folder_id or organizations/org_id. | string | ✓ | | -| prefix | Prefix used to generate project id and name. | string | ✓ | | +| name | Project name and id suffix. | string | ✓ | | +| parent | The resource name of the parent Folder or Organization. Must be of the form folders/folder_id or organizations/org_id. | string | ✓ | | +| prefix | Prefix used to generate project id and name. | string | ✓ | | | *auto_create_network* | Whether to create the default network for the project | bool | | false | | *billing_account* | Billing account id. | string | | | | *custom_roles* | Map of role name => list of permissions to create in this project. | map(list(string)) | | {} | From 535457d9acd558c6f042db72c881e7eb44d12fe0 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sat, 21 Dec 2019 02:02:34 +0100 Subject: [PATCH 066/106] modules/net-vpc: minimal output refactoring --- modules/net-vpc/outputs.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/net-vpc/outputs.tf b/modules/net-vpc/outputs.tf index 6886286b56..64649135e8 100644 --- a/modules/net-vpc/outputs.tf +++ b/modules/net-vpc/outputs.tf @@ -42,9 +42,10 @@ output "project_id" { ] } +# TODO(ludoo): use input names as keys output "subnets" { description = "Subnet resources." - value = google_compute_subnetwork.subnetwork + value = { for k, v in google_compute_subnetwork.subnetwork : k => v } } output "subnet_ips" { From 36d46f45a47ff9b233f17e7a6427cb31c17f381a Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sat, 21 Dec 2019 02:04:02 +0100 Subject: [PATCH 067/106] modules/vm-cos: initial import, base resources working, no outputs --- modules/vm-cos/README.md | 31 +++++++ modules/vm-cos/main.tf | 169 ++++++++++++++++++++++++++++++++++++ modules/vm-cos/outputs.tf | 0 modules/vm-cos/variables.tf | 142 ++++++++++++++++++++++++++++++ 4 files changed, 342 insertions(+) create mode 100644 modules/vm-cos/README.md create mode 100644 modules/vm-cos/main.tf create mode 100644 modules/vm-cos/outputs.tf create mode 100644 modules/vm-cos/variables.tf diff --git a/modules/vm-cos/README.md b/modules/vm-cos/README.md new file mode 100644 index 0000000000..1890ae2804 --- /dev/null +++ b/modules/vm-cos/README.md @@ -0,0 +1,31 @@ +# Container Optimized VM module + + +## Variables + +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| cloud_config_path | None | string | ✓ | | +| name | None | string | ✓ | | +| network | None | string | ✓ | | +| project_id | None | string | ✓ | | +| region | None | string | ✓ | | +| subnetwork | None | string | ✓ | | +| zone | None | string | ✓ | | +| *addresses* | None | list(string) | | [] | +| *attached_disks* | Additional disks (only for non-MIG usage). | string # pd-standard pd-ssd | | {} | +| *boot_disk* | None | "pd-ssd" | | {} | +| *cloud_config_vars* | None | map(any) | | {} | +| *docker_log_driver* | None | string | | gcplogs | +| *instance_count* | None | number | | 1 | +| *instance_type* | None | string | | f1-micro | +| *nat* | None | object({...}) | | {} | +| *options* | None | object({...}) | | {} | +| *service_account* | None | string | | | +| *stackdriver* | None | object({...}) | | {} | +| *tags* | None | list(string) | | ["ssh"] | +| *use_instance_template* | None | bool | | false | + +## Outputs + + diff --git a/modules/vm-cos/main.tf b/modules/vm-cos/main.tf new file mode 100644 index 0000000000..7573e64bd3 --- /dev/null +++ b/modules/vm-cos/main.tf @@ -0,0 +1,169 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + addresses = ( + length(var.addresses) > 0 && ! var.use_instance_template + ? { for name, i in local.names : name => var.addresses[i] } + : {} + ) + disks = { + for pair in setproduct(values(local.names), values(var.attached_disks)) : + "${pair[0]}-${pair[1]}" => { name = pair[0], disk_name = pair[1] } + } + names = ( + var.use_instance_template + ? { "${var.name}" = 0 } + : { for i in range(0, var.instance_count) : "${var.name}-${i + 1}" => i } + ) + service_account = ( + var.service_account == "" + ? google_service_account.service_account[0].email + : var.service_account + ) +} + +data "template_file" "cloud_config" { + for_each = local.names + template = file(var.cloud_config_path) + vars = merge(var.cloud_config_vars, { + address = lookup(local.addresses, each.key, "") + attached_disks = join(",", keys(var.attached_disks)) + docker_log_driver = var.docker_log_driver + id = each.value + name = each.key + }) +} + +resource "google_compute_disk" "disks" { + for_each = local.disks + project = var.project_id + zone = var.zone + name = "${each.value.name}-${each.value.disk_name}" + type = var.attached_disks[each.value.disk_name].type + size = var.attached_disks[each.value.disk_name].size + # TODO(ludoo): labels +} + +resource "google_compute_instance" "default" { + for_each = var.use_instance_template ? {} : local.names + project = var.project_id + zone = var.zone + name = each.key + description = "Managed by the cos-workload Terraform module." + tags = var.tags + machine_type = var.instance_type + can_ip_forward = var.options.can_ip_forward + allow_stopping_for_update = var.options.allow_stopping_for_update + + dynamic attached_disk { + for_each = { + for k, v in values(local.disks) : + k => v.disk_name if v.name == each.key + } + iterator = config + content { + source = google_compute_disk.disks[config.key].name + device_name = config.value + } + } + + boot_disk { + initialize_params { + type = var.boot_disk.type + image = var.boot_disk.image + size = var.boot_disk.size + } + } + + metadata = { + user-data = data.template_file.cloud_config[each.key].rendered + google-logging-enabled = var.stackdriver.enable_logging ? true : null + google-monitoring-enabled = var.stackdriver.enable_monitoring ? true : null + } + + network_interface { + network = var.network + subnetwork = var.subnetwork + network_ip = lookup(local.addresses, each.key, null) + dynamic access_config { + for_each = var.nat.enabled ? [var.nat.address] : [] + content { + nat_ip = each.value + } + } + } + + scheduling { + automatic_restart = var.options.automatic_restart + on_host_maintenance = var.options.on_host_maintenance + } + + service_account { + email = local.service_account + scopes = ["https://www.googleapis.com/auth/cloud-platform"] + } + +} + +resource "google_compute_instance_template" "default" { + count = var.use_instance_template ? 1 : 0 + project = var.project_id + region = var.region + name_prefix = "${var.name}-" + description = "Managed by the cos-workload Terraform module." + tags = var.tags + machine_type = var.instance_type + can_ip_forward = var.options.can_ip_forward + + disk { + source_image = var.boot_disk.image + disk_type = var.boot_disk.type + disk_size_gb = var.boot_disk.size + boot = true + } + + metadata = { + user-data = data.template_file.cloud_config[var.name].rendered + google-logging-enabled = var.stackdriver.enable_logging + google-monitoring-enabled = var.stackdriver.enable_monitoring + } + + network_interface { + network = var.network + subnetwork = var.subnetwork + dynamic access_config { + for_each = var.nat.enabled ? [null] : [] + content {} + } + } + + service_account { + email = local.service_account + scopes = ["https://www.googleapis.com/auth/cloud-platform"] + } + + lifecycle { + create_before_destroy = true + } +} + +resource "google_service_account" "service_account" { + count = var.service_account == "" ? 1 : 0 + project = var.project_id + account_id = "tf-vm-cos-${var.name}" + display_name = "COS ${var.name}." +} diff --git a/modules/vm-cos/outputs.tf b/modules/vm-cos/outputs.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/modules/vm-cos/variables.tf b/modules/vm-cos/variables.tf new file mode 100644 index 0000000000..1e3092a8d8 --- /dev/null +++ b/modules/vm-cos/variables.tf @@ -0,0 +1,142 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "addresses" { + type = list(string) + default = [] +} + +variable "attached_disks" { + description = "Additional disks (only for non-MIG usage)." + type = map(object({ + type = string # pd-standard pd-ssd + size = string + })) + default = {} +} + +variable "boot_disk" { + type = object({ + image = string + size = number + type = string + }) + default = { + image = "projects/cos-cloud/global/images/family/cos-stable" + type = "pd-ssd" + size = 10 + } +} + +variable "cloud_config_path" { + type = string +} + +variable "cloud_config_vars" { + type = map(any) + default = {} +} + +variable "docker_log_driver" { + type = string + default = "gcplogs" +} + +variable "instance_count" { + type = number + default = 1 +} + +variable "instance_type" { + type = string + default = "f1-micro" +} + +variable "name" { + type = string +} + +variable "network" { + type = string +} + +variable "nat" { + type = object({ + enabled = bool + address = string + }) + default = { + enabled = false + address = null + } +} + +variable "options" { + type = object({ + allow_stopping_for_update = bool + automatic_restart = bool + can_ip_forward = bool + on_host_maintenance = string + }) + default = { + allow_stopping_for_update = true + automatic_restart = true + can_ip_forward = false + on_host_maintenance = "MIGRATE" + } +} + +variable "project_id" { + type = string +} + +variable "region" { + type = string +} + +variable "service_account" { + type = string + default = "" +} + +variable "stackdriver" { + type = object({ + enable_logging = bool + enable_monitoring = bool + }) + default = { + enable_logging = false + enable_monitoring = false + } +} + +variable "subnetwork" { + type = string +} + +variable "tags" { + type = list(string) + default = ["ssh"] +} + +variable "use_instance_template" { + type = bool + default = false +} + +variable "zone" { + type = string +} From 567acb4ddf3068a4a4caa2b81e2f412ec5c16806 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sat, 21 Dec 2019 02:15:20 +0100 Subject: [PATCH 068/106] modules/vm-cos: add variable descriptions --- modules/vm-cos/README.md | 39 ++++++++++--------- modules/vm-cos/main.tf | 14 +++---- modules/vm-cos/variables.tf | 75 ++++++++++++++++++++++--------------- 3 files changed, 70 insertions(+), 58 deletions(-) diff --git a/modules/vm-cos/README.md b/modules/vm-cos/README.md index 1890ae2804..012cdb0467 100644 --- a/modules/vm-cos/README.md +++ b/modules/vm-cos/README.md @@ -5,26 +5,25 @@ | name | description | type | required | default | |---|---|:---: |:---:|:---:| -| cloud_config_path | None | string | ✓ | | -| name | None | string | ✓ | | -| network | None | string | ✓ | | -| project_id | None | string | ✓ | | -| region | None | string | ✓ | | -| subnetwork | None | string | ✓ | | -| zone | None | string | ✓ | | -| *addresses* | None | list(string) | | [] | -| *attached_disks* | Additional disks (only for non-MIG usage). | string # pd-standard pd-ssd | | {} | -| *boot_disk* | None | "pd-ssd" | | {} | -| *cloud_config_vars* | None | map(any) | | {} | -| *docker_log_driver* | None | string | | gcplogs | -| *instance_count* | None | number | | 1 | -| *instance_type* | None | string | | f1-micro | -| *nat* | None | object({...}) | | {} | -| *options* | None | object({...}) | | {} | -| *service_account* | None | string | | | -| *stackdriver* | None | object({...}) | | {} | -| *tags* | None | list(string) | | ["ssh"] | -| *use_instance_template* | None | bool | | false | +| cloud_config_path | Path to cloud config template file to set in metadata. | string | ✓ | | +| name | Instance name. | string | ✓ | | +| network | Network name (self link for shared vpc). | string | ✓ | | +| project_id | Project id. | string | ✓ | | +| region | Compute region. | string | ✓ | | +| subnetwork | Subnetwork name (self link for shared vpc). | string | ✓ | | +| zone | Compute zone. | string | ✓ | | +| *addresses* | Optional internal addresses (only for non-MIG usage). | list(string) | | [] | +| *attached_disks* | Additional disks (only for non-MIG usage). | string | | {} | +| *boot_disk* | Boot disk properties. | "pd-ssd" | | {} | +| *cloud_config_vars* | Custom variables passed to cloud config template. | map(any) | | {} | +| *instance_count* | Number of instances to create (only for non-MIG usage). | number | | 1 | +| *instance_type* | Instance type. | string | | f1-micro | +| *nat* | External address properties (addresses only for non-MIG usage). | object({...}) | | {} | +| *options* | Instance options. | object({...}) | | {} | +| *service_account* | Service account email (leave empty to auto-create). | string | | | +| *stackdriver* | Stackdriver options set in metadata. | object({...}) | | {} | +| *tags* | Instance tags. | list(string) | | ["ssh"] | +| *use_instance_template* | Create instance template instead of instances. | bool | | false | ## Outputs diff --git a/modules/vm-cos/main.tf b/modules/vm-cos/main.tf index 7573e64bd3..1dbe5c06cb 100644 --- a/modules/vm-cos/main.tf +++ b/modules/vm-cos/main.tf @@ -40,11 +40,10 @@ data "template_file" "cloud_config" { for_each = local.names template = file(var.cloud_config_path) vars = merge(var.cloud_config_vars, { - address = lookup(local.addresses, each.key, "") - attached_disks = join(",", keys(var.attached_disks)) - docker_log_driver = var.docker_log_driver - id = each.value - name = each.key + address = lookup(local.addresses, each.key, "") + attached_disks = join(",", keys(var.attached_disks)) + id = each.value + name = each.key }) } @@ -100,9 +99,10 @@ resource "google_compute_instance" "default" { subnetwork = var.subnetwork network_ip = lookup(local.addresses, each.key, null) dynamic access_config { - for_each = var.nat.enabled ? [var.nat.address] : [] + for_each = var.nat.enabled ? [var.nat.addresses] : [] + iterator = config content { - nat_ip = each.value + nat_ip = length(config.value) > 0 ? config.value[each.value] : null } } } diff --git a/modules/vm-cos/variables.tf b/modules/vm-cos/variables.tf index 1e3092a8d8..966fb8dd04 100644 --- a/modules/vm-cos/variables.tf +++ b/modules/vm-cos/variables.tf @@ -15,20 +15,22 @@ */ variable "addresses" { - type = list(string) - default = [] + description = "Optional internal addresses (only for non-MIG usage)." + type = list(string) + default = [] } variable "attached_disks" { description = "Additional disks (only for non-MIG usage)." type = map(object({ - type = string # pd-standard pd-ssd + type = string size = string })) default = {} } variable "boot_disk" { + description = "Boot disk properties." type = object({ image = string size = number @@ -42,49 +44,52 @@ variable "boot_disk" { } variable "cloud_config_path" { - type = string + description = "Path to cloud config template file to set in metadata." + type = string } variable "cloud_config_vars" { - type = map(any) - default = {} -} - -variable "docker_log_driver" { - type = string - default = "gcplogs" + description = "Custom variables passed to cloud config template." + type = map(any) + default = {} } variable "instance_count" { - type = number - default = 1 + description = "Number of instances to create (only for non-MIG usage)." + type = number + default = 1 } variable "instance_type" { - type = string - default = "f1-micro" + description = "Instance type." + type = string + default = "f1-micro" } variable "name" { - type = string + description = "Instance name." + type = string } variable "network" { - type = string + description = "Network name (self link for shared vpc)." + type = string } variable "nat" { + description = "External address properties (addresses only for non-MIG usage)." type = object({ - enabled = bool - address = string + enabled = bool + addresses = list(string) }) default = { - enabled = false - address = null + enabled = false + addresses = [] } } variable "options" { + description = "Instance options." type = object({ allow_stopping_for_update = bool automatic_restart = bool @@ -100,19 +105,23 @@ variable "options" { } variable "project_id" { - type = string + description = "Project id." + type = string } variable "region" { - type = string + description = "Compute region." + type = string } variable "service_account" { - type = string - default = "" + description = "Service account email (leave empty to auto-create)." + type = string + default = "" } variable "stackdriver" { + description = "Stackdriver options set in metadata." type = object({ enable_logging = bool enable_monitoring = bool @@ -124,19 +133,23 @@ variable "stackdriver" { } variable "subnetwork" { - type = string + description = "Subnetwork name (self link for shared vpc)." + type = string } variable "tags" { - type = list(string) - default = ["ssh"] + description = "Instance tags." + type = list(string) + default = ["ssh"] } variable "use_instance_template" { - type = bool - default = false + description = "Create instance template instead of instances." + type = bool + default = false } variable "zone" { - type = string + description = "Compute zone." + type = string } From 7013fd943a8a6b0557c4fb50df0c025b3e0a25ef Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sat, 21 Dec 2019 02:30:49 +0100 Subject: [PATCH 069/106] tfdoc: fix parsing in type and default blocks --- tools/tfdoc/tfdoc.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/tfdoc/tfdoc.py b/tools/tfdoc/tfdoc.py index 2b95c47f97..a32672fad5 100755 --- a/tools/tfdoc/tfdoc.py +++ b/tools/tfdoc/tfdoc.py @@ -115,6 +115,9 @@ def _close(self, strip=False): setattr(self, self._data_context, ('\n'.join(data)).strip()) def _start(self, context, data): + if context == self._data_context or getattr(self, context): + self._data.append(data) + return self._close() self._data = [data] self._data_context = context From 26aa479983c02c645b23bbc4f8a2a523ea2c5af8 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sat, 21 Dec 2019 02:31:01 +0100 Subject: [PATCH 070/106] modules/vm-cos: fix README --- modules/vm-cos/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/vm-cos/README.md b/modules/vm-cos/README.md index 012cdb0467..424921f63a 100644 --- a/modules/vm-cos/README.md +++ b/modules/vm-cos/README.md @@ -13,8 +13,8 @@ | subnetwork | Subnetwork name (self link for shared vpc). | string | ✓ | | | zone | Compute zone. | string | ✓ | | | *addresses* | Optional internal addresses (only for non-MIG usage). | list(string) | | [] | -| *attached_disks* | Additional disks (only for non-MIG usage). | string | | {} | -| *boot_disk* | Boot disk properties. | "pd-ssd" | | {} | +| *attached_disks* | Additional disks (only for non-MIG usage). | map(object({...})) | | {} | +| *boot_disk* | Boot disk properties. | object({...}) | | {} | | *cloud_config_vars* | Custom variables passed to cloud config template. | map(any) | | {} | | *instance_count* | Number of instances to create (only for non-MIG usage). | number | | 1 | | *instance_type* | Instance type. | string | | f1-micro | From 554747eebac5138fbcc157097ed146f3428ff385 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sat, 21 Dec 2019 02:32:24 +0100 Subject: [PATCH 071/106] tfdoc: fix parsing in type and default blocks --- tools/tfdoc/tfdoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/tfdoc/tfdoc.py b/tools/tfdoc/tfdoc.py index a32672fad5..5e03814de7 100755 --- a/tools/tfdoc/tfdoc.py +++ b/tools/tfdoc/tfdoc.py @@ -116,7 +116,7 @@ def _close(self, strip=False): def _start(self, context, data): if context == self._data_context or getattr(self, context): - self._data.append(data) + self._data.append("%s = %s" % (context, data)) return self._close() self._data = [data] From 18ae75d41f02759c041b86246be02379951a0a28 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sat, 21 Dec 2019 02:32:29 +0100 Subject: [PATCH 072/106] modules/vm-cos: fix README --- modules/vm-cos/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/vm-cos/README.md b/modules/vm-cos/README.md index 424921f63a..716b4a0147 100644 --- a/modules/vm-cos/README.md +++ b/modules/vm-cos/README.md @@ -13,8 +13,8 @@ | subnetwork | Subnetwork name (self link for shared vpc). | string | ✓ | | | zone | Compute zone. | string | ✓ | | | *addresses* | Optional internal addresses (only for non-MIG usage). | list(string) | | [] | -| *attached_disks* | Additional disks (only for non-MIG usage). | map(object({...})) | | {} | -| *boot_disk* | Boot disk properties. | object({...}) | | {} | +| *attached_disks* | Additional disks (only for non-MIG usage). | map(object({...})) | | {} | +| *boot_disk* | Boot disk properties. | object({...}) | | {} | | *cloud_config_vars* | Custom variables passed to cloud config template. | map(any) | | {} | | *instance_count* | Number of instances to create (only for non-MIG usage). | number | | 1 | | *instance_type* | Instance type. | string | | f1-micro | From 5e7e9fb815f18b613e6de601aa63bfebff2df1f4 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 22 Dec 2019 01:02:59 +0100 Subject: [PATCH 073/106] modules/compute-vm: initial working import (not fully tested) --- modules/compute-vm/README.md | 40 ++++++ modules/compute-vm/main.tf | 209 ++++++++++++++++++++++++++++++++ modules/compute-vm/outputs.tf | 68 +++++++++++ modules/compute-vm/variables.tf | 182 +++++++++++++++++++++++++++ 4 files changed, 499 insertions(+) create mode 100644 modules/compute-vm/README.md create mode 100644 modules/compute-vm/main.tf create mode 100644 modules/compute-vm/outputs.tf create mode 100644 modules/compute-vm/variables.tf diff --git a/modules/compute-vm/README.md b/modules/compute-vm/README.md new file mode 100644 index 0000000000..11114c40e1 --- /dev/null +++ b/modules/compute-vm/README.md @@ -0,0 +1,40 @@ +# Compute Engine VM module + + +## Variables + +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| name | Instances base name. | string | ✓ | | +| network_interfaces | Network interfaces configuration. Use self links for Shared VPC, set addresses to null if not needed. | list(object({...})) | ✓ | | +| project_id | Project id. | string | ✓ | | +| region | Compute region. | string | ✓ | | +| zone | Compute zone. | string | ✓ | | +| *addresses* | Optional internal addresses (only for non-template usage). | list(string) | | [] | +| *attached_disk_defaults* | Defaults for attached disks options. | object({...}) | | {} | +| *attached_disks* | Additional disks, if options is null defaults will be used in its place. | list(object({...})) | | [] | +| *boot_disk* | Boot disk properties. | object({...}) | | {} | +| *hostname* | Instance FQDN name. | string | | null | +| *instance_count* | Number of instances to create (only for non-template usage). | number | | 1 | +| *instance_type* | Instance type. | string | | f1-micro | +| *labels* | Instance labels. | map(string) | | {} | +| *metadata* | Instance metadata. | map(string) | | {} | +| *min_cpu_platform* | Minimum CPU platform. | string | | null | +| *options* | Instance options. | object({...}) | | {} | +| *scratch_disks* | Scratch disks configuration. | object({...}) | | {} | +| *service_account* | Service account email (leave empty to auto-create). | string | | | +| *tags* | Instance tags. | list(string) | | ["ssh"] | +| *use_instance_template* | Create instance template instead of instances. | bool | | false | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| external_ips | Instance main interface external IP addresses. | | +| instances | Instance resources. | | +| internal_ips | Instance main interface internal IP addresses. | | +| names | Instance names. | | +| self_links | Instance self links. | | +| template | Template resource. | | +| template_name | Template name. | | + diff --git a/modules/compute-vm/main.tf b/modules/compute-vm/main.tf new file mode 100644 index 0000000000..f6e36034f7 --- /dev/null +++ b/modules/compute-vm/main.tf @@ -0,0 +1,209 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + attached_disks = { + for disk in var.attached_disks : + disk.name => merge(disk, { + options = disk.options == null ? var.attached_disk_defaults : disk.options + }) + } + attached_disks_pairs = { + for pair in setproduct(keys(local.names), keys(local.attached_disks)) : + "${pair[0]}-${pair[1]}" => { name = pair[0], disk_name = pair[1] } + } + names = ( + var.use_instance_template + ? { "${var.name}" = 0 } + : { for i in range(0, var.instance_count) : "${var.name}-${i + 1}" => i } + ) + service_account = ( + var.service_account == "" + ? google_service_account.service_account[0].email + : var.service_account + ) +} + +resource "google_compute_disk" "disks" { + for_each = var.use_instance_template ? {} : local.attached_disks_pairs + project = var.project_id + zone = var.zone + name = each.key + type = local.attached_disks[each.value.disk_name].options.type + size = local.attached_disks[each.value.disk_name].size + labels = merge(var.labels, { + disk_name = local.attached_disks[each.value.disk_name].name + disk_type = local.attached_disks[each.value.disk_name].options.type + image = local.attached_disks[each.value.disk_name].image + }) +} + +resource "google_compute_instance" "default" { + for_each = var.use_instance_template ? {} : local.names + project = var.project_id + zone = var.zone + name = each.key + hostname = var.hostname + description = "Managed by the compute-vm Terraform module." + tags = var.tags + machine_type = var.instance_type + min_cpu_platform = var.min_cpu_platform + can_ip_forward = var.options.can_ip_forward + allow_stopping_for_update = var.options.allow_stopping_for_update + deletion_protection = var.options.deletion_protection + metadata = var.metadata + labels = var.labels + + dynamic attached_disk { + for_each = { + for resource_name, pair in local.attached_disks_pairs : + resource_name => local.attached_disks[pair.disk_name] if pair.name == each.key + } + iterator = config + content { + device_name = config.value.name + mode = config.value.options.mode + source = google_compute_disk.disks[config.key].name + } + } + + boot_disk { + initialize_params { + type = var.boot_disk.type + image = var.boot_disk.image + size = var.boot_disk.size + } + } + + dynamic network_interface { + for_each = var.network_interfaces + iterator = config + content { + network = config.value.network + subnetwork = config.value.subnetwork + network_ip = config.value.addresses == null ? null : ( + length(config.value.addresses.internal) == 0 + ? null + : config.value.addresses.internal[each.value] + ) + dynamic access_config { + for_each = config.value.nat ? [config.value.addresses] : [] + iterator = nat_addresses + content { + nat_ip = nat_addresses.value == null ? null : ( + length(nat_addresses.value) == 0 ? null : nat_addresses.value[each.value] + ) + } + } + } + } + + scheduling { + automatic_restart = ! var.options.preemptible + on_host_maintenance = var.options.preemptible ? "TERMINATE" : "MIGRATE" + preemptible = var.options.preemptible + } + + dynamic scratch_disk { + for_each = [ + for i in range(0, var.scratch_disks.count) : var.scratch_disks.interface + ] + iterator = config + content { + interface = config.value + } + } + + service_account { + email = local.service_account + scopes = ["https://www.googleapis.com/auth/cloud-platform"] + } + + # guest_accelerator + # shielded_instance_config + +} + +resource "google_compute_instance_template" "default" { + count = var.use_instance_template ? 1 : 0 + project = var.project_id + region = var.region + name_prefix = "${var.name}-" + description = "Managed by the compute-vm Terraform module." + tags = var.tags + machine_type = var.instance_type + min_cpu_platform = var.min_cpu_platform + can_ip_forward = var.options.can_ip_forward + metadata = var.metadata + labels = var.labels + + disk { + source_image = var.boot_disk.image + disk_type = var.boot_disk.type + disk_size_gb = var.boot_disk.size + boot = true + } + + dynamic disk { + for_each = local.attached_disks + iterator = config + content { + auto_delete = config.value.options.auto_delete + device_name = config.value.name + disk_type = config.value.options.type + disk_size_gb = config.value.size + mode = config.value.options.mode + source_image = config.value.image + source = config.value.options.source + type = "PERSISTENT" + } + } + + dynamic network_interface { + for_each = var.network_interfaces + iterator = config + content { + network = config.value.network + subnetwork = config.value.subnetwork + dynamic access_config { + for_each = config.value.nat ? [""] : [] + content {} + } + } + } + + scheduling { + automatic_restart = ! var.options.preemptible + on_host_maintenance = var.options.preemptible ? "TERMINATE" : "MIGRATE" + preemptible = var.options.preemptible + } + + service_account { + email = local.service_account + scopes = ["https://www.googleapis.com/auth/cloud-platform"] + } + + lifecycle { + create_before_destroy = true + } +} + +resource "google_service_account" "service_account" { + count = var.service_account == "" ? 1 : 0 + project = var.project_id + account_id = "tf-vm-${var.name}" + display_name = "Terraform VM ${var.name}." +} diff --git a/modules/compute-vm/outputs.tf b/modules/compute-vm/outputs.tf new file mode 100644 index 0000000000..8f1427fc65 --- /dev/null +++ b/modules/compute-vm/outputs.tf @@ -0,0 +1,68 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "instances" { + description = "Instance resources." + value = [for name, instance in google_compute_instance.default : instance] +} + +output "names" { + description = "Instance names." + value = [for name, instance in google_compute_instance.default : instance.name] +} + +output "self_links" { + description = "Instance self links." + value = [for name, instance in google_compute_instance.default : instance.self_link] +} + +output "internal_ips" { + description = "Instance main interface internal IP addresses." + value = [ + for name, instance in google_compute_instance.default : + instance.network_interface.0.network_ip + ] +} + +output "external_ips" { + description = "Instance main interface external IP addresses." + value = ( + var.network_interfaces[0].nat + ? [ + for name, instance in google_compute_instance.default : + instance.network_interface.0.network_ip + ] + : [] + ) +} + +output "template" { + description = "Template resource." + value = ( + length(google_compute_instance_template.default) > 0 + ? google_compute_instance_template.default[0] + : null + ) +} + +output "template_name" { + description = "Template name." + value = ( + length(google_compute_instance_template.default) > 0 + ? google_compute_instance_template.default[0].name + : null + ) +} diff --git a/modules/compute-vm/variables.tf b/modules/compute-vm/variables.tf new file mode 100644 index 0000000000..b399140318 --- /dev/null +++ b/modules/compute-vm/variables.tf @@ -0,0 +1,182 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "addresses" { + description = "Optional internal addresses (only for non-template usage)." + type = list(string) + default = [] +} + +variable "attached_disks" { + description = "Additional disks, if options is null defaults will be used in its place." + type = list(object({ + name = string + image = string + size = string + options = object({ + auto_delete = bool + mode = string + source = string + type = string + }) + })) + default = [] +} + +variable "attached_disk_defaults" { + description = "Defaults for attached disks options." + type = object({ + auto_delete = bool + mode = string + type = string + source = string + }) + default = { + auto_delete = true + source = null + mode = "READ_WRITE" + type = "pd-ssd" + } +} + +variable "boot_disk" { + description = "Boot disk properties." + type = object({ + image = string + size = number + type = string + }) + default = { + image = "projects/debian-cloud/global/images/family/debian-10" + type = "pd-ssd" + size = 10 + } +} + +variable "hostname" { + description = "Instance FQDN name." + type = string + default = null +} + +variable "instance_count" { + description = "Number of instances to create (only for non-template usage)." + type = number + default = 1 +} + +variable "instance_type" { + description = "Instance type." + type = string + default = "f1-micro" +} + +variable "labels" { + description = "Instance labels." + type = map(string) + default = {} +} + +variable "metadata" { + description = "Instance metadata." + type = map(string) + default = {} +} + +variable "min_cpu_platform" { + description = "Minimum CPU platform." + type = string + default = null +} + +variable "name" { + description = "Instances base name." + type = string +} + +variable "network_interfaces" { + description = "Network interfaces configuration. Use self links for Shared VPC, set addresses to null if not needed." + type = list(object({ + nat = bool + network = string + subnetwork = string + addresses = object({ + internal = list(string) + external = list(string) + }) + })) +} + +variable "options" { + description = "Instance options." + type = object({ + allow_stopping_for_update = bool + can_ip_forward = bool + deletion_protection = bool + preemptible = bool + }) + default = { + allow_stopping_for_update = true + can_ip_forward = false + deletion_protection = false + preemptible = false + } +} + +variable "project_id" { + description = "Project id." + type = string +} + +variable "region" { + description = "Compute region." + type = string +} + +variable "scratch_disks" { + description = "Scratch disks configuration." + type = object({ + count = number + interface = string + }) + default = { + count = 0 + interface = "NVME" + } +} + +variable "service_account" { + description = "Service account email (leave empty to auto-create)." + type = string + default = "" +} + +variable "tags" { + description = "Instance tags." + type = list(string) + default = ["ssh"] +} + +variable "use_instance_template" { + description = "Create instance template instead of instances." + type = bool + default = false +} + +variable "zone" { + description = "Compute zone." + type = string +} From 5a372b2273c2f5043a44e5ee89b93063ea43f34e Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 22 Dec 2019 01:11:17 +0100 Subject: [PATCH 074/106] modules/vm-cos: move to not-ready --- modules/{ => not-ready}/vm-cos/README.md | 0 modules/{ => not-ready}/vm-cos/main.tf | 0 modules/{ => not-ready}/vm-cos/outputs.tf | 0 modules/{ => not-ready}/vm-cos/variables.tf | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename modules/{ => not-ready}/vm-cos/README.md (100%) rename modules/{ => not-ready}/vm-cos/main.tf (100%) rename modules/{ => not-ready}/vm-cos/outputs.tf (100%) rename modules/{ => not-ready}/vm-cos/variables.tf (100%) diff --git a/modules/vm-cos/README.md b/modules/not-ready/vm-cos/README.md similarity index 100% rename from modules/vm-cos/README.md rename to modules/not-ready/vm-cos/README.md diff --git a/modules/vm-cos/main.tf b/modules/not-ready/vm-cos/main.tf similarity index 100% rename from modules/vm-cos/main.tf rename to modules/not-ready/vm-cos/main.tf diff --git a/modules/vm-cos/outputs.tf b/modules/not-ready/vm-cos/outputs.tf similarity index 100% rename from modules/vm-cos/outputs.tf rename to modules/not-ready/vm-cos/outputs.tf diff --git a/modules/vm-cos/variables.tf b/modules/not-ready/vm-cos/variables.tf similarity index 100% rename from modules/vm-cos/variables.tf rename to modules/not-ready/vm-cos/variables.tf From 905198490d2083d04f8c08ad175b294b5eee461f Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 22 Dec 2019 01:19:09 +0100 Subject: [PATCH 075/106] tfdoc: fix variable defaults formatting --- tools/tfdoc/tfdoc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/tfdoc/tfdoc.py b/tools/tfdoc/tfdoc.py index 5e03814de7..bb6dffb037 100755 --- a/tools/tfdoc/tfdoc.py +++ b/tools/tfdoc/tfdoc.py @@ -180,11 +180,11 @@ def format_variables(variables, required_first=True): for v in variables: default = default_spec = type_spec = '' if not v.required: - default = '{}' + default = '{default}' if '\n' in v.default: - default.format(_escape(v.default), '...') + default = default.format(title=_escape(v.default), default='...') else: - default = default.format('', v.default or '') + default = default.format(title='', default=v.default or '') if v.type and '(' in v.type: type_spec = _escape(v.type) yield row.format( From b5875a2685bb5218c641debcd7df09319976281b Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 22 Dec 2019 01:19:55 +0100 Subject: [PATCH 076/106] modules: update README files with tfdoc fixes --- modules/compute-vm/README.md | 8 ++++---- modules/dns/README.md | 2 +- modules/gke-cluster/README.md | 10 +++++----- modules/kms/README.md | 2 +- modules/net-cloudnat/README.md | 2 +- modules/net-vpc/README.md | 2 +- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/modules/compute-vm/README.md b/modules/compute-vm/README.md index 11114c40e1..ac759e26cd 100644 --- a/modules/compute-vm/README.md +++ b/modules/compute-vm/README.md @@ -11,17 +11,17 @@ | region | Compute region. | string | ✓ | | | zone | Compute zone. | string | ✓ | | | *addresses* | Optional internal addresses (only for non-template usage). | list(string) | | [] | -| *attached_disk_defaults* | Defaults for attached disks options. | object({...}) | | {} | +| *attached_disk_defaults* | Defaults for attached disks options. | object({...}) | | ... | | *attached_disks* | Additional disks, if options is null defaults will be used in its place. | list(object({...})) | | [] | -| *boot_disk* | Boot disk properties. | object({...}) | | {} | +| *boot_disk* | Boot disk properties. | object({...}) | | ... | | *hostname* | Instance FQDN name. | string | | null | | *instance_count* | Number of instances to create (only for non-template usage). | number | | 1 | | *instance_type* | Instance type. | string | | f1-micro | | *labels* | Instance labels. | map(string) | | {} | | *metadata* | Instance metadata. | map(string) | | {} | | *min_cpu_platform* | Minimum CPU platform. | string | | null | -| *options* | Instance options. | object({...}) | | {} | -| *scratch_disks* | Scratch disks configuration. | object({...}) | | {} | +| *options* | Instance options. | object({...}) | | ... | +| *scratch_disks* | Scratch disks configuration. | object({...}) | | ... | | *service_account* | Service account email (leave empty to auto-create). | string | | | | *tags* | Instance tags. | list(string) | | ["ssh"] | | *use_instance_template* | Create instance template instead of instances. | bool | | false | diff --git a/modules/dns/README.md b/modules/dns/README.md index c2bf5597ce..611ced83f2 100644 --- a/modules/dns/README.md +++ b/modules/dns/README.md @@ -42,7 +42,7 @@ module "dns-private-zone" { | *dnssec_config* | DNSSEC configuration: kind, non_existence, state. | any | | {} | | *forwarders* | List of target name servers, only valid for 'forwarding' zone types. | list(string) | | [] | | *peer_network* | Peering network self link, only valid for 'peering' zone types. | string | | | -| *recordsets* | List of DNS record objects to manage. | string | | [] | +| *recordsets* | List of DNS record objects to manage. | list(object({...})) | | [] | | *type* | Type of zone to create, valid values are 'public', 'private', 'forwarding', 'peering'. | string | | private | ## Outputs diff --git a/modules/gke-cluster/README.md b/modules/gke-cluster/README.md index e9d4b6d639..69f1dd587a 100644 --- a/modules/gke-cluster/README.md +++ b/modules/gke-cluster/README.md @@ -18,10 +18,10 @@ TODO(ludoo): add example | secondary_range_pods | Subnet secondary range name used for pods. | string | ✓ | | | secondary_range_services | Subnet secondary range name used for services. | string | ✓ | | | subnetwork | VPC subnetwork name or self link. | string | ✓ | | -| *addons* | Addons enabled in the cluster (true means enabled). | object({...}) | | {} | +| *addons* | Addons enabled in the cluster (true means enabled). | object({...}) | | ... | | *authenticator_security_group* | RBAC security group for Google Groups for GKE, format is gke-security-groups@yourdomain.com. | string | | null | -| *cluster_autoscaling* | Enable and configure limits for Node Auto-Provisioning with Cluster Autoscaler. | object({...}) | | {} | -| *database_encryption* | Enable and configure GKE application-layer secrets encryption. | object({...}) | | {} | +| *cluster_autoscaling* | Enable and configure limits for Node Auto-Provisioning with Cluster Autoscaler. | object({...}) | | ... | +| *database_encryption* | Enable and configure GKE application-layer secrets encryption. | object({...}) | | ... | | *default_max_pods_per_node* | Maximum number of pods per node in this cluster. | number | | 110 | | *description* | Cluster description. | string | | null | | *enable_binary_authorization* | Enable Google Binary Authorization. | bool | | null | @@ -37,9 +37,9 @@ TODO(ludoo): add example | *node_locations* | Zones in which the cluster's nodes are located. | list(string) | | [] | | *pod_security_policy* | Enable the PodSecurityPolicy feature. | bool | | null | | *private_cluster* | Enable private cluster. | bool | | false | -| *private_cluster_config* | Private cluster configuration. | object({...}) | | {} | +| *private_cluster_config* | Private cluster configuration. | object({...}) | | ... | | *release_channel* | Release channel for GKE upgrades. | string | | null | -| *resource_usage_export_config* | Configure the ResourceUsageExportConfig feature. | object({...}) | | {} | +| *resource_usage_export_config* | Configure the ResourceUsageExportConfig feature. | object({...}) | | ... | | *vertical_pod_autoscaling* | Enable the Vertical Pod Autoscaling feature. | bool | | null | | *workload_identity* | Enable the Workload Identity feature. | bool | | true | diff --git a/modules/kms/README.md b/modules/kms/README.md index 3f78989872..a4abebc90b 100644 --- a/modules/kms/README.md +++ b/modules/kms/README.md @@ -19,7 +19,7 @@ The resources/services/activations/deletions that this module will create/trigge | *iam_members* | IAM members keyed by key name and role. | map(map(list(string))) | | {} | | *iam_roles* | IAM roles keyed by key name. | map(list(string)) | | {} | | *key_attributes* | Optional key attributes per key. | map(object({...})) | | {} | -| *key_defaults* | Key attribute defaults. | object({...}) | | {} | +| *key_defaults* | Key attribute defaults. | object({...}) | | ... | | *keys* | Key names. | list(string) | | [] | ## Outputs diff --git a/modules/net-cloudnat/README.md b/modules/net-cloudnat/README.md index d639f4d729..d6eec687b5 100644 --- a/modules/net-cloudnat/README.md +++ b/modules/net-cloudnat/README.md @@ -11,7 +11,7 @@ | *addresses* | Optional list of external address self links. | list(string) | | [] | | *config_min_ports_per_vm* | Minimum number of ports allocated to a VM from this NAT config. | number | | 64 | | *config_source_subnets* | Subnetwork configuration (ALL_SUBNETWORKS_ALL_IP_RANGES, ALL_SUBNETWORKS_ALL_PRIMARY_IP_RANGES, LIST_OF_SUBNETWORKS). | string | | ALL_SUBNETWORKS_ALL_IP_RANGES | -| *config_timeouts* | Timeout configurations. | object({...}) | | {} | +| *config_timeouts* | Timeout configurations. | object({...}) | | ... | | *router_asn* | Router ASN used for auto-created router. | number | | 64514 | | *router_name* | Router name, leave blank to create. | string | | | | *router_network* | Name of the VPC used for auto-created router. | string | | | diff --git a/modules/net-vpc/README.md b/modules/net-vpc/README.md index 6306bd21fb..09f08cb120 100644 --- a/modules/net-vpc/README.md +++ b/modules/net-vpc/README.md @@ -17,7 +17,7 @@ TODO(ludoo): add example | *description* | An optional description of this resource (triggers recreation on change). | string | | Terraform-managed. | | *iam_members* | List of IAM members keyed by subnet and role. | map(map(list(string))) | | {} | | *iam_roles* | List of IAM roles keyed by subnet. | map(list(string)) | | {} | -| *log_config_defaults* | Default configuration for flow logs when enabled. | object({...}) | | {} | +| *log_config_defaults* | Default configuration for flow logs when enabled. | object({...}) | | ... | | *log_configs* | Map of per-subnet optional configurations for flow logs when enabled. | map(map(string)) | | {} | | *peering_config* | VPC peering configuration. | object({...}) | | null | | *routing_mode* | The network routing mode (default 'GLOBAL') | string | | GLOBAL | From aecd9289948edffd9fbc6d60e735d13d5a59637b Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 22 Dec 2019 10:02:43 +0100 Subject: [PATCH 077/106] modules: add initial examples --- modules/compute-vm/README.md | 23 ++++++++ modules/dns/README.md | 28 ++++------ modules/folder/README.md | 18 ++++++ modules/gcs/README.md | 14 +++++ modules/iam-service-accounts/README.md | 25 ++++++--- modules/kms/README.md | 16 ++++-- modules/net-address/README.md | 13 +++++ modules/net-cloudnat/README.md | 17 +++++- modules/net-vpc-firewall/README.md | 77 +++++++++----------------- modules/net-vpc/README.md | 39 ++++++++++++- modules/net-vpn-dynamic/README.md | 25 +++++++++ modules/net-vpn-static/README.md | 22 ++++++++ modules/project/README.md | 30 ++++++---- 13 files changed, 252 insertions(+), 95 deletions(-) diff --git a/modules/compute-vm/README.md b/modules/compute-vm/README.md index ac759e26cd..c70ed63559 100644 --- a/modules/compute-vm/README.md +++ b/modules/compute-vm/README.md @@ -1,5 +1,28 @@ # Compute Engine VM module +## Example + +```hcl +module "debian-test" { + source = "../modules/compute-vm" + project_id = local.project + region = local.subnet.region + zone = "${local.subnet.region}-b" + name = "debian-test" + network_interfaces = [{ + network = local.network.self_link, + subnetwork = local.subnet.self_link, + nat = false, + addresses = null + }] + instance_count = 1 + attached_disks = [ + { name = "disk-1", size = 10, image = null, options = null } + ] + # use_instance_template = true +} +``` + ## Variables diff --git a/modules/dns/README.md b/modules/dns/README.md index 611ced83f2..4da8a7705e 100644 --- a/modules/dns/README.md +++ b/modules/dns/README.md @@ -1,30 +1,22 @@ -# Terraform Google Cloud DNS Module +# Cloud DNS Module This module makes it easy to create Google Cloud DNS zones of different types, and manage their records. It supports creating public, private, forwarding, and peering zones. -The resources/services/activations/deletions that this module will create/trigger are: -- One `google_dns_managed_zone` for the zone -- Zero or more `google_dns_record_set` for the zone records - -## Usage - -Basic usage of this module for a private zone is as follows: +## Example ```hcl -module "dns-private-zone" { - source = "./modules/dns - project_id = "my-project" - type = "private" - name = "example-com" - domain = "example.com." - client_networks = [var.vpc_self_link] +module "private-dns" { + source = "./modules/dns" + project_id = local.projects.host + type = "private" + name = "test-example" + domain = "test.example." + client_networks = [local.vpc_peered.self_link, local.vpc_shared.self_link] recordsets = [ - {name = "", type = "NS", ttl = 300, records = ["127.0.0.1"]}, - {name = "localhost", type = "A", ttl = 300, records = ["127.0.0.1"]}, + { name = "localhost", type = "A", ttl = 300, records = ["127.0.0.1"] } ] } - ``` diff --git a/modules/folder/README.md b/modules/folder/README.md index 0f61024e10..ea87b37171 100644 --- a/modules/folder/README.md +++ b/modules/folder/README.md @@ -1,5 +1,23 @@ # Google Cloud Folder Module +## Example + +```hcl +module "folder" { + source = "./modules/folder" + parent = "organizations/${var.organization_id}" + names = ["TF Test"] + iam_members = { + "TF Test" = { + for role in local.folder_roles : "${role}" => var.admins + } + } + iam_roles = { + "TF Test" = local.folder_roles + } +} +``` + ## Variables diff --git a/modules/gcs/README.md b/modules/gcs/README.md index 484b5bdaa0..56ad39b6b8 100644 --- a/modules/gcs/README.md +++ b/modules/gcs/README.md @@ -1,5 +1,19 @@ # Google Cloud Storage Module +## Example + +```iam +module "buckets" { + source = "./modules/gcs" + project_id = module.project.project_id + prefix = "ludo-tf-playground" + names = ["tfstate"] + bucket_policy_only = { + tfstate = false + } +} +``` + ## Variables diff --git a/modules/iam-service-accounts/README.md b/modules/iam-service-accounts/README.md index 0d9a024d62..2815826757 100644 --- a/modules/iam-service-accounts/README.md +++ b/modules/iam-service-accounts/README.md @@ -2,15 +2,22 @@ This module allows easy creation of one or more service accounts, and granting them basic roles. -The resources/services/activations/deletions that this module will create/trigger are: - -- one or more service accounts -- optional non-autoritative IAM role bindings for each service account for the following resource types - - organization - - billing account - - folder - - project -- one optional service account key per service account +## Example + +```hcl +module "serviceprj-service-accounts" { + source = "./modules/iam-service-accounts" + project_id = module.service-project.project_id + names = ["vm-default", "gke-node-default"] + generate_keys = true + iam_project_roles = { + "${module.service-project.project_id}" = [ + "roles/logging.logWriter", + "roles/monitoring.metricWriter", + ] + } +} +``` ## Variables diff --git a/modules/kms/README.md b/modules/kms/README.md index a4abebc90b..d77bf6708d 100644 --- a/modules/kms/README.md +++ b/modules/kms/README.md @@ -2,11 +2,17 @@ Simple Cloud KMS module that allows managing a keyring, zero or more keys in the keyring, and IAM role bindings on individual keys. -The resources/services/activations/deletions that this module will create/trigger are: - -- Create a KMS keyring in the provided project -- Create zero or more keys in the keyring -- Create IAM role bindings for owners, encrypters, decrypters +## Example + +```hcl +module "kms" { + source = "../modules/kms" + project_id = module.project.project_id + keyring = "playground" + location = "europe" + keys = ["vpn"] +} +``` ## Variables diff --git a/modules/net-address/README.md b/modules/net-address/README.md index 0d35b759e3..9c1169b5b6 100644 --- a/modules/net-address/README.md +++ b/modules/net-address/README.md @@ -1,5 +1,18 @@ # Net Address Reservation Module +## Example + +```hcl +module "addresses" { + source = "./modules/net-address" + project_id = local.projects.host + external_addresses = { + nat-1 = module.vpc.subnet_regions["default"], + vpn-remote = module.vpc.subnet_regions["default"], + } +} +``` + ## Variables diff --git a/modules/net-cloudnat/README.md b/modules/net-cloudnat/README.md index d6eec687b5..08640e62f3 100644 --- a/modules/net-cloudnat/README.md +++ b/modules/net-cloudnat/README.md @@ -1,4 +1,19 @@ -# Google Cloud NAT Module +# Cloud NAT Module + +## Example + +```hcl +module "nat" { + source = "../modules/net-cloudnat" + project_id = local.projects.host + region = module.vpc.subnet_regions["default"] + name = "shared" + router_name = module.vpn-dynamic.router_name + addresses = [ + module.addresses.external_addresses.nat-1.self_link + ] +} +``` ## Variables diff --git a/modules/net-vpc-firewall/README.md b/modules/net-vpc-firewall/README.md index c5bce167c4..c87c6e627f 100644 --- a/modules/net-vpc-firewall/README.md +++ b/modules/net-vpc-firewall/README.md @@ -6,62 +6,39 @@ The HTTP and HTTPS rules use the same network tags that are assigned to instance All IP source ranges are configurable through variables, and are set by default to `0.0.0.0/0` for tag-based rules. Allowed protocols and/or ports for the intra-VPC rule are also configurable through a variable. -Custom rules are set through a map where keys are rule names, and values use this custom type: +## Example ```hcl -map(object({ - description = string - direction = string # (INGRESS|EGRESS) - action = string # (allow|deny) - ranges = list(string) # list of IP CIDR ranges - sources = list(string) # tags or SAs (ignored for EGRESS) - targets = list(string) # tags or SAs - use_service_accounts = bool # use tags or SAs in sources/targets - rules = list(object({ - protocol = string - ports = list(string) - })) - extra_attributes = map(string) # map, optional keys disabled or priority -})) -``` - -The resources created/managed by this module are: - -- one optional ingress rule from internal CIDR ranges, only allowing ICMP by default -- one optional ingress rule from admin CIDR ranges, allowing all protocols on all ports -- one optional ingress rule for SSH on network tag `ssh` -- one optional ingress rule for HTTP on network tag `http-server` -- one optional ingress rule for HTTPS on network tag `https-server` -- one or more optional custom rules - - -## Usage - -Basic usage of this module is as follows: - -```hcl -module "net-firewall" { - source = "terraform-google-modules/network/google//modules/fabric-net-firewall" - project_id = "my-project" - network = "my-vpc" - internal_ranges_enabled = true - internal_ranges = ["10.0.0.0/0"] +module "firewall" { + source = "../modules/net-vpc-firewall" + project_id = local.projects.host + network = module.vpc.name + admin_ranges_enabled = true + admin_ranges = values(var.ip_ranges) custom_rules = { - ingress-sample = { - description = "Dummy sample ingress rule, tag-based." + health-checks = { + description = "HTTP health checks." + direction = "INGRESS" + action = "allow" + sources = [] + ranges = ( + data.google_netblock_ip_ranges.health-checkers.cidr_blocks_ipv4 + ) + targets = ["health-checks"] + use_service_accounts = false + rules = [{ protocol = "tcp", ports = [80] }] + extra_attributes = {} + }, + ntp-svc = { + description = "NTP service." direction = "INGRESS" action = "allow" - ranges = ["192.168.0.0"] - sources = ["spam-tag"] - targets = ["foo-tag", "egg-tag"] + sources = [] + ranges = ["0.0.0.0/0"] + targets = ["ntp-svc"] use_service_accounts = false - rules = [ - { - protocol = "tcp" - ports = [] - } - ] - extra_attributes = {} + rules = [{ protocol = "udp", ports = [123] }] + extra_attributes = {} } } } diff --git a/modules/net-vpc/README.md b/modules/net-vpc/README.md index 09f08cb120..e846bec25d 100644 --- a/modules/net-vpc/README.md +++ b/modules/net-vpc/README.md @@ -2,9 +2,44 @@ TODO(ludoo): add description. -## Example usage +## Example -TODO(ludoo): add example +```hcl +module "vpc" { + source = "../modules/net-vpc" + project_id = local.projects.host + name = "shared" + iam_roles = { + default = ["roles/compute.networkUser", "roles/compute.securityAdmin"] + } + iam_members = { + default = { + "roles/compute.networkUser" = [ + local.service_accounts.service_cloudsvc, + local.service_accounts.service_gke + ] + "roles/compute.securityAdmin" = [ + local.service_accounts.service_gke + ] + } + } + shared_vpc_host = true + shared_vpc_service_projects = [ + local.projects.service, + local.projects.gae + ] + subnets = { + default = { + ip_cidr_range = var.ip_ranges.shared-default + region = var.region + secondary_ip_range = { + pods = var.ip_secondary_ranges.shared-default-pods + services = var.ip_secondary_ranges.shared-default-services + } + } + } +} +``` ## Variables diff --git a/modules/net-vpn-dynamic/README.md b/modules/net-vpn-dynamic/README.md index 73541592a1..b055cf0728 100644 --- a/modules/net-vpn-dynamic/README.md +++ b/modules/net-vpn-dynamic/README.md @@ -1,5 +1,30 @@ # Cloud VPN Dynamic Module +## Example + +```hcl +module "vpn-dynamic" { + source = "./modules/net-vpn-dynamic" + project_id = local.projects.host + region = module.vpc.subnet_regions["default"] + network = module.vpc.name + name = "shared-to-remote" + gateway_address = module.addresses.external_addresses.vpn-remote.address + tunnels = { + remote-0 = { + bgp_peer = { + address = "169.254.139.134" + asn = 64513 + } + bgp_peer_options = null + bgp_session_range = "169.254.139.133/30" + ike_version = 2 + peer_ip = var.remote_vpn_gateway.address + shared_secret = "" + } + } +}``` + ## Variables diff --git a/modules/net-vpn-static/README.md b/modules/net-vpn-static/README.md index 12df1aec6c..1032608868 100644 --- a/modules/net-vpn-static/README.md +++ b/modules/net-vpn-static/README.md @@ -1,5 +1,27 @@ # Cloud VPN Route-based Module +## Example + +```hcl +module "vpn" { + source = "./modules/net-vpn-static" + project_id = var.project_id + region = var.region + network = var.network + name = "remote" + # gateway_address = var.gateway_address + remote_ranges = [var.remote_ranges] + tunnels = { + remote-0 = { + ike_version = 2 + peer_ip = var.remote_vpn_gateway_address + shared_secret = "" + traffic_selectors = { local = ["0.0.0.0/0"], remote = null } + } + } +} +``` + ## Variables diff --git a/modules/project/README.md b/modules/project/README.md index 4864bb7d40..7ec77035ed 100644 --- a/modules/project/README.md +++ b/modules/project/README.md @@ -1,15 +1,25 @@ -# Google Cloud Simple Project Creation +# Project Module -This module allows simple Google Cloud Platform project creation, with minimal service and project-level IAM binding management. It's designed to be used for architectural design and rapid prototyping, as part of the [Cloud Foundation Fabric](https://github.com/terraform-google-modules/cloud-foundation-fabric) environments. +## Example -The resources/services/activations/deletions that this module will create/trigger are: - -- one project -- zero or one project metadata items for OSLogin activation -- zero or more project service activations -- zero or more project-level IAM bindings -- zero or more project-level custom roles -- zero or one project liens +```hcl +module "project" { + source = "./modules/project" + parent = var.folder.id + billing_account = var.billing_account_id + prefix = "foo" + name = "project-example" + oslogin = true + oslogin_admins = var.admins + services = concat(var.project_services, [ + "cloudkms.googleapis.com", "accesscontextmanager.googleapis.com" + ]) + iam_roles = ["roles/container.hostServiceAgentUser"] + iam_members = { "roles/container.hostServiceAgentUser" = [ + "serviceAccount:${var.gke_service_account}" + ] } +} +``` ## Variables From 386acf5bfd050477ea7b1f398f590ed8f71272f5 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sat, 4 Jan 2020 13:55:05 +0100 Subject: [PATCH 078/106] gke-nodepool: initial import, untested --- modules/gke-nodepool/main.tf | 115 ++++++++++++++++++ modules/gke-nodepool/outputs.tf | 19 +++ modules/gke-nodepool/variables.tf | 189 ++++++++++++++++++++++++++++++ modules/gke-nodepool/versions.tf | 19 +++ 4 files changed, 342 insertions(+) create mode 100644 modules/gke-nodepool/main.tf create mode 100644 modules/gke-nodepool/outputs.tf create mode 100644 modules/gke-nodepool/variables.tf create mode 100644 modules/gke-nodepool/versions.tf diff --git a/modules/gke-nodepool/main.tf b/modules/gke-nodepool/main.tf new file mode 100644 index 0000000000..a37c7144a8 --- /dev/null +++ b/modules/gke-nodepool/main.tf @@ -0,0 +1,115 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_container_node_pool" "nodepool" { + provider = google-beta + + project = var.project_id + cluster = var.cluster_name + location = var.location + name = var.name + + initial_node_count = var.initial_node_count + max_pods_per_node = var.max_pods_per_node + node_count = var.autoscaling_config == null ? var.node_count : null + node_locations = var.node_locations + version = ( + var.management_config == null || ! var.management_config.auto_upgrade + ? var.gke_version + : null + ) + + node_config { + disk_size_gb = var.node_config_disk_size + disk_type = var.node_config_disk_type + image_type = var.node_config_image_type + labels = var.node_config_labels + local_ssd_count = var.node_config_local_ssd_count + machine_type = var.node_config_machine_type + metadata = var.node_config_metadata + min_cpu_platform = var.node_config_min_cpu_platform + oauth_scopes = var.node_config_oauth_scopes + preemptible = var.node_config_preemptible + service_account = var.node_config_service_account + tags = var.node_config_tags + + dynamic guest_accelerator { + for_each = var.node_config_guest_accelerator + iterator = config + content { + type = config.key + number = config.value + } + } + + dynamic sandbox_config { + for_each = ( + var.node_config_sandbox_config != null + ? [var.node_config_sandbox_config] + : [] + ) + iterator = config + content { + sandbox_type = config.value + } + } + + dynamic shielded_instance_config { + for_each = ( + var.node_config_shielded_instance_config != null + ? [var.node_config_shielded_instance_config] + : [] + ) + iterator = config + content { + enable_secure_boot = config.value.enable_secure_boot + enable_integrity_monitoring = config.value.enable_integrity_monitoring + } + } + + workload_metadata_config { + node_metadata = var.node_config_workload_metadata_config + } + + } + + dynamic autoscaling { + for_each = var.autoscaling_config != null ? [var.autoscaling_config] : [] + iterator = config + content { + min_node_count = config.min_node_count + max_node_count = config.max_node_count + } + } + + dynamic management { + for_each = var.management_config != null ? [var.management_config] : [] + iterator = config + content { + auto_repair = config.auto_repair + auto_upgrade = config.auto_upgrade + } + } + + dynamic upgrade_settings { + for_each = var.upgrade_config != null ? [var.upgrade_config] : [] + iterator = config + content { + max_surge = config.max_surge + max_unavailable = config.max_unavailable + } + } +} diff --git a/modules/gke-nodepool/outputs.tf b/modules/gke-nodepool/outputs.tf new file mode 100644 index 0000000000..43b23bd833 --- /dev/null +++ b/modules/gke-nodepool/outputs.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "name" { + value = google_container_node_pool.nodepool.name +} diff --git a/modules/gke-nodepool/variables.tf b/modules/gke-nodepool/variables.tf new file mode 100644 index 0000000000..a9a28cc8af --- /dev/null +++ b/modules/gke-nodepool/variables.tf @@ -0,0 +1,189 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "autoscaling_config" { + description = "Optional autoscaling configuration." + type = object({ + min_node_count = number + max_node_count = number + }) + default = null +} + +variable "gke_version" { + description = "Kubernetes nodes version. Ignored if auto_upgrade is set in management_config." + type = string + default = null +} + +variable "initial_node_count" { + description = "Initial number of nodes for the pool." + type = number + default = null +} + +variable "initial_node_count" { + description = "Cluster location." + type = string + default = null +} + +variable "management_config" { + description = "Optional node management configuration." + type = object({ + auto_repair = bool + auto_upgrade = bool + }) + default = null +} + +variable "max_pods_per_node" { + description = "Maximum number of pods per node." + type = number + default = null +} + +variable "name" { + description = "Optional nodepool name." + type = string + default = null +} + +variable "node_config_disk_size" { + description = "Node disk size, defaults to 100GB." + type = number + default = 100 +} + +variable "node_config_disk_type" { + description = "Node disk type, defaults to pd-standard." + type = string + default = "pd-standard" +} + +variable "node_config_guest_accelerator" { + description = "Map of type and count of attached accelerator cards." + type = map(number) + default = {} +} + +variable "node_config_image_type" { + description = "Nodes image type." + type = string + default = null +} + +variable "node_config_labels" { + description = "Kubernetes labels attached to nodes." + type = map(string) + default = {} +} + +variable "node_config_local_ssd_count" { + description = "Number of local SSDs attached to nodes." + type = number + default = 0 +} + +variable "node_config_machine_type" { + description = "Nodes machine type." + type = string + default = "n1-standard-1" +} + +variable "node_config_metadata" { + description = "Metadata key/value pairs assigned to nodes. Set disable-legacy-endpoints to true when using this variable." + type = map(string) + default = null +} + +variable "node_config_min_cpu_platform" { + description = "Minimum CPU platform for nodes." + type = string + default = null +} + +variable "node_config_oauth_scopes" { + description = "Set of Google API scopes for the nodes service account. Include logging-write, monitoring, and storage-ro when using this variable." + type = list(string) + default = null +} + +variable "node_config_preemptible" { + description = "Use preemptible VMs for nodes." + type = bool + default = null +} + +variable "node_config_sandbox_config" { + description = "GKE Sandbox configuration. Needs image_type set to COS_CONTAINERD and node_version set to 1.12.7-gke.17 when using this variable." + type = string + default = null +} + +variable "node_config_service_account" { + description = "Service account used for nodes." + type = string + default = null +} + +variable "node_config_shielded_instance_config" { + description = "Shielded instance options." + type = object({ + enable_secure_boot = bool + enable_integrity_monitoring = bool + }) + default = null +} + +variable "node_config_tags" { + description = "Network tags applied to nodes." + type = list(string) + default = null +} + +# variable "node_config_taint" { +# description = "Kubernetes taints applied to nodes." +# type = string +# default = null +# } + +variable "node_config_workload_metadata_config" { + description = "Metadata configuration to expose to workloads on the node pool." + type = string + default = "SECURE" +} + +variable "node_count" { + description = "Number of nodes per instance group, can be updated after creation. Ignored when autoscaling is set." + type = number + default = null +} + +variable "node_locations" { + description = "Optional list of zones in which nodes should be located. Uses cluster locations if unset." + type = list(string) + default = null +} + +variable "upgrade_config" { + description = "Optional node upgrade configuration." + type = object({ + max_surge = number + max_unavailable = number + }) + default = null +} diff --git a/modules/gke-nodepool/versions.tf b/modules/gke-nodepool/versions.tf new file mode 100644 index 0000000000..ce6918e09d --- /dev/null +++ b/modules/gke-nodepool/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12.6" +} From b004c3702733fe9ebb5b94744c01df6379a36a9b Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sat, 4 Jan 2020 14:17:45 +0100 Subject: [PATCH 079/106] gke nodepool: add README, fix location variable, set node count default to 1 --- modules/gke-nodepool/README.md | 46 +++++++++++++++++++++++++++++++ modules/gke-nodepool/outputs.tf | 3 +- modules/gke-nodepool/variables.tf | 4 +-- 3 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 modules/gke-nodepool/README.md diff --git a/modules/gke-nodepool/README.md b/modules/gke-nodepool/README.md new file mode 100644 index 0000000000..731c92c916 --- /dev/null +++ b/modules/gke-nodepool/README.md @@ -0,0 +1,46 @@ +# Minimalistic GKE module + +TODO(ludoo): add description. + +## Example usage + +TODO(ludoo): add example + + +## Variables + +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| *autoscaling_config* | Optional autoscaling configuration. | object({...}) | | null | +| *gke_version* | Kubernetes nodes version. Ignored if auto_upgrade is set in management_config. | string | | null | +| *initial_node_count* | Initial number of nodes for the pool. | number | | 1 | +| *location* | Cluster location. | string | | null | +| *management_config* | Optional node management configuration. | object({...}) | | null | +| *max_pods_per_node* | Maximum number of pods per node. | number | | null | +| *name* | Optional nodepool name. | string | | null | +| *node_config_disk_size* | Node disk size, defaults to 100GB. | number | | 100 | +| *node_config_disk_type* | Node disk type, defaults to pd-standard. | string | | pd-standard | +| *node_config_guest_accelerator* | Map of type and count of attached accelerator cards. | map(number) | | {} | +| *node_config_image_type* | Nodes image type. | string | | null | +| *node_config_labels* | Kubernetes labels attached to nodes. | map(string) | | {} | +| *node_config_local_ssd_count* | Number of local SSDs attached to nodes. | number | | 0 | +| *node_config_machine_type* | Nodes machine type. | string | | n1-standard-1 | +| *node_config_metadata* | Metadata key/value pairs assigned to nodes. Set disable-legacy-endpoints to true when using this variable. | map(string) | | null | +| *node_config_min_cpu_platform* | Minimum CPU platform for nodes. | string | | null | +| *node_config_oauth_scopes* | Set of Google API scopes for the nodes service account. Include logging-write, monitoring, and storage-ro when using this variable. | list(string) | | null | +| *node_config_preemptible* | Use preemptible VMs for nodes. | bool | | null | +| *node_config_sandbox_config* | GKE Sandbox configuration. Needs image_type set to COS_CONTAINERD and node_version set to 1.12.7-gke.17 when using this variable. | string | | null | +| *node_config_service_account* | Service account used for nodes. | string | | null | +| *node_config_shielded_instance_config* | Shielded instance options. | object({...}) | | null | +| *node_config_tags* | Network tags applied to nodes. | list(string) | | null | +| *node_config_workload_metadata_config* | Metadata configuration to expose to workloads on the node pool. | string | | SECURE | +| *node_count* | Number of nodes per instance group, can be updated after creation. Ignored when autoscaling is set. | number | | null | +| *node_locations* | Optional list of zones in which nodes should be located. Uses cluster locations if unset. | list(string) | | null | +| *upgrade_config* | Optional node upgrade configuration. | object({...}) | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| name | Nodepool name. | | + diff --git a/modules/gke-nodepool/outputs.tf b/modules/gke-nodepool/outputs.tf index 43b23bd833..8645a91da2 100644 --- a/modules/gke-nodepool/outputs.tf +++ b/modules/gke-nodepool/outputs.tf @@ -15,5 +15,6 @@ */ output "name" { - value = google_container_node_pool.nodepool.name + description = "Nodepool name." + value = google_container_node_pool.nodepool.name } diff --git a/modules/gke-nodepool/variables.tf b/modules/gke-nodepool/variables.tf index a9a28cc8af..655956f152 100644 --- a/modules/gke-nodepool/variables.tf +++ b/modules/gke-nodepool/variables.tf @@ -32,10 +32,10 @@ variable "gke_version" { variable "initial_node_count" { description = "Initial number of nodes for the pool." type = number - default = null + default = 1 } -variable "initial_node_count" { +variable "location" { description = "Cluster location." type = string default = null From 0f3d340f37fee3dab74a3d7e398e91d2f7a11279 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sat, 4 Jan 2020 18:24:22 +0100 Subject: [PATCH 080/106] gke cluster: fix private cluster variables --- modules/gke-cluster/README.md | 3 +-- modules/gke-cluster/main.tf | 2 +- modules/gke-cluster/variables.tf | 14 ++------------ 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/modules/gke-cluster/README.md b/modules/gke-cluster/README.md index 69f1dd587a..5068bf8686 100644 --- a/modules/gke-cluster/README.md +++ b/modules/gke-cluster/README.md @@ -36,8 +36,7 @@ TODO(ludoo): add example | *monitoring_service* | Monitoring service (disable with an empty string). | string | | monitoring.googleapis.com/kubernetes | | *node_locations* | Zones in which the cluster's nodes are located. | list(string) | | [] | | *pod_security_policy* | Enable the PodSecurityPolicy feature. | bool | | null | -| *private_cluster* | Enable private cluster. | bool | | false | -| *private_cluster_config* | Private cluster configuration. | object({...}) | | ... | +| *private_cluster_config* | Enable and configure private cluster. | object({...}) | | null | | *release_channel* | Release channel for GKE upgrades. | string | | null | | *resource_usage_export_config* | Configure the ResourceUsageExportConfig feature. | object({...}) | | ... | | *vertical_pod_autoscaling* | Enable the Vertical Pod Autoscaling feature. | bool | | null | diff --git a/modules/gke-cluster/main.tf b/modules/gke-cluster/main.tf index 9c57d67303..67e95dd3c7 100644 --- a/modules/gke-cluster/main.tf +++ b/modules/gke-cluster/main.tf @@ -106,7 +106,7 @@ resource "google_container_cluster" "cluster" { } dynamic private_cluster_config { - for_each = var.private_cluster ? list(var.private_cluster_config) : [] + for_each = var.private_cluster_config != null ? [var.private_cluster_config] : [] iterator = config content { enable_private_nodes = config.value.enable_private_nodes diff --git a/modules/gke-cluster/variables.tf b/modules/gke-cluster/variables.tf index c66e60fa3c..e39667711b 100644 --- a/modules/gke-cluster/variables.tf +++ b/modules/gke-cluster/variables.tf @@ -175,24 +175,14 @@ variable "pod_security_policy" { default = null } -variable "private_cluster" { - description = "Enable private cluster." - type = bool - default = false -} - variable "private_cluster_config" { - description = "Private cluster configuration." + description = "Enable and configure private cluster." type = object({ enable_private_nodes = bool enable_private_endpoint = bool master_ipv4_cidr_block = string }) - default = { - enable_private_nodes = true - enable_private_endpoint = false - master_ipv4_cidr_block = null - } + default = null } variable "project_id" { From 5b8d9dd8c839cc5768a626be5289edb88d8cf886 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sat, 4 Jan 2020 18:24:39 +0100 Subject: [PATCH 081/106] gke nodepool: fix README title --- modules/gke-nodepool/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/gke-nodepool/README.md b/modules/gke-nodepool/README.md index 731c92c916..6b820719f8 100644 --- a/modules/gke-nodepool/README.md +++ b/modules/gke-nodepool/README.md @@ -1,4 +1,4 @@ -# Minimalistic GKE module +# Minimalistic GKE nodepool module TODO(ludoo): add description. From c6708d92555d5093b1a4484e21065a0a19263edc Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sat, 4 Jan 2020 18:36:11 +0100 Subject: [PATCH 082/106] gke cluster: add output for cluster location --- modules/gke-cluster/README.md | 1 + modules/gke-cluster/outputs.tf | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/modules/gke-cluster/README.md b/modules/gke-cluster/README.md index 5068bf8686..19ad2bcdc4 100644 --- a/modules/gke-cluster/README.md +++ b/modules/gke-cluster/README.md @@ -48,6 +48,7 @@ TODO(ludoo): add example |---|---|:---:| | cluster | Cluster resource. | ✓ | | endpoint | Cluster endpoint. | | +| location | Cluster location. | | | master_version | Master version. | | | name | Cluster name. | | diff --git a/modules/gke-cluster/outputs.tf b/modules/gke-cluster/outputs.tf index f21b680955..e27acfd114 100644 --- a/modules/gke-cluster/outputs.tf +++ b/modules/gke-cluster/outputs.tf @@ -27,6 +27,11 @@ output "endpoint" { value = google_container_cluster.cluster.endpoint } +output "location" { + description = "Cluster location." + value = google_container_cluster.cluster.location +} + output "master_version" { description = "Master version." value = google_container_cluster.cluster.master_version From 30984a8116abee745030dad51070c4b72b1c7c58 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sat, 4 Jan 2020 18:53:28 +0100 Subject: [PATCH 083/106] gke nodepool: add missing variables for project id and cluster name, remove default from location variable, fix gke version assignment --- modules/gke-nodepool/main.tf | 10 +++------- modules/gke-nodepool/variables.tf | 11 ++++++++++- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/modules/gke-nodepool/main.tf b/modules/gke-nodepool/main.tf index a37c7144a8..40019e09c4 100644 --- a/modules/gke-nodepool/main.tf +++ b/modules/gke-nodepool/main.tf @@ -26,11 +26,7 @@ resource "google_container_node_pool" "nodepool" { max_pods_per_node = var.max_pods_per_node node_count = var.autoscaling_config == null ? var.node_count : null node_locations = var.node_locations - version = ( - var.management_config == null || ! var.management_config.auto_upgrade - ? var.gke_version - : null - ) + version = var.gke_version node_config { disk_size_gb = var.node_config_disk_size @@ -50,8 +46,8 @@ resource "google_container_node_pool" "nodepool" { for_each = var.node_config_guest_accelerator iterator = config content { - type = config.key - number = config.value + type = config.key + count = config.value } } diff --git a/modules/gke-nodepool/variables.tf b/modules/gke-nodepool/variables.tf index 655956f152..ee58c00470 100644 --- a/modules/gke-nodepool/variables.tf +++ b/modules/gke-nodepool/variables.tf @@ -23,6 +23,11 @@ variable "autoscaling_config" { default = null } +variable "cluster_name" { + description = "Cluster name." + type = string +} + variable "gke_version" { description = "Kubernetes nodes version. Ignored if auto_upgrade is set in management_config." type = string @@ -38,7 +43,6 @@ variable "initial_node_count" { variable "location" { description = "Cluster location." type = string - default = null } variable "management_config" { @@ -179,6 +183,11 @@ variable "node_locations" { default = null } +variable "project_id" { + description = "Cluster project id." + type = string +} + variable "upgrade_config" { description = "Optional node upgrade configuration." type = object({ From 7abd9c3fccdc0cbacb01a975e2178b0491923a6c Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sat, 4 Jan 2020 20:07:54 +0100 Subject: [PATCH 084/106] gke nodepool: update README --- modules/gke-nodepool/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/gke-nodepool/README.md b/modules/gke-nodepool/README.md index 6b820719f8..a793f94b16 100644 --- a/modules/gke-nodepool/README.md +++ b/modules/gke-nodepool/README.md @@ -11,10 +11,12 @@ TODO(ludoo): add example | name | description | type | required | default | |---|---|:---: |:---:|:---:| +| cluster_name | Cluster name. | string | ✓ | | +| location | Cluster location. | string | ✓ | | +| project_id | Cluster project id. | string | ✓ | | | *autoscaling_config* | Optional autoscaling configuration. | object({...}) | | null | | *gke_version* | Kubernetes nodes version. Ignored if auto_upgrade is set in management_config. | string | | null | | *initial_node_count* | Initial number of nodes for the pool. | number | | 1 | -| *location* | Cluster location. | string | | null | | *management_config* | Optional node management configuration. | object({...}) | | null | | *max_pods_per_node* | Maximum number of pods per node. | number | | null | | *name* | Optional nodepool name. | string | | null | From 31423c08ff7367663454457a73b80977533dce89 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 5 Jan 2020 09:02:33 +0100 Subject: [PATCH 085/106] net-cloudnat: fix router name when creating default router --- modules/net-cloudnat/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/net-cloudnat/main.tf b/modules/net-cloudnat/main.tf index 70944a55a8..bb1158dcd7 100644 --- a/modules/net-cloudnat/main.tf +++ b/modules/net-cloudnat/main.tf @@ -24,7 +24,7 @@ locals { resource "google_compute_router" "router" { count = var.router_name == "" ? 1 : 0 - name = var.router_name + name = var.router_name == "" ? "${var.name}-nat" : var.router_name project = var.project_id region = var.region network = var.router_network From 07c8643e617f4a42f34781b0162b4c95e984ded3 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 5 Jan 2020 11:34:18 +0100 Subject: [PATCH 086/106] fix variables used for address and router optional creation --- modules/net-vpn-dynamic/README.md | 6 ++++-- modules/net-vpn-dynamic/main.tf | 10 +++++----- modules/net-vpn-dynamic/outputs.tf | 2 +- modules/net-vpn-dynamic/variables.tf | 16 ++++++++++++++-- modules/net-vpn-static/README.md | 3 ++- modules/net-vpn-static/main.tf | 4 ++-- modules/net-vpn-static/variables.tf | 8 +++++++- 7 files changed, 35 insertions(+), 14 deletions(-) diff --git a/modules/net-vpn-dynamic/README.md b/modules/net-vpn-dynamic/README.md index b055cf0728..2024278d1c 100644 --- a/modules/net-vpn-dynamic/README.md +++ b/modules/net-vpn-dynamic/README.md @@ -34,11 +34,13 @@ module "vpn-dynamic" { | network | VPC used for the gateway and routes. | string | ✓ | | | project_id | Project where resources will be created. | string | ✓ | | | region | Region used for resources. | string | ✓ | | -| *gateway_address* | Optional address assigned to the VPN, leave blank to create one. | string | | | +| *gateway_address* | Optional address assigned to the VPN gateway. Ignored unless gateway_address_create is set to false. | string | | | +| *gateway_address_create* | Create external address assigned to the VPN gateway. Needs to be explicitly set to false to use address in gateway_address variable. | bool | | true | | *route_priority* | Route priority, defaults to 1000. | number | | 1000 | | *router_advertise_config* | Router custom advertisement configuration, ip_ranges is a map of address ranges and descriptions. | object({...}) | | null | | *router_asn* | Router ASN used for auto-created router. | number | | 64514 | -| *router_name* | Name of router, leave blank to create one. | string | | | +| *router_create* | Create router. | bool | | true | +| *router_name* | Router name used for auto created router, or to specify existing router to use. Leave blank to use VPN name for auto created router. | string | | | | *tunnels* | VPN tunnel configurations, bgp_peer_options is usually null. | map(object({...})) | | {} | ## Outputs diff --git a/modules/net-vpn-dynamic/main.tf b/modules/net-vpn-dynamic/main.tf index bfab01a4fc..eb0c1eb47b 100644 --- a/modules/net-vpn-dynamic/main.tf +++ b/modules/net-vpn-dynamic/main.tf @@ -16,12 +16,12 @@ locals { gateway_address = ( - var.gateway_address == "" + var.gateway_address_create ? google_compute_address.gateway[0].address : var.gateway_address ) router = ( - var.router_name == "" + var.router_create ? google_compute_router.router[0].name : var.router_name ) @@ -29,7 +29,7 @@ locals { } resource "google_compute_address" "gateway" { - count = var.gateway_address == "" ? 1 : 0 + count = var.gateway_address_create ? 1 : 0 name = "vpn-${var.name}" project = var.project_id region = var.region @@ -65,8 +65,8 @@ resource "google_compute_forwarding_rule" "udp-4500" { } resource "google_compute_router" "router" { - count = var.router_name == "" ? 1 : 0 - name = "vpn-${var.name}" + count = var.router_create ? 1 : 0 + name = var.router_name == "" ? "vpn-${var.name}" : var.router_name project = var.project_id region = var.region network = var.network diff --git a/modules/net-vpn-dynamic/outputs.tf b/modules/net-vpn-dynamic/outputs.tf index 93106a9a6d..7d9956e43e 100644 --- a/modules/net-vpn-dynamic/outputs.tf +++ b/modules/net-vpn-dynamic/outputs.tf @@ -31,7 +31,7 @@ output "name" { output "router" { description = "Router resource (only if auto-created)." - value = var.router_name == "" ? google_compute_router.router[0] : null + value = var.router_create == "" ? google_compute_router.router[0] : null } output "router_name" { diff --git a/modules/net-vpn-dynamic/variables.tf b/modules/net-vpn-dynamic/variables.tf index 9ae408908e..eea77c00bf 100644 --- a/modules/net-vpn-dynamic/variables.tf +++ b/modules/net-vpn-dynamic/variables.tf @@ -14,8 +14,14 @@ * limitations under the License. */ +variable "gateway_address_create" { + description = "Create external address assigned to the VPN gateway. Needs to be explicitly set to false to use address in gateway_address variable." + type = bool + default = true +} + variable "gateway_address" { - description = "Optional address assigned to the VPN, leave blank to create one." + description = "Optional address assigned to the VPN gateway. Ignored unless gateway_address_create is set to false." type = string default = "" } @@ -62,8 +68,14 @@ variable "router_asn" { default = 64514 } +variable "router_create" { + description = "Create router." + type = bool + default = true +} + variable "router_name" { - description = "Name of router, leave blank to create one." + description = "Router name used for auto created router, or to specify existing router to use. Leave blank to use VPN name for auto created router." type = string default = "" } diff --git a/modules/net-vpn-static/README.md b/modules/net-vpn-static/README.md index 1032608868..5961e88506 100644 --- a/modules/net-vpn-static/README.md +++ b/modules/net-vpn-static/README.md @@ -31,7 +31,8 @@ module "vpn" { | network | VPC used for the gateway and routes. | string | ✓ | | | project_id | Project where resources will be created. | string | ✓ | | | region | Region used for resources. | string | ✓ | | -| *gateway_address* | Optional address assigned to the VPN, leave blank to create one. | string | | | +| *gateway_address* | Optional address assigned to the VPN gateway. Ignored unless gateway_address_create is set to false. | string | | | +| *gateway_address_create* | Create external address assigned to the VPN gateway. Needs to be explicitly set to false to use address in gateway_address variable. | bool | | true | | *remote_ranges* | Remote IP CIDR ranges. | list(string) | | [] | | *route_priority* | Route priority, defaults to 1000. | number | | 1000 | | *tunnels* | VPN tunnel configurations. | map(object({...})) | | {} | diff --git a/modules/net-vpn-static/main.tf b/modules/net-vpn-static/main.tf index 6eadd8db6e..559d8d9592 100644 --- a/modules/net-vpn-static/main.tf +++ b/modules/net-vpn-static/main.tf @@ -16,7 +16,7 @@ locals { gateway_address = ( - var.gateway_address == "" + var.gateway_address_create ? google_compute_address.gateway[0].address : var.gateway_address ) @@ -30,7 +30,7 @@ locals { } resource "google_compute_address" "gateway" { - count = var.gateway_address == "" ? 1 : 0 + count = var.gateway_address_create ? 1 : 0 name = "vpn-${var.name}" project = var.project_id region = var.region diff --git a/modules/net-vpn-static/variables.tf b/modules/net-vpn-static/variables.tf index a3ba01d896..f3cebacc58 100644 --- a/modules/net-vpn-static/variables.tf +++ b/modules/net-vpn-static/variables.tf @@ -14,8 +14,14 @@ * limitations under the License. */ +variable "gateway_address_create" { + description = "Create external address assigned to the VPN gateway. Needs to be explicitly set to false to use address in gateway_address variable." + type = bool + default = true +} + variable "gateway_address" { - description = "Optional address assigned to the VPN, leave blank to create one." + description = "Optional address assigned to the VPN gateway. Ignored unless gateway_address_create is set to false." type = string default = "" } From 384e23d5d67efafd3d3629250212f217194373bb Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 5 Jan 2020 11:48:19 +0100 Subject: [PATCH 087/106] vpn dynamic: fix README --- modules/net-vpn-dynamic/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/net-vpn-dynamic/README.md b/modules/net-vpn-dynamic/README.md index 2024278d1c..29f41cce97 100644 --- a/modules/net-vpn-dynamic/README.md +++ b/modules/net-vpn-dynamic/README.md @@ -23,7 +23,8 @@ module "vpn-dynamic" { shared_secret = "" } } -}``` +} +``` ## Variables From 6619333079349b402e97490e566ce61f705dddea Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Mon, 6 Jan 2020 07:38:44 +0100 Subject: [PATCH 088/106] modules/net-vpn-dynamic: fix router name output --- modules/net-vpn-dynamic/outputs.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/net-vpn-dynamic/outputs.tf b/modules/net-vpn-dynamic/outputs.tf index 7d9956e43e..78e435f3f6 100644 --- a/modules/net-vpn-dynamic/outputs.tf +++ b/modules/net-vpn-dynamic/outputs.tf @@ -31,7 +31,7 @@ output "name" { output "router" { description = "Router resource (only if auto-created)." - value = var.router_create == "" ? google_compute_router.router[0] : null + value = var.router_create ? google_compute_router.router[0] : null } output "router_name" { From ef8c3c72b4450147657ddcfd5b2d3990311b8e4c Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Mon, 6 Jan 2020 18:46:08 +0100 Subject: [PATCH 089/106] modules/compute-vm: remove unused variable --- modules/compute-vm/README.md | 1 - modules/compute-vm/variables.tf | 6 ------ 2 files changed, 7 deletions(-) diff --git a/modules/compute-vm/README.md b/modules/compute-vm/README.md index c70ed63559..6cb0fda436 100644 --- a/modules/compute-vm/README.md +++ b/modules/compute-vm/README.md @@ -33,7 +33,6 @@ module "debian-test" { | project_id | Project id. | string | ✓ | | | region | Compute region. | string | ✓ | | | zone | Compute zone. | string | ✓ | | -| *addresses* | Optional internal addresses (only for non-template usage). | list(string) | | [] | | *attached_disk_defaults* | Defaults for attached disks options. | object({...}) | | ... | | *attached_disks* | Additional disks, if options is null defaults will be used in its place. | list(object({...})) | | [] | | *boot_disk* | Boot disk properties. | object({...}) | | ... | diff --git a/modules/compute-vm/variables.tf b/modules/compute-vm/variables.tf index b399140318..b9abf82968 100644 --- a/modules/compute-vm/variables.tf +++ b/modules/compute-vm/variables.tf @@ -14,12 +14,6 @@ * limitations under the License. */ -variable "addresses" { - description = "Optional internal addresses (only for non-template usage)." - type = list(string) - default = [] -} - variable "attached_disks" { description = "Additional disks, if options is null defaults will be used in its place." type = list(object({ From c791718823df5b0d1ac1e1a2f3b578bf8f3e3d95 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Mon, 6 Jan 2020 18:46:33 +0100 Subject: [PATCH 090/106] modules/compute-vm-cos-coredns: initial import --- modules/compute-vm-cos-coredns/Corefile | 5 + modules/compute-vm-cos-coredns/README.md | 44 +++++ .../compute-vm-cos-coredns/cloud-config.yaml | 72 ++++++++ modules/compute-vm-cos-coredns/main.tf | 53 ++++++ modules/compute-vm-cos-coredns/outputs.tf | 50 ++++++ modules/compute-vm-cos-coredns/variables.tf | 162 ++++++++++++++++++ 6 files changed, 386 insertions(+) create mode 100644 modules/compute-vm-cos-coredns/Corefile create mode 100644 modules/compute-vm-cos-coredns/README.md create mode 100644 modules/compute-vm-cos-coredns/cloud-config.yaml create mode 100644 modules/compute-vm-cos-coredns/main.tf create mode 100644 modules/compute-vm-cos-coredns/outputs.tf create mode 100644 modules/compute-vm-cos-coredns/variables.tf diff --git a/modules/compute-vm-cos-coredns/Corefile b/modules/compute-vm-cos-coredns/Corefile new file mode 100644 index 0000000000..f7309f221f --- /dev/null +++ b/modules/compute-vm-cos-coredns/Corefile @@ -0,0 +1,5 @@ +. { + forward . /etc/resolv.conf + log + errors +} diff --git a/modules/compute-vm-cos-coredns/README.md b/modules/compute-vm-cos-coredns/README.md new file mode 100644 index 0000000000..86157d7fb6 --- /dev/null +++ b/modules/compute-vm-cos-coredns/README.md @@ -0,0 +1,44 @@ +# Container Optimized OS CoreDNS module + +## Example + +TODO + + +## Variables + +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| name | Instances base name. | string | ✓ | | +| network_interfaces | Network interfaces configuration. Use self links for Shared VPC, set addresses to null if not needed. | list(object({...})) | ✓ | | +| project_id | Project id. | string | ✓ | | +| region | Compute region. | string | ✓ | | +| zone | Compute zone. | string | ✓ | | +| *boot_disk* | Boot disk properties. | object({...}) | | ... | +| *coredns_corefile* | CoreDNS Corefile path, defaults to sample configuration. | string | | null | +| *coredns_image* | CoreDNS container image. | string | | coredns/coredns | +| *coredns_log_driver* | CoreDNS container log driver (local, gcplogs, etc.). | string | | gcplogs | +| *cos_config* | Configure COS logging and monitoring. | object({...}) | | ... | +| *hostname* | Instance FQDN name. | string | | null | +| *instance_count* | Number of instances to create (only for non-template usage). | number | | 1 | +| *instance_type* | Instance type. | string | | f1-micro | +| *labels* | Instance labels. | map(string) | | {} | +| *metadata* | Instance metadata. | map(string) | | {} | +| *min_cpu_platform* | Minimum CPU platform. | string | | null | +| *options* | Instance options. | object({...}) | | ... | +| *service_account* | Service account email (leave empty to auto-create). | string | | | +| *tags* | Instance tags. | list(string) | | ["dns", "ssh"] | +| *use_instance_template* | Create instance template instead of instances. | bool | | false | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| external_ips | Instance main interface external IP addresses. | | +| instances | Instance resources. | | +| internal_ips | Instance main interface internal IP addresses. | | +| names | Instance names. | | +| self_links | Instance self links. | | +| template | Template resource. | | +| template_name | Template name. | | + diff --git a/modules/compute-vm-cos-coredns/cloud-config.yaml b/modules/compute-vm-cos-coredns/cloud-config.yaml new file mode 100644 index 0000000000..66bd9b4ed6 --- /dev/null +++ b/modules/compute-vm-cos-coredns/cloud-config.yaml @@ -0,0 +1,72 @@ +#cloud-config + +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# https://hub.docker.com/r/coredns/coredns/ +# https://coredns.io/manual/toc/#installation + +# TODO: switch to the gcplogs logging driver, and set driver labels + +write_files: +- path: /var/lib/docker/daemon.json + permissions: 0644 + owner: root + content: | + { + "live-restore": true, + "storage-driver": "overlay2", + "log-opts": { + "max-size": "1024m" + } + } + +# disable systemd-resolved to free port 53 on the loopback interface +- path: /etc/systemd/resolved.conf + permissions: 0644 + owner: root + content: | + [Resolve] + LLMNR=no + DNSStubListener=no + +# Write coreDNS config +- path: /etc/coredns/Corefile + permissions: 0644 + owner: root + content: | + ${indent(4, corefile)} + +# coredns container service +- path: /etc/systemd/system/coredns.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Start CoreDNS container + After=gcr-online.target docker.socket + Wants=gcr-online.target docker.socket docker-events-collector.service + [Service] + ExecStart=/usr/bin/docker run --rm --name=coredns \ + --log-driver=${log_driver} --network host \ + -v /etc/coredns:/etc/coredns \ + ${image} -conf /etc/coredns/Corefile + ExecStop=/usr/bin/docker stop coredns + +runcmd: +- iptables -I INPUT 1 -p tcp -m tcp --dport 53 -m state --state NEW,ESTABLISHED -j ACCEPT +- iptables -I INPUT 1 -p udp -m udp --dport 53 -m state --state NEW,ESTABLISHED -j ACCEPT +- systemctl daemon-reload +- systemctl restart systemd-resolved.service +- systemctl start coredns \ No newline at end of file diff --git a/modules/compute-vm-cos-coredns/main.tf b/modules/compute-vm-cos-coredns/main.tf new file mode 100644 index 0000000000..3a5f0d28db --- /dev/null +++ b/modules/compute-vm-cos-coredns/main.tf @@ -0,0 +1,53 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +data "template_file" "cloud-config" { + template = file("${path.module}/cloud-config.yaml") + + vars = { + corefile = file( + var.coredns_corefile == null + ? "${path.module}/Corefile" + : var.coredns_corefile + ) + image = var.coredns_image + log_driver = var.coredns_log_driver + } +} + +module "cos-coredns" { + source = "../compute-vm" + project_id = var.project_id + region = var.region + zone = var.zone + name = var.name + boot_disk = var.boot_disk + hostname = var.hostname + instance_count = var.instance_count + instance_type = var.instance_type + labels = var.labels + metadata = merge(var.metadata, { + google-logging-enabled = var.cos_config.logging + google-monitoring-enabled = var.cos_config.monitoring + user-data = data.template_file.cloud-config.rendered + }) + min_cpu_platform = var.min_cpu_platform + network_interfaces = var.network_interfaces + options = var.options + service_account = var.service_account + tags = var.tags + use_instance_template = var.use_instance_template +} diff --git a/modules/compute-vm-cos-coredns/outputs.tf b/modules/compute-vm-cos-coredns/outputs.tf new file mode 100644 index 0000000000..4d2154e5a7 --- /dev/null +++ b/modules/compute-vm-cos-coredns/outputs.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "instances" { + description = "Instance resources." + value = module.cos-coredns.instances +} + +output "names" { + description = "Instance names." + value = module.cos-coredns.names +} + +output "self_links" { + description = "Instance self links." + value = module.cos-coredns.self_links +} + +output "internal_ips" { + description = "Instance main interface internal IP addresses." + value = module.cos-coredns.internal_ips +} + +output "external_ips" { + description = "Instance main interface external IP addresses." + value = module.cos-coredns.external_ips +} + +output "template" { + description = "Template resource." + value = module.cos-coredns.template +} + +output "template_name" { + description = "Template name." + value = module.cos-coredns.template_name +} diff --git a/modules/compute-vm-cos-coredns/variables.tf b/modules/compute-vm-cos-coredns/variables.tf new file mode 100644 index 0000000000..61fb473a24 --- /dev/null +++ b/modules/compute-vm-cos-coredns/variables.tf @@ -0,0 +1,162 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "boot_disk" { + description = "Boot disk properties." + type = object({ + image = string + size = number + type = string + }) + default = { + image = "projects/cos-cloud/global/images/family/cos-stable" + type = "pd-ssd" + size = 10 + } +} + +variable "coredns_corefile" { + description = "CoreDNS Corefile path, defaults to sample configuration." + type = string + default = null +} + +variable "coredns_image" { + description = "CoreDNS container image." + type = string + default = "coredns/coredns" +} + +variable "coredns_log_driver" { + description = "CoreDNS container log driver (local, gcplogs, etc.)." + type = string + default = "gcplogs" +} + +variable "cos_config" { + description = "Configure COS logging and monitoring." + type = object({ + logging = bool + monitoring = bool + }) + default = { + logging = false + monitoring = true + } +} + +variable "hostname" { + description = "Instance FQDN name." + type = string + default = null +} + +variable "instance_count" { + description = "Number of instances to create (only for non-template usage)." + type = number + default = 1 +} + +variable "instance_type" { + description = "Instance type." + type = string + default = "f1-micro" +} + +variable "labels" { + description = "Instance labels." + type = map(string) + default = {} +} + +variable "metadata" { + description = "Instance metadata." + type = map(string) + default = {} +} + +variable "min_cpu_platform" { + description = "Minimum CPU platform." + type = string + default = null +} + +variable "name" { + description = "Instances base name." + type = string +} + +variable "network_interfaces" { + description = "Network interfaces configuration. Use self links for Shared VPC, set addresses to null if not needed." + type = list(object({ + nat = bool + network = string + subnetwork = string + addresses = object({ + internal = list(string) + external = list(string) + }) + })) +} + +variable "options" { + description = "Instance options." + type = object({ + allow_stopping_for_update = bool + can_ip_forward = bool + deletion_protection = bool + preemptible = bool + }) + default = { + allow_stopping_for_update = true + can_ip_forward = false + deletion_protection = false + preemptible = false + } +} + +variable "project_id" { + description = "Project id." + type = string +} + +variable "region" { + description = "Compute region." + type = string +} + +variable "service_account" { + description = "Service account email (leave empty to auto-create)." + type = string + default = "" +} + +variable "tags" { + description = "Instance tags." + type = list(string) + default = ["dns", "ssh"] +} + +variable "use_instance_template" { + description = "Create instance template instead of instances." + type = bool + default = false +} + +variable "zone" { + description = "Compute zone." + type = string +} From b48094a43a132e0c755f0aba5a16e26ce65f43ca Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Fri, 10 Jan 2020 09:07:23 +0100 Subject: [PATCH 091/106] Update foundations modules versions (#26) * update foundations modules versions * update Terraform version to v0.12.19 in CI test configuration --- .ci/cloudbuild.test.yaml | 2 +- foundations/business-units/main.tf | 8 ++++---- .../business-units/modules/business-unit-folders/main.tf | 6 +++--- foundations/environments/main.tf | 8 ++++---- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.ci/cloudbuild.test.yaml b/.ci/cloudbuild.test.yaml index afb23d2a95..24de3dfad4 100644 --- a/.ci/cloudbuild.test.yaml +++ b/.ci/cloudbuild.test.yaml @@ -44,7 +44,7 @@ steps: - PYTHONDONTWRITEBYTECODE=true substitutions: - _TERRAFORM_VERSION: 0.12.8 + _TERRAFORM_VERSION: 0.12.19 tags: - "ci" diff --git a/foundations/business-units/main.tf b/foundations/business-units/main.tf index 3252e67c06..2123126170 100644 --- a/foundations/business-units/main.tf +++ b/foundations/business-units/main.tf @@ -26,7 +26,7 @@ locals { module "shared-folder" { source = "terraform-google-modules/folders/google" - version = "2.0.0" + version = "2.0.2" parent = var.root_node names = ["shared"] } @@ -53,7 +53,7 @@ module "project-tf" { module "service-accounts-tf-environments" { source = "terraform-google-modules/service-accounts/google" - version = "2.0.1" + version = "2.0.2" project_id = module.project-tf.project_id org_id = var.organization_id billing_account_id = var.billing_account_id @@ -151,7 +151,7 @@ module "project-audit" { module "bq-audit-export" { source = "terraform-google-modules/log-export/google//modules/bigquery" - version = "3.1.0" + version = "3.2.0" project_id = module.project-audit.project_id dataset_name = "${replace(local.log_sink_name, "-", "_")}" log_sink_writer_identity = module.log-sink-audit.writer_identity @@ -161,7 +161,7 @@ module "bq-audit-export" { module "log-sink-audit" { source = "terraform-google-modules/log-export/google" - version = "3.1.0" + version = "3.2.0" filter = "logName: \"/logs/cloudaudit.googleapis.com%2Factivity\" OR logName: \"/logs/cloudaudit.googleapis.com%2Fsystem_event\"" log_sink_name = local.log_sink_name parent_resource_type = local.log_sink_parent_resource_type diff --git a/foundations/business-units/modules/business-unit-folders/main.tf b/foundations/business-units/modules/business-unit-folders/main.tf index e08616b73d..0b48b5444c 100644 --- a/foundations/business-units/modules/business-unit-folders/main.tf +++ b/foundations/business-units/modules/business-unit-folders/main.tf @@ -18,7 +18,7 @@ module "business-unit-folder" { source = "terraform-google-modules/folders/google" - version = "2.0.0" + version = "2.0.2" parent = var.root_node names = [var.business_unit_folder_name] } @@ -29,7 +29,7 @@ module "business-unit-folder" { module "environment-folders" { source = "terraform-google-modules/folders/google" - version = "2.0.0" + version = "2.0.2" parent = module.business-unit-folder.id names = var.environments set_roles = true @@ -41,4 +41,4 @@ module "environment-folders" { "roles/compute.networkAdmin", "roles/compute.xpnAdmin" ] -} \ No newline at end of file +} diff --git a/foundations/environments/main.tf b/foundations/environments/main.tf index d96745cc54..3aba9f7ef7 100644 --- a/foundations/environments/main.tf +++ b/foundations/environments/main.tf @@ -34,7 +34,7 @@ module "project-tf" { module "service-accounts-tf-environments" { source = "terraform-google-modules/service-accounts/google" - version = "2.0.1" + version = "2.0.2" project_id = module.project-tf.project_id org_id = var.organization_id billing_account_id = var.billing_account_id @@ -78,7 +78,7 @@ module "gcs-tf-environments" { module "folders-top-level" { source = "terraform-google-modules/folders/google" - version = "2.0.0" + version = "2.0.2" parent = var.root_node names = var.environments set_roles = true @@ -118,7 +118,7 @@ module "project-audit" { module "bq-audit-export" { source = "terraform-google-modules/log-export/google//modules/bigquery" - version = "3.1.0" + version = "3.2.0" project_id = module.project-audit.project_id dataset_name = "logs_audit_${replace(var.environments[0], "-", "_")}" log_sink_writer_identity = module.log-sink-audit.writer_identity @@ -129,7 +129,7 @@ module "bq-audit-export" { module "log-sink-audit" { source = "terraform-google-modules/log-export/google" - version = "3.1.0" + version = "3.2.0" filter = "logName: \"/logs/cloudaudit.googleapis.com%2Factivity\" OR logName: \"/logs/cloudaudit.googleapis.com%2Fsystem_event\"" log_sink_name = "logs-audit-${var.environments[0]}" parent_resource_type = "folder" From 0a63efa5ea75a408a8acda9eeb3a6f945c1c9fd4 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Fri, 10 Jan 2020 13:31:19 +0100 Subject: [PATCH 092/106] backport tfdoc from Ludo's branch (#27) --- tools/tfdoc/REQUIREMENTS.txt | 1 + tools/tfdoc/tfdoc.py | 260 +++++++++++++++++++++++++++++++++++ 2 files changed, 261 insertions(+) create mode 100644 tools/tfdoc/REQUIREMENTS.txt create mode 100644 tools/tfdoc/tfdoc.py diff --git a/tools/tfdoc/REQUIREMENTS.txt b/tools/tfdoc/REQUIREMENTS.txt new file mode 100644 index 0000000000..dca9a90964 --- /dev/null +++ b/tools/tfdoc/REQUIREMENTS.txt @@ -0,0 +1 @@ +click diff --git a/tools/tfdoc/tfdoc.py b/tools/tfdoc/tfdoc.py new file mode 100644 index 0000000000..bb6dffb037 --- /dev/null +++ b/tools/tfdoc/tfdoc.py @@ -0,0 +1,260 @@ +#! /usr/bin/env python3 + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections +import enum +import os +import re +import string + +import click + + +MARK_BEGIN = '' +MARK_END = '' +RE_OUTPUTS = re.compile(r'''(?smx) + (?:^\s*output\s*"([^"]+)"\s*\{$) | + (?:^\s*description\s*=\s*"([^"]+)"\s*$) | + (?:^\s*sensitive\s*=\s*(\S+)\s*$) +''') +RE_TYPE = re.compile(r'([\(\{\}\)])') +RE_VARIABLES = re.compile(r'''(?smx) + # empty lines + (^\s*$) | + # block comment + (^\s*/\*.*?\*/$) | + # line comment + (^\s*\#.*?$) | + # variable declaration start + (?:^\s*variable\s*"([^"]+)"\s*\{$) | + # variable description start + (?:^\s*description\s*=\s*"([^"]+)"\s*$) | + # variable type start + (?:^\s*type\s*=\s*(.*?)$) | + # variable default start + (?:^\s*default\s*=\s*"?(.*?)"?\s*$) | + # variable body + (?:^\s*(\S.*?)$) +''') +REPL_VALID = string.digits + string.ascii_letters + ' .,;:_-' + + +OutputData = collections.namedtuple('Output', 'name description sensitive') +OutputToken = enum.Enum('OutputToken', 'NAME DESCRIPTION SENSITIVE') +VariableData = collections.namedtuple( + 'Variable', 'name description type default required') +VariableToken = enum.Enum( + 'VariableToken', + 'EMPTY BLOCK_COMMENT LINE_COMMENT NAME DESCRIPTION TYPE DEFAULT REST') + + +class ItemParsed(Exception): + pass + + +class Output(object): + "Output parsing helper class." + + def __init__(self): + self.in_progress = False + self.name = self.description = self.sensitive = None + + def parse_token(self, token_type, token_data): + if token_type == 'NAME': + if self.in_progress: + raise ItemParsed(self.close()) + self.in_progress = True + self.name = token_data + else: + setattr(self, token_type.lower(), token_data) + + def close(self): + return OutputData(self.name, self.description, self.sensitive) + + +class Variable(object): + "Variable parsing helper class." + + def __init__(self): + self.in_progress = False + self.name = self.description = self.type = self.default = None + self._data = [] + self._data_context = None + + def parse_token(self, token_type, token_data): + if token_type == 'NAME': + if self.in_progress: + raise ItemParsed(self.close()) + self.in_progress = True + self.name = token_data + elif token_type == 'DESCRIPTION': + setattr(self, token_type.lower(), token_data) + elif token_type in ('DEFAULT', 'TYPE'): + self._start(token_type.lower(), token_data) + elif token_type == 'REST': + self._data.append(token_data) + + def _close(self, strip=False): + if self._data_context: + data = self._data + if strip and '}' in data[-1]: + data = data[:-1] + setattr(self, self._data_context, ('\n'.join(data)).strip()) + + def _start(self, context, data): + if context == self._data_context or getattr(self, context): + self._data.append("%s = %s" % (context, data)) + return + self._close() + self._data = [data] + self._data_context = context + + def close(self): + self._close(strip=True) + return VariableData(self.name, self.description, self.type, self.default, + self.default is None) + + +def _escape(s): + "Basic, minimal HTML escaping" + return ''.join(c if c in REPL_VALID else ('&#%s;' % ord(c)) for c in s) + + +def format_outputs(outputs): + "Format outputs." + if not outputs: + return + outputs.sort(key=lambda v: v.name) + yield '| name | description | sensitive |' + yield '|---|---|:---:|' + for o in outputs: + yield '| {name} | {description} | {sensitive} |'.format( + name=o.name, description=o.description, + sensitive='✓' if o.sensitive else '') + + +def format_type(type_spec): + "Format variable type." + if not type_spec: + return '' + buffer = [] + stack = [] + for t in RE_TYPE.split(type_spec.split("\n")[0]): + if not t: + continue + if t in '({': + stack.append(t) + elif t in '})': + stack.pop() + buffer.append(t) + for t in reversed(stack): + buffer.append(')' if t == '(' else '}') + return ''.join(buffer).replace('object({})', 'object({...})') + + +def format_variables(variables, required_first=True): + "Format variables." + if not variables: + return + variables.sort(key=lambda v: v.name) + variables.sort(key=lambda v: v.required, reverse=True) + yield '| name | description | type | required | default |' + yield '|---|---|:---: |:---:|:---:|' + row = ( + '| {name} | {description} | {type} ' + '| {required} | {default} |' + ) + for v in variables: + default = default_spec = type_spec = '' + if not v.required: + default = '{default}' + if '\n' in v.default: + default = default.format(title=_escape(v.default), default='...') + else: + default = default.format(title='', default=v.default or '') + if v.type and '(' in v.type: + type_spec = _escape(v.type) + yield row.format( + name=v.name if v.required else '*%s*' % v.name, + description=v.description, required='✓' if v.required else '', + type=format_type(v.type), type_spec=type_spec, + default=default + ) + + +def get_doc(variables, outputs): + "Return formatted documentation." + buffer = ['## Variables\n'] + for line in format_variables(variables): + buffer.append(line) + buffer.append('\n## Outputs\n') + for line in format_outputs(outputs): + buffer.append(line) + return '\n'.join(buffer) + + +def parse_items(content, item_re, item_enum, item_class, item_data_class): + "Parse variable or output items in data." + item = item_class() + for m in item_re.finditer(content): + try: + item.parse_token(item_enum(m.lastindex).name, m.group(m.lastindex)) + except ItemParsed as e: + item = item_class() + item.parse_token(item_enum(m.lastindex).name, m.group(m.lastindex)) + yield e.args[0] + if item.in_progress: + yield item.close() + + +def replace_doc(module, doc): + "Replace document in module's README.md file." + try: + readme = open(os.path.join(module, 'README.md')).read() + m = re.search('(?sm)%s.*%s' % (MARK_BEGIN, MARK_END), readme) + if not m: + raise SystemExit('Pattern not found in README file.') + replacement = "{pre}{begin}\n{doc}\n{end}{post}".format( + pre=readme[:m.start()], begin=MARK_BEGIN, doc=doc, + end=MARK_END, post=readme[m.end():]) + open(os.path.join(module, 'README.md'), 'w').write(replacement) + except (IOError, OSError) as e: + raise SystemExit('Error replacing in README: %s' % e) + + +@click.command() +@click.argument('module', type=click.Path(exists=True)) +@click.option('--replace/--no-replace', default=True) +def main(module=None, replace=True): + "Program entry point." + try: + with open(os.path.join(module, 'variables.tf')) as file: + variables = [v for v in parse_items( + file.read(), RE_VARIABLES, VariableToken, Variable, VariableData)] + with open(os.path.join(module, 'outputs.tf')) as file: + outputs = [o for o in parse_items( + file.read(), RE_OUTPUTS, OutputToken, Output, OutputData)] + except (IOError, OSError) as e: + raise SystemExit(e) + doc = get_doc(variables, outputs) + if replace: + replace_doc(module, doc) + else: + print(doc) + + +if __name__ == '__main__': + main() From 253c51d07ccfbfc58a5552fe8a9fd1123af65d98 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Fri, 10 Jan 2020 13:44:54 +0100 Subject: [PATCH 093/106] Update docs using tfdoc format (#28) * update README files * set all types on variables --- foundations/business-units/README.md | 75 ++++++++++----------- foundations/business-units/variables.tf | 7 ++ foundations/environments/README.md | 63 +++++++++-------- foundations/environments/variables.tf | 9 +++ infrastructure/hub-and-spoke-vpns/README.md | 54 +++++++-------- infrastructure/shared-vpc/README.md | 53 +++++++-------- infrastructure/shared-vpc/variables.tf | 18 +++++ tools/tfdoc/tfdoc.py | 0 8 files changed, 153 insertions(+), 126 deletions(-) mode change 100644 => 100755 tools/tfdoc/tfdoc.py diff --git a/foundations/business-units/README.md b/foundations/business-units/README.md index 297a0ab7fa..684579bc06 100644 --- a/foundations/business-units/README.md +++ b/foundations/business-units/README.md @@ -25,45 +25,44 @@ The number of resources in this sample is kept to a minimum so as to make it gen This sample uses a top-level folder to encapsulate projects that host resources that are not specific to a single environment. If no shared services are needed,the Terraform and audit modules can be easily attached to the root node, and the shared services folder and project removed from `main.tf`. - -## Inputs + +## Variables -| Name | Description | Type | Default | Required | -|------|-------------|:----:|:-----:|:-----:| -| audit\_viewers | Audit project viewers, in IAM format. | list | `` | no | -| billing\_account\_id | Billing account id used as default for new projects. | string | n/a | yes | -| business\_unit\_1\_name | Business unit 1 short name. | string | n/a | yes | -| business\_unit\_2\_name | Business unit 2 short name. | string | n/a | yes | -| business\_unit\_3\_name | Business unit 3 short name. | string | n/a | yes | -| environments | Environment short names. | list(string) | n/a | yes | -| gcs\_location | GCS bucket location. | string | `"EU"` | no | -| generate\_service\_account\_keys | Generate and store service account keys in the state file. | string | `"false"` | no | -| organization\_id | Organization id. | string | n/a | yes | -| prefix | Prefix used for resources that need unique names. | string | n/a | yes | -| project\_services | Service APIs enabled by default in new projects. | list | `` | no | -| root\_node | Root node for the new hierarchy, either 'organizations/org_id' or 'folders/folder_id'. | string | n/a | yes | -| shared\_bindings\_members | List of comma-delimited IAM-format members for the additional shared project bindings. | list | `` | no | -| shared\_bindings\_roles | List of roles for additional shared project bindings. | list | `` | no | -| terraform\_owners | Terraform project owners, in IAM format. | list | `` | no | +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| billing_account_id | Billing account id used as default for new projects. | string | ✓ | | +| business_unit_1_name | Business unit 1 short name. | string | ✓ | | +| business_unit_2_name | Business unit 2 short name. | string | ✓ | | +| business_unit_3_name | Business unit 3 short name. | string | ✓ | | +| environments | Environment short names. | list(string) | ✓ | | +| organization_id | Organization id. | string | ✓ | | +| prefix | Prefix used for resources that need unique names. | string | ✓ | | +| root_node | Root node for the new hierarchy, either 'organizations/org_id' or 'folders/folder_id'. | string | ✓ | | +| *audit_viewers* | Audit project viewers, in IAM format. | list(string) | | [] | +| *gcs_location* | GCS bucket location. | string | | EU | +| *generate_service_account_keys* | Generate and store service account keys in the state file. | bool | | false | +| *project_services* | Service APIs enabled by default in new projects. | list(string) | | ... | +| *shared_bindings_members* | List of comma-delimited IAM-format members for the additional shared project bindings. | list(string) | | [] | +| *shared_bindings_roles* | List of roles for additional shared project bindings. | list(string) | | [] | +| *terraform_owners* | Terraform project owners, in IAM format. | list(string) | | [] | ## Outputs -| Name | Description | -|------|-------------| -| audit\_logs\_bq\_dataset | Bigquery dataset for the audit logs export. | -| audit\_logs\_project | Project that holds the audit logs export resources. | -| bootstrap\_tf\_gcs\_bucket | GCS bucket used for the bootstrap Terraform state. | -| business\_unit\_1\_environment\_folders\_ids | Business unit 1 environment folders. | -| business\_unit\_1\_folder\_id | Business unit 1 top-level folder ID. | -| business\_unit\_2\_environment\_folders\_ids | Business unit 2 environment folders. | -| business\_unit\_2\_folder\_id | Business unit 2 top-level folder ID. | -| business\_unit\_3\_environment\_folders\_ids | Business unit 3 environment folders. | -| business\_unit\_3\_folder\_id | Business unit 3 top-level folder ID. | -| environment\_service\_account\_keys | Service account keys used to run each environment Terraform modules. | -| environment\_service\_accounts | Service accounts used to run each environment Terraform modules. | -| environment\_tf\_gcs\_buckets | GCS buckets used for each environment Terraform state. | -| shared\_folder\_id | Shared folder ID. | -| shared\_resources\_project | Project that holdes resources shared across business units. | -| terraform\_project | Project that holds the base Terraform resources. | - - +| name | description | sensitive | +|---|---|:---:| +| audit_logs_bq_dataset | Bigquery dataset for the audit logs export. | | +| audit_logs_project | Project that holds the audit logs export resources. | | +| bootstrap_tf_gcs_bucket | GCS bucket used for the bootstrap Terraform state. | | +| business_unit_1_environment_folders_ids | Business unit 1 environment folders. | | +| business_unit_1_folder_id | Business unit 1 top-level folder ID. | | +| business_unit_2_environment_folders_ids | Business unit 2 environment folders. | | +| business_unit_2_folder_id | Business unit 2 top-level folder ID. | | +| business_unit_3_environment_folders_ids | Business unit 3 environment folders. | | +| business_unit_3_folder_id | Business unit 3 top-level folder ID. | | +| environment_service_account_keys | Service account keys used to run each environment Terraform modules. | ✓ | +| environment_service_accounts | Service accounts used to run each environment Terraform modules. | | +| environment_tf_gcs_buckets | GCS buckets used for each environment Terraform state. | | +| shared_folder_id | Shared folder ID. | | +| shared_resources_project | Project that holdes resources shared across business units. | | +| terraform_project | Project that holds the base Terraform resources. | | + diff --git a/foundations/business-units/variables.tf b/foundations/business-units/variables.tf index c25a175570..3dbecffd1a 100644 --- a/foundations/business-units/variables.tf +++ b/foundations/business-units/variables.tf @@ -14,6 +14,7 @@ variable "audit_viewers" { description = "Audit project viewers, in IAM format." + type = list(string) default = [] } @@ -44,11 +45,13 @@ variable "environments" { variable "generate_service_account_keys" { description = "Generate and store service account keys in the state file." + type = bool default = false } variable "gcs_location" { description = "GCS bucket location." + type = string default = "EU" } @@ -70,21 +73,25 @@ variable "root_node" { variable "shared_bindings_members" { description = "List of comma-delimited IAM-format members for the additional shared project bindings." # example: ["user:a@example.com,b@example.com", "user:c@example.com"] + type = list(string) default = [] } variable "shared_bindings_roles" { description = "List of roles for additional shared project bindings." # example: ["roles/storage.objectViewer", "roles/storage.admin"] + type = list(string) default = [] } variable "terraform_owners" { description = "Terraform project owners, in IAM format." + type = list(string) default = [] } variable "project_services" { description = "Service APIs enabled by default in new projects." + type = list(string) default = [ "resourceviews.googleapis.com", "stackdriver.googleapis.com", diff --git a/foundations/environments/README.md b/foundations/environments/README.md index 70208f6326..3e13134f20 100644 --- a/foundations/environments/README.md +++ b/foundations/environments/README.md @@ -27,38 +27,37 @@ For more complex setups where multiple shared services projects are needed to en If no shared services are needed, the shared service project module can of course be removed from `main.tf`. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|:----:|:-----:|:-----:| -| audit\_viewers | Audit project viewers, in IAM format. | list | `` | no | -| billing\_account\_id | Billing account id used as default for new projects. | string | n/a | yes | -| environments | Environment short names. | list(string) | n/a | yes | -| gcs\_location | GCS bucket location. | string | `"EU"` | no | -| generate\_service\_account\_keys | Generate and store service account keys in the state file. | string | `"false"` | no | -| grant\_xpn\_folder\_roles | Grant roles needed for Shared VPC creation to service accounts at the environment folder level. | string | `"true"` | no | -| grant\_xpn\_org\_roles | Grant roles needed for Shared VPC creation to service accounts at the organization level. | string | `"false"` | no | -| organization\_id | Organization id. | string | n/a | yes | -| prefix | Prefix used for resources that need unique names. | string | n/a | yes | -| project\_services | Service APIs enabled by default in new projects. | list | `` | no | -| root\_node | Root node for the new hierarchy, either 'organizations/org_id' or 'folders/folder_id'. | string | n/a | yes | -| shared\_bindings\_members | List of comma-delimited IAM-format members for the additional shared project bindings. | list | `` | no | -| shared\_bindings\_roles | List of roles for additional shared project bindings. | list | `` | no | -| terraform\_owners | Terraform project owners, in IAM format. | list | `` | no | + +## Variables + +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| billing_account_id | Billing account id used as default for new projects. | string | ✓ | | +| environments | Environment short names. | list(string) | ✓ | | +| organization_id | Organization id. | string | ✓ | | +| prefix | Prefix used for resources that need unique names. | string | ✓ | | +| root_node | Root node for the new hierarchy, either 'organizations/org_id' or 'folders/folder_id'. | string | ✓ | | +| *audit_viewers* | Audit project viewers, in IAM format. | list(string) | | [] | +| *gcs_location* | GCS bucket location. | string | | EU | +| *generate_service_account_keys* | Generate and store service account keys in the state file. | bool | | false | +| *grant_xpn_folder_roles* | Grant roles needed for Shared VPC creation to service accounts at the environment folder level. | bool | | true | +| *grant_xpn_org_roles* | Grant roles needed for Shared VPC creation to service accounts at the organization level. | bool | | false | +| *project_services* | Service APIs enabled by default in new projects. | list(string) | | ... | +| *shared_bindings_members* | List of comma-delimited IAM-format members for the additional shared project bindings. | list(string) | | [] | +| *shared_bindings_roles* | List of roles for additional shared project bindings. | list(string) | | [] | +| *terraform_owners* | Terraform project owners, in IAM format. | list(string) | | [] | ## Outputs -| Name | Description | -|------|-------------| -| audit\_logs\_bq\_dataset | Bigquery dataset for the audit logs export. | -| audit\_logs\_project | Project that holds the audit logs export resources. | -| bootstrap\_tf\_gcs\_bucket | GCS bucket used for the bootstrap Terraform state. | -| environment\_folders | Top-level environment folders. | -| environment\_service\_account\_keys | Service account keys used to run each environment Terraform modules. | -| environment\_service\_accounts | Service accounts used to run each environment Terraform modules. | -| environment\_tf\_gcs\_buckets | GCS buckets used for each environment Terraform state. | -| shared\_resources\_project | Project that holdes resources shared across environments. | -| terraform\_project | Project that holds the base Terraform resources. | - - +| name | description | sensitive | +|---|---|:---:| +| audit_logs_bq_dataset | Bigquery dataset for the audit logs export. | | +| audit_logs_project | Project that holds the audit logs export resources. | | +| bootstrap_tf_gcs_bucket | GCS bucket used for the bootstrap Terraform state. | | +| environment_folders | Top-level environment folders. | | +| environment_service_account_keys | Service account keys used to run each environment Terraform modules. | ✓ | +| environment_service_accounts | Service accounts used to run each environment Terraform modules. | | +| environment_tf_gcs_buckets | GCS buckets used for each environment Terraform state. | | +| shared_resources_project | Project that holdes resources shared across environments. | | +| terraform_project | Project that holds the base Terraform resources. | | + diff --git a/foundations/environments/variables.tf b/foundations/environments/variables.tf index 2b8e6f098f..113fcddb4a 100644 --- a/foundations/environments/variables.tf +++ b/foundations/environments/variables.tf @@ -14,6 +14,7 @@ variable "audit_viewers" { description = "Audit project viewers, in IAM format." + type = list(string) default = [] } @@ -29,21 +30,25 @@ variable "environments" { variable "generate_service_account_keys" { description = "Generate and store service account keys in the state file." + type = bool default = false } variable "gcs_location" { description = "GCS bucket location." + type = string default = "EU" } variable "grant_xpn_org_roles" { description = "Grant roles needed for Shared VPC creation to service accounts at the organization level." + type = bool default = false } variable "grant_xpn_folder_roles" { description = "Grant roles needed for Shared VPC creation to service accounts at the environment folder level." + type = bool default = true } @@ -65,21 +70,25 @@ variable "root_node" { variable "shared_bindings_members" { description = "List of comma-delimited IAM-format members for the additional shared project bindings." # example: ["user:a@example.com,b@example.com", "user:c@example.com"] + type = list(string) default = [] } variable "shared_bindings_roles" { description = "List of roles for additional shared project bindings." # example: ["roles/storage.objectViewer", "roles/storage.admin"] + type = list(string) default = [] } variable "terraform_owners" { description = "Terraform project owners, in IAM format." + type = list(string) default = [] } variable "project_services" { description = "Service APIs enabled by default in new projects." + type = list(string) default = [ "resourceviews.googleapis.com", "stackdriver.googleapis.com", diff --git a/infrastructure/hub-and-spoke-vpns/README.md b/infrastructure/hub-and-spoke-vpns/README.md index 855dd684a3..adbc4fcd37 100644 --- a/infrastructure/hub-and-spoke-vpns/README.md +++ b/infrastructure/hub-and-spoke-vpns/README.md @@ -48,34 +48,32 @@ SSH access to instances is configured via [OS Login](https://cloud.google.com/co - Please refer to the [bug](https://github.com/terraform-providers/terraform-provider-google/issues/3753) for more details. - Please refer to the [documentation](https://cloud.google.com/dns/zones/#creating_a_dns_policy_that_enables_inbound_dns_forwarding) on how to get the IPs with `gcloud`. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|:----:|:-----:|:-----:| -| forwarding\_dns\_zone\_domain | Forwarding DNS Zone Domain. | string | `"on-prem.local."` | no | -| forwarding\_dns\_zone\_name | Forwarding DNS Zone Name. | string | `"on-prem-local"` | no | -| forwarding\_zone\_server\_addresses | Forwarding DNS Zone Server Addresses | list(string) | `` | no | -| hub\_bgp\_asn | Hub BGP ASN. | number | `"64515"` | no | -| hub\_project\_id | Hub Project id. Same project can be used for hub and spokes. | string | n/a | yes | -| hub\_subnets | Hub VPC subnets configuration. | object | `` | no | -| private\_dns\_zone\_domain | Private DNS Zone Domain. | string | `"gcp.local."` | no | -| private\_dns\_zone\_name | Private DNS Zone Name. | string | `"gcp-local"` | no | -| spoke\_1\_bgp\_asn | Spoke 1 BGP ASN. | number | `"64516"` | no | -| spoke\_1\_project\_id | Spoke 1 Project id. Same project can be used for hub and spokes. | string | n/a | yes | -| spoke\_1\_subnets | Spoke 1 VPC subnets configuration. | list | `` | no | -| spoke\_2\_bgp\_asn | Spoke 2 BGP ASN. | number | `"64517"` | no | -| spoke\_2\_project\_id | Spoke 2 Project id. Same project can be used for hub and spokes. | string | n/a | yes | -| spoke\_2\_subnets | Spoke 2 VPC subnets configuration. | list | `` | no | -| spoke\_to\_spoke\_route\_advertisement | Use custom route advertisement in hub routers to advertise all spoke subnets. | bool | `"true"` | no | + +## Variables + +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| hub_project_id | Hub Project id. Same project can be used for hub and spokes. | string | ✓ | | +| spoke_1_project_id | Spoke 1 Project id. Same project can be used for hub and spokes. | string | ✓ | | +| spoke_2_project_id | Spoke 2 Project id. Same project can be used for hub and spokes. | string | ✓ | | +| *forwarding_dns_zone_domain* | Forwarding DNS Zone Domain. | string | | on-prem.local. | +| *forwarding_dns_zone_name* | Forwarding DNS Zone Name. | string | | on-prem-local | +| *forwarding_zone_server_addresses* | Forwarding DNS Zone Server Addresses | list(string) | | ["8.8.8.8", "8.8.4.4"] | +| *hub_bgp_asn* | Hub BGP ASN. | number | | 64515 | +| *hub_subnets* | Hub VPC subnets configuration. | list(object({...})) | | ... | +| *private_dns_zone_domain* | Private DNS Zone Domain. | string | | gcp.local. | +| *private_dns_zone_name* | Private DNS Zone Name. | string | | gcp-local | +| *spoke_1_bgp_asn* | Spoke 1 BGP ASN. | number | | 64516 | +| *spoke_1_subnets* | Spoke 1 VPC subnets configuration. | | | ... | +| *spoke_2_bgp_asn* | Spoke 2 BGP ASN. | number | | 64517 | +| *spoke_2_subnets* | Spoke 2 VPC subnets configuration. | | | ... | +| *spoke_to_spoke_route_advertisement* | Use custom route advertisement in hub routers to advertise all spoke subnets. | bool | | true | ## Outputs -| Name | Description | -|------|-------------| -| hub | Hub network resources. | -| spoke-1 | Spoke1 network resources. | -| spoke-2 | Spoke2 network resources. | -| test-instances | Test instance attributes. | - - \ No newline at end of file +| name | description | sensitive | +|---|---|:---:| +| hub | Hub network resources. | | +| spoke-1 | Spoke1 network resources. | | +| spoke-2 | Spoke2 network resources. | | + diff --git a/infrastructure/shared-vpc/README.md b/infrastructure/shared-vpc/README.md index 75ef68fde5..9bec065a64 100644 --- a/infrastructure/shared-vpc/README.md +++ b/infrastructure/shared-vpc/README.md @@ -31,34 +31,31 @@ The networking and GKE instances have `dig` and the `mysql` client installed via There's a minor glitch that can surface running `terraform destroy`, with a simple workaround. The glitch is due to a delay between the API reporting service project removal from the Shared VPC as successful (`google_compute_shared_vpc_service_project` resources destroyed), and the Shared VPC resource being aligned with that event. This results in an error that prevents disabling the Shared VPC feature: `Error disabling Shared VPC Host [...] Cannot disable project as a shared VPC host because it has active service projects.`. The workaround is to run `terraform destroy` again after a few seconds, giving the Shared VPC resource time to be in sync with service project removal. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|:----:|:-----:|:-----:| -| billing\_account\_id | Billing account id used as default for new projects. | string | n/a | yes | -| kms\_keyring\_location | Location used for the KMS keyring. | string | `"europe"` | no | -| kms\_keyring\_name | Name used for the KMS keyring. | string | `"svpc-example"` | no | -| oslogin\_admins\_gce | GCE project oslogin admin members, in IAM format. | list | `` | no | -| oslogin\_users\_gce | GCE project oslogin user members, in IAM format. | list | `` | no | -| owners\_gce | GCE project owners, in IAM format. | list | `` | no | -| owners\_gke | GKE project owners, in IAM format. | list | `` | no | -| owners\_host | Host project owners, in IAM format. | list | `` | no | -| prefix | Prefix used for resources that need unique names. | string | n/a | yes | -| project\_services | Service APIs enabled by default in new projects. | list | `` | no | -| root\_node | Hierarchy node where projects will be created, 'organizations/org_id' or 'folders/folder_id'. | string | n/a | yes | -| subnet\_secondary\_ranges | Shared VPC subnets secondary range definitions. | map | `` | no | -| subnets | Shared VPC subnet definitions. | list | `` | no | + +## Variables + +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| billing_account_id | Billing account id used as default for new projects. | string | ✓ | | +| prefix | Prefix used for resources that need unique names. | string | ✓ | | +| root_node | Hierarchy node where projects will be created, 'organizations/org_id' or 'folders/folder_id'. | string | ✓ | | +| *kms_keyring_location* | Location used for the KMS keyring. | string | | europe | +| *kms_keyring_name* | Name used for the KMS keyring. | string | | svpc-example | +| *oslogin_admins_gce* | GCE project oslogin admin members, in IAM format. | list(string) | | [] | +| *oslogin_users_gce* | GCE project oslogin user members, in IAM format. | list(string) | | [] | +| *owners_gce* | GCE project owners, in IAM format. | list(string) | | [] | +| *owners_gke* | GKE project owners, in IAM format. | list(string) | | [] | +| *owners_host* | Host project owners, in IAM format. | list(string) | | [] | +| *project_services* | Service APIs enabled by default in new projects. | list(string) | | ... | +| *subnet_secondary_ranges* | Shared VPC subnets secondary range definitions. | map(list(object({...}))) | | ... | +| *subnets* | Shared VPC subnet definitions. | list(object({...})) | | ... | ## Outputs -| Name | Description | -|------|-------------| -| host\_project\_id | VPC host project id. | -| mysql-root-password | Password for the test MySQL db root user. | -| service\_project\_ids | Service project ids. | -| test-instances | Test instance names. | -| vpc\_name | Shared VPC name | -| vpc\_subnets | Shared VPC subnets. | - - +| name | description | sensitive | +|---|---|:---:| +| host_project_id | VPC host project id. | | +| service_project_ids | Service project ids. | | +| vpc_name | Shared VPC name | | +| vpc_subnets | Shared VPC subnets. | | + diff --git a/infrastructure/shared-vpc/variables.tf b/infrastructure/shared-vpc/variables.tf index 0efe0babea..0f1f03fa30 100644 --- a/infrastructure/shared-vpc/variables.tf +++ b/infrastructure/shared-vpc/variables.tf @@ -19,36 +19,43 @@ variable "billing_account_id" { variable "kms_keyring_location" { description = "Location used for the KMS keyring." + type = string default = "europe" } variable "kms_keyring_name" { description = "Name used for the KMS keyring." + type = string default = "svpc-example" } variable "oslogin_admins_gce" { description = "GCE project oslogin admin members, in IAM format." + type = list(string) default = [] } variable "oslogin_users_gce" { description = "GCE project oslogin user members, in IAM format." + type = list(string) default = [] } variable "owners_gce" { description = "GCE project owners, in IAM format." + type = list(string) default = [] } variable "owners_gke" { description = "GKE project owners, in IAM format." + type = list(string) default = [] } variable "owners_host" { description = "Host project owners, in IAM format." + type = list(string) default = [] } @@ -64,6 +71,12 @@ variable "root_node" { variable "subnets" { description = "Shared VPC subnet definitions." + type = list(object({ + subnet_name = string + subnet_ip = string + subnet_region = string + subnet_private_access = string + })) default = [ { subnet_name = "networking" @@ -88,6 +101,10 @@ variable "subnets" { variable "subnet_secondary_ranges" { description = "Shared VPC subnets secondary range definitions." + type = map(list(object({ + range_name = string + ip_cidr_range = string + }))) default = { networking = [], gce = [], @@ -106,6 +123,7 @@ variable "subnet_secondary_ranges" { variable "project_services" { description = "Service APIs enabled by default in new projects." + type = list(string) default = [ "resourceviews.googleapis.com", "stackdriver.googleapis.com", diff --git a/tools/tfdoc/tfdoc.py b/tools/tfdoc/tfdoc.py old mode 100644 new mode 100755 From 9877cb1b0b842ad630055f18ef31145157153a7e Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Fri, 10 Jan 2020 18:22:46 +0100 Subject: [PATCH 094/106] foundations/environments: move log filter to a variable, use org for xpn by default --- foundations/environments/main.tf | 2 +- foundations/environments/variables.tf | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/foundations/environments/main.tf b/foundations/environments/main.tf index 3aba9f7ef7..913817ab2e 100644 --- a/foundations/environments/main.tf +++ b/foundations/environments/main.tf @@ -130,7 +130,7 @@ module "bq-audit-export" { module "log-sink-audit" { source = "terraform-google-modules/log-export/google" version = "3.2.0" - filter = "logName: \"/logs/cloudaudit.googleapis.com%2Factivity\" OR logName: \"/logs/cloudaudit.googleapis.com%2Fsystem_event\"" + filter = var.audit_filter log_sink_name = "logs-audit-${var.environments[0]}" parent_resource_type = "folder" parent_resource_id = split("/", module.folders-top-level.ids_list[0])[1] diff --git a/foundations/environments/variables.tf b/foundations/environments/variables.tf index 113fcddb4a..720b6333a7 100644 --- a/foundations/environments/variables.tf +++ b/foundations/environments/variables.tf @@ -12,6 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +variable "audit_filter" { + description = "Audit log filter used for the log sink." + type = string + default = < Date: Fri, 10 Jan 2020 18:53:02 +0100 Subject: [PATCH 095/106] foundations/environments: do not use liens by default --- foundations/environments/main.tf | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/foundations/environments/main.tf b/foundations/environments/main.tf index 913817ab2e..1d8da9229b 100644 --- a/foundations/environments/main.tf +++ b/foundations/environments/main.tf @@ -25,9 +25,9 @@ module "project-tf" { billing_account = var.billing_account_id prefix = var.prefix name = "terraform" - lien_reason = "terraform" - owners = var.terraform_owners - activate_apis = var.project_services + # lien_reason = "terraform" + owners = var.terraform_owners + activate_apis = var.project_services } # per-environment service accounts @@ -107,7 +107,7 @@ module "project-audit" { billing_account = var.billing_account_id prefix = var.prefix name = "audit" - lien_reason = "audit" + # lien_reason = "audit" activate_apis = concat(var.project_services, [ "bigquery.googleapis.com", ]) @@ -153,7 +153,6 @@ module "project-shared-resources" { billing_account = var.billing_account_id prefix = var.prefix name = "shared" - lien_reason = "shared" activate_apis = var.project_services extra_bindings_roles = var.shared_bindings_roles extra_bindings_members = var.shared_bindings_members From c32ea3102245786a977370e973f146a54c1774d4 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sat, 11 Jan 2020 23:10:18 +0100 Subject: [PATCH 096/106] modules/ntp-vpc: better shared_vpc_host variable description --- modules/logging-sinks/main.tf | 15 +++++++++++++++ modules/logging-sinks/outputs.tf | 0 modules/logging-sinks/variables.tf | 15 +++++++++++++++ modules/logging-sinks/versions.tf | 19 +++++++++++++++++++ modules/net-vpc/variables.tf | 2 +- 5 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 modules/logging-sinks/main.tf create mode 100644 modules/logging-sinks/outputs.tf create mode 100644 modules/logging-sinks/variables.tf create mode 100644 modules/logging-sinks/versions.tf diff --git a/modules/logging-sinks/main.tf b/modules/logging-sinks/main.tf new file mode 100644 index 0000000000..83185dca24 --- /dev/null +++ b/modules/logging-sinks/main.tf @@ -0,0 +1,15 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/modules/logging-sinks/outputs.tf b/modules/logging-sinks/outputs.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/modules/logging-sinks/variables.tf b/modules/logging-sinks/variables.tf new file mode 100644 index 0000000000..83185dca24 --- /dev/null +++ b/modules/logging-sinks/variables.tf @@ -0,0 +1,15 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/modules/logging-sinks/versions.tf b/modules/logging-sinks/versions.tf new file mode 100644 index 0000000000..ce6918e09d --- /dev/null +++ b/modules/logging-sinks/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12.6" +} diff --git a/modules/net-vpc/variables.tf b/modules/net-vpc/variables.tf index 8863e9957c..d44f2f76f8 100644 --- a/modules/net-vpc/variables.tf +++ b/modules/net-vpc/variables.tf @@ -85,7 +85,7 @@ variable "routing_mode" { } variable "shared_vpc_host" { - description = "Makes this project a Shared VPC host if 'true' (default 'false')" + description = "Enable shared VPC for this project." type = bool default = false } From 77c90fb28124c7f16e69920ffeb32cbcbc8ed43e Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 12 Jan 2020 00:09:18 +0100 Subject: [PATCH 097/106] modules/logging-sinks: initial version --- modules/logging-sinks/README.md | 27 +++++ modules/logging-sinks/main.tf | 102 ++++++++++++++++++ modules/logging-sinks/outputs.tf | 30 ++++++ modules/logging-sinks/variables.tf | 17 +++ .../fixtures/test_variable/main.tf | 76 +++++++++++++ 5 files changed, 252 insertions(+) create mode 100644 modules/logging-sinks/README.md create mode 100644 tests/modules/logging_sinks/fixtures/test_variable/main.tf diff --git a/modules/logging-sinks/README.md b/modules/logging-sinks/README.md new file mode 100644 index 0000000000..bc12869d48 --- /dev/null +++ b/modules/logging-sinks/README.md @@ -0,0 +1,27 @@ +# Terraform Logging Sinks Module + +This module allows easy creation of one or more logging sinks. + +## Example + +```hcl +module "sinks" { +} +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| *sinks* | Logging sinks that will be created, options default to true except for unique_writer_identity. | list(object({...})) | | [] | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| names | Log sink names. | | +| sinks | Log sink resources. | | +| writer_identities | Log sink writer identities. | | + + diff --git a/modules/logging-sinks/main.tf b/modules/logging-sinks/main.tf index 83185dca24..f2d772a864 100644 --- a/modules/logging-sinks/main.tf +++ b/modules/logging-sinks/main.tf @@ -13,3 +13,105 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +locals { + sinks = [ + for sink in var.sinks : merge( + sink, + zipmap(["resource_type", "resource_id"], split("/", sink.resource)), + { + to_bigquery = substr(sink.destination, 0, 8) == "bigquery", + options = sink.options == null ? {} : sink.options + } + ) + ] + sink_resources = concat( + [for _, sink in google_logging_organization_sink.sinks : sink], + [for _, sink in google_logging_billing_account_sink.sinks : sink], + [for _, sink in google_logging_folder_sink.sinks : sink], + [for _, sink in google_logging_project_sink.sinks : sink], + ) +} + +resource "google_logging_organization_sink" "sinks" { + for_each = { + for sink in local.sinks : + "${sink.resource}/${sink.name}" => sink if sink.resource_type == "organizations" + } + name = each.value.name + org_id = each.value.resource_id + filter = each.value.filter + destination = each.value.destination + include_children = lookup(each.value.options, "include_children", true) + dynamic bigquery_options { + for_each = each.value.to_bigquery ? ["1"] : [] + iterator = config + content { + use_partitioned_tables = lookup( + each.value.options, "bigquery_partitioned_tables", true + ) + } + } +} + +resource "google_logging_billing_account_sink" "sinks" { + for_each = { + for sink in local.sinks : + "${sink.resource}/${sink.name}" => sink if sink.resource_type == "billing_accounts" + } + name = each.value.name + billing_account = each.value.resource_id + filter = each.value.filter + destination = each.value.destination + dynamic bigquery_options { + for_each = each.value.to_bigquery ? ["1"] : [] + iterator = config + content { + use_partitioned_tables = lookup( + each.value.options, "bigquery_partitioned_tables", true + ) + } + } +} + +resource "google_logging_folder_sink" "sinks" { + for_each = { + for sink in local.sinks : + "${sink.resource}/${sink.name}" => sink if sink.resource_type == "folders" + } + name = each.value.name + folder = each.value.resource + filter = each.value.filter + destination = each.value.destination + include_children = lookup(each.value.options, "include_children", true) + dynamic bigquery_options { + for_each = each.value.to_bigquery ? ["1"] : [] + iterator = config + content { + use_partitioned_tables = lookup( + each.value.options, "bigquery_partitioned_tables", true + ) + } + } +} + +resource "google_logging_project_sink" "sinks" { + for_each = { + for sink in local.sinks : + "${sink.resource}/${sink.name}" => sink if sink.resource_type == "projects" + } + name = each.value.name + project = each.value.resource_id + filter = each.value.filter + destination = each.value.destination + unique_writer_identity = lookup(each.value.options, "unique_writer_identity", false) + dynamic bigquery_options { + for_each = each.value.to_bigquery ? ["1"] : [] + iterator = config + content { + use_partitioned_tables = lookup( + each.value.options, "bigquery_partitioned_tables", true + ) + } + } +} diff --git a/modules/logging-sinks/outputs.tf b/modules/logging-sinks/outputs.tf index e69de29bb2..dd8fc8dba7 100644 --- a/modules/logging-sinks/outputs.tf +++ b/modules/logging-sinks/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "sinks" { + description = "Log sink resources." + value = local.sink_resources +} + +output "names" { + description = "Log sink names." + value = [for sink in local.sink_resources : sink.name] +} + +output "writer_identities" { + description = "Log sink writer identities." + value = [for sink in local.sink_resources : sink.writer_identity] +} diff --git a/modules/logging-sinks/variables.tf b/modules/logging-sinks/variables.tf index 83185dca24..569936c415 100644 --- a/modules/logging-sinks/variables.tf +++ b/modules/logging-sinks/variables.tf @@ -13,3 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +variable "sinks" { + description = "Logging sinks that will be created, options default to true except for unique_writer_identity." + type = list(object({ + # organizations/nnn, billing_accounts/nnn, folders/nnn, projects/nnn + name = string + resource = string + filter = string + destination = string + options = object({ + bigquery_partitioned_tables = bool + include_children = bool + unique_writer_identity = bool + }) + })) + default = [] +} diff --git a/tests/modules/logging_sinks/fixtures/test_variable/main.tf b/tests/modules/logging_sinks/fixtures/test_variable/main.tf new file mode 100644 index 0000000000..5f37c1ffe0 --- /dev/null +++ b/tests/modules/logging_sinks/fixtures/test_variable/main.tf @@ -0,0 +1,76 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "sinks" { + source = "../../../../../modules/logging-sinks" + sinks = [ + { + name = "organization-1" + resource = "organizations/12345678" + filter = "filter: org_simple" + destination = "destination: org_simple" + options = null + }, + { + name = "billing-1" + resource = "billing_accounts/12345678" + filter = "filter: billing_simple" + destination = "bigquery destination: billing_simple" + options = { + bigquery_partitioned_tables = true + include_children = false + unique_writer_identity = null + } + }, + { + name = "folder-1" + resource = "folders/12345678" + filter = "filter: folder_simple" + destination = "bigquery destination: folder_simple" + options = { + bigquery_partitioned_tables = false + include_children = true + unique_writer_identity = null + } + }, + { + name = "project-1" + resource = "projects/12345678" + filter = "filter: project-1" + destination = "destination: project-1" + options = { + bigquery_partitioned_tables = null + include_children = true + unique_writer_identity = null + } + }, + { + name = "project-2" + resource = "projects/12345678" + filter = "filter: project-2" + destination = "destination: project-2" + options = { + bigquery_partitioned_tables = null + include_children = null + unique_writer_identity = true + } + }, + ] +} + +output "names" { + value = module.sinks.names +} From 26f7aebacc3010272466892ae5f0fc4d67126f3f Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 12 Jan 2020 00:39:58 +0100 Subject: [PATCH 098/106] modules/logging-sinks: streamline options in sinks variable --- modules/logging-sinks/main.tf | 29 +++++++++---------- .../fixtures/test_variable/main.tf | 2 +- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/modules/logging-sinks/main.tf b/modules/logging-sinks/main.tf index f2d772a864..e32264cbb0 100644 --- a/modules/logging-sinks/main.tf +++ b/modules/logging-sinks/main.tf @@ -15,13 +15,18 @@ */ locals { + default_options = { + bigquery_partitioned_tables = true + include_children = true + unique_writer_identity = false + } sinks = [ for sink in var.sinks : merge( sink, zipmap(["resource_type", "resource_id"], split("/", sink.resource)), { to_bigquery = substr(sink.destination, 0, 8) == "bigquery", - options = sink.options == null ? {} : sink.options + options = sink.options == null ? local.default_options : sink.options } ) ] @@ -42,14 +47,12 @@ resource "google_logging_organization_sink" "sinks" { org_id = each.value.resource_id filter = each.value.filter destination = each.value.destination - include_children = lookup(each.value.options, "include_children", true) + include_children = each.value.options.include_children dynamic bigquery_options { for_each = each.value.to_bigquery ? ["1"] : [] iterator = config content { - use_partitioned_tables = lookup( - each.value.options, "bigquery_partitioned_tables", true - ) + use_partitioned_tables = each.value.options.bigquery_partitioned_tables } } } @@ -67,9 +70,7 @@ resource "google_logging_billing_account_sink" "sinks" { for_each = each.value.to_bigquery ? ["1"] : [] iterator = config content { - use_partitioned_tables = lookup( - each.value.options, "bigquery_partitioned_tables", true - ) + use_partitioned_tables = each.value.options.bigquery_partitioned_tables } } } @@ -83,14 +84,12 @@ resource "google_logging_folder_sink" "sinks" { folder = each.value.resource filter = each.value.filter destination = each.value.destination - include_children = lookup(each.value.options, "include_children", true) + include_children = each.value.options.include_children dynamic bigquery_options { for_each = each.value.to_bigquery ? ["1"] : [] iterator = config content { - use_partitioned_tables = lookup( - each.value.options, "bigquery_partitioned_tables", true - ) + use_partitioned_tables = each.value.options.bigquery_partitioned_tables } } } @@ -104,14 +103,12 @@ resource "google_logging_project_sink" "sinks" { project = each.value.resource_id filter = each.value.filter destination = each.value.destination - unique_writer_identity = lookup(each.value.options, "unique_writer_identity", false) + unique_writer_identity = each.value.options.unique_writer_identity dynamic bigquery_options { for_each = each.value.to_bigquery ? ["1"] : [] iterator = config content { - use_partitioned_tables = lookup( - each.value.options, "bigquery_partitioned_tables", true - ) + use_partitioned_tables = each.value.options.bigquery_partitioned_tables } } } diff --git a/tests/modules/logging_sinks/fixtures/test_variable/main.tf b/tests/modules/logging_sinks/fixtures/test_variable/main.tf index 5f37c1ffe0..2d613e5598 100644 --- a/tests/modules/logging_sinks/fixtures/test_variable/main.tf +++ b/tests/modules/logging_sinks/fixtures/test_variable/main.tf @@ -42,7 +42,7 @@ module "sinks" { destination = "bigquery destination: folder_simple" options = { bigquery_partitioned_tables = false - include_children = true + include_children = false unique_writer_identity = null } }, From b125f0be6540c12793d451efb9c9751e059c05f8 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 12 Jan 2020 01:11:30 +0100 Subject: [PATCH 099/106] modules/compute-vm-cos-coredns: add support for additional files --- .../cloud-config-file.yaml | 5 +++ .../compute-vm-cos-coredns/cloud-config.yaml | 2 ++ modules/compute-vm-cos-coredns/main.tf | 33 +++++++++++++++++-- modules/compute-vm-cos-coredns/variables.tf | 14 +++++++- 4 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 modules/compute-vm-cos-coredns/cloud-config-file.yaml diff --git a/modules/compute-vm-cos-coredns/cloud-config-file.yaml b/modules/compute-vm-cos-coredns/cloud-config-file.yaml new file mode 100644 index 0000000000..47b3df2f74 --- /dev/null +++ b/modules/compute-vm-cos-coredns/cloud-config-file.yaml @@ -0,0 +1,5 @@ +- path: ${path} + content: | + ${indent(4, content)} + owner: ${owner} + permissions: ${permissions} diff --git a/modules/compute-vm-cos-coredns/cloud-config.yaml b/modules/compute-vm-cos-coredns/cloud-config.yaml index 66bd9b4ed6..06cc63c9cb 100644 --- a/modules/compute-vm-cos-coredns/cloud-config.yaml +++ b/modules/compute-vm-cos-coredns/cloud-config.yaml @@ -64,6 +64,8 @@ write_files: ${image} -conf /etc/coredns/Corefile ExecStop=/usr/bin/docker stop coredns +${files} + runcmd: - iptables -I INPUT 1 -p tcp -m tcp --dport 53 -m state --state NEW,ESTABLISHED -j ACCEPT - iptables -I INPUT 1 -p udp -m udp --dport 53 -m state --state NEW,ESTABLISHED -j ACCEPT diff --git a/modules/compute-vm-cos-coredns/main.tf b/modules/compute-vm-cos-coredns/main.tf index 3a5f0d28db..b562f784c1 100644 --- a/modules/compute-vm-cos-coredns/main.tf +++ b/modules/compute-vm-cos-coredns/main.tf @@ -14,15 +14,42 @@ * limitations under the License. */ +locals { + files = { + for path, data in var.files : + path => merge(data, { + attributes = ( + data.attributes == null + ? { owner = "root", permissions = "0644" } + : data.attributes + ) + }) + } + files_yaml = join("\n", [ + for _, data in data.template_file.cloud-config-files : data.rendered + ]) +} + +data "template_file" "cloud-config-files" { + for_each = local.files + template = file("${path.module}/cloud-config-file.yaml") + vars = { + path = each.key + content = each.value.content + owner = each.value.attributes.owner + permissions = each.value.attributes.permissions + } +} + data "template_file" "cloud-config" { template = file("${path.module}/cloud-config.yaml") - vars = { - corefile = file( + corefile = ( var.coredns_corefile == null - ? "${path.module}/Corefile" + ? file("${path.module}/Corefile") : var.coredns_corefile ) + files = local.files_yaml image = var.coredns_image log_driver = var.coredns_log_driver } diff --git a/modules/compute-vm-cos-coredns/variables.tf b/modules/compute-vm-cos-coredns/variables.tf index 61fb473a24..fd18ff8e1e 100644 --- a/modules/compute-vm-cos-coredns/variables.tf +++ b/modules/compute-vm-cos-coredns/variables.tf @@ -29,7 +29,7 @@ variable "boot_disk" { } variable "coredns_corefile" { - description = "CoreDNS Corefile path, defaults to sample configuration." + description = "CoreDNS Corefile content, defaults to sample configuration." type = string default = null } @@ -58,6 +58,18 @@ variable "cos_config" { } } +variable "files" { + description = "Map of files to create on the instances, path as key. Attributes default to 'root' and '0644', set to null if not needed." + type = map(object({ + content = string + attributes = object({ + owner = string + permissions = string + }) + })) + default = {} +} + variable "hostname" { description = "Instance FQDN name." type = string From 0bb9ac0e151128a144069277d2f3afd728977861 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 12 Jan 2020 20:11:00 +0100 Subject: [PATCH 100/106] modules/folders: rename from 'folder' --- modules/{folder => folders}/README.md | 0 modules/{folder => folders}/main.tf | 0 modules/{folder => folders}/outputs.tf | 0 modules/{folder => folders}/variables.tf | 0 modules/{folder => folders}/versions.tf | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename modules/{folder => folders}/README.md (100%) rename modules/{folder => folders}/main.tf (100%) rename modules/{folder => folders}/outputs.tf (100%) rename modules/{folder => folders}/variables.tf (100%) rename modules/{folder => folders}/versions.tf (100%) diff --git a/modules/folder/README.md b/modules/folders/README.md similarity index 100% rename from modules/folder/README.md rename to modules/folders/README.md diff --git a/modules/folder/main.tf b/modules/folders/main.tf similarity index 100% rename from modules/folder/main.tf rename to modules/folders/main.tf diff --git a/modules/folder/outputs.tf b/modules/folders/outputs.tf similarity index 100% rename from modules/folder/outputs.tf rename to modules/folders/outputs.tf diff --git a/modules/folder/variables.tf b/modules/folders/variables.tf similarity index 100% rename from modules/folder/variables.tf rename to modules/folders/variables.tf diff --git a/modules/folder/versions.tf b/modules/folders/versions.tf similarity index 100% rename from modules/folder/versions.tf rename to modules/folders/versions.tf From b7608f8e21f114dafff7286f20291d5c979681f8 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 12 Jan 2020 20:11:36 +0100 Subject: [PATCH 101/106] modules/logging-sinks: fix circular dependencies and improve variables --- modules/logging-sinks/main.tf | 97 ++++++++++++------------------ modules/logging-sinks/variables.tf | 50 ++++++++++----- 2 files changed, 76 insertions(+), 71 deletions(-) diff --git a/modules/logging-sinks/main.tf b/modules/logging-sinks/main.tf index e32264cbb0..cc9d831768 100644 --- a/modules/logging-sinks/main.tf +++ b/modules/logging-sinks/main.tf @@ -15,21 +15,16 @@ */ locals { - default_options = { - bigquery_partitioned_tables = true - include_children = true - unique_writer_identity = false + bigquery_destinations = { + for name, destination in var.destinations : + name => substr(destination, 0, 8) == "bigquery" + } + resource_type = element(split("/", var.parent), 0) + resource_id = element(split("/", var.parent), 1) + sink_options = { + for name, _ in var.sinks : + name => lookup(var.sink_options, name, var.default_options) } - sinks = [ - for sink in var.sinks : merge( - sink, - zipmap(["resource_type", "resource_id"], split("/", sink.resource)), - { - to_bigquery = substr(sink.destination, 0, 8) == "bigquery", - options = sink.options == null ? local.default_options : sink.options - } - ) - ] sink_resources = concat( [for _, sink in google_logging_organization_sink.sinks : sink], [for _, sink in google_logging_billing_account_sink.sinks : sink], @@ -39,76 +34,64 @@ locals { } resource "google_logging_organization_sink" "sinks" { - for_each = { - for sink in local.sinks : - "${sink.resource}/${sink.name}" => sink if sink.resource_type == "organizations" - } - name = each.value.name - org_id = each.value.resource_id - filter = each.value.filter - destination = each.value.destination - include_children = each.value.options.include_children + for_each = local.resource_type == "organizations" ? var.sinks : {} + name = each.key + org_id = local.resource_id + filter = each.value + destination = var.destinations[each.key] + include_children = local.sink_options[each.key].include_children dynamic bigquery_options { - for_each = each.value.to_bigquery ? ["1"] : [] + for_each = local.bigquery_destinations[each.key] ? ["1"] : [] iterator = config content { - use_partitioned_tables = each.value.options.bigquery_partitioned_tables + use_partitioned_tables = local.sink_options[each.key].bigquery_partitioned_tables } } } resource "google_logging_billing_account_sink" "sinks" { - for_each = { - for sink in local.sinks : - "${sink.resource}/${sink.name}" => sink if sink.resource_type == "billing_accounts" - } - name = each.value.name - billing_account = each.value.resource_id - filter = each.value.filter - destination = each.value.destination + for_each = local.resource_type == "billing_accounts" ? var.sinks : {} + name = each.key + billing_account = local.resource_id + filter = each.value + destination = var.destinations[each.key] dynamic bigquery_options { - for_each = each.value.to_bigquery ? ["1"] : [] + for_each = local.bigquery_destinations[each.key] ? ["1"] : [] iterator = config content { - use_partitioned_tables = each.value.options.bigquery_partitioned_tables + use_partitioned_tables = local.sink_options[each.key].bigquery_partitioned_tables } } } resource "google_logging_folder_sink" "sinks" { - for_each = { - for sink in local.sinks : - "${sink.resource}/${sink.name}" => sink if sink.resource_type == "folders" - } - name = each.value.name - folder = each.value.resource - filter = each.value.filter - destination = each.value.destination - include_children = each.value.options.include_children + for_each = local.resource_type == "folders" ? var.sinks : {} + name = each.key + folder = var.parent + filter = each.value + destination = var.destinations[each.key] + include_children = local.sink_options[each.key].include_children dynamic bigquery_options { - for_each = each.value.to_bigquery ? ["1"] : [] + for_each = local.bigquery_destinations[each.key] ? ["1"] : [] iterator = config content { - use_partitioned_tables = each.value.options.bigquery_partitioned_tables + use_partitioned_tables = local.sink_options[each.key].bigquery_partitioned_tables } } } resource "google_logging_project_sink" "sinks" { - for_each = { - for sink in local.sinks : - "${sink.resource}/${sink.name}" => sink if sink.resource_type == "projects" - } - name = each.value.name - project = each.value.resource_id - filter = each.value.filter - destination = each.value.destination - unique_writer_identity = each.value.options.unique_writer_identity + for_each = local.resource_type == "projects" ? var.sinks : {} + name = each.key + project = local.resource_id + filter = each.value + destination = var.destinations[each.key] + unique_writer_identity = local.sink_options[each.key].unique_writer_identity dynamic bigquery_options { - for_each = each.value.to_bigquery ? ["1"] : [] + for_each = local.bigquery_destinations[each.key] ? ["1"] : [] iterator = config content { - use_partitioned_tables = each.value.options.bigquery_partitioned_tables + use_partitioned_tables = local.sink_options[each.key].bigquery_partitioned_tables } } } diff --git a/modules/logging-sinks/variables.tf b/modules/logging-sinks/variables.tf index 569936c415..24294e7d1d 100644 --- a/modules/logging-sinks/variables.tf +++ b/modules/logging-sinks/variables.tf @@ -14,19 +14,41 @@ * limitations under the License. */ -variable "sinks" { - description = "Logging sinks that will be created, options default to true except for unique_writer_identity." - type = list(object({ - # organizations/nnn, billing_accounts/nnn, folders/nnn, projects/nnn - name = string - resource = string - filter = string - destination = string - options = object({ - bigquery_partitioned_tables = bool - include_children = bool - unique_writer_identity = bool - }) +variable "default_options" { + description = "Default options used for sinks where no specific options are set." + type = object({ + bigquery_partitioned_tables = bool + include_children = bool + unique_writer_identity = bool + }) + default = { + bigquery_partitioned_tables = true + include_children = true + unique_writer_identity = false + } +} + +variable "destinations" { + description = "Map of destinations by sink name." + type = map(string) +} + +variable "parent" { + description = "Resource where the sink will be created, eg 'organizations/nnnnnnnn'." + type = string +} + +variable "sink_options" { + description = "Optional map of sink name / sink options. If no options are specified for a sink defaults will be used." + type = map(object({ + bigquery_partitioned_tables = bool + include_children = bool + unique_writer_identity = bool })) - default = [] + default = {} +} + +variable "sinks" { + description = "Map of sink name / sink filter." + type = map(string) } From c3d16b27c2447ecfeedbc4247253d74919a0837e Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 12 Jan 2020 20:12:02 +0100 Subject: [PATCH 102/106] modules/project: remove extra variable --- modules/project/README.md | 1 - modules/project/variables.tf | 6 ------ 2 files changed, 7 deletions(-) diff --git a/modules/project/README.md b/modules/project/README.md index 7ec77035ed..e4a5ef9d7e 100644 --- a/modules/project/README.md +++ b/modules/project/README.md @@ -41,7 +41,6 @@ module "project" { | *oslogin* | Enable OS Login. | bool | | false | | *oslogin_admins* | List of IAM-style identities that will be granted roles necessary for OS Login administrators. | list(string) | | [] | | *oslogin_users* | List of IAM-style identities that will be granted roles necessary for OS Login users. | list(string) | | [] | -| *owners* | IAM-style identities that will be granted non-authoritative viewer role. | list(string) | | [] | | *services* | Service APIs to enable. | list(string) | | [] | ## Outputs diff --git a/modules/project/variables.tf b/modules/project/variables.tf index bc8459c58f..6d71978d69 100644 --- a/modules/project/variables.tf +++ b/modules/project/variables.tf @@ -91,12 +91,6 @@ variable "oslogin_users" { default = [] } -variable "owners" { - description = "IAM-style identities that will be granted non-authoritative viewer role." - type = list(string) - default = [] -} - variable "parent" { description = "The resource name of the parent Folder or Organization. Must be of the form folders/folder_id or organizations/org_id." type = string From 331dea0ef2b77bf9ae959f172dc065f1e5e085bc Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 12 Jan 2020 20:13:04 +0100 Subject: [PATCH 103/106] modules/bigquery: new module with dataset support only --- modules/bigquery/README.md | 31 +++++++++++++++ modules/bigquery/main.tf | 66 ++++++++++++++++++++++++++++++ modules/bigquery/outputs.tf | 47 ++++++++++++++++++++++ modules/bigquery/variables.tf | 75 +++++++++++++++++++++++++++++++++++ modules/bigquery/versions.tf | 19 +++++++++ 5 files changed, 238 insertions(+) create mode 100644 modules/bigquery/README.md create mode 100644 modules/bigquery/main.tf create mode 100644 modules/bigquery/outputs.tf create mode 100644 modules/bigquery/variables.tf create mode 100644 modules/bigquery/versions.tf diff --git a/modules/bigquery/README.md b/modules/bigquery/README.md new file mode 100644 index 0000000000..86935a019a --- /dev/null +++ b/modules/bigquery/README.md @@ -0,0 +1,31 @@ +# Google Cloud Bigquery Module + +TODO(ludoo): add support for tables + +## Example + +```iam +module "bigquery-datasets" { +} +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| datasets | Map of datasets to create keyed by id. Labels and options can be null. | map(object({...})) | ✓ | | +| project_id | Id of the project where datasets will be created. | string | ✓ | | +| *default_access* | Access rules applied to all dataset if no specific ones are defined. | list(object({...})) | | [] | +| *default_options* | Options used for all dataset if no specific ones are defined. | object({...}) | | ... | +| *kms_key* | Self link of the KMS key that will be used to protect destination table. | string | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| datasets | Dataset resources. | | +| ids | Dataset ids. | | +| names | Dataset names. | | +| self_links | Dataset self links. | | + diff --git a/modules/bigquery/main.tf b/modules/bigquery/main.tf new file mode 100644 index 0000000000..75f43fe330 --- /dev/null +++ b/modules/bigquery/main.tf @@ -0,0 +1,66 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + datasets = { + for id, data in var.datasets : + id => merge(data, { + options = data.options != null ? data.options : var.default_options + access = lookup(var.dataset_access, id, var.default_access) + }) + } +} + +resource "google_bigquery_dataset" "datasets" { + for_each = local.datasets + project = var.project_id + dataset_id = each.key + friendly_name = each.value.name + description = each.value.description + labels = each.value.labels + location = each.value.location + + delete_contents_on_destroy = each.value.options.delete_contents_on_destroy + default_table_expiration_ms = each.value.options.default_table_expiration_ms + default_partition_expiration_ms = each.value.options.default_partition_expiration_ms + + dynamic access { + for_each = each.value.access + iterator = config + content { + role = config.value.role + domain = each.value.identity_type == "domain" ? each.value.identity : null + group_by_email = each.value.identity_type == "group_by_email" ? each.value.identity : null + special_group = each.value.identity_type == "special_group" ? each.value.identity : null + user_by_email = each.value.identity_type == "user_by_email" ? each.value.identity : null + dynamic view { + for_each = each.value.identity_type == "view" ? [""] : [] + content { + project_id = view.value.project_id + dataset_id = view.value.dataset_id + table_id = view.value.table_id + } + } + } + } + + dynamic default_encryption_configuration { + for_each = var.kms_key == null ? [] : [""] + content { + kms_key_name = var.kms_key + } + } +} diff --git a/modules/bigquery/outputs.tf b/modules/bigquery/outputs.tf new file mode 100644 index 0000000000..bffb0b39a2 --- /dev/null +++ b/modules/bigquery/outputs.tf @@ -0,0 +1,47 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "datasets" { + description = "Dataset resources." + value = [ + for _, resource in google_bigquery_dataset.datasets : + resource + ] +} + +output "ids" { + description = "Dataset ids." + value = [ + for _, resource in google_bigquery_dataset.datasets : + resource.id + ] +} + +output "names" { + description = "Dataset names." + value = [ + for _, resource in google_bigquery_dataset.datasets : + resource.friendly_name + ] +} + +output "self_links" { + description = "Dataset self links." + value = [ + for _, resource in google_bigquery_dataset.datasets : + resource.self_link + ] +} diff --git a/modules/bigquery/variables.tf b/modules/bigquery/variables.tf new file mode 100644 index 0000000000..1b02ab7fcc --- /dev/null +++ b/modules/bigquery/variables.tf @@ -0,0 +1,75 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "datasets" { + description = "Map of datasets to create keyed by id. Labels and options can be null." + type = map(object({ + description = string + location = string + name = string + labels = map(string) + options = object({ + default_table_expiration_ms = number + default_partition_expiration_ms = number + delete_contents_on_destroy = bool + }) + })) +} + +variable "dataset_access" { + description = "Optional map of dataset access rules by dataset id." + type = map(list(object({ + role = string + identity_type = string + identity = any + }))) + default = {} +} + +variable "default_access" { + description = "Access rules applied to all dataset if no specific ones are defined." + type = list(object({ + role = string + identity_type = string + identity = any + })) + default = [] +} + +variable "default_options" { + description = "Options used for all dataset if no specific ones are defined." + type = object({ + default_table_expiration_ms = number + default_partition_expiration_ms = number + delete_contents_on_destroy = bool + }) + default = { + default_table_expiration_ms = null + default_partition_expiration_ms = null + delete_contents_on_destroy = false + } +} + +variable "kms_key" { + description = "Self link of the KMS key that will be used to protect destination table." + type = string + default = null +} + +variable "project_id" { + description = "Id of the project where datasets will be created." + type = string +} diff --git a/modules/bigquery/versions.tf b/modules/bigquery/versions.tf new file mode 100644 index 0000000000..ce6918e09d --- /dev/null +++ b/modules/bigquery/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12.6" +} From f3c965c57b566f64d4a4ada193da2a520631f806 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 12 Jan 2020 20:13:26 +0100 Subject: [PATCH 104/106] foundations/environments: refactor using local modules --- foundations/environments/locals.tf | 38 +++++ foundations/environments/main.tf | 200 ++++++++++++++------------ foundations/environments/outputs.tf | 3 +- foundations/environments/variables.tf | 109 ++++++++------ foundations/environments/versions.tf | 2 +- 5 files changed, 217 insertions(+), 135 deletions(-) create mode 100644 foundations/environments/locals.tf diff --git a/foundations/environments/locals.tf b/foundations/environments/locals.tf new file mode 100644 index 0000000000..6c3ae0409d --- /dev/null +++ b/foundations/environments/locals.tf @@ -0,0 +1,38 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + folder_roles = concat(var.iam_folder_roles, local.sa_xpn_folder_role) + sa_billing_account_role = ( + var.iam_billing_config.target_org ? [] : ["roles/billing.user"] + ) + sa_billing_org_role = ( + ! var.iam_billing_config.target_org ? [] : ["roles/billing.user"] + ) + sa_xpn_folder_role = ( + local.sa_xpn_target_org ? [] : ["roles/compute.xpnAdmin"] + ) + sa_xpn_org_roles = ( + local.sa_xpn_target_org + ? ["roles/compute.xpnAdmin", "roles/resourcemanager.organizationViewer"] + : ["roles/resourcemanager.organizationViewer"] + ) + sa_xpn_target_org = ( + var.iam_xpn_config.target_org + || + substr(var.root_node, 0, 13) == "organizations" + ) +} diff --git a/foundations/environments/main.tf b/foundations/environments/main.tf index 1d8da9229b..6332e34596 100644 --- a/foundations/environments/main.tf +++ b/foundations/environments/main.tf @@ -18,80 +18,84 @@ # Terraform project -module "project-tf" { - source = "terraform-google-modules/project-factory/google//modules/fabric-project" - version = "5.0.0" - parent = var.root_node - billing_account = var.billing_account_id - prefix = var.prefix - name = "terraform" - # lien_reason = "terraform" - owners = var.terraform_owners - activate_apis = var.project_services +module "tf-project" { + source = "../../modules/project" + name = "terraform" + parent = var.root_node + prefix = var.prefix + billing_account = var.billing_account_id + iam_nonauth_members = { "roles/owner" = var.iam_terraform_owners } + iam_nonauth_roles = ["roles/owner"] + services = var.project_services } # per-environment service accounts -module "service-accounts-tf-environments" { - source = "terraform-google-modules/service-accounts/google" - version = "2.0.2" - project_id = module.project-tf.project_id - org_id = var.organization_id - billing_account_id = var.billing_account_id - prefix = var.prefix - names = var.environments - grant_billing_role = true - grant_xpn_roles = var.grant_xpn_org_roles - generate_keys = var.generate_service_account_keys +module "tf-service-accounts" { + source = "../../modules/iam-service-accounts" + project_id = module.tf-project.project_id + names = var.environments + prefix = var.prefix + iam_billing_roles = { + (var.billing_account_id) = ( + var.iam_billing_config.grant ? local.sa_billing_account_role : [] + ) + } + # folder roles are set in the folders module using authoritative bindings + iam_organization_roles = { + (var.organization_id) = concat( + var.iam_billing_config.grant ? local.sa_billing_org_role : [], + var.iam_xpn_config.grant ? local.sa_xpn_org_roles : [] + ) + } + generate_keys = var.service_account_keys } # bootstrap Terraform state GCS bucket -module "gcs-tf-bootstrap" { - source = "terraform-google-modules/cloud-storage/google" - version = "1.0.0" - project_id = module.project-tf.project_id - prefix = "${var.prefix}-tf" +module "tf-gcs-bootstrap" { + source = "../../modules/gcs" + project_id = module.tf-project.project_id names = ["tf-bootstrap"] + prefix = "${var.prefix}-tf" location = var.gcs_location } # per-environment Terraform state GCS buckets -module "gcs-tf-environments" { - source = "terraform-google-modules/cloud-storage/google" - version = "1.0.0" - project_id = module.project-tf.project_id - prefix = "${var.prefix}-tf" - names = var.environments - location = var.gcs_location - set_admin_roles = true - bucket_admins = zipmap( - var.environments, - module.service-accounts-tf-environments.iam_emails_list - ) +module "tf-gcs-environments" { + source = "../../modules/gcs" + project_id = module.tf-project.project_id + names = var.environments + prefix = "${var.prefix}-tf" + location = var.gcs_location + iam_roles = { + for name in var.environments : (name) => ["roles/storage.objectAdmin"] + } + iam_members = { + for name in var.environments : (name) => { + "roles/storage.objectAdmin" = [module.tf-service-accounts.iam_emails[name]] + } + } } ############################################################################### # Top-level folders # ############################################################################### -module "folders-top-level" { - source = "terraform-google-modules/folders/google" - version = "2.0.2" - parent = var.root_node - names = var.environments - set_roles = true - per_folder_admins = module.service-accounts-tf-environments.iam_emails_list - folder_admin_roles = compact( - [ - "roles/compute.networkAdmin", - "roles/owner", - "roles/resourcemanager.folderViewer", - "roles/resourcemanager.projectCreator", - var.grant_xpn_folder_roles ? "roles/compute.xpnAdmin" : "" - ] - ) +module "environment-folders" { + source = "../../modules/folders" + parent = var.root_node + names = var.environments + iam_roles = { + for name in var.environments : (name) => local.folder_roles + } + iam_members = { + for name in var.environments : (name) => { + for role in local.folder_roles : + (role) => [module.tf-service-accounts.iam_emails[name]] + } + } } ############################################################################### @@ -100,43 +104,50 @@ module "folders-top-level" { # audit logs project -module "project-audit" { - source = "terraform-google-modules/project-factory/google//modules/fabric-project" - version = "5.0.0" +module "audit-project" { + source = "../../modules/project" + name = "audit" parent = var.root_node - billing_account = var.billing_account_id prefix = var.prefix - name = "audit" - # lien_reason = "audit" - activate_apis = concat(var.project_services, [ + billing_account = var.billing_account_id + iam_members = { + "roles/bigquery.dataEditor" = [module.audit-log-sinks.writer_identities[0]] + "roles/viewer" = var.iam_audit_viewers + } + iam_roles = [ + "roles/bigquery.dataEditor", + "roles/viewer" + ] + services = concat(var.project_services, [ "bigquery.googleapis.com", ]) - viewers = var.audit_viewers } -# audit logs destination on BigQuery - -module "bq-audit-export" { - source = "terraform-google-modules/log-export/google//modules/bigquery" - version = "3.2.0" - project_id = module.project-audit.project_id - dataset_name = "logs_audit_${replace(var.environments[0], "-", "_")}" - log_sink_writer_identity = module.log-sink-audit.writer_identity +# audit logs dataset and sink + +module "audit-datasets" { + source = "../../modules/bigquery" + project_id = module.audit-project.project_id + datasets = { + audit_export = { + name = "Audit logs export." + description = "Terraform managed." + location = "EU" + labels = null + options = null + } + } } -# audit log sink -# set the organization as parent to export audit logs for all environments - -module "log-sink-audit" { - source = "terraform-google-modules/log-export/google" - version = "3.2.0" - filter = var.audit_filter - log_sink_name = "logs-audit-${var.environments[0]}" - parent_resource_type = "folder" - parent_resource_id = split("/", module.folders-top-level.ids_list[0])[1] - include_children = "true" - unique_writer_identity = "true" - destination_uri = "${module.bq-audit-export.destination_uri}" +module "audit-log-sinks" { + source = "../../modules/logging-sinks" + parent = var.root_node + destinations = { + audit-logs = "bigquery.googleapis.com/projects/${module.audit-project.project_id}/datasets/${module.audit-datasets.names[0]}" + } + sinks = { + audit-logs = var.audit_filter + } } ############################################################################### @@ -146,16 +157,19 @@ module "log-sink-audit" { # shared resources project # see the README file for additional options on managing shared services -module "project-shared-resources" { - source = "terraform-google-modules/project-factory/google//modules/fabric-project" - version = "5.0.0" - parent = var.root_node - billing_account = var.billing_account_id - prefix = var.prefix - name = "shared" - activate_apis = var.project_services - extra_bindings_roles = var.shared_bindings_roles - extra_bindings_members = var.shared_bindings_members +module "sharedsvc-project" { + source = "../../modules/project" + name = "sharedsvc" + parent = var.root_node + prefix = var.prefix + billing_account = var.billing_account_id + iam_members = { + "roles/owner" = var.iam_sharedsvc_owners + } + iam_roles = [ + "roles/owner" + ] + services = var.project_services } # Add further modules here for resources that are common to all environments diff --git a/foundations/environments/outputs.tf b/foundations/environments/outputs.tf index cca71987d0..f083d1c8a0 100644 --- a/foundations/environments/outputs.tf +++ b/foundations/environments/outputs.tf @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -output "terraform_project" { +/* output "terraform_project" { description = "Project that holds the base Terraform resources." value = module.project-tf.project_id } @@ -60,3 +60,4 @@ output "shared_resources_project" { # Add further outputs here for the additional modules that manage shared # resources, like GCR, GCS buckets, KMS, etc. + */ diff --git a/foundations/environments/variables.tf b/foundations/environments/variables.tf index 720b6333a7..f84890a21e 100644 --- a/foundations/environments/variables.tf +++ b/foundations/environments/variables.tf @@ -22,14 +22,8 @@ variable "audit_filter" { END } -variable "audit_viewers" { - description = "Audit project viewers, in IAM format." - type = list(string) - default = [] -} - variable "billing_account_id" { - description = "Billing account id used as default for new projects." + description = "Billing account id used as to create projects." type = string } @@ -38,33 +32,81 @@ variable "environments" { type = list(string) } -variable "generate_service_account_keys" { - description = "Generate and store service account keys in the state file." - type = bool - default = false -} - variable "gcs_location" { description = "GCS bucket location." type = string default = "EU" } -variable "grant_xpn_org_roles" { - description = "Grant roles needed for Shared VPC creation to service accounts at the organization level." - type = bool - default = true +variable "iam_assets_editors" { + description = "Shared assets project editors, in IAM format." + type = list(string) + default = [] } -variable "grant_xpn_folder_roles" { - description = "Grant roles needed for Shared VPC creation to service accounts at the environment folder level." - type = bool - default = false +variable "iam_assets_owners" { + description = "Shared assets project owners, in IAM format." + type = list(string) + default = [] +} + +variable "iam_audit_viewers" { + description = "Audit project viewers, in IAM format." + type = list(string) + default = [] +} + +variable "iam_billing_config" { + description = "Control granting billing user role to service accounts. Target the billing account by default." + type = object({ + grant = bool + target_org = bool + }) + default = { + grant = true + target_org = false + } +} + +variable "iam_folder_roles" { + description = "List of roles granted to each service account on its respective folder (excluding XPN roles)." + type = list(string) + default = [ + "roles/compute.networkAdmin", + "roles/owner", + "roles/resourcemanager.folderViewer", + "roles/resourcemanager.projectCreator", + ] +} + +variable "iam_sharedsvc_owners" { + description = "Shared services project owners, in IAM format." + type = list(string) + default = [] +} + +variable "iam_terraform_owners" { + description = "Terraform project owners, in IAM format." + type = list(string) + default = [] +} + +variable "iam_xpn_config" { + description = "Control granting Shared VPC creation roles to service accounts. Target the root node by default." + type = object({ + grant = bool + target_org = string + }) + default = { + grant = true + target_org = false + } } variable "organization_id" { description = "Organization id." type = string + default = null } variable "prefix" { @@ -77,25 +119,6 @@ variable "root_node" { type = string } -variable "shared_bindings_members" { - description = "List of comma-delimited IAM-format members for the additional shared project bindings." - # example: ["user:a@example.com,b@example.com", "user:c@example.com"] - type = list(string) - default = [] -} -variable "shared_bindings_roles" { - description = "List of roles for additional shared project bindings." - # example: ["roles/storage.objectViewer", "roles/storage.admin"] - type = list(string) - default = [] -} - -variable "terraform_owners" { - description = "Terraform project owners, in IAM format." - type = list(string) - default = [] -} - variable "project_services" { description = "Service APIs enabled by default in new projects." type = list(string) @@ -104,3 +127,9 @@ variable "project_services" { "stackdriver.googleapis.com", ] } + +variable "service_account_keys" { + description = "Generate and store service account keys in the state file." + type = bool + default = true +} diff --git a/foundations/environments/versions.tf b/foundations/environments/versions.tf index 4eb1500c5a..92787679e0 100644 --- a/foundations/environments/versions.tf +++ b/foundations/environments/versions.tf @@ -13,5 +13,5 @@ # limitations under the License. terraform { - required_version = ">= 0.12" + required_version = ">= 0.12.6" } From 09a754c91d4eae4a73adcc9103c2ede56333920c Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Mon, 13 Jan 2020 12:37:52 +0100 Subject: [PATCH 105/106] modules/bigquery: better variables, README description and example --- modules/bigquery/README.md | 44 ++++++++++++++++++++++++++++++++--- modules/bigquery/main.tf | 5 ++-- modules/bigquery/variables.tf | 21 +++++++++++++---- 3 files changed, 60 insertions(+), 10 deletions(-) diff --git a/modules/bigquery/README.md b/modules/bigquery/README.md index 86935a019a..b894f1ae0f 100644 --- a/modules/bigquery/README.md +++ b/modules/bigquery/README.md @@ -1,11 +1,42 @@ # Google Cloud Bigquery Module -TODO(ludoo): add support for tables +Simple Bigquery module offering support for multiple dataset creation and access configuration. + +The module interface is designed to allow setting default values for dataset access configurations and options (eg table or partition expiration), and optionally override them for individual datasets. Common labels applied to all datasets can also be specified with a single variable, and overridden individually. + +Access configuration supports specifying different [identity types](https://www.terraform.io/docs/providers/google/r/bigquery_dataset.html#access) via the `identity_type` attribute in access variables. The supported identity types are: `domain`, `group_by_email`, `special_group` (eg `projectOwners`), `user_by_email`. ## Example -```iam +```hcl module "bigquery-datasets" { + source = "./modules/bigquery" + project_id = "my-project + datasets = { + dataset_1 = { + name = "Dataset 1." + description = "Terraform managed." + location = "EU" + labels = null + }, + dataset_2 = { + name = "Dataset 2." + description = "Terraform managed." + location = "EU" + labels = null + }, + } + default_access = [ + { + role = "OWNER" + identity_type = "special_group" + identity = "projectOwners" + } + ] + default_labels = { + eggs = "spam", + bar = "baz + } } ``` @@ -14,9 +45,12 @@ module "bigquery-datasets" { | name | description | type | required | default | |---|---|:---: |:---:|:---:| -| datasets | Map of datasets to create keyed by id. Labels and options can be null. | map(object({...})) | ✓ | | +| datasets | Map of datasets to create keyed by id. Labels and options can be null. | map(object({...})) | ✓ | | | project_id | Id of the project where datasets will be created. | string | ✓ | | +| *dataset_access* | Optional map of dataset access rules by dataset id. | map(list(object({...}))) | | {} | +| *dataset_options* | Optional map of dataset option by dataset id. | map(object({...})) | | {} | | *default_access* | Access rules applied to all dataset if no specific ones are defined. | list(object({...})) | | [] | +| *default_labels* | Labels set on all datasets. | map(string) | | {} | | *default_options* | Options used for all dataset if no specific ones are defined. | object({...}) | | ... | | *kms_key* | Self link of the KMS key that will be used to protect destination table. | string | | null | @@ -29,3 +63,7 @@ module "bigquery-datasets" { | names | Dataset names. | | | self_links | Dataset self links. | | + +## TODO + +- [ ] add support for tables diff --git a/modules/bigquery/main.tf b/modules/bigquery/main.tf index 75f43fe330..9f42c231db 100644 --- a/modules/bigquery/main.tf +++ b/modules/bigquery/main.tf @@ -18,8 +18,9 @@ locals { datasets = { for id, data in var.datasets : id => merge(data, { - options = data.options != null ? data.options : var.default_options + options = lookup(var.dataset_options, id, var.default_options) access = lookup(var.dataset_access, id, var.default_access) + labels = data.labels == null ? {} : data.labels }) } } @@ -30,7 +31,7 @@ resource "google_bigquery_dataset" "datasets" { dataset_id = each.key friendly_name = each.value.name description = each.value.description - labels = each.value.labels + labels = merge(var.default_labels, each.value.labels) location = each.value.location delete_contents_on_destroy = each.value.options.delete_contents_on_destroy diff --git a/modules/bigquery/variables.tf b/modules/bigquery/variables.tf index 1b02ab7fcc..ad9ae4ae23 100644 --- a/modules/bigquery/variables.tf +++ b/modules/bigquery/variables.tf @@ -21,11 +21,6 @@ variable "datasets" { location = string name = string labels = map(string) - options = object({ - default_table_expiration_ms = number - default_partition_expiration_ms = number - delete_contents_on_destroy = bool - }) })) } @@ -39,6 +34,16 @@ variable "dataset_access" { default = {} } +variable "dataset_options" { + description = "Optional map of dataset option by dataset id." + type = map(object({ + default_table_expiration_ms = number + default_partition_expiration_ms = number + delete_contents_on_destroy = bool + })) + default = {} +} + variable "default_access" { description = "Access rules applied to all dataset if no specific ones are defined." type = list(object({ @@ -49,6 +54,12 @@ variable "default_access" { default = [] } +variable "default_labels" { + description = "Labels set on all datasets." + type = map(string) + default = {} +} + variable "default_options" { description = "Options used for all dataset if no specific ones are defined." type = object({ From dc0e065cae13ce5f2c4e3bdd4581174e63bf4030 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Thu, 16 Jan 2020 13:27:37 -0800 Subject: [PATCH 106/106] modules: fix a few READMEs --- modules/compute-vm-cos-coredns/README.md | 36 +++++++++++++++++++++++- modules/compute-vm/README.md | 10 ++++++- modules/dns/README.md | 28 +++--------------- modules/folders/README.md | 12 ++++---- modules/gcs/README.md | 18 ++++++++---- modules/gke-cluster/README.md | 31 +++++++++++++++++--- modules/gke-nodepool/README.md | 14 +++++++-- modules/iam-service-accounts/README.md | 20 ++++--------- 8 files changed, 111 insertions(+), 58 deletions(-) diff --git a/modules/compute-vm-cos-coredns/README.md b/modules/compute-vm-cos-coredns/README.md index 86157d7fb6..8bedf436ab 100644 --- a/modules/compute-vm-cos-coredns/README.md +++ b/modules/compute-vm-cos-coredns/README.md @@ -1,8 +1,42 @@ # Container Optimized OS CoreDNS module +This module allows creating instances (or an instance template) runnning a containerized DNS server using CoreDNS. A service account will be created if none is set. + ## Example -TODO +```hcl +module "onprem-dns" { + source = "./modules/compute-vm-cos-coredns" + project_id = var.project_id + region = var.region + zone = var.zone + name = "coredns" + network_interfaces = [{ + network = var.vpc_self_link + subnetwork = var.subnet_self_link + nat = false, + addresses = null + }] + coredns_corefile = < ## Variables diff --git a/modules/compute-vm/README.md b/modules/compute-vm/README.md index 6cb0fda436..fa5e594b1b 100644 --- a/modules/compute-vm/README.md +++ b/modules/compute-vm/README.md @@ -1,4 +1,8 @@ -# Compute Engine VM module +# Google Compute Engine VM module + +This module allows creating on or multiple instances for a specific configuration, optionally creating an instance template instead of instances. + +A service account is optionally created and assigned to instances if no service account email is specified. ## Example @@ -60,3 +64,7 @@ module "debian-test" { | template | Template resource. | | | template_name | Template name. | | + +## TODO + +- [ ] add support for instance groups diff --git a/modules/dns/README.md b/modules/dns/README.md index 4da8a7705e..6993e56958 100644 --- a/modules/dns/README.md +++ b/modules/dns/README.md @@ -1,18 +1,17 @@ -# Cloud DNS Module - -This module makes it easy to create Google Cloud DNS zones of different types, and manage their records. It supports creating public, private, forwarding, and peering zones. +# Google Cloud DNS Module +This module allows simple management of Google Cloud DNS zones and records. It supports creating public, private, forwarding, and peering zones. For DNSSEC configuration, refer to the [`dns_managed_zone` documentation](https://www.terraform.io/docs/providers/google/r/dns_managed_zone.html#dnssec_config). ## Example ```hcl module "private-dns" { source = "./modules/dns" - project_id = local.projects.host + project_id = "myproject" type = "private" name = "test-example" domain = "test.example." - client_networks = [local.vpc_peered.self_link, local.vpc_shared.self_link] + client_networks = [var.vpc_self_link] recordsets = [ { name = "localhost", type = "A", ttl = 300, records = ["127.0.0.1"] } ] @@ -47,22 +46,3 @@ module "private-dns" { | type | The DNS zone type. | | | zone | DNS zone resource. | | - -## Requirements - -### IAM Roles - -The following roles must be used to provision the resources in this module: - -- Storage Admin: `roles/dns.admin` - -### APIs - -A project with the following APIs enabled must be used to host the -resources of this module: - -- Google Cloud DNS API: `dns.googleapis.com` - -## DNSSEC - -For DNSSEC configuration, refer to the [`dns_managed_zone` documentation](https://www.terraform.io/docs/providers/google/r/dns_managed_zone.html#dnssec_config). \ No newline at end of file diff --git a/modules/folders/README.md b/modules/folders/README.md index ea87b37171..3a788952e3 100644 --- a/modules/folders/README.md +++ b/modules/folders/README.md @@ -1,19 +1,21 @@ # Google Cloud Folder Module +This module allow creation and management of sets of folders sharing a common parent, and their individual IAM bindings. + ## Example ```hcl module "folder" { source = "./modules/folder" - parent = "organizations/${var.organization_id}" - names = ["TF Test"] + parent = "organizations/1234567890" + names = ["Folder one", "Folder two] iam_members = { - "TF Test" = { - for role in local.folder_roles : "${role}" => var.admins + "Folder one" = { + "roles/owner" => ["group:users@example.com"] } } iam_roles = { - "TF Test" = local.folder_roles + "Folder one" = ["roles/owner"] } } ``` diff --git a/modules/gcs/README.md b/modules/gcs/README.md index 56ad39b6b8..19538625bc 100644 --- a/modules/gcs/README.md +++ b/modules/gcs/README.md @@ -2,14 +2,22 @@ ## Example -```iam +```hcl module "buckets" { source = "./modules/gcs" - project_id = module.project.project_id - prefix = "ludo-tf-playground" - names = ["tfstate"] + project_id = "myproject" + prefix = "test" + names = ["bucket-one", "bucket-two"] bucket_policy_only = { - tfstate = false + bucket-one = false + } + iam_members = { + bucket-two = { + "roles/storage.admin" = ["group:storage@example.com"] + } + } + iam_roles = { + bucket-two = ["roles/storage.admin"] } } ``` diff --git a/modules/gke-cluster/README.md b/modules/gke-cluster/README.md index 19ad2bcdc4..41800b4501 100644 --- a/modules/gke-cluster/README.md +++ b/modules/gke-cluster/README.md @@ -1,10 +1,33 @@ -# Minimalistic GKE module +# GKE cluster module -TODO(ludoo): add description. +This module allows simplified creation and management of GKE cluster, and should be used together with the GKE nodepool module as the default nodepool is turned off in this module and cannot be re-enabled. Some sensible defaults are set initially, in order to allow less verbose usage for most use cases. -## Example usage +## Example -TODO(ludoo): add example +```hcl +module "cluster-1" { + source = "./modules/gke-cluster" + project_id = "myproject" + name = "cluster-1" + location = "europe-west1-b" + network = var.network_self_link + subnetwork = var.subnet_self_link + secondary_range_pods = "pods" + secondary_range_services = "services" + default_max_pods_per_node = 32 + labels = { + environment = "dev" + } + master_authorized_ranges = { + internal-vms = "10.0.0.0/8" + } + private_cluster_config = { + enable_private_nodes = true + enable_private_endpoint = true + master_ipv4_cidr_block = "192.168.0.0/28" + } +} +``` ## Variables diff --git a/modules/gke-nodepool/README.md b/modules/gke-nodepool/README.md index a793f94b16..959c1d3a57 100644 --- a/modules/gke-nodepool/README.md +++ b/modules/gke-nodepool/README.md @@ -1,10 +1,18 @@ -# Minimalistic GKE nodepool module +# GKE nodepool module -TODO(ludoo): add description. +This module allows simplified creation and management of individual GKE nodepools, setting sensible defaults (eg a service account is created for nodes if none is set) and allowing for less verbose usage in most use cases. ## Example usage -TODO(ludoo): add example +```hcl +module "cluster-1-nodepool-1" { + source = "../modules/gke-nodepool" + project_id = "myproject" + cluster_name = "cluster-1" + location = "europe-west1-b" + name = "nodepool-1" +} +``` ## Variables diff --git a/modules/iam-service-accounts/README.md b/modules/iam-service-accounts/README.md index 2815826757..c174edc381 100644 --- a/modules/iam-service-accounts/README.md +++ b/modules/iam-service-accounts/README.md @@ -1,17 +1,17 @@ -# Terraform Service Accounts Module +# Google Service Accounts Module -This module allows easy creation of one or more service accounts, and granting them basic roles. +This module allows simplified creation and management of one or more service accounts and their IAM bindings. Keys can optionally be generated and will be stored in Terraform state. To use them create a sensitive output in your root modules referencing the `keys` or `key` outputs, then extract the private key from the JSON formatted outputs. ## Example ```hcl -module "serviceprj-service-accounts" { +module "myproject-default-service-accounts" { source = "./modules/iam-service-accounts" - project_id = module.service-project.project_id + project_id = "myproject" names = ["vm-default", "gke-node-default"] generate_keys = true iam_project_roles = { - "${module.service-project.project_id}" = [ + "myproject" = [ "roles/logging.logWriter", "roles/monitoring.metricWriter", ] @@ -48,13 +48,3 @@ module "serviceprj-service-accounts" { | service_account | Service account resource (for single use). | | | service_accounts | Service account resources. | | - -## Requirements - -### IAM - -Service account or user credentials with the following roles must be used to provision the resources of this module: - -- Service Account Admin: `roles/iam.serviceAccountAdmin` -- (optional) Service Account Key Admin: `roles/iam.serviceAccountAdmin` when `generate_keys` is set to `true` -- (optional) roles needed to grant optional IAM roles at the project or organizational level