From 1cd3c4dc86d33fbece94ee069006b77300f3f386 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Fri, 20 Mar 2020 13:04:15 +0100 Subject: [PATCH 1/6] Create generic COS module and update CoreDNS module to use it --- modules/compute-vm-cos-coredns/main.tf | 74 +++---- modules/compute-vm-cos/README.md | 83 +++++++ .../cloud-config-file.yaml | 5 +- .../cloud-config.yaml | 60 +++--- modules/compute-vm-cos/main.tf | 105 +++++++++ modules/compute-vm-cos/outputs.tf | 50 +++++ modules/compute-vm-cos/variables.tf | 203 ++++++++++++++++++ 7 files changed, 504 insertions(+), 76 deletions(-) create mode 100644 modules/compute-vm-cos/README.md rename modules/{compute-vm-cos-coredns => compute-vm-cos}/cloud-config-file.yaml (80%) rename modules/{compute-vm-cos-coredns => compute-vm-cos}/cloud-config.yaml (55%) create mode 100644 modules/compute-vm-cos/main.tf create mode 100644 modules/compute-vm-cos/outputs.tf create mode 100644 modules/compute-vm-cos/variables.tf diff --git a/modules/compute-vm-cos-coredns/main.tf b/modules/compute-vm-cos-coredns/main.tf index b562f784c1..36c5bbe010 100644 --- a/modules/compute-vm-cos-coredns/main.tf +++ b/modules/compute-vm-cos-coredns/main.tf @@ -14,49 +14,9 @@ * limitations under the License. */ -locals { - files = { - for path, data in var.files : - path => merge(data, { - attributes = ( - data.attributes == null - ? { owner = "root", permissions = "0644" } - : data.attributes - ) - }) - } - files_yaml = join("\n", [ - for _, data in data.template_file.cloud-config-files : data.rendered - ]) -} - -data "template_file" "cloud-config-files" { - for_each = local.files - template = file("${path.module}/cloud-config-file.yaml") - vars = { - path = each.key - content = each.value.content - owner = each.value.attributes.owner - permissions = each.value.attributes.permissions - } -} - -data "template_file" "cloud-config" { - template = file("${path.module}/cloud-config.yaml") - vars = { - corefile = ( - var.coredns_corefile == null - ? file("${path.module}/Corefile") - : var.coredns_corefile - ) - files = local.files_yaml - image = var.coredns_image - log_driver = var.coredns_log_driver - } -} module "cos-coredns" { - source = "../compute-vm" + source = "../compute-vm-cos" project_id = var.project_id region = var.region zone = var.zone @@ -69,12 +29,42 @@ module "cos-coredns" { metadata = merge(var.metadata, { google-logging-enabled = var.cos_config.logging google-monitoring-enabled = var.cos_config.monitoring - user-data = data.template_file.cloud-config.rendered }) min_cpu_platform = var.min_cpu_platform network_interfaces = var.network_interfaces options = var.options service_account = var.service_account tags = var.tags + log_driver = var.coredns_log_driver use_instance_template = var.use_instance_template + cos_config = var.cos_config + image = var.coredns_image + files = { + "/etc/systemd/resolved.conf" : { + content = <<-EOT + [Resolve] + LLMNR=no + DNSStubListener=no + EOT + attributes = null + } + "/etc/coredns/Corefile" : { + content = (var.coredns_corefile == null + ? file("${path.module}/Corefile") + : var.coredns_corefile + ) + attributes = null + } + } + volumes = { + "/etc/coredns" : "/etc/coredns" + } + pre_runcmds = [ + "systemctl restart systemd-resolved.service" + ] + extra_args = "-conf /etc/coredns/Corefile" + exposed_ports = { + tcp = [53] + udp = [53] + } } diff --git a/modules/compute-vm-cos/README.md b/modules/compute-vm-cos/README.md new file mode 100644 index 0000000000..1c09486c31 --- /dev/null +++ b/modules/compute-vm-cos/README.md @@ -0,0 +1,83 @@ +# Container Optimized OS CoreDNS module + +This module allows creating instances (or an instance template) runnning a containerized DNS server using CoreDNS. A service account will be created if none is set. + +## Example + +```hcl +module "onprem-dns" { + source = "./modules/compute-vm-cos-coredns" + project_id = var.project_id + region = var.region + zone = var.zone + name = "coredns" + network_interfaces = [{ + network = var.vpc_self_link + subnetwork = var.subnet_self_link + nat = false, + addresses = null + }] + coredns_corefile = < +## Variables + +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| image | Container image. | string | ✓ | | +| name | Instances base name. | string | ✓ | | +| network_interfaces | Network interfaces configuration. Use self links for Shared VPC, set addresses to null if not needed. | list(object({...})) | ✓ | | +| project_id | Project id. | string | ✓ | | +| region | Compute region. | string | ✓ | | +| zone | Compute zone. | string | ✓ | | +| *boot_disk* | Boot disk properties. | object({...}) | | ... | +| *cos_config* | Configure COS logging and monitoring. | object({...}) | | ... | +| *exposed_ports* | Ports to expose in the host | object({...}) | | ... | +| *extra_args* | Extra arguments to pass to the container | string | | | +| *files* | Map of files to create on the instances, path as key. Attributes default to 'root' and '0644', set to null if not needed. | map(object({...})) | | {} | +| *hostname* | Instance FQDN name. | string | | null | +| *instance_count* | Number of instances to create (only for non-template usage). | number | | 1 | +| *instance_type* | Instance type. | string | | f1-micro | +| *labels* | Instance labels. | map(string) | | {} | +| *log_driver* | Container log driver (local, gcplogs, etc.). | string | | gcplogs | +| *metadata* | Instance metadata. | map(string) | | {} | +| *min_cpu_platform* | Minimum CPU platform. | string | | null | +| *options* | Instance options. | object({...}) | | ... | +| *post_runcmds* | Extra commands to run (in the host) after starting the container. | list(string) | | [] | +| *pre_runcmds* | Extra commands to run (in the host) before starting the container. | list(string) | | [] | +| *service_account* | Service account email (leave empty to auto-create). | string | | | +| *tags* | Instance tags. | list(string) | | ["dns", "ssh"] | +| *use_instance_template* | Create instance template instead of instances. | bool | | false | +| *volumes* | Map of volumes to mount in the container, key is the host path, value is the mount location inside the container | map(string) | | {} | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| external_ips | Instance main interface external IP addresses. | | +| instances | Instance resources. | | +| internal_ips | Instance main interface internal IP addresses. | | +| names | Instance names. | | +| self_links | Instance self links. | | +| template | Template resource. | | +| template_name | Template name. | | + diff --git a/modules/compute-vm-cos-coredns/cloud-config-file.yaml b/modules/compute-vm-cos/cloud-config-file.yaml similarity index 80% rename from modules/compute-vm-cos-coredns/cloud-config-file.yaml rename to modules/compute-vm-cos/cloud-config-file.yaml index 63dbc9d860..6aaceba382 100644 --- a/modules/compute-vm-cos-coredns/cloud-config-file.yaml +++ b/modules/compute-vm-cos/cloud-config-file.yaml @@ -1,6 +1,5 @@ -# skip boilerplate check - path: ${path} - content: | - ${indent(4, content)} owner: ${owner} permissions: ${permissions} + content: | + ${indent(4, content)} diff --git a/modules/compute-vm-cos-coredns/cloud-config.yaml b/modules/compute-vm-cos/cloud-config.yaml similarity index 55% rename from modules/compute-vm-cos-coredns/cloud-config.yaml rename to modules/compute-vm-cos/cloud-config.yaml index 06cc63c9cb..ed7cd34994 100644 --- a/modules/compute-vm-cos-coredns/cloud-config.yaml +++ b/modules/compute-vm-cos/cloud-config.yaml @@ -14,11 +14,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -# https://hub.docker.com/r/coredns/coredns/ -# https://coredns.io/manual/toc/#installation - # TODO: switch to the gcplogs logging driver, and set driver labels +# create a system user, otherwise docker-credential-gcr fails as root +users: +- name: cosuser + uid: 2000 + groups: docker + write_files: - path: /var/lib/docker/daemon.json permissions: 0644 @@ -32,43 +35,38 @@ write_files: } } -# disable systemd-resolved to free port 53 on the loopback interface -- path: /etc/systemd/resolved.conf - permissions: 0644 - owner: root - content: | - [Resolve] - LLMNR=no - DNSStubListener=no - -# Write coreDNS config -- path: /etc/coredns/Corefile - permissions: 0644 - owner: root - content: | - ${indent(4, corefile)} +${files} -# coredns container service -- path: /etc/systemd/system/coredns.service +# container service +- path: /etc/systemd/system/cosapp.service permissions: 0644 owner: root content: | [Unit] - Description=Start CoreDNS container + Description=Start Application container After=gcr-online.target docker.socket Wants=gcr-online.target docker.socket docker-events-collector.service [Service] - ExecStart=/usr/bin/docker run --rm --name=coredns \ + User=cosuser + ExecStartPre=/usr/bin/docker-credential-gcr configure-docker + ExecStart=/usr/bin/docker run --rm --name=cosapp \ --log-driver=${log_driver} --network host \ - -v /etc/coredns:/etc/coredns \ - ${image} -conf /etc/coredns/Corefile - ExecStop=/usr/bin/docker stop coredns - -${files} + %{~ if volumes != "" ~} + ${volumes} \ + %{~ endif ~} + ${image} ${extra_args} + ExecStop=/usr/bin/docker stop cosapp + ExecStopPost=/usr/bin/docker rm cosapp runcmd: -- iptables -I INPUT 1 -p tcp -m tcp --dport 53 -m state --state NEW,ESTABLISHED -j ACCEPT -- iptables -I INPUT 1 -p udp -m udp --dport 53 -m state --state NEW,ESTABLISHED -j ACCEPT - systemctl daemon-reload -- systemctl restart systemd-resolved.service -- systemctl start coredns \ No newline at end of file +%{ if fw_runcmds != "" ~} +${fw_runcmds} +%{ endif ~} +%{ if pre_runcmds != "" ~} +${pre_runcmds} +%{ endif ~} +- systemctl start cosapp +%{ if post_runcmds != "" ~} +${post_runcmds} +%{ endif ~} diff --git a/modules/compute-vm-cos/main.tf b/modules/compute-vm-cos/main.tf new file mode 100644 index 0000000000..4896236611 --- /dev/null +++ b/modules/compute-vm-cos/main.tf @@ -0,0 +1,105 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + files = { + for path, data in var.files : + path => merge(data, { + attributes = ( + data.attributes == null + ? { owner = "root", permissions = "0644" } + : data.attributes + ) + }) + } + files_yaml = join("\n", [ + for _, data in data.template_file.cloud-config-files : data.rendered + ]) + volumes = join(" ", [ + for host_path, mount_path in var.volumes : "-v ${host_path}:${mount_path}" + ]) + tcp_ports = [ + for port in coalesce(var.exposed_ports.tcp, []) + : "- iptables -I INPUT 1 -p tcp -m tcp --dport ${port} -m state --state NEW,ESTABLISHED -j ACCEPT" + ] + udp_ports = [ + for port in coalesce(var.exposed_ports.udp, []) + : "- iptables -I INPUT 1 -p udp -m udp --dport ${port} -m state --state NEW,ESTABLISHED -j ACCEPT" + ] + pre_runcmds = [ + for cmd in var.pre_runcmds : "- ${cmd}" + ] + post_runcmds = [ + for cmd in var.post_runcmds : "- ${cmd}" + ] + fw_runcmds_yaml = join("\n", concat(local.tcp_ports, local.udp_ports)) + pre_runcmds_yaml = join("\n", local.pre_runcmds) + post_runcmds_yaml = join("\n", local.post_runcmds) +} + +data "template_file" "cloud-config-files" { + for_each = local.files + template = file("${path.module}/cloud-config-file.yaml") + vars = { + path = each.key + content = each.value.content + owner = each.value.attributes.owner + permissions = each.value.attributes.permissions + } +} + +data "template_file" "cloud-config" { + template = file("${path.module}/cloud-config.yaml") + vars = { + files = local.files_yaml + volumes = local.volumes + extra_args = var.extra_args + image = var.image + log_driver = var.log_driver + fw_runcmds = local.fw_runcmds_yaml + pre_runcmds = local.pre_runcmds_yaml + post_runcmds = local.post_runcmds_yaml + } +} + +# output "rendered" { +# value = data.template_file.cloud-config.rendered +# } + + +module "vm" { + source = "../compute-vm" + project_id = var.project_id + region = var.region + zone = var.zone + name = var.name + boot_disk = var.boot_disk + hostname = var.hostname + instance_count = var.instance_count + instance_type = var.instance_type + labels = var.labels + metadata = merge(var.metadata, { + google-logging-enabled = var.cos_config.logging + google-monitoring-enabled = var.cos_config.monitoring + user-data = data.template_file.cloud-config.rendered + }) + min_cpu_platform = var.min_cpu_platform + network_interfaces = var.network_interfaces + options = var.options + service_account = var.service_account + tags = var.tags + use_instance_template = var.use_instance_template +} diff --git a/modules/compute-vm-cos/outputs.tf b/modules/compute-vm-cos/outputs.tf new file mode 100644 index 0000000000..c70585e697 --- /dev/null +++ b/modules/compute-vm-cos/outputs.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "instances" { + description = "Instance resources." + value = module.vm.instances +} + +output "names" { + description = "Instance names." + value = module.vm.names +} + +output "self_links" { + description = "Instance self links." + value = module.vm.self_links +} + +output "internal_ips" { + description = "Instance main interface internal IP addresses." + value = module.vm.internal_ips +} + +output "external_ips" { + description = "Instance main interface external IP addresses." + value = module.vm.external_ips +} + +output "template" { + description = "Template resource." + value = module.vm.template +} + +output "template_name" { + description = "Template name." + value = module.vm.template_name +} diff --git a/modules/compute-vm-cos/variables.tf b/modules/compute-vm-cos/variables.tf new file mode 100644 index 0000000000..dcd6d96ec3 --- /dev/null +++ b/modules/compute-vm-cos/variables.tf @@ -0,0 +1,203 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "boot_disk" { + description = "Boot disk properties." + type = object({ + image = string + size = number + type = string + }) + default = { + image = "projects/cos-cloud/global/images/family/cos-stable" + type = "pd-ssd" + size = 10 + } +} + +variable "image" { + description = "Container image." + type = string +} + +variable "log_driver" { + description = "Container log driver (local, gcplogs, etc.)." + type = string + default = "gcplogs" +} + +variable "cos_config" { + description = "Configure COS logging and monitoring." + type = object({ + logging = bool + monitoring = bool + }) + default = { + logging = false + monitoring = true + } +} + +variable "files" { + description = "Map of files to create on the instances, path as key. Attributes default to 'root' and '0644', set to null if not needed." + type = map(object({ + content = string + attributes = object({ + owner = string + permissions = string + }) + })) + default = {} +} + +variable "hostname" { + description = "Instance FQDN name." + type = string + default = null +} + +variable "instance_count" { + description = "Number of instances to create (only for non-template usage)." + type = number + default = 1 +} + +variable "instance_type" { + description = "Instance type." + type = string + default = "f1-micro" +} + +variable "labels" { + description = "Instance labels." + type = map(string) + default = {} +} + +variable "metadata" { + description = "Instance metadata." + type = map(string) + default = {} +} + +variable "min_cpu_platform" { + description = "Minimum CPU platform." + type = string + default = null +} + +variable "name" { + description = "Instances base name." + type = string +} + +variable "network_interfaces" { + description = "Network interfaces configuration. Use self links for Shared VPC, set addresses to null if not needed." + type = list(object({ + nat = bool + network = string + subnetwork = string + addresses = object({ + internal = list(string) + external = list(string) + }) + })) +} + +variable "options" { + description = "Instance options." + type = object({ + allow_stopping_for_update = bool + can_ip_forward = bool + deletion_protection = bool + preemptible = bool + }) + default = { + allow_stopping_for_update = true + can_ip_forward = false + deletion_protection = false + preemptible = false + } +} + +variable "project_id" { + description = "Project id." + type = string +} + +variable "region" { + description = "Compute region." + type = string +} + +variable "service_account" { + description = "Service account email (leave empty to auto-create)." + type = string + default = "" +} + +variable "tags" { + description = "Instance tags." + type = list(string) + default = ["dns", "ssh"] +} + +variable "use_instance_template" { + description = "Create instance template instead of instances." + type = bool + default = false +} + +variable "zone" { + description = "Compute zone." + type = string +} + +variable "volumes" { + description = "Map of volumes to mount in the container, key is the host path, value is the mount location inside the container" + type = map(string) + default = {} +} + +variable "exposed_ports" { + description = "Ports to expose in the host" + type = object({ + tcp = list(number) + udp = list(number) + }) + default = { + tcp = [] + udp = [] + } +} + +variable "extra_args" { + description = "Extra arguments to pass to the container" + type = string + default = "" +} + +variable "pre_runcmds" { + description = "Extra commands to run (in the host) before starting the container." + type = list(string) + default = [] +} + +variable "post_runcmds" { + description = "Extra commands to run (in the host) after starting the container." + type = list(string) + default = [] +} From 657b7665afe6101cfdd7d478c4c97ceabc8ead12 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Fri, 20 Mar 2020 13:06:50 +0100 Subject: [PATCH 2/6] Update compute-vm-cos README --- modules/compute-vm-cos/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/compute-vm-cos/README.md b/modules/compute-vm-cos/README.md index 1c09486c31..b4b65989ff 100644 --- a/modules/compute-vm-cos/README.md +++ b/modules/compute-vm-cos/README.md @@ -1,6 +1,6 @@ -# Container Optimized OS CoreDNS module +# Container Optimized OS module -This module allows creating instances (or an instance template) runnning a containerized DNS server using CoreDNS. A service account will be created if none is set. +This module allows creating instances (or an instance template) runnning a containerized application using CoreDNS. A service account will be created if none is set. ## Example From 274e556b7e17f93b318fca8ff370668d8ec6f838 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Fri, 20 Mar 2020 13:07:55 +0100 Subject: [PATCH 3/6] Fix COS README --- modules/compute-vm-cos/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/compute-vm-cos/README.md b/modules/compute-vm-cos/README.md index b4b65989ff..fe5e0ab1b1 100644 --- a/modules/compute-vm-cos/README.md +++ b/modules/compute-vm-cos/README.md @@ -1,6 +1,6 @@ # Container Optimized OS module -This module allows creating instances (or an instance template) runnning a containerized application using CoreDNS. A service account will be created if none is set. +This module allows creating instances (or an instance template) runnning a containerized application using COS. A service account will be created if none is set. ## Example From 1f04a853543686c68f590dcd46583c34581bc7d0 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Fri, 20 Mar 2020 13:33:37 +0100 Subject: [PATCH 4/6] Update COS example --- modules/compute-vm-cos/README.md | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/modules/compute-vm-cos/README.md b/modules/compute-vm-cos/README.md index fe5e0ab1b1..bafc36d30f 100644 --- a/modules/compute-vm-cos/README.md +++ b/modules/compute-vm-cos/README.md @@ -5,36 +5,32 @@ This module allows creating instances (or an instance template) runnning a conta ## Example ```hcl -module "onprem-dns" { - source = "./modules/compute-vm-cos-coredns" +module "nginx" { + source = "./modules/compute-vm-cos" project_id = var.project_id region = var.region zone = var.zone - name = "coredns" + name = "nginx" + image = "nginx:1.17" network_interfaces = [{ network = var.vpc_self_link subnetwork = var.subnet_self_link nat = false, addresses = null }] - coredns_corefile = < Date: Fri, 20 Mar 2020 13:38:28 +0100 Subject: [PATCH 5/6] Skip boilerplate check for COS file template --- modules/compute-vm-cos/cloud-config-file.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/compute-vm-cos/cloud-config-file.yaml b/modules/compute-vm-cos/cloud-config-file.yaml index 6aaceba382..673b6c703f 100644 --- a/modules/compute-vm-cos/cloud-config-file.yaml +++ b/modules/compute-vm-cos/cloud-config-file.yaml @@ -1,3 +1,4 @@ +# skip boilerplate check - path: ${path} owner: ${owner} permissions: ${permissions} From de3480b877d21ca2f31163a932339f0edbb78b1a Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Thu, 26 Mar 2020 15:27:35 +0100 Subject: [PATCH 6/6] Make COS module more generic and provide preset configurations --- modules/compute-vm-cos/README.md | 58 +++++++++---- modules/compute-vm-cos/cloud-config-file.yaml | 6 -- .../compute-vm-cos/cloud-config/coredns.yaml | 87 +++++++++++++++++++ .../generic.yaml} | 34 +++++--- modules/compute-vm-cos/main.tf | 81 +++++------------ modules/compute-vm-cos/variables.tf | 51 ++--------- 6 files changed, 174 insertions(+), 143 deletions(-) delete mode 100644 modules/compute-vm-cos/cloud-config-file.yaml create mode 100644 modules/compute-vm-cos/cloud-config/coredns.yaml rename modules/compute-vm-cos/{cloud-config.yaml => cloud-config/generic.yaml} (73%) diff --git a/modules/compute-vm-cos/README.md b/modules/compute-vm-cos/README.md index bafc36d30f..204c47fd29 100644 --- a/modules/compute-vm-cos/README.md +++ b/modules/compute-vm-cos/README.md @@ -11,27 +11,55 @@ module "nginx" { region = var.region zone = var.zone name = "nginx" - image = "nginx:1.17" network_interfaces = [{ network = var.vpc_self_link subnetwork = var.subnet_self_link nat = false, addresses = null }] - files = { - "/etc/nginx/htdocs/index.html" = { - content = "hello world" - attributes = null + cloud_config = { + template_path = "preset:generic.yaml" + variables = { + image = "nginx:1.17" + files = { + "/etc/nginx/htdocs/index.html" : { + content = "hello world" + } + } + exposed_ports = { + tcp = [80] + } + volumes = { + "/etc/nginx/htdocs" : "/usr/share/nginx/html" + } } } - volumes = { - "/etc/nginx/htdocs" : "/usr/share/nginx/html" - } - exposed_ports = { - tcp = [80] - udp = null +} + +module "coredns" { + source = "../../tmp/cloud-foundation-fabric/modules/compute-vm-cos/" + project_id = "jccb-testenv" + zone = "europe-west1-b" + region = "europe-west1" + name = "coredns" + network_interfaces = [{ + network = "net1" + subnetwork = "https://www.googleapis.com/compute/v1/projects/jccb-testenv/regions/europe-west1/subnetworks/europe-west1" + nat = false, + addresses = null + }] + instance_type = "n1-standard-1" + + cloud_config = { + template_path = "preset:coredns.yaml" + variables = { + corefile = "default" + log_driver = "gcplogs" + files = {} + } } } + ``` @@ -39,7 +67,7 @@ module "nginx" { | name | description | type | required | default | |---|---|:---: |:---:|:---:| -| image | Container image. | string | ✓ | | +| cloud_config | None | | ✓ | | | name | Instances base name. | string | ✓ | | | network_interfaces | Network interfaces configuration. Use self links for Shared VPC, set addresses to null if not needed. | list(object({...})) | ✓ | | | project_id | Project id. | string | ✓ | | @@ -47,23 +75,17 @@ module "nginx" { | zone | Compute zone. | string | ✓ | | | *boot_disk* | Boot disk properties. | object({...}) | | ... | | *cos_config* | Configure COS logging and monitoring. | object({...}) | | ... | -| *exposed_ports* | Ports to expose in the host | object({...}) | | ... | -| *extra_args* | Extra arguments to pass to the container | string | | | | *files* | Map of files to create on the instances, path as key. Attributes default to 'root' and '0644', set to null if not needed. | map(object({...})) | | {} | | *hostname* | Instance FQDN name. | string | | null | | *instance_count* | Number of instances to create (only for non-template usage). | number | | 1 | | *instance_type* | Instance type. | string | | f1-micro | | *labels* | Instance labels. | map(string) | | {} | -| *log_driver* | Container log driver (local, gcplogs, etc.). | string | | gcplogs | | *metadata* | Instance metadata. | map(string) | | {} | | *min_cpu_platform* | Minimum CPU platform. | string | | null | | *options* | Instance options. | object({...}) | | ... | -| *post_runcmds* | Extra commands to run (in the host) after starting the container. | list(string) | | [] | -| *pre_runcmds* | Extra commands to run (in the host) before starting the container. | list(string) | | [] | | *service_account* | Service account email (leave empty to auto-create). | string | | | | *tags* | Instance tags. | list(string) | | ["dns", "ssh"] | | *use_instance_template* | Create instance template instead of instances. | bool | | false | -| *volumes* | Map of volumes to mount in the container, key is the host path, value is the mount location inside the container | map(string) | | {} | ## Outputs diff --git a/modules/compute-vm-cos/cloud-config-file.yaml b/modules/compute-vm-cos/cloud-config-file.yaml deleted file mode 100644 index 673b6c703f..0000000000 --- a/modules/compute-vm-cos/cloud-config-file.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# skip boilerplate check -- path: ${path} - owner: ${owner} - permissions: ${permissions} - content: | - ${indent(4, content)} diff --git a/modules/compute-vm-cos/cloud-config/coredns.yaml b/modules/compute-vm-cos/cloud-config/coredns.yaml new file mode 100644 index 0000000000..2734868c41 --- /dev/null +++ b/modules/compute-vm-cos/cloud-config/coredns.yaml @@ -0,0 +1,87 @@ +#cloud-config + +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# https://hub.docker.com/r/coredns/coredns/ +# https://coredns.io/manual/toc/#installation + +# TODO: switch to the gcplogs logging driver, and set driver labels + +write_files: +- path: /var/lib/docker/daemon.json + permissions: 0644 + owner: root + content: | + { + "live-restore": true, + "storage-driver": "overlay2", + "log-opts": { + "max-size": "1024m" + } + } + +# disable systemd-resolved to free port 53 on the loopback interface +- path: /etc/systemd/resolved.conf + permissions: 0644 + owner: root + content: | + [Resolve] + LLMNR=no + DNSStubListener=no + +- path: /etc/coredns/Corefile + permissions: 0644 + owner: root + content: | +%{ if corefile == "default" ~} + . { + forward . /etc/resolv.conf + log + errors + } +%{ else ~} + ${indent(4, corefile)} +%{ endif ~} + +# coredns container service +- path: /etc/systemd/system/coredns.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Start CoreDNS container + After=gcr-online.target docker.socket + Wants=gcr-online.target docker.socket docker-events-collector.service + [Service] + ExecStart=/usr/bin/docker run --rm --name=coredns \ + --log-driver=${log_driver} --network host \ + -v /etc/coredns:/etc/coredns \ + coredns/coredns -conf /etc/coredns/Corefile + ExecStop=/usr/bin/docker stop coredns + +%{ for path, data in files } +- path: ${path} + owner: ${lookup(data, "owner", "root")} + permissions: ${lookup(data, "permissions", "0644")} + content: | + ${indent(4, data.content)} +%{ endfor } + +runcmd: +- iptables -I INPUT 1 -p tcp -m tcp --dport 53 -m state --state NEW,ESTABLISHED -j ACCEPT +- iptables -I INPUT 1 -p udp -m udp --dport 53 -m state --state NEW,ESTABLISHED -j ACCEPT +- systemctl daemon-reload +- systemctl restart systemd-resolved.service +- systemctl start coredns diff --git a/modules/compute-vm-cos/cloud-config.yaml b/modules/compute-vm-cos/cloud-config/generic.yaml similarity index 73% rename from modules/compute-vm-cos/cloud-config.yaml rename to modules/compute-vm-cos/cloud-config/generic.yaml index ed7cd34994..588a7d385d 100644 --- a/modules/compute-vm-cos/cloud-config.yaml +++ b/modules/compute-vm-cos/cloud-config/generic.yaml @@ -35,7 +35,13 @@ write_files: } } -${files} +%{ for path, data in files } +- path: ${path} + owner: ${lookup(data, "owner", "root")} + permissions: ${lookup(data, "permissions", "0644")} + content: | + ${indent(4, data.content)} +%{ endfor } # container service - path: /etc/systemd/system/cosapp.service @@ -51,22 +57,24 @@ ${files} ExecStartPre=/usr/bin/docker-credential-gcr configure-docker ExecStart=/usr/bin/docker run --rm --name=cosapp \ --log-driver=${log_driver} --network host \ - %{~ if volumes != "" ~} - ${volumes} \ - %{~ endif ~} + %{~ for host_path, mount_path in volumes ~} + -v ${host_path}:${mount_path} \ + %{~ endfor ~} ${image} ${extra_args} ExecStop=/usr/bin/docker stop cosapp ExecStopPost=/usr/bin/docker rm cosapp runcmd: - systemctl daemon-reload -%{ if fw_runcmds != "" ~} -${fw_runcmds} -%{ endif ~} -%{ if pre_runcmds != "" ~} -${pre_runcmds} -%{ endif ~} +%{ for proto, ports in fw_runcmds ~} + %{~ for port in ports ~} +- iptables -I INPUT 1 -p ${proto} -m ${proto} --dport ${port} -m state --state NEW,ESTABLISHED -j ACCEPT + %{~ endfor ~} +%{ endfor ~} +%{ for cmd in pre_runcmds ~} +- ${cmd} +%{ endfor ~} - systemctl start cosapp -%{ if post_runcmds != "" ~} -${post_runcmds} -%{ endif ~} +%{ for cmd in pre_runcmds ~} +- ${cmd} +%{ endfor ~} diff --git a/modules/compute-vm-cos/main.tf b/modules/compute-vm-cos/main.tf index 4896236611..433440f876 100644 --- a/modules/compute-vm-cos/main.tf +++ b/modules/compute-vm-cos/main.tf @@ -15,70 +15,29 @@ */ locals { - files = { - for path, data in var.files : - path => merge(data, { - attributes = ( - data.attributes == null - ? { owner = "root", permissions = "0644" } - : data.attributes - ) - }) - } - files_yaml = join("\n", [ - for _, data in data.template_file.cloud-config-files : data.rendered - ]) - volumes = join(" ", [ - for host_path, mount_path in var.volumes : "-v ${host_path}:${mount_path}" - ]) - tcp_ports = [ - for port in coalesce(var.exposed_ports.tcp, []) - : "- iptables -I INPUT 1 -p tcp -m tcp --dport ${port} -m state --state NEW,ESTABLISHED -j ACCEPT" - ] - udp_ports = [ - for port in coalesce(var.exposed_ports.udp, []) - : "- iptables -I INPUT 1 -p udp -m udp --dport ${port} -m state --state NEW,ESTABLISHED -j ACCEPT" - ] - pre_runcmds = [ - for cmd in var.pre_runcmds : "- ${cmd}" - ] - post_runcmds = [ - for cmd in var.post_runcmds : "- ${cmd}" - ] - fw_runcmds_yaml = join("\n", concat(local.tcp_ports, local.udp_ports)) - pre_runcmds_yaml = join("\n", local.pre_runcmds) - post_runcmds_yaml = join("\n", local.post_runcmds) -} + cc = var.cloud_config + ccvars = var.cloud_config.variables -data "template_file" "cloud-config-files" { - for_each = local.files - template = file("${path.module}/cloud-config-file.yaml") - vars = { - path = each.key - content = each.value.content - owner = each.value.attributes.owner - permissions = each.value.attributes.permissions - } -} + use_generic_template = local.cc.template_path == "preset:generic.yaml" + template_path = replace(local.cc.template_path, "/preset:/", "${path.module}/cloud-config/") -data "template_file" "cloud-config" { - template = file("${path.module}/cloud-config.yaml") - vars = { - files = local.files_yaml - volumes = local.volumes - extra_args = var.extra_args - image = var.image - log_driver = var.log_driver - fw_runcmds = local.fw_runcmds_yaml - pre_runcmds = local.pre_runcmds_yaml - post_runcmds = local.post_runcmds_yaml + generic_config_vars = { + image = lookup(local.ccvars, "image", "") + extra_args = lookup(local.ccvars, "extra_args", "") + log_driver = lookup(local.ccvars, "log_driver", "gcplogs") + volumes = lookup(local.ccvars, "volumes", {}) + pre_runcmds = lookup(local.ccvars, "pre_runcmds", []) + post_runcmds = lookup(local.ccvars, "post_runcmds", []) + fw_runcmds = lookup(local.ccvars, "exposed_ports", []) + files = lookup(local.ccvars, "files", {}) } -} - -# output "rendered" { -# value = data.template_file.cloud-config.rendered -# } + cloud_config_content = ( + local.use_generic_template + ? templatefile(local.template_path, local.generic_config_vars) + : templatefile(local.template_path, var.cloud_config.variables) + ) +} module "vm" { source = "../compute-vm" @@ -94,7 +53,7 @@ module "vm" { metadata = merge(var.metadata, { google-logging-enabled = var.cos_config.logging google-monitoring-enabled = var.cos_config.monitoring - user-data = data.template_file.cloud-config.rendered + user-data = local.cloud_config_content }) min_cpu_platform = var.min_cpu_platform network_interfaces = var.network_interfaces diff --git a/modules/compute-vm-cos/variables.tf b/modules/compute-vm-cos/variables.tf index dcd6d96ec3..85acce2654 100644 --- a/modules/compute-vm-cos/variables.tf +++ b/modules/compute-vm-cos/variables.tf @@ -28,17 +28,6 @@ variable "boot_disk" { } } -variable "image" { - description = "Container image." - type = string -} - -variable "log_driver" { - description = "Container log driver (local, gcplogs, etc.)." - type = string - default = "gcplogs" -} - variable "cos_config" { description = "Configure COS logging and monitoring." type = object({ @@ -166,38 +155,10 @@ variable "zone" { type = string } -variable "volumes" { - description = "Map of volumes to mount in the container, key is the host path, value is the mount location inside the container" - type = map(string) - default = {} -} - -variable "exposed_ports" { - description = "Ports to expose in the host" - type = object({ - tcp = list(number) - udp = list(number) - }) - default = { - tcp = [] - udp = [] - } -} - -variable "extra_args" { - description = "Extra arguments to pass to the container" - type = string - default = "" -} - -variable "pre_runcmds" { - description = "Extra commands to run (in the host) before starting the container." - type = list(string) - default = [] -} - -variable "post_runcmds" { - description = "Extra commands to run (in the host) after starting the container." - type = list(string) - default = [] +variable "cloud_config" { + description = "Configuration parameters for cloud-config" + # type = object({ + # template_path = string + # variables = map(any) + # }) }