Extended simple-nva module to manage BGP service running on FR routing docker container

modules/cloud-config-container/simple-nva/README.md

@@ -62,16 +62,88 @@ module "vm" {
 }
 # tftest modules=1 resources=1
 ```
+
+### Example with advanced routing capabilities
+
+Find below a sample terraform example for bootstrapping a simple NVA powered by [COS](https://cloud.google.com/container-optimized-os/docs) and running [FRRouting](https://frrouting.org/) container.
+Please find below a sample frr.conf file based on the documentation available [here](https://docs.frrouting.org/en/latest/basic.html) for hosting a BGP service with ASN 65001 on FRR container establishing a BGP session with a remote neighbor with IP address 10.128.0.2 and ASN 65002.
+
+```
+# tftest-file id=frr_conf path=./frr.conf
+# Example frr.conmf file
+
+log syslog informational
+no ipv6 forwarding
+router bgp 65001
+ neighbor 10.128.0.2 remote-as 65002
+line vty
+```
+
+Following code assumes a file in the same folder named frr.conf exists. 
+
+```hcl
+locals {
+  network_interfaces = [
+    {
+      addresses           = null
+      name                = "dev"
+      nat                 = false
+      network             = "dev_vpc_self_link"
+      routes              = ["10.128.0.0/9"]
+      subnetwork          = "dev_vpc_nva_subnet_self_link"
+      enable_masquerading = true
+      non_masq_cidrs      = ["10.0.0.0/8"]
+    },
+    {
+      addresses  = null
+      name       = "prod"
+      nat        = false
+      network    = "prod_vpc_self_link"
+      routes     = ["10.0.0.0/9"]
+      subnetwork = "prod_vpc_nva_subnet_self_link"
+    }
+  ]
+}
+
+module "cos-nva" {
+  source               = "./fabric/modules/cloud-config-container/simple-nva"
+  enable_health_checks = true
+  network_interfaces   = local.network_interfaces
+  frr_config           = { config_file = "./frr.conf", daemons_enabled = ["bgpd"] }
+  optional_run_cmds    = ["ls -l"]
+}
+
+module "vm" {
+  source             = "./fabric/modules/compute-vm"
+  project_id         = "my-project"
+  zone               = "europe-west8-b"
+  name               = "cos-nva"
+  network_interfaces = local.network_interfaces
+  metadata = {
+    user-data              = module.cos-nva.cloud_config
+    google-logging-enabled = true
+  }
+  boot_disk = {
+    image = "projects/cos-cloud/global/images/family/cos-stable"
+    type  = "pd-ssd"
+    size  = 10
+  }
+  tags = ["nva", "ssh"]
+}
+# tftest modules=1 resources=1 files=frr_conf
+```
 <!-- BEGIN TFDOC -->
 
 ## Variables
 
 | name | description | type | required | default |
 |---|---|:---:|:---:|:---:|
-| [network_interfaces](variables.tf#L39) | Network interfaces configuration. | <code title="list&#40;object&#40;&#123;&#10;  routes &#61; optional&#40;list&#40;string&#41;&#41;&#10;&#125;&#41;&#41;">list&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | ✓ |  |
+| [network_interfaces](variables.tf#L54) | Network interfaces configuration. | <code title="list&#40;object&#40;&#123;&#10;  routes              &#61; optional&#40;list&#40;string&#41;&#41;&#10;  enable_masquerading &#61; optional&#40;bool, false&#41;&#10;  non_masq_cidrs      &#61; optional&#40;list&#40;string&#41;&#41;&#10;&#125;&#41;&#41;">list&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | ✓ |  |
 | [cloud_config](variables.tf#L17) | Cloud config template path. If null default will be used. | <code>string</code> |  | <code>null</code> |
 | [enable_health_checks](variables.tf#L23) | Configures routing to enable responses to health check probes. | <code>bool</code> |  | <code>false</code> |
 | [files](variables.tf#L29) | Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null. | <code title="map&#40;object&#40;&#123;&#10;  content     &#61; string&#10;  owner       &#61; string&#10;  permissions &#61; string&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
+| [frr_config](variables.tf#L39) | FRR configuration for container running on the NVA. | <code title="object&#40;&#123;&#10;  daemons_enabled &#61; optional&#40;list&#40;string&#41;&#41;&#10;  config_file     &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
+| [optional_run_cmds](variables.tf#L63) | Optional Cloud Init run commands to execute. | <code>list&#40;string&#41;</code> |  | <code>&#91;&#93;</code> |
 
 ## Outputs
 

modules/cloud-config-container/simple-nva/cloud-config.yaml

@@ -1,6 +1,6 @@
 #cloud-config
 
-# Copyright 2022 Google LLC
+# Copyright 2023 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -22,6 +22,7 @@ write_files:
     content: |
       ${indent(6, data.content)}
 %{ endfor }
+
   - path: /etc/systemd/system/routing.service
     permissions: 0644
     owner: root
@@ -34,6 +35,7 @@ write_files:
       Wants=network-online.target
       [Service]
       ExecStart=/bin/sh -c "/var/run/nva/start-routing.sh"
+
   - path: /var/run/nva/start-routing.sh
     permissions: 0744
     owner: root
@@ -43,6 +45,12 @@ write_files:
 %{ if enable_health_checks ~}
       /var/run/nva/policy_based_routing.sh ${interface.name}
 %{ endif ~}
+%{ if interface.enable_masquerading ~}
+%{ for cidr in interface.non_masq_cidrs ~}
+      iptables -t nat -A POSTROUTING -o ${interface.name} -d ${cidr} -j ACCEPT
+%{ endfor ~}
+      iptables -t nat -A POSTROUTING -o ${interface.name} -j MASQUERADE
+%{ endif ~}
 %{ for route in interface.routes ~}
       ip route add ${route} via `curl http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/${interface.number}/gateway -H "Metadata-Flavor:Google"` dev ${interface.name}
 %{ endfor ~}
@@ -55,4 +63,6 @@ runcmd:
   - systemctl daemon-reload
   - systemctl enable routing
   - systemctl start routing
-
+%{ for cmd in optional_run_cmds ~}
+  - ${cmd}
+%{ endfor ~}

modules/cloud-config-container/simple-nva/files/frr/daemons

@@ -0,0 +1,65 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+zebra=${zebra_enabled}
+bgpd=${bgpd_enabled}
+ospfd=${ospfd_enabled}
+ospf6d=${ospf6d_enabled}
+ripd=${ripd_enabled}
+ripngd=${ripngd_enabled}
+isisd=${isisd_enabled}
+pimd=${pimd_enabled}
+ldpd=${ldpd_enabled}
+nhrpd=${nhrpd_enabled}
+eigrpd=${eigrpd_enabled}
+babeld=${babeld_enabled}
+sharpd=${sharpd_enabled}
+staticd=${staticd_enabled}
+pbrd=${pbrd_enabled}
+bfdd=${bfdd_enabled}
+fabricd=${fabricd_enabled}
+
+# If this option is set the /etc/init.d/frr script automatically loads
+# the config via "vtysh -b" when the servers are started.
+# Check /etc/pam.d/frr if you intend to use "vtysh"!
+
+vtysh_enable=yes
+zebra_options="  -A 127.0.0.1 -s 90000000"
+bgpd_options="   -A 127.0.0.1"
+ospfd_options="  --daemon -A 127.0.0.1"
+ospf6d_options=" --daemon -A ::1"
+ripd_options="   --daemon -A 127.0.0.1"
+ripngd_options=" --daemon -A ::1"
+isisd_options="  --daemon -A 127.0.0.1"
+pimd_options="  --daemon -A 127.0.0.1"
+ldpd_options="  --daemon -A 127.0.0.1"
+nhrpd_options="  --daemon -A 127.0.0.1"
+eigrpd_options="  --daemon -A 127.0.0.1"
+babeld_options="  --daemon -A 127.0.0.1"
+sharpd_options="  --daemon -A 127.0.0.1"
+staticd_options="  --daemon -A 127.0.0.1"
+pbrd_options="  --daemon -A 127.0.0.1"
+bfdd_options="  --daemon -A 127.0.0.1"
+fabricd_options="  --daemon -A 127.0.0.1"
+
+#MAX_FDS=1024
+# The list of daemons to watch is automatically generated by the init script.
+#watchfrr_options=""
+
+# for debugging purposes, you can specify a "wrap" command to start instead
+# of starting the daemon directly, e.g. to use valgrind on ospfd:
+#   ospfd_wrap="/usr/bin/valgrind"
+# or you can use "all_wrap" for all daemons, e.g. to use perf record:
+#   all_wrap="/usr/bin/perf record --call-graph -"
+# the normal daemon command is added to this at the end.

modules/cloud-config-container/simple-nva/files/frr/frr.service

@@ -0,0 +1,27 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[Unit]
+Description=Start FRR container
+After=gcr-online.target docker.socket
+Wants=gcr-online.target docker.socket docker-events-collector.service
+[Service]
+Environment="HOME=/home/frr"
+ExecStart=/usr/bin/docker run --rm --name=frr \
+--privileged \
+--network host \
+-v /etc/frr:/etc/frr \
+frrouting/frr
+ExecStop=/usr/bin/docker stop frr
+ExecStopPost=/usr/bin/docker rm frr

modules/cloud-config-container/simple-nva/files/ipprefix_by_netmask.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2022 Google LLC
+# Copyright 2023 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

modules/cloud-config-container/simple-nva/files/policy_based_routing.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2022 Google LLC
+# Copyright 2023 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

modules/cloud-config-container/simple-nva/main.tf

@@ -1,5 +1,5 @@
 /**
- * Copyright 2022 Google LLC
+ * Copyright 2023 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,42 +15,109 @@
  */
 
 locals {
-  cloud_config = templatefile(local.template, merge({
-    files                = local.files
-    enable_health_checks = var.enable_health_checks
-    network_interfaces   = local.network_interfaces
-  }))
+  _files = merge(
+    {
+      "/var/run/nva/ipprefix_by_netmask.sh" = {
+        content     = file("${path.module}/files/ipprefix_by_netmask.sh")
+        owner       = "root"
+        permissions = "0744"
+      }
+      "/var/run/nva/policy_based_routing.sh" = {
+        content     = file("${path.module}/files/policy_based_routing.sh")
+        owner       = "root"
+        permissions = "0744"
+      }
+      }, {
+      for path, attrs in var.files : path => {
+        content     = attrs.content,
+        owner       = attrs.owner,
+        permissions = attrs.permissions
+      }
+    },
+    try(var.frr_config != null, false) ? {
+      "/etc/frr/daemons" = {
+        content     = templatefile("${path.module}/files/frr/daemons", local._frr_daemons_enabled)
+        owner       = "root"
+        permissions = "0744"
+      }
+      "/etc/frr/frr.conf" = {
+        content     = file(var.frr_config.config_file)
+        owner       = "root"
+        permissions = "0744"
+      }
+      "/etc/systemd/system/frr.service" = {
+        content     = file("${path.module}/files/frr/frr.service")
+        owner       = "root"
+        permissions = "0644"
+      }
+      "/var/lib/docker/daemon.json" = {
+        content     = <<EOF
+        {
+          "live-restore": true,
+          "storage-driver": "overlay2",
+          "log-opts": {
+            "max-size": "1024m"
+          }
+        }
+        EOF
+        owner       = "root"
+        permissions = "0644"
+      }
+    } : {}
+  )
 
-  files = merge({
-    "/var/run/nva/ipprefix_by_netmask.sh" = {
-      content     = file("${path.module}/files/ipprefix_by_netmask.sh")
-      owner       = "root"
-      permissions = "0744"
-    }
-    "/var/run/nva/policy_based_routing.sh" = {
-      content     = file("${path.module}/files/policy_based_routing.sh")
-      owner       = "root"
-      permissions = "0744"
-    }
-    }, {
-    for path, attrs in var.files : path => {
-      content     = attrs.content,
-      owner       = attrs.owner,
-      permissions = attrs.permissions
-    }
-  })
+  _frr_daemons = [
+    "zebra",
+    "bgpd",
+    "ospfd",
+    "ospf6d",
+    "ripd",
+    "ripngd",
+    "isisd",
+    "pimd",
+    "ldpd",
+    "nhrpd",
+    "eigrpd",
+    "babeld",
+    "sharpd",
+    "staticd",
+    "pbrd",
+    "bfdd",
+    "fabricd"
+  ]
 
-  network_interfaces = [
+  _frr_daemons_enabled = try(
+    {
+      for daemon in local._frr_daemons :
+      "${daemon}_enabled" => contains(var.frr_config.daemons_enabled, daemon) ? "yes" : "no"
+  }, {})
+
+  _network_interfaces = [
     for index, interface in var.network_interfaces : {
-      name   = "eth${index}"
-      number = index
-      routes = interface.routes
+      name                = "eth${index}"
+      number              = index
+      routes              = interface.routes
+      enable_masquerading = interface.enable_masquerading != null ? interface.enable_masquerading : false
+      non_masq_cidrs      = interface.non_masq_cidrs != null ? interface.non_masq_cidrs : []
     }
   ]
 
-  template = (
+  _optional_run_cmds = (
+    try(var.frr_config != null, false)
+    ? concat(["systemctl start frr"], var.optional_run_cmds)
+    : var.optional_run_cmds
+  )
+
+  _template = (
     var.cloud_config == null
     ? "${path.module}/cloud-config.yaml"
     : var.cloud_config
   )
+
+  cloud_config = templatefile(local._template, {
+    enable_health_checks = var.enable_health_checks
+    files                = local._files
+    network_interfaces   = local._network_interfaces
+    optional_run_cmds    = local._optional_run_cmds
+  })
 }

modules/cloud-config-container/simple-nva/outputs.tf

@@ -1,5 +1,5 @@
 /**
- * Copyright 2022 Google LLC
+ * Copyright 2023 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.