From ebb4bf789eb8d5b1d688e088f5e98835387546eb Mon Sep 17 00:00:00 2001 From: bruzzechesse Date: Mon, 27 Feb 2023 17:20:24 +0100 Subject: [PATCH 01/39] add frr configuration for running bgp on the nva add optional run cmds during cloud init bootstrap add masquerading configuration on network interfaces variable skip route configuration when no LB is in front of the NVA --- .../simple-nva/files/frr/daemons | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 modules/cloud-config-container/simple-nva/files/frr/daemons diff --git a/modules/cloud-config-container/simple-nva/files/frr/daemons b/modules/cloud-config-container/simple-nva/files/frr/daemons new file mode 100644 index 0000000000..2a5c0ac9e4 --- /dev/null +++ b/modules/cloud-config-container/simple-nva/files/frr/daemons @@ -0,0 +1,66 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +zebra=no +bgpd=yes +ospfd=no +ospf6d=no +ripd=no +ripngd=no +isisd=no +pimd=no +ldpd=no +nhrpd=no +eigrpd=no +babeld=no +sharpd=no +staticd=no +pbrd=no +bfdd=no +fabricd=no + +# +# If this option is set the /etc/init.d/frr script automatically loads +# the config via "vtysh -b" when the servers are started. +# Check /etc/pam.d/frr if you intend to use "vtysh"! +# +vtysh_enable=yes +zebra_options=" -A 127.0.0.1 -s 90000000" +bgpd_options=" -A 127.0.0.1" +ospfd_options=" --daemon -A 127.0.0.1" +ospf6d_options=" --daemon -A ::1" +ripd_options=" --daemon -A 127.0.0.1" +ripngd_options=" --daemon -A ::1" +isisd_options=" --daemon -A 127.0.0.1" +pimd_options=" --daemon -A 127.0.0.1" +ldpd_options=" --daemon -A 127.0.0.1" +nhrpd_options=" --daemon -A 127.0.0.1" +eigrpd_options=" --daemon -A 127.0.0.1" +babeld_options=" --daemon -A 127.0.0.1" +sharpd_options=" --daemon -A 127.0.0.1" +staticd_options=" --daemon -A 127.0.0.1" +pbrd_options=" --daemon -A 127.0.0.1" +bfdd_options=" --daemon -A 127.0.0.1" +fabricd_options=" --daemon -A 127.0.0.1" + +#MAX_FDS=1024 +# The list of daemons to watch is automatically generated by the init script. +#watchfrr_options="" + +# for debugging purposes, you can specify a "wrap" command to start instead +# of starting the daemon directly, e.g. to use valgrind on ospfd: +# ospfd_wrap="/usr/bin/valgrind" +# or you can use "all_wrap" for all daemons, e.g. to use perf record: +# all_wrap="/usr/bin/perf record --call-graph -" +# the normal daemon command is added to this at the end. \ No newline at end of file From c7fefa8f42dd4cdcd2e1c4d7f0ef0164984cd9c7 Mon Sep 17 00:00:00 2001 From: bruzzechesse Date: Tue, 28 Feb 2023 11:15:06 +0100 Subject: [PATCH 02/39] add ip masquerading config on network interfaces add bgp config for bgp server configuration on the nva via FRR container add optional run cmds to execute during cloud init bootstrap add condition to skip some rouiting configs when no LB is in front of the NVA --- .../simple-nva/cloud-config.yaml | 10 ++- .../simple-nva/files/frr/frr.conf | 23 ++++++ .../simple-nva/files/frr/frr.service | 27 +++++++ .../simple-nva/files/frr/vtysh.conf | 18 +++++ .../simple-nva/files/policy_based_routing.sh | 18 +++-- .../cloud-config-container/simple-nva/main.tf | 79 ++++++++++++++----- .../simple-nva/variables.tf | 19 +++++ 7 files changed, 166 insertions(+), 28 deletions(-) create mode 100644 modules/cloud-config-container/simple-nva/files/frr/frr.conf create mode 100644 modules/cloud-config-container/simple-nva/files/frr/frr.service create mode 100644 modules/cloud-config-container/simple-nva/files/frr/vtysh.conf diff --git a/modules/cloud-config-container/simple-nva/cloud-config.yaml b/modules/cloud-config-container/simple-nva/cloud-config.yaml index f1d71e8262..492f4965d2 100644 --- a/modules/cloud-config-container/simple-nva/cloud-config.yaml +++ b/modules/cloud-config-container/simple-nva/cloud-config.yaml @@ -43,6 +43,12 @@ write_files: %{ if enable_health_checks ~} /var/run/nva/policy_based_routing.sh ${interface.name} %{ endif ~} +%{ if interface.enable_masquerading ~} +%{ for cidr in interface.non_masq_cidrs ~} + iptables -t nat -A POSTROUTING -o ${interface.name} -d ${cidr} -j ACCEPT +%{ endfor ~} + iptables -t nat -A POSTROUTING -o ${interface.name} -j MASQUERADE +%{ endif ~} %{ for route in interface.routes ~} ip route add ${route} via `curl http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/${interface.number}/gateway -H "Metadata-Flavor:Google"` dev ${interface.name} %{ endfor ~} @@ -55,4 +61,6 @@ runcmd: - systemctl daemon-reload - systemctl enable routing - systemctl start routing - +%{ for cmd in optional_run_cmds ~} + - ${cmd} +%{ endfor ~} diff --git a/modules/cloud-config-container/simple-nva/files/frr/frr.conf b/modules/cloud-config-container/simple-nva/files/frr/frr.conf new file mode 100644 index 0000000000..bf997a3ca8 --- /dev/null +++ b/modules/cloud-config-container/simple-nva/files/frr/frr.conf @@ -0,0 +1,23 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default FRR configuration which enables a BGP protocol process with a default ASN of 65500. +# For more information on how to update this file and/or configure bgp sessions please refer to +# the official documentation available at the following link: +# https://docs.frrouting.org/en/latest/overview.html + +log syslog informational +no ipv6 forwarding +router bgp 65500 +line vty \ No newline at end of file diff --git a/modules/cloud-config-container/simple-nva/files/frr/frr.service b/modules/cloud-config-container/simple-nva/files/frr/frr.service new file mode 100644 index 0000000000..0a64c2ad8c --- /dev/null +++ b/modules/cloud-config-container/simple-nva/files/frr/frr.service @@ -0,0 +1,27 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[Unit] +Description=Start FRR container +After=gcr-online.target docker.socket +Wants=gcr-online.target docker.socket docker-events-collector.service +[Service] +Environment="HOME=/home/frr" +ExecStart=/usr/bin/docker run --rm --name=frr \ +--privileged \ +--network host \ +-v /etc/frr:/etc/frr \ +frrouting/frr +ExecStop=/usr/bin/docker stop frr +ExecStopPost=/usr/bin/docker rm frr \ No newline at end of file diff --git a/modules/cloud-config-container/simple-nva/files/frr/vtysh.conf b/modules/cloud-config-container/simple-nva/files/frr/vtysh.conf new file mode 100644 index 0000000000..2220b1ebfd --- /dev/null +++ b/modules/cloud-config-container/simple-nva/files/frr/vtysh.conf @@ -0,0 +1,18 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Configure vtysh to read configuration from frr.conf file and send the appropiate portions to only the daemons interested +# Official documentation available at the following link: https://docs.frrouting.org/en/latest/basic.html#config-methods + +service integrated-vtysh-config \ No newline at end of file diff --git a/modules/cloud-config-container/simple-nva/files/policy_based_routing.sh b/modules/cloud-config-container/simple-nva/files/policy_based_routing.sh index 951396d356..f1edbaeb71 100644 --- a/modules/cloud-config-container/simple-nva/files/policy_based_routing.sh +++ b/modules/cloud-config-container/simple-nva/files/policy_based_routing.sh @@ -17,11 +17,14 @@ IF_NAME=$1 IP_LB=$(ip r show table local | grep "$IF_NAME proto 66" | cut -f 2 -d " ") -# Sleep while there's no load balancer IP route for this IF -while [ -z $IP_LB ] ; do - sleep 2 - IP_LB=$(ip r show table local | grep "$IF_NAME proto 66" | cut -f 2 -d " ") -done +# Skip this step if no LB is configured for this NVA +if [[ ! -z "$IP_LB" ]]; then + # Sleep while there's no load balancer IP route for this IF + while [ -z $IP_LB ] ; do + sleep 2 + IP_LB=$(ip r show table local | grep "$IF_NAME proto 66" | cut -f 2 -d " ") + done +fi IF_NUMBER=$(echo $IF_NAME | sed -e s/eth//) IF_GW=$(curl http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/$IF_NUMBER/gateway -H "Metadata-Flavor: Google") @@ -31,4 +34,7 @@ IF_IP_PREFIX=$(/var/run/nva/ipprefix_by_netmask.sh $IF_NETMASK) grep -qxF "$((200 + $IF_NUMBER)) hc-$IF_NAME" /etc/iproute2/rt_tables || echo "$((200 + $IF_NUMBER)) hc-$IF_NAME" >>/etc/iproute2/rt_tables ip route add $IF_GW src $IF_IP dev $IF_NAME table hc-$IF_NAME ip route add default via $IF_GW dev $IF_NAME table hc-$IF_NAME -ip rule add from $IP_LB/32 table hc-$IF_NAME +# Skip adding this route if no LB is configured for this NVA +if [[ ! -z "$IP_LB" ]]; then + ip rule add from $IP_LB/32 table hc-$IF_NAME +fi \ No newline at end of file diff --git a/modules/cloud-config-container/simple-nva/main.tf b/modules/cloud-config-container/simple-nva/main.tf index 4ff0afe29b..fc927e7243 100644 --- a/modules/cloud-config-container/simple-nva/main.tf +++ b/modules/cloud-config-container/simple-nva/main.tf @@ -19,35 +19,72 @@ locals { files = local.files enable_health_checks = var.enable_health_checks network_interfaces = local.network_interfaces + optional_run_cmds = local.optional_run_cmds })) - files = merge({ - "/var/run/nva/ipprefix_by_netmask.sh" = { - content = file("${path.module}/files/ipprefix_by_netmask.sh") - owner = "root" - permissions = "0744" - } - "/var/run/nva/policy_based_routing.sh" = { - content = file("${path.module}/files/policy_based_routing.sh") - owner = "root" - permissions = "0744" - } - }, { - for path, attrs in var.files : path => { - content = attrs.content, - owner = attrs.owner, - permissions = attrs.permissions - } - }) + frr_config = ( + try(var.bgp_config.frr_config != null,false) ? var.bgp_config.frr_config : "${path.module}/files/frr/frr.conf" + ) + daemons = ( + try(var.bgp_config.daemons != null,false) ? var.bgp_config.daemons : "${path.module}/files/frr/daemons" + ) + files = merge( + { + "/var/run/nva/ipprefix_by_netmask.sh" = { + content = file("${path.module}/files/ipprefix_by_netmask.sh") + owner = "root" + permissions = "0744" + } + "/var/run/nva/policy_based_routing.sh" = { + content = file("${path.module}/files/policy_based_routing.sh") + owner = "root" + permissions = "0744" + } + }, { + for path, attrs in var.files : path => { + content = attrs.content, + owner = attrs.owner, + permissions = attrs.permissions + } + }, + try(var.bgp_config.enable, false) ? { + "/etc/frr/daemons" = { + content = file(local.daemons) + owner = "root" + permissions = "0744" + } + "/etc/frr/frr.conf" = { + content = file(local.frr_config) + owner = "root" + permissions = "0744" + } + "/etc/frr/vtysh.conf" = { + content = file("${path.module}/files/frr/vtysh.conf") + owner = "root" + permissions = "0744" + } + "/etc/systemd/system/frr.service" = { + content = file("${path.module}/files/frr/frr.service") + owner = "root" + permissions = "0644" + } + } : {} + ) network_interfaces = [ for index, interface in var.network_interfaces : { - name = "eth${index}" - number = index - routes = interface.routes + name = "eth${index}" + number = index + routes = interface.routes + enable_masquerading = interface.enable_masquerading + non_masq_cidrs = interface.non_masq_cidrs } ] + optional_run_cmds = try(var.bgp_config.enable, false) ? concat( + ["systemctl start frr"],var.optional_run_cmds + ) : var.optional_run_cmds + template = ( var.cloud_config == null ? "${path.module}/cloud-config.yaml" diff --git a/modules/cloud-config-container/simple-nva/variables.tf b/modules/cloud-config-container/simple-nva/variables.tf index 39d96d913f..427f429a43 100644 --- a/modules/cloud-config-container/simple-nva/variables.tf +++ b/modules/cloud-config-container/simple-nva/variables.tf @@ -14,6 +14,18 @@ * limitations under the License. */ + variable "bgp_config" { + description = "BGP configuration for FR Routing container running on the NVA" + type = object({ + daemons = string + enable = bool + frr_config = string + }) + default = { + enable = false + } +} + variable "cloud_config" { description = "Cloud config template path. If null default will be used." type = string @@ -40,5 +52,12 @@ variable "network_interfaces" { description = "Network interfaces configuration." type = list(object({ routes = optional(list(string)) + enable_masquerading = optional(bool) + non_masq_cidrs = optional(list(string)) })) } + +variable "optional_run_cmds" { + description = "Optional Cloud Init run commands to execute" + type = list(string) +} From 12fd6de8ac03a2e35745a11cc37a264ebc557f09 Mon Sep 17 00:00:00 2001 From: bruzzechesse Date: Tue, 28 Feb 2023 15:19:10 +0100 Subject: [PATCH 03/39] small fixes --- .../simple-nva/files/frr/vtysh.conf | 18 --------------- .../cloud-config-container/simple-nva/main.tf | 22 +++++++++++++------ .../simple-nva/variables.tf | 7 +++--- 3 files changed, 19 insertions(+), 28 deletions(-) delete mode 100644 modules/cloud-config-container/simple-nva/files/frr/vtysh.conf diff --git a/modules/cloud-config-container/simple-nva/files/frr/vtysh.conf b/modules/cloud-config-container/simple-nva/files/frr/vtysh.conf deleted file mode 100644 index 2220b1ebfd..0000000000 --- a/modules/cloud-config-container/simple-nva/files/frr/vtysh.conf +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Configure vtysh to read configuration from frr.conf file and send the appropiate portions to only the daemons interested -# Official documentation available at the following link: https://docs.frrouting.org/en/latest/basic.html#config-methods - -service integrated-vtysh-config \ No newline at end of file diff --git a/modules/cloud-config-container/simple-nva/main.tf b/modules/cloud-config-container/simple-nva/main.tf index fc927e7243..5741a854c7 100644 --- a/modules/cloud-config-container/simple-nva/main.tf +++ b/modules/cloud-config-container/simple-nva/main.tf @@ -58,16 +58,24 @@ locals { owner = "root" permissions = "0744" } - "/etc/frr/vtysh.conf" = { - content = file("${path.module}/files/frr/vtysh.conf") - owner = "root" - permissions = "0744" - } "/etc/systemd/system/frr.service" = { content = file("${path.module}/files/frr/frr.service") owner = "root" permissions = "0644" } + "/var/lib/docker/daemon.json" = { + content = < Date: Tue, 28 Feb 2023 15:24:57 +0100 Subject: [PATCH 04/39] fmt terraform code update documentation for simple-nva module --- modules/cloud-config-container/simple-nva/README.md | 10 ++++++---- modules/cloud-config-container/simple-nva/main.tf | 6 +++--- .../cloud-config-container/simple-nva/variables.tf | 12 ++++++------ 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/modules/cloud-config-container/simple-nva/README.md b/modules/cloud-config-container/simple-nva/README.md index f70842e8c4..31d3b995e2 100644 --- a/modules/cloud-config-container/simple-nva/README.md +++ b/modules/cloud-config-container/simple-nva/README.md @@ -68,10 +68,12 @@ module "vm" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [network_interfaces](variables.tf#L39) | Network interfaces configuration. | list(object({…})) | ✓ | | -| [cloud_config](variables.tf#L17) | Cloud config template path. If null default will be used. | string | | null | -| [enable_health_checks](variables.tf#L23) | Configures routing to enable responses to health check probes. | bool | | false | -| [files](variables.tf#L29) | Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null. | map(object({…})) | | {} | +| [network_interfaces](variables.tf#L51) | Network interfaces configuration. | list(object({…})) | ✓ | | +| [bgp_config](variables.tf#L17) | BGP configuration for FR Routing container running on the NVA | object({…}) | | {…} | +| [cloud_config](variables.tf#L29) | Cloud config template path. If null default will be used. | string | | null | +| [enable_health_checks](variables.tf#L35) | Configures routing to enable responses to health check probes. | bool | | false | +| [files](variables.tf#L41) | Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null. | map(object({…})) | | {} | +| [optional_run_cmds](variables.tf#L60) | Optional Cloud Init run commands to execute | list(string) | | [] | ## Outputs diff --git a/modules/cloud-config-container/simple-nva/main.tf b/modules/cloud-config-container/simple-nva/main.tf index 5741a854c7..c2798e2bb3 100644 --- a/modules/cloud-config-container/simple-nva/main.tf +++ b/modules/cloud-config-container/simple-nva/main.tf @@ -23,10 +23,10 @@ locals { })) frr_config = ( - try(var.bgp_config.frr_config != null,false) ? var.bgp_config.frr_config : "${path.module}/files/frr/frr.conf" + try(var.bgp_config.frr_config != null, false) ? var.bgp_config.frr_config : "${path.module}/files/frr/frr.conf" ) daemons = ( - try(var.bgp_config.daemons != null,false) ? var.bgp_config.daemons : "${path.module}/files/frr/daemons" + try(var.bgp_config.daemons != null, false) ? var.bgp_config.daemons : "${path.module}/files/frr/daemons" ) files = merge( { @@ -90,7 +90,7 @@ locals { ] optional_run_cmds = try(var.bgp_config.enable, false) ? concat( - ["systemctl start frr"],var.optional_run_cmds + ["systemctl start frr"], var.optional_run_cmds ) : var.optional_run_cmds template = ( diff --git a/modules/cloud-config-container/simple-nva/variables.tf b/modules/cloud-config-container/simple-nva/variables.tf index 58b7780e71..638b851e33 100644 --- a/modules/cloud-config-container/simple-nva/variables.tf +++ b/modules/cloud-config-container/simple-nva/variables.tf @@ -14,14 +14,14 @@ * limitations under the License. */ - variable "bgp_config" { +variable "bgp_config" { description = "BGP configuration for FR Routing container running on the NVA" - type = object({ + type = object({ daemons = optional(string) enable = optional(bool) frr_config = optional(string) }) - default = { + default = { enable = false } } @@ -51,7 +51,7 @@ variable "files" { variable "network_interfaces" { description = "Network interfaces configuration." type = list(object({ - routes = optional(list(string)) + routes = optional(list(string)) enable_masquerading = optional(bool) non_masq_cidrs = optional(list(string)) })) @@ -59,6 +59,6 @@ variable "network_interfaces" { variable "optional_run_cmds" { description = "Optional Cloud Init run commands to execute" - type = list(string) - default = [] + type = list(string) + default = [] } From 2fe6572bfb4a3583091e4a45d1e69e1a99357132 Mon Sep 17 00:00:00 2001 From: bruzzechesse Date: Tue, 28 Feb 2023 15:41:40 +0100 Subject: [PATCH 05/39] fix variable description --- modules/cloud-config-container/simple-nva/README.md | 4 ++-- modules/cloud-config-container/simple-nva/variables.tf | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/cloud-config-container/simple-nva/README.md b/modules/cloud-config-container/simple-nva/README.md index 31d3b995e2..efafffdeaa 100644 --- a/modules/cloud-config-container/simple-nva/README.md +++ b/modules/cloud-config-container/simple-nva/README.md @@ -69,11 +69,11 @@ module "vm" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| | [network_interfaces](variables.tf#L51) | Network interfaces configuration. | list(object({…})) | ✓ | | -| [bgp_config](variables.tf#L17) | BGP configuration for FR Routing container running on the NVA | object({…}) | | {…} | +| [bgp_config](variables.tf#L17) | BGP configuration for FR Routing container running on the NVA. | object({…}) | | {…} | | [cloud_config](variables.tf#L29) | Cloud config template path. If null default will be used. | string | | null | | [enable_health_checks](variables.tf#L35) | Configures routing to enable responses to health check probes. | bool | | false | | [files](variables.tf#L41) | Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null. | map(object({…})) | | {} | -| [optional_run_cmds](variables.tf#L60) | Optional Cloud Init run commands to execute | list(string) | | [] | +| [optional_run_cmds](variables.tf#L60) | Optional Cloud Init run commands to execute. | list(string) | | [] | ## Outputs diff --git a/modules/cloud-config-container/simple-nva/variables.tf b/modules/cloud-config-container/simple-nva/variables.tf index 638b851e33..fba9a12763 100644 --- a/modules/cloud-config-container/simple-nva/variables.tf +++ b/modules/cloud-config-container/simple-nva/variables.tf @@ -15,7 +15,7 @@ */ variable "bgp_config" { - description = "BGP configuration for FR Routing container running on the NVA" + description = "BGP configuration for FR Routing container running on the NVA." type = object({ daemons = optional(string) enable = optional(bool) @@ -58,7 +58,7 @@ variable "network_interfaces" { } variable "optional_run_cmds" { - description = "Optional Cloud Init run commands to execute" + description = "Optional Cloud Init run commands to execute." type = list(string) default = [] } From 3aafecff7df532a0fc90ff0819470b1e3adddfc1 Mon Sep 17 00:00:00 2001 From: bruzzechesse Date: Tue, 28 Feb 2023 16:18:23 +0100 Subject: [PATCH 06/39] remove redundant check on LB ip address --- .../simple-nva/files/policy_based_routing.sh | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/modules/cloud-config-container/simple-nva/files/policy_based_routing.sh b/modules/cloud-config-container/simple-nva/files/policy_based_routing.sh index f1edbaeb71..3746d4b20f 100644 --- a/modules/cloud-config-container/simple-nva/files/policy_based_routing.sh +++ b/modules/cloud-config-container/simple-nva/files/policy_based_routing.sh @@ -17,14 +17,11 @@ IF_NAME=$1 IP_LB=$(ip r show table local | grep "$IF_NAME proto 66" | cut -f 2 -d " ") -# Skip this step if no LB is configured for this NVA -if [[ ! -z "$IP_LB" ]]; then - # Sleep while there's no load balancer IP route for this IF - while [ -z $IP_LB ] ; do - sleep 2 - IP_LB=$(ip r show table local | grep "$IF_NAME proto 66" | cut -f 2 -d " ") - done -fi +# Sleep while there's no load balancer IP route for this IF +while [ -z $IP_LB ] ; do + sleep 2 + IP_LB=$(ip r show table local | grep "$IF_NAME proto 66" | cut -f 2 -d " ") +done IF_NUMBER=$(echo $IF_NAME | sed -e s/eth//) IF_GW=$(curl http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/$IF_NUMBER/gateway -H "Metadata-Flavor: Google") @@ -34,7 +31,4 @@ IF_IP_PREFIX=$(/var/run/nva/ipprefix_by_netmask.sh $IF_NETMASK) grep -qxF "$((200 + $IF_NUMBER)) hc-$IF_NAME" /etc/iproute2/rt_tables || echo "$((200 + $IF_NUMBER)) hc-$IF_NAME" >>/etc/iproute2/rt_tables ip route add $IF_GW src $IF_IP dev $IF_NAME table hc-$IF_NAME ip route add default via $IF_GW dev $IF_NAME table hc-$IF_NAME -# Skip adding this route if no LB is configured for this NVA -if [[ ! -z "$IP_LB" ]]; then - ip rule add from $IP_LB/32 table hc-$IF_NAME -fi \ No newline at end of file +ip rule add from $IP_LB/32 table hc-$IF_NAME \ No newline at end of file From 4cdc5c108cfadf4db2dd2b2052f23319e1c73a53 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Wed, 1 Mar 2023 08:08:07 +0100 Subject: [PATCH 07/39] Fix secondary ranges in net-vpc readme (#1198) Fixes #1197 --- modules/net-vpc/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/net-vpc/README.md b/modules/net-vpc/README.md index bd5675d239..25dfaa58fd 100644 --- a/modules/net-vpc/README.md +++ b/modules/net-vpc/README.md @@ -174,7 +174,7 @@ module "vpc-host" { ip_cidr_range = "10.0.0.0/24" name = "subnet-1" region = "europe-west1" - secondary_ip_range = { + secondary_ip_ranges = { pods = "172.16.0.0/20" services = "192.168.0.0/24" } From c11b07e6b2cf5e9bd5652d203d6ccdb50e50b060 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Wed, 1 Mar 2023 09:58:50 +0100 Subject: [PATCH 08/39] Add test for #1197 --- tests/modules/net_vpc/examples/shared-vpc.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/modules/net_vpc/examples/shared-vpc.yaml b/tests/modules/net_vpc/examples/shared-vpc.yaml index b004e31511..6467fd389e 100644 --- a/tests/modules/net_vpc/examples/shared-vpc.yaml +++ b/tests/modules/net_vpc/examples/shared-vpc.yaml @@ -24,7 +24,12 @@ values: module.vpc-host.google_compute_shared_vpc_service_project.service_projects["project2"]: host_project: my-project service_project: project2 - module.vpc-host.google_compute_subnetwork.subnetwork["europe-west1/subnet-1"]: {} + module.vpc-host.google_compute_subnetwork.subnetwork["europe-west1/subnet-1"]: + secondary_ip_range: + - ip_cidr_range: 172.16.0.0/20 + range_name: pods + - ip_cidr_range: 192.168.0.0/24 + range_name: services module.vpc-host.google_compute_subnetwork_iam_binding.binding["europe-west1/subnet-1.roles/compute.networkUser"]: condition: [] members: From b842bd70203d9876a990a56a9003d431e21fba3e Mon Sep 17 00:00:00 2001 From: Aleksandr Averbukh Date: Wed, 1 Mar 2023 10:33:08 +0100 Subject: [PATCH 09/39] Add missing tfvars template to the tfc blueprint --- .../terraform.auto.tfvars.template | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 blueprints/cloud-operations/terraform-cloud-dynamic-credentials/gcp-workload-identity-provider/terraform.auto.tfvars.template diff --git a/blueprints/cloud-operations/terraform-cloud-dynamic-credentials/gcp-workload-identity-provider/terraform.auto.tfvars.template b/blueprints/cloud-operations/terraform-cloud-dynamic-credentials/gcp-workload-identity-provider/terraform.auto.tfvars.template new file mode 100644 index 0000000000..2d2167a3ec --- /dev/null +++ b/blueprints/cloud-operations/terraform-cloud-dynamic-credentials/gcp-workload-identity-provider/terraform.auto.tfvars.template @@ -0,0 +1,18 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +billing_account = "015647-1A8CBC-A002D9" +parent = "folders/0123456789" +tfc_organization_id = "org-W3bx9naazHrUz99U" +tfc_workspace_id = "ws-2GnGJtqTfvxKptHn" From cbb40a2c7e907a3cd40d4c757aff62b21c100b54 Mon Sep 17 00:00:00 2001 From: Aleksandr Averbukh Date: Wed, 1 Mar 2023 10:42:39 +0100 Subject: [PATCH 10/39] Add more explicit template --- .../terraform.auto.tfvars.template | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/blueprints/cloud-operations/terraform-cloud-dynamic-credentials/gcp-workload-identity-provider/terraform.auto.tfvars.template b/blueprints/cloud-operations/terraform-cloud-dynamic-credentials/gcp-workload-identity-provider/terraform.auto.tfvars.template index 2d2167a3ec..918d837547 100644 --- a/blueprints/cloud-operations/terraform-cloud-dynamic-credentials/gcp-workload-identity-provider/terraform.auto.tfvars.template +++ b/blueprints/cloud-operations/terraform-cloud-dynamic-credentials/gcp-workload-identity-provider/terraform.auto.tfvars.template @@ -16,3 +16,13 @@ billing_account = "015647-1A8CBC-A002D9" parent = "folders/0123456789" tfc_organization_id = "org-W3bx9naazHrUz99U" tfc_workspace_id = "ws-2GnGJtqTfvxKptHn" + +billing_account = "xxx" +project_create = false +project_id = "xxx" +parent = "organizations/xxx" +tfc_organization_id = "org-xxxxxxxxxxxxx" +tfc_workspace_id = "ws-xxxxxxxxxxxxx" +workload_identity_pool_id = "tfc-pool" +workload_identity_pool_provider_id = "tfc-provider" +issuer_uri = "https://app.terraform.io/" \ No newline at end of file From 2efdef52e7083ba405363798e5b92e05b5f6c1a6 Mon Sep 17 00:00:00 2001 From: Aleksandr Averbukh Date: Wed, 1 Mar 2023 10:43:32 +0100 Subject: [PATCH 11/39] Missing newline --- .../terraform.auto.tfvars.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blueprints/cloud-operations/terraform-cloud-dynamic-credentials/gcp-workload-identity-provider/terraform.auto.tfvars.template b/blueprints/cloud-operations/terraform-cloud-dynamic-credentials/gcp-workload-identity-provider/terraform.auto.tfvars.template index 918d837547..327a51d651 100644 --- a/blueprints/cloud-operations/terraform-cloud-dynamic-credentials/gcp-workload-identity-provider/terraform.auto.tfvars.template +++ b/blueprints/cloud-operations/terraform-cloud-dynamic-credentials/gcp-workload-identity-provider/terraform.auto.tfvars.template @@ -25,4 +25,4 @@ tfc_organization_id = "org-xxxxxxxxxxxxx" tfc_workspace_id = "ws-xxxxxxxxxxxxx" workload_identity_pool_id = "tfc-pool" workload_identity_pool_provider_id = "tfc-provider" -issuer_uri = "https://app.terraform.io/" \ No newline at end of file +issuer_uri = "https://app.terraform.io/" From 88af3bf08310a9cf6426d7273e9b66579df1dbdb Mon Sep 17 00:00:00 2001 From: Aleksandr Averbukh Date: Wed, 1 Mar 2023 11:34:37 +0100 Subject: [PATCH 12/39] Fix tfvars template --- .../terraform.auto.tfvars.template | 5 ----- 1 file changed, 5 deletions(-) diff --git a/blueprints/cloud-operations/terraform-cloud-dynamic-credentials/gcp-workload-identity-provider/terraform.auto.tfvars.template b/blueprints/cloud-operations/terraform-cloud-dynamic-credentials/gcp-workload-identity-provider/terraform.auto.tfvars.template index 327a51d651..57787bf930 100644 --- a/blueprints/cloud-operations/terraform-cloud-dynamic-credentials/gcp-workload-identity-provider/terraform.auto.tfvars.template +++ b/blueprints/cloud-operations/terraform-cloud-dynamic-credentials/gcp-workload-identity-provider/terraform.auto.tfvars.template @@ -12,11 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -billing_account = "015647-1A8CBC-A002D9" -parent = "folders/0123456789" -tfc_organization_id = "org-W3bx9naazHrUz99U" -tfc_workspace_id = "ws-2GnGJtqTfvxKptHn" - billing_account = "xxx" project_create = false project_id = "xxx" From 0f6e94afd345dff56068398b4f25ed9d30348140 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Wed, 1 Mar 2023 07:54:10 +0100 Subject: [PATCH 13/39] Fix Variables --- modules/dataproc/README.md | 29 +++++++++++++++++++++++++++++ modules/dataproc/main.tf | 26 +++++++++++++------------- modules/dataproc/variables.tf | 6 +++--- 3 files changed, 45 insertions(+), 16 deletions(-) diff --git a/modules/dataproc/README.md b/modules/dataproc/README.md index 80835dd1a7..982d043c51 100644 --- a/modules/dataproc/README.md +++ b/modules/dataproc/README.md @@ -46,6 +46,35 @@ module "processing-dp-cluster" { # tftest modules=1 resources=1 ``` +### Cluster with CMEK encrypotion + +To set cluster configuration use the Customer Managed Encryption key, set '' variable. The Compute Engine service agent and the Cloud Storage service agent needs to have 'CryptoKey Encrypter/Decrypter' role on they configured KMS key ([Documentation](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/customer-managed-encryption)). + +```hcl +module "processing-dp-cluster" { + source = "./fabric/modules/dataproc" + project_id = "my-project" + name = "my-cluster" + region = "europe-west1" + prefix = "prefix" + dataproc_config = { + cluster_config = { + gce_cluster_config = { + subnetwork = "https://www.googleapis.com/compute/v1/projects/PROJECT/regions/europe-west1/subnetworks/SUBNET" + zone = "europe-west1-b" + service_account = "" + service_account_scopes = ["cloud-platform"] + internal_ip_only = true + } + } + } + encryption_config = try({ + kms_key_name = "projects/project-id/locations/region/keyRings/key-ring-name/cryptoKeys/key-name" + }, null) +} +# tftest modules=1 resources=1 +``` + ## IAM Examples IAM is managed via several variables that implement different levels of control: diff --git a/modules/dataproc/main.tf b/modules/dataproc/main.tf index ab09cbea0c..55bef5c70e 100644 --- a/modules/dataproc/main.tf +++ b/modules/dataproc/main.tf @@ -59,9 +59,9 @@ resource "google_dataproc_cluster" "cluster" { dynamic "shielded_instance_config" { for_each = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config == null ? [] : [""] content { - enable_secure_boot = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config.value.enable_secure_boot - enable_vtpm = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config.value.enable_vtpm - enable_integrity_monitoring = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config.value.enable_integrity_monitoring + enable_secure_boot = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config.enable_secure_boot + enable_vtpm = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config.enable_vtpm + enable_integrity_monitoring = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config.enable_integrity_monitoring } } } @@ -99,9 +99,9 @@ resource "google_dataproc_cluster" "cluster" { dynamic "disk_config" { for_each = var.dataproc_config.cluster_config.worker_config.disk_config == null ? [] : [""] content { - boot_disk_type = var.dataproc_config.cluster_config.worker_config.disk_config.value.boot_disk_type - boot_disk_size_gb = var.dataproc_config.cluster_config.worker_config.disk_config.value.boot_disk_size_gb - num_local_ssds = var.dataproc_config.cluster_config.worker_config.disk_config.value.num_local_ssds + boot_disk_type = var.dataproc_config.cluster_config.worker_config.disk_config.boot_disk_type + boot_disk_size_gb = var.dataproc_config.cluster_config.worker_config.disk_config.boot_disk_size_gb + num_local_ssds = var.dataproc_config.cluster_config.worker_config.disk_config.num_local_ssds } } image_uri = var.dataproc_config.cluster_config.worker_config.image_uri @@ -165,20 +165,20 @@ resource "google_dataproc_cluster" "cluster" { dynamic "autoscaling_config" { for_each = var.dataproc_config.cluster_config.autoscaling_config == null ? [] : [""] content { - policy_uri = var.dataproc_config.cluster_config.autoscaling_config.value.policy_uri + policy_uri = var.dataproc_config.cluster_config.autoscaling_config.policy_uri } } dynamic "initialization_action" { for_each = var.dataproc_config.cluster_config.initialization_action == null ? [] : [""] content { - script = var.dataproc_config.cluster_config.initialization_action.value.script - timeout_sec = var.dataproc_config.cluster_config.initialization_action.value.timeout_sec + script = var.dataproc_config.cluster_config.initialization_action.script + timeout_sec = var.dataproc_config.cluster_config.initialization_action.timeout_sec } } dynamic "encryption_config" { - for_each = var.dataproc_config.cluster_config.encryption_config == null ? [] : [""] + for_each = try(var.dataproc_config.cluster_config.encryption_config.kms_key_name == null ? [] : [""], []) content { - kms_key_name = var.dataproc_config.cluster_config.encryption_config.value.kms_key_name + kms_key_name = var.dataproc_config.cluster_config.encryption_config.kms_key_name } } dynamic "dataproc_metric_config" { @@ -243,8 +243,8 @@ resource "google_dataproc_cluster" "cluster" { dynamic "kubernetes_software_config" { for_each = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.kubernetes_software_config == null ? [] : [""] content { - component_version = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.kubernetes_software_config.value.component_version - properties = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.kubernetes_software_config.value.properties + component_version = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.kubernetes_software_config.component_version + properties = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.kubernetes_software_config.properties } } diff --git a/modules/dataproc/variables.tf b/modules/dataproc/variables.tf index 3636a70637..314d243125 100644 --- a/modules/dataproc/variables.tf +++ b/modules/dataproc/variables.tf @@ -84,9 +84,9 @@ variable "dataproc_config" { }), null) }), null) software_config = optional(object({ - image_version = string - override_properties = list(map(string)) - optional_components = list(string) + image_version = optional(string, null) + override_properties = map(string) + optional_components = optional(list(string), null) }), null) security_config = optional(object({ kerberos_config = object({ From d53ea840cbde6e7427efee4e0c75d74169a3b227 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Wed, 1 Mar 2023 08:00:46 +0100 Subject: [PATCH 14/39] Fix linting --- modules/dataproc/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/dataproc/README.md b/modules/dataproc/README.md index 982d043c51..0a0c4a4bb3 100644 --- a/modules/dataproc/README.md +++ b/modules/dataproc/README.md @@ -148,7 +148,7 @@ module "processing-dp-cluster" { | [name](variables.tf#L211) | Cluster name. | string | ✓ | | | [project_id](variables.tf#L226) | Project ID. | string | ✓ | | | [region](variables.tf#L231) | Dataproc region. | string | ✓ | | -| [dataproc_config](variables.tf#L17) | Dataproc cluster config. | object({…}) | | {} | +| [dataproc_config](variables.tf#L17) | Dataproc cluster config. | object({…}) | | {} | | [group_iam](variables.tf#L184) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | | [iam](variables.tf#L191) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_additive](variables.tf#L198) | IAM additive bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | From 7fd3ebc21f2b77f2f1cf5b0ab75e6eab4939bf1b Mon Sep 17 00:00:00 2001 From: lcaggio Date: Wed, 1 Mar 2023 10:43:33 +0100 Subject: [PATCH 15/39] Update README. --- .../demo/orchestrate_pyspark.py | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 blueprints/data-solutions/data-platform-spark/demo/orchestrate_pyspark.py diff --git a/blueprints/data-solutions/data-platform-spark/demo/orchestrate_pyspark.py b/blueprints/data-solutions/data-platform-spark/demo/orchestrate_pyspark.py new file mode 100644 index 0000000000..a8e0f2d3ce --- /dev/null +++ b/blueprints/data-solutions/data-platform-spark/demo/orchestrate_pyspark.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import datetime +import os + +from airflow import models +from airflow.providers.google.cloud.operators.dataproc import ( + DataprocCreateBatchOperator, DataprocDeleteBatchOperator, DataprocGetBatchOperator, DataprocListBatchesOperator + +) +from airflow.utils.dates import days_ago + +# -------------------------------------------------------------------------------- +# Get variables +# -------------------------------------------------------------------------------- +BQ_LOCATION = os.environ.get("BQ_LOCATION") +CURATED_BQ_DATASET = os.environ.get("CURATED_BQ_DATASET") +CURATED_GCS = os.environ.get("CURATED_GCS") +CURATED_PRJ = os.environ.get("CURATED_PRJ") +DP_KMS_KEY = os.environ.get("DP_KMS_KEY", "") +DP_REGION = os.environ.get("DP_REGION") +GCP_REGION = os.environ.get("GCP_REGION") +LAND_PRJ = os.environ.get("LAND_PRJ") +LAND_BQ_DATASET = os.environ.get("LAND_BQ_DATASET") +LAND_GCS = os.environ.get("LAND_GCS") +PHS_NAME = os.environ.get("PHS_NAME") +PROCESSING_GCS = os.environ.get("PROCESSING_GCS") +PROCESSING_PRJ = os.environ.get("PROCESSING_PRJ") +PROCESSING_SA_DP = os.environ.get("PROCESSING_SA_DP") +PROCESSING_SA_SUBNET = os.environ.get("PROCESSING_SUBNET") +PROCESSING_SA_VPC = os.environ.get("PROCESSING_VPC") + +PYTHON_FILE_LOCATION = "gs://"+PROCESSING_GCS+"/pyspark_sort.py" +PHS_CLUSTER_PATH = "projects/"+PROCESSING_PRJ+"/regions/"+DP_REGION+"/clusters/"+PHS_NAME + +default_args = { + # Tell airflow to start one day ago, so that it runs as soon as you upload it + "start_date": days_ago(1), + "region": DP_REGION, +} +with models.DAG( + "dataproc_batch_operators", # The id you will see in the DAG airflow page + default_args=default_args, # The interval with which to schedule the DAG + schedule_interval=None, # Override to match your needs +) as dag: + + create_batch = DataprocCreateBatchOperator( + task_id="batch_create", + project_id=PROCESSING_PRJ, + batch={ + "environment_config": { + "execution_config": { + "service_account": PROCESSING_SA_DP, + "subnetwork_uri": PROCESSING_SA_SUBNET + } + }, + "pyspark_batch": { + "main_python_file_uri": PYTHON_FILE_LOCATION, + }, + "history_server_cluster": PHS_NAME, + }, + batch_id="batch-create-phs", + ) + + list_batches = DataprocListBatchesOperator( + task_id="list-all-batches", + ) + + get_batch = DataprocGetBatchOperator( + task_id="get_batch", + batch_id="batch-create-phs", + ) + + create_batch >> list_batches >> get_batch \ No newline at end of file From 6c12a33efbb07c642fc14aa7646a2a42464a6573 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Wed, 1 Mar 2023 10:44:01 +0100 Subject: [PATCH 16/39] Update README --- modules/dataproc/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/dataproc/README.md b/modules/dataproc/README.md index 0a0c4a4bb3..9ba449470b 100644 --- a/modules/dataproc/README.md +++ b/modules/dataproc/README.md @@ -46,9 +46,9 @@ module "processing-dp-cluster" { # tftest modules=1 resources=1 ``` -### Cluster with CMEK encrypotion +### Cluster with CMEK encryption PIPPO -To set cluster configuration use the Customer Managed Encryption key, set '' variable. The Compute Engine service agent and the Cloud Storage service agent needs to have 'CryptoKey Encrypter/Decrypter' role on they configured KMS key ([Documentation](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/customer-managed-encryption)). +To set cluster configuration use the Customer Managed Encryption key, set `dataproc_config.encryption_config.` variable. The Compute Engine service agent and the Cloud Storage service agent need to have `CryptoKey Encrypter/Decrypter` role on they configured KMS key ([Documentation](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/customer-managed-encryption)). ```hcl module "processing-dp-cluster" { @@ -67,10 +67,10 @@ module "processing-dp-cluster" { internal_ip_only = true } } + encryption_config = { + kms_key_name = "projects/project-id/locations/region/keyRings/key-ring-name/cryptoKeys/key-name" + } } - encryption_config = try({ - kms_key_name = "projects/project-id/locations/region/keyRings/key-ring-name/cryptoKeys/key-name" - }, null) } # tftest modules=1 resources=1 ``` From bffdb29d4a58116804cda68fb44b8003c6b09a31 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Wed, 1 Mar 2023 10:46:33 +0100 Subject: [PATCH 17/39] Remove wrongly submitted file. --- .../demo/orchestrate_pyspark.py | 89 ------------------- 1 file changed, 89 deletions(-) delete mode 100644 blueprints/data-solutions/data-platform-spark/demo/orchestrate_pyspark.py diff --git a/blueprints/data-solutions/data-platform-spark/demo/orchestrate_pyspark.py b/blueprints/data-solutions/data-platform-spark/demo/orchestrate_pyspark.py deleted file mode 100644 index a8e0f2d3ce..0000000000 --- a/blueprints/data-solutions/data-platform-spark/demo/orchestrate_pyspark.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import datetime -import datetime -import os - -from airflow import models -from airflow.providers.google.cloud.operators.dataproc import ( - DataprocCreateBatchOperator, DataprocDeleteBatchOperator, DataprocGetBatchOperator, DataprocListBatchesOperator - -) -from airflow.utils.dates import days_ago - -# -------------------------------------------------------------------------------- -# Get variables -# -------------------------------------------------------------------------------- -BQ_LOCATION = os.environ.get("BQ_LOCATION") -CURATED_BQ_DATASET = os.environ.get("CURATED_BQ_DATASET") -CURATED_GCS = os.environ.get("CURATED_GCS") -CURATED_PRJ = os.environ.get("CURATED_PRJ") -DP_KMS_KEY = os.environ.get("DP_KMS_KEY", "") -DP_REGION = os.environ.get("DP_REGION") -GCP_REGION = os.environ.get("GCP_REGION") -LAND_PRJ = os.environ.get("LAND_PRJ") -LAND_BQ_DATASET = os.environ.get("LAND_BQ_DATASET") -LAND_GCS = os.environ.get("LAND_GCS") -PHS_NAME = os.environ.get("PHS_NAME") -PROCESSING_GCS = os.environ.get("PROCESSING_GCS") -PROCESSING_PRJ = os.environ.get("PROCESSING_PRJ") -PROCESSING_SA_DP = os.environ.get("PROCESSING_SA_DP") -PROCESSING_SA_SUBNET = os.environ.get("PROCESSING_SUBNET") -PROCESSING_SA_VPC = os.environ.get("PROCESSING_VPC") - -PYTHON_FILE_LOCATION = "gs://"+PROCESSING_GCS+"/pyspark_sort.py" -PHS_CLUSTER_PATH = "projects/"+PROCESSING_PRJ+"/regions/"+DP_REGION+"/clusters/"+PHS_NAME - -default_args = { - # Tell airflow to start one day ago, so that it runs as soon as you upload it - "start_date": days_ago(1), - "region": DP_REGION, -} -with models.DAG( - "dataproc_batch_operators", # The id you will see in the DAG airflow page - default_args=default_args, # The interval with which to schedule the DAG - schedule_interval=None, # Override to match your needs -) as dag: - - create_batch = DataprocCreateBatchOperator( - task_id="batch_create", - project_id=PROCESSING_PRJ, - batch={ - "environment_config": { - "execution_config": { - "service_account": PROCESSING_SA_DP, - "subnetwork_uri": PROCESSING_SA_SUBNET - } - }, - "pyspark_batch": { - "main_python_file_uri": PYTHON_FILE_LOCATION, - }, - "history_server_cluster": PHS_NAME, - }, - batch_id="batch-create-phs", - ) - - list_batches = DataprocListBatchesOperator( - task_id="list-all-batches", - ) - - get_batch = DataprocGetBatchOperator( - task_id="get_batch", - batch_id="batch-create-phs", - ) - - create_batch >> list_batches >> get_batch \ No newline at end of file From 06a6a2c56f1d7a7af5fba0be3607dd1ca7d9947a Mon Sep 17 00:00:00 2001 From: lcaggio Date: Wed, 1 Mar 2023 10:48:33 +0100 Subject: [PATCH 18/39] Fix README --- modules/dataproc/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/dataproc/README.md b/modules/dataproc/README.md index 9ba449470b..d071ecdaad 100644 --- a/modules/dataproc/README.md +++ b/modules/dataproc/README.md @@ -46,7 +46,7 @@ module "processing-dp-cluster" { # tftest modules=1 resources=1 ``` -### Cluster with CMEK encryption PIPPO +### Cluster with CMEK encryption To set cluster configuration use the Customer Managed Encryption key, set `dataproc_config.encryption_config.` variable. The Compute Engine service agent and the Cloud Storage service agent need to have `CryptoKey Encrypter/Decrypter` role on they configured KMS key ([Documentation](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/customer-managed-encryption)). From 0ed0a5ab7e934d6f6b601684bf94ec211edd8cb9 Mon Sep 17 00:00:00 2001 From: erabusi <72961506+erabusi@users.noreply.github.com> Date: Thu, 2 Mar 2023 12:21:39 +0530 Subject: [PATCH 19/39] Fix url_redirect issue on net-glb module (#1204) --- modules/net-glb/urlmap.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/net-glb/urlmap.tf b/modules/net-glb/urlmap.tf index a7f01d55b1..ec1acaf694 100644 --- a/modules/net-glb/urlmap.tf +++ b/modules/net-glb/urlmap.tf @@ -921,9 +921,9 @@ resource "google_compute_url_map" "default" { } dynamic "url_redirect" { for_each = ( - route_rules.value.default_url_redirect == null + route_rules.value.url_redirect == null ? [] - : [route_rules.value.default_url_redirect] + : [route_rules.value.url_redirect] ) content { host_redirect = url_redirect.value.host From 4d948b30d2e760f68506f10911b61aae42ac773d Mon Sep 17 00:00:00 2001 From: Luca Prete Date: Thu, 2 Mar 2023 09:53:07 +0100 Subject: [PATCH 20/39] Blueprint: GLB hybrid NEG internal --- blueprints/README.md | 2 +- blueprints/networking/README.md | 38 +++-- .../glb-hybrid-neg-internal/README.md | 100 ++++++++++++ .../data/nva-startup-script.tftpl | 42 +++++ .../glb-hybrid-neg-internal/diagram.png | Bin 0 -> 63567 bytes .../glb-hybrid-neg-internal/diagram.svg | 1 + .../networking/glb-hybrid-neg-internal/glb.tf | 71 +++++++++ .../glb-hybrid-neg-internal/main.tf | 122 +++++++++++++++ .../networking/glb-hybrid-neg-internal/nva.tf | 87 +++++++++++ .../glb-hybrid-neg-internal/outputs.tf | 20 +++ .../glb-hybrid-neg-internal/spoke.tf | 146 ++++++++++++++++++ .../glb-hybrid-neg-internal/variables.tf | 76 +++++++++ 12 files changed, 688 insertions(+), 17 deletions(-) create mode 100644 blueprints/networking/glb-hybrid-neg-internal/README.md create mode 100644 blueprints/networking/glb-hybrid-neg-internal/data/nva-startup-script.tftpl create mode 100644 blueprints/networking/glb-hybrid-neg-internal/diagram.png create mode 100644 blueprints/networking/glb-hybrid-neg-internal/diagram.svg create mode 100644 blueprints/networking/glb-hybrid-neg-internal/glb.tf create mode 100644 blueprints/networking/glb-hybrid-neg-internal/main.tf create mode 100644 blueprints/networking/glb-hybrid-neg-internal/nva.tf create mode 100644 blueprints/networking/glb-hybrid-neg-internal/outputs.tf create mode 100644 blueprints/networking/glb-hybrid-neg-internal/spoke.tf create mode 100644 blueprints/networking/glb-hybrid-neg-internal/variables.tf diff --git a/blueprints/README.md b/blueprints/README.md index b108f86199..e7136d9cec 100644 --- a/blueprints/README.md +++ b/blueprints/README.md @@ -9,7 +9,7 @@ Currently available blueprints: - **data solutions** - [GCE and GCS CMEK via centralized Cloud KMS](./data-solutions/cmek-via-centralized-kms), [Cloud Composer version 2 private instance, supporting Shared VPC and external CMEK key](./data-solutions/composer-2), [Cloud SQL instance with multi-region read replicas](./data-solutions/cloudsql-multiregion), [Data Platform](./data-solutions/data-platform-foundations), [Spinning up a foundation data pipeline on Google Cloud using Cloud Storage, Dataflow and BigQuery](./data-solutions/gcs-to-bq-with-least-privileges), [#SQL Server Always On Groups blueprint](./data-solutions/sqlserver-alwayson), [Data Playground](./data-solutions/data-playground), [MLOps with Vertex AI](./data-solutions/vertex-mlops), [Shielded Folder](./data-solutions/shielded-folder) - **factories** - [The why and the how of Resource Factories](./factories), [Google Cloud Identity Group Factory](./factories/cloud-identity-group-factory), [Google Cloud BQ Factory](./factories/bigquery-factory), [Google Cloud VPC Firewall Factory](./factories/net-vpc-firewall-yaml), [Minimal Project Factory](./factories/project-factory) - **GKE** - [Binary Authorization Pipeline Blueprint](./gke/binauthz), [Storage API](./gke/binauthz/image), [Multi-cluster mesh on GKE (fleet API)](./gke/multi-cluster-mesh-gke-fleet-api), [GKE Multitenant Blueprint](./gke/multitenant-fleet), [Shared VPC with GKE support](./networking/shared-vpc-gke/) -- **networking** - [Decentralized firewall management](./networking/decentralized-firewall), [Decentralized firewall validator](./networking/decentralized-firewall/validator), [Network filtering with Squid](./networking/filtering-proxy), [Network filtering with Squid with isolated VPCs using Private Service Connect](./networking/filtering-proxy-psc), [HTTP Load Balancer with Cloud Armor](./networking/glb-and-armor), [Hub and Spoke via VPN](./networking/hub-and-spoke-vpn), [Hub and Spoke via VPC Peering](./networking/hub-and-spoke-peering), [Internal Load Balancer as Next Hop](./networking/ilb-next-hop), On-prem DNS and Google Private Access, [Calling a private Cloud Function from On-premises](./networking/private-cloud-function-from-onprem), [Hybrid connectivity to on-premise services through PSC](./networking/psc-hybrid), [PSC Producer](./networking/psc-hybrid/psc-producer), [PSC Consumer](./networking/psc-hybrid/psc-consumer), [Shared VPC with optional GKE cluster](./networking/shared-vpc-gke) +- **networking** - [Calling a private Cloud Function from On-premises](./networking/private-cloud-function-from-onprem), [Decentralized firewall management](./networking/decentralized-firewall), [Decentralized firewall validator](./networking/decentralized-firewall/validator), [Network filtering with Squid](./networking/filtering-proxy), [GLB and multi-regional daisy-chaining through hybrid NEGs](./networking/glb-hybrid-neg-internal), [Hybrid connectivity to on-premise services through PSC](./networking/psc-hybrid), [HTTP Load Balancer with Cloud Armor](./networking/glb-and-armor), [Hub and Spoke via VPN](./networking/hub-and-spoke-vpn), [Hub and Spoke via VPC Peering](./networking/hub-and-spoke-peering), [Internal Load Balancer as Next Hop](./networking/ilb-next-hop), [Network filtering with Squid with isolated VPCs using Private Service Connect](./networking/filtering-proxy-psc), On-prem DNS and Google Private Access, [PSC Producer](./networking/psc-hybrid/psc-producer), [PSC Consumer](./networking/psc-hybrid/psc-consumer), [Shared VPC with optional GKE cluster](./networking/shared-vpc-gke) - **serverless** - [Creating multi-region deployments for API Gateway](./serverless/api-gateway), [Cloud Run series](./serverless/cloud-run-explore) - **third party solutions** - [OpenShift on GCP user-provisioned infrastructure](./third-party-solutions/openshift), [Wordpress deployment on Cloud Run](./third-party-solutions/wordpress/cloudrun) diff --git a/blueprints/networking/README.md b/blueprints/networking/README.md index e7c0b1ae6b..05f4933553 100644 --- a/blueprints/networking/README.md +++ b/blueprints/networking/README.md @@ -6,15 +6,27 @@ They are meant to be used as minimal but complete starting points to create actu ## Blueprints +### Calling a private Cloud Function from on-premises + + This [blueprint](./private-cloud-function-from-onprem/) shows how to invoke a [private Google Cloud Function](https://cloud.google.com/functions/docs/networking/network-settings) from the on-prem environment via a [Private Service Connect endpoint](https://cloud.google.com/vpc/docs/private-service-connect#benefits-apis). + +
+ +### Calling on-premise services through PSC and hybrid NEGs + + This [blueprint](./psc-hybrid/) shows how to privately connect to on-premise services (IP + port) from GCP, leveraging [Private Service Connect (PSC)](https://cloud.google.com/vpc/docs/private-service-connect) and [Hybrid Network Endpoint Groups](https://cloud.google.com/load-balancing/docs/negs/hybrid-neg-concepts). + +
+ ### Decentralized firewall management This [blueprint](./decentralized-firewall/) shows how a decentralized firewall management can be organized using the [firewall factory](../factories/net-vpc-firewall-yaml/).
-### Network filtering with Squid +### GLB and multi-regional daisy-chaining through hybrid NEGs - This [blueprint](./filtering-proxy/) how to deploy a filtering HTTP proxy to restrict Internet access, in a simplified setup using a VPC with two subnets and a Cloud DNS zone, and an optional MIG for scaling. + This [blueprint](./glb-hybrid-neg-internal/) shows the experimental use of hybrid NEGs behind external Global Load Balancers (GLBs) to connect to GCP instances living in spoke VPCs and behind Network Virtual Appliances (NVAs).
@@ -24,19 +36,19 @@ They are meant to be used as minimal but complete starting points to create actu
-### Hub and Spoke via Peering +### Hub and Spoke via Dynamic VPN - This [blueprint](./hub-and-spoke-peering/) implements a hub and spoke topology via VPC peering, a common design where a landing zone VPC (hub) is connected to on-premises, and then peered with satellite VPCs (spokes) to further partition the infrastructure. + This [blueprint](./hub-and-spoke-vpn/) implements a hub and spoke topology via dynamic VPN tunnels, a common design where peering cannot be used due to limitations on the number of spokes or connectivity to managed services. -The sample highlights the lack of transitivity in peering: the absence of connectivity between spokes, and the need create workarounds for private service access to managed services. One such workaround is shown for private GKE, allowing access from hub and all spokes to GKE masters via a dedicated VPN. +The blueprint shows how to implement spoke transitivity via BGP advertisements, how to expose hub DNS zones to spokes via DNS peering, and allows easy testing of different VPN and BGP configurations.
-### Hub and Spoke via Dynamic VPN +### Hub and Spoke via Peering - This [blueprint](./hub-and-spoke-vpn/) implements a hub and spoke topology via dynamic VPN tunnels, a common design where peering cannot be used due to limitations on the number of spokes or connectivity to managed services. + This [blueprint](./hub-and-spoke-peering/) implements a hub and spoke topology via VPC peering, a common design where a landing zone VPC (hub) is connected to on-premises, and then peered with satellite VPCs (spokes) to further partition the infrastructure. -The blueprint shows how to implement spoke transitivity via BGP advertisements, how to expose hub DNS zones to spokes via DNS peering, and allows easy testing of different VPN and BGP configurations. +The sample highlights the lack of transitivity in peering: the absence of connectivity between spokes, and the need create workarounds for private service access to managed services. One such workaround is shown for private GKE, allowing access from hub and all spokes to GKE masters via a dedicated VPN.
@@ -63,15 +75,9 @@ The emulated on-premises environment can be used to test access to different ser --> -### Calling a private Cloud Function from on-premises - - This [blueprint](./private-cloud-function-from-onprem/) shows how to invoke a [private Google Cloud Function](https://cloud.google.com/functions/docs/networking/network-settings) from the on-prem environment via a [Private Service Connect endpoint](https://cloud.google.com/vpc/docs/private-service-connect#benefits-apis). - -
- -### Calling on-premise services through PSC and hybrid NEGs +### Network filtering with Squid - This [blueprint](./psc-hybrid/) shows how to privately connect to on-premise services (IP + port) from GCP, leveraging [Private Service Connect (PSC)](https://cloud.google.com/vpc/docs/private-service-connect) and [Hybrid Network Endpoint Groups](https://cloud.google.com/load-balancing/docs/negs/hybrid-neg-concepts). + This [blueprint](./filtering-proxy/) how to deploy a filtering HTTP proxy to restrict Internet access, in a simplified setup using a VPC with two subnets and a Cloud DNS zone, and an optional MIG for scaling.
diff --git a/blueprints/networking/glb-hybrid-neg-internal/README.md b/blueprints/networking/glb-hybrid-neg-internal/README.md new file mode 100644 index 0000000000..b6bd3d78f4 --- /dev/null +++ b/blueprints/networking/glb-hybrid-neg-internal/README.md @@ -0,0 +1,100 @@ +# GLB and multi-regional daisy-chaining through hybrid NEGs + +The blueprint shows the experimental use of hybrid NEGs behind eXternal Global Load Balancers (GLBs) to connect to GCP instances living in spoke VPCs and behind Network Virtual Appliances (NVAs). + +

+ +This allows users to not configure per-destination-VM NAT rules in the NVAs. + +The user traffic will enter the GLB, it will go across the NVAs and it will be routed to the destination VMs (or the ILBs behind the VMs) in the spokes. + +## What the blueprint creates + +This is what the blueprint brings up, using the default module values. +The ids `primary` and `secondary` are used to identify two regions. By default, `europe-west1` and `europe-west4`. + +- Projects: landing, spoke-01 + +- VPCs and subnets + + landing-untrusted: primary - 192.168.1.0/24 and secondary - 192.168.2.0/24 + + landing-trusted: primary - 192.168.11.0/24 and secondary - 192.168.22.0/24 + + spoke-01: primary - 192.168.101.0/24 and secondary - 192.168.102.0/24 + +- Cloud NAT + + landing-untrusted (both for primary and secondary) + + in spoke-01 (both for primary and secondary) - this is just for test purposes, so you VMs can automatically install nginx, even if NVAs are still not ready + +- VMs + + NVAs in MIGs in the landing project, both in primary and secondary, with NICs in the untrusted and in the trusted VPCs + + Test VMs, in spoke-01, both in primary and secondary. Optionally, deployed in MIGs + +- Hybrid NEGs in the untrusted VPC, both in primary and secondary, either pointing to the test VMs in the spoke or -optionally- to ILBs in the spokes (if test VMs are deployed as MIGs) + +- Internal Load balancers (L4 ILBs) + + in the untrusted VPC, pointing to NVA MIGs, both in primary and secondary. Their VIPs are used by custom routes in the untrusted VPC, so that all traffic that arrives in the untrusted VPC destined for the test VMs in the spoke is sent through the NVAs + + optionally, in the spokes. They are created if the user decides to deploy the test VMs as MIGs + +- External Global Load balancer (GLB) in the untrusted VPC, using the hybrid NEGs as its backends + +## Health Checks + +Google Cloud sends [health checks](https://cloud.google.com/load-balancing/docs/health-checks) using [specific IP ranges](https://cloud.google.com/load-balancing/docs/health-checks#fw-netlb). Each VPC uses [implicit routes](https://cloud.google.com/vpc/docs/routes#special_return_paths) to send the health check replies back to Google. + +At the moment of writing, when Google Cloud sends out [health checks](https://cloud.google.com/load-balancing/docs/health-checks) against backend services, it expects replies to come back from the same VPC where they have been sent out to. + +Given the GLB lives in the untrusted VPC, its backend service health checks are sent out to that VPC, and so the replies are expected from it. Anyway, the destinations of the health checks are the test VMs in the spokes. + +The blueprint configures some custom routes in the untrusted VPC and routing/NAT rules in the NVAs, so that health checks reach the test VMs through the NVAs, and replies come back through the NVAs in the untrusted VPC. Without these configurations health checks will fail and backend services won't be reachable. + +Specifically: + +- we create two custom routes in the untrusted VPC (one per region) so that traffic for the spoke subnets is sent to the VIP of the L4 ILBs in front of the NVAs + +- we configure the NVAs so they know how to route traffic to the spokes via the trusted VPC gateway + +- we configure the NVAs to s-NAT (specifically, masquerade) health checks traffic destined to the test VMs + +## Change the ilb_create variable + +Through the `ilb_create` variable you can decide whether test VMs in the spoke will be deployed as MIGs with ILBs in front. This will also configure NEGs, so they point to the ILB VIPs, instead of the VM IPs. + +At the moment, every time a user changes the configuration of a NEG, the NEG is recreated. When this happens, the provider doesn't check if it is used by other resources, such as GLB backend services. Until this doesn't get fixed, every time you'll need to change the NEG configuration (i.e. when changing the variable `ilb_create`) you'll have to workaround it. Here is how: + +- Destroy the existing backend service: `terraform destroy -target 'module.hybrid-glb.google_compute_backend_service.default["default"]'` + +- Change the variable `ilb_create` + +- run `terraform apply` + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [prefix](variables.tf#L36) | Prefix used for resource names. | string | ✓ | | +| [ilb_create](variables.tf#L17) | Whether we should create an ILB L4 in front of the test VMs in the spoke. | string | | "false" | +| [ip_config](variables.tf#L23) | The subnet IP configurations. | object({…}) | | {} | +| [project_names](variables.tf#L45) | The project names. | object({…}) | | {…} | +| [projects_create](variables.tf#L57) | Parameters for the creation of the new project. | object({…}) | | null | +| [regions](variables.tf#L66) | Region definitions. | object({…}) | | {…} | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [glb_ip_address](outputs.tf#L17) | Load balancer IP address. | | + + +## Test +```hcl +module "test" { + source = "./fabric/blueprints/networking/glb-hybrid-neg-internal" + prefix = "prefix" + projects_create = { + billing_account_id = "123456-123456-123456" + parent = "folders/123456789" + } +} + +# tftest modules=21 resources=64 +``` diff --git a/blueprints/networking/glb-hybrid-neg-internal/data/nva-startup-script.tftpl b/blueprints/networking/glb-hybrid-neg-internal/data/nva-startup-script.tftpl new file mode 100644 index 0000000000..62b9988a6d --- /dev/null +++ b/blueprints/networking/glb-hybrid-neg-internal/data/nva-startup-script.tftpl @@ -0,0 +1,42 @@ +#!/bin/bash + +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo 'Enabling IP forwarding' +sed '/net.ipv4.ip_forward=1/s/^#//g' -i /etc/sysctl.conf && +sysctl -p /etc/sysctl.conf && +/etc/init.d/procps restart + +echo 'Setting routes' +ip route add ${spoke-primary} via ${gateway-trusted} dev ens5 +ip route add ${spoke-secondary} via ${gateway-trusted} dev ens5 + +echo 'Setting NAT masquerade, so that HCs can reach the spoke through the NVA using the trusted intf source IP' +iptables \ + -t nat \ + -A POSTROUTING \ + -s 130.211.0.0/22,35.191.0.0/16 \ + -d ${spoke-primary} \ + -p tcp \ + --dport 80 \ + -j MASQUERADE +iptables \ + -t nat \ + -A POSTROUTING \ + -s 130.211.0.0/22,35.191.0.0/16 \ + -d ${spoke-secondary} \ + -p tcp \ + --dport 80 \ + -j MASQUERADE diff --git a/blueprints/networking/glb-hybrid-neg-internal/diagram.png b/blueprints/networking/glb-hybrid-neg-internal/diagram.png new file mode 100644 index 0000000000000000000000000000000000000000..bf82cd80a407c1b4af6b46789467ff206f9ed5d3 GIT binary patch literal 63567 zcmeEtWmuG5xb6%c0wNs(f^;{M0}@JiO4k6=-GitoNJ)dh(A_27T|<|EG)Q;DdHue< z_qF$Re(mf0J^zMz7w@ci))V)0-(OXg<#4dbu|OaY&f7QA>L3tu8wi9%iGc)sQ!Am$ z1_HrAZ>1$Qy^Qvn(PQa#{0~^4)KZ%Sg$G%$S(fTK`Ijwk%#F_Upr!oAekMr@mZAxQ zV3_g^=+W7Hw2~cP|M5C9H_tVBW4?cz`NF-aDjh^{j@=3-!F48AVL4W=g^G+64P(t@b6zShCE=t zB*jez7z6Mn&m8^#`r*<`SQ?e%v%JE;zrL60`}yXq!i~8O2X}V{VialTV|~^;M*wQ5 zaD3NVXTjqgEXuo-|45;2PYy|8+?`W=^>=B8)u#D6)CvhMy}zRZaMsU%=6dP}{tyGh zQ*L_Cd=(gC1pDIq_3xKeZy2BboqQ#FGRVhH38cgjL-u$24894N@VaRf1TY1_bbtY# zN&jyH)>BQ^6lc1IwlB&}&1(G>-NwWS81BiN=5v#9@2?jXfo(FZ_RqC4vywLc+;6mP z+N)TzdG~j6>-&cpWLc&?l1pe*NlDY`%fozNzB!o`>-jU(#JwD1D}BDL1W>XfgJU28 z#F2skvRgNz1lqS@fGsKia|$A3D2Rm~6vR!5D*?ne8CX~hCcNL`M41nKR?7{p4%q#L zf=IJM+TOSLdxlK}N}y(YMNrAIupqcP4_LQI(%;t~O8+lke|7>|6cEi)`Nda8Vn0ACrC;B@HBh0+=?;BTHqq$&%`;0 z9TXjGo|)fJP?;l_ZYr}}nm@?dxjaUdlK6|;cHG*j?BDIHW(6&Zi!KQ81*_uHG129C z4rtH)!ZAm3<#oYU+n~X-s(GRqKUpI{)|35y;-Y6Mdb8zt{H8V{x*n$nSNX+j*GH7c zdHxr%f^L)5mqm33W0P@F$?bV|A~$L{CP=$o0~KnDS=w#Fb|PqARFjH}F`3mT88&-( z(_hccQ+0V&mN6M%(~hYrp!tbCT1YS*zSdKYr+imz?q_Yk+TbQ#=$)xwG?!*yk)WgH zJw<;iTe$GVubs*Qx+59tB8OdE9HHC-L=XVm%aa zeZ4{b`;6;rN;nQrSmP|AUgUpmRrn3tofV-t!Gy!H6J`33cpddPgLjzKsFMuV7IfQL z-}7w_!y;6qa7c(!U^gIl6$r${Du6rFTUYueaq%wYmv3J=OKo~Se7Q1QBlcs@FIK6K z3$9^a=%%HwVNXt##Q7LUqAdV7`uZi5xzM)DqY6tNM_ntSV^?#JhzEXwg@xNLD|w+S zU6r z+5Cn%*pRVEv;<#_{3c+AM$u{i;wTG`P>}AV`uc{s*!D@}H*He#Gcro@+_|LBY50#Y zi1f2I?rB(J@O``b7bzOAF0nLrrkAmfs1+4&j7n~DDQm;1Eu-F&h`@9c3^l1@qP~gs zvzD}wJr9#9FM89v2|6IFN2Rg3!@AS8#HS>%#HRvPuoutKREhI2Nf?8nszMQonJX%e+9BaTuAJk;zmJ^1=`RLgCOmuZmsTC0-08$arV<|{}^ zbyg6GL0D&eRkN6MzSh=Z!p{x17+@ECjTi{HdcYv|k5E;OB?MT?o;oe=@atRu*jKgW z>s}wa<|DvQokp{X+R;ngvUO_DXcFPkqy5j&oPxXY6P^;6e>d=!5ltjr7w=sW%(_D9 z!0pu&65#HpS!R7*y>c5IM~B6Zg%Fi}F+iWYxLRcb;y(Nq1W#~b4Pm5xt>h~KO|0U- zV9|_-PrB~D%0a6q1HpPez&%HqNk3lC+h+L@(*<*)9wIk<4W2 z?)c<5^t`QK+T#`P)wJlAY5(AjOq;CsEa5A{h1Cv*mcB*UQKadi!M&%7ioGdPK>Drj zKxkB{+64`3AdWWgYY>@e*|NR#Cy$Na!Cc8{2CBMKUkE=wk4QsfrX^G7b;Soy3Sr+~ z-=3Buv)+I6&3o_nNf%iFU7%Y&wg?Tw=Zk8rSHA70^Unq||q+qS?@ zO<$^sS}-bYK;#l3A$mJoS@&=x=p7$DlH-qmebN~@Xr3nIYWwP?CeqIgWh+Gyh|d;P z)5iI2L1MQExpwWD$?N0L^>O)z|(Ub6y#Q!vd(|J%seISx4%A=j%2|oMx zabEnlUT80A8DP!^=y`mg;TdmUVcg@7F@gw!c5qV|yxlk0DXSuU_QMwxbNiDw$*v&# zC=nAHrWen#KWJZbk+@vRuP{ZrW&b8HRrNj^qml4Qj5cV?t3aN$hI8Q+UP)7SMy}qk zJ~^SAOf0oI)CfQ|Mj2r(SPg@gLKv+rM(^*+F}k!+;T6x$Q=4)A$=v|+pn@kjAI64D zJR2#YD$z@A$>*mtKPr5au9WVYq$gH*uh!tiR&{m3`=7}J5M6z|?H;c@X-7j%(IsVh z^Y7G;O_RQ48_TyihZc%kbC-G#^uP7Bo#k z)3_a`Iq45y+&$@09=31Bsq{pyc5j*$owNF}&81u&GqaC-s);UzELHfR>RXK8_i;>p z0-9u#9ozJRioNPvX>-SsuA>5MDe0ex>MH}QRf$)|)3Tj;qp$6e0wr%Lkw`4{YD79a zSz4cAb(r65B89hm6MXjD_j$WJ0d@RppD7(!EjEo|fnhzLF6mMvZR#;h^tl(vDi{7a z`vW4dJ|!#pHp^IPboF=HM&5Ju38F(q*iz|*8O%d54!7Fcc1A%z{Yl9+M!`+ieg)O; zh?_9wzwc@nU5d$(xH7Uw+O{|UCwS?gKnLHLs@Mg2glEU`&8C$C(@O*rwYfoXLpW%D z_>WEhIR}jD;Em@~h+{cQm`(L={KIeG8X#j@vJvsNE0JJUNczVn6F?20Kt5CRLa^ya z{Jxa;_iyzEmT(aFC3V%i^d%}Nh?mVE1YIiGP4MBA5f&UD(#X&iq2{_!Ug7Q)-*nv* z-FtjUP7)t@WAvY?7DaYA2pa1VdwqdVAd^c|^l6Zw-;G4D;F6>E zoxHdq8oq>YQi>ihF|!N{`Z4~EDN8i<&w=Ogx@em!4~6XExT#GjS@OxZoQH^zBmTjCL+@QSPk8 zXEqaBi?7zrCQ?Dk)sVmI^1ub0pneKkE=zJ_O$e-d$5!ykv>T3Rgr0Wsf13;mT&+X$ zz1BXm|7-Jk_Gke^8Jtx1*Sw`?)O0WgcA(09Tb7E9FeESwHf{P@tL2OFCZ%*!`L&Fl z@$HwdARljhh-1qyY{XrUj*W(j%3~C;CM6vsE!T;*Sk`7CjQ?Db9N5kOP}kJG>Yw$5 z?~7RuH-oTu3^xXO`2M+O6kvpQN|o>Am}q&MjQz#BZOS}q$%Wn^n)n+U3@HPH6ag*9 zu;^&O3x{SXuQFTau@GO^H3!y=bxL!~sUdKAGDGL>uhF2UkK%^@XO z`VRqpt0Ae;pUwq#QvVbejwp~6X={fV_R`$W3&d!NK0e3>;WrmFcuU-`lXOABpIr;gCjy5$3Ai={Z8ZqEvl0;Md_pdu+ zczeiVxWw&tC=E`UKcgVBQRgv1h7;y0h3YboSi3dvkB6NGs{*XL9ha-u1?FlL4m7<5 zA%WY$dz=#80T+fIW`cv*nqKpUphe+5p%j+1H^hIQc%%d;$yook-z~6sQ8?67))pi% zl7~&g^YLJ8ig@aI$ckqF4A^H1g;WQp9vSd90vzyYHI5I?2Pi2i?o=h8elZ${DOiiDk=;6t{?^WAyuI4X0#Qj8mNvd7vfdx zX>N>12#s+n^0OW(OszDZ&C`1pZh~%x8D|YWhsh@H2)zPm)31W@$|D1GliRl1N#-mH z8iD`qeJF0#v~sAMFZMu#25@SB`0Bc|nynW4BAR}h1ZkKPHHqSO|6+=0>Nz_Oq|S{@ z|FhLu-Vg-TYzX{wCC|&3*@cCFCJQvQi_Uw(DWapJySuw-<_ij#O%A%lNVrKTe<2WU zU~)gdhdb}zxP25mz|yxyQZ+buFVD|W(b0K{AR!?k39+%icZY`VyhzY-DZhk;t-Ztb zMup%hzlteTi=Ol8q5Q&<8NMN~@T~BC_r5@t)&OGm@J&fY?({rFoR49vFdMx6=OXAi z4uVNk!^%3K>MT97sGO)~j?D#gqR-5oVP^I~!t`$X#hS`UqS>ImNO>)ApJCI3jb#v5 z*aHH6j!}HX$SZcyoADYK3=QCRye1rtMT4h^_@#a@MZZ%=P;G$}c`zN#u>X^D!0!a!HI zPbVK8Ix=17u*kyB-fcTEF;O9;?P5*x5?HIwxscavzhgz10At5UkkAg{4P9& zD}OaCk@(L6A;ZC#^AXPrua_Zl@_YYqulxlX3IB_6F0mjNk(N)kMgQT+(rhP7KkUGog?!(k(F*& zW@Y;HjQ*H;&x<*NNhdo{JKU}!7v?J$uq?ei=P9_{q4JV zl#Flk8yXt&^71k>k>K!ku!ozc!rH+L13$l}?<4DZxoO^;WbRn8>FI&1U6@LlviVZ+ zhwKfQv@j2~lqZM}N`Fvw5N2yFi)MJNl`T|O-t3w}skCWIDXzNFLL4!_3MZ$g;$man zcE;%gP?M3}f{LS}o)Tg*)PXpSnlNBXt%0p&Wt9*+h08~#DbrmoIutS z(-IRwMK;1>0FIID#6s=6$Bify#?L#f-%8WCa#sWHU`9SvWAP=@ws*$*nTk+M5xtLc zaisvxn@f@S@52Z0_dNF+93D0$*Zg}Gd!Kz|jtPlx!WmZvZI`N{!}GGuM=;}RUmtfI zs16>#T(QmVd;Sh;KkGf5m$SYI^56Pi)w7ENjdIqiYtCMb*EVnd&TP~`&!A-xHvhiP zz&^lxda$nRH?_aAGewx zt~t56)6>&P@7j7H23*csT5(LFS6GA`tgK<*zM&lE=Hvv$d6VD{iFqDSV!cWvRZ?OL z<3F++f6oo$U}xt~8P?4}nj3g0llBB51P77+tfbiY7}OAa36m8~;(LYHjS1~ZmuvK& zwuzbFpVs*>T=r2HaM0D><$Z<9Xc1fywF2(kslx7|%VfI22o$Ndv0NTveEgu89HqFa zxw-3<<-5&6;|CWA1m=;JA`KE_L|G!MB2coZKmA;jzi9O=N4h9H2_>(G>66Db8T<-s z7f&Ur#T{`qQI-2mxG_&)NkCcz498be1rOq!jmnj+DTwPnZndd=GqdW?TH z)&`Oy1+P>PyRM#o389wRKyh@=cYF)8!IBE1X{KvC&7?`FoIKyCD(yzs%mRBYSwhGq z66A5W^j$U#l%HLsJ?0$PMOXaTQU3N<>djU(m`?Sb_vi33h)Xl8nCh zD!i89ZAUZ;SS>$8gM||C@XihdZzul5LC*i9g8I!p>N9vAGV^HaottH<>|NHY9XX2~MFBDDKI6TI! z0nhV(Zw{xxR|J>+s&aGDdWlOeV`B*!(hm*;E*t#{($dmihu}UJ^C1lM_N&n^S$?>v zixY8tibP2MMt22Xj7`Gv6k=DWw&4IHwjcQ{j$X(&X+d^gn_;u#2X~(4*@nEBAeC`) zlg->1BikB7Sy}4rp-$riF5BJbz7r|S+y|A-t*s&}d8y;j@bIaZE*alW8ltdw)r=`S zsEz8|#}EjZ$Mw*V;>~kQ3k$Ambpm3^G3j#M%^vjRwL9%Y9w0#}!N=-Z6W^S9&3?Y* z(O+Ltp>{9y>TqFxe4IyEIP~en*cf-ZyUhztn;17*nD(Uj-fssgr~hhQ z`r)Auk4ARj`gH5HlvJXh!kNhOwX%T$#UD}4k8j$IH;at#3oD7(ZtDVX>r72eG4Mr! z++XX00cIprm?a0UmX#;L#fvQhl{DVhD8tZDmJ4GVXfl}ts=CP4Xe*Tbawfk=#7nB@5W{=Q@D{Y8#Kf|D$)7HV~S zb+xzDOhJSw3%s{8saB5!eWDJyGG#EoEGlc62Xddt$E!#h0=|*ft8Pv+ob&VZ%U`Z| zmB&x~n5EgH@RA|Rj}MpL19Xw1nFrZH;Z(0&fEuXv8N?9{f~P1PW4As+5vR@90qH%O zEhca`(kz%}irz>a4-ST|Pm%47UL~cU*GUtG#Tlwo)9n|MV0CW~#IX+OROe{&(q}OI zzWjbBHKysAgwD-63gNmw_@16logVS+o3xdt3)83lN}-qZ3=Ek9GX22y(>}N;{B#c7 zQX*HXOzG4AD=kH~p-V zQ{1LY3FijG&}3Ob;R9APrG^;C#*a5M;j(n(!;Bs& zi{Cn)KU*q-{L^sQYu{a^s~@Irr+bji0y(fkroX8|XlW z$MVNDw@{~L?ui}33hVdnY+e>U6=wVOu}{0Ma0vby#(i&K7u5}wg0JygJqie(l1M$y z+DJAF^In(E{J5?Cjyo%s1*-V6mmHkG2a8Z#s1*vC_2jHp0&i>PS;8A zjX0f?lar?du_L^AqDP>PgpOXsAZKea+6z$iO~Df&H3w~vD;J!ieIurD4EQRbJ-s57 z#FnN=%qMI4xgp$|_p3TL5OGgfW$AcX35CQUYum17XiKI@{Oxm9)??YSoe>h4LDEs& zEK*PL3L(;1#(Fbl#xo$kL#!T6(Ej0JnMK=8kmSHHZh%f$QAr6-*bh}G&-XELsjtI-R5g+YsPWsXF*kcc&kat z$yj)JaxYfyucOFH=)}X94Opu&hPC-wKt4z4!uaf-qh9il-ueho*z^(F-I$XB+IwV8~tAXz)8>;4bd zTdyFz^|Ei}<)!#LTOYRNByNr)1Mg0OsP%Ar7`! zY~Sz1g}c75=l|Ij>M=5g(%=4xb=h4LbOWS*$fME&tH&rb6vV`zRgn`-0oPiSCQ1AT z*$YQnlBT1Y9Nu~oSLVdxe?Fz-o$hXVN5B>Vy;u$7JeaQu%QIo_b`@T9zXdOmG{0S3~6t56L}Qc?^IVia%?rW>|61gn(+=V$xO>gsA@BI%v)*rsGTgvu{9 zHMLtlGb<}PLV%Q5MMY zX?1PQ5WGE?w1ZCN^?R$5HhA>-^aEzvYR=by_7^v>pR|w=B7^$Qu$%>LTNqWEv)i{J z_aoI`la7Ml6wIBOKsYFpgLi%n7R8c{L7&t??lgqE|M4@f&FF9y(B z?FZbK{V!kqhEE;~qaClG<-Ep%6%%^wB9s5Vbkex7Hz7U)7WihkPl!Cy?mgODh{< zTsn1h{rY%Kefm}6`r_THV<1oZ)wsFq*TG&|zO~4fOr@sX()NUlqtGV=r(uW75|=St z713V@rhOv~f80Gpq8lwNc#6SnO5;w3W&>f1rRjS58iuV%;oqAWJ5I98uH#Gw4-noG zBs7A|$ZTbwh8St_Tw2IXcSt?>#eJhUOCfk0RX1MKv7(;uBbx^vHp@06j*e$GBL(h) z@ZO|Tw;FtFDdlWqMXGLCq++XE2-BW$h%A9L8s6=sdfZ}zSv@zueEA~ebH)_oXrB>> zn=Gw`3X^ren71p-7?J62Hy^1Q6)$2i#E4s*X}REkiW?K6rH2JqdmEIPq`2N^&XY7` z#d~t}c&id{LU4{_fV7P|PYI(E6xOgSK9n{Ha2xOv*83r#fR~A@{OU*oox|=frN}B) zA#bu^@D@`ZR6WOCO6Mm^uU3xdh-k6@JjqF+k)J$RY>W$;9us<8JPhc?mar8$ zk5PeEtg#OC#Hfh(uNx3Ul``(j?PfTZJfxggQ#0zH${KlZ5N)u=n8!9Cug3Se!;J2* zs$Upf2HbKgv6CS zv9nLW@plXxdySm&o5S~m6QvMVqik$++O5`mI)$G@t)+!{0C1J?VMtKHUN79^z~KBM zhIy64c(C7sJIjFszeUeh2D(V_;=MJ@Xe__v;Y4FkBVBikXq&fy_VSwVMWzK9qGIra zZem>H(3Vd4MJ-J*5+cBRwS|fAJ<64O&j39E(S%CijRxP0_Zet}HWB{Nk&b^(@}ZwV zMYMfkR!>havFO0ReLY%H73cW8>2yRy^mr#Pee?Lw$;Kc*KY#vaz)>*u(Ex7EEN8%R zSTB|DHwoh}w%AphI*h!6hebU^1Ozeh@wv%|%J`5`Ow|0`-iKKt?>FJDo0H@PVO3D` zjns=H+M=FgqUkX*X6;ekAE+hj%aOA#1-ti|W$cqsshLl~{bG?p;v}3#UGi!KM`#3n zUr$TX;O&Bdn^Qap&OnDMUR_^57x&X$28Je3#Rd7Qp4z*@WKqj^Uu;wk09_`*nb50% zM#~X!UH77Cyl3X8L>C)Hje|-3g?CzuNG-%8By}doxt&t!KLRimg z;^ylgAM#ZLgw&m2n^{R_pTdy%W-&nFD(jLUplWCmfC+?+JBf||RI`eI*9?NQHttur zc)mV3GII`$-C(p1<-Vk!2M8bG?;r?Dmm4%1uQjQedjH z+6ZQp=Ub_3$*0i3;vU4*_$t9`PwPTLy!T<9Zu~hHjgX4s@lFf*UVN8yg$deyeL}98b;6=e0rD_tAn=l91}Z3vz@7mJ(w3aP4-_$0J(wzzPeTWA2;qdblU~gYkKlb zL>S)`Tkh&VM+-lVQp=~MrHyT$(Pq^F&nsm)9eE103w1ENG#Y}TAD=%Y(p(Nd+fCA3 zNk|~BUcgUqN!iPvL`mU`2$`C4_)5Vsxd_y>YCU9YQrMx(Im;}-rBxtT&zd>@8F}Uq zdq^#q`j*H#;ovbi@BtCa`w8!tnq|?ULH#Kj$w2$i(6VNQTUh zy7YK~_RKN(WTfFg+rH}c=YlH6pzD#yEdDYy;v|sIILFE~q2(+alD_vdG8?F%W&{UP z_0)m%#J1Vzqjxa+W@T~#u<^zM{z#_BZ6kO51hZLlJGi`dF6VORiIT``>>Pur=p5}V zn(pU1lK?6ppa}RZOH29~$JkRh@t~Mb(*o4Vv4^~^FP{LQUl#EZoqIa7KG6C4n zm05YR)Z3Ho?Cg^KsM=RSmJe+3O7|9hOVY+afBwwQ>PFT1d`p;~)^_&~9xBz8KtY72 zo!2>KITJM;y^unMtA#vQA`I|R&b0pc90wAd2Bey9%iR5CpNfwHfAyb0VMg269F&W$ z1fMUJu`c~m?z@f48xO7K=3FHTaz5F&`A!Og;q2PhGhdoB2LpKYP#{@R%MLY>LvK>i z=RfbMSxyUfuvx2rG&ryA3FwWT!<^*Ut1b3WI{WXEBF!M)fW-1&3!r zhHrt|ST@npWxoIxA>LZaGR)VS8PxE%1t7m#xXuFE1^m^BU9EF)@HOHD@MN@j5wEtd z0qg~KhYEVG!+{f!pi5t0K4p$EPbULvhQKrXp6%i(leW5Vk2I=P2Ufot*8lOz-ZfQF zb#MBb1qTtCTlQ(ot`Uh3$!8G0D6!liGPxQ<&!9Os|CusaBVq(!&MJ7)U`_SM(|0t@ zZ%4QeNE6*OFK0VLyPBipCko8;OG-*T<>V-lpfc!Ys0Vb%@lN%;nP~89BKQ{a5MEkb zA-5RBcog3lrxw6{KdZd7A2$qejMYJ^9&j}j%FRvmE72V13}_cW!N2ef?4=4w#CbCG zIC*YE03t8`!Gy<7>1EKsdD-QDsvx;}qU>wa$XET!kL|5bB;R=)G;89~b&$c-pyXqgwH0HAsZb^hHJ1c$?Utw$^@ENDJmhOes-1L#Q7 zuoX3RIt&JS88L`NYmSqroVq&0^Zonxa)G7u_yUJVT1$Jn4SOC)>r<1>0Cu<1#ZvU` zKK0$4Co4J$TOAQ-a8Bzltu61-P}`yVP(g?~(5npkaq{?RQhXE^wrOn0a~P?Ahq~wy zQswyztS?_uVZ}dt!>HX;xsmhE{DRhA2R2fiFpH{YWn6tUiNlQO)tOfXXAcBDxNN?I zOAvA%&2Z9jSNHh{89`Qg1L9$K;muPAD?r%NbS!tx`r2)rorwk|OyOl#pSd<{+3fWn zQYVS%)Q2e-))H0ED~P-lyeLy76472j0btJZ!V*g`SkasIPnz#6?`ZjN69x(7Dl~fv z*vQb(dXs^gTKh7t5I48;N(U;#WvDbaBwqqZctbzB{B05d&Nk56NKOu&`X+{xUqHa) z{>Gg_0x`NyrKD44WXS>pfSz{-3A;*Y+kzeJzZ~WCJJwgW_(3>6E%EFvlok(T1@#oh zy|T1eaQoCY3xo;!Wc(Z~88*!{{e2_0sE#AjO_ypZ4 zr&~!}W8QX2q7B+DfZS|tSr`o!HFb1OrHA2KATyH+H2AR%7ir?J{r$bU zsyl86VCCywKcb>we;cfR+h&&bZ}iTX7l>;5L8)qvYszIYXztS>zRXaS_$!}8K8N3B zWr+t8%-q#z@T`Gs9ikIYf=#|sKMs#dpe=1j2nHI@Mu68*b#z{Kh~v0}t1^QYXAdDs z_3`YPJQ{w?ugT?qv5g?Rz59fd!%?E`Qh=THEf~|q)s>Epj)I)L`cy#u)vH%Kp4kVs zt@%Lj;(FLw&^8ZLZDi8^c(FWdd{5+irbbLvnp!kb3*H8xtkqRB1_}ATGDvXTs}mL- zczcU>5z!|N9E6RHjh`gK2@1_egDWu*1YUM*Zv!6zh+F&ILstgq9EHaIO(r%BJ(l0? z?q)m_JM|DDy+LPGk}RY!25mQ4iL}`2)D;U03j>AbSfZN5a?P03>o=GpIXOA=bHXnK z3e(A4!~Vyzmm*>hGCK~tF-&E!OCebSf6|@Kl!W+Y8`ef(#p5-Z?~h*v4w~x>9cBC^ zBE`0(&=U7Dv#{tI9gS>_o@9R}a||2`fCA!DiOvSd-T5aC_Cfh@f0k)*&WxZ!OD{Y& z*E-j^Rp{M4tBQ;36(+H3>_I+IDAPL!uGx*tE2Iyc?|!x!lQjoE1`G)Uo*4%AVKPz3{mp3^m$)M1w~ z?rKQ0xxlWzrAY?wraM+|t+2jl&&RB``}G#M_-F56Nw;xP^?BY{3ws}?nDLe>e<*$6 zZPjbI_u+o+qxf*!8-8?vAQE;35q5NRT=9`6<2fazr$;vI&)3-9t&;J;2f$dr8r-ob zwno#pkb?kBqPwdLBnbczZBOAx9@ko@@T2(n_`beA|J#e>nMxDMVw|W9G)qf1FSbJeIq4>6hnRV`xd5DS6l1h=}AIF1Q5RWASNa>7vL0=lZW8$*-+>J#zIy)$D3k*2Eny7OU;F#t-ffJ1mh%B}%!luy zpWzuUJL5SYJ|Jahoqu?rm6hcuWY~rE0Lp*X&NVhYjYcV_Tuk{Y8V8$}k9h$VVx5uU zB?r|oWfgSzrJAfhOPpI#_~i=*L#vt$q0C2VaoO_L)>cF9_wU(g@iJpN+DpR#vWzU? z>Sdq4Es8V3^<>amf1R{cST{qySFbeAn@tUrWKq zr$M;egzHER=#%$e)34LXMS%S51s0Z?eHj@Tjyxe)<7$C0yJ&(bJW`U~m3w)BYf+)U zfyKunrOB%p;G!UAarGA~lS}@KI?WX6l%4dJK>o;P;fEd70R4C^ZEXPo0i-guT}ly; zC%zx%@y}^(X*AU3|G(Q$vaUyM#X!> z_toj2Zo46_eW_|fPO_@!?2hfio@={ipRZ1_HxDWK+d0uo5DpMC9ZH78f!al!=37H+WB;9HsFlU%p_{fd`G5o11^p!H^yRHR@uTp+<`S zc9iB4uY9rPPKfuB@JHj^oF3H%XyQvi8+0iI-6Xu$$JF#io)d}~e`+HW5_Fur#t@ub zO$8j!vts1~<|;lDziQES%XgpnTU_XM-l%ZVNzA{e<8>|Ledp}wtf>joaF zd{FoFJ1J|D4O0Fg6eNKPO1?B486D+hV$#Ke^YZX;7&d(T!@|Y&^!%wY@`PcKY);>@ zC&&-IG0S${aaty>01f&e&a7#ZKMPb#c$7kIyHXqHNr0XCSgw2vJc+60yjOTkLw4sE zyYY7-fjH)B8z@f~pTCF3fNxxK`~2w@zQxLo4mng_hAa5@#Cskl(p4FSss&xIt_Ybr zX0KmX>2M@{VM(r&W#SaH0dRF;y3a=kd{?)pvf|@0-@k(pFJ8R(BNLCb{ntbMh6!9B z9dV6j)g$4+a1(sbkK|+)=I0S+TUA%{Kn2d`v880%2K`vzbS)*+ky_A=cGy>~@tG6l z#<7hMqn@4~)yAXbmX?+WpvQUjEiBCZgr3hxf*lz_W5|FfscK*lX8YCn8~CIUP%M7LY*vdxZm+L{9FqLPt(jp0Ns4(Y>6>9?E9%tXYK6)L>+j>eRNhT3nW>I-wJj=h z=#D7RG6Vy1wVIPlBt>dd;H!=Z7s2;rjGFRiY+%nUdz0~3_R)I-*c^~_c@@i#G<@C1 zuUK2rm^eA{?5`|Al3d&xCH87C z*{(Gb?5Aim2t^>xDN*}Nssl)TE<$r{u@hXIeff3byNu_BD*2V};X?Xo&u{720oq9< zJy!TVZs*TV&e0n)_(l%I)u^i3dgv9;+PkSDH^78ct>I=ufe&}v+e{q|Ku{vR`Ro0e z&n-d1OiUQ={XjDbvLGfVmOrZ6`CKA^GA>;sU`liv=7Srj3b0H74|cOcm4l6r5;X;K z^Ys;({@mOlACClOXlnp758wDnCzX^;zihGr83 z){pKUs3b3btEFD{ElSGH2^?rwi85d+W9Ae@s`kx>f=4=zHkeeO3C!0*Tr+3QRs`t> z{4xsVN9?=6Yo+qsT!y=X=#C_JGMMZFMcBI*za-Bh3=E7fp`loqm@8l`?vUQ6#bEEY z2@Fel`qG1rXaJ%DLltRAY3bS6_JIE2drTPGLAsbPaXY%*X4wa$xc|sY-f|*sTgEP% z-7azO45Ri>35@kH4397_2`;<#UCAbL3Dd<5vDS6ns*nbHjq5Ji)Rh$ElZW>J2X^jG z?x(c`@2QV3So+na0YY?kcD{c7dZ6)F1(hEWN|5oDWOwJS*KcY*E-tt?J%uget4O7O z3E*JwB6}Fi*#rc6c%D6fzBoT0`s9tIa3Io?zM?&%>9v8~)$_m9on&zYzkQY=$$f6? z`QlvQt?=KSECP{(v+euFX0269YEDADS6NHk(d4x1)}%9!v_Ee6HNb>|bA164UJKLU z1CxSPbIH>0Z7O$+X)O?6OefOnlh!{&pB{zOy(l41R{TY|+Ns4iC1IW`ZOiG)59-sw zL-LA{cdL?^-00DAhQa3c^0Sfw7HfvxPw%f#p5b?Saq>J69fNmQSi5j_H==Eii0r1% zq?`dW&E^jUC1(R&8=iR-m{CfsID#T0A|irSrxbwlGAiG7Dyi%IHIaCqFXat(wz%-u zDwTEVR_x3-VvNA}S}ZuxqC?FxV@PVYZKH`R?VhN&A(lv4h@Pxf_G6ez{CNnYl7y%w z%Wk#7KE(;1q2`~zzW{*nFc=IVClv4F2$g>M8(9Hu*%>X&NOr^U;<3Rh;H>gh9VQV` zN`=tn5@CSrlppQeA6)Y1+PcyE%*F*8a;fbO3th=W3o?J_hP2I+gXcDvJ88|L2dCl&oYOIBpQODAw+W%^Xti2w5eYf88QCS0e49eq2~W8w9d5@fwv zf2U_AcJA6GYNIz#@ivRiylVWh8@SJ5VYzY~$t;Zm=RGq505#So{Ez)GK&!I>NW1fX z0C|{62duw;Y<7{PQiXm{!|?9FVI;$2vGAqtx6bEw-$D+Mw$r%foHB+bmgfbw%N5qd zLgGx&@9<|S5~5Ict99#{~(n!Gyx_-cNg8UZbzN&|9MzfvPa zo^Px(sfa#{FL_MKmy(9RZ0h|6eqR1ucB7Ydwtm%tO^gbGN6qsn2r3RFC`k=5>@)hw z?AOJS)_aPGWwqYqh%aV+>zvi1mf(#FiU0Zl^)0qWKP_|!v7k5V|I#S! z(dgK1JfPg5uR&@X>v-kB2x#(tV!#b2$-(*pg=!VTXhK3`Dg1H!5_->qa(TST)*u=2 z)k?A3c)6^A;Kpws17>xSPQ9T^$$COqms1hB9Im2tBked6cz?ldZ#7y4@ZKA@$p8ZT zhyC0e7Y3ZU^>D5Tc7R7!^_Kw-$ew@D?rR?V2uRDBLP>3xllkl@9X#a2KQS1KNEd~T1PLM zB0)!i>qLBgnr<0*1irX#oz71AjID0wU!4Rf4X2Yp^PPW<_7^7N*MNWDn3RWAB%fc1 zIG(rC-!%bj*H0Q{6)OH87Trm1*VErl+&lraqvlOdASBv?cF4pw6P8@0z$QiCg7uy4 zSZ>=NqyN+h2K3f03W*x5T7nv~D*zcBfi$R@DBw!`Vn-q!8LGI;8`WnABCV{jBcvuZ8KwJh4WA zd(ISUZ*LFKt_F^Nq97;^)7UP017D8u3g&8c284T9Z)D%tkh}o)W=jU#U_F2)&uY7^ z*v}gNb{IEI2{S_Z>=;jp!9E^9lYw`Hg@vUgC;yHA=|GxUhJy|bAAL+q<9Sz=08oL| zT-~=+t=`(3X`4!$$wC!R!potLG@N0g-a4fgXs9+C&C);-Twh-gaE#G{+bwvQgR=l4 zW2WS2Ls!H1?cl?fQWZvECL1!~OZEvsGg)R={!)z}ANnh*~za_h<2{enguBqx{^;Uxk^j)?jqvZ}H`&JE7sGkhR zxG2zmugY9)Vm@(eK99f8qGoUOHxTJm&@?hKGB8+}n&KB1kML+bd8CCcNw;2q6tK)~ zs-#(+&8R!u0W@kwO-zu^#h7V;vk&m(8(i}X{ozgBPDOCrsOqoeR8+Fh(R^z4O9tF& zkhXKG=Ri0rv0Tf+d$-Fb3l46B;TP&49O+%kD)`5l^KaVjPqb^#w{+?V;; zlwjwN3d;_|cD~;z2UULsh&tQv>lUWU5&=S#ya5P7o_F;oco}EYW};u16BQwP0zWU6 zPD?&m`K-+H-mMpkeSB@p^vFh+dj1WjHt2S{Bs2abPdcK8!P!2MBme zPSimqq>?c@HgP3rj5ZTF1w0@GAR)M!u%Ll0xxhHWP4IrPiith`Su3V96iFyP;Q0p$R5>6=mJH-jULjzO*tTf}i&Is&ygoB;#>s4M`D^=4_fxys$Q4OH8S zewH!}M5x;QDGRwv=z!Wo%_D}2%g!A32ws5d)jH#T;9YNbg8vy=)6AFaIT<2%u`Cos z%0@R8?aP?{G%FG?`2i`vj{6g0K@TE+x9@_>LMI#SpK~Aj58f6%>;czS3^#FS$1lS;@$lbG<0mTZ@o3A*BZWgvScSsB^}Tect5wo z0Q0CLLEhdR_}TEC$zK={udy@&Vly2+Z@cD685(48H9FFnZZw~6@8cEa?c+lPD)r3B zV!Ow4IL^{jZ`pgP{q%>{ThUCttG*Qo3;ZN3Gm!2`Uj<N%J*9Gw@x^zMvRlJNX1yXM^s-1m|%Cb>i-=ncJ@ceGP^@pYa4 z&n}xF9trfny)F$E@qB0RQl+Bxj_@yoJC$s{UZm3q2f;hqym!@=KH5~>*lMmS;rUnO z&ihX9>6ltVzRD#rGTc}Hq&hE3`sCSpZ9?*AM?3lRp>N>dHDd1*tKF1m-?@!u!;OP= zd_8gvsA~i!P!L5&UfkHiJzfcg{p!>4xx?ip_UQ%BQ?`;K(wMAFj~<;nN= z$F}B@s8W-vM<<~poZz#YCS4L-M?o-Oe^Ct+#F3HWv(Gso>i2wtyYpAn59BP{t4#fr(f||uSFP>)nm2R$20bml~@ArXw+K7HS znzh-$$AZ0G}jHK??Hk(Uicln zzAFPjK~9ntLHhxR-uK-jH!uIMi7e?N=eAxg=NtnJS&jIM77q?@8M>SEtq~=OtLJ|m zyQ9Q#<=w1ifdHeoKX9j7d7qW4TyE_ErusiNCx&dPf*=UR&J_PF{_G9?OUwV3=XT7afqc*~<@KcwEZOwLXX~RE7B* z$Wski@660aC*;s|YM61BH>*?;QVY0jft60s{yCPt7gGNkg$$@f09*~|`kVQw*kff4 zfU^YH2^^^9iTk4r^3|{Ea(t_Fz1M%at?ON^JcivLK8Ec3yjivKw>rrMbZ>tTc#W1r zuQ%*`XK6JY85+|@eoM(PR*))5TMua1Y=J!nq|#}tbevVKj|K(39WxJK7jef%&TUyS z#uB?)Zu$Raq>=ZSx7U|j&&J}9ta@ezF34dX$_aCf3gFx}kx_1te)`p}T%aXoLy5{I z)7qPPhW|s?TgOG!t#RKo3?f)JS)i zNO$ut?{m)m+~+*+<9~dFfxXw>YhCgE{jNu85m8W1huiR&KFL?OB?2p>w_=O%K!iaT zpd23`KRFTc(!*o)h0CYY4>ft%$PsYg!5x$7fqC<+US1M;ncP@~i<|hxoiXf1FXYAR zkH6!M>bvaYe@v(za4h5+etOHaK(~=H=67|}7HCGyr_$Me4rk-$xh>OnJh$_cT`45& zZGxVEy_KFY6??%s8>f$dEY;Cqbd-}A(_hcK;lgrC5o;=gi zdg;3EcC;}98sYX_FFW5r^zG5C!Fx5CGn2$Yyf&p)dC|<-c>0e0`doQ)LV3JmQ<@*Co?D!p*;FhTcn(+}!d0dLTB%~p!>iHluXbeMjWL>@wqWB|? zL>(@psszWQew0GoED~jM`t!fIqnIYEywg-g?Je=IqWAfvJS|^srCXbhxgSKxC)iGy zX*Pea7>u_jK>3|_y=MV24_oZ7#rBA@#$)QHte_#P5BgSdA>=e-7TCYu`roH;6s{0ptD}ajA1$XLOO(m_|&b73Mys56vIdqynk=6)85fUSSId)K`e@Jcg04_K^Uwz}%G(rDjT4YQx%{z;1j>G{>= zCC~;8jg0{zlpD~>092r@(rUm4GWp95v_@j+q!N;n-smALd)vpBmzOIlDh?0bQdK%H za<`cOekJV}gNY~tAKshw%*?d1U_1iPJ-~l7*l{f8mkF)BK~a+b?9M={Zi$YP(sMw!K3zCqyODN&p??pk5vuVf z`CnkG&7kg8PO6>j&qh}**B0ZUk~p}yCczy#JQQw5vlpfg`f1gl_UqR#s2CwWKKLch zh~vGzZ#)e%0jIkQA3t(Lw4=F!kdv607zTrZj%t^vlvEMWi8lQGMgZ3nA++`M;v>l6 zrxy!iI(@sa0+kc`4DPU-<21D60XI}zhO^&2&*yR?&~ z<$obIi9Y3Wg(OF;!fEN~>;;qgOq%epW!=`*9Ub`x2QuYiq)(S2>7d6^210wl%y+%| zv&gC{DnaEZTT`{3JI3nO9zwbHcmA`az@eoV8|nFZ6ovxu(lE7wP1KcH{b|wTe8+Bx z{{YhxE;QlN@?y7&C9k7K>KU3f>R?ZiDW^YS>c!*Y&s$L>9%m_;xOX!jl5;g|^>`xK z-j4!N?!R{Me|+}a*9EjX4&j6u0(%jtISDhri+uGdTd_!~R}Kay1cD5am2qz4-fPyf zku#6?iD*j{4(Re9-VuBIpUvdow~nV4A`HY0;NiGl+pCMFl@m9WjORIwl$d%=n-w~0 zaO9B}+p!)wB&LJMS5BZ>Djq-={--SR?~@TY@6(Y*qP3Tu4TpFI)c4 zhf+flN1i|aSj831Q9-XlW{h3V5y08yBH|j0mRZnk1nFBg6%*VjR{QGMbh2w`65Uyv zLds>&_O$26KuVDz@KNZRTvY|IDC5G&?{Hko2wZ(K)vkJ!AUaUtbR_<+JRe1vcOei)~NSnR!ol z`ZcsWRPY%RhcC1J&n@$D>(CpwY0#2ei?b;*$@l{3RhM!Lli{?vs0wsS2aWjy;Swxl ziATZnS9wh0{rR`l&QmfE#FWVlI-`mU4SbNh8B*z44C40nE|1fy#rv{D#`v-H{*q$3 z<|hfJK6M&Y_b|??s1Q)|^<3KBE$^=KdiT%!V&@C1vUXKXPFt#U1#%f$H8T1Jv!)Dl zdj^+Y?{R^<4|AM&%DExw`nHV;yoNm#b5X<@d#Md+Hf}^jR_3Vi-7p-D zF+V>a@CH1+;$=Y93O3!mH}-w48P>50@jtDrrV@P*>dx12gNbNpBEmHbR443e0Lty* z?#}5-IWj;09CRK9i%9p1z|OAt^U4px;VHg)<1fzX@9$rD)^n@JtZX=f`+u)j$$zQ; z#4IE;L*#zEk2>M>-k!7id(oq4A79_70WU=2cz`NWl*^K0y326oSvpXun}WP%M(PEE;&u&ZTM>GD)pR|7#1 zyJ}ieQWDBC9#jd)iHV&z$15zbeeGKbcnI65F)%TMovS}_y$rZsUcTc>g?#+zQFeAV zXm49r^Q{}haV z2>Jw8S87;?hS`cva&>j(@8@?~4Dd%FV7K@*Jv|NfL1tGt93B|h{2?PHMPEm!z;DEw zleTwYz!7v%n}RDr1zYRDXiyTWKNC+Y8^`$UX#8v`Se2b7)qS>$d`C*uv(cL~@Z{s( zi$NE1LVZdAd*I)?^(_mwnO{&~(%^v~My)O`A`o)|rKi}=e&ti39`wjEjn3)rU%nIlk#f? zV*s7#A3o-?z^ST&=UQ`o5uB9c)KRp>Sjw+&EG(_8-pfx^{T(~}SuJTwq%y7uXA#f< z&b%Z^S27I(`}U3^uAGz=0rzJc%Vfo0CETT+U0dLk4P}&rgLn6RcX|6$J3Bi(#o%!8 zhWz;P$l`rfm2f{566=NbAQT0jG{)xvbovX@ei*t4as=@>?VK3Vs|h=LI=XiD0$O>Z zavtOhTd0p(2y1OO+s7}s7gHi!N6B?o3tCpy&7Qq(JWp?@>TQ>c2@4}G@BW;gj$|M} zU}`SfIy(zQ--W+D(P(+P7)Gaz7e-xqHwqU0@(GePHZCp%HZnUKorVMN1jK;NSrr-z zBJ$#Uc*{!-lQB<>KdsB*M!`b}KC*KxA%UW4>Jmp7LT%lp1|S=Lqk3UMLA1l=-wQg( z3TzXiUrBJPmc>C`c2#z41G07xJq})K#A;M2kOa;P%29W{TFRfb!iVS3qP;&8?$j>M zWUyIogMQQLA2Ni$m_@Yhq00b)yE*q0+2PX>@z&G(NaIV6nsIXL3@Tozu;5ReR+CrV z1^kkd)N2@Ceako4qJ5O#2z4kxQ{vxw5;%2|*U%2bWZSM6V>b)tEZ99NzruG93nBIX z= zZ*d|1;MwSv@z}iG9NE`kk9gXm1%z0z6U03qEtvt zTl<|g-5uSpGiI5hwls{^9aVVZey;7BoMwd-{yB5$~@7`mR7mF1QWHlcF&Jn~#D>q{KJ`>(=kamWy>xjk+W z{M+}bzCT>s5+*=FG%ufAH!vc@&hscGB}oJlWgirN zCY*vbPBl9d+Fbx?C$WgpZ#_{L))e+ghm_*PY5EgMKC&V8jnGO#j2|z*6($Cb;i;D=KI=CRq>b92+aL`+e>jbU6=-lwKsqN?(r2c4Q3Hc#6-q( zqfXnLzw-nJ2C~U{ci%XSW35;Q>;Ii50<*_DDrDklVHunoa#bN`8Y9;t=~%LX15^;T zETQl28ThrQDMZ*^Wb+Q3r4Hol#kY;SOjXF?hMbqpT;9`g=I^EO}G zi+?4aM81h_d0($p@+lX9;*zJo)Bmz|<0V9OAxxqakKR%Dz=UICU;~7u?MPl5kNLUZ(lvMekmUlh-6U`;2Bp= z@SOF=a%-2WL$eiz;77I)+&v>n*MS_4M+(*VYCD!k*!o8bU8zzM4Cdd;!=)`#nQ{Bn z;N+Br(Q{#7`#+m(6~U%77BXFx-8$aH5;n7Mir(_&>C>k=7S05ix(B71>fYYo;0(_s zz{7hMGoq!T(aIlm?FRyHkdXZb1mjuq5vgg|MiYrLHMGL)&X%t^2@WN;!vi4rAKhw=t|7kJcSi=M$ znsbLpgkmoae~m}9_`BO$mW>%gVugD}ObN-n80&oiQGIFf-#Mo>Fv!XRzNg;R|J;ZF zvUXtr&W|5-0F3*ude>hm>i_?@PXz#L&*cZ7D%iC3{9W^)VjxE_d_ZaIvjwbX?kL6U ztKErsaCxP|R5~~+vZBw1MZ-8xkl z)&Lv)qk$&z=l`9O@Ev8#BlPX;Ho@C{)>AaY(U*#=6Zcm}LSyyx>HtH2en&!QR&jh} zB(HY_S5A(L3n%RURkkDBjh~#H+))qi4C^)D!=U9Y`ndEZxWf;{ZBkkIb_T8Qb_*ygfd~XVN|UHWt2~nI18(JXXvf zCJ3{yn#h7IB@dW}{pF~c zObp+7afSg;?OO%dy`qxR>yQwL2Y_J)GGu)B7VpCsrl-?-R?sfYM8w6D6zJKzK%@&G zOh$eWJX~BU@BKT07?PCb@%%tE2+EQND=3sY9egR0QJNO(S*NcgDG3RDc2e55sJIyK zU1!{h3h)bZ_w>ZVz%Z(F!C)Zy_%?2)!E-5zBEn*>$-hP0?{+-m?4F7WzU*z;ThT*6 zXkwShXVmI$ZD-eJA@2hMsm}`v`z(Kix*HSDzjkAU@Zn49)-YN?M=7gU(gE6$vb?Ga zq_Xb{*5SwNKXd%dT+%QkMq(3yD&>6M{J?YAbz%FtXYP{WI4*k3K)38!o+0 z%YcUk1oDJ|@7d=lr52mNa{}-W8f~Jh+tSqHlja-{K+kY1ik22j2N=PGa;BrYkQw^q z&?;b_ocfn1^k3DgOEe_l>fiyoUjnYqLHY=xe$pK$mS^?i1tj;B1EYlk{*H4xIX*sA z<%zeHWI_dMaxx_&g8+_x&0K(k0(l$8@V*5(5^B?hIgeQz6%%t%f&$>)spFRCG`RQH z$`-ew)bN40+UkRtK3^mh1y=}oK?l#2bhj}xLtF2T`R&)6%A_J9e5<7+zW{&H)LTK`xt2&B*52O6wf2(Fka^ZO z>h~?5=%ds!5r!nxx(}JW1`oBVVL;I@Wl+@H@mkSb7QfHG>tnVe?KArKM zc8<(w5yNdkd*jz%j?9q|ANC%82(@8SYAWtBT2Od4Hp0QqPVpR^=5B6oKs2`Ya=j%8 zcG>_E$=X?;2to%?c0hj~%F!y1s2rzzk*}cxNH@tl3K-T%iO)g&6^W8u)Txl1N%=tl zhE8Cz-nlcsY2#Q+jNCJMUR_oeQ;YNLPfUylVqy%H3@Ojj#>Vs+bZZ3XwaRs%d{+)L z|1k)Q316ud!K`INUWu`vjriaVzFQgD%F!5S0~TCCij7je4>B6_A9rsh_E^+TNeBj;1bv-1D&Bujc-jfpdck*Vrr~6@ zdj@Y|o-Dd1q&zc)IJ4_7i2hbkhl!z>UA^C#pr)*qlha*_U8-RQ(kf=eDColu{pDdz z1wB+vFo6h*-|oVwLTlY`(t5`yPV%L9%?ZGQi%UKdb^CnXpir`mr3d5_=G<@eE_N9b zDx}o;d3jY-3}Hw>e1BW{N+SZK^r_11GsVN8inyh&uI{5pID^hkxh7w|ob3@xn3rP@hE+dhp$WgEH0-}o zp;hl|DZ8Wjg(4H5eZ}fqV@p`4`%>ekp(H}72v+dj2(D^KTcz9b4Poi0Uqy@97e_K; z`#ls1|qo;zELWtUgPN2tGv)$uoo5bXoTngWx)GaZ9TbrAz9u8%R2qD zBf$creKL4|zU`e4^Co&=W9fiU$fJfx3mv5oP?!2YzO8}>EC^#-Uiyv1zyVhKg!~RH zOBIZc$6GxR{0~cwF}Womx0tkXy|hmTpv{Na*b$KR&khc1^c`FL^9}5cflNvoj~b_{ z>&2ws$~y+oy?x?nM!<{T)g5b0|5&+XK;D|y zOiF7Dj%13>rfx}z%ALIt*vkr|7i2q6VZU&pRC?>I6_uC*`o&}aT=G9Z7<2BwGgQDc zjE0N~4*;g?A51_{VvDw}0y^Y>^*m~A(X2_jJ3%D+&zm(>Erp1V!4_pZe-9e30if|( z@{bd1uGHi@O1Aw5JXrg-|GZXEO|}D;@!e(reebOulU_T$doTrX@c;gp8T`e2@`^D$x+pH8Tqq15-f_6OPP;=m3|Cf6#Xc+w& z8(;!<7vsN14=6#l@GmZi2m~c|?F`maOm5@grRG~nv7-~t$TdEf4mL5+{o?So+2AFb z_2)1TK4Bi7q)3aTP0tR0H{Rjj+W=nJnwTLX2n>Al?g@c(3~!POuBr>f%gl7vE3kGD z{5C>+qjgb==btkt$KBWwO)?F>Z{C6HJr*H_|GADRt2KbQ{rTcUXsGJU6)X6bKY!Tu zbF_7EbE@_>f%|?FaNmauLJuvc=-@-Qtpz8>o{@AwiuA@QG4 z1x>5fP)VN4w*o)cN3SaqXjnQ9r<|W!_#2vOXy_~5)36y^PZNOY0g&+Df4eQr&0E31 z)FNGAW_@td(tL62b{3m}#FmYXk6o))2&4IHe5r==stVt(BnLP1+FPd)p^JsDCu@w& zU%MOWk7;NOp}$bR+{~<|5wa@GgUtg_YFzRty42Z@yWrqC zxNI>3HK+H#&R7`C2>o;#!aM$~oTVCs`lqz$f{lYSh7B1H3kH*{P_wvD*}A?1Q?`eT zkEP4G$TJ$`)R(s9>>obM$}_JxVyO9K*yS3^$$qh|)ZVw_;wbx;s!;uyhYO3gGWUqnuBMcX69|AVK}00Joh?=n3z0W^I5`& z_wnyz%PtDgM*e;bWzg5VJ|-P*e*OC4$Hydb2y4^Qo=$AVEdlhoz0c>P<>gL(EAL4w zu!(Ouvxx{{6}=(Z)6pbSGFv3IopNMkE%+$S%PrRM5ihVpe@^0EQrc@gairvP*uwgw z7jfqTsC{L0SSZUIrc*{FQw`EI(?BDCvsA&)L8wffR0jR3gt+Gvp^c2{7xwlEKu(?z(S zAE+e!@wwoz-^6!DViv$a6vqsqfzNVhA$X|3%Uj+PNkLE8E+WB&2EWyh9n|WD33G|g zTJ;8+Dyn!084tibp8jTR%gOkT22WtXiAOJ_eFi%kyHLS-nBxFHR2F%zasJF~k!1J) z4dq=Owt?<_DyZ6r^>)6?!kw2buQehv`kM(0{ME0KgY9?3jQ_Z?*6N%LL6Ex`G0bL+ z2)zD>r=Jfpo1e|zx1G)f^A8Bn&`_1xSu?qp0T_i!l8cam@CTuAbp4g|PQhF*JR6OG zuEnR=>ye6ITwhQ?@U2B(W=B0_dggbp7Sd-<*57lML^I#NWNcm=&eHVI;n-@?oaI>p z*H1hKK}k`1T~KxC=nvW|Ye@*DW}3>qe3TUD(U*(oquTRB<%#N&dkhZh>u2aS^CHVR4Cx&Y=`>N1 z_fvb+gNla9vxea5XbvsJMe>NqlvTa;e30`Re*ZQ(sg(w#-smif#2s^k>8RYu{6Gn@ ziX_Z=J$1M@(FD$de{WD3Fk!Wa1Kp0>bWq79+>Cc|Deq$k0^>D=kaW>y(u+!`@z^1B z)jJeHXubGN%X9;U{6w!Jtf0 z@mOUF8+k>AB$V#CE<35X8rT?oiH()KX*&ynpA1ZTpRUsNejI_Iq!m0&Id^z5+v_pb zdoRsqNncH3<06;SvRyLdrMRf$qa(t=FAd3BQ(yyj-{0>gI74NXXZGDmQ9rNZPWYP0iLczxZw|0f8xBRHY zOEIixVST=BK?V2YkG5XL`jMJ3c8%oz3M&1*4l^wZYhPP9j^*>K+^PyzRN1}GU<3v-oXc)!p98G&Z3Dv{M>oWLLIXWNN4XiAk=S5gBM3Hfi#}0x5w3BXOFUak z)MC%iT_a0mXDis7e1>5yizx}VhEy+QkwU6mME_F*CxlX8ism!SNTs7sGIkJf_Wo-t zKj?r^e|@u?jc2I(t?=RM44$`o=T}%3BMUnN6B=WLcn(0$20d}d@4uP%?~vAjz9)WD z6|?9b-OK!;Yrt!noJnJkiFPB#oE(3vE2isJbBp#oCqb||^Q%c%AJ#(_qP^$sQR8o1 zljGPWccEZkx#vs~D~P0u^4knn*kd>(8X`L19VooP@L1@mJLRjVx0ba#Uu~8vLlYLX zR{r4C4bI>#MX_`c3%vMT=53GIrcY$x#lkw+Ha!bE49~m#s9&Y_#XNZpI%B36AZyh9 zNj54PV|M73%E#E@f*uu|!~@b_kz9rLPWQHwJKm3kKgC+jJ^uF4y_7I^KTV3OHNc{k zNHzp!wS1)|)%N!mGKOg_GQazY_sI{>OhVaJSKE&Q(4tGJXau4|!b5RfUe*2jZ9J(( z7n9i;&yeU{KDpviY|Z^h1u<8fR~Zk_?!1w7O>>}J(on?~{BNH~f0BWr>61CP3jsI?C__GR{eBYWT|Xne zG8By`W&h+*Lao*yKb5X>Y3iu-=#n6 zN|G_=VxZKlUOr#s0}Ti5yMDz{D7}qJoJ^Fl<#MV^JHH93`b4rdT9kfPe zOoyTw=x^kMPZe-=VA&oi-zh1e(hS-&^I3#rtf#+}!?ZU2?Dw(u(Z4>=?aA>9HGeg~ z@1j!l&_l?U(Jj%4Ir2z%mH)()zD5Wo5u1%t6vO+fj1C{yNXjA7KLP61ZKOTv3e0Hb zeCdOO0%Uz}*6vRZv;+eh6$6O1@UuS`-zW+v?3e1AvRzWq0}mVs7t3%_E+Uh9`EtxJ zWulJl%CikAkx?e-`xH}Nn%ZFE$V)xInAT1{g@Dlna@?~?pN~}aec5=H?FC8$=OX>* zSEpHQ|MRNxchf+k2BQVlOs#05BEY+Hy2GRd;^hROWpXK^!$5uf?!(qA`t)-IfqYz- zfuo^!!}48~)Thbb8nMKG?h-(1S?wj^gk3Q=`kkxA9A_Ti#e%)(3+vTEup0MQw6Ew?X|;BBE+#V^e6v zlknt;E;e$E0(v-V3v9iMM`avJ>tx063YN-|zUU1Cya|cdATP{qZf=1XP zxZI2Eb`(iVqiB_KFNc76fL}#nV{@SOvp}}Ngrf4&Msu$ONQ#^7BokytdcDhzWQZog z|C6qtki3^@YyNq9(Y=QSxpS@sHU&^OJ%zBpe$Py!*rcSdGMi=oD(_H4&GpM38@WP~ zKZ~@5kEj_p%1bA(zYI}Z9?y(0B)%78R2kFfT|sDOW3$m3ihDjB4@v>t+_5(eC$qkU zfuQs%1^B#y+@HWC!pFzQ(9qD)u`FQfkv9yFH}G?D!IU zO|NodWk|Q@G*lt>(o(gakQ63SnSan3WT~LOFJze&XTbffI}< z0kz+^(b3wrA%OV=NN+xL9Viz=LP8h``JWkzw*!xT;3)(eP~LxhR}nvLFo9qhrYnhX z*rb~H3E6V1_$gs!8g`kkX8SG$i%#@sAEoc}AMSTmPPd^w={x$yZ%9BZNmZ;44ZODi zlLuZrDN1XGYZ03qcRMS$UJa=>EOTu-NmDuaN{l9y#bwSquLV6@*n2#RLi=ep_ZpF{ z4YSQX&3A+NUvFDT~Gl_{&p3tdF-DMmIg8}z^;I#}8*XQItdwYAd zNa%#L05^A0X(<6NV1Idec`p$F^Fi7Gg)A8tkK+CN?rv_qz=XHA_rZrM$2nR1Kvy+2 z!Z}-fdnvdWo_JuOUQ<77IzdSK{Zw1E&wjYUpF^lB{U%6l+6r1r)J!U>@ugT9l&H+b z1@rAH+#oY3nXfrepV^JV->!E?-a6?en*03pH@!zywBao6@$I~%an+Z=K=s8GJH_-G z-m`x%;i1jGC_Ey&s{&Ef+#KlQ^0~Iw)6ehhrGIgBRMg%IAecnxsOjkZ7sJ`BIG@Nz zRi7ct%F42{gKv%-K^N>;T^0w$*}Bdf9fVnJSm&~Gd3FG}7+nnb!M!(wx`u|4DiE_` zG>;etIRZogaK4@OUZb1U!b>6m|3Kda%JUd9qe6J}@ySdlhpr{lzghkw%vcU)=J z`2`Rm4tJZk20&K#mS~PX@Y31HN={#;togcL^geqVoGjeE^OK`4I;!XM<6__5Mn!}xmwVYd58l^M z=7bhin{DB{my+0#XTYN58_c}F&Byv-`5 zH9j3_SVJLlh6zxgZ>z0*DVfn@lr|>=f!7BU6BB+%TeWkl4<5ivZh#s3C+}^*2lw}@ z@(2qHUmls<97WNH4Xga|hps9r^wsJsJSXl>W87q6Fuu# zgVaaLD{FvTLR%omI3gOr+;MTaY8H4WWPLRLh0MaG+x5Ts;@(~CG|Nd!>Nk4}0d3vcI!ECFj)RTGu11+bx(o&>+rQ@_Lg5(W7gbH zsqJ2y1oc?DN1)P?7xd7|K6A;f)L-p`Q=OvX;{fZ9@IuwVFC!R;>x<@_i@>X~z?FDO ztaUKtqs(uG`@@|Rr`qc38UK?R=>uTxeGY6!>BOB7WoAP_n5A;xafi^m}82-e;X@_A%DJxXbdJ6szArHkws)8%af&CAL(s{+!8s zvGycYZS{?%DQCbwkq<70(Nm!DZ&}|MDVk>|bA$-b<1v!c(ngK|o_eV>Cg*aAzeV5w zO*sjJvHg4AiR*h^ z*pNNf0hyDjx$A`5hXB!Z)&s=*a0%%DcZzh16qlANT}t?!?)LWL%}vhC^qsVhQP|U? z7XjO@;$ohvCr{W&Q{ducDGEW&zR~2R59SJb!A$m>Zmk^=H$LSk@$fq^vR0Dw z_~z3odyr$btZIGtq`p|`QkJn4hP?t~F*+C@vTj+~D z;!n3!vdRUi{L-;ybkj48k7Sg9y;B!5;6d#^A;OrdP85QYT&a50j2#c9=5@cSQ6a~{ zd~iRrz8+m(_aag88Si9fMGIV9_8#6NvL7_sBHl;$z9~rGlX?2=S=Bbi9mMD@Ka|Ac zm(e@+*Xn9C_?-7z z`g4>123lmrZX2oE^4gj?^a~za3))*88YJrKAyE`d3$#cQu(Z|g2HOSF- z=ZjE@JfD@Ml}H4{maIY|g1i2n-*Vhap0$Bb&ahnF=tsg*U|BJ&=11k3uJcmgl$&8J zkF@DvWrh11x2o`G(rOP!+<@H8*GaO~c8sul&cncouMZ4p-uV6dXZb3pBp6_{#8O>T zlg-)OQO0`>MKPQ+P92rM{TYCc0*KhD<%Xt?&L3fhbA_xH3mrTsD&*}q1%%Rqd<)vp zQducFFfiz|sBW<2{aiqyr_c-&8Pcjd{!l7XvJ|PrSt5VW^74LcfXlaAYN;z@YU=|P z8nj?m*G}WfESRD}j>N^q%?vnyIlb-Zw?y-Gqt$=^%LfK3_uJ1y`EzPcM{Aqf6YJEwvzmtU?SP$$%y0?x5KFFw{!4^K`chF7cr=_5 zqXp-$r7?h&K0@Zyabv`i2E&@(V}8$D=XJ2~Lw9CiyWMfy8$SL6JmKA-Cx-P;V$Vx0 z7Y8*NY$2FTWPK`#Q+0lct{L!=my7EMaDrs!_v1LZOt$#5O9aeaj8n>X0?hGvT%lzh zSV+GPuyQDhYaC>E-E5Y%^5>eS(B|3@&H8v4>Fwh*EkYpnhEYI3;Ny`q_s?MoFe%JI z5g6fbn_FA2188~jfljI6w-SIgKI>d&ar{5XS7-6ux-;$P zfupjwVXVCN#?{`0!I|0M^B|3Xo*U=9Azy{goNiHVBaK^clE=K0~wuZ6qmWT+4si)$H9seC)4j1VaFoA=stTzb_H6Fn) z#Oi?TH}IvO21Dek!sV!R^sU*CVX`KyMt>m&w2b@8gMquD}zy@0j~w_u_6iI0F}%i49K$P+|L~@yYc&qgKF^D@;RQclAr~qvw|4Lg!IyHo}=Yjib1J24YIIJ z#sPj0e7ImaKNZK&4t1LL$yM{f$z}*BYilfb=DCpi9S~=B)maVDXY4P>s@eE@Ku{aB zv!El5w5nC}w0nzPiBOp)CCTWxxnrgz6E^O=x3zIZ@sQU{^lU$xDY3 zGFbi~+kVbdwV^It4Z+7Q`t)gmRSU!VE`=PW%jnpcYpEd4;cICdZhT}n)WM?EqHcDa zH*lfOQ0}6wGk<8A{lVc)HDv0<*(Z}!$@tK>jWc}we= z{Fz&5)O+VS#mv(x*AWt>cXcVMucsWFqokr4TRT4#%AnpTUQ)n<&ydtr=?GlEBuY(8 zbL8bm85Gj~x&ex~eMX$&B_zYcg9qtjH*XDB&#r>>J-zjfpaJ&!9abJU=;KeFw;BWY zPW_Xt@OIx+iS+TgK7I)3b{d(zl%)%?X*CnZ=t7G85{%U_tEnqssYgQLA zhGFgQ5G!X`qZHS3+vw-k?y}0S?6yZzdAX8%UNu+BKQ^&Cp&na>YC8Y*3H+gMzmJ* zNKVTB?Ar7Ihe2@Z!UV-PYWXbvZwlL3LJ!8YLF*=zq}Vm z6jz<*C8_2o(|>$OZlE#K0o89T@RP?y`Q5D|K%TuK+u%{t zIoX=a`~tzPZ{Nkv8BTfKK`oR((QL=ks`fbmFXF4Q0)I!Q}XZ zFGl|no?KUei`d1a+~>eEUysxFyf)8_=O(9*sXBLVL?dMCzxrxz?*DX{9PmNj?8{Z& ztPO282W)HmB2|V;b!MARLaPPQ|8F~KNAPyl`9f2c_ z8a(zCt9rdH8i9E9IZ1S}EF_BJnlSKGf%X6(K~KD|vD&`6@!XG*-pAg-LC4+8ocuDn zPBdK(_UqgykF5e3$6E`A*JXWvH)PL}&p!&c^YwWpY@V6}6Dq8EJ7SE~0rZ`_Is#`nbR|N}0i6maLINpN zDJH7rlY8K&6GB8&lmLnrIYg89?d4co>xW8%ys<^QoxEGs{;DDsw!_{{sOv*=+s~bl zC`ZAYeYGb~iZM}_OuXbs2{EC(lF-nO4kaXBQ|%iz@e~{s(#rVGspB>WiFDljwN04Q zIwVD`NJ&Ylieppu z4%?jdJ5Z5zXRQf-+k!&OY;QY&268D#QXf5fR31T$OcwM@zG2fqdcS!mc0odBN*-8U z$yEr!S8de;-KmFiRUqVRiqaDohqE;4if9>Vl`Zut8mQ+tditFitMS$|_iPR*j(_+) z`itdRXiSg+9vh}o1BP`~-5JD_90{;LV9^9BpD1Q5zN+f#J-sv(4jRtf7w08l2;lz~ z_yl{^kJz-+Er0)cyffE~2{$MAJ|!NO=7KMMpU?GcWs9D)1@>w^dD5YHQ;MUa0OU5n zT(9ufXdpvaDuwJZ7m=$kh$2btXHhc164=& zaM_RcnCsOt&$^T%5)%~WGH&`$>;f<5(iyHF-0b0gW&R!hC_VhK<)ee~DGzbQs z%^R|);)4Wfb$GA0gGFi{k>swR&4?}BI|~$1{UWgN($Dfo*%`Lm`S6H{pkTHv?(aH5 zK|vxe;`7mr&*u0bqDvLV#UhltdwQqxB?f$qTTQ6wJ$sWrUU#LOPX~z}U!Q>y7o5^0 z(S*s<^@BV=h&EZVTT6`^+y>Bq=x{AFaQ48)e+XoI#opWzJd`^U%rVzz_dkjh>FJw` zDn3lkVKI8SU?CczdtCnd`--;mm!*s1<8x&8;>VB@FoCp%TD9sU`RS1PM;b4NJ8vJ` z+H#|;?ChSTZ{nG_TyL{x#Kxp_W}TkmKS*E0c!w@NnoXjKNxm!l7@k@m;RDaPgdue% z6%e2V*3y=Bc3e<4?&Lc4I!!a8AJk+!Ho1Pqt@TL9Iq+{d;q`qXa-*>0L%&+b*~_~< zl}A!x^&~@4Gf>gfr9@1bWILVjg#p>8qDj3pMnwwa+v+4$%9u*N`;mzgcE-NU(u5#E z6p>uB5s_HPmuF8gous$sg|iQL9bWpR;YS^J@)tv_pSgRNL>JOeREyE(e>#fTw*MjS zx`>G|H&`BmG!FJCh@U4?JlUb8*gh4#IFPeqxDjudsBM*$npn(UHJPRbC4sUkK=M2y z&HC&=Ucs*ifhQU!DvP4665V5`Fyz=BFsw-DCYuUNH+NMnnh|MSw-b;c$i*q^n0#B> zbC7cob~-i!rr=2@>qA9283=T=w5o&l&c7<>l$6+SeR@0kt^i`c+W|=dudQ=Sa`soH zFPSa`{@DR%N{@XJ@;L_Z~wZ3#}i~@34+>SxetD6KDU219#YjRs`F_L*v7lZR3Z=eSmq@q5B4KjQL>slvVG$_2 zr|6Gp0}-*Q6ph;@4HDFPJl5oF!d9beW|(eMV<0&q8q2_7 zURhPI^|2=~I5B{tp(L4fMZ_x0@8*AXRvD%9EH`SR0OyUzhWdUC_A5#+UrIksj8^&PUx&wnOvhjTxY7bGb)9eF9YO zAe08PEp}g883$gu4;KJ?a!^+zLx*x0>gv9EB+ATe23*^#t3`Nt>?|!`tGS5C$XxYH z?I159Ir7UKau5gFl24z))!On2O22PAFpz)%p8&Ow5e7xQ--b{h`h6O|whKAh{=VwC zWq^Sk05(A05*?M9FpolDWd<6ZT>l?$Ul~?qx3x=1w@8g~cb|#xyZ5*EKHraXUFSRJr@~syXFhX|G3JG=%8$-Ioy*^1RCGg5}dJ8Zl+A>2OT9Rjj zIDuNgm?Ao(uak-h?&BhXX^|jmERhxx^oS2?{Ol>bczXV(P8yw4Asj9wEI;2sBI3vO zo}u0a-14&3!To$6NgNy^1PJ_HmKOoJd-oU5W?ivaN1rmK1y}xF6S*XXT-smC5LI-g zwFr3K@hJzW9p!Aj>E+^jeQf$=ER&js2EOm|h~YyBtn162o=DTp&j1DEj_&G7JrSUN zP>$W-+)Tp?Dssbul{`h~70JVk*3^`j;Ko-RsA|IOs!79GC9gWP&W+_X#pW`CI`3#9 z!ZUu?0v7Y2r=K$Yv@dTM2`!qwZLxKk#bQYGYB18mqq4m{O zV3NbS>LIUcrcUdxhXe)ZSm6(2unxcon#mJP2@qT6@s54I36AX=h`QS_#JpSNq(={z zGx?Soca&{)0!Y&VNxcZ#?>9VT*fqTA$wVkPS-gf8mOt^ zZ#^EnO_iD6O;>oi)N`d02DEsVxrN~0aZE&42RJbb+4~@ii2NEzsD6HG$Lg7x*9Tv0V@gk81FIIRyj8yT3 z=~x%s%+f^gZ8s5O5)%@(RGNdU07gdfOk-TH2d~|CRN<;>YD+I%F`+ecNkix0{yB`+ z8z@TaNU4ff!$&H@g8SQkvu|71tAPlS&Y;xRN=|mB6%l%$zia=LrAdk}nUW93MC2~$ zhwR;#t2hvNY#x^nk-%(3$w?^0l3@-#X2k1XW>N9ht*8ZQ2V$O5644XgmP$|Oc>Dq zJq_Fk-O-aXz*Ve~(f6`4oM3R5$o2Z?rKRBV^72Egu$h^IBbz^wW_lB*2-kW^y{S#$ z5ap}|-g0%U7DPbhkYbB4yPMErY`x$Zk=1^dsxDZGl*e?+TcYkSQ|q4P=rQ8Bb){VB zey63nizK3a@^+G^!tp$N-bHApUSVrXOioUkrF?C+V%UptWw_$dLf0-D3xF;0R^ zc0&W7KvB+|_4**U(Q;wHQs52kL(r7DWMoDt0dJcF!hrbDK3@KHgr-ml!~c5QlK3O8 zi@52ks@z_wTZ0Zv06c7Tf#* zYG9VweHHZJE??PF*IYR2yJHI*lc-H}n0Z^P>rPGt%w*NV@oSO^eH#zzJ~%sY~rLNzClIPr@yH8n8?M zn39N&2frV!?Ehtbr8hd4Rj3_^pNsd^?>B{*o7mbOj;re-h-(z7-SsIkv(S;OhS=4E_53bb>3pFQd+``H8KY>_bqnyD} z_%?$UhN5Ho#IO2KOadrV1g9C&39B&QA`&B-rEg+^9<(?BNhKM37Z-Kj&cJ~q`)Y3? zHXabfK-jg*?a=FxC#PH7A_*s@F6O}EF<~es!L@rf9EzPd`NHKwL7?xkP5S2dGa{Q< zg68XqeyC z+#+E?r&%I(WJJ+O2>nf|@D4b^!9kXvKDDtFo;B=yg^m*rf>NJXUJDBjl$e+p)I?=p z9OjcgNo&BIXW`Be$Egk@(Wj7CxqbR%9Wd50MIQ4n|p;MJMBNK-NQrVM}`kY-tG?- zudQV~Ueb9peeMJAe>rgIp5Z!Xo~u&&@qwgUpAeh_nw#XNjQ;|(udN_rFu(L+YkJ;x8OI= z;!$gn>S{(=DmPvS5U0?rrTFhIjcXmc`;fr~N~E@LZpX=d%(rg40F?3r8v_ukns$OH z`LGwahk1QR-pR|$BMBYn$xRIgRS*j`A9j)OaqR#%%9XXk@E{v^QC>e}-!|+_K1AMSqV@>-&Z)Q3ESlfKmL0Rl z5TeMMMH_)wy#2(uc;W;xuS{<*!*{)iv7r8vtx~wL&CM19s7zyHV*vq(;bXLCxoHYn zMQBhENOM=XHVxPX1@+LNEfxY@A~b-n0_di{Z&Vd5VA#|+p@)TEV5X_KNzpccDGOeT zS?FL1+SqS@y&}57f!qrJwFtdU02)@)g|n|t&H_97+nX%}tDn%&XbEys2`@NbXsnuq z+Iu3sOAtX0ChdG|kB^Qfz=?eq>m}8NI=mTn?Ovt90JEP667)s1q(>*GSKSzckYP;o z$4`cTR?-6k5YV6`?Mk<)!cW9m^3+m&*sGMtvBFkYcrJhC)zG>8q}A=E9+x^Y&zjW< zLr;)3SnnyqQy-(q#iFC51G%yF8;l-3j7#K@S0!M+fymz9*Qa2UzP7!y!{>3u7VcY= zdn1(96OnJAufGTij-J@4C^#PV;R4J>@0LCQe=;{;l4x%Qqs^9yg4yV9U0zxmqjCfM zC#)>8wh;%9cUNO+jPASI@ZG{mJl!ae#_wS9sB%QER`e9vvd!VBV!M3fqM{t5wH3z{ z)lkzddL}`Bk^eYOig#6#`y^0X2gF^o_seME`%GTwnc3NEK(_Gh;_{CPn2koa^QBi< zp=&_=KOF*rS@f^&b=eV~g7t?VEXdFQs8RWN5_7So)&V9=;p5GPE!*^Ao`VQi1xN<3 z?ZCAmDVTvTf)}=b?aIIBvUf8oXU>e+;oG}7fVWv_qVrop@!vf{CxNVdiuM;Fg4-$z z@RZO^a(Fh=#3{+8_r7G$u>ze(eBC;ocv+a?b6>EaK<78p?^;?~0C%9L>J_Y3fSL;D zEiPqYQNR<3=(YOjAT9?SqK?!072&~UZox3Rtv>EOCEgDW`rK&eu+NFAQP1g@;UU@J zLQ*`;A03ozZ&w66mtOTo1fXK!k-WBipAQOEtSl1vxiS7?#0l{%YpT1HOp$oDDgVMO3E z8wbachYcVe{&TMue0bki%k=9E;G*IT{q|ULWF4p{CBbQr&;6%srJ8tG6=gRC<4jZe zLTJ8qtFN9Mr{WeircIFGr!;UI{rK?@Sgtcspg3<{8Hh}By%r0@v&HOoYdhgGc+m@( zOWC_{W~x~_;a;kWby@=<|@ zw|;oQ`ORDpmUJ?}I!L8lv{S`O$Z1ELuM%-g2d2me&NQ8;8r%F-gRf`~0ceUMEa>dA zG5w0{#J7qH93oZ`liKZ@c|=O}Rb!8GZ7csrd02dcLn^P&^A z3~a?0;X`$zy{dEDTwk9(GDg)9LL9BwHi+cA)Nzk7(>5LA`H6aoBb3HRarjl9Zp9$} zaI`PA*eLB59UL53PEVemipL$05R0GVXSEvT>1w;brEW=2;%<1CkuD`f48V42ZB=W%k9cWgPQkkv=w8LPx!K z$;VyDLKDP}gCEz~$9TI!HiH4PP!Ieyon!X*W;3GP@evI##TA73*mceIIKKE8RuNPC z?C>w6F7uEuHR%EHcK5^5vPOa=yzt|RXzmg?m7Hw+#B;<*K_;aZn~$|1^WA*ohJ1p0V9D0f(H`pixOivGYVig5I8O4y{6_B zm`rclts9xM&%8!y(*qcjOmGihrrU1;hUsK;Mf=ddh-$WI^ZW1nTVU+#I-z7J>m(%-+3Sd_a@z43DfmicM|! za%F*F{8AG^a?&nyDH`pqcJ9xgKd-K?fVT)Rh^e2GgK4a4z=ZNpc!l8=y>4W4;Rjvs z1I_*KmFb_@O7gjz8yNYkxws3Sa;$2#sS1YhjmI&@barnm-vZpL7?2bW3kw5UrT8IY z9&bqgdXiFYZ*j;@LMhuk;VbK!R7IueWOXO@`V~7FJMIq!K>AlvS$T9~f`g6i`z{i& z8j;n;fW9dyZN8zu-V!XmczXNd$khoA3K)DP;fue(Br!W^=^HCqe^4=Zvdj<^OWm8) zwRbH?7Ga0Xvc?-XW>2|#*N?qXmYDryLzuG;iV?I zi~Ru&^tWGgbECMb)u#cj`_{C%cYtaBlz-#SLi3Qxt>yHn6_{XtW*Wcb8tRn2~)aJ#?G8upND9xMStnS+(puu^N>i5CdE0CCV1kjy)zm`}o9 zl5CZyqDW6p8K_E^gjM)IegwNgFBx_9iVo1lb!GrUO%E-GrkR9cA}1O{7Xu=TDsnzJ zh2w(lw6f58)uqLI`}7F;^t71CbL7<2t84yTp!@cGoAWvb+eK9I5x+&659PxjT)_jM zt;J!fg&jTBc|q`@Kdaz1&$Ud7AzBx&F)%)G@-7ZHKA-OuxTV>t)hnI1yjAMPz8^x@alCy&A@aJ>E-k!bMHG{Z`)0~ z8iT@PNmE>FD7yHv_}&wb&y$tTi!T+HsfWqOa^``E6z|N zr=SPjfFn6JScjHB+}yogB(owfN5Sh=kHXjDX-&~P`!M`J=bJBP7NpbenXiB{E8`e9yxna0Izw80FJ&E2f^JO1L0SO4CWa$0?wR1z6yTA=wURzO5Z z`<)qyPAOr3>9DJ%fm6e$uT1hereRp&#K$`_5eypN{qY~@;}7fJ>&FL6D512v{r*P$ zA@a%$AlW185VhZ3;O?Z_{P>P5{Z{sj%rw}Y=2^)f?G~9cU{7OH0_v02=v8U7-eg;XMt&b zRBTek^wi-o8|9Ilf7tH-99g49%gOyk$*n^RE3BQ< zZp}Mt4)vm8$-jvAMn41rwbTvp3h}I^3~Y7`nC?wJR0>8Upq{w&XvMvYbW6NH^eT;T zk5BOwX^z=J_#-3m&ktcVl}32nWbSx*1vegP6W6|SEA%r?GrdlWEJ2r-G9v0u&Q1q!)~^^tC2KUkWzFA+)$pK z--q3vy-7$4OX$@$__xn{bY7?F>s@8=87vmuw@6W)YTr(e#z>>>Ztm3U8G|YYpqZvb z?wh^^Rrw|QGys|ZKT8G33IG9y>1sZufnsvKRJeVMds^IeWkJKZ^Xu-DpIb5m*>En_ zYc_Wjm#}@!H#k-(Gf9)D@^V+@%UCr}1qs!~HVb1`MHD&4G3t-2<gWBp_2$(lam^h~ck(+lpZe#3*^7?dQ5ssFT8I` zMpAPjxD=T;LQ0FK!E(7-%<4E)y6mLhcT1ZsxoK{$_{OE&In=};|8p`jU6B5l*PjPy zG@En}V^gMX!0hhTt@gm2Ietj{uG2x=;{MWuGJ^YE=FcU>NzmAu=ozMdT@rBz+%bWgW$qH@J zR++ONrx2Gq>36r3l>eqm@v9(}0uUaE#w8y8OiU!)WGlI=)A`TeAR0(ZAf0&l4PtUL zZNl2io9Na@&>5fy@<;c(*XV~jhIi@^`E0i9O2)n-vIp z>v25%>yH}7uY$8j5E_DarpKKD0)Z<~o9P5VkTCwUti>Nb*k~>fbEOr1Qa&2HkXo&F z`nCfKvP|{Aeknif26xD|Xw_|#VQfx&5rZ#goOGIg4!}&u{p+u(!*5hkN*EMr%&eDA z8NW}}av+z%+v7Lqc9JDbblS4tXT}k_{_#i0q9* zRt_3~4%y>WAw_7HCsMze9%%$Dq^3W631~?@%m665>cvx-x~CZNX7qx~?oPg~f^l8@ z7Mh~%tfTG#(A8WV{cQZyNXmcnNd~x({IBz}I6?!fbK%)3EAgG;s}oPI0uCbq}UcBIN4=fC-ggkg2L z)sqV8W*!UvWXgDJ0jVHM&aM=lOu1g>LPG$g1iftliUD^dmzS4vIpqxv?%-yfj?Fb5?u1}0|yb-o=JDNt&@8q)^yfJ}c& z>VS2!qq+@K_i2_$%>auDudMXFz2=T(md>DWIJk+|4a1ubNaj$KV?(3Ql5^+ReQWU* z-?AV0zjprq{Tm=XUWilySu#s&YeshVO3+c~9(CQ?*Z^J&YU)6c{u5C8jQe~R$E25< zl46XCO`-lsX#JNt=$6sZjWt%ZF~V$vR>v0SEI@yH{Q8q}z!9?7s07@mnI8Eq|2p&N zZA%JnLsP6;f2MX3_6R`D{Z35VidgUrK4$C4MO570RIDy6u-@}^xR$*oP0-n@DbCA# zNlU8`G0+L?>gjn0nkq1w&;u|*Ku7GS3*vj0yQ<2{6fWB(fNlcyu9i1q=0d1&EFrc83RN1dRjY9zfc-lVNrP z`6_$@0#)y(kBe;s0|UUM7$|EEn>;d0&Z?|b)uHyreE+X|l}rdIvohj`5PaE<0Sg#b z;`ldeH5Tu-HCJk*0lbL%85iJK@!hHvdrXRLXBk|Fq2z36z?nU@)YMFHHDbhswzjsK zqXuarK|#9;V65F$IsJTnnZmJgaA4md3Jnhr0}2AKI1c_ru30sV6{xmV;Wl9YHJnVq zR_GXIe^`fxjxyGI1(jxKyY75Goij@Jvb#)Z4c9jXBsv2hEIHlB-ltP#hSAZ{(k?ta zh)77>f7EHe4OoZ2?b3r`RP$*DV!z=}ksES|UJ?-ziF*ftM#_qX%lU72w*R_{OdciHeK{4mE*9m{ndd-7}hx@b=cj_KoBBosXB_FSfvR1gFqJ- zXz{H8QCdJ)VZfKRw%&t6p#VQ#LqzDY)DavRiL(+K6a=%pv=5}aN~fVvY*JG9!=+B3 z%Lz!Pr`~TD(ICeU9nZ2cr|ZWgix=+5j$k!;c-#)A7>P+p`1X&ho^p`@Blv&Yec(_B zwO0&usDhHKT9@p9atiMwk9PL2D zNfq*u0h--J0lSh?3LnZz|47xDzKeTjIMCMnkH5A-RBo^Lg@IPtnMEu0=yF_RMjNH- zyVhoy^I|tmp{?qxHMe=-FYxanzZ!;t``xOJl4t^dnRwxTG|ht9#;4@L>id=Nj-MKe z>^nBkF56P5{v(rsv))%7yRhNfdt+4Y0ZV&rI}RS&+MBC$WUG#A;d0@F5=kDgb%k`>*JOm z`B{Cnab3HBj&TRj1O-7<3?5>N46#w9^4&$F>g)>X1r=i>Sb#vssrM-1P|ky!s%Wet z?TZw^63u9`HhW_!Xp_T=lNqoI|)w&CV!fJN+~(sJj@U*$$LeD7LW)Z(4V zpN}hiieyvH`p3tC-T;mD%1LGR325H%+j?nlod`9tXgmz~LCXrhHzcz&n!C&N^z;Y7Dww~+h97gk`}nUyt?{oy4a^lLWO!xT z%d7WnyId{hSc|&1^>IG+ayCu7Wu$RN_E$s*$j&|=z*s4 zUu6#W*Ft!+!02DpakMOeTv_%$Zlzhnkw^Xed!k?s?3iR9aICd^PmEs8HO}~p;hD=e z1HIM@x(O4rjkX&b62e zl&}6dG1pOk6y%k&j?m6K9Fs>9IV^Tf>@ zwtufc!7tiiMZDLi1F+*&2`L-n`$AgEYnLd;8o`GK`Tu;2Ai1Pi+|MCwgN*|isc#Gj zzKAyEp8{xqFndPlUY})D=~ZCA8Zmt1Qqo)CK}Rr3&o@)R4ntgRGUlMu4F5rPP$q03 z5jB)R8K}cOh^vvlolu2DiI6W@st2(1>h$Fs9A>Yo3Ot|gBrRT5t#-XVofLT@CN92J zT{H%?JmpjQdRdbOP39&5yLM;CQfEmG_~DWQ+rxR?B3LL8S)=kLyCQpQVeV?@Y{I3!DwB5v8D>IFky-&Q zqoWQygaJQ+cYAvq;6chHqn;NoF~zEScbVJR?1NamAdT(U)FjZ9sSXq#Fl9;*p(A)p z4>`7^=95CUhncwN9%a3lE0r#@$!~#px{U~Ft(U;?5JW+6`v2?6-J<;Ju@ahZi{~XP zGgjWOAMJJlm!8UG4P*2Qe2Afe0nkn?I{-#RK2c>KfqZLBzN$AP6BF<(e=94CKx>HS z1wfBLsi#3cO#q{zE1aF39e8zuGzloApg5yba+7>mNhjtox4Mb9#qeu3dye}HdOsW^ zdSS?P>E^u#3X z_W_!Dde;e{y+B5dj2r=j9v??QO-oGVv0dtDY4PUM35xs2B>M0Etc7EFKG}$C@mfF}n`_>sOPtLecKwq@}#Faqc#(^vCi7kk$HJ~>4F9toND zW|b?W47kUdrA)>nY0bbhNd5$R0x+#0nG!JR0(oP4Iy$B5BXToK1am`Ggbh$U;z8cb zgA2*ZFI!t%rH;V?}Gyaxk+3?yrkjLH1cwCa-WOPP*FdI z1N$>P2vA7?>jP#97Z(@Q;NEKhtFZ#8iLJ14^2-RIa)lo)fIUg}4;$)Ru$p?&hb814 zPqU`O*I7hhW!EEQNc_ex=V|l9Yo8_*`V`K^n)tm7HqhPTou$!4C8uW1e(tJDxQCHb z%zS7HCFrLCeMb-j3xu$~qY9!{x!;#&mFheebrJ2p?OstWqCl^D_cCVPN3Ia3?CJoe zKY{OjRgcqA+{VTRX#e7aBGIRFTGrp+->KgH>&VCmAZI2GBC3J9Q5d+?8X8hPelt9! z1-Jpk#HiRv?(iu-BR$bTbo~4j;6!_UHpgLj8dTir2I{smc6eMYVMtQ!8|_tpzqIx! zlKg>emvdSR7n!ponGuLQIf4j7jFCh&=|@G>mim0Am2I(+4R$tpgqGi<^DJ6yRjc&e`OID>{$omKl9;;i?kEHXRfpNt%4r%n2om zFMg=-gjV*|%lt^#lbxuMaE2mO*gIw5vBAyUNGvpY!c#o)_E7bQ>?mVkG=qkw+NjbO zZNTjwS*t=u53@;)nrraA7UmxXlTq}o^zmV%18hg~3>|>bOYxGWWQK3I55;Brr=?YW zbKILXMFIC3|JdndY4&HbTsEQG3|OwQA!+?Jd0s!cv-lz@TEJ~MQZ{H?zcC8=eDe*S zo@a%IJXww?+~te5Zz>MeS6mE>)FqepD*)_qZ%RElIJmpJ1O6!<;o4*O3-fM*Vr8fM z(+h%o4T358%8pwesa8|No-#9}%#e7YH)3xd!4tyZiQ3^mUr&xqwig#|k;U||UdwEB zlZSuJ!_#_QT#%bO{vkVOQfqQ+lW4`1Nm)q=2yle0Zvstv(|0H3M;LHnRD3**6C6Ky z*lIbdY43lV-}0t#-NAB)H1o%Y(ds1LN(vh|rVD9vKQS|qC%g~U*;<-6wXgRGlPu8_ z$0bwy@jcCv`8^};!jM<-7dY9S=)p&>@9iqny|S!vdBoM%@F!QDEnSOT%v77bp&Sjp z&9viU*N)2o?H3%3GB{wh35*!eDdA0AtCOG}ztX35u z!TkDTK3AqpDaeU}q-HX5qjf^vYV@0?DJTgMv{^zH=Fbokr%;nnP72zxqFavx(}t~r zBz_9tchp5T7U|SsH*Fde+v+u6y&6G10r)JSFG0$~#`Yu|Aq5!)`ZfIk} zOi3xq;A;^$Thk#%hXe=vH0s0B$EKzhTRw+^(-BZdnNkG$DVCOnPrzB~mENPe%uK4M zPhm)U`}*42fn>7F#oioZclSFMNyzmJwaorF1G?Dj7ZdlKJ3C8PjSh5#KgUjsxEd1e z!*yG7P4s(9TiUex>Q}Xc(KN&QS&zL)ICT@R>HxqpARyp<;}PCI@EC?}PnL`Wwxz(y z>8XE^ArCcmR%4@1ALWKK?)ZH?K!4H*6k(y0NVoyVpZ(WW@A=%^{;@JFCwyK{gZcCL zp?TY!)Ns+fKXg|z<7L&;3*o+$_bUm(_j0S*H1I;SJeT$ZOs{CPXyB&=?zgbt1$_$) z?2e-R%6ijjz|1kw*$(i-&QlHqw8{^iREbll1gqRo^ zm!2+Hwg<5MY_Y{eMU6rzMa8u7@)E=(K%{dvdk)1u1c!JrDkzr&Fc$>X1gIl^#Oppl zgflZTJ|2umkTHw5X+v~OuK&BavLV&zwbTgsYW3D-y>DQ)?u%ns(Hc9+u*Ez^GL=Cb^h7Me}v@=oJ=JUeCF^S-M4P1S?UQc`F`Q!ATliH(J z#EAI3IO{MWqs_zsn~5fA{C+j2auv_`gNv0&th(SraD5$6>zO*cFFy%@BO@XT4g?JX zD3f!H!3DJoUxyx-nw%Wwz`(+S9%H1di%XH$xS}FtBngFWR*q$s?%?9ku=95i&}33a zhij~<30i;Zwz;?0rTcz;>Z#iK;bA1hO;G%EsBucaPi@E@cL=v^e-_y*aTUiB6{jMe zfT8yEkm*?oQVXrVn7M~j1(Uqk$sZ!y%H@{4HG;# z?uHPO8^?wak=51AF)g$M^Op=nJ79u|L1e6D>V#O-#mTm)aM`{$;3Tv}ID0*Af*4UP z>wANJy*gyo+9cQ&tKv%gp(A)u1$uc02OrId8t|2PfiJrz=<~!E5Z>V_3kktA5QTvD zp)fX(7XXOcwKahS@u{N zb(49#aoJf}d^o;$IDAi{l=h`KCX;Dx78k!XR4b^Os=(IeVmVRYp_H6d$k|FlB9N~m zD66Syegm~4RQn^TkUIl_268x(yV`6$GKkD03gXj;+O1#b6qP+g-Rq?=ZI9bco!ADpA zH)vU=+bT{a_btZ<-Q%wmkClE<+T_RGX(`MP7hW1OO7)KC?s^6v6!^4zoGyeJz}6x+ zru6I%UG+@20N|z&0bE4G{$t;y||v=r%ZC5(-1G4N+KjfR0N%TNfdr9T2G!c3LGjLstamI^!7m zB2OC-2zyUXPJobC-%|+&zi>lKL$;@_?dj>1GC;Y&&Dpt!fo-drPZk4Vx-VS_*Uezn zuygGFyn?p5*Rl2wN-DBVd&5qP(U^Ze^E9WuZQTCsy_Z+Y(pijuR;3SC0cOcsmXSzC zKVW4~cI8Yq%!B7nm^8sPa~tGDX4|Q4u&1tYW+`*6wf%Ve*MKP;DIbHQf6-O_|UtgPo zPqn%5V`J9RRbS@ql^uJPp4?s{`rL7aQ1-QzCsW5wxlD4Ee&lze%yWIYI(HOFV87nJ zMuOz2>X9v^&HuzqI7ry%g;R?OT%*awX9#4T1`=x- zyexYMN7wM92J-*?r)J_HLOlfKe|`k~+vQ&;0tWfyJr=^W!7pKmX#g56G&|s5XCJIn zt9_-UtuN-U?0bh-bX0ZiJr>PO*G%B72xrEKIFSZoiAeSmi3dh0WbjZcjKOLbeuBzY z1|Lh)PjhYWO1$9E5MomjMkRcDSAX#`B=<=*r_%mu1_=!sd# z!#g(nGpI`Ndbu73^)OC94c`A3KQuA2x)CvUDsD>ty8jujh*FT-xXSCTUUAmGj&IXJ zJwzy)qW!vRC}G~y;?fp^tD~Y`IT>@hm!okOn#4r*d>-O@ z#jiP;Zw`tl3rz2>H2o~UV4zmC2jgC~6>a7gzS)Jwk=Tx|PCIXBr@+l#H3hEcEq9-m zH=&`jU9xmEV;1!Ejl|i!Js34rPhsRqv8R`k?QV}5IWK{SGZfL?j#H5g3-?&E*VB`0 zA^^)%@M@l&9y3DUK;P`iC}GL@*aQX#+j^5^*FxnX)@AQYGheMfI_84) z2l)jJi+aoSf;xN*Ts0=Sei&)0<>(ath=x5e1OzP3qH|98Q`spk`wzAB5mpmx1o+wC zWgX8q5r~t-Tt0~7pJ56*yWYbhB*q66p*ZONobhlw%V=tGU+rG$DZ=0R{ zZsZ^-o+QRihwvg+q6@fFW@x`TbvgT;-X!v7we8~*w+<(TVC(uiA_k!W7!ZRgKFwMO zt+j>nThAu%Qt&wchM7*Oh_6|e2$!q)gm870?j=_4FF;bxXEQH_j1l-dK+x-@cFVA3yc1@Y)d>P}70FV|%stI42;v>Z)wo$#;<;ONxrLfYZ< zlI^+3h%Xw78c?wc?2b-$$tbn;E#SiLu6xo&!aB2ZJtj-uiD9&{vBBZJ!4Jap^tZJP z@)COYmNp^W8fIS>~|@hdVwFC3OY&vv2TS?YY5(?isq7UAIkc)saH& zRg2rXQMoxv!al`WLfXhDWCTO0BJ!}6yM_6!3l?!D@4b;C5j0x1V|nZFPPo(=xx(10 zzX)+YiGf^e5E6IN;>5t&D+>IR>c}tT)SW(YRje(o1d4;*UvDkixMG<3cq)&Vr z4tFa85yHI|VS;+?Eij3Av9S?$V6j(0NU_8qtFf;eg)(pZJfitHnd$v;m_AEk?2f4B zgqYPrEz>^*Qp-4oE*W?U+PV}=Yn{f2pV4Amz##2gN+qtGIhiVQ^!WHvtyaFk(v zyz#j@FIP4mTT#JgQAk#lAsiJIF<0T#f14Pi`m3WN3ndgA*%{?+L|CAZblAlHoTUP! z_pBxO3ps3NYw#H(pP!IOxG{bo3w^`>;!CU~slLx?0__Q;Co7v^qm59=o}Z-fIg#FC zdYfjNfA8#&RR4KhOb)M@C-oG(yYg>LaABTO_|ZeF9V3QY$LT`%v>=V$DIgHb#vq^g zm#-VezAaUQjH7kYs@sH$Q@Wmj$l!1s$Mnwm@*=OU*uX65nO^ayUAVI)kL&Ql8n)Mr zB%-6-^AoVK;$5Y`mMYLsHx(hly;C(p%{JRA=xOiL96x?-!@h^0Sa>JA=ITrL2RQf&9JO#^*3v22e?Obh z@pD0cpMVb{KI9fw5;?^;$`CMXP(OV*k7ICMDR_zrsf-f1K4|r2Htca2N_x(BQopVo zl^PtJx7SpyIHF*5GC{#prx-;}(!9t(`o!!=X9^<3n_}-r(ehlyM-5$f2Fx&)JUK6B zJ2s^TSyIjZVlWTy6cjz0(CWi>esSlcuCxP`;iPK(0C5rcw9E-eFE#0-5c`@m88kvX?7ivP)2?a0Cr~jI)-Tw>n0>mw|M{b|{TIq%31w z)l}t6dRMI5UiD(rG(#p2gvo-Dfk{TPU3|q*6}6)CVI{-!pFAuG^hQi%cM#F)KC+p< zMq@~Hk8Zy{BV!ZVh@UH|@1uW`;V5$KmrCfb@rs&fExCR36+tp&=*wb=uUX8Ju&}A? zRat9Dc(^m|0+lFkKowSoH-A2fiX?i7kbRy&ArpOwfpGY8yl{-iNPzT zL{bV$wA-F`9Fv-|w{W4Zb3*^5X!{%606v%AF3rr!lhV~gMe_2+MBm1BygH@EWAV>g zp**>B1j)Rt0*cyjixUED@K4NsSV4Q1pLc79QBpjrd(O`aox~b!nho&V!p3_-deB4Q zD9SSSq?QF9muqI^^tVd)(d$WM`e4n)VZD4u)a8M?&jy*DKX4~YPLV;lwyS1iVf|Az zIDygbVt{<~;whBB((%g9;P_9)Fbzs@(ue=fw>!$N3-kIKJ_QB=$_yu3C_Xq5A&)#% z9@rLva8|Yt)e6J^)DI0v;KEF~e@P%yHq%mQhiQvptYMq6m0KA}hw@6}fihzAQ=PgL z@-M{EO7Y5GI#A*W_q>Rgfb>=y%z$CB;Qkp1_5W=#)G3AFS~qktDS}M7GO1tJTBADl zp4?~R-idtVsI2`W$fLCifjB4juJf^QQ=dxXp9*MAP>Is-rtg3vPlNQ13J;Vrb2jbD z5A|0csK3B$Qi?v5WWt4HU|BvCJr(~t)&H|UWTC2+>_rpFyt0cqw@RtJ^aokyPk~Z| za*{LelND*}aXI0>u`xkFte70Dn37b&q`v7k2^Lc_QCiLGD_z0!nRHMOwam7Bl-Bfb za%X+P>sFU1CtXrUlVP0+Fyt zdq55wor&1oOZ=qo!^B*u&SatQE;v8ovxd@d$WXy+7Vv&}l2PyF@PHb6T#YXOu|z-n z=qOXcHxr}uM{`xW5J9KYsu*?SE}Au(ocH~9VGK;03{LqvpvXcOu|n zkLFtJP3oX2|5PwuLkuWuF_TVuM=4%Z>%Q1#XhWewJr;C-cNka_$4N(JzDib20ur}z z2XzkzdRS>mNEzB&QElByvTJ3vlqn0 zXv9R*Oei!Zj-rd!I5}et6 zK;Ub0(r2d7U|nlU-7td6#$)5q3tc#zz^st{f&Y$(+H4;R#}V^w?)*SBq8ODek=wGA zqZxZJZIoMr&D4yv%-E8mpamxM5%DV0*{%p{P9x0k`-8|PkLO0pmQoPY>p*gsxmV6f zSLzE-zY|qj7FLpt1}qv=;0tx%9~7&DJb1UHFQz6ZZ$)PQ>yS!<#i=tYJZHFt52RjD z@rmk)q%>~YVH}0kYKSvI-OhL36iv>PftCc8luQOoS?Ie%y-bOpfi|D6323>NvNiu&yUnhr1V>rUt@2@2b{ z^Qr<8o_*u?qGaX|M6H>-?T-MveYLIU4s4MvkKw7;kBH@A`Y>0dU{18wP|bR*b?Tcq z?%rs>?NOWhCiE&%o9ijm+=K1$Mubjd?i+dcJ@wUv7 z24lEjl(oDjgnsUx7og>{+xY6VZ%m{K)|BvzL=~|;&mdMF$DIAkoq%fflquN*A^OA_dMtD{FQL*=AWmle@5#6lFLh|)K`J* z@a}Cc{bf@TE?iNgd>t19d=*1naWFRsaJ=wvVQ(5_3^z&$6Tizh!haF}TnWcNS3aLP zjDc>*^3whC;zQozd_?RI)C!M-<=!E3N1gaL!e(5+)Gt<^Ot6?EXbN(N%J$@qN08cS zKuCzIzV}xveP1|?ZdG)+BHg$3o~v7;ZgB=R-6F;O{hY~rpi)m!YEqFI<8}5fe8!H3 zsS_DPMNQbcRU*`{pxF{tYFew_k^Hc0G>D11UG6yOh!N3!Jom0O`qG;>G#OI_nFx&E zS3HACV37qZ3kTU&iDbJij7L;EhE`PoZE9U_4i@22_cnVb!9{IeCFgKt^TU+51@ zQ|;J#!aQse6_a(aXMJOH`B<;d=jvm^9udpkq-T;0CeuMR?xq&0%t|56-4Uc>tOS>@ zHLUAEg`}CJH>Ig+4{C&yzEl5BGkg}7(a{JeR1DmzgeN#zgca|*!^yOf*|a3PM@vp#gb&$R8lq`Q2`oiZk z%kJM``bRoy&zGM(zLd19-ht!jPF)`R6z1%^wLuRaV25%HKR}4Kh)+Hj;|7 zXV5?_a+!#*VQJTpR5N`uHgNnA59PWXDTlmkM7kPhy|0MpDqOW)e&NdEz<7mwQC5wG z7@bQDLca2;E1J()LZx!Nyg;|olQ16qs2s9Bs125G0HQ!QhyK*Y_396z zE0b@AXzVTsUlzBee40Erj>^2R*k9u9ATVUhPLh_ZGd19EPtc!&o9sK@v_qF4q*^k96%IaMNKe67W98@WSK(ueGPOl+Oe zJ^T!6tiwDdSTss_CXpRPHX4}Kt^FULjKh+A!{Szq8N-rhK^?dG>JGl@p{lZ!h3sOZ zvaygMJFN5rH|H{>hpNf|1b7C}nvWkcGL@INN@+^||5t_a&p~9IaD;o9`YLRti#kc+ zbSmeGz82+j3K-~>TKS|Fk2fj__i)7LvFFRGy##|5=4j=ElWhVu8tCFAfU;XSh~fXF z(*zo(|HG}V4ZW{x`4_?S?@sk~J`9{Fys?s0vEGt~su1N25t|POMI3nGIahPNGFmnC z_f($Aq32Z5t3g1+%?iV_r9e8W^k;4NzoHJ-1kWIS)Wn&S#q-94Yd(5TE1GLIGOtRi zKsn|$8m_884of=L2pxk5t-AW9MFp~KfM>tN`jM#MB77qA7n=2 zqV-UZQ5|G$pU_2KVaFIa71r=Eq2ra(vOda5=0Sste0)#;at$Rv!U-+6?JLS{Z1r!Q z4{SVcZ&C6aJc|)pN1z-`B*H#-2)$CCcUS&#GE5${yiO}HL%4CTOrs3bAlWtb+|=Ti z`fB=X1tIJ58PVpn5gvypL!%O$r%BTSG~r}p1lHnu_FF`ORKxBzJgNKvG~SO-Y|r=W zoabsY^y1?|Gp6jhvhQV%JeeYefR|nUQz|ajm(amZ7P=Jytu&I!p%Phj19XpHy_7u* zR7?!Wn7htrT#1O9SviMAc^INrl4Jd*o9Z2n~k(wTbc-pQ1se}=-swjT)fN0_?R?uzs-{lnB;k=|5Rz<7CyO) zUU!Xp^D2W7LSQ(J=vC|YQ~PGrvs|WGVzws=UZB<-iW_!!WEKLm)B1FDaYa9&>p9qf6E?0cW-p{+sf&nQ_I~x?xEbHk8eicF)On3l2lvsX66p?aS35E?+g}$)CHze@@}bq z34oxlZ>`(>beud4na4)O4Y?w8&DdJr5H(=LJPA%9?8c*22G5Mm$Sk-to=rUERB{8^ zURiGh+aph3L3gB=*K?uU&0o2=&7G+WWbx>EXxB>@`i#>YPpv&2Yy(U~Bwc zFRdDANWzF)4A?srzq+yg&>*nd-X_rXwc3pDD$H*SI{RdEpg^p~mgfwPtah)Rv`n2{ z8)aX1hm)GoMs(trqFG|d1GnfvM4V+*wi@RG%o_z0! zV{bDhDen7G#{X*XyQ7-w-Zd2ny?5!TNS7idQUjqVMUdXB0v{kCy+bIW7g6aYf(i=K zr6auv3PC|2^rn&CLKV21@B7W%J8Raxvu4fxb6EM~%@1cmsR>wIS#XdK+X(^lzln+?_JHqJJpQuwM{6HXR; zK%+B{7?*Ul5DTVE?y2;1C6ZPB9@jJ9M|7~Iw3P5^Zf?tH-rkRmpf=3}LbNcQWxqr7 z>B?{_p7s|(^c6$+W=?k&KRf8Xl_sMaqb&Q=Ckzl8Cj$aLJbE_sNUL`IWDbRJmJI6O zG5YN0k(E}yp6z#2cD6ed4-2oOoNY{E8LFD4M^d^XTUA%87@friR6S0IoZeTliYPdningK!QzgR)q-9QtM-T-l^N~DEUPHfc0Kz_6gj9>$4W|DOE zjpo4L68Vkw#d`%r2LO2?=LwMGg-Qw`tFfb$`&`hqa!dJy8nU8l5EP(OI6+e8?8PX9 ziZ+zKe#H>#c_;sSH%8P~^@8Y4d`JEVnmwX!DzEmTB~BA8O$_TS1PYBwV^1kdPn8w< z4t4ZVVV6lJ+n;k`95O}bs>P+u1T3w$Fc1krp2-TaI<=T$>xa4(Fe<-C{R~bfx3am3 zYlHQWMucY)awe^j!jk7%Mkw2IPdM?3K`fq}$xDY?{YukSI2krxKr7X7}eZ>xa~c-br&)YQ#XocZOEkp^Yi&+XHwL^is*I ziQmn|EZTeTmoqp$q3HB|&B2f4-d}*9(s!vs-$i;VnVpHU;|ces@FWq)2=&X(SIMfq z){8Y0dP$>(*P1GT92*HoeMl8CvmIBb+$e@Ur!YOyMH;a<`Y&gwnY7{w?@PTY;rxm1 z;J19nNg1Q2nrf~vDl6^QXa(B}_w=#-HRz~oda!v+H54Ee_nno26t499Y50h?{={h2 z4I+b!-V!|p;zHPkE&z4Au`&rq0@;FS0jD~twJ{acj2W93 zneBa>?o0WdDvw`gFXDnwh6tmrJMhvMpg&qMhJ1Jd#`YCNuWxTqb;qNyd?VCIqJ}x) z^K{y0HXmU2Wa63(P9-%G)lTzPa;=eu%N+WqOj6-Yg>`9AAyEfIYycV2!_aUT>uRT| zLDTZYupddOF$$8o!)VP+jX+*&&v`ggdGmp5Zqq83CN%uJ)kmv&=Z0wbW!q+hdJ6{K zjP5%R$%l1UQPXOL3{Fq2!H?EZW5?cUxK4gQ6Srcidq*m9z8LV{ko+afr}E9s)i>XB zBqastvDS>a%S0E`rO5+{1ZHBvFClqkL()ClD;Qa6k#{&gxrrb!TjcCn25UUBe1qA$Tr&9oXQgTY25&Ko1v4&4f)F|9$xRw&uE z=jaJxo%Eu--2?AgWmJMVTol`9=e5KG=6JEDXt>z)F2ZyIBC}2bVfkvO@#oG;0bbGr zcFzjt^01|kLZW&|v}m80liI&SWKXlD383O;$=o1!$gTWIL8 zE!FOQY3Df*7fNLH5U@e_$Dn@2F;?o)(?ZwZ@T{fDC2H&_K^P6YfujKram?@xE~OKA zbScFz6k2u*a*jeJxWr>Oo@h)qdJ#dgU6JJUV=>U-q|lp`zZRw%ic82!l%R9-6{|wI z->^?bbK%C(Jsy;8aiNJZZnu+5XiToy-`&^dYYaLah+5#?(|!ENGOW!U`@0fy+41Qj z@#V%!#|27guBOcBa3~TeWsEU&G5ORr2WNT!-nl z(;_(nUrAmT+=vBGDEV4ld^wZClYOm9as&J+s$fsn(krf}4l@nD9L-3<`FD23{zI#I zMF480;!A|_wS|pCW~2@_m;Gr!yt3FHlAJD2a!in7DFtN7)5ph6kAkBr7P1PInj3pQ zlq@qXs1r*Fl+{*IxSpgsh)NI zRYn>2Fa=!%*LvvU;RZLk>dk#?9{}sbPlQUr4-=kt#(ZjtNmn|Rh^i$BGf3Y=)|cE^ zJ*0dNK&f3JJ1kPyoEBo=yQg#((bq1_^&^wVc&Rh8waV00N#SM0VNvx`+$)X1>896b z$-77}KOEYB(;1~%i^9|HSM;CDDLP*yYH}KlqXUUu(O{WdR18{VTR1x=i&0?clB2Jyx{^_x9A%t z|H_0{b3Z=|us*w)L7iZ8i1&dxbjPNJcM2;-D~R}4RvzTh|B&K6`Fgoe``^Dz`KR#Y zKV;y07V&1J_CMMCCxT6FslV9%l(=h37uniqv_W>|Th$|N7%~_!?$vyQ_g$Z|!^G=p`xMX4GQ`!`L2gZ0)U`?yq#ezg*`Lb6*w_ z5KZp}?u)?B&$hyvgHFfxpKNn_)mnHRVru-_J_WhODnj$d5@`r86H0J?2yH=c$odA@9&wfmGLo!_ordT$BVOy-{A6fuMfp<{tJc%GId+N_CDF zDx?|DaHp%qscqnlm4cFrjWJS2X!M1^ZSA%z+$x}Mg)$sg;L{Wl*V4qn=V0KxU|28P z$3C<ZEU%C{9OC|06&(}FelZDK;^=bz18>=hiXd%hru>b*Fa zq7;hT?H^Un$jPrR6cTB7srthibYRD1D9@RaCl|6ldd>99F_I|Tu6CC1McQ6Qg;KUE zN>D~|{@6yiWG|ur(YoD5LCpElR(`!IPOSoa--F)a^PBz3*SUGJezxRSvU+{bRT1a5 zpr?o_GL<)JxWqODOplA$5SV$zcrop-uzDUo46mILIHlrHgTJ%(Bts`JHTFDyc??lK zKdqkwPYE|jt(=I0Ok%|nuXSx*9OF|sSXqj6vj{RuQk>V>tHVr`wYMa+PP9WQq(U(}i>-abADdAEgl)A?gv9DepNQ0f+Kod&Sxt2pde)`*A{5l#!ai%2p z?jw@wnwLxSj+QcA&GJ&eS(jr5_s;L=Bz%8LjE*qLvE-}E_WaS+_W%W<)EO#W&o8rV zn_5U7)Yi#rK0TZpi67WMSETAlQAO3fmq9zoKxOip{TVF0ec>)~v_mHkU z7RPCW6#jQDCKV+fIIzokuHW|<&DzUo`J+(bd5qDXsI#~`+vsw*V9&b}N-NiGSH0%4 zX+?b&bF~6eJ9U)6^O(%(V&UwY#XqoH`Adob+V=ikr?%?%n2~c1s6X>&LY*w13?zv^ z`zNlVa(b`F&V*SQddsFaY<)5T={deRXFDYsa%V_d&UbVAqu)hA+r|XVOC#@|F?KqW z-kB+#nLJ{N=|IqBk~1t0)@M?BbVJW~KZX!dE^^m}*Rk?c+OD(kOmKnZC03xa!mNwJ z3vib``i3lbkALp<-;{&P=^VrKF946KjfLJx7SfeWEYc9`UvYwY>jCF{GsPZl5bL4lY!PD8kRH8~@^5a+M{9%mSHXt(#sd7)+L@4WKOtTZF$ zXx&Fbsa8LP@RFpMlDKLmxr=G3Mu>0yG%O-Wy1_`PGa&5DQg-I~P^DQG2Qn?1wd*N( z@G2wKUj%g?tLu<;dI~|7K#5N##+$2fP2mrXMyM1vMZ+0co*BZx1o>0S>BF!$3OZm? z1TAFaf_Na}e%E7L)U0LBcRgHY+<6m`ti^wHrWj_!cpiV7sONRznaby|!%etX5^Zg} z$xcI;sn1fmU+`;@aOn&`Ef`7%n)|%saArTy#O25r)zwH=O}@;URC&VFIX_Vwc5%Rp znN2Z9^S({?CQ1Hb&z8kYiV(@(_e_zB7z;>WlU&$vf- z_GQHj%R<=5f^o*jv5kL6k5N!C&xQ6BEBb*Bm$CFo8h*F!zwP>~e8x#l6*cm@Gn|tk zYI)RqrtyMcH(P&-B0=@OXGinkxVtJ#c>tS&Z|BMn%SH@$EUbBMPgT%*=8n(Sj+Y^- zDi0%qM`5k#bxxa(xIQQ@gRr{TDH_I`k%=GMF^II&t7vEfiRX83U>AXMq~xkMgfbMr zL=e}4{bNwG0S0PLU&wIGIffRy^?cqbYpBdsHULAT(R48kU+i9$M~nULIi)DzP;OYc zEknFHdlpvMy^~h+$7rUsq4u)FG#%cXIr^(FpMyCWJnmB(}N?gj)SD4u#c~dbg=} zJQNsXm3|Ro$cgcdigM}3%Av#zABr}7`En6nDFE-OK>}5Lae3YM>x`!2V0HT_Cj_b7 z9BKa@gpN0%sQdjQm&x@@pi?CZqWWbd$*&D{sWzm^GUJ=eM932%c~_aZmW?9;xT67c zwn=7M`m?kj4?`LybtiG$#dSHHP};wN(*F?^{a=Ep|MlY$S&%>@?fSaNwI1>h4 zdS_-+KJN}>hi*ktui5%O?P-{Gd$B<((Ira$_X|KzTV~|tOY1)z694RP`lqMw}2iS&}2ni@L%YK#6@uwTG(t=C8y;Q)U4%kz_tHs@2ttbvIyX*kc6o5H1Uob;r%+&};hFj`i-e*Sc5 zf#7hs)n+M4NuC)D!ghI8((wVCYD@O*eS`dZ4V_IWTaNw*weFnV-CWe_I*Mh>VDEaBvtkBqbwr zXI|XeQk0V`OLym;kUVg*aAyD9-=8|p^OuXu9V_MMfVyc)SRB>dESpSXm2qGpu3=%3 z>ne5g=A*zsjDKArsK>u8R5q@jYXg{`y*2gh|7xjBO%!0#+R_3^ZtU!Bi-yYDM=jw} zd3kxjd%E`xE5vPlWW?6OS?|kDH1-^bwT{MYk2E#O$kj8$ZO8!)MSKk~8SdY|KgFgQ zEbl<;6gv+DC&oYhk>~?TDBFy%SliF5&4eQBo14C#o}I)*EHe`m#KkmEm8xrMUZvUF z+XI{H3ai$WpzqcZ9~LA;Z=ab}T36}0+yggmVDLig5Qke>vB&FsGEI%OR8b*F=2oMj z25!5mUjW^$m&~}Hnv=62H&;k~3+SIfU$2%`5XP{$T+1+1KR;O$Y2iyG1=hNv+!Y~R zsLcWyAJSi}x3+)U*=o)EyyHz%fL1KO5ai=)2-thO2duQ-=j^OMiNX5{ z`hO;`1HF^s4HQ#q3g9+03IS#Fso7Z$YU*v^T77;VDs)Bjxu8jAswZyM0BFz~k&uz) zWMwH82LYz_{-n9}=7|PqI<(zU>It=FXh?{>RdqoD=Q0-P7ihAu~4Jn!%cS5r)l#zyB|gBDS(SjAIWZ4 z7X6s(U1lmRn=?>VWs|;U2{W%E9cr4Mp59{D3sv7XT3oTg^Pv;@J~&7=Othu&fJ08c zB`1ENT7Fjl_|Z~30$&_~Qd{~sKhu|XxHXrtrxY;`?w{v-UkYcR9dK{;KUPpss9XiZ zsumxrfOc8O`wB7g^YMAe*yID#cOlScSs~Wfs!_G z*Vg(nJPdztqil9Hni~iYp>i@F-*jlGb5=lDj-3|AqZnTL zrq5#*EgvRimr3xcatWE8@<3OVLi$-Zi;xj+iRX)|F+sU@-{H-=wD4g?)R0hCG3EB- zzgjREFcf!~2}cEc3>neYPqAXlqoef6!b8h|VMnf786vdw^C!}fl3gaMhPXx{_=roL z1dr6Tn*><|w5T8V)hOq6%0}M+7T&Hc;{^iuxjah7tsct^zhK-7bS!(q1+hplAd2BBKXB?!0O1d&X%yy-7ePG3`AlQ;ZY$< zsNvkZdpk?&t`v^RCvgK(@PFbo-5pX1p=)`ZO)x%j4K{Z6o_O8sE>LOZmOyf8Jmrb*YT1wod{TtO?)P7i-VEzt&^KsyXckjD_0{7nLprK#R_0Rr(j7N zk>1$UG=Ni-3uE)bS$3U%jD~4qFc>bc$^r_A*=>{nQiql++dmV80iNxV#>TrPuUeKS zmOhx*)87F<;JB;!Rz2h#GjGX=Z{)6ytQqh^_6rcI{+QKvqyGYY;y4%|Xliz?_0XgS zBIaK*GBPfP+tYLNoE~n)Fal$By&r~k*SuM!#hsQd@FL-4neVX77imKAn0axi|+g!ZXfX- zpEY$Z&IC4YB>6 zp{1>@B$*hfQcCNzzXq(eV;SSC*IW zk!TS|P$n78?d0ck^l0Hd>22T6&CShM2r)A@=H*!xW=jLY6>P9X_$cj0iB1q1QJO=n zRs!9UD}v8Ob9AE=??bwcJ``DU%X5sAgp~Az5h0~tizj@zH~+Ed?O+`Dq2v>8G=`@7 zKzn?)6EV-z*VDs~WUW}yg`vc5>YoQtNjy;0S8PDR?troX6BtTjfmepKv~)jWAGJ+I z;}px{{lvqAx`qpdC3|H(oG+hKP!QuWh-(8BVla8Q9rgU#=nck8k_-LAY#%OYT;30F z4F|01hhsPPy36pA4(;YXg_xR}?&@Cr0p16;wScPXU$Z>RiaYSWN8WpfFGlb(j_m%L zpBHuN2R`X4sD(a%sF)Z+nnmN$O|Wx$egu3tphlja)n6hA0TV7OkbftJP=xLVf#EYy zcGEFI^)?}c-1>#N=a{J)UXm^M0}^sx7FO1$qOVe*VII_G=H?H{f`fx!JRz2#o(K<{vO4V0RmV+%4Q1O;=G-DIwJ}HYQRq1OTjWc@Y? z2nbrjE;x8}#k9h@>Z8w-O(85I%kv$PtA4-t@!}!Ny>30*rH|^}=Vlkjo9h^ANF|a* zqjbiSDZ>2S9zD7Vg)-kFiW`|7PV)Kw?Hd=2d91%Vl8KC|yVJw2<&i6eZ|ou2VE+u| z(bV*`Q0Y>zE_s{!!3oXAZJmz_-!Sn+die<`MF7nRaI=_2M{CGwT}sNZ-`VRW5zc?XvOmFy^U2FE+W&*C#dP@4IhRoQ z^dAKNzrQzAcgR6%J1o*cP8=Txm}cdkFiI#Id=X9j2S8_T^!~U6?02I7hBcSS{#Mlg gKm6}+hvkc_{Q}O%oKH!pu7JP$st;7km2D#a4Uz-4ApigX literal 0 HcmV?d00001 diff --git a/blueprints/networking/glb-hybrid-neg-internal/diagram.svg b/blueprints/networking/glb-hybrid-neg-internal/diagram.svg new file mode 100644 index 0000000000..3c90fc424e --- /dev/null +++ b/blueprints/networking/glb-hybrid-neg-internal/diagram.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/blueprints/networking/glb-hybrid-neg-internal/glb.tf b/blueprints/networking/glb-hybrid-neg-internal/glb.tf new file mode 100644 index 0000000000..4d67d68a30 --- /dev/null +++ b/blueprints/networking/glb-hybrid-neg-internal/glb.tf @@ -0,0 +1,71 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description External Global Load Balancer. + +module "hybrid-glb" { + source = "../../../modules/net-glb" + project_id = module.project_landing.project_id + name = "hybrid-glb" + backend_service_configs = { + default = { + backends = [ + { + backend = "neg-primary" + balancing_mode = "RATE" + max_rate = { per_endpoint = 100 } + }, + { + backend = "neg-secondary" + balancing_mode = "RATE" + max_rate = { per_endpoint = 100 } + } + ] + } + } + neg_configs = { + neg-primary = { + hybrid = { + network = module.vpc_landing_untrusted.name + zone = local.zones["primary"] + endpoints = { + primary = { + ip_address = (var.ilb_create + ? module.test_vm_ilbs["primary"].forwarding_rule_address + : module.test_vms["primary"].internal_ip + ) + port = 80 + } + } + } + } + neg-secondary = { + hybrid = { + network = module.vpc_landing_untrusted.name + zone = local.zones["secondary"] + endpoints = { + secondary = { + ip_address = (var.ilb_create + ? module.test_vm_ilbs["secondary"].forwarding_rule_address + : module.test_vms["secondary"].internal_ip + ) + port = 80 + } + } + } + } + } +} diff --git a/blueprints/networking/glb-hybrid-neg-internal/main.tf b/blueprints/networking/glb-hybrid-neg-internal/main.tf new file mode 100644 index 0000000000..55600156bd --- /dev/null +++ b/blueprints/networking/glb-hybrid-neg-internal/main.tf @@ -0,0 +1,122 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + zones = { + primary = "${var.regions.primary}-b" + secondary = "${var.regions.secondary}-b" + } +} + +module "project_landing" { + source = "../../../modules/project" + billing_account = (var.projects_create != null + ? var.projects_create.billing_account_id + : null + ) + name = var.project_names.landing + parent = (var.projects_create != null + ? var.projects_create.parent + : null + ) + prefix = var.prefix + project_create = var.projects_create != null + + services = [ + "compute.googleapis.com", + "networkmanagement.googleapis.com", + # Logging and Monitoring + "logging.googleapis.com", + "monitoring.googleapis.com" + ] +} + +module "vpc_landing_untrusted" { + source = "../../../modules/net-vpc" + project_id = module.project_landing.project_id + name = "landing-untrusted" + + routes = { + spoke1-primary = { + dest_range = var.ip_config.spoke_primary + next_hop_type = "ilb" + next_hop = module.nva_untrusted_ilbs["primary"].forwarding_rule_self_link + } + spoke1-secondary = { + dest_range = var.ip_config.spoke_secondary + next_hop_type = "ilb" + next_hop = module.nva_untrusted_ilbs["secondary"].forwarding_rule_self_link + } + } + + subnets = [ + { + ip_cidr_range = var.ip_config.untrusted_primary + name = "untrusted-${var.regions.primary}" + region = var.regions.primary + }, + { + ip_cidr_range = var.ip_config.untrusted_secondary + name = "untrusted-${var.regions.secondary}" + region = var.regions.secondary + } + ] +} + +module "vpc_landing_trusted" { + source = "../../../modules/net-vpc" + project_id = module.project_landing.project_id + name = "landing-trusted" + subnets = [ + { + ip_cidr_range = var.ip_config.trusted_primary + name = "trusted-${var.regions.primary}" + region = var.regions.primary + }, + { + ip_cidr_range = var.ip_config.trusted_secondary + name = "trusted-${var.regions.secondary}" + region = var.regions.secondary + } + ] +} + +module "firewall_landing_untrusted" { + source = "../../../modules/net-vpc-firewall" + project_id = module.project_landing.project_id + network = module.vpc_landing_untrusted.name + + ingress_rules = { + allow-ssh-from-hcs = { + description = "Allow health checks to NVAs coming on port 22." + targets = ["ssh"] + source_ranges = [ + "130.211.0.0/22", + "35.191.0.0/16" + ] + rules = [{ protocol = "tcp", ports = [22] }] + } + } +} + +module "nats_landing" { + for_each = var.regions + source = "../../../modules/net-cloudnat" + project_id = module.project_landing.project_id + region = each.value + name = "nat-${each.value}" + router_network = module.vpc_landing_untrusted.self_link +} diff --git a/blueprints/networking/glb-hybrid-neg-internal/nva.tf b/blueprints/networking/glb-hybrid-neg-internal/nva.tf new file mode 100644 index 0000000000..1d2a508f87 --- /dev/null +++ b/blueprints/networking/glb-hybrid-neg-internal/nva.tf @@ -0,0 +1,87 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Network Virtual Appliances (NVAs). + +module "nva_instance_templates" { + for_each = var.regions + source = "../../../modules/compute-vm" + project_id = module.project_landing.project_id + can_ip_forward = true + create_template = true + name = "nva-${each.value}" + service_account_create = true + zone = local.zones[each.key] + + metadata = { + startup-script = templatefile( + "${path.module}/data/nva-startup-script.tftpl", + { + gateway-trusted = cidrhost(module.vpc_landing_trusted.subnet_ips["${each.value}/trusted-${each.value}"], 1) + spoke-primary = var.ip_config.spoke_primary + spoke-secondary = var.ip_config.spoke_secondary + } + ) + } + + network_interfaces = [ + { + network = module.vpc_landing_untrusted.self_link + subnetwork = module.vpc_landing_untrusted.subnet_self_links["${each.value}/untrusted-${each.value}"] + }, + { + network = module.vpc_landing_trusted.self_link + subnetwork = module.vpc_landing_trusted.subnet_self_links["${each.value}/trusted-${each.value}"] + } + ] + + tags = [ + "http-server", + "https-server", + "ssh" + ] +} + +module "nva_migs" { + for_each = var.regions + source = "../../../modules/compute-mig" + project_id = module.project_landing.project_id + location = local.zones[each.key] + name = "nva-${each.value}" + target_size = 1 + instance_template = module.nva_instance_templates[each.key].template.self_link +} + +module "nva_untrusted_ilbs" { + for_each = var.regions + source = "../../../modules/net-ilb" + project_id = module.project_landing.project_id + region = each.value + name = "nva-ilb-${local.zones[each.key]}" + service_label = "nva-ilb-${local.zones[each.key]}" + vpc_config = { + network = module.vpc_landing_untrusted.self_link + subnetwork = module.vpc_landing_untrusted.subnet_self_links["${each.value}/untrusted-${each.value}"] + } + backends = [{ + group = module.nva_migs[each.key].group_manager.instance_group + }] + health_check_config = { + tcp = { + port = 22 + } + } +} diff --git a/blueprints/networking/glb-hybrid-neg-internal/outputs.tf b/blueprints/networking/glb-hybrid-neg-internal/outputs.tf new file mode 100644 index 0000000000..7d8ce185ff --- /dev/null +++ b/blueprints/networking/glb-hybrid-neg-internal/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "glb_ip_address" { + description = "Load balancer IP address." + value = module.hybrid-glb.address +} diff --git a/blueprints/networking/glb-hybrid-neg-internal/spoke.tf b/blueprints/networking/glb-hybrid-neg-internal/spoke.tf new file mode 100644 index 0000000000..1769d4c682 --- /dev/null +++ b/blueprints/networking/glb-hybrid-neg-internal/spoke.tf @@ -0,0 +1,146 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description VPC Spoke(s) and test VMs. + +module "project_spoke_01" { + source = "../../../modules/project" + billing_account = (var.projects_create != null + ? var.projects_create.billing_account_id + : null + ) + name = var.project_names.spoke_01 + parent = (var.projects_create != null + ? var.projects_create.parent + : null + ) + prefix = var.prefix + + services = [ + "compute.googleapis.com", + "networkmanagement.googleapis.com", + # Logging and Monitoring + "logging.googleapis.com", + "monitoring.googleapis.com" + ] +} + +module "vpc_spoke_01" { + source = "../../../modules/net-vpc" + project_id = module.project_spoke_01.project_id + name = "spoke-01" + subnets = [ + { + ip_cidr_range = var.ip_config.spoke_primary + name = "spoke-01-${var.regions.primary}" + region = var.regions.primary + }, + { + ip_cidr_range = var.ip_config.spoke_secondary + name = "spoke-01-${var.regions.secondary}" + region = var.regions.secondary + } + ] + peering_config = { + peer_vpc_self_link = module.vpc_landing_trusted.self_link + import_routes = true + } +} + +module "firewall_spoke_01" { + source = "../../../modules/net-vpc-firewall" + project_id = module.project_spoke_01.project_id + network = module.vpc_spoke_01.name + + ingress_rules = { + allow-nva-hcs = { + description = "Allow health checks coming on port 80 and 443 from NVAs." + targets = ["http-server", "https-server"] + source_ranges = [ + var.ip_config.trusted_primary, + var.ip_config.trusted_secondary + ] + rules = [{ protocol = "tcp", ports = [80, 443] }] + } + } +} + +# NAT is used to install nginx for test purposed, even if NVAs are still not ready + +module "nats_spoke_01" { + for_each = var.regions + source = "../../../modules/net-cloudnat" + name = "spoke-01-${each.value}" + project_id = module.project_spoke_01.project_id + region = each.value + router_network = module.vpc_spoke_01.name +} + +module "test_vms" { + for_each = var.regions + source = "../../../modules/compute-vm" + name = "spoke-01-${each.value}" + project_id = module.project_spoke_01.project_id + create_template = var.ilb_create + service_account_create = true + zone = local.zones[each.key] + + metadata = { + startup-script = "apt update && apt install -y nginx" + } + + network_interfaces = [{ + network = module.vpc_spoke_01.self_link + subnetwork = module.vpc_spoke_01.subnet_self_links["${each.value}/spoke-01-${each.value}"] + }] + + tags = [ + "http-server", + "https-server", + "ssh" + ] +} + +module "test_vm_migs" { + for_each = var.ilb_create ? var.regions : {} + source = "../../../modules/compute-mig" + project_id = module.project_spoke_01.project_id + location = local.zones[each.key] + name = "test-vm-${each.value}" + target_size = 1 + instance_template = module.test_vms[each.key].template.self_link +} + +module "test_vm_ilbs" { + for_each = var.ilb_create ? var.regions : {} + source = "../../../modules/net-ilb" + project_id = module.project_spoke_01.project_id + region = each.value + name = "test-vm-ilb-${each.value}" + service_label = "test-vm-ilb-${each.value}" + vpc_config = { + network = module.vpc_spoke_01.self_link + subnetwork = module.vpc_spoke_01.subnet_self_links["${each.value}/spoke-01-${each.value}"] + } + backends = [{ + group = module.test_vm_migs[each.key].group_manager.instance_group + }] + health_check_config = { + tcp = { + port = 80 + } + } +} diff --git a/blueprints/networking/glb-hybrid-neg-internal/variables.tf b/blueprints/networking/glb-hybrid-neg-internal/variables.tf new file mode 100644 index 0000000000..3f96b86797 --- /dev/null +++ b/blueprints/networking/glb-hybrid-neg-internal/variables.tf @@ -0,0 +1,76 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "ilb_create" { + description = "Whether we should create an ILB L4 in front of the test VMs in the spoke." + type = string + default = false +} + +variable "ip_config" { + description = "The subnet IP configurations." + type = object({ + spoke_primary = optional(string, "192.168.101.0/24") + spoke_secondary = optional(string, "192.168.102.0/24") + trusted_primary = optional(string, "192.168.11.0/24") + trusted_secondary = optional(string, "192.168.22.0/24") + untrusted_primary = optional(string, "192.168.1.0/24") + untrusted_secondary = optional(string, "192.168.2.0/24") + }) + default = {} +} + +variable "prefix" { + description = "Prefix used for resource names." + type = string + validation { + condition = var.prefix != "" + error_message = "Prefix cannot be empty." + } +} + +variable "project_names" { + description = "The project names." + type = object({ + landing = string + spoke_01 = string + }) + default = { + landing = "landing" + spoke_01 = "spoke-01" + } +} + +variable "projects_create" { + description = "Parameters for the creation of the new project." + type = object({ + billing_account_id = string + parent = string + }) + default = null +} + +variable "regions" { + description = "Region definitions." + type = object({ + primary = string + secondary = string + }) + default = { + primary = "europe-west1" + secondary = "europe-west4" + } +} From 29151771139cb260e97ca46979fd804918a02554 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taneli=20Lepp=C3=A4?= Date: Thu, 2 Mar 2023 10:21:52 +0100 Subject: [PATCH 21/39] Fix issue with GKE cluster notifications topic, change pubsub module output to static. --- modules/gke-cluster/main.tf | 6 ++++-- modules/pubsub/README.md | 10 +++++----- modules/pubsub/main.tf | 1 + modules/pubsub/outputs.tf | 3 ++- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/modules/gke-cluster/main.tf b/modules/gke-cluster/main.tf index 107d8341cf..a42c4fb3f5 100644 --- a/modules/gke-cluster/main.tf +++ b/modules/gke-cluster/main.tf @@ -406,9 +406,11 @@ resource "google_compute_network_peering_routes_config" "gke_master" { resource "google_pubsub_topic" "notifications" { count = ( - try(var.enable_features.upgrade_notifications.topic_id, null) == null ? 0 : 1 + try(var.enable_features.upgrade_notifications, null) != null && + try(var.enable_features.upgrade_notifications.topic_id, null) == null ? 1 : 0 ) - name = "gke-pubsub-notifications" + project = var.project_id + name = "gke-pubsub-notifications" labels = { content = "gke-notifications" } diff --git a/modules/pubsub/README.md b/modules/pubsub/README.md index 81e4336599..09fa1b3aec 100644 --- a/modules/pubsub/README.md +++ b/modules/pubsub/README.md @@ -169,10 +169,10 @@ module "pubsub" { | name | description | sensitive | |---|---|:---:| | [id](outputs.tf#L17) | Topic id. | | -| [schema](outputs.tf#L25) | Schema resource. | | -| [schema_id](outputs.tf#L30) | Schema resource id. | | -| [subscription_id](outputs.tf#L35) | Subscription ids. | | -| [subscriptions](outputs.tf#L45) | Subscription resources. | | -| [topic](outputs.tf#L53) | Topic resource. | | +| [schema](outputs.tf#L26) | Schema resource. | | +| [schema_id](outputs.tf#L31) | Schema resource id. | | +| [subscription_id](outputs.tf#L36) | Subscription ids. | | +| [subscriptions](outputs.tf#L46) | Subscription resources. | | +| [topic](outputs.tf#L54) | Topic resource. | | diff --git a/modules/pubsub/main.tf b/modules/pubsub/main.tf index 7e5630a9af..ccb6f5d79a 100644 --- a/modules/pubsub/main.tf +++ b/modules/pubsub/main.tf @@ -33,6 +33,7 @@ locals { options = try(v.options, v, null) == null ? var.defaults : v.options } } + topic_id_static = "projects/${var.project_id}/topics/${var.name}" } resource "google_pubsub_schema" "default" { diff --git a/modules/pubsub/outputs.tf b/modules/pubsub/outputs.tf index 4aea42c53b..3e99889bbe 100644 --- a/modules/pubsub/outputs.tf +++ b/modules/pubsub/outputs.tf @@ -16,8 +16,9 @@ output "id" { description = "Topic id." - value = google_pubsub_topic.default.id + value = local.topic_id_static depends_on = [ + google_pubsub_topic.default, google_pubsub_topic_iam_binding.default ] } From 5b433d8b97fb3da494a8fb51ec3c460bfbcdb1bb Mon Sep 17 00:00:00 2001 From: bruzzechesse Date: Thu, 2 Mar 2023 11:56:05 +0100 Subject: [PATCH 22/39] small fixes add excample for simple nva with bgp enable --- .../simple-nva/README.md | 65 +++++++++++++++++++ .../simple-nva/files/frr/frr.service | 2 +- .../simple-nva/files/policy_based_routing.sh | 6 +- 3 files changed, 69 insertions(+), 4 deletions(-) diff --git a/modules/cloud-config-container/simple-nva/README.md b/modules/cloud-config-container/simple-nva/README.md index efafffdeaa..d6d357d584 100644 --- a/modules/cloud-config-container/simple-nva/README.md +++ b/modules/cloud-config-container/simple-nva/README.md @@ -62,6 +62,71 @@ module "vm" { } # tftest modules=1 resources=1 ``` + +### Example with BGP service + +```hcl +locals { + network_interfaces = [ + { + addresses = null + name = "dev" + nat = false + network = "dev_vpc_self_link" + routes = ["10.128.0.0/9"] + subnetwork = "dev_vpc_nva_subnet_self_link" + enable_masquerading = true + non_masq_cidrs = ["10.0.0.0/8"] + }, + { + addresses = null + name = "prod" + nat = false + network = "prod_vpc_self_link" + routes = ["10.0.0.0/9"] + subnetwork = "prod_vpc_nva_subnet_self_link" + } + ] +} + +module "cos-nva" { + source = "../../simple-nva" + enable_health_checks = true + network_interfaces = local.network_interfaces + bgp_config = { + enable = true + # frr_config = "./frr.conf" + } + optional_run_cmds = ["ls -l"] + # files = { + # "/var/lib/cloud/scripts/per-boot/firewall-rules.sh" = { + # content = file("./your_path/to/firewall-rules.sh") + # owner = "root" + # permissions = 0700 + # } + # } +} + +module "vm" { + source = "./fabric/modules/compute-vm" + project_id = "my-project" + zone = "europe-west8-b" + name = "cos-nva" + network_interfaces = local.network_interfaces + metadata = { + user-data = module.cos-nva.cloud_config + google-logging-enabled = true + } + boot_disk = { + image = "projects/cos-cloud/global/images/family/cos-stable" + type = "pd-ssd" + size = 10 + } + tags = ["nva", "ssh"] +} +# tftest modules=1 resources=1 +``` + ## Variables diff --git a/modules/cloud-config-container/simple-nva/files/frr/frr.service b/modules/cloud-config-container/simple-nva/files/frr/frr.service index 0a64c2ad8c..a560602eb9 100644 --- a/modules/cloud-config-container/simple-nva/files/frr/frr.service +++ b/modules/cloud-config-container/simple-nva/files/frr/frr.service @@ -24,4 +24,4 @@ ExecStart=/usr/bin/docker run --rm --name=frr \ -v /etc/frr:/etc/frr \ frrouting/frr ExecStop=/usr/bin/docker stop frr -ExecStopPost=/usr/bin/docker rm frr \ No newline at end of file +ExecStopPost=/usr/bin/docker rm frr diff --git a/modules/cloud-config-container/simple-nva/files/policy_based_routing.sh b/modules/cloud-config-container/simple-nva/files/policy_based_routing.sh index 3746d4b20f..951396d356 100644 --- a/modules/cloud-config-container/simple-nva/files/policy_based_routing.sh +++ b/modules/cloud-config-container/simple-nva/files/policy_based_routing.sh @@ -19,8 +19,8 @@ IP_LB=$(ip r show table local | grep "$IF_NAME proto 66" | cut -f 2 -d " ") # Sleep while there's no load balancer IP route for this IF while [ -z $IP_LB ] ; do - sleep 2 - IP_LB=$(ip r show table local | grep "$IF_NAME proto 66" | cut -f 2 -d " ") + sleep 2 + IP_LB=$(ip r show table local | grep "$IF_NAME proto 66" | cut -f 2 -d " ") done IF_NUMBER=$(echo $IF_NAME | sed -e s/eth//) @@ -31,4 +31,4 @@ IF_IP_PREFIX=$(/var/run/nva/ipprefix_by_netmask.sh $IF_NETMASK) grep -qxF "$((200 + $IF_NUMBER)) hc-$IF_NAME" /etc/iproute2/rt_tables || echo "$((200 + $IF_NUMBER)) hc-$IF_NAME" >>/etc/iproute2/rt_tables ip route add $IF_GW src $IF_IP dev $IF_NAME table hc-$IF_NAME ip route add default via $IF_GW dev $IF_NAME table hc-$IF_NAME -ip rule add from $IP_LB/32 table hc-$IF_NAME \ No newline at end of file +ip rule add from $IP_LB/32 table hc-$IF_NAME From ac6aa5f4ad81a88e8b9c2d18cc7b8343400e5172 Mon Sep 17 00:00:00 2001 From: bruzzechesse Date: Thu, 2 Mar 2023 11:57:29 +0100 Subject: [PATCH 23/39] small fixes add example of simple-nva usage with bgp --- .../simple-nva/cloud-config.yaml | 2 +- .../simple-nva/files/frr/daemons | 2 +- .../simple-nva/files/frr/frr.conf | 2 +- modules/cloud-config-container/simple-nva/main.tf | 10 ++++++---- .../cloud-config-container/simple-nva/variables.tf | 11 +++++------ 5 files changed, 14 insertions(+), 13 deletions(-) diff --git a/modules/cloud-config-container/simple-nva/cloud-config.yaml b/modules/cloud-config-container/simple-nva/cloud-config.yaml index 492f4965d2..0d9f754d9a 100644 --- a/modules/cloud-config-container/simple-nva/cloud-config.yaml +++ b/modules/cloud-config-container/simple-nva/cloud-config.yaml @@ -1,6 +1,6 @@ #cloud-config -# Copyright 2022 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/modules/cloud-config-container/simple-nva/files/frr/daemons b/modules/cloud-config-container/simple-nva/files/frr/daemons index 2a5c0ac9e4..52f5ac54ed 100644 --- a/modules/cloud-config-container/simple-nva/files/frr/daemons +++ b/modules/cloud-config-container/simple-nva/files/frr/daemons @@ -63,4 +63,4 @@ fabricd_options=" --daemon -A 127.0.0.1" # ospfd_wrap="/usr/bin/valgrind" # or you can use "all_wrap" for all daemons, e.g. to use perf record: # all_wrap="/usr/bin/perf record --call-graph -" -# the normal daemon command is added to this at the end. \ No newline at end of file +# the normal daemon command is added to this at the end. diff --git a/modules/cloud-config-container/simple-nva/files/frr/frr.conf b/modules/cloud-config-container/simple-nva/files/frr/frr.conf index bf997a3ca8..352ab77756 100644 --- a/modules/cloud-config-container/simple-nva/files/frr/frr.conf +++ b/modules/cloud-config-container/simple-nva/files/frr/frr.conf @@ -20,4 +20,4 @@ log syslog informational no ipv6 forwarding router bgp 65500 -line vty \ No newline at end of file +line vty diff --git a/modules/cloud-config-container/simple-nva/main.tf b/modules/cloud-config-container/simple-nva/main.tf index c2798e2bb3..8ff338cd10 100644 --- a/modules/cloud-config-container/simple-nva/main.tf +++ b/modules/cloud-config-container/simple-nva/main.tf @@ -1,5 +1,5 @@ /** - * Copyright 2022 Google LLC + * Copyright 2023 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -89,9 +89,11 @@ locals { } ] - optional_run_cmds = try(var.bgp_config.enable, false) ? concat( - ["systemctl start frr"], var.optional_run_cmds - ) : var.optional_run_cmds + optional_run_cmds = ( + try(var.bgp_config.enable, false) + ? concat(["systemctl start frr"], var.optional_run_cmds) + : var.optional_run_cmds + ) template = ( var.cloud_config == null diff --git a/modules/cloud-config-container/simple-nva/variables.tf b/modules/cloud-config-container/simple-nva/variables.tf index fba9a12763..6e00f6b624 100644 --- a/modules/cloud-config-container/simple-nva/variables.tf +++ b/modules/cloud-config-container/simple-nva/variables.tf @@ -1,5 +1,5 @@ /** - * Copyright 2022 Google LLC + * Copyright 2023 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,12 +18,11 @@ variable "bgp_config" { description = "BGP configuration for FR Routing container running on the NVA." type = object({ daemons = optional(string) - enable = optional(bool) + enable = optional(bool, false) frr_config = optional(string) }) - default = { - enable = false - } + default = {} + nullable = false } variable "cloud_config" { @@ -52,7 +51,7 @@ variable "network_interfaces" { description = "Network interfaces configuration." type = list(object({ routes = optional(list(string)) - enable_masquerading = optional(bool) + enable_masquerading = optional(bool, false) non_masq_cidrs = optional(list(string)) })) } From 740d8cdc0e11859e4894ca9a5c3bd89b8e240c2d Mon Sep 17 00:00:00 2001 From: bruzzechesse Date: Thu, 2 Mar 2023 12:06:41 +0100 Subject: [PATCH 24/39] fmt terraform --- modules/cloud-config-container/simple-nva/main.tf | 2 +- modules/cloud-config-container/simple-nva/variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/cloud-config-container/simple-nva/main.tf b/modules/cloud-config-container/simple-nva/main.tf index 8ff338cd10..4dff8d5c9b 100644 --- a/modules/cloud-config-container/simple-nva/main.tf +++ b/modules/cloud-config-container/simple-nva/main.tf @@ -91,7 +91,7 @@ locals { optional_run_cmds = ( try(var.bgp_config.enable, false) - ? concat(["systemctl start frr"], var.optional_run_cmds) + ? concat(["systemctl start frr"], var.optional_run_cmds) : var.optional_run_cmds ) diff --git a/modules/cloud-config-container/simple-nva/variables.tf b/modules/cloud-config-container/simple-nva/variables.tf index 6e00f6b624..38115507a7 100644 --- a/modules/cloud-config-container/simple-nva/variables.tf +++ b/modules/cloud-config-container/simple-nva/variables.tf @@ -21,7 +21,7 @@ variable "bgp_config" { enable = optional(bool, false) frr_config = optional(string) }) - default = {} + default = {} nullable = false } From db5dd1400f0a00522cec6f40850afe5c4c8085ca Mon Sep 17 00:00:00 2001 From: bruzzechesse Date: Thu, 2 Mar 2023 12:20:46 +0100 Subject: [PATCH 25/39] update README --- modules/cloud-config-container/simple-nva/README.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/modules/cloud-config-container/simple-nva/README.md b/modules/cloud-config-container/simple-nva/README.md index d6d357d584..3bdd6b25b1 100644 --- a/modules/cloud-config-container/simple-nva/README.md +++ b/modules/cloud-config-container/simple-nva/README.md @@ -126,19 +126,18 @@ module "vm" { } # tftest modules=1 resources=1 ``` - ## Variables | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [network_interfaces](variables.tf#L51) | Network interfaces configuration. | list(object({…})) | ✓ | | -| [bgp_config](variables.tf#L17) | BGP configuration for FR Routing container running on the NVA. | object({…}) | | {…} | -| [cloud_config](variables.tf#L29) | Cloud config template path. If null default will be used. | string | | null | -| [enable_health_checks](variables.tf#L35) | Configures routing to enable responses to health check probes. | bool | | false | -| [files](variables.tf#L41) | Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null. | map(object({…})) | | {} | -| [optional_run_cmds](variables.tf#L60) | Optional Cloud Init run commands to execute. | list(string) | | [] | +| [network_interfaces](variables.tf#L50) | Network interfaces configuration. | list(object({…})) | ✓ | | +| [bgp_config](variables.tf#L17) | BGP configuration for FR Routing container running on the NVA. | object({…}) | | {} | +| [cloud_config](variables.tf#L28) | Cloud config template path. If null default will be used. | string | | null | +| [enable_health_checks](variables.tf#L34) | Configures routing to enable responses to health check probes. | bool | | false | +| [files](variables.tf#L40) | Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null. | map(object({…})) | | {} | +| [optional_run_cmds](variables.tf#L59) | Optional Cloud Init run commands to execute. | list(string) | | [] | ## Outputs From 8f2e0ac9e66709b6d2422c596d723070fe980427 Mon Sep 17 00:00:00 2001 From: bruzzechesse Date: Thu, 2 Mar 2023 14:14:07 +0100 Subject: [PATCH 26/39] fix issue in README --- modules/cloud-config-container/simple-nva/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/cloud-config-container/simple-nva/README.md b/modules/cloud-config-container/simple-nva/README.md index 3bdd6b25b1..68dcccb876 100644 --- a/modules/cloud-config-container/simple-nva/README.md +++ b/modules/cloud-config-container/simple-nva/README.md @@ -90,7 +90,7 @@ locals { } module "cos-nva" { - source = "../../simple-nva" + source = "./fabric/modules/cloud-config-container/simple-nva" enable_health_checks = true network_interfaces = local.network_interfaces bgp_config = { From 04377fc4dc2f25d04c8369c8aac780efed1b2a3b Mon Sep 17 00:00:00 2001 From: bruzzechesse Date: Thu, 2 Mar 2023 16:07:50 +0100 Subject: [PATCH 27/39] fmt terraform code in example in README --- .../simple-nva/README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/cloud-config-container/simple-nva/README.md b/modules/cloud-config-container/simple-nva/README.md index 68dcccb876..618e21f8e9 100644 --- a/modules/cloud-config-container/simple-nva/README.md +++ b/modules/cloud-config-container/simple-nva/README.md @@ -69,14 +69,14 @@ module "vm" { locals { network_interfaces = [ { - addresses = null - name = "dev" - nat = false - network = "dev_vpc_self_link" - routes = ["10.128.0.0/9"] - subnetwork = "dev_vpc_nva_subnet_self_link" + addresses = null + name = "dev" + nat = false + network = "dev_vpc_self_link" + routes = ["10.128.0.0/9"] + subnetwork = "dev_vpc_nva_subnet_self_link" enable_masquerading = true - non_masq_cidrs = ["10.0.0.0/8"] + non_masq_cidrs = ["10.0.0.0/8"] }, { addresses = null @@ -93,8 +93,8 @@ module "cos-nva" { source = "./fabric/modules/cloud-config-container/simple-nva" enable_health_checks = true network_interfaces = local.network_interfaces - bgp_config = { - enable = true + bgp_config = { + enable = true # frr_config = "./frr.conf" } optional_run_cmds = ["ls -l"] From a085213597711edbb8812e8b77509d310901a6d1 Mon Sep 17 00:00:00 2001 From: bruzzechesse Date: Fri, 3 Mar 2023 12:25:16 +0100 Subject: [PATCH 28/39] update copyright on outdated files update bgp_config variable to frr_config replac daemons input file with daemons_enabled to manage FRR services --- fast/stages/2-networking-c-nva/bgp-files/ew1b | 67 +++++++++++++++++++ .../simple-nva/README.md | 15 ++++- .../cloud-config-container/simple-nva/main.tf | 13 ++-- 3 files changed, 88 insertions(+), 7 deletions(-) create mode 100644 fast/stages/2-networking-c-nva/bgp-files/ew1b diff --git a/fast/stages/2-networking-c-nva/bgp-files/ew1b b/fast/stages/2-networking-c-nva/bgp-files/ew1b new file mode 100644 index 0000000000..e0a212f907 --- /dev/null +++ b/fast/stages/2-networking-c-nva/bgp-files/ew1b @@ -0,0 +1,67 @@ +log syslog informational +no ipv6 forwarding +service integrated-vtysh-config +interface lo + ip address 10.128.0.101/32 +route-map ALLOW-ALL permit 100 +route-map PRIMARY permit 10 + set metric 100 +route-map SECONDARY permit 20 + set metric 10100 +route-map NVA-TO-NVA permit 30 + set metric 50 +router bgp 65514 view trusted + no bgp ebgp-requires-policy + no bgp network import-check + neighbor 10.128.64.201 remote-as 64515 + neighbor 10.128.64.202 remote-as 64515 + address-family ipv4 unicast + neighbor 10.128.64.201 activate + neighbor 10.128.64.201 soft-reconfiguration inbound + neighbor 10.128.64.202 activate + neighbor 10.128.64.202 soft-reconfiguration inbound + network 10.128.0.0/24 route-map PRIMARY + network 0.0.0.0/0 route-map PRIMARY + network 10.128.32.0/24 route-map SECONDARY + exit-address-family +router bgp 65514 + bgp router-id 10.128.0.101 + no bgp ebgp-requires-policy + no bgp network import-check + neighbor 10.128.32.101 remote-as 65513 + neighbor 10.128.32.101 update-source lo + neighbor 10.128.32.101 ebgp-multihop 2 + neighbor 10.128.32.101 next-hop-self + neighbor 10.128.32.102 remote-as 65513 + neighbor 10.128.32.102 update-source lo + neighbor 10.128.32.102 ebgp-multihop 2 + neighbor 10.128.32.102 next-hop-self + address-family ipv4 unicast + neighbor 10.128.32.101 activate + neighbor 10.128.32.101 soft-reconfiguration inbound + neighbor 10.128.32.102 activate + neighbor 10.128.32.102 soft-reconfiguration inbound + network 10.128.192.0/24 route-map NVA-TO-NVA + network 10.128.128.0/24 route-map NVA-TO-NVA + network 10.128.48.0/24 route-map NVA-TO-NVA + exit-address-family +router bgp 65514 view untrusted + no bgp ebgp-requires-policy + no bgp network import-check + neighbor 10.128.0.201 remote-as 64515 + neighbor 10.128.0.202 remote-as 64515 + address-family ipv4 unicast + neighbor 10.128.0.201 activate + neighbor 10.128.0.201 soft-reconfiguration inbound + neighbor 10.128.0.202 activate + neighbor 10.128.0.202 soft-reconfiguration inbound + network 10.128.0.0/27 route-map PRIMARY + network 10.0.0.0/8 route-map PRIMARY + network 172.16.0.0/12 route-map PRIMARY + network 192.168.0.0/16 route-map PRIMARY + network 10.128.96.0/24 route-map SECONDARY + network 10.128.128.0/24 route-map PRIMARY + network 10.128.48.0/24 route-map PRIMARY + network 10.128.192.0/24 route-map PRIMARY + exit-address-family +line vty \ No newline at end of file diff --git a/modules/cloud-config-container/simple-nva/README.md b/modules/cloud-config-container/simple-nva/README.md index 618e21f8e9..9aac71d427 100644 --- a/modules/cloud-config-container/simple-nva/README.md +++ b/modules/cloud-config-container/simple-nva/README.md @@ -63,7 +63,20 @@ module "vm" { # tftest modules=1 resources=1 ``` -### Example with BGP service +### Example with advanced routing capabilities + +Below a sample terraform example for bootstrapping a simple-nva based on COS running [FRRouting](https://frrouting.org/) container. +Please find below s sample frr.conf file based on documentation available at the following link for hosting a BGP service on FRR container, by default enabling + +``` +Example frr.conf file + +log syslog informational +no ipv6 forwarding +router bgp 65500 +line vty + +``` ```hcl locals { diff --git a/modules/cloud-config-container/simple-nva/main.tf b/modules/cloud-config-container/simple-nva/main.tf index 4dff8d5c9b..3d0b45aa06 100644 --- a/modules/cloud-config-container/simple-nva/main.tf +++ b/modules/cloud-config-container/simple-nva/main.tf @@ -22,9 +22,6 @@ locals { optional_run_cmds = local.optional_run_cmds })) - frr_config = ( - try(var.bgp_config.frr_config != null, false) ? var.bgp_config.frr_config : "${path.module}/files/frr/frr.conf" - ) daemons = ( try(var.bgp_config.daemons != null, false) ? var.bgp_config.daemons : "${path.module}/files/frr/daemons" ) @@ -47,14 +44,14 @@ locals { permissions = attrs.permissions } }, - try(var.bgp_config.enable, false) ? { + try(var.frr_config, false) ? { "/etc/frr/daemons" = { - content = file(local.daemons) + content = templatefile(local.daemons, local.frr_daemons_enabled) owner = "root" permissions = "0744" } "/etc/frr/frr.conf" = { - content = file(local.frr_config) + content = file(var.frr_config.config_file) owner = "root" permissions = "0744" } @@ -79,6 +76,10 @@ locals { } : {} ) + frr_daemons = ["zebra", "bgpd", "ospfd", "ospf6d", "ripd", "ripngd", "isisd", "pimd", "ldpd", "nhrpd", "eigrpd", "babeld", "sharpd", "staticd", "pbrd", "bfdd", "fabricd"] + + frr_daemons_enabled = try(merge([for daemon in local.frr_daemons : {daemon: contains(var.frr_config.daemons_enabled, daemon)}]),{}) + network_interfaces = [ for index, interface in var.network_interfaces : { name = "eth${index}" From c147c806def581f514700c64ab6c8d7c2ccf1511 Mon Sep 17 00:00:00 2001 From: bruzzechesse Date: Fri, 3 Mar 2023 12:30:28 +0100 Subject: [PATCH 29/39] update copyright on poutdated ones replace daemons file in frr_config with daemons_enabled to manage FRR services small fixes --- .../simple-nva/cloud-config.yaml | 2 ++ .../simple-nva/files/frr/daemons | 34 +++++++++---------- .../simple-nva/files/frr/frr.conf | 23 ------------- .../simple-nva/files/ipprefix_by_netmask.sh | 2 +- .../simple-nva/files/policy_based_routing.sh | 2 +- .../simple-nva/outputs.tf | 2 +- .../simple-nva/variables.tf | 26 ++++++++------ .../simple-nva/versions.tf | 2 +- 8 files changed, 38 insertions(+), 55 deletions(-) delete mode 100644 modules/cloud-config-container/simple-nva/files/frr/frr.conf diff --git a/modules/cloud-config-container/simple-nva/cloud-config.yaml b/modules/cloud-config-container/simple-nva/cloud-config.yaml index 0d9f754d9a..f44cd08e6b 100644 --- a/modules/cloud-config-container/simple-nva/cloud-config.yaml +++ b/modules/cloud-config-container/simple-nva/cloud-config.yaml @@ -22,6 +22,7 @@ write_files: content: | ${indent(6, data.content)} %{ endfor } + - path: /etc/systemd/system/routing.service permissions: 0644 owner: root @@ -34,6 +35,7 @@ write_files: Wants=network-online.target [Service] ExecStart=/bin/sh -c "/var/run/nva/start-routing.sh" + - path: /var/run/nva/start-routing.sh permissions: 0744 owner: root diff --git a/modules/cloud-config-container/simple-nva/files/frr/daemons b/modules/cloud-config-container/simple-nva/files/frr/daemons index 52f5ac54ed..6303fc23da 100644 --- a/modules/cloud-config-container/simple-nva/files/frr/daemons +++ b/modules/cloud-config-container/simple-nva/files/frr/daemons @@ -12,23 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -zebra=no -bgpd=yes -ospfd=no -ospf6d=no -ripd=no -ripngd=no -isisd=no -pimd=no -ldpd=no -nhrpd=no -eigrpd=no -babeld=no -sharpd=no -staticd=no -pbrd=no -bfdd=no -fabricd=no +zebra=%{frr_daemons_enabled.zebra ? "yes" : "no"} +bgpd=%{frr_daemons_enabled.bgpd ? "yes" : "no"} +ospfd=%{frr_daemons_enabled.ospfd ? "yes" : "no"} +ospf6d=%{frr_daemons_enabled.ospf6d ? "yes" : "no"} +ripd=%{frr_daemons_enabled.ripd ? "yes" : "no"} +ripngd=%{frr_daemons_enabled.ripngd ? "yes" : "no"} +isisd=%{frr_daemons_enabled.isisd ? "yes" : "no"} +pimd=%{frr_daemons_enabled.pimd ? "yes" : "no"} +ldpd=%{frr_daemons_enabled.ldpd ? "yes" : "no"} +nhrpd=%{frr_daemons_enabled.nhrpd ? "yes" : "no"} +eigrpd=%{frr_daemons_enabled.eigrpd ? "yes" : "no"} +babeld=%{frr_daemons_enabled.babeld ? "yes" : "no"} +sharpd=%{frr_daemons_enabled.sharpd ? "yes" : "no"} +staticd=%{frr_daemons_enabled.staticd ? "yes" : "no"} +pbrd=%{frr_daemons_enabled.pbrd ? "yes" : "no"} +bfdd=%{frr_daemons_enabled.bfdd ? "yes" : "no"} +fabricd=%{frr_daemons_enabled.fabricd ? "yes" : "no"} # # If this option is set the /etc/init.d/frr script automatically loads diff --git a/modules/cloud-config-container/simple-nva/files/frr/frr.conf b/modules/cloud-config-container/simple-nva/files/frr/frr.conf deleted file mode 100644 index 352ab77756..0000000000 --- a/modules/cloud-config-container/simple-nva/files/frr/frr.conf +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default FRR configuration which enables a BGP protocol process with a default ASN of 65500. -# For more information on how to update this file and/or configure bgp sessions please refer to -# the official documentation available at the following link: -# https://docs.frrouting.org/en/latest/overview.html - -log syslog informational -no ipv6 forwarding -router bgp 65500 -line vty diff --git a/modules/cloud-config-container/simple-nva/files/ipprefix_by_netmask.sh b/modules/cloud-config-container/simple-nva/files/ipprefix_by_netmask.sh index a1c69822df..1694382535 100644 --- a/modules/cloud-config-container/simple-nva/files/ipprefix_by_netmask.sh +++ b/modules/cloud-config-container/simple-nva/files/ipprefix_by_netmask.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2022 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/modules/cloud-config-container/simple-nva/files/policy_based_routing.sh b/modules/cloud-config-container/simple-nva/files/policy_based_routing.sh index 951396d356..49f3828837 100644 --- a/modules/cloud-config-container/simple-nva/files/policy_based_routing.sh +++ b/modules/cloud-config-container/simple-nva/files/policy_based_routing.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2022 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/modules/cloud-config-container/simple-nva/outputs.tf b/modules/cloud-config-container/simple-nva/outputs.tf index 7d8d41656b..54942c1ad5 100644 --- a/modules/cloud-config-container/simple-nva/outputs.tf +++ b/modules/cloud-config-container/simple-nva/outputs.tf @@ -1,5 +1,5 @@ /** - * Copyright 2022 Google LLC + * Copyright 2023 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/cloud-config-container/simple-nva/variables.tf b/modules/cloud-config-container/simple-nva/variables.tf index 38115507a7..33e070ccbf 100644 --- a/modules/cloud-config-container/simple-nva/variables.tf +++ b/modules/cloud-config-container/simple-nva/variables.tf @@ -14,17 +14,6 @@ * limitations under the License. */ -variable "bgp_config" { - description = "BGP configuration for FR Routing container running on the NVA." - type = object({ - daemons = optional(string) - enable = optional(bool, false) - frr_config = optional(string) - }) - default = {} - nullable = false -} - variable "cloud_config" { description = "Cloud config template path. If null default will be used." type = string @@ -47,6 +36,21 @@ variable "files" { default = {} } +variable "frr_config" { + description = "FRR configuration for container running on the NVA." + type = object({ + daemons_enabled = optional(list()) + config_file = string + }) + default = null + validation { + condition = try(var.frr_config.daemons_enabled && alltrue([ + for daemon in var.frr_config.daemons_enabled : contains(local.frr_daemons, daemon) + ]), true) + error_message = "Invalid entry specified in daemons_enabled list, must be one of ${local.frr_daemons}" + } +} + variable "network_interfaces" { description = "Network interfaces configuration." type = list(object({ diff --git a/modules/cloud-config-container/simple-nva/versions.tf b/modules/cloud-config-container/simple-nva/versions.tf index cef924ea40..fe423d229f 100644 --- a/modules/cloud-config-container/simple-nva/versions.tf +++ b/modules/cloud-config-container/simple-nva/versions.tf @@ -1,4 +1,4 @@ -# Copyright 2022 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From cccf5e6bfabf5b269d6aa3711405752cd299ba0c Mon Sep 17 00:00:00 2001 From: bruzzechesse Date: Fri, 3 Mar 2023 17:31:08 +0100 Subject: [PATCH 30/39] small fixes --- .../simple-nva/README.md | 23 ++++++------- .../simple-nva/files/frr/daemons | 34 +++++++++---------- .../cloud-config-container/simple-nva/main.tf | 17 ++++------ .../simple-nva/variables.tf | 10 +++--- .../simple-nva/versions.tf | 4 +-- 5 files changed, 42 insertions(+), 46 deletions(-) diff --git a/modules/cloud-config-container/simple-nva/README.md b/modules/cloud-config-container/simple-nva/README.md index 9aac71d427..1226ff35e0 100644 --- a/modules/cloud-config-container/simple-nva/README.md +++ b/modules/cloud-config-container/simple-nva/README.md @@ -65,8 +65,8 @@ module "vm" { ### Example with advanced routing capabilities -Below a sample terraform example for bootstrapping a simple-nva based on COS running [FRRouting](https://frrouting.org/) container. -Please find below s sample frr.conf file based on documentation available at the following link for hosting a BGP service on FRR container, by default enabling +Below a sample terraform example for bootstrapping a simple-nva powered by [COS](https://cloud.google.com/container-optimized-os/docs) and running [FRRouting](https://frrouting.org/) container. +Please find below s sample frr.conf file based on documentation available at the following [link](https://docs.frrouting.org/en/latest/basic.html) for hosting a BGP service on FRR container. ``` Example frr.conf file @@ -78,6 +78,8 @@ line vty ``` +Following code assumes a file in the same folder named frr.conf exists. + ```hcl locals { network_interfaces = [ @@ -106,10 +108,7 @@ module "cos-nva" { source = "./fabric/modules/cloud-config-container/simple-nva" enable_health_checks = true network_interfaces = local.network_interfaces - bgp_config = { - enable = true - # frr_config = "./frr.conf" - } + frr_config = {config_file = "./frr.conf}", daemons_enabled = ["bgpd"]} optional_run_cmds = ["ls -l"] # files = { # "/var/lib/cloud/scripts/per-boot/firewall-rules.sh" = { @@ -145,12 +144,12 @@ module "vm" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [network_interfaces](variables.tf#L50) | Network interfaces configuration. | list(object({…})) | ✓ | | -| [bgp_config](variables.tf#L17) | BGP configuration for FR Routing container running on the NVA. | object({…}) | | {} | -| [cloud_config](variables.tf#L28) | Cloud config template path. If null default will be used. | string | | null | -| [enable_health_checks](variables.tf#L34) | Configures routing to enable responses to health check probes. | bool | | false | -| [files](variables.tf#L40) | Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null. | map(object({…})) | | {} | -| [optional_run_cmds](variables.tf#L59) | Optional Cloud Init run commands to execute. | list(string) | | [] | +| [network_interfaces](variables.tf#L54) | Network interfaces configuration. | list(object({…})) | ✓ | | +| [cloud_config](variables.tf#L17) | Cloud config template path. If null default will be used. | string | | null | +| [enable_health_checks](variables.tf#L23) | Configures routing to enable responses to health check probes. | bool | | false | +| [files](variables.tf#L29) | Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null. | map(object({…})) | | {} | +| [frr_config](variables.tf#L39) | FRR configuration for container running on the NVA. | object({…}) | | null | +| [optional_run_cmds](variables.tf#L63) | Optional Cloud Init run commands to execute. | list(string) | | [] | ## Outputs diff --git a/modules/cloud-config-container/simple-nva/files/frr/daemons b/modules/cloud-config-container/simple-nva/files/frr/daemons index 6303fc23da..efa760bed6 100644 --- a/modules/cloud-config-container/simple-nva/files/frr/daemons +++ b/modules/cloud-config-container/simple-nva/files/frr/daemons @@ -12,23 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -zebra=%{frr_daemons_enabled.zebra ? "yes" : "no"} -bgpd=%{frr_daemons_enabled.bgpd ? "yes" : "no"} -ospfd=%{frr_daemons_enabled.ospfd ? "yes" : "no"} -ospf6d=%{frr_daemons_enabled.ospf6d ? "yes" : "no"} -ripd=%{frr_daemons_enabled.ripd ? "yes" : "no"} -ripngd=%{frr_daemons_enabled.ripngd ? "yes" : "no"} -isisd=%{frr_daemons_enabled.isisd ? "yes" : "no"} -pimd=%{frr_daemons_enabled.pimd ? "yes" : "no"} -ldpd=%{frr_daemons_enabled.ldpd ? "yes" : "no"} -nhrpd=%{frr_daemons_enabled.nhrpd ? "yes" : "no"} -eigrpd=%{frr_daemons_enabled.eigrpd ? "yes" : "no"} -babeld=%{frr_daemons_enabled.babeld ? "yes" : "no"} -sharpd=%{frr_daemons_enabled.sharpd ? "yes" : "no"} -staticd=%{frr_daemons_enabled.staticd ? "yes" : "no"} -pbrd=%{frr_daemons_enabled.pbrd ? "yes" : "no"} -bfdd=%{frr_daemons_enabled.bfdd ? "yes" : "no"} -fabricd=%{frr_daemons_enabled.fabricd ? "yes" : "no"} +zebra=${zebra_enabled} +bgpd=${bgpd_enabled} +ospfd=${ospfd_enabled} +ospf6d=${ospf6d_enabled} +ripd=${ripd_enabled} +ripngd=${ripngd_enabled} +isisd=${isisd_enabled} +pimd=${pimd_enabled} +ldpd=${ldpd_enabled} +nhrpd=${nhrpd_enabled} +eigrpd=${eigrpd_enabled} +babeld=${babeld_enabled} +sharpd=${sharpd_enabled} +staticd=${staticd_enabled} +pbrd=${pbrd_enabled} +bfdd=${bfdd_enabled} +fabricd=${fabricd_enabled} # # If this option is set the /etc/init.d/frr script automatically loads diff --git a/modules/cloud-config-container/simple-nva/main.tf b/modules/cloud-config-container/simple-nva/main.tf index 3d0b45aa06..ef439a2c5b 100644 --- a/modules/cloud-config-container/simple-nva/main.tf +++ b/modules/cloud-config-container/simple-nva/main.tf @@ -15,16 +15,13 @@ */ locals { - cloud_config = templatefile(local.template, merge({ + cloud_config = templatefile(local.template, { files = local.files enable_health_checks = var.enable_health_checks network_interfaces = local.network_interfaces optional_run_cmds = local.optional_run_cmds - })) + }) - daemons = ( - try(var.bgp_config.daemons != null, false) ? var.bgp_config.daemons : "${path.module}/files/frr/daemons" - ) files = merge( { "/var/run/nva/ipprefix_by_netmask.sh" = { @@ -44,9 +41,9 @@ locals { permissions = attrs.permissions } }, - try(var.frr_config, false) ? { + try(var.frr_config != null, false) ? { "/etc/frr/daemons" = { - content = templatefile(local.daemons, local.frr_daemons_enabled) + content = templatefile("${path.module}/files/frr/daemons", local.frr_daemons_enabled) owner = "root" permissions = "0744" } @@ -77,8 +74,8 @@ locals { ) frr_daemons = ["zebra", "bgpd", "ospfd", "ospf6d", "ripd", "ripngd", "isisd", "pimd", "ldpd", "nhrpd", "eigrpd", "babeld", "sharpd", "staticd", "pbrd", "bfdd", "fabricd"] - - frr_daemons_enabled = try(merge([for daemon in local.frr_daemons : {daemon: contains(var.frr_config.daemons_enabled, daemon)}]),{}) + + frr_daemons_enabled = try({ for daemon in local.frr_daemons : "${daemon}_enabled" => contains(var.frr_config.daemons_enabled, daemon) ? "yes" : "no" }, {}) network_interfaces = [ for index, interface in var.network_interfaces : { @@ -91,7 +88,7 @@ locals { ] optional_run_cmds = ( - try(var.bgp_config.enable, false) + try(var.frr_config != null, false) ? concat(["systemctl start frr"], var.optional_run_cmds) : var.optional_run_cmds ) diff --git a/modules/cloud-config-container/simple-nva/variables.tf b/modules/cloud-config-container/simple-nva/variables.tf index 33e070ccbf..774eccb9d1 100644 --- a/modules/cloud-config-container/simple-nva/variables.tf +++ b/modules/cloud-config-container/simple-nva/variables.tf @@ -39,15 +39,15 @@ variable "files" { variable "frr_config" { description = "FRR configuration for container running on the NVA." type = object({ - daemons_enabled = optional(list()) + daemons_enabled = optional(list(string)) config_file = string }) - default = null + default = null validation { - condition = try(var.frr_config.daemons_enabled && alltrue([ - for daemon in var.frr_config.daemons_enabled : contains(local.frr_daemons, daemon) + condition = try(alltrue([ + for daemon in var.frr_config.daemons_enabled : contains(["zebra", "bgpd", "ospfd", "ospf6d", "ripd", "ripngd", "isisd", "pimd", "ldpd", "nhrpd", "eigrpd", "babeld", "sharpd", "staticd", "pbrd", "bfdd", "fabricd"], daemon) ]), true) - error_message = "Invalid entry specified in daemons_enabled list, must be one of ${local.frr_daemons}" + error_message = "Invalid entry specified in daemons_enabled list, must be one of [zebra, bgpd, ospfd, ospf6d, ripd, ripngd, isisd, pimd, ldpd, nhrpd, eigrpd, babeld, sharpd, staticd, pbrd, bfdd, fabricd]" } } diff --git a/modules/cloud-config-container/simple-nva/versions.tf b/modules/cloud-config-container/simple-nva/versions.tf index fe423d229f..d9c1d37c73 100644 --- a/modules/cloud-config-container/simple-nva/versions.tf +++ b/modules/cloud-config-container/simple-nva/versions.tf @@ -17,11 +17,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 4.55.0" # tftest + version = ">= 4.50.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 4.55.0" # tftest + version = ">= 4.50.0" # tftest } } } From a2fcce054cff108d2af678ff16b768ad8c42b601 Mon Sep 17 00:00:00 2001 From: bruzzechesse Date: Fri, 3 Mar 2023 17:38:55 +0100 Subject: [PATCH 31/39] upgrade terraform required version for simple-nva module --- modules/cloud-config-container/simple-nva/versions.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/cloud-config-container/simple-nva/versions.tf b/modules/cloud-config-container/simple-nva/versions.tf index d9c1d37c73..fe423d229f 100644 --- a/modules/cloud-config-container/simple-nva/versions.tf +++ b/modules/cloud-config-container/simple-nva/versions.tf @@ -17,11 +17,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 4.50.0" # tftest + version = ">= 4.55.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 4.50.0" # tftest + version = ">= 4.55.0" # tftest } } } From 4a0b0805eb88f3f5cbe079561e2eba4510a9ba1d Mon Sep 17 00:00:00 2001 From: bruzzechesse Date: Fri, 3 Mar 2023 17:55:20 +0100 Subject: [PATCH 32/39] add simple BGP configuration for BGP session with neighbor --- modules/cloud-config-container/simple-nva/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/cloud-config-container/simple-nva/README.md b/modules/cloud-config-container/simple-nva/README.md index 1226ff35e0..2cfe194288 100644 --- a/modules/cloud-config-container/simple-nva/README.md +++ b/modules/cloud-config-container/simple-nva/README.md @@ -66,14 +66,15 @@ module "vm" { ### Example with advanced routing capabilities Below a sample terraform example for bootstrapping a simple-nva powered by [COS](https://cloud.google.com/container-optimized-os/docs) and running [FRRouting](https://frrouting.org/) container. -Please find below s sample frr.conf file based on documentation available at the following [link](https://docs.frrouting.org/en/latest/basic.html) for hosting a BGP service on FRR container. +Please find below s sample frr.conf file based on documentation available at the following [link](https://docs.frrouting.org/en/latest/basic.html) for hosting a BGP service with ASN 65001 on FRR container establishing a BGP session with a remote neighbor with IP address 10.128.0.2 and ASN 65002. ``` Example frr.conf file log syslog informational no ipv6 forwarding -router bgp 65500 +router bgp 65001 + neighbor 10.128.0.2 remote-as 65002 line vty ``` From 0dd334922fdee0af20ad71e7ff460225ae645310 Mon Sep 17 00:00:00 2001 From: bruzzechesse Date: Fri, 3 Mar 2023 18:12:13 +0100 Subject: [PATCH 33/39] add _ on local values not referenced outside locals --- .../cloud-config-container/simple-nva/main.tf | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/modules/cloud-config-container/simple-nva/main.tf b/modules/cloud-config-container/simple-nva/main.tf index ef439a2c5b..91817cf1ab 100644 --- a/modules/cloud-config-container/simple-nva/main.tf +++ b/modules/cloud-config-container/simple-nva/main.tf @@ -15,14 +15,14 @@ */ locals { - cloud_config = templatefile(local.template, { - files = local.files + cloud_config = templatefile(local._template, { + files = local._files enable_health_checks = var.enable_health_checks - network_interfaces = local.network_interfaces - optional_run_cmds = local.optional_run_cmds + network_interfaces = local._network_interfaces + optional_run_cmds = local._optional_run_cmds }) - files = merge( + _files = merge( { "/var/run/nva/ipprefix_by_netmask.sh" = { content = file("${path.module}/files/ipprefix_by_netmask.sh") @@ -43,7 +43,7 @@ locals { }, try(var.frr_config != null, false) ? { "/etc/frr/daemons" = { - content = templatefile("${path.module}/files/frr/daemons", local.frr_daemons_enabled) + content = templatefile("${path.module}/files/frr/daemons", local._frr_daemons_enabled) owner = "root" permissions = "0744" } @@ -73,11 +73,11 @@ locals { } : {} ) - frr_daemons = ["zebra", "bgpd", "ospfd", "ospf6d", "ripd", "ripngd", "isisd", "pimd", "ldpd", "nhrpd", "eigrpd", "babeld", "sharpd", "staticd", "pbrd", "bfdd", "fabricd"] + _frr_daemons = ["zebra", "bgpd", "ospfd", "ospf6d", "ripd", "ripngd", "isisd", "pimd", "ldpd", "nhrpd", "eigrpd", "babeld", "sharpd", "staticd", "pbrd", "bfdd", "fabricd"] - frr_daemons_enabled = try({ for daemon in local.frr_daemons : "${daemon}_enabled" => contains(var.frr_config.daemons_enabled, daemon) ? "yes" : "no" }, {}) + _frr_daemons_enabled = try({ for daemon in local._frr_daemons : "${daemon}_enabled" => contains(var.frr_config.daemons_enabled, daemon) ? "yes" : "no" }, {}) - network_interfaces = [ + _network_interfaces = [ for index, interface in var.network_interfaces : { name = "eth${index}" number = index @@ -87,13 +87,13 @@ locals { } ] - optional_run_cmds = ( + _optional_run_cmds = ( try(var.frr_config != null, false) ? concat(["systemctl start frr"], var.optional_run_cmds) : var.optional_run_cmds ) - template = ( + _template = ( var.cloud_config == null ? "${path.module}/cloud-config.yaml" : var.cloud_config From 696115854f0dc89742d3e4022aceb4a8d02f89f3 Mon Sep 17 00:00:00 2001 From: bruzzechesse Date: Mon, 6 Mar 2023 09:11:30 +0100 Subject: [PATCH 34/39] add sample frr.conf file in README --- modules/cloud-config-container/simple-nva/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/cloud-config-container/simple-nva/README.md b/modules/cloud-config-container/simple-nva/README.md index 2cfe194288..9e1ab7d060 100644 --- a/modules/cloud-config-container/simple-nva/README.md +++ b/modules/cloud-config-container/simple-nva/README.md @@ -69,14 +69,14 @@ Below a sample terraform example for bootstrapping a simple-nva powered by [COS] Please find below s sample frr.conf file based on documentation available at the following [link](https://docs.frrouting.org/en/latest/basic.html) for hosting a BGP service with ASN 65001 on FRR container establishing a BGP session with a remote neighbor with IP address 10.128.0.2 and ASN 65002. ``` -Example frr.conf file +# tftest-file id=frr_conf path=./frr.conf +# Example frr.conmf file log syslog informational no ipv6 forwarding router bgp 65001 neighbor 10.128.0.2 remote-as 65002 line vty - ``` Following code assumes a file in the same folder named frr.conf exists. @@ -137,7 +137,7 @@ module "vm" { } tags = ["nva", "ssh"] } -# tftest modules=1 resources=1 +# tftest modules=1 resources=1 files=frr_conf ``` From c7ae02cc9c83d5f00b1746f9a15ef265fdb853f3 Mon Sep 17 00:00:00 2001 From: bruzzechesse Date: Mon, 6 Mar 2023 10:47:33 +0100 Subject: [PATCH 35/39] remove typo --- modules/cloud-config-container/simple-nva/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/cloud-config-container/simple-nva/README.md b/modules/cloud-config-container/simple-nva/README.md index 9e1ab7d060..6470192de6 100644 --- a/modules/cloud-config-container/simple-nva/README.md +++ b/modules/cloud-config-container/simple-nva/README.md @@ -109,7 +109,7 @@ module "cos-nva" { source = "./fabric/modules/cloud-config-container/simple-nva" enable_health_checks = true network_interfaces = local.network_interfaces - frr_config = {config_file = "./frr.conf}", daemons_enabled = ["bgpd"]} + frr_config = {config_file = "./frr.conf", daemons_enabled = ["bgpd"]} optional_run_cmds = ["ls -l"] # files = { # "/var/lib/cloud/scripts/per-boot/firewall-rules.sh" = { From 7caaf2887fc9cf445872ad3253cdc6b91b3969a3 Mon Sep 17 00:00:00 2001 From: bruzzechesse Date: Mon, 6 Mar 2023 11:19:48 +0100 Subject: [PATCH 36/39] formatted example code --- modules/cloud-config-container/simple-nva/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/cloud-config-container/simple-nva/README.md b/modules/cloud-config-container/simple-nva/README.md index 6470192de6..987e359ac1 100644 --- a/modules/cloud-config-container/simple-nva/README.md +++ b/modules/cloud-config-container/simple-nva/README.md @@ -109,8 +109,8 @@ module "cos-nva" { source = "./fabric/modules/cloud-config-container/simple-nva" enable_health_checks = true network_interfaces = local.network_interfaces - frr_config = {config_file = "./frr.conf", daemons_enabled = ["bgpd"]} - optional_run_cmds = ["ls -l"] + frr_config = { config_file = "./frr.conf", daemons_enabled = ["bgpd"] } + optional_run_cmds = ["ls -l"] # files = { # "/var/lib/cloud/scripts/per-boot/firewall-rules.sh" = { # content = file("./your_path/to/firewall-rules.sh") From 42f963a577612f036fb4233da0a67e743dc8de40 Mon Sep 17 00:00:00 2001 From: bruzzechesse Date: Tue, 7 Mar 2023 12:05:24 +0100 Subject: [PATCH 37/39] small fixes --- modules/cloud-config-container/simple-nva/files/frr/daemons | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/cloud-config-container/simple-nva/files/frr/daemons b/modules/cloud-config-container/simple-nva/files/frr/daemons index efa760bed6..0a388df012 100644 --- a/modules/cloud-config-container/simple-nva/files/frr/daemons +++ b/modules/cloud-config-container/simple-nva/files/frr/daemons @@ -30,11 +30,10 @@ pbrd=${pbrd_enabled} bfdd=${bfdd_enabled} fabricd=${fabricd_enabled} -# # If this option is set the /etc/init.d/frr script automatically loads # the config via "vtysh -b" when the servers are started. # Check /etc/pam.d/frr if you intend to use "vtysh"! -# + vtysh_enable=yes zebra_options=" -A 127.0.0.1 -s 90000000" bgpd_options=" -A 127.0.0.1" From 648e7dcfaf74823582e13f0ec0984b71300f48bc Mon Sep 17 00:00:00 2001 From: bruzzechesse Date: Tue, 7 Mar 2023 12:06:37 +0100 Subject: [PATCH 38/39] small fixes --- fast/stages/2-networking-c-nva/bgp-files/ew1b | 67 ------------------- .../simple-nva/README.md | 11 +-- .../cloud-config-container/simple-nva/main.tf | 40 ++++++++--- .../simple-nva/variables.tf | 20 +++++- 4 files changed, 52 insertions(+), 86 deletions(-) delete mode 100644 fast/stages/2-networking-c-nva/bgp-files/ew1b diff --git a/fast/stages/2-networking-c-nva/bgp-files/ew1b b/fast/stages/2-networking-c-nva/bgp-files/ew1b deleted file mode 100644 index e0a212f907..0000000000 --- a/fast/stages/2-networking-c-nva/bgp-files/ew1b +++ /dev/null @@ -1,67 +0,0 @@ -log syslog informational -no ipv6 forwarding -service integrated-vtysh-config -interface lo - ip address 10.128.0.101/32 -route-map ALLOW-ALL permit 100 -route-map PRIMARY permit 10 - set metric 100 -route-map SECONDARY permit 20 - set metric 10100 -route-map NVA-TO-NVA permit 30 - set metric 50 -router bgp 65514 view trusted - no bgp ebgp-requires-policy - no bgp network import-check - neighbor 10.128.64.201 remote-as 64515 - neighbor 10.128.64.202 remote-as 64515 - address-family ipv4 unicast - neighbor 10.128.64.201 activate - neighbor 10.128.64.201 soft-reconfiguration inbound - neighbor 10.128.64.202 activate - neighbor 10.128.64.202 soft-reconfiguration inbound - network 10.128.0.0/24 route-map PRIMARY - network 0.0.0.0/0 route-map PRIMARY - network 10.128.32.0/24 route-map SECONDARY - exit-address-family -router bgp 65514 - bgp router-id 10.128.0.101 - no bgp ebgp-requires-policy - no bgp network import-check - neighbor 10.128.32.101 remote-as 65513 - neighbor 10.128.32.101 update-source lo - neighbor 10.128.32.101 ebgp-multihop 2 - neighbor 10.128.32.101 next-hop-self - neighbor 10.128.32.102 remote-as 65513 - neighbor 10.128.32.102 update-source lo - neighbor 10.128.32.102 ebgp-multihop 2 - neighbor 10.128.32.102 next-hop-self - address-family ipv4 unicast - neighbor 10.128.32.101 activate - neighbor 10.128.32.101 soft-reconfiguration inbound - neighbor 10.128.32.102 activate - neighbor 10.128.32.102 soft-reconfiguration inbound - network 10.128.192.0/24 route-map NVA-TO-NVA - network 10.128.128.0/24 route-map NVA-TO-NVA - network 10.128.48.0/24 route-map NVA-TO-NVA - exit-address-family -router bgp 65514 view untrusted - no bgp ebgp-requires-policy - no bgp network import-check - neighbor 10.128.0.201 remote-as 64515 - neighbor 10.128.0.202 remote-as 64515 - address-family ipv4 unicast - neighbor 10.128.0.201 activate - neighbor 10.128.0.201 soft-reconfiguration inbound - neighbor 10.128.0.202 activate - neighbor 10.128.0.202 soft-reconfiguration inbound - network 10.128.0.0/27 route-map PRIMARY - network 10.0.0.0/8 route-map PRIMARY - network 172.16.0.0/12 route-map PRIMARY - network 192.168.0.0/16 route-map PRIMARY - network 10.128.96.0/24 route-map SECONDARY - network 10.128.128.0/24 route-map PRIMARY - network 10.128.48.0/24 route-map PRIMARY - network 10.128.192.0/24 route-map PRIMARY - exit-address-family -line vty \ No newline at end of file diff --git a/modules/cloud-config-container/simple-nva/README.md b/modules/cloud-config-container/simple-nva/README.md index 987e359ac1..25d80956c5 100644 --- a/modules/cloud-config-container/simple-nva/README.md +++ b/modules/cloud-config-container/simple-nva/README.md @@ -65,8 +65,8 @@ module "vm" { ### Example with advanced routing capabilities -Below a sample terraform example for bootstrapping a simple-nva powered by [COS](https://cloud.google.com/container-optimized-os/docs) and running [FRRouting](https://frrouting.org/) container. -Please find below s sample frr.conf file based on documentation available at the following [link](https://docs.frrouting.org/en/latest/basic.html) for hosting a BGP service with ASN 65001 on FRR container establishing a BGP session with a remote neighbor with IP address 10.128.0.2 and ASN 65002. +Find below a sample terraform example for bootstrapping a simple NVA powered by [COS](https://cloud.google.com/container-optimized-os/docs) and running [FRRouting](https://frrouting.org/) container. +Please find below a sample frr.conf file based on the documentation available [here](https://docs.frrouting.org/en/latest/basic.html) for hosting a BGP service with ASN 65001 on FRR container establishing a BGP session with a remote neighbor with IP address 10.128.0.2 and ASN 65002. ``` # tftest-file id=frr_conf path=./frr.conf @@ -111,13 +111,6 @@ module "cos-nva" { network_interfaces = local.network_interfaces frr_config = { config_file = "./frr.conf", daemons_enabled = ["bgpd"] } optional_run_cmds = ["ls -l"] - # files = { - # "/var/lib/cloud/scripts/per-boot/firewall-rules.sh" = { - # content = file("./your_path/to/firewall-rules.sh") - # owner = "root" - # permissions = 0700 - # } - # } } module "vm" { diff --git a/modules/cloud-config-container/simple-nva/main.tf b/modules/cloud-config-container/simple-nva/main.tf index 91817cf1ab..110983bf93 100644 --- a/modules/cloud-config-container/simple-nva/main.tf +++ b/modules/cloud-config-container/simple-nva/main.tf @@ -15,13 +15,6 @@ */ locals { - cloud_config = templatefile(local._template, { - files = local._files - enable_health_checks = var.enable_health_checks - network_interfaces = local._network_interfaces - optional_run_cmds = local._optional_run_cmds - }) - _files = merge( { "/var/run/nva/ipprefix_by_netmask.sh" = { @@ -73,9 +66,31 @@ locals { } : {} ) - _frr_daemons = ["zebra", "bgpd", "ospfd", "ospf6d", "ripd", "ripngd", "isisd", "pimd", "ldpd", "nhrpd", "eigrpd", "babeld", "sharpd", "staticd", "pbrd", "bfdd", "fabricd"] + _frr_daemons = [ + "zebra", + "bgpd", + "ospfd", + "ospf6d", + "ripd", + "ripngd", + "isisd", + "pimd", + "ldpd", + "nhrpd", + "eigrpd", + "babeld", + "sharpd", + "staticd", + "pbrd", + "bfdd", + "fabricd" + ] - _frr_daemons_enabled = try({ for daemon in local._frr_daemons : "${daemon}_enabled" => contains(var.frr_config.daemons_enabled, daemon) ? "yes" : "no" }, {}) + _frr_daemons_enabled = try( + { + for daemon in local._frr_daemons : + "${daemon}_enabled" => contains(var.frr_config.daemons_enabled, daemon) ? "yes" : "no" + }, {}) _network_interfaces = [ for index, interface in var.network_interfaces : { @@ -98,4 +113,11 @@ locals { ? "${path.module}/cloud-config.yaml" : var.cloud_config ) + + cloud_config = templatefile(local._template, { + enable_health_checks = var.enable_health_checks + files = local._files + network_interfaces = local._network_interfaces + optional_run_cmds = local._optional_run_cmds + }) } diff --git a/modules/cloud-config-container/simple-nva/variables.tf b/modules/cloud-config-container/simple-nva/variables.tf index 774eccb9d1..9e568f5b01 100644 --- a/modules/cloud-config-container/simple-nva/variables.tf +++ b/modules/cloud-config-container/simple-nva/variables.tf @@ -45,7 +45,25 @@ variable "frr_config" { default = null validation { condition = try(alltrue([ - for daemon in var.frr_config.daemons_enabled : contains(["zebra", "bgpd", "ospfd", "ospf6d", "ripd", "ripngd", "isisd", "pimd", "ldpd", "nhrpd", "eigrpd", "babeld", "sharpd", "staticd", "pbrd", "bfdd", "fabricd"], daemon) + for daemon in var.frr_config.daemons_enabled : contains([ + "zebra", + "bgpd", + "ospfd", + "ospf6d", + "ripd", + "ripngd", + "isisd", + "pimd", + "ldpd", + "nhrpd", + "eigrpd", + "babeld", + "sharpd", + "staticd", + "pbrd", + "bfdd", + "fabricd" + ], daemon) ]), true) error_message = "Invalid entry specified in daemons_enabled list, must be one of [zebra, bgpd, ospfd, ospf6d, ripd, ripngd, isisd, pimd, ldpd, nhrpd, eigrpd, babeld, sharpd, staticd, pbrd, bfdd, fabricd]" } From 7e41adf9e801903383a8660b6b9650335eb2977a Mon Sep 17 00:00:00 2001 From: bruzzechesse Date: Tue, 7 Mar 2023 16:37:07 +0100 Subject: [PATCH 39/39] fix documentation --- modules/cloud-config-container/simple-nva/README.md | 4 ++-- modules/cloud-config-container/simple-nva/variables.tf | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/modules/cloud-config-container/simple-nva/README.md b/modules/cloud-config-container/simple-nva/README.md index 25d80956c5..e0800d17e1 100644 --- a/modules/cloud-config-container/simple-nva/README.md +++ b/modules/cloud-config-container/simple-nva/README.md @@ -138,12 +138,12 @@ module "vm" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [network_interfaces](variables.tf#L54) | Network interfaces configuration. | list(object({…})) | ✓ | | +| [network_interfaces](variables.tf#L75) | Network interfaces configuration. | list(object({…})) | ✓ | | | [cloud_config](variables.tf#L17) | Cloud config template path. If null default will be used. | string | | null | | [enable_health_checks](variables.tf#L23) | Configures routing to enable responses to health check probes. | bool | | false | | [files](variables.tf#L29) | Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null. | map(object({…})) | | {} | | [frr_config](variables.tf#L39) | FRR configuration for container running on the NVA. | object({…}) | | null | -| [optional_run_cmds](variables.tf#L63) | Optional Cloud Init run commands to execute. | list(string) | | [] | +| [optional_run_cmds](variables.tf#L84) | Optional Cloud Init run commands to execute. | list(string) | | [] | ## Outputs diff --git a/modules/cloud-config-container/simple-nva/variables.tf b/modules/cloud-config-container/simple-nva/variables.tf index 9e568f5b01..84f62f6989 100644 --- a/modules/cloud-config-container/simple-nva/variables.tf +++ b/modules/cloud-config-container/simple-nva/variables.tf @@ -65,7 +65,10 @@ variable "frr_config" { "fabricd" ], daemon) ]), true) - error_message = "Invalid entry specified in daemons_enabled list, must be one of [zebra, bgpd, ospfd, ospf6d, ripd, ripngd, isisd, pimd, ldpd, nhrpd, eigrpd, babeld, sharpd, staticd, pbrd, bfdd, fabricd]" + error_message = <