From 15487078d438e5fd13a7b32427db7bda7dad7392 Mon Sep 17 00:00:00 2001
From: James D'Alfonso
Date: Mon, 13 Feb 2023 10:00:50 +0100
Subject: [PATCH 001/169] add missing iam properties to factory_subnets
---
modules/net-vpc/subnets.tf | 3 +++
1 file changed, 3 insertions(+)
diff --git a/modules/net-vpc/subnets.tf b/modules/net-vpc/subnets.tf
index ae094ecfa0..7c03bfca3b 100644
--- a/modules/net-vpc/subnets.tf
+++ b/modules/net-vpc/subnets.tf
@@ -31,6 +31,9 @@ locals {
flow_logs_config = try(v.flow_logs, null)
ipv6 = try(v.ipv6, null)
secondary_ip_ranges = try(v.secondary_ip_ranges, null)
+ iam_groups = try(v.iam_groups, [])
+ iam_users = try(v.iam_users, [])
+ iam_service_accounts = try(v.iam_service_accounts, [])
}
}
_factory_subnets_iam = [
From ebc4bc51a519247f4a900d021e4d913bebd4e8d9 Mon Sep 17 00:00:00 2001
From: lcaggio
Date: Mon, 13 Feb 2023 15:25:24 +0100
Subject: [PATCH 002/169] Workaround to mitigate issue 9164
---
blueprints/data-solutions/data-playground/main.tf | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/blueprints/data-solutions/data-playground/main.tf b/blueprints/data-solutions/data-playground/main.tf
index b87e8e7301..548bee37d1 100644
--- a/blueprints/data-solutions/data-playground/main.tf
+++ b/blueprints/data-solutions/data-playground/main.tf
@@ -217,6 +217,11 @@ resource "google_notebooks_instance" "playground" {
service_account = module.service-account-notebook.email
+ # Remove once terraform-provider-google/issues/9164 is fixed
+ lifecycle {
+ ignore_changes = [disk_encryption, kms_key]
+ }
+
#TODO Uncomment once terraform-provider-google/issues/9273 is fixed
# tags = ["ssh"]
depends_on = [
From 3f0271480b85a80c6966be1b1a413a694d88c5f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Wiktor=20Niesiob=C4=99dzki?=
Date: Sat, 11 Feb 2023 10:59:50 +0000
Subject: [PATCH 003/169] Add documentation about JIT-ed service accounts
---
modules/project/README.md | 32 +++++++++++++++++++++++++++++
modules/project/service-accounts.tf | 21 +++++++++++--------
2 files changed, 44 insertions(+), 9 deletions(-)
diff --git a/modules/project/README.md b/modules/project/README.md
index 3753a5da27..de37503e2d 100644
--- a/modules/project/README.md
+++ b/modules/project/README.md
@@ -138,6 +138,38 @@ module "project" {
# tftest modules=1 resources=2
```
+### Service identities requiring manual IAM grants
+The module will create service identities at the time of the creation of the project instead creation of them at the time of first use.
+This allows granting these service identities roles in other projects which is usually necessary in Shared VPC context.
+You can grant those roles using following construct:
+
+```hcl
+module "project" {
+ source = "./fabric/modules/project"
+ name = "project-example"
+ iam = {
+ "roles/apigee.serviceAgent" = [
+ "serviceAccount:${module.project.service_accounts.robots.apigee}"
+ ]
+ }
+}
+# tftest modules=1 resources=2
+```
+
+This table lists all affected services and roles that you need to grant to service identities
+
+| service | service identity | role |
+|---|---|---|
+| apigee.googleapis.com | apigee | roles/apigee.serviceAgent |
+| artifactregistry.googleapis.com | artifactregistry | roles/artifactregistry.serviceAgent |
+| cloudasset.googleapis.com | cloudasset | roles/cloudasset.serviceAgent |
+| cloudbuild.googleapis.com | cloudbuild | roles/cloudbuild.builds.builder |
+| gkehub.googleapis.com | fleet | roles/gkehub.serviceAgent |
+| multiclusteringress.googleapis.com | multicluster-ingress | roles/multiclusteringress.serviceAgent |
+| pubsub.googleapis.com | pubsub | roles/pubsub.serviceAgent |
+| sqladmin.googleapis.com | sqladmin | roles/cloudsql.serviceAgent |
+
+
## Shared VPC
The module allows managing Shared VPC status for both hosts and service projects, and includes a simple way of assigning Shared VPC roles to service identities.
diff --git a/modules/project/service-accounts.tf b/modules/project/service-accounts.tf
index 1979958ba2..e93978a860 100644
--- a/modules/project/service-accounts.tf
+++ b/modules/project/service-accounts.tf
@@ -70,16 +70,19 @@ locals {
gke-mcs-importer = "${local.project.project_id}.svc.id.goog[gke-mcs/gke-mcs-importer]"
}
)
+ # JIT-ed service accounts are created without default roles granted, these needs to be assigned manually to them
+ # Roles can be found here: https://cloud.google.com/iam/docs/service-agents
+ # Remember to update "Service identities requiring manual IAM grants" in README.md when updating this list
service_accounts_jit_services = [
- "apigee.googleapis.com",
- "artifactregistry.googleapis.com",
- "cloudasset.googleapis.com",
- "gkehub.googleapis.com",
- "multiclusteringress.googleapis.com",
- "pubsub.googleapis.com",
- "secretmanager.googleapis.com",
- "sqladmin.googleapis.com",
- "cloudbuild.googleapis.com",
+ "apigee.googleapis.com", # grant roles/apigee.serviceAgent to apigee
+ "artifactregistry.googleapis.com", # grant roles/artifactregistry.serviceAgent to artifactregistry
+ "cloudasset.googleapis.com", # grant roles/cloudasset.serviceAgent to cloudasset
+ "cloudbuild.googleapis.com", # grant roles/cloudbuild.builds.builder to cloudbuild
+ "gkehub.googleapis.com", # grant roles/gkehub.serviceAgent to fleet
+ "multiclusteringress.googleapis.com", # grant roles/multiclusteringress.serviceAgent to multicluster-ingress
+ "pubsub.googleapis.com", # grant roles/pubsub.serviceAgent to pubsub
+ "secretmanager.googleapis.com", # no grants needed
+ "sqladmin.googleapis.com", # grant roles/cloudsql.serviceAgent to sqladmin (TODO: verify)
]
service_accounts_cmek_service_keys = distinct(flatten([
for s in keys(var.service_encryption_key_ids) : [
From 57f43f0f1e69a8689e3ba25e2ac9be6a40c6fe46 Mon Sep 17 00:00:00 2001
From: Ludovico Magnocavallo
Date: Sat, 11 Feb 2023 13:20:24 +0100
Subject: [PATCH 004/169] Update README.md
---
modules/project/README.md | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/modules/project/README.md b/modules/project/README.md
index de37503e2d..fbc4ab294d 100644
--- a/modules/project/README.md
+++ b/modules/project/README.md
@@ -139,9 +139,10 @@ module "project" {
```
### Service identities requiring manual IAM grants
-The module will create service identities at the time of the creation of the project instead creation of them at the time of first use.
-This allows granting these service identities roles in other projects which is usually necessary in Shared VPC context.
-You can grant those roles using following construct:
+
+The module will create service identities at project creation instead of creating of them at the time of first use. This allows granting these service identities roles in other projects, something which is usually necessary in a Shared VPC context.
+
+You can grant roles to service identities using the following construct:
```hcl
module "project" {
From 52468e6d0e7848a7856a6866c537943143fc56eb Mon Sep 17 00:00:00 2001
From: Luca Prete
Date: Sat, 11 Feb 2023 17:45:16 +0100
Subject: [PATCH 005/169] net-ilb: add example about ref existing MIG example
(#1151)
---
modules/net-ilb/README.md | 56 +++++++++++++++++++++++++++++++++++++++
1 file changed, 56 insertions(+)
diff --git a/modules/net-ilb/README.md b/modules/net-ilb/README.md
index c284c4c631..48c1d9081d 100644
--- a/modules/net-ilb/README.md
+++ b/modules/net-ilb/README.md
@@ -12,6 +12,62 @@ One other issue is a `Provider produced inconsistent final plan` error which is
## Examples
+### Reference existing MIGs
+
+This example shows how to reference existing Managed Infrastructure Groups (MIGs).
+
+```hcl
+module "instance_template" {
+ source = "./fabric/modules/compute-vm"
+ project_id = var.project_id
+ create_template = true
+ name = "vm-test"
+ service_account_create = true
+ zone = "europe-west1-b"
+
+ network_interfaces = [
+ {
+ network = var.vpc.self_link
+ subnetwork = var.subnet.self_link
+ }
+ ]
+
+ tags = [
+ "http-server"
+ ]
+}
+
+module "mig" {
+ source = "./fabric/modules/compute-mig"
+ project_id = var.project_id
+ location = "europe-west1"
+ name = "mig-test"
+ target_size = 1
+ instance_template = module.instance_template.template.self_link
+}
+
+module "ilb" {
+ source = "./fabric/modules/net-ilb"
+ project_id = var.project_id
+ region = "europe-west1"
+ name = "ilb-test"
+ service_label = "ilb-test"
+ vpc_config = {
+ network = var.vpc.self_link
+ subnetwork = var.subnet.self_link
+ }
+ backends = [{
+ group = module.mig.group_manager.instance_group
+ }]
+ health_check_config = {
+ http = {
+ port = 80
+ }
+ }
+}
+# tftest modules=3 resources=6
+```
+
### Externally managed instances
This examples shows how to create an ILB by combining externally managed instances (in a custom module or even outside of the current root module) in an unmanaged group. When using internally managed groups, remember to run `terraform apply` each time group instances change.
From 92a57becd869cd1abe33f85b11df033c2a26d72f Mon Sep 17 00:00:00 2001
From: Julio Diez
Date: Mon, 13 Feb 2023 21:54:03 +0100
Subject: [PATCH 006/169] Accessing Cloud Run privately, first use case
---
.../serverless/cloud-run-corporate/main.tf | 153 ++++++++++++++++++
.../serverless/cloud-run-corporate/outputs.tf | 20 +++
.../cloud-run-corporate/variables.tf | 65 ++++++++
3 files changed, 238 insertions(+)
create mode 100644 blueprints/serverless/cloud-run-corporate/main.tf
create mode 100644 blueprints/serverless/cloud-run-corporate/outputs.tf
create mode 100644 blueprints/serverless/cloud-run-corporate/variables.tf
diff --git a/blueprints/serverless/cloud-run-corporate/main.tf b/blueprints/serverless/cloud-run-corporate/main.tf
new file mode 100644
index 0000000000..5e16a9a1c6
--- /dev/null
+++ b/blueprints/serverless/cloud-run-corporate/main.tf
@@ -0,0 +1,153 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+locals {
+ domain_cr_host = format("%s.",
+ trimprefix(module.cloud_run_host.service.status[0].url, "https://"))
+}
+
+###############################################################################
+# Projects #
+###############################################################################
+
+# Main or host project, depending on if there are service projects
+module "project_host" {
+ source = "../../../modules/project"
+ name = var.prj_host_id
+ project_create = var.prj_host_create != null
+ billing_account = try(var.prj_host_create.billing_account_id, null)
+ parent = try(var.prj_host_create.parent, null)
+ shared_vpc_host_config = {
+ enabled = true
+ }
+ services = [
+ "run.googleapis.com",
+ "compute.googleapis.com",
+ "dns.googleapis.com"
+ ]
+}
+
+###############################################################################
+# Cloud Run #
+###############################################################################
+
+# Cloud Run service in main project
+module "cloud_run_host" {
+ source = "../../../modules/cloud-run"
+ project_id = module.project_host.project_id
+ name = var.run_svc_name
+ region = var.region
+ containers = [{
+ image = var.image
+ options = null
+ ports = null
+ resources = null
+ volume_mounts = null
+ }]
+ iam = {
+ "roles/run.invoker" = ["allUsers"]
+ }
+ ingress_settings = var.ingress_settings
+}
+
+###############################################################################
+# VPCs #
+###############################################################################
+
+# VPC in main or host project
+module "vpc_host" {
+ source = "../../../modules/net-vpc"
+ project_id = module.project_host.project_id
+ name = "vpc-host"
+ subnets = [
+ {
+ ip_cidr_range = var.ip_ranges_host.subnet
+ name = "subnet-host"
+ region = var.region
+ enable_private_access = true # PGA enabled
+ }
+ ]
+}
+
+# VPC Firewall with default config, IAP for SSH enabled
+module "firewall_host" {
+ source = "../../../modules/net-vpc-firewall"
+ project_id = module.project_host.project_id
+ network = module.vpc_host.name
+ default_rules_config = {
+ http_ranges = []
+ https_ranges = []
+ }
+}
+
+###############################################################################
+# PSC #
+###############################################################################
+
+module "psc_addr_host" {
+ source = "../../../modules/net-address"
+ project_id = module.project_host.project_id
+ psc_addresses = {
+ psc-addr-host = {
+ address = var.ip_ranges_host.psc_addr
+ network = module.vpc_host.self_link
+ }
+ }
+}
+
+resource "google_compute_global_forwarding_rule" "psc_endpoint_host" {
+ provider = google-beta
+ project = module.project_host.project_id
+ name = "pscaddrhost"
+ network = module.vpc_host.self_link
+ ip_address = module.psc_addr_host.psc_addresses["psc-addr-host"].self_link
+ target = "vpc-sc"
+ load_balancing_scheme = ""
+}
+
+###############################################################################
+# VMs #
+###############################################################################
+
+module "vm_test_host" {
+ source = "../../../modules/compute-vm"
+ project_id = module.project_host.project_id
+ zone = "${var.region}-b"
+ name = "vm-test-host"
+ instance_type = "e2-micro"
+ network_interfaces = [{
+ network = module.vpc_host.self_link
+ subnetwork = module.vpc_host.subnet_self_links["${var.region}/subnet-host"]
+ }]
+ tags = ["ssh"]
+}
+
+###############################################################################
+# DNS #
+###############################################################################
+
+module "private_dns_host" {
+ source = "../../../modules/dns"
+ project_id = module.project_host.project_id
+ type = "private"
+ name = "dns-host"
+ client_networks = [module.vpc_host.self_link]
+ domain = local.domain_cr_host
+ recordsets = {
+ "A " = { records = [module.psc_addr_host.psc_addresses["psc-addr-host"].address] }
+ }
+}
diff --git a/blueprints/serverless/cloud-run-corporate/outputs.tf b/blueprints/serverless/cloud-run-corporate/outputs.tf
new file mode 100644
index 0000000000..abf2c03b04
--- /dev/null
+++ b/blueprints/serverless/cloud-run-corporate/outputs.tf
@@ -0,0 +1,20 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "default_URL" {
+ description = "Cloud Run service default URL."
+ value = module.cloud_run_host.service.status[0].url
+}
diff --git a/blueprints/serverless/cloud-run-corporate/variables.tf b/blueprints/serverless/cloud-run-corporate/variables.tf
new file mode 100644
index 0000000000..729ca57561
--- /dev/null
+++ b/blueprints/serverless/cloud-run-corporate/variables.tf
@@ -0,0 +1,65 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "image" {
+ description = "Container image to deploy."
+ type = string
+ default = "us-docker.pkg.dev/cloudrun/container/hello"
+}
+
+variable "ingress_settings" {
+ description = "Ingress traffic sources allowed to call the service."
+ type = string
+ default = "all"
+}
+
+variable "ip_ranges_host" {
+ description = "IPs or IP ranges used by VPCs"
+ type = object({
+ subnet = string
+ psc_addr = string
+ })
+ default = {
+ subnet = "10.0.1.0/24"
+ psc_addr = "10.0.0.100"
+ }
+}
+
+variable "prj_host_create" {
+ description = "Parameters for the creation of a host project."
+ type = object({
+ billing_account_id = string
+ parent = string
+ })
+ default = null
+}
+
+variable "prj_host_id" {
+ description = "Host Project ID."
+ type = string
+}
+
+variable "region" {
+ description = "Cloud region where resource will be deployed."
+ type = string
+ default = "europe-west1"
+}
+
+variable "run_svc_name" {
+ description = "Cloud Run service name."
+ type = string
+ default = "hello"
+}
From 28f4c3170e87a0efee9ff2c23ec1d38fc7a6b279 Mon Sep 17 00:00:00 2001
From: Julio Diez
Date: Mon, 13 Feb 2023 21:57:54 +0100
Subject: [PATCH 007/169] Initial README
---
.../serverless/cloud-run-corporate/README.md | 71 +++++++++++++++++++
1 file changed, 71 insertions(+)
create mode 100644 blueprints/serverless/cloud-run-corporate/README.md
diff --git a/blueprints/serverless/cloud-run-corporate/README.md b/blueprints/serverless/cloud-run-corporate/README.md
new file mode 100644
index 0000000000..e250f55b58
--- /dev/null
+++ b/blueprints/serverless/cloud-run-corporate/README.md
@@ -0,0 +1,71 @@
+# Cloud Run Corporate
+
+## Introduction
+
+This blueprint contains all the necessary Terraform modules to build and privately expose a Cloud Run service in a variety of use cases.
+
+The content of this blueprint corresponds to the chapter '_Developing an enterprise application - The corporate environment_' of the __Serverless Networking Guide__ (to be released soon). This guide is an easy to follow introduction to Cloud Run, where a couple of friendly characters will guide you from the basics to more advanced topics with a very practical approach and in record time! The code here complements this learning and allows you to test the scenarios presented and your knowledge.
+
+## Architecture
+
+## Prerequisites
+
+## Spinning up the architecture
+
+### General steps
+
+1. Clone the repo to your local machine or Cloud Shell:
+```bash
+git clone https://github.com/GoogleCloudPlatform/cloud-foundation-fabric
+```
+
+2. Change to the directory of the blueprint:
+```bash
+cd cloud-foundation-fabric/blueprints/serverless/cloud-run-corporate
+```
+You should see this README and some terraform files.
+
+3. To deploy a specific use case, you will need to create a file in this directory called `terraform.tfvars` and follow the corresponding instructions to set variables. Sometimes values that are meant to be substituted will be shown inside brackets but you need to omit these brackets. E.g.:
+```tfvars
+project_id = "[your-project_id]"
+```
+may become
+```tfvars
+project_id = "spiritual-hour-331417"
+```
+
+Although each use case is somehow built around the previous one they are self-contained so you can deploy any of them at will.
+
+4. The usual terraform commands will do the work:
+```bash
+terraform init
+terraform plan
+terraform apply
+```
+
+The resource creation will take a few minutes but when it’s complete, you should see an output stating the command completed successfully with a list of the created resources, and some output variables with information to access your service.
+
+__Congratulations!__ You have successfully deployed the use case you chose based on the variables configuration.
+
+### Use case 1: Access to Cloud Run from a VM in the project
+
+### Use case 2:
+
+### Use case 3:
+
+### Use case 4:
+
+### Use case 5:
+
+## Cleaning up your environment
+
+The easiest way to remove all the deployed resources is to run the following command:
+```bash
+terraform destroy
+```
+The above command will delete the associated resources so there will be no billable charges made afterwards. IAP Brands, though, can only be created once per project and not deleted. Destroying a Terraform-managed IAP Brand will remove it from state but will not delete it from Google Cloud.
+
+
+
+
+## Tests
From 7bbeac805e1dde5d1c14e52368606073755342a9 Mon Sep 17 00:00:00 2001
From: lcaggio
Date: Tue, 14 Feb 2023 08:43:15 +0100
Subject: [PATCH 008/169] Add 'max_time_travel_hours ' on BQ module
---
modules/bigquery-dataset/README.md | 9 +++++----
modules/bigquery-dataset/main.tf | 2 +-
modules/bigquery-dataset/variables.tf | 13 +++++--------
3 files changed, 11 insertions(+), 13 deletions(-)
diff --git a/modules/bigquery-dataset/README.md b/modules/bigquery-dataset/README.md
index 381ffab7ce..6dbd120c20 100644
--- a/modules/bigquery-dataset/README.md
+++ b/modules/bigquery-dataset/README.md
@@ -67,6 +67,7 @@ module "bigquery-dataset" {
default_table_expiration_ms = 3600000
default_partition_expiration_ms = null
delete_contents_on_destroy = false
+ max_time_travel_hours = 168
}
}
# tftest modules=1 resources=1
@@ -178,7 +179,7 @@ module "bigquery-dataset" {
| name | description | type | required | default |
|---|---|:---:|:---:|:---:|
| [id](variables.tf#L69) | Dataset id. | string
| ✓ | |
-| [project_id](variables.tf#L100) | Id of the project where datasets will be created. | string
| ✓ | |
+| [project_id](variables.tf#L97) | Id of the project where datasets will be created. | string
| ✓ | |
| [access](variables.tf#L17) | Map of access rules with role and identity type. Keys are arbitrary and must match those in the `access_identities` variable, types are `domain`, `group`, `special_group`, `user`, `view`. | map(object({…}))
| | {}
|
| [access_identities](variables.tf#L33) | Map of access identities used for basic access roles. View identities have the format 'project_id\|dataset_id\|table_id'. | map(string)
| | {}
|
| [dataset_access](variables.tf#L39) | Set access in the dataset resource instead of using separate resources. | bool
| | false
|
@@ -188,9 +189,9 @@ module "bigquery-dataset" {
| [iam](variables.tf#L63) | IAM bindings in {ROLE => [MEMBERS]} format. Mutually exclusive with the access_* variables used for basic roles. | map(list(string))
| | {}
|
| [labels](variables.tf#L74) | Dataset labels. | map(string)
| | {}
|
| [location](variables.tf#L80) | Dataset location. | string
| | "EU"
|
-| [options](variables.tf#L86) | Dataset options. | object({…})
| | {…}
|
-| [tables](variables.tf#L105) | Table definitions. Options and partitioning default to null. Partitioning can only use `range` or `time`, set the unused one to null. | map(object({…}))
| | {}
|
-| [views](variables.tf#L133) | View definitions. | map(object({…}))
| | {}
|
+| [options](variables.tf#L86) | Dataset options. | object({…})
| | {}
|
+| [tables](variables.tf#L102) | Table definitions. Options and partitioning default to null. Partitioning can only use `range` or `time`, set the unused one to null. | map(object({…}))
| | {}
|
+| [views](variables.tf#L130) | View definitions. | map(object({…}))
| | {}
|
## Outputs
diff --git a/modules/bigquery-dataset/main.tf b/modules/bigquery-dataset/main.tf
index 47f8fcb53b..f832cd85c3 100644
--- a/modules/bigquery-dataset/main.tf
+++ b/modules/bigquery-dataset/main.tf
@@ -42,7 +42,7 @@ resource "google_bigquery_dataset" "default" {
delete_contents_on_destroy = var.options.delete_contents_on_destroy
default_table_expiration_ms = var.options.default_table_expiration_ms
default_partition_expiration_ms = var.options.default_partition_expiration_ms
-
+ max_time_travel_hours = var.options.max_time_travel_hours
dynamic "access" {
for_each = var.dataset_access ? local.access_domain : {}
content {
diff --git a/modules/bigquery-dataset/variables.tf b/modules/bigquery-dataset/variables.tf
index 5f8028abfb..b44b66585b 100644
--- a/modules/bigquery-dataset/variables.tf
+++ b/modules/bigquery-dataset/variables.tf
@@ -86,15 +86,12 @@ variable "location" {
variable "options" {
description = "Dataset options."
type = object({
- default_table_expiration_ms = number
- default_partition_expiration_ms = number
- delete_contents_on_destroy = bool
+ default_table_expiration_ms = optional(number, null)
+ default_partition_expiration_ms = optional(number, null)
+ delete_contents_on_destroy = optional(bool, false)
+ max_time_travel_hours = optional(number, 168)
})
- default = {
- default_table_expiration_ms = null
- default_partition_expiration_ms = null
- delete_contents_on_destroy = false
- }
+ default = {}
}
variable "project_id" {
From 742b5bab62ab29f573f707cd9f43d417893f91e0 Mon Sep 17 00:00:00 2001
From: Julio Castillo
Date: Tue, 14 Feb 2023 11:29:19 +0200
Subject: [PATCH 009/169] Fix tfvars sample for fast bootstrap stage
---
fast/stages/0-bootstrap/terraform.tfvars.sample | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/fast/stages/0-bootstrap/terraform.tfvars.sample b/fast/stages/0-bootstrap/terraform.tfvars.sample
index 66710ba4a6..a134f8dc5d 100644
--- a/fast/stages/0-bootstrap/terraform.tfvars.sample
+++ b/fast/stages/0-bootstrap/terraform.tfvars.sample
@@ -1,15 +1,14 @@
# use `gcloud beta billing accounts list`
# if you have too many accounts, check the Cloud Console :)
billing_account = {
- id = "012345-67890A-BCDEF0"
- organization_id = 1234567890
+ id = "012345-67890A-BCDEF0"
}
# use `gcloud organizations list`
organization = {
- domain = "example.org"
- id = 1234567890
- customer_id = "C000001"
+ domain = "example.org"
+ id = 1234567890
+ customer_id = "C000001"
}
outputs_location = "~/fast-config"
From d0934903aa990a547d8eeadec4e4df510ecf3a8f Mon Sep 17 00:00:00 2001
From: Julio Diez
Date: Tue, 14 Feb 2023 12:57:43 +0100
Subject: [PATCH 010/169] Use a more curl friendly image
---
blueprints/serverless/cloud-run-corporate/variables.tf | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/blueprints/serverless/cloud-run-corporate/variables.tf b/blueprints/serverless/cloud-run-corporate/variables.tf
index 729ca57561..540a764fd4 100644
--- a/blueprints/serverless/cloud-run-corporate/variables.tf
+++ b/blueprints/serverless/cloud-run-corporate/variables.tf
@@ -17,7 +17,7 @@
variable "image" {
description = "Container image to deploy."
type = string
- default = "us-docker.pkg.dev/cloudrun/container/hello"
+ default = "us-docker.pkg.dev/google-samples/containers/gke/whereami:v1.2.19"
}
variable "ingress_settings" {
From e8334857ff46b12c2396214675bb8e15c6284eec Mon Sep 17 00:00:00 2001
From: Chema Polo
Date: Wed, 15 Feb 2023 06:28:47 +0100
Subject: [PATCH 011/169] Update main.tf (#1158)
replaced .secondary_pod_range by var.pod_range.secondary_pod_range that is the object which contins create, cidr an name attributes.
---
modules/gke-nodepool/main.tf | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/modules/gke-nodepool/main.tf b/modules/gke-nodepool/main.tf
index ad0c053f35..9ae4cf2844 100644
--- a/modules/gke-nodepool/main.tf
+++ b/modules/gke-nodepool/main.tf
@@ -115,9 +115,9 @@ resource "google_container_node_pool" "nodepool" {
dynamic "network_config" {
for_each = var.pod_range != null ? [""] : []
content {
- create_pod_range = var.pod_range.create
- pod_ipv4_cidr_block = var.pod_range.cidr
- pod_range = var.pod_range.name
+ create_pod_range = var.pod_range.secondary_pod_range.create
+ pod_ipv4_cidr_block = var.pod_range.secondary_pod_range.cidr
+ pod_range = var.pod_range.secondary_pod_range.name
}
}
From 36a7347744b84fe38e463feff18d785d890afc32 Mon Sep 17 00:00:00 2001
From: Ludovico Magnocavallo
Date: Wed, 15 Feb 2023 06:42:14 +0100
Subject: [PATCH 012/169] FAST stage docs cleanup (#1145)
* top-level and stage 0
* stage 1
* net peering
* networking
* networking
* security
* gke, dp
* checks
---
fast/stage-links.sh | 5 +
fast/stages/0-bootstrap/README.md | 111 ++++++-----
fast/stages/1-resman/README.md | 74 +++++---
fast/stages/2-networking-a-peering/README.md | 170 ++++++++++-------
fast/stages/2-networking-b-vpn/README.md | 166 ++++++++++-------
fast/stages/2-networking-c-nva/README.md | 173 +++++++++++-------
.../2-networking-d-separate-envs/README.md | 163 ++++++++++-------
fast/stages/2-security/README.md | 82 +++++----
fast/stages/3-data-platform/dev/README.md | 103 ++++++-----
fast/stages/3-gke-multitenant/dev/README.md | 108 ++++++-----
fast/stages/CLEANUP.md | 14 +-
fast/stages/COMPANION.md | 56 +++---
fast/stages/FAQ.md | 42 ++---
13 files changed, 740 insertions(+), 527 deletions(-)
diff --git a/fast/stage-links.sh b/fast/stage-links.sh
index 79d1973fa1..52c9e5ae6d 100755
--- a/fast/stage-links.sh
+++ b/fast/stage-links.sh
@@ -78,6 +78,11 @@ case $STAGE_NAME in
TFVARS="tfvars/0-bootstrap.auto.tfvars.json
tfvars/1-resman.auto.tfvars.json"
;;
+"2-security"*)
+ PROVIDER="providers/2-security-providers.tf"
+ TFVARS="tfvars/0-bootstrap.auto.tfvars.json
+ tfvars/1-resman.auto.tfvars.json"
+ ;;
*)
# check for a "dev" stage 3
echo "no stage found, trying for parent stage 3..."
diff --git a/fast/stages/0-bootstrap/README.md b/fast/stages/0-bootstrap/README.md
index 2cab11510d..e1bb2948a4 100644
--- a/fast/stages/0-bootstrap/README.md
+++ b/fast/stages/0-bootstrap/README.md
@@ -14,6 +14,28 @@ Use the following diagram as a simple high level reference for the following sec
+## Table of contents
+
+- [Design overview and choices](#design-overview-and-choices)
+ - [User groups](#user-groups)
+ - [Organization-level IAM](#organization-level-iam)
+ - [Automation project and resources](#automation-project-and-resources)
+ - [Billing account](#billing-account)
+ - [Organization-level logging](#organization-level-logging)
+ - [Naming](#naming)
+ - [Workload Identity Federation and CI/CD](#workload-identity-federation-and-cicd)
+- [How to run this stage](#how-to-run-this-stage)
+ - [Prerequisites](#prerequisites)
+ - [Output files and cross-stage variables](#output-files-and-cross-stage-variables)
+ - [Running the stage](#running-the-stage)
+- [Customizations](#customizations)
+ - [Group names](#group-names)
+ - [IAM](#iam)
+ - [Log sinks and log destinations](#log-sinks-and-log-destinations)
+ - [Names and naming convention](#names-and-naming-convention)
+ - [Workload Identity Federation](#workload-identity-federation)
+ - [CI/CD repositories](#cicd-repositories)
+
## Design overview and choices
As mentioned above, this stage only does the bare minimum required to bootstrap automation, and ensure that base audit and billing exports are in place from the start to provide some measure of accountability, even before the security configurations are applied in a later stage.
@@ -80,7 +102,7 @@ The convention is used in its full form only for specific resources with globall
The [Customizations](#names-and-naming-convention) section on names below explains how to configure tokens, or implement a different naming convention.
-## Workload Identity Federation and CI/CD
+### Workload Identity Federation and CI/CD
This stage also implements initial support for two interrelated features
@@ -124,7 +146,7 @@ To quickly self-grant the above roles, run the following code snippet as the ini
export FAST_BU=$(gcloud config list --format 'value(core.account)')
# find and set your org id
-gcloud organizations list --filter display_name:$partofyourdomain
+gcloud organizations list
export FAST_ORG_ID=123456
# set needed roles
@@ -139,25 +161,6 @@ done
Then make sure the same user is also part of the `gcp-organization-admins` group so that impersonating the automation service account later on will be possible.
-#### Billing account in a different organization
-
-If you are using a billing account belonging to a different organization (e.g. in multiple organization setups), some initial configurations are needed to ensure the identities running this stage can assign billing-related roles.
-
-If the billing organization is managed by another version of this stage, we leverage the `organizationIamAdmin` role created there, to allow restricted granting of billing roles at the organization level.
-
-If that's not the case, an equivalent role needs to exist, or the predefined `resourcemanager.organizationAdmin` role can be used if not managed authoritatively. The role name then needs to be manually changed in the `billing.tf` file, in the `google_organization_iam_binding` resource.
-
-The identity applying this stage for the first time also needs two roles in billing organization, they can be removed after the first `apply` completes successfully:
-
-```bash
-export FAST_BILLING_ORG_ID=789012
-export FAST_ROLES=(roles/billing.admin roles/resourcemanager.organizationAdmin)
-for role in $FAST_ROLES; do
- gcloud organizations add-iam-policy-binding $FAST_BILLING_ORG_ID \
- --member user:$FAST_BU --role $role
-done
-```
-
#### Standalone billing account
If you are using a standalone billing account, the identity applying this stage for the first time needs to be a billing account administrator:
@@ -187,7 +190,7 @@ Please note that FAST also supports an additional group for users with permissio
Then make sure you have configured the correct values for the following variables by providing a `terraform.tfvars` file:
- `billing_account`
- an object containing `id` as the id of your billing account, derived from the Cloud Console UI or by running `gcloud beta billing accounts list`, and `organization_id` as the id of the organization owning it, or `null` to use the billing account in isolation
+ an object containing `id` as the id of your billing account, derived from the Cloud Console UI or by running `gcloud beta billing accounts list`, and the `is_org_level` flag that controls whether organization or account-level bindings are used, and a billing export project and dataset are created
- `groups`
the name mappings for your groups, if you're following the default convention you can leave this to the provided default
- `organization.id`, `organization.domain`, `organization.customer_id`
@@ -202,7 +205,6 @@ You can also adapt the example that follows to your needs:
# if you have too many accounts, check the Cloud Console :)
billing_account = {
id = "012345-67890A-BCDEF0"
- organization_id = 1234567890
}
# use `gcloud organizations list`
@@ -237,18 +239,18 @@ Below is the outline of the output files generated by all stages, which is ident
```bash
[path specified in outputs_location]
├── providers
-│ ├── 00-bootstrap-providers.tf
-│ ├── 01-resman-providers.tf
-│ ├── 02-networking-providers.tf
-│ ├── 02-security-providers.tf
-│ ├── 03-project-factory-dev-providers.tf
-│ ├── 03-project-factory-prod-providers.tf
-│ └── 99-sandbox-providers.tf
+│ ├── 0-bootstrap-providers.tf
+│ ├── 1-resman-providers.tf
+│ ├── 2-networking-providers.tf
+│ ├── 2-security-providers.tf
+│ ├── 3-project-factory-dev-providers.tf
+│ ├── 3-project-factory-prod-providers.tf
+│ └── 9-sandbox-providers.tf
└── tfvars
-│ ├── 00-bootstrap.auto.tfvars.json
-│ ├── 01-resman.auto.tfvars.json
-│ ├── 02-networking.auto.tfvars.json
-│ └── 02-security.auto.tfvars.json
+│ ├── 0-bootstrap.auto.tfvars.json
+│ ├── 1-resman.auto.tfvars.json
+│ ├── 2-networking.auto.tfvars.json
+│ └── 2-security.auto.tfvars.json
└── workflows
└── [optional depending on the configured CI/CD repositories]
```
@@ -267,17 +269,34 @@ terraform apply \
> If you see an error related to project name already exists, please make sure the project name is unique or the project was not deleted recently
-Once the initial `apply` completes successfully, configure a remote backend using the new GCS bucket, and impersonation on the automation service account for this stage. To do this you can use the generated `providers.tf` file if you have configured output files as described above, or extract its contents from Terraform's output, then migrate state with `terraform init`:
+Once the initial `apply` completes successfully, configure a remote backend using the new GCS bucket, and impersonation on the automation service account for this stage. To do this you can use the generated `providers.tf` file from either
+
+- the local filesystem if you have configured output files as described above
+- the GCS bucket where output files are always stored
+- Terraform outputs (not recommended as it's more complex)
+
+The following two snippets show how to leverage the `stage-links.sh` script in the root FAST folder to fetch the commands required for output files linking or copying, using either the local output folder configured via Terraform variables, or the GCS bucket which can be derived from the `automation` output.
+
+```bash
+../../stage-links.sh ~/fast-config
+
+# copy and paste the following commands for '0-bootstrap'
+
+ln -s ~/fast-config/providers/0-bootstrap-providers.tf ./
+```
+
+```bash
+../../stage-links.sh gs://xxx-prod-iac-core-outputs-0
+
+# copy and paste the following commands for '0-bootstrap'
+
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/providers/0-bootstrap-providers.tf ./
+```
+
+Copy/paste the command returned by the script to link or copy the provider file, then migrate state with `terraform init` and run `terraform apply`:
```bash
-# if using output files via the outputs_location and set to `~/fast-config`
-ln -s ~/fast-config/providers/00-bootstrap* ./
-# or from outputs if not using output files
-terraform output -json providers | jq -r '.["00-bootstrap"]' \
- > providers.tf
-# migrate state to GCS bucket configured in providers file
terraform init -migrate-state
-# run terraform apply to remove the bootstrap_user iam binding
terraform apply
```
@@ -334,7 +353,7 @@ You can customize organization-level logs through the `log_sinks` variable in tw
- creating additional log sinks to capture more logs
- changing the destination of captured logs
-By default, all logs are exported to Bigquery, but FAST can create sinks to Cloud Logging Buckets, GCS, or PubSub.
+By default, all logs are exported to a log bucket, but FAST can create sinks to BigQuery, GCS, or PubSub.
If you need to capture additional logs, please refer to GCP's documentation on [scenarios for exporting logging data](https://cloud.google.com/architecture/exporting-stackdriver-logging-for-security-and-access-analytics), where you can find ready-made filter expressions for different use cases.
@@ -400,12 +419,6 @@ cicd_repositories = {
name = "my-gh-org/fast-bootstrap"
type = "github"
}
- cicd = {
- branch = null
- identity_provider = "github-sample"
- name = "my-gh-org/fast-cicd"
- type = "github"
- }
resman = {
branch = "main"
identity_provider = "github-sample"
diff --git a/fast/stages/1-resman/README.md b/fast/stages/1-resman/README.md
index c2091eb50a..971c69633d 100644
--- a/fast/stages/1-resman/README.md
+++ b/fast/stages/1-resman/README.md
@@ -13,6 +13,22 @@ The following diagram is a high level reference of the resources created and man
+## Table of contents
+
+- [Design overview and choices](#design-overview-and-choices)
+ - [Multitenancy](#multitenancy)
+ - [Workload Identity Federation and CI/CD](#workload-identity-federation-and-cicd)
+- [How to run this stage](#how-to-run-this-stage)
+ - [Provider and Terraform variables](#provider-and-terraform-variables)
+ - [Impersonating the automation service account](#impersonating-the-automation-service-account)
+ - [Variable configuration](#variable-configuration)
+ - [Running the stage](#running-the-stage)
+- [Customizations](#customizations)
+ - [Team folders](#team-folders)
+ - [Organization Policies](#organization-policies)
+ - [IAM](#iam)
+ - [Additional folders](#additional-folders)
+
## Design overview and choices
Despite its simplicity, this stage implements the basics of a design that we've seen working well for a variety of customers, where the hierarchy is laid out following two conceptually different approaches:
@@ -54,51 +70,49 @@ It's of course possible to run this stage in isolation, but that's outside the s
Before running this stage, you need to make sure you have the correct credentials and permissions, and localize variables by assigning values that match your configuration.
-### Providers configuration
-
-The default way of making sure you have the right permissions, is to use the identity of the service account pre-created for this stage during bootstrap, and that you are a member of the group that can impersonate it via provider-level configuration (`gcp-devops` or `organization-admins`).
+### Provider and Terraform variables
-To simplify setup, the previous stage pre-configures a valid providers file in its output, and optionally writes it to a local file if the `outputs_location` variable is set to a valid path.
+As all other FAST stages, the [mechanism used to pass variable values and pre-built provider files from one stage to the next](../0-bootstrap/README.md#output-files-and-cross-stage-variables) is also leveraged here.
-If you have set a valid value for `outputs_location` in the bootstrap stage (see the [bootstrap stage README](../0-bootstrap/#output-files-and-cross-stage-variables) for more details), simply link the relevant `providers.tf` file from this stage's folder in the path you specified:
+The commands to link or copy the provider and terraform variable files can be easily derived from the `stage-links.sh` script in the FAST root folder, passing it a single argument with the local output files folder (if configured) or the GCS output bucket in the automation project (derived from stage 0 outputs). The following examples demonstrate both cases, and the resulting commands that then need to be copy/pasted and run.
```bash
-# `outputs_location` is set to `~/fast-config`
-ln -s ~/fast-config/providers/01-resman-providers.tf .
-```
+../../stage-links.sh ~/fast-config
-If you have not configured `outputs_location` in bootstrap, you can derive the providers file from that stage's outputs:
+# copy and paste the following commands for '1-resman'
-```bash
-cd ../0-bootstrap
-terraform output -json providers | jq -r '.["01-resman"]' \
- > ../1-resman/providers.tf
+ln -s ~/fast-config/providers/1-resman-providers.tf ./
+ln -s ~/fast-config/tfvars/globals.auto.tfvars.json ./
+ln -s ~/fast-config/tfvars/0-bootstrap.auto.tfvars.json ./
```
-If you want to continue to rely on `outputs_location` logic, create a `terraform.tfvars` file and configure it as described [here](../0-bootstrap/#output-files-and-cross-stage-variables).
+```bash
+../../stage-links.sh gs://xxx-prod-iac-core-outputs-0
-### Variable configuration
+# copy and paste the following commands for '1-resman'
-There are two broad sets of variables you will need to fill in:
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/providers/1-resman-providers.tf ./
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/tfvars/globals.auto.tfvars.json ./
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/tfvars/0-bootstrap.auto.tfvars.json ./
+```
-- variables shared by other stages (org id, billing account id, etc.), or derived from a resource managed by a different stage (folder id, automation project id, etc.)
-- variables specific to resources managed by this stage
+### Impersonating the automation service account
-To avoid the tedious job of filling in the first group of variable with values derived from other stages' outputs, the same mechanism used above for the provider configuration can be used to leverage pre-configured `.tfvars` files.
+The preconfigured provider file uses impersonation to run with this stage's automation service account's credentials. The `gcp-devops` and `organization-admins` groups have the necessary IAM bindings in place to do that, so make sure the current user is a member of one of those groups.
-If you configured a valid path for `outputs_location` in the bootstrap stage, simply link the relevant `*.auto.tfvars.json` files from the outputs folder. For this stage, you need the `globals.auto.tfvars.json` file containing global values compiled manually for the bootstrap stage, and `0-bootstrap.auto.tfvars.json` containing values derived from resources managed by the bootstrap stage:
+### Variable configuration
-```bash
-# `outputs_location` is set to `~/fast-config`
-ln -s ~/fast-config/tfvars/globals.auto.tfvars.json .
-ln -s ~/fast-config/tfvars/0-bootstrap.auto.tfvars.json .
-```
+Variables in this stage -- like most other FAST stages -- are broadly divided into three separate sets:
+
+- variables which refer to global values for the whole organization (org id, billing account id, prefix, etc.), which are pre-populated via the `globals.auto.tfvars.json` file linked or copied above
+- variables which refer to resources managed by previous stage, which are prepopulated here via the `0-bootstrap.auto.tfvars.json` file linked or copied above
+- and finally variables that optionally control this stage's behaviour and customizations, and can to be set in a custom `terraform.tfvars` file
-A second set of variables is specific to this stage, they are all optional so if you need to customize them, create an extra `terraform.tfvars` file.
+The latter set is explained in the [Customization](#customizations) sections below, and the full list can be found in the [Variables](#variables) table at the bottom of this document.
-Refer to the [Variables](#variables) table at the bottom of this document, for a full list of variables, their origin (e.g. a stage or specific to this one), and descriptions explaining their meaning. The sections below also describe some of the possible customizations. For billing configurations, refer to the [Bootstrap documentation on billing](../0-bootstrap/README.md#billing-account) as the `billing_account` variable is identical across all stages.
+### Running the stage
-Once done, you can run this stage:
+Once provider and variable values are in place and the correct user is configured, the stage can be run:
```bash
terraform init
@@ -139,9 +153,9 @@ This allows to centralize the minimum set of resources to delegate control of ea
### Organization policies
-Organization policies are laid out in an explicit manner in the `organization.tf` file, so it's fairly easy to add or remove specific policies.
+Organization policies leverage -- with one exception -- the built-in factory implemented in the organization module, and configured via the yaml files in the `data` folder. To edit organization policies, check and edit the files there.
-For policies where additional data is needed, a root-level `organization_policy_configs` variable allows passing in specific data. Its built-in use to add additional organizations to the [Domain Restricted Sharing](https://cloud.google.com/resource-manager/docs/organization-policy/restricting-domains) policy, can be taken as an example on how to leverage it for additional customizations.
+The one exception is [Domain Restricted Sharing](https://cloud.google.com/resource-manager/docs/organization-policy/restricting-domains), which is made dynamic and implemented in code so as to auto-add the current organization's customer id. The `organization_policy_configs` variable allow to easily add ids from third party organizations if needed.
### IAM
diff --git a/fast/stages/2-networking-a-peering/README.md b/fast/stages/2-networking-a-peering/README.md
index 7966ce80bb..c066423cdd 100644
--- a/fast/stages/2-networking-a-peering/README.md
+++ b/fast/stages/2-networking-a-peering/README.md
@@ -15,6 +15,33 @@ The following diagram illustrates the high-level design, and should be used as a
+## Table of contents
+
+- [Design overview and choices](#design-overview-and-choices)
+ - [VPC design](#vpc-design)
+ - [External connectivity](#external-connectivity)
+ - [Internal connectivity](#internal-connectivity)
+ - [IP ranges, subnetting, routing](#ip-ranges-subnetting-routing)
+ - [Internet egress](#internet-egress)
+ - [VPC and Hierarchical Firewall](#vpc-and-hierarchical-firewall)
+ - [DNS](#dns)
+- [Stage structure and files layout](#stage-structure-and-files-layout)
+ - [VPCs](#vpcs)
+ - [VPNs](#vpns)
+ - [Routing and BGP](#routing-and-bgp)
+ - [Firewall](#firewall)
+ - [DNS architecture](#dns-architecture)
+ - [Private Google Access](#private-google-access)
+- [How to run this stage](#how-to-run-this-stage)
+ - [Provider and Terraform variables](#provider-and-terraform-variables)
+ - [Impersonating the automation service account](#impersonating-the-automation-service-account)
+ - [Variable configuration](#variable-configuration)
+ - [Running the stage](#running-the-stage)
+ - [Post-deployment activities](#post-deployment-activities)
+- [Customizations](#customizations)
+ - [Changing default regions](#changing-default-regions)
+ - [Adding an environment](#adding-an-environment)
+
## Design overview and choices
### VPC design
@@ -44,13 +71,13 @@ As mentioned initially, there are of course other ways to implement internal con
This is a summary of the main options:
-- [HA VPN](https://cloud.google.com/network-connectivity/docs/vpn/concepts/topologies) (implemented by [02-networking-vpn](../2-networking-b-vpn/))
+- [HA VPN](https://cloud.google.com/network-connectivity/docs/vpn/concepts/topologies) (implemented by [2-networking-vpn](../2-networking-b-vpn/))
- Pros: simple compatibility with GCP services that leverage peering internally, better control on routes, avoids peering groups shared quotas and limits
- Cons: additional cost, marginal increase in latency, requires multiple tunnels for full bandwidth
- [VPC Peering](https://cloud.google.com/vpc/docs/vpc-peering) (implemented here)
- Pros: no additional costs, full bandwidth with no configurations, no extra latency, total environment isolation
- Cons: no transitivity (e.g. to GKE masters, Cloud SQL, etc.), no selective exchange of routes, several quotas and limits shared between VPCs in a peering group
-- [Multi-NIC appliances](https://cloud.google.com/architecture/best-practices-vpc-design#multi-nic) (implemented by [02-networking-nva](../2-networking-c-nva/))
+- [Multi-NIC appliances](https://cloud.google.com/architecture/best-practices-vpc-design#multi-nic) (implemented by [2-networking-nva](../2-networking-c-nva/))
- Pros: additional security features (e.g. IPS), potentially better integration with on-prem systems by using the same vendor
- Cons: complex HA/failover setup, limited by VM bandwidth and scale, additional costs for VMs and licenses, out of band management of a critical cloud component
@@ -120,58 +147,7 @@ From cloud, the `example.com` domain (used as a placeholder) is forwarded to on-
This configuration is battle-tested, and flexible enough to lend itself to simple modifications without subverting its design, for example by forwarding and peering root zones to bypass Cloud DNS external resolution.
-## How to run this stage
-
-This stage is meant to be executed after the [resman](../1-resman) stage has run, as it leverages the automation service account and bucket created there, and additional resources configured in the [bootstrap](../0-bootstrap) stage.
-
-It's of course possible to run this stage in isolation, but that's outside the scope of this document, and you would need to refer to the code for the previous stages for the environmental requirements.
-
-Before running this stage, you need to make sure you have the correct credentials and permissions, and localize variables by assigning values that match your configuration.
-
-### Providers configuration
-
-The default way of making sure you have the right permissions, is to use the identity of the service account pre-created for this stage during the [resource management](../1-resman) stage, and that you are a member of the group that can impersonate it via provider-level configuration (`gcp-devops` or `organization-admins`).
-
-To simplify setup, the previous stage pre-configures a valid providers file in its output, and optionally writes it to a local file if the `outputs_location` variable is set to a valid path.
-
-If you have set a valid value for `outputs_location` in the bootstrap stage, simply link the relevant `providers.tf` file from this stage's folder in the path you specified:
-
-```bash
-# `outputs_location` is set to `~/fast-config`
-ln -s ~/fast-config/providers/02-networking-providers.tf .
-```
-
-If you have not configured `outputs_location` in bootstrap, you can derive the providers file from that stage's outputs:
-
-```bash
-cd ../1-resman
-terraform output -json providers | jq -r '.["02-networking"]' \
- > ../02-networking/providers.tf
-```
-
-### Variable configuration
-
-There are two broad sets of variables you will need to fill in:
-
-- variables shared by other stages (org id, billing account id, etc.), or derived from a resource managed by a different stage (folder id, automation project id, etc.)
-- variables specific to resources managed by this stage
-
-To avoid the tedious job of filling in the first group of variables with values derived from other stages' outputs, the same mechanism used above for the provider configuration can be used to leverage pre-configured `.tfvars` files.
-
-If you have set a valid value for `outputs_location` in the bootstrap and in the resman stage, simply link the relevant `*.auto.tfvars.json` files from this stage's folder in the path you specified.
-The `*` above is set to the name of the stage that produced it, except for `globals.auto.tfvars.json` which is also generated by the bootstrap stage, containing global values compiled manually for the bootstrap stage.
-For this stage, link the following files:
-
-```bash
-# `outputs_location` is set to `~/fast-config`
-ln -s ~/fast-config/tfvars/globals.auto.tfvars.json .
-ln -s ~/fast-config/tfvars/00-bootstrap.auto.tfvars.json .
-ln -s ~/fast-config/tfvars/01-resman.auto.tfvars.json .
-```
-
-A second set of variables is specific to this stage, they are all optional so if you need to customize them, create an extra `terraform.tfvars` file.
-
-Please refer to the [Variables](#variables) table below for a map of the variable origins, and to the sections below on how to adapt this stage to your networking configuration.
+## Stage structure and files layout
### VPCs
@@ -224,27 +200,64 @@ DNS queries sent to the on-premises infrastructure come from the `35.199.192.0/1
The [Inbound DNS Policy](https://cloud.google.com/dns/docs/server-policies-overview#dns-server-policy-in) defined in module `landing-vpc` ([`landing.tf`](./landing.tf)) automatically reserves the first available IP address on each created subnet (typically the third one in a CIDR) to expose the Cloud DNS service so that it can be consumed from outside of GCP.
-### Private Google Access
+## How to run this stage
-[Private Google Access](https://cloud.google.com/vpc/docs/private-google-access) (or PGA) enables VMs and on-prem systems to consume Google APIs from within the Google network, and is already fully configured on this environment.
+This stage is meant to be executed after the [resource management](../1-resman) stage has run, as it leverages the automation service account and bucket created there, and additional resources configured in the [bootstrap](../0-bootstrap) stage.
-For PGA to work:
+It's of course possible to run this stage in isolation, but that's outside the scope of this document, and you would need to refer to the code for the previous stages for the environmental requirements.
-- Private Google Access should be enabled on the subnet. \
-Subnets created by the `net-vpc` module are PGA-enabled by default.
+Before running this stage, you need to make sure you have the correct credentials and permissions, and localize variables by assigning values that match your configuration.
-- 199.36.153.4/30 (`restricted.googleapis.com`) and 199.36.153.8/30 (`private.googleapis.com`) should be routed from on-prem to VPC, and from there to the `default-internet-gateway`. \
-Per variable `vpn_onprem_configs` such ranges are advertised to onprem - furthermore every VPC (e.g. see `landing-vpc` in [`landing.tf`](./landing.tf)) has explicit routes set in case the `0.0.0.0/0` route is changed.
+### Provider and Terraform variables
-- A private DNS zone for `googleapis.com` should be created and configured per [this article](https://cloud.google.com/vpc/docs/configure-private-google-access-hybrid#config-domain), as implemented in module `googleapis-private-zone` in [`dns-landing.tf`](./dns-landing.tf)
+As all other FAST stages, the [mechanism used to pass variable values and pre-built provider files from one stage to the next](../0-bootstrap/README.md#output-files-and-cross-stage-variables) is also leveraged here.
+
+The commands to link or copy the provider and terraform variable files can be easily derived from the `stage-links.sh` script in the FAST root folder, passing it a single argument with the local output files folder (if configured) or the GCS output bucket in the automation project (derived from stage 0 outputs). The following examples demonstrate both cases, and the resulting commands that then need to be copy/pasted and run.
+
+```bash
+../../stage-links.sh ~/fast-config
-### Preliminar activities
+# copy and paste the following commands for '2-networking-a-peering'
-Before running `terraform apply` on this stage, make sure to adapt all of `variables.tf` to your needs, to update all reference to regions (e.g. `europe-west1` or `ew1`) in the whole directory to match your preferences.
+ln -s ~/fast-config/providers/2-networking-providers.tf ./
+ln -s ~/fast-config/tfvars/globals.auto.tfvars.json ./
+ln -s ~/fast-config/tfvars/0-bootstrap.auto.tfvars.json ./
+ln -s ~/fast-config/tfvars/1-resman.auto.tfvars.json ./
+```
+
+```bash
+../../stage-links.sh gs://xxx-prod-iac-core-outputs-0
-If you're not using FAST, you'll also need to create a `providers.tf` file to configure the GCS backend and the service account to use to run the deployment.
+# copy and paste the following commands for '2-networking-a-peering'
+
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/providers/2-networking-providers.tf ./
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/tfvars/globals.auto.tfvars.json ./
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/tfvars/0-bootstrap.auto.tfvars.json ./
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/tfvars/1-resman.auto.tfvars.json ./
+```
-You're now ready to run `terraform init` and `apply`.
+### Impersonating the automation service account
+
+The preconfigured provider file uses impersonation to run with this stage's automation service account's credentials. The `gcp-devops` and `organization-admins` groups have the necessary IAM bindings in place to do that, so make sure the current user is a member of one of those groups.
+
+### Variable configuration
+
+Variables in this stage -- like most other FAST stages -- are broadly divided into three separate sets:
+
+- variables which refer to global values for the whole organization (org id, billing account id, prefix, etc.), which are pre-populated via the `globals.auto.tfvars.json` file linked or copied above
+- variables which refer to resources managed by previous stage, which are prepopulated here via the `0-bootstrap.auto.tfvars.json` and `1-resman.auto.tfvars.json` files linked or copied above
+- and finally variables that optionally control this stage's behaviour and customizations, and can to be set in a custom `terraform.tfvars` file
+
+The latter set is explained in the [Customization](#customizations) sections below, and the full list can be found in the [Variables](#variables) table at the bottom of this document.
+
+### Running the stage
+
+Once provider and variable values are in place and the correct user is configured, the stage can be run:
+
+```bash
+terraform init
+terraform apply
+```
### Post-deployment activities
@@ -252,8 +265,29 @@ You're now ready to run `terraform init` and `apply`.
- On-prem routers should accept BGP sessions from their cloud peers.
- On-prem DNS servers should have forward zones for GCP-managed ones.
+#### Private Google Access
+
+[Private Google Access](https://cloud.google.com/vpc/docs/private-google-access) (or PGA) enables VMs and on-prem systems to consume Google APIs from within the Google network, and is already fully configured on this environment.
+
+For PGA to work:
+
+- Private Google Access should be enabled on the subnet. \
+Subnets created by the `net-vpc` module are PGA-enabled by default.
+
+- 199.36.153.4/30 (`restricted.googleapis.com`) and 199.36.153.8/30 (`private.googleapis.com`) should be routed from on-prem to VPC, and from there to the `default-internet-gateway`. \
+Per variable `vpn_onprem_configs` such ranges are advertised to onprem - furthermore every VPC (e.g. see `landing-vpc` in [`landing.tf`](./landing.tf)) has explicit routes set in case the `0.0.0.0/0` route is changed.
+
+- A private DNS zone for `googleapis.com` should be created and configured per [this article](https://cloud.google.com/vpc/docs/configure-private-google-access-hybrid#config-domain), as implemented in module `googleapis-private-zone` in [`dns-landing.tf`](./dns-landing.tf)
+
## Customizations
+### Changing default regions
+
+Regions are defined via the `regions` variable which sets up a mapping between the `regions.primary` and `regions.secondary` logical names and actual GCP region names. If you need to change regions from the defaults:
+
+- change the values of the mappings in the `regions` variable to the regions you are going to use
+- change the regions in the factory subnet files in the `data` folder
+
### Adding an environment
To create a new environment (e.g. `staging`), a few changes are required.
@@ -262,10 +296,10 @@ Create a `spoke-staging.tf` file by copying `spoke-prod.tf` file,
and adapt the new file by replacing the value "prod" with the value "staging".
Running `diff spoke-dev.tf spoke-prod.tf` can help to see how environment files differ.
-The new VPC requires a set of dedicated CIDRs, one per region, added to variable `custom_adv` (for example as `spoke_staging_ew1` and `spoke_staging_ew4`).
+The new VPC requires a set of dedicated CIDRs, one per region, added to variable `custom_adv` (for example as `spoke_staging_primary` and `spoke_staging_secondary`).
>`custom_adv` is a map that "resolves" CIDR names to actual addresses, and will be used later to configure routing.
>
-Variables managing L7 Interal Load Balancers (`l7ilb_subnets`) and Private Service Access (`psa_ranges`) should also be adapted, and subnets and firewall rules for the new spoke should be added as described above.
+Variables managing L7 Internal Load Balancers (`l7ilb_subnets`) and Private Service Access (`psa_ranges`) should also be adapted, and subnets and firewall rules for the new spoke should be added as described above.
DNS configurations are centralised in the `dns-*.tf` files. Spokes delegate DNS resolution to Landing through DNS peering, and optionally define a private zone (e.g. `dev.gcp.example.com`) which the landing peers to. To configure DNS for a new environment, copy one of the other environments DNS files [e.g. (dns-dev.tf](dns-dev.tf)) into a new `dns-*.tf` file suffixed with the environment name (e.g. `dns-staging.tf`), and update its content accordingly. Don't forget to add a peering zone from the landing to the newly created environment private zone.
diff --git a/fast/stages/2-networking-b-vpn/README.md b/fast/stages/2-networking-b-vpn/README.md
index 2177f3113b..7a8983d81b 100644
--- a/fast/stages/2-networking-b-vpn/README.md
+++ b/fast/stages/2-networking-b-vpn/README.md
@@ -15,6 +15,33 @@ The following diagram illustrates the high-level design, and should be used as a
+## Table of contents
+
+- [Design overview and choices](#design-overview-and-choices)
+ - [VPC design](#vpc-design)
+ - [External connectivity](#external-connectivity)
+ - [Internal connectivity](#internal-connectivity)
+ - [IP ranges, subnetting, routing](#ip-ranges-subnetting-routing)
+ - [Internet egress](#internet-egress)
+ - [VPC and Hierarchical Firewall](#vpc-and-hierarchical-firewall)
+ - [DNS](#dns)
+- [Stage structure and files layout](#stage-structure-and-files-layout)
+ - [VPCs](#vpcs)
+ - [VPNs](#vpns)
+ - [Routing and BGP](#routing-and-bgp)
+ - [Firewall](#firewall)
+ - [DNS architecture](#dns-architecture)
+ - [Private Google Access](#private-google-access)
+- [How to run this stage](#how-to-run-this-stage)
+ - [Provider and Terraform variables](#provider-and-terraform-variables)
+ - [Impersonating the automation service account](#impersonating-the-automation-service-account)
+ - [Variable configuration](#variable-configuration)
+ - [Running the stage](#running-the-stage)
+ - [Post-deployment activities](#post-deployment-activities)
+- [Customizations](#customizations)
+ - [Changing default regions](#changing-default-regions)
+ - [Adding an environment](#adding-an-environment)
+
## Design overview and choices
### VPC design
@@ -45,10 +72,10 @@ This is a summary of the main options:
- [HA VPN](https://cloud.google.com/network-connectivity/docs/vpn/concepts/topologies) (implemented here)
- Pros: simple compatibility with GCP services that leverage peering internally, better control on routes, avoids peering groups shared quotas and limits
- Cons: additional cost, marginal increase in latency, requires multiple tunnels for full bandwidth
-- [VPC Peering](https://cloud.google.com/vpc/docs/vpc-peering) (implemented by [02-networking-peering](../2-networking-a-peering/))
+- [VPC Peering](https://cloud.google.com/vpc/docs/vpc-peering) (implemented by [2-networking-peering](../2-networking-a-peering/))
- Pros: no additional costs, full bandwidth with no configurations, no extra latency
- Cons: no transitivity (e.g. to GKE masters, Cloud SQL, etc.), no selective exchange of routes, several quotas and limits shared between VPCs in a peering group
-- [Multi-NIC appliances](https://cloud.google.com/architecture/best-practices-vpc-design#multi-nic) (implemented by [02-networking-nva](../2-networking-c-nva/))
+- [Multi-NIC appliances](https://cloud.google.com/architecture/best-practices-vpc-design#multi-nic) (implemented by [2-networking-nva](../2-networking-c-nva/))
- Pros: additional security features (e.g. IPS), potentially better integration with on-prem systems by using the same vendor
- Cons: complex HA/failover setup, limited by VM bandwidth and scale, additional costs for VMs and licenses, out of band management of a critical cloud component
@@ -126,58 +153,7 @@ From cloud, the `example.com` domain (used as a placeholder) is forwarded to on-
This configuration is battle-tested, and flexible enough to lend itself to simple modifications without subverting its design, for example by forwarding and peering root zones to bypass Cloud DNS external resolution.
-## How to run this stage
-
-This stage is meant to be executed after the [resman](../1-resman) stage has run, as it leverages the automation service account and bucket created there, and additional resources configured in the [bootstrap](../0-bootstrap) stage.
-
-It's of course possible to run this stage in isolation, but that's outside the scope of this document, and you would need to refer to the code for the previous stages for the environmental requirements.
-
-Before running this stage, you need to make sure you have the correct credentials and permissions, and localize variables by assigning values that match your configuration.
-
-### Providers configuration
-
-The default way of making sure you have the right permissions, is to use the identity of the service account pre-created for this stage during the [resource management](../1-resman) stage, and that you are a member of the group that can impersonate it via provider-level configuration (`gcp-devops` or `organization-admins`).
-
-To simplify setup, the previous stage pre-configures a valid providers file in its output, and optionally writes it to a local file if the `outputs_location` variable is set to a valid path.
-
-If you have set a valid value for `outputs_location` in the bootstrap stage, simply link the relevant `providers.tf` file from this stage's folder in the path you specified:
-
-```bash
-# `outputs_location` is set to `~/fast-config`
-ln -s ~/fast-config/providers/02-networking-providers.tf .
-```
-
-If you have not configured `outputs_location` in bootstrap, you can derive the providers file from that stage's outputs:
-
-```bash
-cd ../1-resman
-terraform output -json providers | jq -r '.["02-networking"]' \
- > ../02-networking/providers.tf
-```
-
-### Variable configuration
-
-There are two broad sets of variables you will need to fill in:
-
-- variables shared by other stages (org id, billing account id, etc.), or derived from a resource managed by a different stage (folder id, automation project id, etc.)
-- variables specific to resources managed by this stage
-
-To avoid the tedious job of filling in the first group of variables with values derived from other stages' outputs, the same mechanism used above for the provider configuration can be used to leverage pre-configured `.tfvars` files.
-
-If you have set a valid value for `outputs_location` in the bootstrap and in the resman stage, simply link the relevant `*.auto.tfvars.json` files from this stage's folder in the path you specified.
-The `*` above is set to the name of the stage that produced it, except for `globals.auto.tfvars.json` which is also generated by the bootstrap stage, containing global values compiled manually for the bootstrap stage.
-For this stage, link the following files:
-
-```bash
-# `outputs_location` is set to `~/fast-config`
-ln -s ~/fast-config/tfvars/globals.auto.tfvars.json .
-ln -s ~/fast-config/tfvars/00-bootstrap.auto.tfvars.json .
-ln -s ~/fast-config/tfvars/01-resman.auto.tfvars.json .
-```
-
-A second set of variables is specific to this stage, they are all optional so if you need to customize them, create an extra `terraform.tfvars` file.
-
-Please refer to the [Variables](#variables) table below for a map of the variable origins, and to the sections below on how to adapt this stage to your networking configuration.
+## Stage structure and files layout
### VPCs
@@ -238,27 +214,64 @@ DNS queries sent to the on-premises infrastructure come from the `35.199.192.0/1
The [Inbound DNS Policy](https://cloud.google.com/dns/docs/server-policies-overview#dns-server-policy-in) defined in module `landing-vpc` ([`landing.tf`](./landing.tf)) automatically reserves the first available IP address on each created subnet (typically the third one in a CIDR) to expose the Cloud DNS service so that it can be consumed from outside of GCP.
-### Private Google Access
+## How to run this stage
-[Private Google Access](https://cloud.google.com/vpc/docs/private-google-access) (or PGA) enables VMs and on-prem systems to consume Google APIs from within the Google network, and is already fully configured on this environment.
+This stage is meant to be executed after the [resource management](../1-resman) stage has run, as it leverages the automation service account and bucket created there, and additional resources configured in the [bootstrap](../0-bootstrap) stage.
-For PGA to work:
+It's of course possible to run this stage in isolation, but that's outside the scope of this document, and you would need to refer to the code for the previous stages for the environmental requirements.
-- Private Google Access should be enabled on the subnet. \
-Subnets created by the `net-vpc` module are PGA-enabled by default.
+Before running this stage, you need to make sure you have the correct credentials and permissions, and localize variables by assigning values that match your configuration.
-- 199.36.153.4/30 (`restricted.googleapis.com`) and 199.36.153.8/30 (`private.googleapis.com`) should be routed from on-prem to VPC, and from there to the `default-internet-gateway`. \
-Per variable `vpn_onprem_configs` such ranges are advertised to onprem - furthermore every VPC (e.g. see `landing-vpc` in [`landing.tf`](./landing.tf)) has explicit routes set in case the `0.0.0.0/0` route is changed.
+### Provider and Terraform variables
+
+As all other FAST stages, the [mechanism used to pass variable values and pre-built provider files from one stage to the next](../0-bootstrap/README.md#output-files-and-cross-stage-variables) is also leveraged here.
+
+The commands to link or copy the provider and terraform variable files can be easily derived from the `stage-links.sh` script in the FAST root folder, passing it a single argument with the local output files folder (if configured) or the GCS output bucket in the automation project (derived from stage 0 outputs). The following examples demonstrate both cases, and the resulting commands that then need to be copy/pasted and run.
+
+```bash
+../../stage-links.sh ~/fast-config
+
+# copy and paste the following commands for '2-networking-a-peering'
+
+ln -s ~/fast-config/providers/2-networking-providers.tf ./
+ln -s ~/fast-config/tfvars/globals.auto.tfvars.json ./
+ln -s ~/fast-config/tfvars/0-bootstrap.auto.tfvars.json ./
+ln -s ~/fast-config/tfvars/1-resman.auto.tfvars.json ./
+```
+
+```bash
+../../stage-links.sh gs://xxx-prod-iac-core-outputs-0
+
+# copy and paste the following commands for '2-networking-a-peering'
+
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/providers/2-networking-providers.tf ./
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/tfvars/globals.auto.tfvars.json ./
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/tfvars/0-bootstrap.auto.tfvars.json ./
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/tfvars/1-resman.auto.tfvars.json ./
+```
+
+### Impersonating the automation service account
+
+The preconfigured provider file uses impersonation to run with this stage's automation service account's credentials. The `gcp-devops` and `organization-admins` groups have the necessary IAM bindings in place to do that, so make sure the current user is a member of one of those groups.
+
+### Variable configuration
-- A private DNS zone for `googleapis.com` should be created and configured per [this article](https://cloud.google.com/vpc/docs/configure-private-google-access-hybrid#config-domain), as implemented in module `googleapis-private-zone` in [dns-landing.tf](./dns-landing.tf)
+Variables in this stage -- like most other FAST stages -- are broadly divided into three separate sets:
-### Preliminary activities
+- variables which refer to global values for the whole organization (org id, billing account id, prefix, etc.), which are pre-populated via the `globals.auto.tfvars.json` file linked or copied above
+- variables which refer to resources managed by previous stage, which are prepopulated here via the `0-bootstrap.auto.tfvars.json` and `1-resman.auto.tfvars.json` files linked or copied above
+- and finally variables that optionally control this stage's behaviour and customizations, and can to be set in a custom `terraform.tfvars` file
-Before running `terraform apply` on this stage, make sure to adapt all of `variables.tf` and `vpn-variables.tf` to your needs, to update all references to regions (e.g. `europe-west1` or `ew1`) in the whole directory to match your preferences.
+The latter set is explained in the [Customization](#customizations) sections below, and the full list can be found in the [Variables](#variables) table at the bottom of this document.
-If you're not using FAST, you'll also need to create a `providers.tf` file to configure the GCS backend and the service account to use to run the deployment.
+### Running the stage
-You're now ready to run `terraform init` and `apply`.
+Once provider and variable values are in place and the correct user is configured, the stage can be run:
+
+```bash
+terraform init
+terraform apply
+```
### Post-deployment activities
@@ -266,8 +279,29 @@ You're now ready to run `terraform init` and `apply`.
- On-prem routers should accept BGP sessions from their cloud peers.
- On-prem DNS servers should have forward zones for GCP-managed ones.
+#### Private Google Access
+
+[Private Google Access](https://cloud.google.com/vpc/docs/private-google-access) (or PGA) enables VMs and on-prem systems to consume Google APIs from within the Google network, and is already fully configured on this environment.
+
+For PGA to work:
+
+- Private Google Access should be enabled on the subnet. \
+Subnets created by the `net-vpc` module are PGA-enabled by default.
+
+- 199.36.153.4/30 (`restricted.googleapis.com`) and 199.36.153.8/30 (`private.googleapis.com`) should be routed from on-prem to VPC, and from there to the `default-internet-gateway`. \
+Per variable `vpn_onprem_configs` such ranges are advertised to onprem - furthermore every VPC (e.g. see `landing-vpc` in [`landing.tf`](./landing.tf)) has explicit routes set in case the `0.0.0.0/0` route is changed.
+
+- A private DNS zone for `googleapis.com` should be created and configured per [this article](https://cloud.google.com/vpc/docs/configure-private-google-access-hybrid#config-domain), as implemented in module `googleapis-private-zone` in [`dns-landing.tf`](./dns-landing.tf)
+
## Customizations
+### Changing default regions
+
+Regions are defined via the `regions` variable which sets up a mapping between the `regions.primary` and `regions.secondary` logical names and actual GCP region names. If you need to change regions from the defaults:
+
+- change the values of the mappings in the `regions` variable to the regions you are going to use
+- change the regions in the factory subnet files in the `data` folder
+
### Adding an environment
To create a new environment (e.g. `staging`), a few changes are required.
diff --git a/fast/stages/2-networking-c-nva/README.md b/fast/stages/2-networking-c-nva/README.md
index 425e1d195e..d0e62fd62a 100644
--- a/fast/stages/2-networking-c-nva/README.md
+++ b/fast/stages/2-networking-c-nva/README.md
@@ -21,6 +21,34 @@ The final number of subnets, and their IP addressing will depend on the user-spe
+## Table of contents
+
+- [Design overview and choices](#design-overview-and-choices)
+ - [Multi-regional deployment](#multi-regional-deployment)
+ - [VPC design](#vpc-design)
+ - [External connectivity](#external-connectivity)
+ - [Internal connectivity](#internal-connectivity)
+ - [IP ranges, subnetting, routing](#ip-ranges-subnetting-routing)
+ - [Internet egress](#internet-egress)
+ - [VPC and Hierarchical Firewall](#vpc-and-hierarchical-firewall)
+ - [DNS](#dns)
+- [Stage structure and files layout](#stage-structure-and-files-layout)
+ - [VPCs](#vpcs)
+ - [VPNs](#vpns)
+ - [Routing and BGP](#routing-and-bgp)
+ - [Firewall](#firewall)
+ - [DNS architecture](#dns-architecture)
+ - [Private Google Access](#private-google-access)
+- [How to run this stage](#how-to-run-this-stage)
+ - [Provider and Terraform variables](#provider-and-terraform-variables)
+ - [Impersonating the automation service account](#impersonating-the-automation-service-account)
+ - [Variable configuration](#variable-configuration)
+ - [Running the stage](#running-the-stage)
+ - [Post-deployment activities](#post-deployment-activities)
+- [Customizations](#customizations)
+ - [Changing default regions](#changing-default-regions)
+ - [Adding an environment](#adding-an-environment)
+
## Design overview and choices
### Multi-regional deployment
@@ -190,58 +218,7 @@ In GCP, a forwarding zone in the landing project is configured to forward querie
This configuration is battle-tested, and flexible enough to lend itself to simple modifications without subverting its design.
-## How to run this stage
-
-This stage is meant to be executed after the [resman](../1-resman) stage has run. It leverages the automation service account and the storage bucket created there, and additional resources configured in the [bootstrap](../0-bootstrap) stage.
-
-It's possible to run this stage in isolation, but that's outside of the scope of this document. Please, refer to the previous stages for the environment requirements.
-
-Before running this stage, you need to make sure you have the correct credentials and permissions. You'll also need identify the module variables and make sure you assign them the values that match your configuration.
-
-### Providers configuration
-
-The default way of making sure you have the right permissions, is to use the identity of the service account pre-created for this stage, during the [resource management](../1-resman) stage, and that you are a member of the group that can impersonate it via provider-level configuration (`gcp-devops` or `organization-admins`).
-
-To simplify the setup, the previous stage pre-configures a valid providers file in its output and optionally writes it to a local file if the `outputs_location` variable is set to a valid path.
-
-If you have set a valid value for `outputs_location` in the bootstrap stage, simply link the relevant `providers.tf` file from this stage folder in the path you selected:
-
-```bash
-# `outputs_location` is set to `~/fast-config`
-ln -s ~/fast-config/providers/02-networking-providers.tf .
-```
-
-If you have not configured `outputs_location` in bootstrap, you can derive the providers file from that stage outputs:
-
-```bash
-cd ../1-resman
-terraform output -json providers | jq -r '.["02-networking"]' \
- > ../2-networking-c-nva/providers.tf
-```
-
-### Variable configuration
-
-There are two broad sets of variables you will need to fill in:
-
-- variables shared by other stages (org id, billing account id, etc.), or derived from a resource managed by a different stage (folder id, automation project id, etc.)
-- variables specific to resources managed by this stage
-
-To avoid the tedious job of filling in the first group of variables with values derived from other stages outputs, the same mechanism used above for the provider configuration can be used to leverage pre-configured `.tfvars` files.
-
-If you have set a valid value for `outputs_location` in the bootstrap and in the resman stage, simply link the relevant `*.auto.tfvars.json` files from this stage's folder in the path you specified.
-The `*` above is set to the name of the stage that produced it, except for `globals.auto.tfvars.json` which is also generated by the bootstrap stage, containing global values compiled manually for the bootstrap stage.
-For this stage, link the following files:
-
-```bash
-# `outputs_location` is set to `~/fast-config`
-ln -s ~/fast-config/tfvars/globals.auto.tfvars.json .
-ln -s ~/fast-config/tfvars/00-bootstrap.auto.tfvars.json .
-ln -s ~/fast-config/tfvars/01-resman.auto.tfvars.json .
-```
-
-A second set of variables is specific to this stage, they are all optional so if you need to customize them, create an extra `terraform.tfvars` file.
-
-Please, refer to the [variables](#variables) table below for a map of the variable origins, and use the sections below to understand how to adapt this stage to your networking configuration.
+## Stage structure and files layout
### VPCs
@@ -286,46 +263,104 @@ Cloud DNS manages onprem forwarding, the main GCP zone (in this example `gcp.exa
The root DNS zone defined in the landing project acts as the source of truth for DNS within the Cloud environment. The resources defined in the spoke VPCs consume the landing DNS infrastructure through DNS peering (e.g. `prod-landing-root-dns-peering`).
The spokes can optionally define private zones (e.g. `prod-dns-private-zone`). Granting visibility both to the trusted and untrusted landing VPCs ensures that the whole cloud environment can query such zones.
-#### Cloud to on-premises
+#### Cloud to on-prem
Leveraging the forwarding zone defined in the landing project (e.g. `onprem-example-dns-forwarding` and `reverse-10-dns-forwarding`), the cloud environment can resolve `in-addr.arpa.` and `onprem.example.com.` using the on-premise DNS infrastructure. On-premise resolver IPs are set in the variable `dns.onprem`.
DNS queries sent to the on-premise infrastructure come from the `35.199.192.0/19` source range.
-#### On-premises to cloud
+#### On-prem to cloud
The [Inbound DNS Policy](https://cloud.google.com/dns/docs/server-policies-overview#dns-server-policy-in) defined in the *trusted landing VPC module* ([`landing.tf`](./landing.tf)) automatically reserves the first available IP address on each subnet (typically the third one in a CIDR) to expose the Cloud DNS service, so that it can be consumed from outside of GCP.
-### Private Google Access
+## How to run this stage
-[Private Google Access](https://cloud.google.com/vpc/docs/private-google-access) (or PGA) is configured in this environment. It enables VMs and on-premise systems to consume Google APIs from within the Google network.
+This stage is meant to be executed after the [resource management](../1-resman) stage has run, as it leverages the automation service account and bucket created there, and additional resources configured in the [bootstrap](../0-bootstrap) stage.
-For PGA to work:
+It's of course possible to run this stage in isolation, but that's outside the scope of this document, and you would need to refer to the code for the previous stages for the environmental requirements.
-- Private Google Access should be enabled on the subnet. \
-Subnets created using the `net-vpc` module are PGA-enabled by default.
+Before running this stage, you need to make sure you have the correct credentials and permissions, and localize variables by assigning values that match your configuration.
-- 199.36.153.4/30 (`restricted.googleapis.com`) and 199.36.153.8/30 (`private.googleapis.com`) should be routed from on-premises to the trusted landing VPC, and from there to the `default-internet-gateway`. \
-The `vpn_onprem_configs` variable contains the ranges advertised from GCP to on-premises. Furthermore, the trusted landing VPC (e.g. see `landing-trusted-vpc` in [`landing.tf`](./landing.tf)) has explicit routes to send traffic destined to restricted and private - googleapis.com to the Internet gateway (which works for Google APIs only, and not for the whole Internet, since Cloud NAT is not configured in the trusted landing VPC).
+### Provider and Terraform variables
-- On-premises, a private DNS zone for `googleapis.com` should be created and configured per [this article](https://cloud.google.com/vpc/docs/configure-private-google-access-hybrid#config-domain). Its configuration can be copied from the module `googleapis-private-zone` in [`dns-landing.tf`](./dns-landing.tf)
+As all other FAST stages, the [mechanism used to pass variable values and pre-built provider files from one stage to the next](../0-bootstrap/README.md#output-files-and-cross-stage-variables) is also leveraged here.
-### Preliminar activities
+The commands to link or copy the provider and terraform variable files can be easily derived from the `stage-links.sh` script in the FAST root folder, passing it a single argument with the local output files folder (if configured) or the GCS output bucket in the automation project (derived from stage 0 outputs). The following examples demonstrate both cases, and the resulting commands that then need to be copy/pasted and run.
-Before running `terraform apply`, make sure to adapt `variables.tf` to your needs, to update the variable values using a new `terraform.tfvars` file, and to update the references to the regions in the whole directory, in order to match your preferences (e.g. `europe-west1` or `ew1`).
+```bash
+../../stage-links.sh ~/fast-config
+
+# copy and paste the following commands for '2-networking-a-peering'
+
+ln -s ~/fast-config/providers/2-networking-providers.tf ./
+ln -s ~/fast-config/tfvars/globals.auto.tfvars.json ./
+ln -s ~/fast-config/tfvars/0-bootstrap.auto.tfvars.json ./
+ln -s ~/fast-config/tfvars/1-resman.auto.tfvars.json ./
+```
+
+```bash
+../../stage-links.sh gs://xxx-prod-iac-core-outputs-0
+
+# copy and paste the following commands for '2-networking-a-peering'
+
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/providers/2-networking-providers.tf ./
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/tfvars/globals.auto.tfvars.json ./
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/tfvars/0-bootstrap.auto.tfvars.json ./
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/tfvars/1-resman.auto.tfvars.json ./
+```
+
+### Impersonating the automation service account
+
+The preconfigured provider file uses impersonation to run with this stage's automation service account's credentials. The `gcp-devops` and `organization-admins` groups have the necessary IAM bindings in place to do that, so make sure the current user is a member of one of those groups.
+
+### Variable configuration
+
+Variables in this stage -- like most other FAST stages -- are broadly divided into three separate sets:
+
+- variables which refer to global values for the whole organization (org id, billing account id, prefix, etc.), which are pre-populated via the `globals.auto.tfvars.json` file linked or copied above
+- variables which refer to resources managed by previous stage, which are prepopulated here via the `0-bootstrap.auto.tfvars.json` and `1-resman.auto.tfvars.json` files linked or copied above
+- and finally variables that optionally control this stage's behaviour and customizations, and can to be set in a custom `terraform.tfvars` file
-If you're not using other FAST stages, you'll also need to create a `providers.tf` file to configure the GCS backend and the service account to use to run the deployment.
+The latter set is explained in the [Customization](#customizations) sections below, and the full list can be found in the [Variables](#variables) table at the bottom of this document.
-You're now ready to run `terraform init` and `terraform apply`.
+### Running the stage
+
+Once provider and variable values are in place and the correct user is configured, the stage can be run:
+
+```bash
+terraform init
+terraform apply
+```
### Post-deployment activities
-- On-premise routers should be configured to advertise all relevant CIDRs to the GCP environments. To avoid hitting GCP quotas, we recommend aggregating routes as much as possible
-- On-premise routers should accept BGP sessions from their cloud peers
-- On-premise DNS servers should have forward zones configured, in order to resolve GCP-managed domains
+- On-prem routers should be configured to advertise all relevant CIDRs to the GCP environments. To avoid hitting GCP quotas, we recomment aggregating routes as much as possible.
+- On-prem routers should accept BGP sessions from their cloud peers.
+- On-prem DNS servers should have forward zones for GCP-managed ones.
+
+#### Private Google Access
+
+[Private Google Access](https://cloud.google.com/vpc/docs/private-google-access) (or PGA) enables VMs and on-prem systems to consume Google APIs from within the Google network, and is already fully configured on this environment.
+
+For PGA to work:
+
+- Private Google Access should be enabled on the subnet. \
+Subnets created by the `net-vpc` module are PGA-enabled by default.
+
+- 199.36.153.4/30 (`restricted.googleapis.com`) and 199.36.153.8/30 (`private.googleapis.com`) should be routed from on-prem to VPC, and from there to the `default-internet-gateway`. \
+Per variable `vpn_onprem_configs` such ranges are advertised to onprem - furthermore every VPC (e.g. see `landing-vpc` in [`landing.tf`](./landing.tf)) has explicit routes set in case the `0.0.0.0/0` route is changed.
+
+- A private DNS zone for `googleapis.com` should be created and configured per [this article](https://cloud.google.com/vpc/docs/configure-private-google-access-hybrid#config-domain), as implemented in module `googleapis-private-zone` in [`dns-landing.tf`](./dns-landing.tf)
## Customizations
+### Changing default regions
+
+Regions are defined via the `regions` variable which sets up a mapping between the `regions.primary` and `regions.secondary` logical names and actual GCP region names. If you need to change regions from the defaults:
+
+- change the values of the mappings in the `regions` variable to the regions you are going to use
+- change the regions in the factory subnet files in the `data` folder
+
### Adding an environment
To create a new environment (e.g. `staging`), a few changes are required:
diff --git a/fast/stages/2-networking-d-separate-envs/README.md b/fast/stages/2-networking-d-separate-envs/README.md
index a461dc97bf..dfc199cd0f 100644
--- a/fast/stages/2-networking-d-separate-envs/README.md
+++ b/fast/stages/2-networking-d-separate-envs/README.md
@@ -1,4 +1,4 @@
-# Networking
+# Networking with separated single environment
This stage sets up the shared network infrastructure for the whole organization. It implements a single shared VPC per environment, where each environment is independently connected to the on-premise environment, to maintain a fully separated routing domain on GCP.
@@ -14,6 +14,31 @@ The following diagram illustrates the high-level design, and should be used as a
+## Table of contents
+
+- [Design overview and choices](#design-overview-and-choices)
+ - [VPC design](#vpc-design)
+ - [External connectivity](#external-connectivity)
+ - [IP ranges, subnetting, routing](#ip-ranges-subnetting-routing)
+ - [Internet egress](#internet-egress)
+ - [VPC and Hierarchical Firewall](#vpc-and-hierarchical-firewall)
+ - [DNS](#dns)
+- [Stage structure and files layout](#stage-structure-and-files-layout)
+ - [VPCs](#vpcs)
+ - [VPNs](#vpns)
+ - [Routing and BGP](#routing-and-bgp)
+ - [Firewall](#firewall)
+ - [DNS architecture](#dns-architecture)
+ - [Private Google Access](#private-google-access)
+- [How to run this stage](#how-to-run-this-stage)
+ - [Provider and Terraform variables](#provider-and-terraform-variables)
+ - [Impersonating the automation service account](#impersonating-the-automation-service-account)
+ - [Variable configuration](#variable-configuration)
+ - [Running the stage](#running-the-stage)
+ - [Post-deployment activities](#post-deployment-activities)
+- [Customizations](#customizations)
+ - [Changing default regions](#changing-default-regions)
+
## Design overview and choices
### VPC design
@@ -87,57 +112,7 @@ From cloud, the `example.com` domain (used as a placeholder) is forwarded to on-
This configuration is battle-tested, and flexible enough to lend itself to simple modifications without subverting its design, for example by forwarding and peering root zones to bypass Cloud DNS external resolution.
-## How to run this stage
-
-This stage is meant to be executed after the [resman](../1-resman) stage has run, as it leverages the automation service account and bucket created there, and additional resources configured in the [bootstrap](../0-bootstrap) stage.
-
-It's of course possible to run this stage in isolation, but that's outside the scope of this document, and you would need to refer to the code for the previous stages for the environmental requirements.
-
-Before running this stage, you need to make sure you have the correct credentials and permissions, and localize variables by assigning values that match your configuration.
-
-### Providers configuration
-
-The default way of making sure you have the right permissions, is to use the identity of the service account pre-created for this stage during the [resource management](../1-resman) stage, and that you are a member of the group that can impersonate it via provider-level configuration (`gcp-devops` or `organization-admins`).
-
-To simplify setup, the previous stage pre-configures a valid providers file in its output, and optionally writes it to a local file if the `outputs_location` variable is set to a valid path.
-
-If you have set a valid value for `outputs_location` in the bootstrap stage, simply link the relevant `providers.tf` file from this stage's folder in the path you specified:
-
-```bash
-# `outputs_location` is set to `~/fast-config`
-ln -s ~/fast-config/providers/02-networking-providers.tf .
-```
-
-If you have not configured `outputs_location` in bootstrap, you can derive the providers file from that stage's outputs:
-
-```bash
-cd ../1-resman
-terraform output -json providers | jq -r '.["02-networking"]' \
- > ../02-networking/providers.tf
-```
-
-### Variable configuration
-
-There are two broad sets of variables you will need to fill in:
-
-- variables shared by other stages (org id, billing account id, etc.), or derived from a resource managed by a different stage (folder id, automation project id, etc.)
-- variables specific to resources managed by this stage
-
-To avoid the tedious job of filling in the first group of variables with values derived from other stages' outputs, the same mechanism used above for the provider configuration can be used to leverage pre-configured `.tfvars` files.
-
-If you have set a valid value for `outputs_location` in the bootstrap and in the resman stage, simply link the relevant `terraform-*.auto.tfvars.json` files from this stage's folder in the path you specified, where the `*` above is set to the name of the stage that produced it. For this stage, a single `.tfvars` file is available:
-
-```bash
-# `outputs_location` is set to `~/fast-config`
-ln -s ../../configs/example/02-networking/terraform-bootstrap.auto.tfvars.json
-ln -s ../../configs/example/02-networking/terraform-resman.auto.tfvars.json
-# also copy the tfvars file used for the bootstrap stage
-cp ../0-bootstrap/terraform.tfvars .
-```
-
-A second set of variables is specific to this stage, they are all optional so if you need to customize them, add them to the file copied from bootstrap.
-
-Please refer to the [Variables](#variables) table below for a map of the variable origins, and to the sections below on how to adapt this stage to your networking configuration.
+## Stage structure and files layout
### VPCs
@@ -187,27 +162,64 @@ When implementing this architecture, make sure you'll be able to route packets c
The [Inbound DNS Policy](https://cloud.google.com/dns/docs/server-policies-overview#dns-server-policy-in) defined on eachVPC automatically reserves the first available IP address on each created subnet (typically the third one in a CIDR) to expose the Cloud DNS service so that it can be consumed from outside of GCP.
-### Private Google Access
+## How to run this stage
-[Private Google Access](https://cloud.google.com/vpc/docs/private-google-access) (or PGA) enables VMs and on-prem systems to consume Google APIs from within the Google network, and is already fully configured on this environment.
+This stage is meant to be executed after the [resource management](../1-resman) stage has run, as it leverages the automation service account and bucket created there, and additional resources configured in the [bootstrap](../0-bootstrap) stage.
-For PGA to work:
+It's of course possible to run this stage in isolation, but that's outside the scope of this document, and you would need to refer to the code for the previous stages for the environmental requirements.
-- Private Google Access should be enabled on the subnet. \
-Subnets created by the `net-vpc` module are PGA-enabled by default.
+Before running this stage, you need to make sure you have the correct credentials and permissions, and localize variables by assigning values that match your configuration.
-- 199.36.153.4/30 (`restricted.googleapis.com`) and 199.36.153.8/30 (`private.googleapis.com`) should be routed from on-prem to VPC, and from there to the `default-internet-gateway`. \
-Per variable `vpn_onprem_configs` such ranges are advertised to onprem - furthermore every VPC has explicit routes set in case the `0.0.0.0/0` route is changed.
+### Provider and Terraform variables
+
+As all other FAST stages, the [mechanism used to pass variable values and pre-built provider files from one stage to the next](../0-bootstrap/README.md#output-files-and-cross-stage-variables) is also leveraged here.
+
+The commands to link or copy the provider and terraform variable files can be easily derived from the `stage-links.sh` script in the FAST root folder, passing it a single argument with the local output files folder (if configured) or the GCS output bucket in the automation project (derived from stage 0 outputs). The following examples demonstrate both cases, and the resulting commands that then need to be copy/pasted and run.
+
+```bash
+../../stage-links.sh ~/fast-config
+
+# copy and paste the following commands for '2-networking-a-peering'
+
+ln -s ~/fast-config/providers/2-networking-providers.tf ./
+ln -s ~/fast-config/tfvars/globals.auto.tfvars.json ./
+ln -s ~/fast-config/tfvars/0-bootstrap.auto.tfvars.json ./
+ln -s ~/fast-config/tfvars/1-resman.auto.tfvars.json ./
+```
+
+```bash
+../../stage-links.sh gs://xxx-prod-iac-core-outputs-0
+
+# copy and paste the following commands for '2-networking-a-peering'
+
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/providers/2-networking-providers.tf ./
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/tfvars/globals.auto.tfvars.json ./
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/tfvars/0-bootstrap.auto.tfvars.json ./
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/tfvars/1-resman.auto.tfvars.json ./
+```
+
+### Impersonating the automation service account
+
+The preconfigured provider file uses impersonation to run with this stage's automation service account's credentials. The `gcp-devops` and `organization-admins` groups have the necessary IAM bindings in place to do that, so make sure the current user is a member of one of those groups.
+
+### Variable configuration
+
+Variables in this stage -- like most other FAST stages -- are broadly divided into three separate sets:
-- A private DNS zone for `googleapis.com` should be created and configured per [this article](https://cloud.google.com/vpc/docs/configure-private-google-access-hybrid#config-domain), as implemented in module `googleapis-private-zone` in `dns-xxx.tf`
+- variables which refer to global values for the whole organization (org id, billing account id, prefix, etc.), which are pre-populated via the `globals.auto.tfvars.json` file linked or copied above
+- variables which refer to resources managed by previous stage, which are prepopulated here via the `0-bootstrap.auto.tfvars.json` and `1-resman.auto.tfvars.json` files linked or copied above
+- and finally variables that optionally control this stage's behaviour and customizations, and can to be set in a custom `terraform.tfvars` file
-### Preliminar activities
+The latter set is explained in the [Customization](#customizations) sections below, and the full list can be found in the [Variables](#variables) table at the bottom of this document.
-Before running `terraform apply` on this stage, make sure to adapt all of `variables.tf` to your needs, to update all reference to regions (e.g. `europe-west1` or `ew1`) in the whole directory to match your preferences.
+### Running the stage
-If you're not using FAST, you'll also need to create a `providers.tf` file to configure the GCS backend and the service account to use to run the deployment.
+Once provider and variable values are in place and the correct user is configured, the stage can be run:
-You're now ready to run `terraform init` and `apply`.
+```bash
+terraform init
+terraform apply
+```
### Post-deployment activities
@@ -215,6 +227,29 @@ You're now ready to run `terraform init` and `apply`.
- On-prem routers should accept BGP sessions from their cloud peers.
- On-prem DNS servers should have forward zones for GCP-managed ones.
+#### Private Google Access
+
+[Private Google Access](https://cloud.google.com/vpc/docs/private-google-access) (or PGA) enables VMs and on-prem systems to consume Google APIs from within the Google network, and is already fully configured on this environment.
+
+For PGA to work:
+
+- Private Google Access should be enabled on the subnet. \
+Subnets created by the `net-vpc` module are PGA-enabled by default.
+
+- 199.36.153.4/30 (`restricted.googleapis.com`) and 199.36.153.8/30 (`private.googleapis.com`) should be routed from on-prem to VPC, and from there to the `default-internet-gateway`. \
+Per variable `vpn_onprem_configs` such ranges are advertised to onprem - furthermore every VPC has explicit routes set in case the `0.0.0.0/0` route is changed.
+
+- A private DNS zone for `googleapis.com` should be created and configured per [this article](https://cloud.google.com/vpc/docs/configure-private-google-access-hybrid#config-domain)
+
+## Customizations
+
+### Changing default regions
+
+Regions are defined via the `regions` variable which sets up a mapping between the `regions.primary` and `regions.secondary` logical names and actual GCP region names. If you need to change regions from the defaults:
+
+- change the values of the mappings in the `regions` variable to the regions you are going to use
+- change the regions in the factory subnet files in the `data` folder
+
diff --git a/fast/stages/2-security/README.md b/fast/stages/2-security/README.md
index a609cd8144..6486cd7418 100644
--- a/fast/stages/2-security/README.md
+++ b/fast/stages/2-security/README.md
@@ -12,6 +12,24 @@ The following diagram illustrates the high-level design of created resources and
+## Table of contents
+
+- [Design overview and choices](#design-overview-and-choices)
+ - [Cloud KMS](#cloud-kms)
+ - [VPC Service Controls](#vpc-service-controls)
+- [How to run this stage](#how-to-run-this-stage)
+ - [Provider and Terraform variables](#provider-and-terraform-variables)
+ - [Impersonating the automation service account](#impersonating-the-automation-service-account)
+ - [Variable configuration](#variable-configuration)
+ - [Running the stage](#running-the-stage)
+- [Customizations](#customizations)
+ - [KMS keys](#kms-keys)
+ - [VPC Service Controls configuration](#vpc-service-controls-configuration)
+ - [Dry-run vs. enforced](#dry-run-vs-enforced)
+ - [Access levels](#access-levels)
+ - [Ingress and Egress policies](#ingress-and-egress-policies)
+ - [Perimeters](#perimeters)
+
## Design overview and choices
Project-level security resources are grouped into two separate projects, one per environment. This setup matches requirements we frequently observe in real life and provides enough separation without needlessly complicating operations.
@@ -42,57 +60,57 @@ Some care needs to be taken with project membership in perimeters, which can onl
## How to run this stage
-This stage is meant to be executed after the [resource management](../1-resman) stage has run, as it leverages the folder and automation resources created there. The relevant user groups must also exist, but that's one of the requirements for the previous stages too, so if you ran those successfully, you're good to go.
-
-It's possible to run this stage in isolation, but that's outside the scope of this document, and you would need to refer to the code for the bootstrap stage for the required roles.
+This stage is meant to be executed after the [resource management](../1-resman) stage has run, as it leverages the automation service account and bucket created there, and additional resources configured in the [bootstrap](../0-bootstrap) stage.
-Before running this stage, you need to ensure you have the correct credentials and permissions, and customize variables by assigning values that match your configuration.
+It's of course possible to run this stage in isolation, but that's outside the scope of this document, and you would need to refer to the code for the previous stages for the environmental requirements.
-### Providers configuration
+Before running this stage, you need to make sure you have the correct credentials and permissions, and localize variables by assigning values that match your configuration.
-The default way of making sure you have the correct permissions is to use the identity of the service account pre-created for this stage during bootstrap, and that you are a member of the group that can impersonate it via provider-level configuration (`gcp-devops` or `organization-admins`).
+### Provider and Terraform variables
-To simplify setup, the previous stage pre-configures a valid providers file in its output, and optionally writes it to a local file if the `outputs_location` variable is set to a valid path.
+As all other FAST stages, the [mechanism used to pass variable values and pre-built provider files from one stage to the next](../0-bootstrap/README.md#output-files-and-cross-stage-variables) is also leveraged here.
-If you have set a valid value for `outputs_location` in the resource management stage, simply link the relevant `providers.tf` file from this stage's folder in the path you specified:
+The commands to link or copy the provider and terraform variable files can be easily derived from the `stage-links.sh` script in the FAST root folder, passing it a single argument with the local output files folder (if configured) or the GCS output bucket in the automation project (derived from stage 0 outputs). The following examples demonstrate both cases, and the resulting commands that then need to be copy/pasted and run.
```bash
-# `outputs_location` is set to `~/fast-config`
-ln -s ~/fast-config/providers/02-security-providers.tf .
-```
+../../stage-links.sh ~/fast-config
-If you have not configured `outputs_location` in resource management, you can derive the providers file from that stage's outputs:
+# copy and paste the following commands for '2-security'
-```bash
-cd ../1-resman
-terraform output -json providers | jq -r '.["02-security"]' \
- > ../02-security/providers.tf
+ln -s ~/fast-config/providers/2-security-providers.tf ./
+ln -s ~/fast-config/tfvars/globals.auto.tfvars.json ./
+ln -s ~/fast-config/tfvars/0-bootstrap.auto.tfvars.json ./
+ln -s ~/fast-config/tfvars/1-resman.auto.tfvars.json ./
```
-### Variable configuration
+```bash
+../../stage-links.sh gs://xxx-prod-iac-core-outputs-0
-There are two broad sets of variables you will need to fill in:
+# copy and paste the following commands for '2-security'
-- variables shared by other stages (organization id, billing account id, etc.), or derived from a resource managed by a different stage (folder id, automation project id, etc.)
-- variables specific to resources managed by this stage
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/providers/2-security-providers.tf ./
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/tfvars/globals.auto.tfvars.json ./
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/tfvars/0-bootstrap.auto.tfvars.json ./
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/tfvars/1-resman.auto.tfvars.json ./
+```
-To avoid the tedious job of filling in the first group of variables with values derived from other stages' outputs, the same mechanism used above for the provider configuration can be used to leverage pre-configured `.tfvars` files.
+### Impersonating the automation service account
-If you configured a valid path for `outputs_location` in the previous stages, simply link the relevant `terraform-*.auto.tfvars.json` files from this stage's output folder (under the path you specified), where the `*` above is set to the name of the stage that produced it. For this stage, two `.tfvars` files are available:
+The preconfigured provider file uses impersonation to run with this stage's automation service account's credentials. The `gcp-devops` and `organization-admins` groups have the necessary IAM bindings in place to do that, so make sure the current user is a member of one of those groups.
-```bash
-# `outputs_location` is set to `~/fast-config`
-ln -s ~/fast-config/tfvars/00-bootstrap.auto.tfvars.json .
-ln -s ~/fast-config/tfvars/01-resman.auto.tfvars.json .
-# also copy the tfvars file used for the bootstrap stage
-cp ../0-bootstrap/terraform.tfvars .
-```
+### Variable configuration
+
+Variables in this stage -- like most other FAST stages -- are broadly divided into three separate sets:
+
+- variables which refer to global values for the whole organization (org id, billing account id, prefix, etc.), which are pre-populated via the `globals.auto.tfvars.json` file linked or copied above
+- variables which refer to resources managed by previous stage, which are prepopulated here via the `0-bootstrap.auto.tfvars.json` and `1-resman.auto.tfvars.json` files linked or copied above
+- and finally variables that optionally control this stage's behaviour and customizations, and can to be set in a custom `terraform.tfvars` file
-A second set of optional variables is specific to this stage. If you need to customize them add them to the file copied from bootstrap.
+The latter set is explained in the [Customization](#customizations) sections below, and the full list can be found in the [Variables](#variables) table at the bottom of this document.
-Refer to the [Variables](#variables) table at the bottom of this document, for a full list of variables, their origin (e.g., a stage or specific to this one), and descriptions explaining their meaning. The sections below also describe some of the possible customizations.
+### Running the stage
-Once done, you can run this stage:
+Once provider and variable values are in place and the correct user is configured, the stage can be run:
```bash
terraform init
diff --git a/fast/stages/3-data-platform/dev/README.md b/fast/stages/3-data-platform/dev/README.md
index 615dbde8b0..48d09eafc0 100644
--- a/fast/stages/3-data-platform/dev/README.md
+++ b/fast/stages/3-data-platform/dev/README.md
@@ -78,74 +78,91 @@ In the case your Data Warehouse need to handle confidential data and you have th
## How to run this stage
-This stage can be run in isolation by prviding the necessary variables, but it's really meant to be used as part of the FAST flow after the "foundational stages" ([`00-bootstrap`](../../0-bootstrap), [`01-resman`](../../1-resman), [`02-networking`](../../2-networking-b-vpn) and [`02-security`](../../2-security)).
+This stage is meant to be executed after the FAST "foundational" stages: bootstrap, resource management, security and networking stages.
-When running in isolation, the following roles are needed on the principal used to apply Terraform:
+It's of course possible to run this stage in isolation, refer to the *[Running in isolation](#running-in-isolation)* section below for details.
-- on the organization or network folder level
- - `roles/xpnAdmin` or a custom role which includes the following permissions
- - `"compute.organizations.enableXpnResource"`,
- - `"compute.organizations.disableXpnResource"`,
- - `"compute.subnetworks.setIamPolicy"`,
-- on each folder where projects are created
- - `"roles/logging.admin"`
- - `"roles/owner"`
- - `"roles/resourcemanager.folderAdmin"`
- - `"roles/resourcemanager.projectCreator"`
-- on the host project for the Shared VPC
- - `"roles/browser"`
- - `"roles/compute.viewer"`
-- on the organization or billing account
- - `roles/billing.admin`
+Before running this stage, you need to make sure you have the correct credentials and permissions, and localize variables by assigning values that match your configuration.
-The VPC host project, VPC and subnets should already exist.
+### Provider and Terraform variables
-### Providers configuration
+As all other FAST stages, the [mechanism used to pass variable values and pre-built provider files from one stage to the next](../../0-bootstrap/README.md#output-files-and-cross-stage-variables) is also leveraged here.
-If you're running this on top of Fast, you should run the following commands to create the providers file, and populate the required variables from the previous stage.
+The commands to link or copy the provider and terraform variable files can be easily derived from the `stage-links.sh` script in the FAST root folder, passing it a single argument with the local output files folder (if configured) or the GCS output bucket in the automation project (derived from stage 0 outputs). The following examples demonstrate both cases, and the resulting commands that then need to be copy/pasted and run.
```bash
-# Variable `outputs_location` is set to `~/fast-config` in stage 01-resman
-ln -s ~/fast-config/providers/03-data-platform-dev-providers.tf .
-```
+../../../stage-links.sh ~/fast-config
-If you have not configured `outputs_location` in bootstrap, you can derive the providers file from that stage's outputs:
+# copy and paste the following commands for '3-data-platform'
+
+ln -s /home/ludomagno/fast-config/providers/3-data-platform-providers.tf ./
+ln -s /home/ludomagno/fast-config/tfvars/globals.auto.tfvars.json ./
+ln -s /home/ludomagno/fast-config/tfvars/0-bootstrap.auto.tfvars.json ./
+ln -s /home/ludomagno/fast-config/tfvars/1-resman.auto.tfvars.json ./
+ln -s /home/ludomagno/fast-config/tfvars/2-networking.auto.tfvars.json ./
+ln -s /home/ludomagno/fast-config/tfvars/2-security.auto.tfvars.json ./
+```
```bash
-cd ../../1-resman
-terraform output -json providers | jq -r '.["03-data-platform-dev"]' \
- > ../3-data-platform/dev/providers.tf
+../../../stage-links.sh gs://xxx-prod-iac-core-outputs-0
+
+# copy and paste the following commands for '3-data-platform'
+
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/providers/3-data-platform-providers.tf ./
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/tfvars/globals.auto.tfvars.json ./
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/tfvars/0-bootstrap.auto.tfvars.json ./
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/tfvars/1-resman.auto.tfvars.json ./
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/tfvars/2-networking.auto.tfvars.json ./
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/tfvars/2-security.auto.tfvars.json ./
```
-### Variable configuration
+### Impersonating the automation service account
-There are two broad sets of variables that can be configured:
+The preconfigured provider file uses impersonation to run with this stage's automation service account's credentials. The `gcp-devops` and `organization-admins` groups have the necessary IAM bindings in place to do that, so make sure the current user is a member of one of those groups.
-- variables shared by other stages (organization id, billing account id, etc.) or derived from a resource managed by a different stage (folder id, automation project id, etc.)
-- variables specific to resources managed by this stage
+### Variable configuration
-To avoid the tedious job of filling in the first group of variables with values derived from other stages' outputs, the same mechanism used above for the provider configuration can be used to leverage pre-configured `.tfvars` files.
+Variables in this stage -- like most other FAST stages -- are broadly divided into three separate sets:
-If you configured a valid path for `outputs_location` in the bootstrap security and networking stages, simply link the relevant `terraform-*.auto.tfvars.json` files from this stage's outputs folder under the path you specified. This will also link the providers configuration file:
+- variables which refer to global values for the whole organization (org id, billing account id, prefix, etc.), which are pre-populated via the `globals.auto.tfvars.json` file linked or copied above
+- variables which refer to resources managed by previous stage, which are prepopulated here via the `*.auto.tfvars.json` files linked or copied above
+- and finally variables that optionally control this stage's behaviour and customizations, and can to be set in a custom `terraform.tfvars` file
-```bash
-# Variable `outputs_location` is set to `~/fast-config`
-ln -s ~/fast-config/tfvars/00-bootstrap.auto.tfvars.json .
-ln -s ~/fast-config/tfvars/01-resman.auto.tfvars.json .
-ln -s ~/fast-config/tfvars/02-networking.auto.tfvars.json .
-# also copy the tfvars file used for the bootstrap stage
-cp ../../0-bootstrap/terraform.tfvars .
-```
+The full list can be found in the [Variables](#variables) table at the bottom of this document.
-If you're not using FAST or its output files, refer to the [Variables](#variables) table at the bottom of this document for a full list of variables, their origin (e.g., a stage or specific to this one), and descriptions explaining their meaning.
+### Running the stage
-Once the configuration is complete you can apply this stage:
+Once provider and variable values are in place and the correct user is configured, the stage can be run:
```bash
terraform init
terraform apply
```
+### Running in isolation
+
+This stage can be run in isolation by providing the necessary variables, but it's really meant to be used as part of the FAST flow after the "foundational stages" ([`0-bootstrap`](../../0-bootstrap), [`1-resman`](../../1-resman), [`2-networking`](../../2-networking-b-vpn) and [`2-security`](../../2-security)).
+
+When running in isolation, the following roles are needed on the principal used to apply Terraform:
+
+- on the organization or network folder level
+ - `roles/xpnAdmin` or a custom role which includes the following permissions
+ - `"compute.organizations.enableXpnResource"`,
+ - `"compute.organizations.disableXpnResource"`,
+ - `"compute.subnetworks.setIamPolicy"`,
+- on each folder where projects are created
+ - `"roles/logging.admin"`
+ - `"roles/owner"`
+ - `"roles/resourcemanager.folderAdmin"`
+ - `"roles/resourcemanager.projectCreator"`
+- on the host project for the Shared VPC
+ - `"roles/browser"`
+ - `"roles/compute.viewer"`
+- on the organization or billing account
+ - `roles/billing.admin`
+
+The VPC host project, VPC and subnets should already exist.
+
## Demo pipeline
The application layer is out of scope of this script. As a demo purpuse only, several Cloud Composer DAGs are provided. Demos will import data from the `landing` area to the `DataWarehouse Confidential` dataset suing different features.
diff --git a/fast/stages/3-gke-multitenant/dev/README.md b/fast/stages/3-gke-multitenant/dev/README.md
index 4accf8e1aa..f0460c06c5 100644
--- a/fast/stages/3-gke-multitenant/dev/README.md
+++ b/fast/stages/3-gke-multitenant/dev/README.md
@@ -39,7 +39,68 @@ This stage creates a project containing and as many clusters and node pools as r
## How to run this stage
-This stage is meant to be executed after "foundational stages" (i.e., stages [`00-bootstrap`](../../0-bootstrap), [`01-resman`](../../1-resman), 02-networking (either [VPN](../../2-networking-b-vpn) or [NVA](../../2-networking-c-nva)) and [`02-security`](../../2-security)) have been run.
+This stage is meant to be executed after the FAST "foundational" stages: bootstrap, resource management, security and networking stages.
+
+It's of course possible to run this stage in isolation, refer to the *[Running in isolation](#running-in-isolation)* section below for details.
+
+Before running this stage, you need to make sure you have the correct credentials and permissions, and localize variables by assigning values that match your configuration.
+
+### Provider and Terraform variables
+
+As all other FAST stages, the [mechanism used to pass variable values and pre-built provider files from one stage to the next](../../0-bootstrap/README.md#output-files-and-cross-stage-variables) is also leveraged here.
+
+The commands to link or copy the provider and terraform variable files can be easily derived from the `stage-links.sh` script in the FAST root folder, passing it a single argument with the local output files folder (if configured) or the GCS output bucket in the automation project (derived from stage 0 outputs). The following examples demonstrate both cases, and the resulting commands that then need to be copy/pasted and run.
+
+```bash
+../../../stage-links.sh ~/fast-config
+
+# copy and paste the following commands for '3-gke-multitenant'
+
+ln -s /home/ludomagno/fast-config/providers/3-gke-multitenant-providers.tf ./
+ln -s /home/ludomagno/fast-config/tfvars/globals.auto.tfvars.json ./
+ln -s /home/ludomagno/fast-config/tfvars/0-bootstrap.auto.tfvars.json ./
+ln -s /home/ludomagno/fast-config/tfvars/1-resman.auto.tfvars.json ./
+ln -s /home/ludomagno/fast-config/tfvars/2-networking.auto.tfvars.json ./
+ln -s /home/ludomagno/fast-config/tfvars/2-security.auto.tfvars.json ./
+```
+
+```bash
+../../../stage-links.sh gs://xxx-prod-iac-core-outputs-0
+
+# copy and paste the following commands for '3-gke-multitenant'
+
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/providers/3-gke-multitenant-providers.tf ./
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/tfvars/globals.auto.tfvars.json ./
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/tfvars/0-bootstrap.auto.tfvars.json ./
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/tfvars/1-resman.auto.tfvars.json ./
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/tfvars/2-networking.auto.tfvars.json ./
+gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/tfvars/2-security.auto.tfvars.json ./
+```
+
+### Impersonating the automation service account
+
+The preconfigured provider file uses impersonation to run with this stage's automation service account's credentials. The `gcp-devops` and `organization-admins` groups have the necessary IAM bindings in place to do that, so make sure the current user is a member of one of those groups.
+
+### Variable configuration
+
+Variables in this stage -- like most other FAST stages -- are broadly divided into three separate sets:
+
+- variables which refer to global values for the whole organization (org id, billing account id, prefix, etc.), which are pre-populated via the `globals.auto.tfvars.json` file linked or copied above
+- variables which refer to resources managed by previous stage, which are prepopulated here via the `*.auto.tfvars.json` files linked or copied above
+- and finally variables that optionally control this stage's behaviour and customizations, and can to be set in a custom `terraform.tfvars` file
+
+The latter set is explained in the [Customization](#customizations) sections below, and the full list can be found in the [Variables](#variables) table at the bottom of this document.
+
+### Running the stage
+
+Once provider and variable values are in place and the correct user is configured, the stage can be run:
+
+```bash
+terraform init
+terraform apply
+```
+
+### Running in isolation
It's of course possible to run this stage in isolation, by making sure the architectural prerequisites are satisfied (e.g., networking), and that the Service Account running the stage is granted the roles/permissions below:
@@ -62,39 +123,9 @@ It's of course possible to run this stage in isolation, by making sure the archi
The VPC host project, VPC and subnets should already exist.
-### Providers configuration
-
-If you're running this on top of FAST, you should run the following commands to create the providers file, and populate the required variables from the previous stage.
-
-```bash
-# Variable `outputs_location` is set to `~/fast-config` in stage 01-resman
-$ cd fabric-fast/stages/03-gke-multitenant/dev
-ln -s ~/fast-config/providers/03-gke-dev-providers.tf .
-```
-
-### Variable configuration
-
-There are two broad sets of variables you will need to fill in:
-
-- variables shared by other stages (organization id, billing account id, etc.), or derived from a resource managed by a different stage (folder id, automation project id, etc.)
-- variables specific to resources managed by this stage
-
-#### Variables passed in from other stages
+## Customizations
-To avoid the tedious job of filling in the first group of variables with values derived from other stages' outputs, the same mechanism used above for the provider configuration can be used to leverage pre-configured `.tfvars` files.
-
-If you configured a valid path for `outputs_location` in the bootstrap and networking stage, simply link the relevant `terraform-*.auto.tfvars.json` files from this stage's outputs folder (under the path you specified), where the `*` above is set to the name of the stage that produced it. For this stage, a single `.tfvars` file is available:
-
-```bash
-# Variable `outputs_location` is set to `~/fast-config`
-ln -s ~/fast-config/tfvars/00-bootstrap.auto.tfvars.json .
-ln -s ~/fast-config/tfvars/01-resman.auto.tfvars.json .
-ln -s ~/fast-config/tfvars/02-networking.auto.tfvars.json .
-```
-
-If you're not using FAST, refer to the [Variables](#variables) table at the bottom of this document for a full list of variables, their origin (e.g., a stage or specific to this one), and descriptions explaining their meaning.
-
-#### Cluster and node pools
+### Cluster and node pools
This stage is designed with multi-tenancy in mind, and the expectation is that GKE clusters will mostly share a common set of defaults. Variables are designed to support this approach for both clusters and node pools:
@@ -105,7 +136,7 @@ This stage is designed with multi-tenancy in mind, and the expectation is that
There are two additional variables that influence cluster configuration: `authenticator_security_group` to configure [Google Groups for RBAC](https://cloud.google.com/kubernetes-engine/docs/how-to/google-groups-rbac), `dns_domain` to configure [Cloud DNS for GKE](https://cloud.google.com/kubernetes-engine/docs/how-to/cloud-dns).
-#### Fleet management
+### Fleet management
Fleet management is entirely optional, and uses three separate variables:
@@ -116,15 +147,6 @@ Fleet management is entirely optional, and uses three separate variables:
Leave all these variables unset (or set to `null`) to disable fleet management.
-## Running Terraform
-
-Once the [provider](#providers-configuration) and [variable](#variable-configuration) configuration is complete, you can apply this stage:
-
-```bash
-terraform init
-terraform apply
-```
-
diff --git a/fast/stages/CLEANUP.md b/fast/stages/CLEANUP.md
index 3bc581f92a..4b2667c838 100644
--- a/fast/stages/CLEANUP.md
+++ b/fast/stages/CLEANUP.md
@@ -7,7 +7,7 @@ Destruction must be done in reverse order, from stage 3 to stage 0
## Stage 3 (Project Factory)
```bash
-cd $FAST_PWD/03-project-factory/prod/
+cd $FAST_PWD/3-project-factory/dev/
terraform destroy
```
@@ -16,7 +16,7 @@ terraform destroy
Terraform refuses to delete non-empty GCS buckets and BigQuery datasets, so they need to be removed manually from the state.
```bash
-cd $FAST_PWD/03-project-factory/prod/
+cd $FAST_PWD/3-gke-multitenant/dev/
# remove BQ dataset manually
for x in $(terraform state list | grep google_bigquery_dataset); do
@@ -29,14 +29,14 @@ terraform destroy
## Stage 2 (Security)
```bash
-cd $FAST_PWD/02-security/
+cd $FAST_PWD/2-security/
terraform destroy
```
## Stage 2 (Networking)
```bash
-cd $FAST_PWD/02-networking-XXX/
+cd $FAST_PWD/2-networking-XXX/
terraform destroy
```
@@ -47,7 +47,7 @@ A minor glitch can surface running `terraform destroy`, where the service projec
Stage 1 is a little more complicated because of the GCS buckets containing your terraform statefiles. By default, Terraform refuses to delete non-empty buckets, which is good to protect your terraform state, but it makes destruction a bit harder. Use the commands below to remove the GCS buckets from the state and then execute `terraform destroy`
```bash
-cd $FAST_PWD/01-resman/
+cd $FAST_PWD/1-resman/
# remove buckets from state since terraform refuses to delete them
for x in $(terraform state list | grep google_storage_bucket.bucket); do
@@ -64,10 +64,10 @@ terraform destroy
Just like before, we manually remove several resources (GCS buckets and BQ datasets). Note that `terrafom destroy` will fail. This is expected; just continue with the rest of the steps.
```bash
-cd $FAST_PWD/00-bootstrap/
+cd $FAST_PWD/0-bootstrap/
# remove provider config to execute without SA impersonation
-rm 00-bootstrap-providers.tf
+rm 0-bootstrap-providers.tf
# migrate to local state
terraform init -migrate-state
diff --git a/fast/stages/COMPANION.md b/fast/stages/COMPANION.md
index d5d7752f2e..96506d0083 100644
--- a/fast/stages/COMPANION.md
+++ b/fast/stages/COMPANION.md
@@ -8,7 +8,7 @@ The detailed explanation of each stage, their configuration, possible modificati
## Prerequisites
-1. FAST uses the recommended groups from the [GCP Enterprise Setup checklist](). Go to [Workspace / Cloud Identity](https://admin.google.com) and ensure all the following groups exist:
+1. FAST uses the recommended groups from the [GCP Enterprise Setup checklist](https://cloud.google.com/docs/enterprise/setup-checklist). Go to [Workspace / Cloud Identity](https://admin.google.com) and ensure all the following groups exist:
- `gcp-billing-admins@`
- `gcp-devops@`
@@ -80,8 +80,8 @@ If you are using a billing account in a different organization, please follow [t
This initial stage will create common projects for IaC, Logging & Billing, and bootstrap IAM policies.
```bash
-# move to the 00-bootstrap directory
-cd $FAST_PWD/00-bootstrap
+# move to the 0-bootstrap directory
+cd $FAST_PWD/0-bootstrap
# copy the template terraform tfvars file and save as `terraform.tfvars`
# then edit to match your environment!
@@ -114,11 +114,12 @@ outputs_location = "~/fast-config"
terraform init
terraform apply -var bootstrap_user=$FAST_BU
-# link the generated provider file
-ln -s ~/fast-config/providers/0-0-bootstrap* .
+# link providers file
+ln -s ~/fast-config/providers/0-bootstrap-providers.tf ./
# re-run init and apply to remove user-level IAM
terraform init -migrate-state
+
# answer 'yes' to terraform's question
terraform apply
```
@@ -132,14 +133,14 @@ This stage performs two important tasks:
```bash
# move to the 01-resman directory
-cd $FAST_PWD/01-resman
+cd $FAST_PWD/1-resman
-# Link providers and variables from previous stages
-ln -s ~/fast-config/providers/1-0-resman-providers.tf .
-ln -s ~/fast-config/tfvars/0-0-bootstrap.auto.tfvars.json .
+# link providers and variables from previous stages
+ln -s ~/fast-config/providers/1-resman-providers.tf .
+ln -s ~/fast-config/tfvars/0-bootstrap.auto.tfvars.json .
ln -s ~/fast-config/tfvars/globals.auto.tfvars.json .
-# Edit your terraform.tfvars to append Teams configuration (optional)
+# edit your terraform.tfvars to append Teams configuration (optional)
edit terraform.tfvars
```
@@ -178,15 +179,15 @@ In this stage, we will deploy one of the 3 available Hub&Spoke networking topolo
```bash
# move to the 02-networking-XXX directory (where XXX should be one of vpn|peering|nva)
-cd $FAST_PWD/02-networking-XXX
+cd $FAST_PWD/2-networking-XXX
# setup providers and variables from previous stages
-ln -s ~/fast-config/providers/2-0-networking-providers.tf .
-ln -s ~/fast-config/tfvars/0-0-bootstrap.auto.tfvars.json .
-ln -s ~/fast-config/tfvars/1-0-resman.auto.tfvars.json .
+ln -s ~/fast-config/providers/2-networking-providers.tf .
+ln -s ~/fast-config/tfvars/0-bootstrap.auto.tfvars.json .
+ln -s ~/fast-config/tfvars/1-resman.auto.tfvars.json .
ln -s ~/fast-config/tfvars/globals.auto.tfvars.json .
-# Create terraform.tfvars. output_location variable is required to generate networking stage output file
+# create terraform.tfvars. output_location variable is required to generate networking stage output file
edit terraform.tfvars
```
@@ -212,12 +213,12 @@ This stage sets up security resources (KMS and VPC-SC) and configurations which
cd $FAST_PWD/02-security
# link providers and variables from previous stages
-ln -s ~/fast-config/providers/2-0-security-providers.tf .
-ln -s ~/fast-config/tfvars/0-0-bootstrap.auto.tfvars.json .
-ln -s ~/fast-config/tfvars/1-0-resman.auto.tfvars.json .
+ln -s ~/fast-config/providers/2-security-providers.tf .
+ln -s ~/fast-config/tfvars/0-bootstrap.auto.tfvars.json .
+ln -s ~/fast-config/tfvars/1-resman.auto.tfvars.json .
ln -s ~/fast-config/tfvars/globals.auto.tfvars.json .
-# Edit terraform.tfvars to include KMS and/or VPC-SC configuration
+# edit terraform.tfvars to include KMS and/or VPC-SC configuration
edit terraform.tfvars
```
@@ -234,19 +235,20 @@ terraform apply
The Project Factory stage builds on top of your foundations to create and set up projects (and related resources) to be used for your workloads. It is organized in folders representing environments (e.g. "dev", "prod"), each implemented by a stand-alone terraform resource factory.
```bash
-# Variable `outputs_location` is set to `~/fast-config`
-cd $FAST_PWD/3-0-project-factory/ENVIRONMENT
-ln -s ~/fast-config/providers/3-0-project-factory-ENVIRONMENT-providers.tf .
+# variable `outputs_location` is set to `~/fast-config`
+cd $FAST_PWD/3-project-factory/ENVIRONMENT
+ln -s ~/fast-config/providers/3-project-factory-ENVIRONMENT-providers.tf .
-ln -s ~/fast-config/tfvars/0-0-bootstrap.auto.tfvars.json .
-ln -s ~/fast-config/tfvars/1-0-resman.auto.tfvars.json .
-ln -s ~/fast-config/tfvars/2-0-networking.auto.tfvars.json .
+ln -s ~/fast-config/tfvars/0-bootstrap.auto.tfvars.json .
+ln -s ~/fast-config/tfvars/1-resman.auto.tfvars.json .
+ln -s ~/fast-config/tfvars/2-networking.auto.tfvars.json .
ln -s ~/fast-config/tfvars/globals.auto.tfvars.json .
-# Define your environment default values (eg for billing alerts and labels)
+# define your environment default values (eg for billing alerts and labels)
edit data/defaults.yaml
-# Create one yaml file per project to be created. Yaml file will include project configuration. Projects will be named after the filename
+# create one YAML file per project to be created with project configuration
+# filenames will be used for project ids
cp data/projects/project.yaml.sample data/projects/YOUR_PROJECT_NAME.yaml
edit data/projects/YOUR_PROJECT_NAME.yaml
diff --git a/fast/stages/FAQ.md b/fast/stages/FAQ.md
index bd9559d481..5245c8a965 100644
--- a/fast/stages/FAQ.md
+++ b/fast/stages/FAQ.md
@@ -1,29 +1,13 @@
-
-## 00-bootstrap
-
-1. How to handle requests where automation, logging and/or billing export projects are not under organization but in different folders.
- - Run bootstrap stage and let automation, logging and/or billing projects be created under organization.
- - Run resource manager stage or any other custom stage which creates the folders where these projects will reside.
- - Once folders are created add folder ids to varibale "project_parent_ids" in bootstrap stage and run bootstrap stage.
- - This step will move the projects from organization to the parent folders specificed.
-
-## cicd
-
-1. Why do we need two seperate ServiceAccounts when configuring cicd pipelines (cicd SA and IaC SA)
- - Having seperate service accounts helps shutdown the pipeline incase of any issues and still keep IaC SA and ability to run terraform plan/apply manually.
- - A pipeline can only generate a token that can get access to an SA. It cannot directly call a provider file to impersonate IaC SA.
- - Having providers file that allows impersonation to IaC SA allows flexibility to run terraform manually or from CICD Pipelines.
- ![CICD SA and IaC SA](IaC_SA.png)
-
-## Authenciation
-
-1. If you are seeing "Permission Issues" when doing terraform apply and the identity with which you are running terraform has correct permissions;
- run below command so that correct auth credentials are picked by ADC when terraform commands are executed
-
- ````bash
- gcloud auth application-default login
- ````
-
-
- Refer to [GCP Authentication](https://cloud.google.com/docs/authentication
- ) and [Terraform Provider](https://registry.terraform.io/providers/hashicorp/google/latest/docs/guides/provider_reference) for more information
+# FAST Mini FAQ
+
+- **How can the automation, logging and/or billing export projects be placed under specific folders instead of the org?**
+ - Run the bootstrap stage and let automation, logging and/or billing projects be created under the organization.
+ - Add the needed folders to the resource manager stage, or create them outside the stage in the console/gcloud or from a custom Terraform setup.
+ - Once folders have been created go back to the bootstrap stage, and edit your tfvars file by adding their ids to the `project_parent_ids` variable.
+ - Run the bootstrap stage again, the projects will be moved under the desired folders.
+- **Why do we need two separate service accounts when configuring CI/CD pipelines (CI/CD SA and IaC SA)?**
+ - To have the pipeline workflow follow the same impersonation flow ([CI/CD SA impersonates IaC SA](IaC_SA.png)) used when applying Terraform manually (user impersonates IaC SA), which allows the pipeline to consume the same auto-generated provider files.
+ - To allow disabling pipeline credentials in case of issues with a single operation, by removing the ability of the CI/CD SA to impersonate the IaC SA.
+- **How can I fix permission issues when running Terraform apply?**
+ - Make sure your account is part of the organization admin group defined in variables.
+ - Make sure you have configured [application default credentials](https://cloud.google.com/docs/authentication/application-default-credentials), rerun `gcloud auth login --update-adc` to fix them.
From 636a4cc01cec296b4d1ff8b89b9f27a09bdff2f1 Mon Sep 17 00:00:00 2001
From: Julio Diez
Date: Wed, 15 Feb 2023 13:37:16 +0100
Subject: [PATCH 013/169] Access CR from "onprem" environment
---
.../serverless/cloud-run-corporate/main.tf | 150 +++++++++++++++++-
.../cloud-run-corporate/variables.tf | 31 +++-
2 files changed, 171 insertions(+), 10 deletions(-)
diff --git a/blueprints/serverless/cloud-run-corporate/main.tf b/blueprints/serverless/cloud-run-corporate/main.tf
index 5e16a9a1c6..2644ea81fb 100644
--- a/blueprints/serverless/cloud-run-corporate/main.tf
+++ b/blueprints/serverless/cloud-run-corporate/main.tf
@@ -41,6 +41,20 @@ module "project_host" {
]
}
+# Simulated onprem environment
+module "project_onprem" {
+ source = "../../../modules/project"
+ count = var.prj_onprem_id != null ? 1 : 0
+ name = var.prj_onprem_id
+ project_create = var.prj_onprem_create != null
+ billing_account = try(var.prj_onprem_create.billing_account_id, null)
+ parent = try(var.prj_onprem_create.parent, null)
+ services = [
+ "compute.googleapis.com",
+ "dns.googleapis.com"
+ ]
+}
+
###############################################################################
# Cloud Run #
###############################################################################
@@ -75,7 +89,7 @@ module "vpc_host" {
name = "vpc-host"
subnets = [
{
- ip_cidr_range = var.ip_ranges_host.subnet
+ ip_cidr_range = var.ip_ranges["host"].subnet
name = "subnet-host"
region = var.region
enable_private_access = true # PGA enabled
@@ -83,7 +97,7 @@ module "vpc_host" {
]
}
-# VPC Firewall with default config, IAP for SSH enabled
+# Host VPC Firewall with default config, IAP for SSH enabled
module "firewall_host" {
source = "../../../modules/net-vpc-firewall"
project_id = module.project_host.project_id
@@ -94,16 +108,44 @@ module "firewall_host" {
}
}
+# VPC in simulated onprem environment
+module "vpc_onprem" {
+ source = "../../../modules/net-vpc"
+ count = length(module.project_onprem)
+ project_id = module.project_onprem[0].project_id
+ name = "vpc-onprem"
+ subnets = [
+ {
+ ip_cidr_range = var.ip_ranges["onprem"].subnet
+ name = "subnet-onprem"
+ region = var.region
+ }
+ ]
+}
+
+# Onprem VPC Firewall with default config, IAP for SSH enabled
+module "firewall_onprem" {
+ source = "../../../modules/net-vpc-firewall"
+ count = length(module.project_onprem)
+ project_id = module.project_onprem[0].project_id
+ network = module.vpc_onprem[0].name
+ default_rules_config = {
+ http_ranges = []
+ https_ranges = []
+ }
+}
+
###############################################################################
# PSC #
###############################################################################
+# PSC configured in the host
module "psc_addr_host" {
source = "../../../modules/net-address"
project_id = module.project_host.project_id
psc_addresses = {
psc-addr-host = {
- address = var.ip_ranges_host.psc_addr
+ address = var.ip_ranges["host"].psc_addr
network = module.vpc_host.self_link
}
}
@@ -125,6 +167,7 @@ resource "google_compute_global_forwarding_rule" "psc_endpoint_host" {
module "vm_test_host" {
source = "../../../modules/compute-vm"
+ count = 1 - length(module.project_onprem)
project_id = module.project_host.project_id
zone = "${var.region}-b"
name = "vm-test-host"
@@ -136,12 +179,27 @@ module "vm_test_host" {
tags = ["ssh"]
}
+module "vm_test_onprem" {
+ source = "../../../modules/compute-vm"
+ count = length(module.project_onprem)
+ project_id = module.project_onprem[0].project_id
+ zone = "${var.region}-b"
+ name = "vm-test-onprem"
+ instance_type = "e2-micro"
+ network_interfaces = [{
+ network = module.vpc_onprem[0].self_link
+ subnetwork = module.vpc_onprem[0].subnet_self_links["${var.region}/subnet-onprem"]
+ }]
+ tags = ["ssh"]
+}
+
###############################################################################
# DNS #
###############################################################################
module "private_dns_host" {
source = "../../../modules/dns"
+ count = 1 - length(module.project_onprem)
project_id = module.project_host.project_id
type = "private"
name = "dns-host"
@@ -151,3 +209,89 @@ module "private_dns_host" {
"A " = { records = [module.psc_addr_host.psc_addresses["psc-addr-host"].address] }
}
}
+
+module "private_dns_onprem" {
+ source = "../../../modules/dns"
+ count = length(module.project_onprem)
+ project_id = module.project_onprem[0].project_id
+ type = "private"
+ name = "dns-onprem"
+ client_networks = [module.vpc_onprem[0].self_link]
+ domain = local.domain_cr_host
+ recordsets = {
+ "A " = { records = [module.psc_addr_host.psc_addresses["psc-addr-host"].address] }
+ }
+}
+
+###############################################################################
+# VPN #
+###############################################################################
+
+# VPN between main project and "onprem" environment
+module "vpn_host" {
+ source = "../../../modules/net-vpn-ha"
+ count = length(module.project_onprem)
+ project_id = module.project_host.project_id
+ region = var.region
+ network = module.vpc_host.self_link
+ name = "vpn-host-to-onprem"
+ peer_gateway = { gcp = module.vpn_onprem[0].self_link }
+ router_config = {
+ asn = 65001
+ custom_advertise = {
+ all_subnets = true
+ ip_ranges = {
+ (var.ip_ranges["host"].psc_addr) = "to-psc-endpoint"
+ }
+ }
+ }
+ tunnels = {
+ tunnel-0 = {
+ bgp_peer = {
+ address = "169.254.0.2"
+ asn = 65002
+ }
+ bgp_session_range = "169.254.0.1/30"
+ vpn_gateway_interface = 0
+ }
+ tunnel-1 = {
+ bgp_peer = {
+ address = "169.254.1.2"
+ asn = 65002
+ }
+ bgp_session_range = "169.254.1.1/30"
+ vpn_gateway_interface = 1
+ }
+ }
+}
+
+module "vpn_onprem" {
+ source = "../../../modules/net-vpn-ha"
+ count = length(module.project_onprem)
+ project_id = module.project_onprem[0].project_id
+ region = var.region
+ network = module.vpc_onprem[0].self_link
+ name = "vpn-onprem-to-host"
+ peer_gateway = { gcp = module.vpn_host[0].self_link }
+ router_config = { asn = 65002 }
+ tunnels = {
+ tunnel-0 = {
+ bgp_peer = {
+ address = "169.254.0.1"
+ asn = 65001
+ }
+ bgp_session_range = "169.254.0.2/30"
+ vpn_gateway_interface = 0
+ shared_secret = module.vpn_host[0].random_secret
+ }
+ tunnel-1 = {
+ bgp_peer = {
+ address = "169.254.1.1"
+ asn = 65001
+ }
+ bgp_session_range = "169.254.1.2/30"
+ vpn_gateway_interface = 1
+ shared_secret = module.vpn_host[0].random_secret
+ }
+ }
+}
diff --git a/blueprints/serverless/cloud-run-corporate/variables.tf b/blueprints/serverless/cloud-run-corporate/variables.tf
index 540a764fd4..422e3506f7 100644
--- a/blueprints/serverless/cloud-run-corporate/variables.tf
+++ b/blueprints/serverless/cloud-run-corporate/variables.tf
@@ -26,15 +26,17 @@ variable "ingress_settings" {
default = "all"
}
-variable "ip_ranges_host" {
+variable "ip_ranges" {
description = "IPs or IP ranges used by VPCs"
- type = object({
- subnet = string
- psc_addr = string
- })
+ type = map(map(string))
default = {
- subnet = "10.0.1.0/24"
- psc_addr = "10.0.0.100"
+ host = {
+ subnet = "10.0.1.0/24"
+ psc_addr = "10.0.0.100"
+ }
+ onprem = {
+ subnet = "172.16.1.0/24"
+ }
}
}
@@ -52,6 +54,21 @@ variable "prj_host_id" {
type = string
}
+variable "prj_onprem_create" {
+ description = "Parameters for the creation of an 'onprem' project."
+ type = object({
+ billing_account_id = string
+ parent = string
+ })
+ default = null
+}
+
+variable "prj_onprem_id" {
+ description = "Host Project ID."
+ type = string
+ default = null
+}
+
variable "region" {
description = "Cloud region where resource will be deployed."
type = string
From 15c8f92f46fc2c1d6a91f0f4140af01c835dfa8c Mon Sep 17 00:00:00 2001
From: Julio Diez
Date: Wed, 15 Feb 2023 20:33:45 +0100
Subject: [PATCH 014/169] Delete project resources without deleting the
projects
---
blueprints/serverless/cloud-run-corporate/main.tf | 2 ++
1 file changed, 2 insertions(+)
diff --git a/blueprints/serverless/cloud-run-corporate/main.tf b/blueprints/serverless/cloud-run-corporate/main.tf
index 2644ea81fb..075b9e77d4 100644
--- a/blueprints/serverless/cloud-run-corporate/main.tf
+++ b/blueprints/serverless/cloud-run-corporate/main.tf
@@ -39,6 +39,7 @@ module "project_host" {
"compute.googleapis.com",
"dns.googleapis.com"
]
+ skip_delete = true
}
# Simulated onprem environment
@@ -53,6 +54,7 @@ module "project_onprem" {
"compute.googleapis.com",
"dns.googleapis.com"
]
+ skip_delete = true
}
###############################################################################
From 30821ac58e121d29f5b3d2883ba0ec90df064b6a Mon Sep 17 00:00:00 2001
From: Julio Diez
Date: Wed, 15 Feb 2023 20:44:22 +0100
Subject: [PATCH 015/169] Add first use case description in README
---
.../serverless/cloud-run-corporate/README.md | 37 +++++++++++++++++-
.../images/service-running.png | Bin 0 -> 295552 bytes
.../cloud-run-corporate/images/use-case-1.png | Bin 0 -> 188887 bytes
3 files changed, 36 insertions(+), 1 deletion(-)
create mode 100644 blueprints/serverless/cloud-run-corporate/images/service-running.png
create mode 100644 blueprints/serverless/cloud-run-corporate/images/use-case-1.png
diff --git a/blueprints/serverless/cloud-run-corporate/README.md b/blueprints/serverless/cloud-run-corporate/README.md
index e250f55b58..4e3ce6a7a0 100644
--- a/blueprints/serverless/cloud-run-corporate/README.md
+++ b/blueprints/serverless/cloud-run-corporate/README.md
@@ -6,10 +6,28 @@ This blueprint contains all the necessary Terraform modules to build and private
The content of this blueprint corresponds to the chapter '_Developing an enterprise application - The corporate environment_' of the __Serverless Networking Guide__ (to be released soon). This guide is an easy to follow introduction to Cloud Run, where a couple of friendly characters will guide you from the basics to more advanced topics with a very practical approach and in record time! The code here complements this learning and allows you to test the scenarios presented and your knowledge.
+If you are interested in following this guide, take a look to the chapters' blueprints:
+* [My serverless "Hello, World! - Exploring Cloud Run](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/tree/master/blueprints/serverless/cloud-run-explore)
+* [Developing an enterprise application - The corporate environment](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/tree/master/blueprints/serverless/cloud-run-corporate)
+
## Architecture
+This blueprint creates multiple architectures depending on the use case. Some may have one or two projecs while others may have four or more. Some use [Private Service Connect (PSC)](https://cloud.google.com/vpc/docs/private-service-connect) to access Google APIs, and others a [Layer 7 Internal Load Balancer](https://cloud.google.com/load-balancing/docs/l7-internal). Even security plays a role and [VPC Service Controls](https://cloud.google.com/vpc-service-controls) is introduced.
+
## Prerequisites
+Depending on the use case, you will need one or more projects with [billing enabled](https://cloud.google.com/billing/docs/how-to/modify-project) and a user with the “Project owner” [IAM](https://cloud.google.com/iam) role on those projects. You can use existing projects or let the blueprint creates them for you but in that case you will need to add extra information for each project. E.g.:
+
+```tfvars
+# Create the main or host project
+prj_host_create = {
+ billing_account_id = "ABCDE-12345-ABCDE"
+ parent = "organizations/0123456789"
+}
+```
+
+Below it is explained how to set this information.
+
## Spinning up the architecture
### General steps
@@ -34,7 +52,7 @@ may become
project_id = "spiritual-hour-331417"
```
-Although each use case is somehow built around the previous one they are self-contained so you can deploy any of them at will.
+Use cases are self-contained so you can deploy any of them at will.
4. The usual terraform commands will do the work:
```bash
@@ -49,6 +67,23 @@ __Congratulations!__ You have successfully deployed the use case you chose based
### Use case 1: Access to Cloud Run from a VM in the project
+This use case deploys a Cloud Run service and a VM in the same project. To privately access Cloud Run from the VM, PSC is used. A PSC endpoint is created so that the VM can reach the service through an RFC1918 IP. Also, a DNS entry is created to point the service's default URL to that IP.
+
+
+
+In this case the only variable that you need to set in `terraform.tfvars` is the main or host project ID:
+```tfvars
+prj_host_id = "[your-project-id]"
+```
+Alternatively you can pass this value on the command line:
+```bash
+terraform apply -var prj_host_id="[your-project-id]"
+```
+
+The main project is also referenced as host project because some use cases use it with a Shared VPC. The default URL is automatically created and shown as a terraform output variable. It will be similar to the one shown in the picture above. Now SSH into the VM and run `curl`, you should see the following:
+
+
+
### Use case 2:
### Use case 3:
diff --git a/blueprints/serverless/cloud-run-corporate/images/service-running.png b/blueprints/serverless/cloud-run-corporate/images/service-running.png
new file mode 100644
index 0000000000000000000000000000000000000000..3db8fbf459833f440eac6d71c81ba68fabc7af2f
GIT binary patch
literal 295552
zcma&NWmsHGvj#f2yW8NF;4Z;ExVu9jxVyV+kf4DOJh;0{u;4PdyZa!QWPjgz_Br>*
zy*pjBA6DHx}Np;t+uMe8ccK2&*!fA@2KLH$F9aUTsF<;ZLb^1IUmgC*;fbM62JOh)A
zSyos}uYS1!Qxm@0cd!QJWFZFErXN40veL>Rc*IqD-`{)T?P#y=y$}mn&6xx!QstP3
zX_K##<*b(DXWd2imXnqB?e%@9(NDNfJc=5z&$>cI`oL~em~_==Y{I%W@R6uCs02e1
zQ%X&~$cOx-=9g!5tww3huD}CQu(q%H*CUQ_^g)VLGLev=cTseCL3BU)=y8+kNUzS(
z`1~UE5SycnwzY%_2+ao3Uf?=g>z%N+>{M7_T{;$S2U*Y;Uuoc&-
zoqBMe1Z67g>xMAUmtdJHbTew;^iBa#ieWat;il7^UJFlU>;w9
z%8dFZEDsN~Ekmve@UnE?;n@u-i8I%L{>^1@cQTTt-UDuEeF9t
zTH6@_K*IX-g#ct^;lE|VxyZ;%!0kbzVWGi0;mCTtWf8bYXt{{l+uEAjxd6nR%#B^l
z&B)w8yI7J*$;c~d2BP8t0Av6eaS=6-rQ;P(J?y<_xbul)rYfYM2)!Ug#GcZG0PfYu
z0e03x8AAD~zJjqys?LEIKLMvY`Kf|t5dQS?*lgWoR(>RgJVy>jJUrsQQ#m9eBn9_z
z5AX^cY>kJzU~Z9ibmiCX(~gP%DaUh{w`1Z1`vgGz--*Kve&xxIC*T6024L~J-v7P#
za@z>EJ1X-L9-n|`d3r>aDQ!69)LrD?mBgnojoC8$3kM`sB|JTE@<`E&gl>z3p0A!B
zuFM&UH~>6w-r1zMmxa7#$s5
zlgnO9D^ONmUSQ>N3ls1F7)=GaWszM2STMS(0s;bzTis4Q*qNC8+Q^2;qCRmP-dI}A
z^ct~rnX&aSL4ao0Cfixx?_bJ~zNonO(j8kFk)@{s0#cYUog
zGe7sOG@m%EP_l#mf0Hzfq(GLJw=AxN0(gLtlh-^4v*&TMncF-r)g&PRLVz7W-yApm
zAQZg3v~;?|{(2yq8QWJS&t_tI`Pfmp|B>a}k9>%MH*P5i!BuDZ}{hK`w;o%)8XlU5))zy5&&Ct2j6y(KsBam9zK`2YP4^cm
zX(Ru)hV-8r226u}kywKU-X~GKfHoLFt#i!;6&y4)^z5t}d)mLoG9m)@t8s^4ad~OS
z2XY#>6b3dnHXfhr^ly(#guBQX|4Nf&YKEH8%HJhpk&%#)QeUs?HD7t%uJ+pn?)z)N9e?5~vn7K3>37c%(Y+yu<@ZR+m~O;stKL;l?(OLQy=48eCP
zm2ynUo^1U8eiQ*K)U0>-2CYX;=z5F
z)@kfme~Aozrb)&S?ckEamgo@sD`pOS2hjw}{q+#?zl`35k-?*n@0dHgCH$#i3D_zi
z!*Hzm`zM098$t9hX^1GSVDT9YTExGTVhGsN`~ig){_h{ybdt3K7OaS&!ku|0QyTO?
zRFxaeMj`)J3=B;U7VqN<`rDZ-@~n{^qL9CGf$EPtX}eGqzt!$Yvd6^yQ!CNdC-|RE
z1gvk9VN{Qi{UJd?-*k-9b2PuYF^9F-a3wto!4(k`t-R^P}?~ByP%#o3x#1y|uUu
zLZu{kLfVOKGF{z(D{6DPpT)rylYrV$EWutVRz4?4%)*KO`~n30gGt4ym5E>8ogWMv
z@SY1_R^329N0BLQx|ij+?xRgX3w8|
zv&5F
zA}Pf=VQ8}Zeq+g)p4*3|)@+cBwn4hVeAXISr5J-Dm8&=CXvALJG{)8{I)zPO6dpf_FDyj@cIy7+5L_rU68sO
zbi&T2xOL+$BrKz?ox2T3dPa74r`!CQN0|{4eVs@MCb=yO6c-WkD8m1aZXFLd_C1Y+
zsoXcTZImB8&QI_P*hgeKG4g#Gs`uL?9HYKQ@7lm(E#XV9hpLFkXavDSVYk$I<}Z}r
z%+yaB^kPgk&@sT2yF!x*RKkVCag$pEl%pQFQqE;YK$n}$3)bpaGQ#p-%$n=Xo%Vf?
zaHsuGyfR#kypEUHIj>+z4gzG~RE0EobiO-DQWOK4?QEXVkxzgi?uv9ijCVy1loS+u
zOS%@G#@BWIO`m5eM!&Axw|^Oqoi{lnnlLf73hg&C5O?(=R3Iue$$$gBlDl^FIE=QQ
z$&8xrxRmRmV^j;7ykMbFUrshXR)kkknS98Xf)l33Q?7X?ASSKC3R#)49~kU=35I=S
zF+@*CAVI#n5_6SD3E17yE@S)6*&IYO6y!Iv4=U<~|YHMssk>cq>Mx!+wh4
zW`cSZR%99G$Lc#;??%#a!?tt0Wu8*ZwGYp5PqNb+Tm6;GaSNpkKCP~_x7jnBqM^W`
z3HT7He$$t!6fH*yBW`QOOeN=Ae76(+u(~WrY{OE?*j2!gK;HGOgs>5MPfrg91~2gN
zJ!S&_sp<1d@qk*J!&xSe4eRg;?}^_N*gx4u=)&w>Uo}>mF)SI~Y&xs7o`aF#>~yO_n%S(`Y{@KynGTL(NCyM
zI+M!3I+A~2Q>VP!hd36<2%O$y%C9R$^-4zmWlYKP&IA(EVsvoI(+XOq-qO$hFjnn>
z(V%Cp2-pqBO9DCpv2@(yCw0C6!`YCOH7tUh%%Cf>+M&4(xk*fQ36WIIxjJ6mXgh>*
z!jW)L(A%UaL%XK2l1|D2qNrW;_iX%o^4F`hHEb>p1fi=Q<;3B;9_w}*Ynd2Km
z@gHKDqjW|B0sZ%xRU4J_CUwXGESN~Id%VjZTFgzVwm3DRjk}l37d(Bi#_>V^^oPHc
zH7Fq=AVIIE60enBBOc}H7{sjW@z
z4#gx692M4)(>-RGUlfm=9GTPJHeo#J#SW|y4Tkz!A-3=PM@1Kz@)pK5V1-F|2LC7X
zWo{XAN4;5BA-3ulhTGGH$d$obWrq@6STt|2u?3jmGw3;Ia9E*0v(!;XD2~?4->`3N
z3tNn4)#Kj$xYP<9K_qE#1=r>~UvOJ=-~X=Q!=1ke|MY?$HZ&U5)8(1R=HbSP`Ousn
z?kj}-ioo!r9=ITjA43s=TsjNSWNks^7mj3W#Fab_VS&v`pulI(b_UJ%=)0errZONQ
zcLPNT*F8mwn=iO~h~t!<#mFi3H^tVL!tJZ@-C{6?uZi$PM4|5sX*LeEtKlz0yqEkf
z&akeJUc@#Xg_IRfEZ_uuRja1Ck>D1eW}xoYqW27Z77u+4SMOo{?u5|lCZfl)2|!DMEQ^R
zv;G+BMO<5x+$ss0%ne1gyO?#Ky}!u|#X!9$?a7@VF!mVC5uXuBZiATc`x;B=9G+xh
zC+_P?c~~~$Xy6Sig$jS317;EQBAYBhTgezN+VJ>_5CCAl!KS*0kF(#iY0-CDtzc$;
zyn;Jl)y=Hp?*x)0uP8Yapm3%s>m>wJ!j1RtjooOiue4sOrL|&Jc0N$6$Y$`>OFb@)
z(2aJ)^c-U?)Z4!QwBkbw)B}bZ3&{K2x|yiWkl$s-dhAZK7dP-66h?^_ygC4Bd^
z{S$9$k^>M>X3c-+ybHdaiNAl$JZQS|KOMMqe8=!KLY)j8n&x)4-O=nSqdlXch~aAM4jPOHd5SzW>_vPGYMTBXL#tb@27x$azs+x}4Qv(qJK@
zrQmjK6FQ{bLXIdFNRFXhT?msK@hdMW^3w|P8-O|Sn}0aV=g~5D6`Nt^F_BdqGkRAbGE`JOH>
z=fik@%uS_}Ao>~7oVk537EyAdy6K_o@=42WJ>E$!sIrLQp;f4dsP&yq=kuag{I4l5
zwNzY+qXh$-MLw)fy;oU1_JL{6J?2T@pupe$(tM}1P648MU3mg)?yX~^4@hvXz;c~x
zVy|o<(aQs0U#jGrL$tLszuM)s)qZ|{-ZfI$4v{?kc+u*8suNz{xEzy|e`P1nT;`>@
zBDaFj_B=&%lePltAvU;j{oZZ75QMXLa(FqF{>qr+`6%GyUg3OBVSWxuxWdWt
zvSUXD;g`EEwNzt=zk)Ge9#e3=@P6z~b-Q0Qjuri+fn)S%p34=x9Js5Jn&Mm#*L%4T
z_;3|@v;B1z_mRTqZmzTHDrqtH;oXW~fq)SBpx$unp!1ms%;bWc&%dW_I;`9VUM^d3
z>$tOa;McU*(+-!fF9ZPZMuf90BJHLEyNIvc4^!w05V&P#GPkNenbrrD#
zp~FzpLC5<`SNkn|6a^Ul`eHQs#!RC!W73g%Z
z86XH$NFVA&ZXno)yp^a=5L-N{zZSlWa#C>A;+L1$XjTgWEQ&X^)
zKHGZSFzTXP_&{h`;FR`o`N93p-l6PgMNdYiea~Y!u&$!|6FXPiL)_SKx8X`XYG;cz
ziIsKfN{&HVyGYaH9RG?J#c-|=6aGrK#xChq5RhX;VX#(l40>gi2xcx=JjwW
zou>~@kol=kh_7juA_Hx^n_PczK`-JvdTlj
znANupIO!Oykng$!!j?XM!OoS+*+rgG_>{MB;AlY*m)iuu0*GM55@@0}t}ME=X4f9L
zjd}8G7?_Z`Q>|sx-5Zjxg0%_)4o}LX64eCfeGuc|AGBqdvH6ZOL=T
zz$456K`3@A#h_eG3zcHIh(((|GLeeKM7dwZnAqD`}D+4#r#WiWu}OT<$TbK
zv~br9(M4yVE7$4^y0Yt`wNG0oyu1nc_7a0N9s8ZTs-Akd4$lIb?9_xbzUDPHS`%B`
z=@`#X(g7@meyLMnQQem`3@gqabBiUKCvj83@hr~u3~@yO#V;o(<<-2YuOrThzS`n8
zRe*08ycs@P<(p`W`;qB0kMDGI)2>$gRG+Q+L}xTNYib?cnKyk~r}rGd^LX>B&}OUf
z@wpM7FRc1vNJ4)n`kmjZ0K#C^Qg$j`J
zp?@%?woAuY@w;O4z^2HjG2U(a(KJ>rrX4z50!f*g+gA*!4`uSBRS(Y1wo`QpNgV#QUX!8seHQTO!Wfv%2K5KDoADN3R=sSxw(=Roxtu
zeNpCKKNzf(Qaj3c_a4{y5ai#F1RkM&)M}1ssq^JgGj8Ck*lRkv*v_@@3Yptt(hj)i
z&gof0Q@?i5E4fr}d6xF}L)DwuqF+$AN%ZT6n&f_Bez~rVXxL#C6+M#-(PRYr(W1CA
z1_u|`u5BEZQ+u2vDh|f&skJ-0`l7G1e`d@9_o~F)0LV0qd!qgOGHGw-4sM9UGfDlS
z8zt#8r?FpYfJ+|GoVF<5lyyr;EVQnE``?Og=TrsooW*X9NBZaZF*$kMc#u#cRVQ1#
zKB|r@6Ic82Yo1F@J$Wp2Qp6$rfLj}U&(GX|_Km#D`Izm=YSA(i?IZGkw~@|F0k4>z
zpjKd)0MbB%o*~9#l+%6~g53eB-n0SFG%7s)frFlfc!u~+S-sx!#2Ff9BGL~Z&)5)c{A(2j)EH#v{#)m}5MW*WvQP_8p5REc4{Ml1WIzfz_yVnex
zJY#%IE(f(Ns2xHTLSauQj?@gFfCyAWW#mt{*B)8)xaKQjv30uI4`ynyr@!vM`C__h
z>)&tfr2D-y5uSh(_OMB5-~W}NA+=!+28x50v)GWMHJnwPdC8yl3=6sZrB{E;kD}&>
zYyQL>RUY-oaCe7?H#z`&lg#tybn{i!MPm^2?(!wO-obo_ePyvYRg9d7@z?%ox_b3u
zwrKcN_vWJ1naEpUbLNTdlN>OrUnn!!sJY*82+DC5#ra}(n51c8X&{=$QgQE_ou`s6
z+>uZZ8rw|WhdevU;hejRBo?&k{9X)P%ot_zz;n%ptaQ1^Z^A^pC!8dmzIz{^1}?*$
z6g^z|pfJ*=rV}hMGuMD$onRyp8^&w){!EsuJw|=c18ppYqHM`__3Qk;6zbQ
zUy+a^$+H4QjXz@QUkfp`X(zg@V6eLJYBd=}LyC6XfDw8v#O$48y&VlZisCaC=Ib%W
zG=-jS0*BZqr9XZiuu9)g9h6_r>4X^6VlTbR>9VKRbyV
zcnblwEcHp-At7}io$%)2((3V4u5;u1KPUC6;INWH9H*1cI>s1dJ=qV$=;J*$j
zXL6+*9JYu>{(h{VU<6JG28Tu=E&!QA{JPTVbl;Z^e0J|zA8(+CHoRp=yZNkeQSGvo
z(9x8`){LBAP5AJY)^T%`7d!RK_fmy;$?qlRkrOm(8oH$2T5cFWK4Azt(NC-`zL{@F
zLd&eDv^6WD{5pQH)}Cc$Kf~ZLL;@kuq4TDp^T^=g5l|&<<#FCDw?~F-pWv!8o<(-T
zNGx>{zkR4qV@Rb>$^j_yEZ63I?G)(jh0aaPq*$SJiV%1=CnEI+Dg?jAx>$GtL#&^c
z)M6GjLY`7xA;k4DR5}-9W*S5fA=VP$i)r-(3m&dmxd+xH0~5P@Wf!mDqWi51vVU_i
zi;$7V9z=>X2n8%|y&3F5TXCD3G5e`wNgyLx(^*tro$=Z&&_v
z3pRc*WCQZDg}*}Wfsk&}FNjJ|2=14oa-=ISE{e9tQDE!2#)N-N=_ck(RQ20|o0`3z
z9x3#MV>*w5F{d!e?hzD$5vvH@Adf4DnuaN!y~hZ2bax_rayJ0WXayEDyZVde)3{9^
z72ak{-Ves|iLUPC8|jaifxnBpyRp=9J;?^sd23m_bBaaEB?c@iH~Vcpi);1l7#!Z6
z`fnTP;ZU5J3$YF@`Fn#Uh994qh6athpQ`omwoXq;3#^#Mxo`eB1S4+_!FFmZU)-Hk
zp=|iYl5D19&47v1Z%XWFQoVQ(?W0y>EmM4%%}sA_%V{`uw51aMnoBz&GgZ|nEKs;uDh5i5glO@-1h)1Y8JEt9-*tl$cbbRnN(^ff
zS)07#Ac{~8y3DH|T7xA%aY}CQe*5j@_nQ*&0_y`c6L&>q?n?K?(MC{HYuZJJ^BrUB
z=hxqF7Ld65E?3*_sat`*RgM2gt;(|t(D(Mwui34NEM1Rin(^SsA*R+2FtD=irD5!c
zgoJY6nt>0(PTFbLRKMnpHle=8PWqJlAl^*7w5-OmU(pNjzJqA+NWJTO?;UxuiSmxF^Gv9H
z)VSg(i`Rc*+Z3pZry$(fHz>;EBgNxOFAt1I?k8KeCpr;z8umTuMn8?lc))Jt<`(+&~l
z7?eZh_|7&JYKJa{fc{$F*FoKTqvAz%v*}mO+ktm;
z@b~9LEoa2D_G>}Zlt$)injt8btJAg3)smNNUGKC(^@c}d&-v%75XX|4la8?`cUR(*
zbDp2M16b{`@yh}V)q=|RCA0YNM=W*gC2XcP{bW(y&&_y$EL5>!=>F&
zjP^GVm3$Tr*kxdd&bB!gm$UNq{h6C`1u-(*)H;9Bp?b%-vl1t^>?kGcupE2V;eO6AM(ak%U^K`SzQXg~L1B}Tu(
z5n(wy-2Y~Ma^XW;>cRY?eq?;yL{m}RyZ9ykyuWVGp>b)_gfgFL3woR*60nV*k{-b7
zWYOTvwKA34IPxoc9p;DTpiQZm{7-lf87+Jm=HfO!9HyBg8)B)MygX9}S`U%W&H94{
z1hLtLU&VcK-nIHrjm~)Ew4*M`q3fqewjN2r2)-E*Uq!K)e(<8e!@I?s#QUc#
zI>R721VewE;|IM$pY$|SMIAm}`+^EI5_e`Sn>xJgal%UjiJjYq@0JE1-Cl6+?@CN<
z*Cc`b@(QI{Ok=fvWX-Xed0EP6Z!pc#4?
z`DtqS#d#1qh#OCjM3Z+3O_nh2Bj^+~>R+;DSv2_hnImW4)@S?F@grPK!rS|hFXWXs
zAnRQ#%NnG-=Jd5EjRdm6`vp3Wl9+UCaw)3@jQk5?g}p2>VY}ejJGkIkR`}GLK_|fg
zUL1FRjI-A3jvxG$MGyr1?&P2vK_r8S^iJ4L9ZnRipzv;4Cctj={b8ighUw4Y`n|It
zF!XcFtHWedZp5+k`rwR0a&QpY`dz0!+FSYn-jucNWX~M)cwV_P<{$)b%ZrAV0hO5q
zbA=KHBpUo_-0rglK#@VQMKpm-SBr@!w(ay&S^-&CG^51!D_H4>^*q)$*iisaB^Jog
zaqtClM0e1UN!5BwsxRT8V{C=QQjDSY7qd^AXdhJ%Zv$I_E3
z6t7Kyar%32NiN6O8c_m_iOweeknL`GJ4hzcj`qdi*QJS-sMz<(L$MPlMd~k6sVd~q
zWFOQJM0D3WVoLx**`<9^Wst|-q`f&V@RaDdn=m5QtV1jY!l%tN4D$Qv5C>O@-m5dl
z!uUZ|8=@mG*yn0mcpTLYxb-x3{tHADgf(o5rs%ds^7vs1WcKjI)jVeu+ga15JJLt<
z@MGo=zcr%ubHA(i?~`{LA2sz42^7Kc0UN2%csT`#HA{U`tb|l@bJ=%B#A0111Y*j<
z5aJqVD=N5E=HTd;qE8zRuAQZplp#70oLPF$cy8-v$g7oup}=T8duG}Fu|m|CyYgbf
z+@I0QYR@l(45XZLBytcCp%ptx;$OYU2fW;pU&t3b}EUhtqr@@4GPda6!NH4XH^ZJ_P4
zF2KRG#m7npF_M&iZXkVy6Vq?}9{2EEZ5Ikf+;F7e=QD?uUptI@44aE-2B+6Fd5s@K
zJ!>tRIoo|0giV5qXvrs8r
z4I%M}6{>?dWo2KTqyC3Elf6H-T+&r)1nZUQ)1$%26PWZvXXU^l?UL{ylnbnBf!kLM9d!<6Zt8JaJ1*?CNp*kyv~MML?;}2jCuqqHcMAq^+Iy7+9AMd_>ekEyFz`Oyg%o@(m{W;V9{HQ3k$x$?t#=a92F_iM0c)*6*c7?bq
zq;7gcrnC^`eX&5u8HVs&Lh5L!cbMa5*u$ff^goUUzk*{HoC%Fox2k2QnAOk=2HeOf
zqoW5lP1rER5UU2sN|GJ>uQ<~1oe_;7O^(g)A091u(>6Oa4IODtPm3J0bt{W3#Z}|)
z3P+ndLukcbDBLPL0?xooXuM!PlT?>2Lb6{No_M
z;uAuNj7&Nq4zf@v24~v1nDZrc9G>`kAomRZ$;N`?Qz3KrtYLQF?bO^^WQ65!pE#k{
zdi0T9g+4lUN~TXFqQBxFs&vRjfk6@@m>_%PAxR3aCx@ZScT0lsOc_yCJE28NEWVhc
zN^$I;35F_JUD?UdSr{CFpB{3%a(>-P4P1Nbo~O}HY-KD~!)wfZXL$zPni&LX_dEep
zc}Sfh{dy`IXcsc#!F`5^4z4N;K>~*kr}q%YBPBIkO)5KOF5tx5Ct^B7_EclXOBo;E
zYU*bbtjmVBkin5u)7jnS1H+c9n4U2WjHCuWPAyr6UrkUP*=HO&zD{DO&gd@ikW`2
z{xAu;A6nVMCfZ8X;v5*B&SPy@ZRGCyeaT;W!5U#jkC&s&8ga=JQb&W4KWyeIrVq>;
z5HHh?e^DLxcFsjMp;kB%jIfE0+-%AEg}S8*g{acmy$5q5l@w03xm3V<{-yj)B~RU$
zR^1mFz3&VZT>+9X^kiAw+I^eh7aLNEP2P)C&?D9krwfhucd4pO9(3IjcL=a*ujCtV
z#s#s8vi9uiWgWwNueUX~xU?xI?6IX`O~28JmkeHGBv$9;XRAYylF20tC(VO@EM#=%
z*+7kp&&}EO?I&J8l2v8qkk0k>+lCfyW`Ah9Qb*oBGQkPn1qssPAg`N=YNL?Yh?Wve
zPug3Gv%b}wR+S0q(QM%EdaTOZu@{>XBi9;w4jXk#bfkiJV=~}l0zcuD7qBK`-i2^JTC*RYIRJxWKqTZuUHM)ufv-&&lqH7
zk`lrDIsdUDR(5z`txsD~Gw>QIip+EKn^Bjba<@{Ta%I?Ov#
zgR1H8x3gk1MJ|m-Gx4U!$XXG++7}Ma#C_?<3)0+tl@_IAbm&nhZ#&)RWorC76~S=a
zjd+;XxwnI#kupYPDo4_s|E#&@Z;nrF)^W6D&Ec^`I#X^Jt$p@>ejricPJBP5RZ_nH
zn-&iIIJLZ^maXL2s(Zm#*q?lF@-
zsvC_aHpVh#RoRc-3DHmVR_ier>!I@nEAuST#yquDAqr*q$dXUO3k*o>8~nD=)VSdi
z+kAx^gs411+C{1v;(otgU^=gVj@LQh?8%d{`s9+b&QCCHDq!;(
z+YaK*wxb`{?YW#%eXt~vm0;@4ec?bpYrmnt#h&S9;8Q6)Q4DuKj`i3?TlIMU`IARu
zZpQW_%mkTS=LPUX9UI}hZZFyXdz|~5o1%+RT*gCwypKcACJtA@(Mk_
z@q9C}esdt{Gx>W1jL-D1mAUAX#k+VYUBJ0-P@HU
zyNp|9r*Yqm#PwTyRV&Gf#~C^H)vm-=?URkx8XP7WX>v&D){a7*g5)w>g6x={U(d6SI=^(^`?-p2#~-GNU-2^g6Mp3Z@db1MVy$>PpCKVdu6yZq-SU$X
z1RQugADyY(oj4wzMNjOZWi+YeC2XCqt|***FDQ0U>R*z&)fKW;()T1lxv9@?uzvX(
zK^ko{gnq8*};$s1X!@oae#S=BJUC?{6p
z*Bth-%tY@4qQLe${}2hozz10kqyCgYt{fVSb^arT5A09KZ~kFu!#ZTtuV0SMH9!=3
z*mP(ZwcaNTY(7HPWGoi{ZiYS>z88kb13f+kwG0UO^GMk~vmeeN-dwx8tj23-y1~)j
zU`54l*MsotJ4@J_x;m&MizF!{UaXz7}9u$NGSk2gT7Hrlay6&|hAC?@v<$%rPSl!_%i
zmp`McpSST-)Xf6AGrDdAX&1&A%$IqhjSak@{d?^uyBifbq|sh}QkUW)gO7ADn!9Cc
z;KawLV`f9aExe7!iSe!*A(EaB)2}?f74Xz)*2?w(Yy*Sy3zEQu-z?&i@?8Zoj_LVt
zAw8EJrGIqWdtoikO>koUtFuGD1@HZ^mjG?wsC*B;w!h5Xy5+04e|FZ%UG`b{vhJVr
z$2W~Wj-AeWZ#bUz7A5O`C-vM;>gX!8b?RnV>ZhFVsMGl#ZhYX+`&6yVu8_z{r56Wc
zz7ZuoULY3bwnpRGz+hbXWh|qWnaAJ^@tPGif?5{YbQ5_BV~7b>XnwC6>2
z&mpgL&uP|wt3|LWr^i9AeT%h?$C0+<@f2TSCoQi9LkSga;9h3B%4XJyDz;Soay-z*
z-}Y^Bd^TNH4EEs?AAmGtU*-C$q^b^~FXo#D;+UBi!IrpxfD>jq%RHM!^jy?tI!N^2
z-Q|YwbUH-ofe>wrIhptnbf<$?1V={HjIY@4L_{+JHVqmZ0zL5npIe}5L_!O~4SLQ#
zJuKV-Xkt(#aAQ_&wh$UhEdx^
zz`;(&9MsfMromd~7h+bYxwVR4{Y)Jq(&Cu9H9i=FhB5vgh9MC*nOYqY5nTuRP9)Qw
z#Imi>Sb%QXSc2qsTX&sXJE?7^0g8g$Cd(V>qML7Zgo6#IfpbnU$lXZE6%m1WZ|yq2
zwR`0QzJ6lPD_DNiWn+8AfKRA?anoc}DE(ap@D;r=!l;CW75p0U9#Ok)oF@KD`u-s0Ku96=C!gYtVeZEN&S*Rgo%au}Ld
z)P%+;r0a2nCn93ul;t(@jopZy=!j)`RzN3|RVEK~MXJjhAx%E;CL(BIpHvO1W_;AxmNQmMV{2VS5tqAl78XB^>+v&FDSDC$N4J}q&TMlz3F5Kt~
zvIj+ES<9yC_iDxAll~3UCwdD-(NasFSf{yL3n-VGmeR(
zbwY}2Ky*rdami0H-|z6vY21>~VJ64-_*8P4p`My?VU<~XQsOfL5J7SfDQD6#vGQHB
z?^Cx*ErqH)9Y=Lv7em1Id-y
z=H}FsU|H4liJlhvFE&D&FpKP;`hQDPP!Nh@1NPwl7qTb}k)Srb=|x7Ufw1}
z`w77Au1?1wVF?(~z(~sVg8{#XkXYt>IyvP*J=L+6RHli;nSm^2$y&}GMJ6W+(1X{#p&TKb5qB?gD_k@3h@V8!C7h@c9h5JMjjjGI
zzX4a05)g{m0nr_yLQ5vd1POs$LZV`57%xTYi@i6VZ4K6b)3BtZq&H;=zStVvI$o^h
z@xJ|ojw??k(XY3{Ic?s{5tMDs7c%=s!w)wMY7k4#h5{%J4jA2(ro7#%T;A5!vVE40
zg(s+A=pkjV>CpA!BEGAqW74a+v72+5xFlv4z`>WF9TDmNlz!E0<1cR;?ZJs+cRH>&
zmv5wLJ)-!^HU1~@+K3jFmyTDX_&LBmjnU0mq{D%Z}H9%R4U1l+stu!kia+MNVpR2c<7qKwccoPR*1?DuANm8*w2!YR|iln?yXahh{Ho+3ZLPX`N!GguZO-7oH5wKp@^=F0=AyNxTIR
zOG*rA*nAX3wl&}LriEEzAO>|JVO-e{>X*$vdzt8^HIn3>F$ZHnr29?ZMC=u;b(s$J
zf#dxnFQZNVyYjvMCa=&0KD
zNW*5S4gpYQ3*v%?gWGyMsys`s8+qirdTD)N|L{8X-ZnW{_$p-Min%?AQMc3h;0@2E
zQbI#y4nQxL%r{%>{wX1%qXFLjW&+)lPeW<^$Ur5pC~T8%KY~S9Pq*+&Zt%>5g5#U9
z>Q_L9lA!~D7t8WmmzwGD>ri@^qEv$Y%;E|=W5u5M^=qh{Jh|1fn-=5LB@eGO2ewPS
z&-u5r5m?_WF)9pw2#&AXL!)w12BFpAvF{Tbgm4a)Hpi2Lp(Ih**rPK|#a)Rhmi$g3
zepF(9{K5QaKD7Val|pl1A^;L2QaSi3%!M>o*^B>l-i(}`9Vw$t2VYCYd(v)3xb{_e
zWcwRtw=#lshChymfdJaHY4A5_z`kz~Mp#G_UC2X&%haPN@yk7@PnJe6Lv3&CSqF%Vm&b0Ir|tCxoc$IE{cwN3
zKBsHnUppa)WN&W|=<0mAVKy@}yOwzbb8zq{rpBFhyh6N7!7S6%k1mNs00cmVqnaMg
zgQ{b=f|EP(Z`1J|ijj
zCAi8MQvY&}f-0${1)cwWkm5rVX6xEL?owcqk3JV0VupQ?l(xbpG}T5&QIdye$>2Rf
z6dhIR!-+eg&D(0?i=a`F6=qqcSfv-ALXdfJ2sfyVK#6#JEV3HJxlSRr%?8{^Z~sJ+
zT55`6cjYY1w|(}4PHHosk6S+^w+bu2e4=#4*~65Qn%dy&{2K%!nh>}n_VMxIvvoe0
zqK+Zr9zS%yxVUKA%ke(98WAhX}_^#_#(!t!bR)kvHx5&Bx!|RXBtz3C0
zkvwCQa{G@A?avYzQ=BP)fT}(o=&BIZ0T9ve!q2Q>_Gp+W1lg+;#Mi!yN)gTxkD}Of
zf;Y%HUfbMFXK_c>mTnX?7;hVwu*J9O0x@goZ!w}K*k?3JrWduq0167!EYgDRezZvU
zR|vF9`9l`og-g&qju!UprBHo8U8zxA;(Hh|0m_Q#VG)25xzi+Hd7OH~7a2x2#B}SG
zhwWs=r5A9H_vrx#FFc^t{cPZSykOKDG~MlSWwSq;)p=JAAS@{ftD>s9;)m)>X#Ht#
z8xpWPcJM^CIMWAh^LQQIVEgz{#tTtJMJ1t>UL*ww_;QY>E|#4Tpp9fUlIfim=eOyv*N_W{A9dQ4juCDL|QlnmOe8%F)JhPo$pn(#ZQA^P9(~_
zFNaiRm(|?VF!~?sGyWS3KwnZa#5@W0(LW?iQdGd2Je4ssRGJpGfLTWBqZ8t7VbS*j
z#DFQ%LJW4l4>5^0%A$aS=vp3^C%pqVc&Y7pm|%zfod+~q+u)C9xyCBMI@
ziBuv>^z_VXwSOX?`51{Lp91Th{MMmT5t$U=uiwz4It_%@sU;g|0+*_HqN*@mLJ%XL
zLfw)*AEf0KM4XOX12OXuYO5SJtQazJtoCl^DgEu)l2mUg1Uj3EAH!*k=6cMv8u%dc
zPCtgNt*PHA>Kcj%sO}X8S*;j7TU(0CUe4rB_w~
z+ZU@m`e_Iyavna!v9n9RC>xn+8WdF2LT79~2bJ##ghfOkB036@
z0rreUYmlFoj&w~SDs%=YQ~?N&ibG;zlzrb=EKP@|3;=?lRtH<}IX@ExwE$GXP)7&D
z13Hvuq~n-Ii?Uin^La+aVRoXrn`70WNK=Rk1K<%51$DRzFw`JFvpvmFsj+lzlJ#2_
zh6?0mW+F4M5EZq}`RW@y7SVBWh>PrQ&3PS4^Ry__0SFk2_;9~&2mUqrnMljjqM)J{
zMxzH5zP^ZvjzfH0l-=K`v@Qe9`QqmrjId}m;w>hto6Bttw5`kF8q1)~$ZURlvyCsp
z)g3>+twX6M6PX$0put_T@N(~((F2A#3FUyO*Kj_LM(=gd%pB25<3
zGzF-r)xii4r~<+erH->|lkIEM9qHHYiyNHke+(7K(-cDoz&AV&kpW%rzceqqxi9n`
zi}*;tp7Xt?2pMVV&=i!R)(EJg=417ODC_)^@H!hRv`90(ohf$Sf#C
zO|1bs04kL){DQ*~5k3}CQQ>I6-&TYCj6&2_l(j9%S6h&Y43%}l+TykT(tZ`CA?;W;
z3MzCkDpUvvk40Q!ob5cfk-vFq1uz0oL?^)Nb6i&SDHf3(_GhTjLQ|x(ip>nA8XP;8
ziM+B}7~uim;3%jQ6A<5HemN4$_5L~;YKoxMYLQn^jEV{!j0QBfUlCD=i;wGcjy=Zg
z&Wr01`r-Mxw-FatJ4YuM5qzo)0Y`^MbP9`!qBOV*%}{JATKQ)Y1-nJ@kB796O%Aq
z-JxwYltFXs7&7vj+iHbx1k}@~Au-ag*Gr)GevBf;b#i1(Y(PbRHi~KsP}Wj8jFs8Q
z%rIKtNNyXuI?I5HymX|cX;DzC0~9I*M8qN?F%gk=eU-zy4C(-=e1j3KPC%U6vTg30
z_Fbjw?%JqYKG$LF{@RrmLaQx6K~Zsg8hry09$`m2VbMp1tGkbV@-@Y%?bzS8)UTR?
znzsBd(qtmT*LsSfr@HERyRJj2MvF27;1Mtu>TY)47Ats+K!iHH>loVZ>nveB)Z19F
z&!`Zg4hFId0q9VxGXVS&VV7E$1q})c>ULSt?%aCotvGVzNZbBezg6fi{o-M?*0okG
z1qP1n+eB5RV#B(1cw}of8oS(RIIiE6kK{G}))O+D*JI84w{fJj>)m_B{tLxZ2z>?+f{q#Z(c5_8w;`B$-v)fRVU5iRx~jd;WA)mtX!M$iSJG3lUftDoO2Lz>
z@50VTFU)*JgVgmlrXEUC@W`5Vc%ef^e)`d?c=C}2cyjY*ER(3%l%za@HS0EDU#aCu
ze~OP@eH4$(d>9{YUXT1oR^zw(8sW9{1C*q!vpLC=pS&LH?tiPpegJ&>5njc5L@Zd3
zw?2Fx%et9q(Q7`${cG0aqm%#7-uuV2eW&@puX|(*GrcC*voQ#yn5ntPzKYozsk8>E
zz$B}NQ?kWUQw3(44#~DDX*CU*R;Og=?j1&ZW=gMiZ(-)NP0}c#O=mGG)tV~lLNy^k
zBz}}gIKq*>J_m2@T-gHWj7_k*HnOF?e|*3Eu<;Lmm?X31k&sBRzu(XI^E}_@*ZcE4
zPa<=BIp1}b9DM`r=elXD&EN6o$wU#sN?S`S=SJq&zi(adk)y4l<(!qyy25SoC+&Rz
zIL@A$_Zc2;#gs%RoNH;Ot=Gx@bt3&9XIh`&Xh%2aT9N^G3kIyN=lN1jWr9EgZcZHAOY^7StN&DKrN
z=-q_iX_bZmh3OT3{jwq6bqTw;{;8EgXE3?TJuaFfLz=Z%!DKx{cBKX>8Un7+-%A?Bh4-SMHEjsEQ=gO6S
z9BBtC6jA@qFF2U$MR1EVCjVl$Jam_K>3d44Y}u-Q-WcuY*n7t*mnS68wxhG>$LYCz
zlOQsT$NrupxnFVl)0^wAH8p;NLEZD5JXtN-isTd!`swKzUrlnZS-aovyGWnZV^d*l=GRI54ezK`wEMFC9Q*Zpe|3AlNR;{MFSzpgEqv+TVbwFd`F9QC
zVYsM^q04{3vDQl)@8$A%{8yBfKj+skJ_6ORF+bCvUZ%%>FWz6a-(qx7g}(kJUarxm
zPh3WwOp84t(HRZ#pA9*6zc`7+r8y3FivelXST?{+9cKHijKvDhQ^7cXXN)D;#
z@~t&GIf%Wl8+!u8U_Ac!oG44~_sw44ieC3K{<)NIFg5d=myIc5s`;^AE{}&mVdCU(
zUrKmCx4`FY`a_%c1s^_zL(BcYa#7~)f`h5(U;p)Axpz<0>6MjRZ>9eD@N@Z=nkNW5l=47X9&
zd>-c#ut(1!OC7Z)1Nn*wkp(yQ2|E+R6F4L3vjYpmZPcCX;*}aAmrw}Ix$R6$3^Q?ij=6=%Ca=A~
zKuax_^U}5V=s9GuP;1C%!EI;QW@B=Gp2_n~)Vd=K4>V)gDON5F(^_Yt>-G|mL&4J)
z8ZBl@6cOALHmtT0&RQC=*G95%t5)hPUEE#*DBy_(3yrk~6p=aX12$|UT{K$eGJ=tA
zYa_Kyy?6j}3V5Qx30c63i?joWoLkiotnSWY|(`
zqA-6BVR!)Bpp*N~e$Lj_GB+?xXA+PwThEh;BKA(|>dr_&_#X5%b<~jYNI)!gHJ{xXrDeR@|FdJ
zVUf4@Z=a{J(S+S*prfvl^G*f&nxoX3O2|jt`$6Y@=10!b*lNJuZAq;I!cL~886bu=
zGeGjnp2T2t;)p~LrW}OJ%2PWb0XtJJ(Q8o|D>vVFJmAGAd4Mu?ar$b-6zc`>UcJbr
zQJ;7sS+x&?L5o(cLPm^G$dBLSVb*hxSS-<53ov%+GS^%pSCvKkF_dc1sFcWJA%cEC
zUe8@T{t&TP6jJTP;xs)MCU685sC0Tti`C>tL(IDboc;*0;2o?N`cbzXi9ao!uol6w
zfjLQ0AA^@};_$^#7VXDq(4x)_;hzc<&-OTm(aR8Nd8{LdVW=dKT9
z>GYHqYfuux@0IM8?-tgsAhFY@s7gpgZQZVK%x}n={mCu@LukgzhxR*!P@vI?`!0_L
z@y6sk!spQRXIyYN2`NFK^^fp@L)`s&lF(RB_-U84d`;SD+|%telA?OMG(oKFzV43Yf#E!
z#6khQULPLMU4pUbYI0kyMrYbDRt-4C!%fPf{RxR*IZExLkP3_;*gnSL2qIJM!&IiD
zSeZ-6>tx2^A{Yze?EjbxoZ|JG;*`nRsLO#M_K4iZ47nJd!ES8hcR^W%Ubl~eJUQWj
z2SKFspdYMa_U+_kidY!+%Xi|1Xxv2S)4jP`t$=k&WNa6bp=Y~VL!7oPJEFVK}Ro_Z5mFS99+0?10PC?
z^m?=fxv=QRB{=a%V)&=5T+t?x!41%+W0Y+0X-hG&9JSGeQ5nK3xI|D=wGU&lGCrxS
z*=7|vLB=k9#JDShLZic|D0fZLI9aSY%7-6(xuV~dMS2XS
z8kDjy0grTzpp&5s9}zzBCNC!FriZTIqkhId{Q`&7u}**Z$LqnVMIYW^Otiz=eH3dn
zDAlCnM$T3RG=HcVS%n+OgT)bOs=N?V+;w87kf1oC?5ZE3|)Yo&3ssQAJF
zt&I{mn4{%*wzy%bJ#_ZFC6TG}pDz>$%TJ
zdo%5|6LgqUK!mq@`6Nfr&w2(p*SNM-a68sUGfll7?vJ$7Sf^le&cr)6Y;>&MlZe1M
zi
9)B$0^u`&asY;81^YoyAcz+sON5hj>fEa#~dKIHuLEm0oXOIg)s
zy@Dw2>o=wE%g|QrPkYKT<$9w6qhmg)Lg0B$pR7)P(mm`)@XF$o7T%j&cZnyNiw>OP
z^kHrC3b%j@Pc*dyA97CMQ0SxyggI;4Q9CKTBz<&0KV1_lQYiLwQk}7|Dqv
zs6bc$b50yAUJbYoHc&70@!_Qj{4pP+SH8m3)<7zZFyyook`+;XvYC2Al2&AaaeDfs
z1f*gg2Ts1q;e=igq&g@2TnAvKe@T+;PVeZ
zW85FY@%a}_>k^Zli+uCxXA(FlLsM~-<44Mq$UvC+8G(@W2zg}_h`)aS5RYEVI|RaOnn~hzpx_6vOH2RRC%`c6~#8oh|#5ee&4v
zAk&yQcKj&i$sRh(eD*k;pn8(yryGeYzOZ;1x%3hJ&IloAKc9c2=e0!tWxIUdFljKB
zDjA&$5%V~4Ma$!4Kk9KZ8v|qt6tM`AkO#pFrN!}yEWs;b13F#&er#J7!Z@x=Nk!%U
zG@$GvULnY8>Re0B%g3pizedMQNl7=ELCtrST}QA6r{ddVu=CG(8{u
z0edil=UOkLhE$a&Q9RR5X@;CBUaxF&EmPw%G!;MPY6bFC(8`q~94LINrzgs&_bC%+ACaVS&EO%iy32DA6Bzg%efrhZq7g
zRxVt*g+F$W@t!XzYkO(UjLV0t-=luUE`3CwQ{;`N`ZzBqTR@(U>P8dg<=Uj}FQfBQ
z^jzr09t`2IeZ{P?F%6i$MJlM^#g{8s9Q%m&J{O>-;>F|ZeZ(y5^(=YkTjcZ3_(K_e
zK>+V4muz>^9<0br3XGQH)D&kB!{~CSuNdJ_0ROC=(V?3-f`DuH78SasOTm5I#)$PY?QrUH|=~IB;;I^uPY}CoW$4^nt7~A_xcoIcA!h
zQ&$2hN@}T%>yPQi?iMSI3hG)&UFejLsn(o+{#!u<=Mup|fP6FU9nEL~mh21+&u#tz
z7Y68P9}(LmYJY~+mK+=e$TwNAY^sl`KzD}?u^6m|4r^zUXE4!f?dE6VLom|W$#7%`
z7&&L9!zw}(dk%Hd9S06BZ)Y3U&O_|k=B*ZN9dtVZa;WLB#)D~+VlHXyrn_06_Bs=t
z9Sq(FDBx(PH69$4WT&>nM&}`M#q<3gblMZ-cc6pLNfEWvzSbQN4oXy5cMfaE)8qhn
ztaMo2JMui4$lcw}IT2CkXWh00aByX@bXq%jItQ4y(b?_Z^nHZxcDg+Pd#LHKVQo$^
zzfVy@ZDVb`Lo#QjUEBHWpH@CWj~5tqI^DKp~AC
zt>SYs&*bC+hdQkB;GlRjTH2eXd!^F>QD$dc;>moiDPhpN99@-3bbMkS#vOr_!P?YD
zPGY87!R%L9$E8z_no8=+)6b_%x!#Z;zjPu{TUD954OXrOL#aB!=k|+^AVpMEX;TI>
zar
z#;!3HT;9J|lecxrGxFl`XDI@R+~N9^Kc0u~uNibkR`%s&Q+Wy*$fcmHoYE~Or2IbV
zz8WLdDNnu2$t%&3azw(sQ<^ckOS`m7-%m|XnH%xN0hJE76<&bNg
z9rX$>5p>cWKs>8<57%rU!RK`1i2tM~;Y@CA#gK9K+p0;tQWUx~%m%izwq7Tf1j_G03l
zAx-s53E-gkOzNjz;DB^SIpT8QN!*&58;tox{j97w%F9*jE1cD3jT}2vB-t1{gWm)m
z4gDVVGj`#3^f|>jo0`gIUQf|F8MTp`r?e?&U->SxE*GwC{oKWu31M6u<->Q*^3FSFdFP$Kqy4|V&xh6r368HbfDA_2!ou&s45Qf8BM}oEjEN+SN
z6=LFk$&Fpu%z72%D{=r>5Cq)YSc2Blni61CnCmd_0a!w?Z~iWk;cjeF>&cq7_V~)N
z?UH){t6c=A3M_3jm!#U`I$B$v5P$a!V4cW58yALc3`<~}rL{5TDU`y-RxI0Um>yx+
zHY@_Jd+KOuEJ-OJwXMxOk(25=c5DMu?|FSAt&OQ5;tzGLzNWj)T
zDYm=SHqzc=*rFWeP}`b%?x(`sN^6a{8TW^6sX^}?GcBnO9+a3dFSC759WAL|AX6Qt
zOgbRyVp@=9fRr27Ihyj)3d+SuAJ3Sbq!6Kdoaqgm#7ypkO>yPQO>y3ytcU}R&ry1fY#z>kWj4p*>Kqy1zui7AVIyo*~5Jsfs?yMFO5
z?b7#`GT7df@sZHSg_V=ZKj!#}L|vDCovkc;CS`&|uG6EH`po#f!GuJDgR$inW#tD_
zb|3)At9S;3Ok9i0;mwX+-_TxX$Nubyt?++s#*4Qe>>Oqy6P
zm$It;;=V^*I6OP#KdTIw5)Y*2>Zy_r{Gnct>a-mDQ<0OEsmE-Y0aA_@zg2O$*X^m?EkV
zr1ratt9~rS&)~5;9s&IxW!CmFeNt7Yrm~sSFB-N5K+3g7bP|~EP4wL!I&EFAXNR`I
z=N<5e())rEsrK_!#gkNQ=cm$yLAebnjfDw^A3d=7k@KGt+KMjIV~;&XSy|c2&vJF)
z*IzuC^qrdl#`!
zAc#W^G1Z#W?w84IM$2tH%we~?Az)p)aPcTc*lZh547?!}~LX$Z|8-aXEN(uoFOKv*?P`H`<2&?;QR%~64
z>T1*7wS?L_134qNS-Nj$LI6uLCYG2mrKv71EX)DV5;+EQN-z>d{)!G*NTCCwE;}x1
z21r@bfKy9l`Cdkc?tyQLG4Fwd);F6uiJ9m`3G}SYqEKRNYUXf>JVTx@&3f~67>dB<
z1OMp9bS+j=T~$GuRzs#)$#Sg$t?Ul2Scs`hzhkkgnwpCJ=!)|;)fz>n(V|s^{n3cuaw7_08nk&vpafKk@!B3?WAo=llrDVk^%0)xXQ(LCQ=;Ceqy(68`1n6|=MV1EE`4w5qYnp}OKW?R
z=)d_Rh
z6&f|*hk)Nt6zOFDHthO_{D!R9pPjNb+SE352l%Yk?MwSS#X2pD(I63mc>Q6POMJ4z
z=grVsnGl>T9+dd7=vt{P(&^L`YqThb?h*1hnT=K;b{q8L@Q52H)9T~fU}Z%XH0ZSA
zki?khOS-yzR99D0ZqTCs4|AwqiPn%#;#T*7kRNXV5{rAi4ndkBXTrFx(_9)#i_&bY
zQi-UZVuZsnSR-xQ_vJR7J|!^gK=9nf>kSc#MS&R0-P%ISg{YwbAxQZyZaK~f`|wEj
zkZCg=KwM8=mjtY%NgsO(89-?;Cdmm%op0XOdF@jinOWsIyZOayQ-q1)VecQjZ%`qXo3vx1Z9Iy*@}sGM*S2e
z%V5p9c9Jf^Z@gB@kNoU@g9ETAYZr{fGu*0CE4A&g}FItO@)+XLPbkVl;nVCX^A=a9AvfAbcDG%sm?6F
zBm+R3Php`5K5dQWa4(m%<(Fgtt_mrU0HF1knPX1+?-D~vdN5j1LP;TTU-}+MGBt-=
zx=2o;A;XRqQc{=?%)`PQa|_EC$4Vjk{xiBHO$AWyAI|7>zK
zY9$Z^LLnB{#${;(AHO)v9tcf+#5+?Pm7y5n=yF^T!oR#wwPr`doZPbKc+u$LiX%uY
z=wiIz#dtr+lzY)?br_80l$YyBneSM1;_$+M+CdO=mv(8F9(~GWd*k(ngPsdtia?iQ
z9}Optu5SgoBRX3~Zmuk?QIo}`JXTXR;Xp|8!(uqDoaGl+HnuOZaD>&A&o=D(M*N1X
z*q@ytMP8bt+&f^KRhslZm|U$Q4~P&W6bP+cB^!K>&h*hWcsIq3by!X}P`WmGS1Kj%
zyaJwRC1rAmdK_Y6PNl<6}NCi#F>L-CBUK!mv1@gme8IpOR`XM=mjcW-HiKV
zggm#ncI6h=kfGERi9SMk1?2_}j{;ukzWO`*gjn)F7C*}yP00WhB^g8owPr=&@Gn9<
z+UjT8aZlu=LC?0D^B@xsD5Un-Qo7yS_T@G~VfPR{SH8v(jAj0CG`2HwM!?Uio2=TT
z>WWJ5JiuN0+TW&cqu&fiScW^ZX)I`z5r=uckSO|2_9Ey#b_31=rWUsTBQI_1R@~-B8WR{5MAzjqV9-~*>T3QE>G(mD)f6&yV1;L{Jf
zu?Hi#`ak1Z={s0<^BgJZ;K74jym*oCzWa{;{(gS)lb>w*8f)hj9z1yPA-tu#l#~oD
z1K3U7bab4ME>*$=Apsurnj&2(h&igE}
zgy;4Mp4%hzog{wVF;=V(n^nx)vWXNxQw6on!q$gt^auUUV_Yb(}(A(6<=K4-?I
zWN9N5nLm*7BasDZ|8h34p5=D7ODrYq-DaMf6(Bm2oCVu?o{S7yAhL`r=Imk5MlY1V
zh5KxHU$^Kk=q5BGtD@Jg7irISY%BkzkoWxx9
zo?O^I5S=bipldkAi7L$og4N9Hm6z0W`fVL!gEttTaf!2wViAHKCqa)B`{-9Fb(J(7
zOPr@9r=aE(ZQ5}L2S=H9-6IkMB0+qvAU>C!(S8Nm$`?3hDQ9O67)KGNm*?jgGtZ=w
z%QEhnOtv;{n<|(}ca_-6H)R>iHDoMz4O{1Qz`=36WZA_)AWT^DMXcIkUhX>IR{
zdh>Gi@@LqC04ge8ewCV3tsr+oXKyhIjIB*NZ9=!pWU?&T^$qw9S+PGm6MDxY_UPAW{lj$;eoPJ@CbK_u%@C?z7IpO42LU(
z%i)I-4Ploc%}?BmF0rgFTe6U6;KjFofnoG(hQ=Itf-xXQ$bT1y|1OTH8`xC)IdrTs
z$sD-{Nl_vJKfY8M=Oivvhe87nb7)Do94LB9`Qitdgve{5@9&I+F%KujD9;rIDcj9XBm4QF59
zt;UHPh@a(hDPtKWP7yu?{T_`j+wDvTwj?nhyT}K9E~%Av5&AM6rP?CYxw&g}x1X_|
zuW&}#rDU{4K4;7C-MBBfEF_BrLa}W>cCJ!Q?pEGswUMT#CvkmvixBttW>lcQ$*@~R
zn1cT5ul|aE`lo;5-~R32=;`T6+CKCrKlw>Y;PCzT-{;;v@u1a#0|!>xkv>S;1qjnn
zp}Cn(TQi-J1qAyr6T`y{3=A+Z;^h9)JR@C=)I|ocpG%nYrZ7;~)=ga-MBH{JCMFmd
z9$?ruz~uZA_np0*sarr{AE2e=K|hs(e1!s7g2)03+s82CD1QYYS;&m}MHX0Yj;P4r
z;vmE{_fRZ9=fdiJQeGyqu&q5+NQHPTh+tIeEuCX-Aw{*FBA@&mU}=fST&DFbFejmF
zIdMRsVh!dOVWlWAB@34ClfU}CY&}oLZCifc9G+;Oz}{xqA(1a!+1G`+g)PeT#@?5;
z`>9YU9-i{@uFe1{H>68Z=22du!{%}l3*BMN)j+91&SpXX31`cs^&
z(QM3t*x-8QY6CS*25Oq3EPC(aniUWPCxXjIC>A5+y2+&vWt@JieqBXfUP&b{zE;VL
z(M9GxcX4?H1cwvB6C_3i&-fqsAeKwpPB=YL7Y=DolD15rIsX_AXPn0r3&)lt9pqbC
zuuQBj35LWmv(8FGTB~$0Ml14G9+e2(I0u0
zrgW!A9)!-`sA^v^I!Y=|{)(p3tzT=)c0H5dkQMv0ISOaP0PFytW8pAqOCn>dEems%
zvaIr1rmdu*UYFc9%8J(cYI5ydrhe7JI?svIZM1@aq
ziT6+@&L7OOEXXyK9ejzhgAfgPam~6!c@>=aBM=MT;o65EBWrt;8ui1{KXNorJxhIY
za)Feo_oa}Pm9BFpGrayr1Ickpf$5&_w+9}}A&)M{QH$|>5`G*U3
zwukpBsd#0&~(s|BKONZ%!KTe6M1bT17lE8!=8qL|ec790-
zBDS=EAk0CUnQallzA~?{#1Q|v6=-ECLp#ZZ)%KAIJHOiQG2%vWLtV;XWe!2u4nSzY
zRFVVE0m#;tAy
zLE7EI5^<(oHlHV>AQ~u9z~s^rZo9w&2J&}A;7g?YfZJ|gGp}(&FF%FLagld$NJK~j*i#=+^smetL9IfyrRY$_
z@;H;wyg&V1!LT2Ha5=H0VZ))Vx`^dYr~$8+Fk5NOqM}$Fqyh7H@kT-YppLEOswpW`
zQ&LvJL81g^`nhmr0)H%mp9)}2g6jdMR
zrB7#TcF{#_&`%u*mGyZY~YoB@(;K=%|Mp%fkaI)P`yfZu;D1
z0Ixp^1*x2@D1P6H^NA)-;1&_=Vk=o?X*rKn2eQrhavOH+?;^OwI7wOdYzlBt+*5yW
zr#7oOTPW~TYmJTA8l!l;dbah?Xqz5q%btn(P}0KU}-Y|Ng?L8b{Zvn
z;uZuRM)kL$wpI$BoD5hevbzRTE#^G{OH9}Xm`nM=2*b8v?u#p;)|3Dc7B0_eb0fG@
zuW)X703qdbn#B1BOA`zXr(Owj1K1{(w(-#U)Yh3qdGHKi9Zs`eYd6EGZly3&Ysx`f
zvWnB~HNjR^@{HPxEeq@+tQn@hxB=LfJ&q~WgHhRrT6W{T!*1pCf(x}GmBdbv~{|@lGzZ_?@P*EngVjgC`aY#OqjAFyS_4|-5
zS;(;N(nHTuCV^(b`9ORUO}?g$YwS@?uSIYmr@%k|z$S-f_xBCE1)Fn&JvwL_wNM~-QU?z}jZkTh@o}L?U
zulsXJV+*x=06cWJwvqI)-QBdbr(Q3VU}=_s=Xo3L9VyDmBNMc@4~XFEo?4nM@!+bd
zu^ExQ9=hAQaVNdk0=D)}MwS2ysB5f^tE$V+^JJnzEG-t%k-y(hTbqroVT2LvR!QDF
zt=JdR-Di;Y*3I2#tsLF%Y{#17%kJ(r&Pn*7-qMWuVJWA8z_i%o2ZL!NuCE+j)qXMP
zh`AV>78u*;NzAB+>lZ#|$RoDTEUG@v$-~;soy^|!dQ+`s%?u9t=8EH21D(wCS2j-`
z>|!4@ij2k$f_X8VgJ0uL>ToYIbM2-y+d+k~VylT5HRToi#DFGphiii=g3YLltAkE4
zc_`CSQN9^CGLQ0Vy%>l@oLsX_vz5GDK~*JU?jwTT*3afJLOIGjNg~5Rg4?R_SaWJl
zu2^%HF^X$iJn$pan$SJW(L=kmOS|-tDZMQV3sx>jGh}4i=Qw$+e1mr3LFnv_=4p&n
zmx&3vnCEMH#$Jp31Io$ZyDw{UKK)RRia!0nnRDQ_6kjZ&DeYjcYW$>rX-BK$XF4_kFelanIUJ?J?hhmQu%V5
z;FBzOWzJy0<0ZLk8Q>e+P>NY>*`z$mt0ei3xftx*oP5hmpP@oJln@;If5s5Hn*AX2$JeHTh22yB(#C$0!emPw8Uf)rz~=BUW8jQelz~6NQ+%HcoOC
zw#dxY8`7Z~1%@iarYp&_t>2^GX|7)%;`;R=rsvc1Q=Y4k)*Hs>NdZrV9oPn4>Bn6Z
z=_C+6f-vn&uDiklSBLJVU5_kJDFKZEyuk;75UMF&K8)eB<4DTnDD$IVGrdjWOv>kn
zVp-~3+yu6c7TRo^
zX}_6|rM;b}a{!#2Z?w=h;En^j<|aC6tZQd-2|!D8dkgD|tqLtPN>yL;pR~|?E`EIr
z_HJ5g8~J2m4|`HAMNvyzTaEa4=Xn|}9mE$B&N19!p|Q(JPEJk+p|H|%M%>djYmzFm
z!bUpUo+bxay4{7v+(?IY7`xkzTbN+rTr0Ka7WyN4^l46Tlr(qHA~J^C?^&>P#sj>O
zxdA%rEOa@SfE=D^>7X&Oi>s-H4yjD_{zxlzjotB}Swz6rT1WFIqKmYrrk!?6{*F9P
zMrpBh(9s~eDV|RnFAXhdoh_+7F4&)O;^3buCzhNZj}A-OX5921U@epn>)_^FwPi
z7?`n(!KhMCdC5kD!%E6kVgTp5+RNC6PGSZayYMmBJW^t*$V8QP5n+0o=~O>6o^?U%
zhSodV$3Jvj#x^G4U0m;kMgs!2zHuo)+=p>Zncn>M-_h|wA0snfl6;lL*&%waOX0i9
zfHB)gPrVLlPSRcs#!UpS{+nFBG#c+@MFY&(E=q0bK%sw@gQZ(f#Hg$4sgMp``bWFD
zaNQNRN6~=5)ekQ-E+vFCRnJke$zx-=>FMK}gt)0#SG1Zq`FtF~8z8W_$l~H6
zi;K(i9v2xq{~Nk`hM0B-k|boqc*m>^I;ECUx;1UgqIa2gNi!zP^}8qQc4?Qs&y>!V
z&5v^7(_5mgQkgh*{J^>yY!6#!Z#1b{THk<4DJGd*eZQm2HjOulYL5my%#8GLzT9FH75hUmZ~cORmm%m=6RbKV*@`giD6!1!mUz
z&komWLO$e*|J#g^lqXxF*j@TBe8x9%7mPAL{TUZ*%X1!096G3p^N$3F_%}tV3ByCh
z2e;wNZH7L$yYvx#&d3fNJWMX%hjo&CJN`h|r6GiM)j?tA
zg>gRb8BM4#D(29UXVA!in2(_keoxPsCrQjC`q^W)%ltz}AG6ujh9pn`03ZNKL_t*8
z^R4*Lah-3zq;E!QkgPLP{%7cbZ!AX12A{SY6RXi`C4k0r6WfS@dw$KkvG@=xql*tv
zFB&Rho-es@X*?l*_A~wYMXuN-S{$d)?M~5_;e&5gjsY?(bLt9J)HhJ@**9@(&1gl)qcL2
z^%4jyt~_&i2gE^Jp_i&?eGFddVX-xfe0*Ld8V=x|9^%t~{2hI+<<0?>)F*W|L(GnTCCTzlTq#bdgk3jRr&UzRNFv`AeRE{`okt@CQHm0Y;;drluw?UcAVeGiP}0vB!A)
z@yGemkAAfBxd#mr?#>`g8b?JC`nx#Q-^Hn%J!lOEN(%E)5MjY>$L_o@%^7=w)92dO
zv}TMDnVY2dY!jcHg`5ICCPN|l3I&mcIRyJ{=B0A0f`(2y8}lF9BPwb;88|mbZA&kn
z`4PIB$~o7f#b78QUlC!!Eg*RAvqTPuF1K&);)YhN-4oO`_u`x%;l2OzBhG1`z)+G8
za{_kfyi|2oK+XF$I_r|{Lm|zb?R1;ZGP&fT|5Q1JZCVT^`9v1nI6e2p{qJr@Xg-s&
z0hx1
zx$o@dY*R01*M1~N&ui9m7&_`WyuD>qoXxTZnuH+126u)4Aq01~5Zo;c?(XiE;O?%$
zA-KD{TX1)G7<{;sZ|}3$KIhNRZmqvwJlMl8<2vkLHX^J3T~ed
zIrD*u;?y0v;M*1M%7TqYUVSN-DdD)@sW^KfV^*^RXw7Nf&UuG_d~G{^d1*2Y8LRq-
zMq|W2Q=H$}v%I%gCI0kYqiibv(bsTE>n%UOHR0BAx%?hXIe?}bmMq+1l1Z3*SXkTBfqPojb*e8n$R6|~g0~<5Iqki&Sw*5gJ5l8BS
z-0^}$-<$Df(k|}>OWEx!ZLrA^O}VreEHedlL};rfSG6jq-nk4>6&~oWnJRKMzh&6fsUUIVQrBJsm4@N+99eDqPhx4ay^ZcXqhJ
zFLd*K&)^)5+gT%R$Lu*|(Dw^Wk7DnH`4j?~}y(iYWXwlJz-mxN$Vto7BHp{1w
zV#=-EW}N|0?68rM4=3U-6*_Q04{2-!x9G4?A08-YN&EzN))W=A_W~u$Xs8J?^-hEo
zT)3WmCK%uoOs}wnp;l0Ka_=kRGxYLB!ZbXqU8N_&{P*Z&n|H@Sn0DSlm-p)qIkYWv
zU$q~JaB|~lBGH)E-zU8;(_v9#y9uizhd$5*oVn2+5#Gp9tpBjOe9*fW;*nJN2ILU}
zK83ap=jr){X!#rcej(z55K-7Wn;{vpr!l2*%;}s}4FI>Hw(~BO7tQnOw4sw){Qj)v
z{On%38(v#5Ij!ZkIh6!*(N&A`rPM0#auU_{ZOd8pzcOsMiBYfj|)7Z`+LLA0q&3*{n
z+Dp<0ICU8+nlh{cb~w1Td_jgf+P3pe@aZyi?_bOn7=G&Mq3Z18wAQaeWx241dF1Py
z<{ckGhZFm3H%>Fj(K^y;OIfGZWsX$L!}0S)XKlYuNG_LdK?s%WWQt#KY+`LU>(qH%
z?J6ZqTvyTCd)h<=PaUsvePcCOPCgn&>Mmmih_5FMy(xs4xp(jcBfXF+!L=u=S76mS
zF3^Gbxu!nf`lgRulqF7IjL@GmTN-&s-qs^WTykj(tct7Q+>FisF4mOTQz&PbXD^^WcEnrUyP>E7
z`SeJZ?`lttLw3+YGdz5?HWF`2qrEXU!_aryw!DcY2gb#xid%_SDs