From 9b25b7d7230197bda6522733db05ffef0a7c7e92 Mon Sep 17 00:00:00 2001 From: Callum May Date: Fri, 18 Oct 2024 13:02:54 -0400 Subject: [PATCH 1/5] Add support for CRDB multi-region deployments in tanka config --- deploy/services/tanka/cockroachdb.libsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/services/tanka/cockroachdb.libsonnet b/deploy/services/tanka/cockroachdb.libsonnet index e76bcc4a9..b740685e4 100644 --- a/deploy/services/tanka/cockroachdb.libsonnet +++ b/deploy/services/tanka/cockroachdb.libsonnet @@ -87,7 +87,7 @@ local volumes = import 'volumes.libsonnet'; join: std.join(',', ['cockroachdb-0.cockroachdb'] + if metadata.single_cluster then [] else metadata.cockroach.JoinExisting), logtostderr: true, - locality: 'zone=' + metadata.cockroach.locality, + locality: if std.objectHas(metadata.cockroach, 'region') then std.join(',', ['region=' + metadata.cockroach.region, 'zone=' + metadata.cockroach.locality]) else 'zone=' + metadata.cockroach.locality, 'locality-advertise-addr': 'zone=' + metadata.cockroach.locality + '@$(hostname -f)', 'http-addr': '0.0.0.0', cache: '25%', From 262dc6ddbe69605a186871f9c35c339e90bd5d8a Mon Sep 17 00:00:00 2001 From: Callum May Date: Fri, 18 Oct 2024 13:32:25 -0400 Subject: [PATCH 2/5] add region support to terraform config and populate examples --- build/README.md | 6 ++++++ .../dependencies/terraform-commons-dss/helm.tf | 5 +++-- .../infrastructure/modules/terraform-aws-dss/main.tf | 1 + .../terraform-aws-dss/terraform.dev.example.tfvars | 1 + .../modules/terraform-google-dss/main.tf | 1 + .../terraform.dev.example.tfvars | 1 + .../infrastructure/utils/definitions/crdb_region.tf | 12 ++++++++++++ deploy/infrastructure/utils/variables.py | 1 + deploy/operations/ci/aws-1/main.tf | 1 + deploy/operations/ci/aws-1/terraform.tfvars | 1 + 10 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 deploy/infrastructure/utils/definitions/crdb_region.tf diff --git a/build/README.md b/build/README.md index e45640fbc..23a7e550b 100644 --- a/build/README.md +++ b/build/README.md @@ -302,6 +302,12 @@ a PR to that effect would be greatly appreciated. recommend "_", and the `=` character is not allowed. However, any unique (among all other participating DSS instances) value is acceptable. + + 1. `VAR_CRDB_REGION`:Region of your DSS instance. Regions are a high-level abstraction + of a geographic region, and are meant to correspond directly to the region terminology + used by cloud providers. Each region is broken into multiple zones. Regions are used + to achieve varying survival goals in the face of database failure. More info at + https://www.cockroachlabs.com/docs/stable/multiregion-overview. 1. `VAR_CRDB_NODE_IPn`: IP address (**numeric**) of nth CRDB node (add more entries if you have more than 3 CRDB nodes). Example: `1.1.1.1` diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf index fc5b73ec0..65500d9d9 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf @@ -2,6 +2,7 @@ locals { # Tanka defines itself the variable below. For helm, since we are using the official helm CRDB chart, # the following variable has to be provided here. helm_crdb_statefulset_name = "dss-cockroachdb" + locality_args = var.crdb_region != "" ? "region=${var.crdb_region},zone=${var.crdb_locality}" : "zone=${var.crdb_locality}" } resource "local_file" "helm_chart_values" { @@ -17,13 +18,13 @@ resource "local_file" "helm_chart_values" { join = var.crdb_external_nodes cluster-name = var.crdb_cluster_name single-node = false # Always false. Even with 1 replica, we would expect to keep the ability to pool it with another cluster. - locality = "zone=${var.crdb_locality}" + locality = locals.locality_args } statefulset = { replicas = length(var.crdb_internal_nodes) args = [ - "--locality-advertise-addr=zone=${var.crdb_locality}@$(hostname -f)", + "--locality-advertise-addr=${locals.locality_args}@$(hostname -f)", "--advertise-addr=$${HOSTNAME##*-}.${var.crdb_hostname_suffix}" ] } diff --git a/deploy/infrastructure/modules/terraform-aws-dss/main.tf b/deploy/infrastructure/modules/terraform-aws-dss/main.tf index effdfae82..3fee48ecd 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/main.tf +++ b/deploy/infrastructure/modules/terraform-aws-dss/main.tf @@ -26,6 +26,7 @@ module "terraform-commons-dss" { should_init = var.should_init authorization = var.authorization crdb_locality = var.crdb_locality + crdb_region = var.crdb_region crdb_external_nodes = var.crdb_external_nodes crdb_internal_nodes = module.terraform-aws-kubernetes.crdb_nodes ip_gateway = module.terraform-aws-kubernetes.ip_gateway diff --git a/deploy/infrastructure/modules/terraform-aws-dss/terraform.dev.example.tfvars b/deploy/infrastructure/modules/terraform-aws-dss/terraform.dev.example.tfvars index d9cc764a3..ba6d0af1f 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/terraform.dev.example.tfvars +++ b/deploy/infrastructure/modules/terraform-aws-dss/terraform.dev.example.tfvars @@ -29,4 +29,5 @@ should_init = true crdb_image_tag = "v24.1.3" crdb_cluster_name = "interuss_example" crdb_locality = "interuss_dss-aws-ew1" +crdb_region = "eu-west-1" crdb_external_nodes = [] diff --git a/deploy/infrastructure/modules/terraform-google-dss/main.tf b/deploy/infrastructure/modules/terraform-google-dss/main.tf index ea98a4dd6..9f814457d 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/main.tf +++ b/deploy/infrastructure/modules/terraform-google-dss/main.tf @@ -25,6 +25,7 @@ module "terraform-commons-dss" { should_init = var.should_init authorization = var.authorization crdb_locality = var.crdb_locality + crdb_region = var.crdb_region image_pull_secret = var.image_pull_secret crdb_external_nodes = var.crdb_external_nodes kubernetes_api_endpoint = module.terraform-google-kubernetes.kubernetes_api_endpoint diff --git a/deploy/infrastructure/modules/terraform-google-dss/terraform.dev.example.tfvars b/deploy/infrastructure/modules/terraform-google-dss/terraform.dev.example.tfvars index 70e21eff2..0129a1112 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/terraform.dev.example.tfvars +++ b/deploy/infrastructure/modules/terraform-google-dss/terraform.dev.example.tfvars @@ -30,4 +30,5 @@ should_init = true crdb_image_tag = "v24.1.3" crdb_cluster_name = "interuss_example" crdb_locality = "interuss_dss-dev-w6a" +crdb_region = "europe-west6-a" crdb_external_nodes = [] diff --git a/deploy/infrastructure/utils/definitions/crdb_region.tf b/deploy/infrastructure/utils/definitions/crdb_region.tf new file mode 100644 index 000000000..16ac2fc08 --- /dev/null +++ b/deploy/infrastructure/utils/definitions/crdb_region.tf @@ -0,0 +1,12 @@ +variable "crdb_region" { + type = string + description = <<-EOT + Region of your DSS instance. Regions are a high-level abstraction of a geographic region, + and are meant to correspond directly to the region terminology used by cloud providers. + Each region is broken into multiple zones. Regions are used to achieve varying survival + goals in the face of database failure. More info at + https://www.cockroachlabs.com/docs/stable/multiregion-overview. + + Example: + EOT +} \ No newline at end of file diff --git a/deploy/infrastructure/utils/variables.py b/deploy/infrastructure/utils/variables.py index afc5f3d6d..d2a666120 100755 --- a/deploy/infrastructure/utils/variables.py +++ b/deploy/infrastructure/utils/variables.py @@ -36,6 +36,7 @@ "crdb_image_tag", "crdb_cluster_name", "crdb_locality", + "crdb_region", "crdb_external_nodes", "kubernetes_namespace", ] diff --git a/deploy/operations/ci/aws-1/main.tf b/deploy/operations/ci/aws-1/main.tf index 998b71eb9..f5b020775 100644 --- a/deploy/operations/ci/aws-1/main.tf +++ b/deploy/operations/ci/aws-1/main.tf @@ -21,6 +21,7 @@ module "terraform-aws-dss" { crdb_cluster_name = var.crdb_cluster_name crdb_hostname_suffix = var.crdb_hostname_suffix crdb_locality = var.crdb_locality + crdb_region = var.crdb_region crdb_external_nodes = var.crdb_external_nodes image = var.image kubernetes_version = var.kubernetes_version diff --git a/deploy/operations/ci/aws-1/terraform.tfvars b/deploy/operations/ci/aws-1/terraform.tfvars index 67b1ed2cb..b0a4222fa 100644 --- a/deploy/operations/ci/aws-1/terraform.tfvars +++ b/deploy/operations/ci/aws-1/terraform.tfvars @@ -26,6 +26,7 @@ should_init = true crdb_image_tag = "v24.1.3" crdb_cluster_name = "interuss-ci" crdb_locality = "interuss_dss-ci-aws-ue1" +crdb_region = "us-east-1" crdb_external_nodes = [] aws_iam_permissions_boundary = "arn:aws:iam::301042233698:policy/GithubCIPermissionBoundaries20231130225039606500000001" From f3dd016c63a4e93324f21ecaab82f10346338740 Mon Sep 17 00:00:00 2001 From: Callum May Date: Fri, 18 Oct 2024 13:34:20 -0400 Subject: [PATCH 3/5] add missing var --- .../infrastructure/dependencies/terraform-commons-dss/tanka.tf | 1 + .../terraform-commons-dss/templates/main.jsonnet.tmp | 1 + 2 files changed, 2 insertions(+) diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/tanka.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/tanka.tf index b02801730..684969969 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/tanka.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/tanka.tf @@ -6,6 +6,7 @@ resource "local_file" "tanka_config_main" { VAR_ENABLE_SCD = var.enable_scd VAR_CRDB_HOSTNAME_SUFFIX = var.crdb_hostname_suffix VAR_CRDB_LOCALITY = var.crdb_locality + VAR_CRDB_REGION = var.crdb_region VAR_CRDB_NODE_IPS = join(",", [for i in var.crdb_internal_nodes[*].ip : "'${i}'"]) VAR_INGRESS_NAME = var.ip_gateway VAR_CRDB_EXTERNAL_NODES = join(",", [for a in var.crdb_external_nodes : "'${a}'"]) diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/templates/main.jsonnet.tmp b/deploy/infrastructure/dependencies/terraform-commons-dss/templates/main.jsonnet.tmp index 85c093ed6..695d52aea 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/templates/main.jsonnet.tmp +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/templates/main.jsonnet.tmp @@ -15,6 +15,7 @@ local metadata = metadataBase { image: '${VAR_CRDB_DOCKER_IMAGE_NAME}', hostnameSuffix: '${VAR_CRDB_HOSTNAME_SUFFIX}', locality: '${VAR_CRDB_LOCALITY}', + region: '${VAR_CRDB_REGION}', nodeIPs: [${VAR_CRDB_NODE_IPS}], shouldInit: ${VAR_SHOULD_INIT}, JoinExisting: [${VAR_CRDB_EXTERNAL_NODES}], From cff6b0968031c3d534e66ab955f7601cb1c43b8c Mon Sep 17 00:00:00 2001 From: Callum May Date: Fri, 18 Oct 2024 13:36:13 -0400 Subject: [PATCH 4/5] add default to crdb_region tf variable --- deploy/infrastructure/utils/definitions/crdb_region.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deploy/infrastructure/utils/definitions/crdb_region.tf b/deploy/infrastructure/utils/definitions/crdb_region.tf index 16ac2fc08..9e94ef87e 100644 --- a/deploy/infrastructure/utils/definitions/crdb_region.tf +++ b/deploy/infrastructure/utils/definitions/crdb_region.tf @@ -9,4 +9,6 @@ variable "crdb_region" { Example: EOT + + default = "" } \ No newline at end of file From 445420f17c2f8b3c9b35a44bb509679342ad350e Mon Sep 17 00:00:00 2001 From: Callum May Date: Fri, 18 Oct 2024 13:36:55 -0400 Subject: [PATCH 5/5] add new line --- deploy/infrastructure/utils/definitions/crdb_region.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/infrastructure/utils/definitions/crdb_region.tf b/deploy/infrastructure/utils/definitions/crdb_region.tf index 9e94ef87e..f3143f70a 100644 --- a/deploy/infrastructure/utils/definitions/crdb_region.tf +++ b/deploy/infrastructure/utils/definitions/crdb_region.tf @@ -11,4 +11,4 @@ variable "crdb_region" { EOT default = "" -} \ No newline at end of file +}