terraform k8s separation (aptos-labs#5755)

* [tf/aws] option to separate k8s workloads from TF state

* [tf/gcp] option to separate k8s workloads from TF state

terraform/aptos-node-testnet/addons.tf

@@ -226,13 +226,27 @@ resource "helm_release" "external-dns" {
   ]
 }
 
+locals {
+  # these values are the most likely to be changed by the user and may be managed by terraform to trigger re-deployment
+  testnet_addons_helm_values_managed = {
+    "imageTag"    = var.image_tag
+    "genesis.era" = var.era
+  }
+}
+
 resource "helm_release" "testnet-addons" {
   count       = var.enable_forge ? 0 : 1
   name        = "testnet-addons"
   chart       = local.testnet_addons_helm_chart_path
   max_history = 5
   wait        = false
 
+  lifecycle {
+    ignore_changes = [
+      values,
+    ]
+  }
+
   values = [
     jsonencode({
       imageTag = var.image_tag
@@ -260,6 +274,14 @@ resource "helm_release" "testnet-addons" {
     jsonencode(var.testnet_addons_helm_values)
   ]
 
+  dynamic "set" {
+    for_each = var.manage_via_tf ? local.testnet_addons_helm_values_managed : {}
+    content {
+      name  = set.key
+      value = set.value
+    }
+  }
+
   # inspired by https://stackoverflow.com/a/66501021 to trigger redeployment whenever any of the charts file contents change.
   set {
     name  = "chart_sha1"

terraform/aptos-node-testnet/main.tf

@@ -19,6 +19,8 @@ locals {
 module "validator" {
   source = "../aptos-node/aws"
 
+  manage_via_tf = var.manage_via_tf
+
   maximize_single_az_capacity = var.maximize_single_az_capacity
 
   region   = var.region
@@ -92,14 +94,27 @@ provider "kubernetes" {
 
 locals {
   genesis_helm_chart_path = "${path.module}/../helm/genesis"
+
+  # these values are the most likely to be changed by the user and may be managed by terraform to trigger re-deployment
+  genesis_helm_values_managed = {
+    "imageTag"  = var.image_tag
+    "chain.era" = var.era
+  }
 }
 
+
 resource "helm_release" "genesis" {
   name        = "genesis"
   chart       = local.genesis_helm_chart_path
   max_history = 5
   wait        = false
 
+  lifecycle {
+    ignore_changes = [
+      values,
+    ]
+  }
+
   values = [
     jsonencode({
       chain = {
@@ -124,6 +139,15 @@ resource "helm_release" "genesis" {
     }),
     jsonencode(var.genesis_helm_values)
   ]
+
+  dynamic "set" {
+    for_each = var.manage_via_tf ? local.genesis_helm_values_managed : {}
+    content {
+      name  = set.key
+      value = set.value
+    }
+  }
+
   # inspired by https://stackoverflow.com/a/66501021 to trigger redeployment whenever any of the charts file contents change.
   set {
     name  = "chart_sha1"

terraform/aptos-node-testnet/variables.tf

@@ -207,3 +207,8 @@ variable "fullnode_storage_class" {
     error_message = "Supported storage classes are gp3, io1, io2"
   }
 }
+
+variable "manage_via_tf" {
+  description = "Whether to manage the aptos-node k8s workload via Terraform"
+  default     = true
+}

terraform/aptos-node/aws/kubernetes.tf

@@ -169,6 +169,12 @@ locals {
 
   # override the helm release name if an override exists, otherwise adopt the workspace name
   helm_release_name = var.helm_release_name_override != "" ? var.helm_release_name_override : local.workspace_name
+
+  # these values are the most likely to be changed by the user and may be managed by terraform to trigger re-deployment
+  helm_values_managed = {
+    "imageTag"  = var.image_tag
+    "chain.era" = var.era
+  }
 }
 
 resource "helm_release" "validator" {
@@ -178,12 +184,26 @@ resource "helm_release" "validator" {
   max_history = 5
   wait        = false
 
+  lifecycle {
+    ignore_changes = [
+      values,
+    ]
+  }
+
   values = [
     local.helm_values,
     var.helm_values_file != "" ? file(var.helm_values_file) : "{}",
     jsonencode(var.helm_values),
   ]
 
+  dynamic "set" {
+    for_each = var.manage_via_tf ? local.helm_values_managed : {}
+    content {
+      name  = set.key
+      value = set.value
+    }
+  }
+
   # inspired by https://stackoverflow.com/a/66501021 to trigger redeployment whenever any of the charts file contents change.
   set {
     name  = "chart_sha1"

terraform/aptos-node/aws/variables.tf

@@ -265,3 +265,8 @@ variable "fullnode_storage_class" {
     error_message = "Supported storage classes are gp3, io1, io2"
   }
 }
+
+variable "manage_via_tf" {
+  description = "Whether to manage the aptos-node k8s workload via Terraform"
+  default     = true
+}

terraform/aptos-node/gcp/kubernetes.tf

@@ -28,6 +28,12 @@ locals {
   monitoring_helm_chart_path = "${path.module}/../../helm/monitoring"
   logger_helm_chart_path     = "${path.module}/../../helm/logger"
   aptos_node_helm_chart_path = var.helm_chart != "" ? var.helm_chart : "${path.module}/../../helm/aptos-node"
+
+  # these values are the most likely to be changed by the user and may be managed by terraform to trigger re-deployment
+  helm_values_managed = {
+    "imageTag"  = var.image_tag
+    "chain.era" = var.era
+  }
 }
 
 resource "helm_release" "validator" {
@@ -36,6 +42,12 @@ resource "helm_release" "validator" {
   max_history = 5
   wait        = false
 
+  lifecycle {
+    ignore_changes = [
+      values,
+    ]
+  }
+
   values = [
     jsonencode({
       imageTag = var.image_tag
@@ -76,6 +88,14 @@ resource "helm_release" "validator" {
     jsonencode(var.helm_values),
   ]
 
+  dynamic "set" {
+    for_each = var.manage_via_tf ? local.helm_values_managed : {}
+    content {
+      name  = set.key
+      value = set.value
+    }
+  }
+
   # inspired by https://stackoverflow.com/a/66501021 to trigger redeployment whenever any of the charts file contents change.
   set {
     name  = "chart_sha1"

terraform/aptos-node/gcp/variables.tf

@@ -142,3 +142,8 @@ variable "node_exporter_helm_values" {
   type        = any
   default     = {}
 }
+
+variable "manage_via_tf" {
+  description = "Whether to manage the aptos-node k8s workload via Terraform"
+  default     = true
+}