terraform: use kubectl_manifest

The `kubernetes_manifest` resource in provider `hashicorp/kubernetes`
has a known issue[1] where resources created in a manifest can't depend
on other resources that don't exist yet. To work around this, we instead
use `gavinbunney/kubectl`'s `kubectl_manifest` resource, which does not
have this problem because it uses a different mechanism for planning.

[1] hashicorp/terraform-provider-kubernetes#1380

Resolves #1088

terraform/main.tf

@@ -323,6 +323,10 @@ terraform {
       source  = "hashicorp/tls"
       version = "~> 3.1.0"
     }
+    kubectl = {
+      source = "gavinbunney/kubectl"
+      version = "~> 1.13.1"
+    }
   }
 }
 
@@ -366,9 +370,6 @@ provider "kubernetes" {
   host                   = local.kubernetes_cluster.endpoint
   cluster_ca_certificate = base64decode(local.kubernetes_cluster.certificate_authority_data)
   token                  = local.kubernetes_cluster.token
-  experiments {
-    manifest_resource = true
-  }
 }
 
 provider "helm" {
@@ -379,6 +380,15 @@ provider "helm" {
   }
 }
 
+# We use provider kubectl to manage Kubernetes manifests, as a workaround for an
+# issue in hashicorp/kubernetes:
+# https://github.com/hashicorp/terraform-provider-kubernetes/issues/1380
+provider "kubectl" {
+  host                   = local.kubernetes_cluster.endpoint
+  cluster_ca_certificate = base64decode(local.kubernetes_cluster.certificate_authority_data)
+  token                  = local.kubernetes_cluster.token
+}
+
 module "manifest_gcp" {
   source                  = "./modules/manifest_gcp"
   count                   = var.use_aws ? 0 : 1

terraform/modules/kubernetes/kubernetes.tf

@@ -162,6 +162,20 @@ variable "gcp_workload_identity_pool_provider" {
   type = string
 }
 
+# We repeat this declaration from main.tf to work around an issue where
+# Terraform will look for hashicorp/kubectl instead of gavinbunney/kubectl
+# https://github.com/gavinbunney/terraform-provider-kubectl/issues/39
+terraform {
+  required_version = ">= 0.14.8"
+
+  required_providers {
+    kubectl = {
+      source = "gavinbunney/kubectl"
+      version = "~> 1.13.1"
+    }
+  }
+}
+
 locals {
   workflow_manager_iam_entity = "${var.environment}-${var.data_share_processor_name}-workflow-manager"
 }
@@ -422,9 +436,9 @@ resource "kubernetes_deployment" "intake_batch" {
   }
 }
 
-resource "kubernetes_manifest" "intake_queue_depth_metric" {
+resource "kubectl_manifest" "intake_queue_depth_metric" {
   count = var.use_aws ? 1 : 0
-  manifest = {
+  yaml_body = yamlencode({
     apiVersion = "metrics.aws/v1alpha1"
     kind       = "ExternalMetric"
     metadata = {
@@ -450,7 +464,7 @@ resource "kubernetes_manifest" "intake_queue_depth_metric" {
         }
       }]
     }
-  }
+  })
 }
 
 resource "kubernetes_horizontal_pod_autoscaler" "intake_batch_autoscaler" {
@@ -648,9 +662,9 @@ resource "kubernetes_deployment" "aggregate" {
   }
 }
 
-resource "kubernetes_manifest" "aggregate_queue_depth_metric" {
+resource "kubectl_manifest" "aggregate_queue_depth_metric" {
   count = var.use_aws ? 1 : 0
-  manifest = {
+  yaml_body = yamlencode({
     apiVersion = "metrics.aws/v1alpha1"
     kind       = "ExternalMetric"
     metadata = {
@@ -676,7 +690,7 @@ resource "kubernetes_manifest" "aggregate_queue_depth_metric" {
         }
       }]
     }
-  }
+  })
 }
 
 resource "kubernetes_horizontal_pod_autoscaler" "aggregate_autoscaler" {