Add support for CSE 4.2.1

.changes/v3.12.0/1195-features.md

@@ -1,5 +1,5 @@
 * **New Resource:** `vcd_cse_kubernetes_cluster` to create and manage Kubernetes clusters in a VCD with Container Service Extension
-  4.2.1, 4.2.0, 4.1.1 or 4.1.0 installed and running [GH-1195, GH-1218]
+  4.2.1, 4.2.0, 4.1.1 or 4.1.0 installed and running [GH-1195, GH-1218, GH-1222]
 * **New Data Source:** `vcd_cse_kubernetes_cluster` to read Kubernetes clusters from a VCD with Container Service Extension
-  4.2.1, 4.2.0, 4.1.1 or 4.1.0 installed and running [GH-1195, GH-1218]
+  4.2.1, 4.2.0, 4.1.1 or 4.1.0 installed and running [GH-1195, GH-1218, GH-1222]
 * **New Data Source:** `vcd_version` to get the VCD version and perform additional checks with version constraints [GH-1195, GH-1218]

examples/container-service-extension/v4.2/install/step1/3.11-cse-install-1-provider-config.tf

@@ -44,3 +44,8 @@ data "vcd_version" "gte_1051" {
   condition         = ">= 10.5.1"
   fail_if_not_match = false
 }
+
+# There are some differences between CSE 4.2.0 and 4.2.1, like in the Kubernetes Cluster Rights Bundle and the Kubernetes Cluster Author rights.
+locals {
+  is_cse_420 = var.capvcd_version == "1.2.0" && var.cpi_version == "1.5.0" && var.csi_version == "1.5.0"
+}

examples/container-service-extension/v4.2/install/step1/3.11-cse-install-2-cse-server-prerequisites.tf

@@ -94,7 +94,7 @@ resource "vcd_role" "cse_admin_role" {
   org         = var.administrator_org
   name        = "CSE Admin Role"
   description = "Used for administrative purposes"
-  rights = concat([
+  rights = concat(concat([
     "API Tokens: Manage",
     "${vcd_rde_type.vcdkeconfig_type.vendor}:${vcd_rde_type.vcdkeconfig_type.nss}: Administrator Full access",
     "${vcd_rde_type.vcdkeconfig_type.vendor}:${vcd_rde_type.vcdkeconfig_type.nss}: Administrator View",
@@ -106,7 +106,9 @@ resource "vcd_role" "cse_admin_role" {
     "${vcd_rde_type.capvcdcluster_type.vendor}:${vcd_rde_type.capvcdcluster_type.nss}: Full Access",
     "${vcd_rde_type.capvcdcluster_type.vendor}:${vcd_rde_type.capvcdcluster_type.nss}: Modify",
     "${vcd_rde_type.capvcdcluster_type.vendor}:${vcd_rde_type.capvcdcluster_type.nss}: View"
-  ], data.vcd_version.gte_1051.matches_condition ? ["Organization: Traversal"] : [])
+    ], data.vcd_version.gte_1051.matches_condition ? ["Organization: Traversal"] : []),
+    # CSE 4.2.1 requires a few extra rights for IP Spaces
+  local.is_cse_420 ? [] : ["IP Spaces: Allocate", "Private IP Spaces: View", "Private IP Spaces: Manage"])
 }
 
 # This will allow to have a user with a limited set of rights that can access the Provider area of VCD.
@@ -124,7 +126,7 @@ resource "vcd_org_user" "cse_admin" {
 resource "vcd_rights_bundle" "k8s_clusters_rights_bundle" {
   name        = "Kubernetes Clusters Rights Bundle"
   description = "Rights bundle with required rights for managing Kubernetes clusters"
-  rights = [
+  rights = concat([
     "API Tokens: Manage",
     "Access All Organization VDCs",
     "Catalog: View Published Catalogs",
@@ -152,7 +154,8 @@ resource "vcd_rights_bundle" "k8s_clusters_rights_bundle" {
     "vmware:tkgcluster: View",
     "vmware:tkgcluster: Administrator View",
     "vmware:tkgcluster: Administrator Full access",
-  ]
+    # CSE 4.2.1 requires a few extra rights for IP Spaces
+  ], local.is_cse_420 ? [] : ["IP Spaces: Allocate", "Private IP Spaces: View", "Private IP Spaces: Manage"])
   publish_to_all_tenants = true # This needs to be published to all the Organizations
 }
 
@@ -163,7 +166,7 @@ resource "vcd_rights_bundle" "k8s_clusters_rights_bundle" {
 resource "vcd_global_role" "k8s_cluster_author" {
   name        = "Kubernetes Cluster Author"
   description = "Role to create Kubernetes clusters"
-  rights = [
+  rights = concat([
     "API Tokens: Manage",
     "Access All Organization VDCs",
     "Catalog: Add vApp from My Cloud",
@@ -223,7 +226,8 @@ resource "vcd_global_role" "k8s_cluster_author" {
     "vmware:tkgcluster: Full Access",
     "vmware:tkgcluster: Modify",
     "vmware:tkgcluster: View",
-  ]
+    # CSE 4.2.1 requires a few extra rights for IP Spaces
+  ], local.is_cse_420 ? [] : ["IP Spaces: Allocate", "Private IP Spaces: View", "Private IP Spaces: Manage"])
 
   publish_to_all_tenants = true # This needs to be published to all the Organizations
 

examples/container-service-extension/v4.2/install/step1/terraform.tfvars.example → examples/container-service-extension/v4.2/install/step1/terraform.tfvars.example.4.2.0

@@ -28,11 +28,11 @@ cse_admin_password = "change-me"
 
 # These are required to create the Runtime Defined Entity that will contain the CSE Server configuration (vcdKeConfig)
 # To know more about the specific versions, please refer to the CSE documentation.
-# The values set here correspond to CSE 4.2:
+# The values set here correspond to CSE 4.2.0:
 vcdkeconfig_template_filepath = "../../entities/vcdkeconfig.json.template"
-capvcd_version                = "1.2.0"
-cpi_version                   = "1.5.0"
-csi_version                   = "1.5.0"
+capvcd_version                = "1.2.0" # 4.2.0
+cpi_version                   = "1.5.0" # 4.2.0
+csi_version                   = "1.5.0" # 4.2.0
 rde_projector_version         = "0.7.0"
 
 # Optional but recommended to avoid rate limiting when configuring the TKGm clusters.

examples/container-service-extension/v4.2/install/step1/terraform.tfvars.example.4.2.1

@@ -0,0 +1,61 @@
+# Change configuration to your needs and rename to 'terraform.tfvars'
+# For more details about the variables specified here, please read the guide first:
+# https://registry.terraform.io/providers/vmware/vcd/latest/docs/guides/container_service_extension_4_x_install
+
+# ------------------------------------------------
+# VCD Provider config
+# ------------------------------------------------
+
+vcd_url                = "https://vcd.my-awesome-corp.com"
+administrator_user     = "administrator"
+administrator_password = "change-me"
+administrator_org      = "System"
+insecure_login         = "false"
+
+# ------------------------------------------------
+# CSE Server Pre-requisites
+# ------------------------------------------------
+
+# This user will be created by the Terraform configuration, so you can
+# customise what its username and password will be.
+# This user will have an API token that must be consumed by the CSE Server.
+cse_admin_username = "cse_admin"
+cse_admin_password = "change-me"
+
+# ------------------------------------------------
+# CSE Server Settings
+# ------------------------------------------------
+
+# These are required to create the Runtime Defined Entity that will contain the CSE Server configuration (vcdKeConfig)
+# To know more about the specific versions, please refer to the CSE documentation.
+# The values set here correspond to CSE 4.2.1:
+vcdkeconfig_template_filepath = "../../entities/vcdkeconfig.json.template"
+capvcd_version                = "1.3.0" # 4.2.1
+cpi_version                   = "1.6.0" # 4.2.1
+csi_version                   = "1.6.0" # 4.2.1
+rde_projector_version         = "0.7.0"
+
+# Optional but recommended to avoid rate limiting when configuring the TKGm clusters.
+# Create this one in https://github.com/settings/tokens
+github_personal_access_token = ""
+
+# Node will be considered unhealthy and remediated if joining the cluster takes longer than this timeout (seconds)
+node_startup_timeout = "900"
+# A newly joined node will be considered unhealthy and remediated if it cannot host workloads for longer than this timeout (seconds)
+node_not_ready_timeout = "300"
+# A healthy node will be considered unhealthy and remediated if it is unreachable for longer than this timeout (seconds)
+node_unknown_timeout = "300"
+# Remediation will be suspended when the number of unhealthy nodes exceeds this percentage.
+# (100% means that unhealthy nodes will always be remediated, while 0% means that unhealthy nodes will never be remediated)
+max_unhealthy_node_percentage = 100
+
+# URL from where TKG clusters will fetch container images
+container_registry_url = "projects.registry.vmware.com"
+
+# Certificate(s) to allow the ephemeral VM (created during cluster creation) to authenticate with.
+# For example, when pulling images from a container registry. (Copy and paste .cert file contents)
+k8s_cluster_certificates = []
+
+# Certificate(s) to allow clusters to authenticate with.
+# For example, when pulling images from a container registry. (Copy and paste .cert file contents)
+bootstrap_vm_certificates = []

examples/container-service-extension/v4.2/install/step1/variables.tf

@@ -59,19 +59,16 @@ variable "vcdkeconfig_template_filepath" {
 variable "capvcd_version" {
   type        = string
   description = "Version of CAPVCD"
-  default     = "1.2.0"
 }
 
 variable "cpi_version" {
   type        = string
   description = "VCDKEConfig: Cloud Provider Interface version"
-  default     = "1.5.0"
 }
 
 variable "csi_version" {
   type        = string
   description = "VCDKEConfig: Container Storage Interface version"
-  default     = "1.5.0"
 }
 
 variable "rde_projector_version" {

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/hashicorp/go-version v1.6.0
 	github.com/hashicorp/terraform-plugin-sdk/v2 v2.31.0
 	github.com/kr/pretty v0.2.1
-	github.com/vmware/go-vcloud-director/v2 v2.23.0-alpha.5
+	github.com/vmware/go-vcloud-director/v2 v2.23.0-alpha.6
 )
 
 require (

go.sum

@@ -142,8 +142,8 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU
 github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
 github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
 github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
-github.com/vmware/go-vcloud-director/v2 v2.23.0-alpha.5 h1:HL3T2EGsR8zaTbBbAPIzqQON7x9GQ+USKuz88I1N3cY=
-github.com/vmware/go-vcloud-director/v2 v2.23.0-alpha.5/go.mod h1:NyNcb2ymhrzwv4GyYXyYOm1NbqRwGNxDWn90AtWniXc=
+github.com/vmware/go-vcloud-director/v2 v2.23.0-alpha.6 h1:H3wZPjF62Q1W5rdJVWcOCrFQ2PUvulj7lwdTEIwWQaY=
+github.com/vmware/go-vcloud-director/v2 v2.23.0-alpha.6/go.mod h1:NyNcb2ymhrzwv4GyYXyYOm1NbqRwGNxDWn90AtWniXc=
 github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
 github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=

vcd/resource_vcd_cse_kubernetes_cluster_test.go

@@ -122,22 +122,26 @@ func TestAccVcdCseKubernetesCluster(t *testing.T) {
 		t.Skip(acceptanceTestsSkipped)
 		return
 	}
+	vcdClient := createSystemTemporaryVCDConnection()
 	cacheId := testCachedFieldValue{}
 	clusterName := "vcd_cse_kubernetes_cluster.my_cluster"
 	dataWithName := "data.vcd_cse_kubernetes_cluster.with_name_ds"
 	dataWithId := "data.vcd_cse_kubernetes_cluster.with_id_ds"
 	resource.Test(t, resource.TestCase{
 		ProviderFactories: testAccProviders,
 		CheckDestroy: func(state *terraform.State) error {
-			if cacheId.fieldValue == "" {
-				return fmt.Errorf("cached ID '%s' is empty", cacheId.fieldValue)
+			org, err := vcdClient.GetOrgByName(testConfig.Cse.TenantOrg)
+			if err != nil {
+				return fmt.Errorf("could not check cluster deletion: %s", err)
+			}
+			clusters, err := org.CseGetKubernetesClustersByName(*cseVersion, strings.ToLower(t.Name()))
+			if err != nil && !govcd.ContainsNotFound(err) {
+				return fmt.Errorf("could not check cluster deletion: %s", err)
 			}
-			conn := testAccProvider.Meta().(*VCDClient)
-			_, err := conn.GetRdeById(cacheId.fieldValue)
-			if err == nil {
-				return fmt.Errorf("cluster with ID '%s' still exists", cacheId.fieldValue)
+			if len(clusters) == 0 || govcd.ContainsNotFound(err) {
+				return nil
 			}
-			return nil
+			return fmt.Errorf("there are still %d clusters with name '%s': %s", len(clusters), clusterName, err)
 		},
 		Steps: []resource.TestStep{
 			// Basic scenario of cluster creation

website/docs/guides/container_service_extension_4_x_install.html.markdown

@@ -79,12 +79,11 @@ To customise it, the [step 1 configuration][step1] asks for the following variab
   [the RDE template file for CSE 4.2](https://github.com/vmware/terraform-provider-vcd/tree/main/examples/container-service-extension/v4.2/entities/vcdkeconfig.json.template)
   used in the step 1 configuration, that can be rendered correctly with the Terraform built-in function `templatefile`.
   (Note: In `terraform.tfvars.example` the path for the CSE 4.2 RDE contents is already provided).
-* `capvcd_version`: The version for CAPVCD. The default value is **"1.1.0"** for CSE 4.2.
-  (Note: Do not confuse with the version of the `capvcdCluster` [RDE Type][rde_type],
-  which **must be "1.3.0"** for CSE 4.2 and cannot be changed through a variable).
-* `cpi_version`: The version for CPI (Cloud Provider Interface). The default value is **"1.5.0"** for CSE 4.2.
-* `csi_version`: The version for CSI (Cloud Storage Interface). The default value is **"1.5.0"** for CSE 4.2.
-* `rde_projector_version`: The version for the RDE Projector. The default value is **"0.7.0"** for CSE 4.2.
+* `capvcd_version`: The version for CAPVCD. Must be **"1.2.0"** for CSE 4.2.0, or **"1.3.0"** for CSE 4.2.1.
+  (Note: Do not confuse with the version of the `capvcdCluster` [RDE Type][rde_type], which **must be "1.3.0"** for CSE 4.2.X, and cannot be changed through a variable).
+* `cpi_version`: The version for CPI (Cloud Provider Interface). Must be **"1.5.0"** for CSE 4.2.0, or **"1.6.0"** for CSE 4.2.1.
+* `csi_version`: The version for CSI (Cloud Storage Interface). Must be **"1.5.0"** for CSE 4.2.0, or **"1.6.0"** for CSE 4.2.1.
+* `rde_projector_version`: The version for the RDE Projector. The default value is **"0.7.0"** for CSE 4.2.X.
 * `github_personal_access_token`: Create this one [here](https://github.com/settings/tokens),
   this will avoid installation errors caused by GitHub rate limiting, as the TKGm cluster creation process requires downloading
   some Kubernetes components from GitHub.
@@ -101,9 +100,17 @@ To customise it, the [step 1 configuration][step1] asks for the following variab
   (100% means that unhealthy nodes will always be remediated, while 0% means that unhealthy nodes will never be remediated). Defaults to 100 in the step 1 configuration.
 * `container_registry_url`: URL from where TKG clusters will fetch container images, useful for VCD appliances that are completely isolated from Internet. Defaults to "projects.registry.vmware.com" in the step 1 configuration.
 * `bootstrap_vm_certificates`: Certificate(s) to allow the ephemeral VM (created during cluster creation) to authenticate with.
-  For example, when pulling images from a container registry. Optional in the step 1 configuration.
+  For instance, when pulling images from a container registry. Optional in the step 1 configuration.
 * `k8s_cluster_certificates`: Certificate(s) to allow clusters to authenticate with.
-  For example, when pulling images from a container registry. Optional in the step 1 configuration.
+  For instance, when pulling images from a container registry. Optional in the step 1 configuration.
+  Here is an example value for the certificates in the configuration:
+  ```hcl
+  k8s_cluster_certificates = [
+    "\"-----BEGIN CERTIFICATE-----\\nABCDEDCCFkgAAhiJkgLUm+1234567aBcdEfghiJKLMNAVZyABCDEFoZIhvcNAPQR\\nzYCDEDCCFkgAAhiJkgLUm+1234567aBcdEfghiJKLMNAVZyABCDEFoZIhvcNAPUs\\n...\\n-----END CERTIFICATE-----\"",
+    "\"-----BEGIN CERTIFICATE-----\\n...\\n-----END CERTIFICATE-----\"",
+    # ... more certificates
+  ]
+  ```
 
 #### Rights, Roles and VM Sizing Policies
 
@@ -338,9 +345,9 @@ The most common issues are:
 * Cluster creation is failing:
   * Please visit the [CSE documentation][cse_docs] to learn how to monitor the logs and troubleshoot possible problems.
 
-## Upgrade from CSE 4.1 to 4.2
+## Upgrade from CSE 4.1 to 4.2.0
 
-In this section you can find the required steps to update from CSE 4.1 to 4.2.
+In this section you can find the required steps to update from CSE 4.1 to 4.2.0.
 
 ~> This section assumes that the old CSE 4.1 installation was done with Terraform by following the 4.1 guide steps.
 Also, you need to meet [the pre-requisites criteria](#pre-requisites).
@@ -372,21 +379,23 @@ resource "vcd_rde" "vcdkeconfig_instance" {
   # Same values as before, except:
   input_entity = templatefile(var.vcdkeconfig_template_filepath, {
     # Same values as before, except:
+    capvcd_version        = "1.2.0"
+    cpi_version           = "1.5.0"
+    csi_version           = "1.5.0"
     rde_projector_version = "0.7.0"
   })
 }
 ```
 
 You can find the meaning of these values in the section ["RDE (CSE Server configuration / VCDKEConfig)"](#rde-cse-server-configuration--vcdkeconfig).
-Please notice that you need to upgrade the CAPVCD, CPI and CSI versions. The new values are stated in the same section.
 
-### Upload the new CSE 4.2 OVA
+### Upload the new CSE 4.2.0 OVA
 
-You need to upload the new CSE 4.2 OVA to the `cse_catalog` that already hosts the CSE 4.1 one.
+You need to upload the new CSE 4.2.0 OVA to the `cse_catalog` that already hosts the CSE 4.1 one.
 To download the required OVAs, please refer to the [CSE documentation][cse_docs].
 
 ```hcl
-resource "vcd_catalog_vapp_template" "cse_ova_4_2" {
+resource "vcd_catalog_vapp_template" "cse_ova_4_2_0" {
   org        = vcd_org.solutions_organization.name # References the Solutions Organization that already exists from 4.1
   catalog_id = vcd_catalog.cse_catalog.id          # References the CSE Catalog that already exists from 4.1
 
@@ -403,11 +412,73 @@ To update the CSE Server, just change the referenced OVA:
 ```hcl
 resource "vcd_vapp_vm" "cse_server_vm" {
   # All values remain the same, except:
-  vapp_template_id = vcd_catalog_vapp_template.cse_ova_4_2.id # Reference the 4.2 OVA
+  vapp_template_id = vcd_catalog_vapp_template.cse_ova_4_2_0.id # Reference the 4.2.0 OVA
+}
+```
+
+This will re-deploy the VM with the new CSE 4.2.0 Server.
+
+## Upgrade from CSE 4.2.0 to 4.2.1
+
+In this section you can find the required steps to update from CSE 4.2.0 to 4.2.1.
+
+Change the `VCDKEConfig` [RDE][rde] to update the `capvcd_version`, `cpi_version` and `csi_version` (follow [the instructions
+in the section below](#update-cse-server-configuration) to know how to upgrade this configuration):
+
+```hcl
+resource "vcd_rde" "vcdkeconfig_instance" {
+  # ...omitted
+  input_entity = templatefile(var.vcdkeconfig_template_filepath, {
+    # ...omitted
+    capvcd_version = "1.3.0" # It was 1.2.0 in 4.2.0
+    cpi_version    = "1.6.0" # It was 1.5.0 in 4.2.0
+    csi_version    = "1.6.0" # It was 1.5.0 in 4.2.0
+  })
+}
+```
+
+The **Kubernetes Clusters Rights Bundle** and **Kubernetes Cluster Author Role** need to have the Right to view and manage IP Spaces:
+
+```hcl
+resource "vcd_role" "cse_admin_role" {
+  name = "CSE Admin Role"
+  # ...omitted
+  rights = [
+    "API Tokens: Manage",
+    # ...omitted
+    "IP Spaces: Allocate",
+    "Private IP Spaces: View",
+    "Private IP Spaces: Manage",
+  ]
+}
+
+resource "vcd_rights_bundle" "k8s_clusters_rights_bundle" {
+  name = "Kubernetes Clusters Rights Bundle"
+  # ...omitted
+  rights = [
+    "API Tokens: Manage",
+    # ...omitted
+    "IP Spaces: Allocate",
+    "Private IP Spaces: View",
+    "Private IP Spaces: Manage",
+  ]
+}
+
+resource "vcd_global_role" "k8s_cluster_author" {
+  name = "Kubernetes Cluster Author"
+  # ...omitted
+  rights = [
+    "API Tokens: Manage",
+    # ...omitted
+    "IP Spaces: Allocate",
+    "Private IP Spaces: View",
+    "Private IP Spaces: Manage",
+  ]
 }
 ```
 
-This will re-deploy the VM with the new CSE 4.2 Server.
+After applying the changes with `terraform apply`, you also need to update the CSE Server OVA to 4.2.1 and restart,
+like it was done [in the previous section](#update-cse-server).
 
 ## Update CSE Server Configuration