k8s-infra-vsphere: add terraform to provision vsphere for ci

infra/gcp/terraform/k8s-infra-vsphere/vsphere/folders.tf

@@ -0,0 +1,39 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+resource "vsphere_folder" "cpi" {
+  path          = "cloud-provider-vsphere"
+  type          = "vm"
+  datacenter_id = data.vsphere_datacenter.datacenter.id
+}
+
+resource "vsphere_folder" "capi" {
+  path          = "cluster-api-provider-vsphere"
+  type          = "vm"
+  datacenter_id = data.vsphere_datacenter.datacenter.id
+}
+
+resource "vsphere_folder" "image-builder" {
+  path          = "image-builder"
+  type          = "vm"
+  datacenter_id = data.vsphere_datacenter.datacenter.id
+}
+
+resource "vsphere_folder" "templates" {
+  path          = "templates"
+  type          = "vm"
+  datacenter_id = data.vsphere_datacenter.datacenter.id
+}

infra/gcp/terraform/k8s-infra-vsphere/vsphere/hack/ensure_ova_from_github.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+if [[ "${DEBUG}" == "true" ]]; then
+  set -x
+fi
+
+# TODO: ensure govc
+# TODO: ensure all env vars are set
+# TODO: set pipefail, etc.
+
+GITHUB_CA_CERTIFICATE_JSON='{"cert_chain": {"cert_chain": ["'${GITHUB_CA_CERTIFICATE}'"]}}'
+
+# Ensure the githubs CA certificate exists
+
+echo "> Ensuring githubs CA certificate exists so library imports work"
+
+if [[ "$(govc session.login -r -X GET "/api/vcenter/certificate-management/vcenter/trusted-root-chains" | grep -e "${GITHUB_CA_THUMBPRINT}")" == "" ]]; then
+  govc session.login -r -X POST "/api/vcenter/certificate-management/vcenter/trusted-root-chains" <<< ${GITHUB_CA_CERTIFICATE_JSON}
+fi
+
+function ensureOVA() {
+  URL="${1}"
+  NAME=${URL##*/}
+  NAME=${NAME%.ova}
+
+  echo "> Ensuring OVA ${NAME} from ${URL} exists"
+
+  if [[ "$(govc library.info "/${CONTENT_LIBRARY_NAME}/${NAME}" || true)" == "" ]]; then
+    echo ">> in content library /${CONTENT_LIBRARY_NAME}"
+    govc library.import -pull "/${CONTENT_LIBRARY_NAME}" "${URL}"
+  fi
+
+  if [[ "$(govc vm.info "${TEMPLATES_FOLDER}/${NAME}" || true)" == "" ]]; then
+    echo ">> as VM template in ${TEMPLATES_FOLDER}"
+    govc library.deploy -folder "${TEMPLATES_FOLDER}" -ds "${DATASTORE}" -pool "${RESOURCE_POOL}" "/${CONTENT_LIBRARY_NAME}/${NAME}"
+    govc snapshot.create -vm "${TEMPLATES_FOLDER}/${NAME}" "for-linkedclone"
+    govc vm.markastemplate "${TEMPLATES_FOLDER}/${NAME}"
+  fi
+}
+
+echo "> Ensuring OVA from ${URL}"
+
+ensureOVA "${URL}"

infra/gcp/terraform/k8s-infra-vsphere/vsphere/permissions.tf

@@ -0,0 +1,99 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+resource "vsphere_role" "capv-ci-content-library" {
+  name = "capv-ci-content-library"
+  role_privileges = [
+    "ContentLibrary.DownloadSession",
+    "ContentLibrary.ReadStorage",
+    "ContentLibrary.SyncLibraryItem",
+  ]
+}
+
+resource "vsphere_role" "capv-ci" {
+  name = "capv-ci"
+  role_privileges = [
+    "Cns.Searchable",
+    "Datastore.AllocateSpace",
+    "Datastore.Browse",
+    "Datastore.FileManagement",
+    "Folder.Create",
+    "Folder.Delete",
+    "Global.SetCustomField",
+    "Network.Assign",
+    "Resource.AssignVMToPool",
+    "Resource.CreatePool",
+    "Resource.DeletePool",
+    "Sessions.GlobalMessage",
+    "Sessions.ValidateSession",
+    "StorageProfile.View",
+    "VApp.ApplicationConfig",
+    "VApp.Import",
+    "VApp.InstanceConfig",
+    "VirtualMachine.Config.AddExistingDisk",
+    "VirtualMachine.Config.AddNewDisk",
+    "VirtualMachine.Config.AddRemoveDevice",
+    "VirtualMachine.Config.AdvancedConfig",
+    "VirtualMachine.Config.Annotation",
+    "VirtualMachine.Config.CPUCount",
+    "VirtualMachine.Config.ChangeTracking",
+    "VirtualMachine.Config.DiskExtend",
+    "VirtualMachine.Config.EditDevice",
+    "VirtualMachine.Config.HostUSBDevice",
+    "VirtualMachine.Config.ManagedBy",
+    "VirtualMachine.Config.Memory",
+    "VirtualMachine.Config.RawDevice",
+    "VirtualMachine.Config.RemoveDisk",
+    "VirtualMachine.Config.Resource",
+    "VirtualMachine.Config.Settings",
+    "VirtualMachine.Config.SwapPlacement",
+    "VirtualMachine.Config.UpgradeVirtualHardware",
+    "VirtualMachine.Interact.ConsoleInteract",
+    "VirtualMachine.Interact.DeviceConnection",
+    "VirtualMachine.Interact.PowerOff",
+    "VirtualMachine.Interact.PowerOn",
+    "VirtualMachine.Interact.SetCDMedia",
+    "VirtualMachine.Interact.SetFloppyMedia",
+    "VirtualMachine.Inventory.Create",
+    "VirtualMachine.Inventory.CreateFromExisting",
+    "VirtualMachine.Inventory.Delete",
+    "VirtualMachine.Provisioning.Clone",
+    "VirtualMachine.Provisioning.CloneTemplate",
+    "VirtualMachine.Provisioning.CreateTemplateFromVM",
+    "VirtualMachine.Provisioning.DeployTemplate",
+    "VirtualMachine.Provisioning.DiskRandomRead",
+    "VirtualMachine.Provisioning.GetVmFiles",
+    "VirtualMachine.State.CreateSnapshot",
+    "VirtualMachine.State.RemoveSnapshot",
+  ]
+}
+
+# resource "vsphere_entity_permissions" "rp-capv" {
+#   entity_id = vsphere_resource_pool.capi.id
+#   entity_type = "ResourcePool"
+#   permissions {
+#     user_or_group = "vsphere.local\\DCClients"
+#     propagate = true
+#     is_group = true
+#     role_id = data.vsphere_role.role1.id
+#   }
+#   permissions {
+#     user_or_group = "vsphere.local\\ExternalIDPUsers"
+#     propagate = true
+#     is_group = true
+#     role_id = vsphere_role.role2.id
+#   }
+# }

infra/gcp/terraform/k8s-infra-vsphere/vsphere/resource-pools.tf

@@ -0,0 +1,35 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+resource "vsphere_resource_pool" "cpi" {
+  name                    = "cloud-provider-vsphere"
+  parent_resource_pool_id = data.vsphere_compute_cluster.compute_cluster.resource_pool_id
+}
+
+resource "vsphere_resource_pool" "capi" {
+  name                    = "cluster-api-provider-vsphere"
+  parent_resource_pool_id = data.vsphere_compute_cluster.compute_cluster.resource_pool_id
+}
+
+resource "vsphere_resource_pool" "image-builder" {
+  name                    = "image-builder"
+  parent_resource_pool_id = data.vsphere_compute_cluster.compute_cluster.resource_pool_id
+}
+
+resource "vsphere_resource_pool" "templates" {
+  name                    = "templates"
+  parent_resource_pool_id = data.vsphere_compute_cluster.compute_cluster.resource_pool_id
+}

infra/gcp/terraform/k8s-infra-vsphere/vsphere/vm-templates.tf

@@ -0,0 +1,49 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+resource "vsphere_content_library" "capv" {
+  name            = "capv"
+  description     = "Content Library for CAPV."
+  storage_backing = [data.vsphere_datastore.datastore.id]
+}
+
+# TODO: consider if this should be pure bash instead?
+
+resource "terraform_data" "ova_templates" {
+  depends_on = [vsphere_content_library.capv]
+
+  triggers_replace = sha512(file("${path.module}/hack/ensure_ova_from_github.sh"))
+
+  provisioner "local-exec" {
+    when    = create
+    command = "${path.module}/hack/ensure_ova_from_github.sh"
+    environment = {
+      "GOVC_URL"              = "${var.vsphere_user}:${var.vsphere_password}@${var.vsphere_server}"
+      "GOVC_INSECURE"         = "true"
+      "DEBUG"                 = "true"
+      "GITHUB_CA_CERTIFICATE" = "${var.github_ca_certificate}"
+      "GITHUB_CA_THUMBPRINT"  = "${var.github_ca_thumbprint}"
+      "CONTENT_LIBRARY_NAME"  = "${vsphere_content_library.capv.name}"
+      "TEMPLATES_FOLDER"      = "${vsphere_folder.templates.path}"
+      "DATASTORE"             = "${data.vsphere_datastore.datastore.name}"
+      "RESOURCE_POOL"         = "/${data.vsphere_datacenter.datacenter.name}/host/${data.vsphere_compute_cluster.compute_cluster.name}/Resources/${vsphere_resource_pool.templates.name}"
+      "URL"                   = "${each.value}"
+    }
+    interpreter = ["/bin/bash", "-c"]
+  }
+
+  for_each = var.ova_templates
+}

infra/gcp/terraform/k8s-infra-vsphere/vsphere/vsphere.tf

@@ -0,0 +1,51 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+# resource "google_vmwareengine_private_cloud" "vsphere-cluster" {
+#   location    = "us-central1"
+#   name        = "sample-pc"
+#   description = "vSphere Cluster for CI."
+#   # TODO: type TIME_LIMITED is for the timely limitted 1 node_count. Change to `STANDARD`.
+#   type        = "TIME_LIMITED"
+#   network_config {
+#     management_cidr       = "192.168.30.0/24"
+#     vmware_engine_network = google_vmwareengine_network.cluster-nw.id
+#   }
+
+#   management_cluster {
+#     cluster_id = "vsphere-ci-cluster"
+#     node_type_configs {
+#       node_type_id = "standard-72"
+#       # TODO: node_count 1 is for the TIME_LIMITED version. Change to `3`.
+#       node_count   = 1
+#     }
+#   }
+# }
+
+# resource "google_vmwareengine_network" "vsphere-network" {
+#   name        = "vsphere-network"
+#   type        = "STANDARD"
+#   # TODO: check if this needs to be set to `global` (it should according tf docs).
+#   location    = "us-central1"
+#   description = "network for vSphere CI."
+
+# }
+
+# resource "google_vmwareengine_subnet" "vmw-engine-subnet" {
+#   name = "vm-subnet"
+#   parent =  google_vmwareengine_private_cloud.vsphere-cluster.id
+#   ip_cidr_range = "192.168.31.0/24"
+# }