Rebuilt citycloud templates.

clusters/raas-citycloud/.secrets/rancher.rc

@@ -1,8 +1,10 @@
-
 export TF_VAR_os_auth_url=$OS_AUTH_URL/v$OS_AUTH_VERSION
 export TF_VAR_os_region=$OS_REGION_NAME
 export TF_VAR_os_username=$OS_USERNAME
 export TF_VAR_os_password=$OS_PASSWORD
-
+export TF_VAR_os_tenant_name=$OS_TENANT_NAME
+export TF_VAR_os_user_domain_name=$OS_USER_DOMAIN_NAME
+export TF_VAR_s3_access_key=""
+export TF_VAR_s3_secret_key=""
 export TF_VAR_rancher_access_key=""
-export TF_VAR_rancher_secret_key=""
+export TF_VAR_rancher_secret_key=""

clusters/raas-citycloud/README.md

@@ -0,0 +1,24 @@
+# Prerequisites
+## Object storage
+These scripts expect an S3-compatible bucket for automated backups of the cluster state.
+Instructions on how to create one can be found here:
+https://cleura.cloud/storage/objectstorage
+
+## Rancher
+You'll need an access key and a secret key from RaaS.
+This can be created under your user (on the top right in the UI) "Account and API Keys" and "Create API Key".
+Take notes of these as they can not to retrieved again.
+
+## OpenStack
+You need to create an OpenStack user with permissions to create resources.
+This can be done here:
+https://cleura.cloud/users/openstack
+
+## SSH
+You'll need to provide a SSH keypair which in turn will be created as an OpenStack keypair.
+This key needs to be placed in $PWD/.secrets
+A key using ed25519 is expected.
+You can either provide your own key or create a new one:
+```bash
+$ ssh-keygen -t ed25519
+```

clusters/raas-citycloud/apps.tf

@@ -1,13 +1,131 @@
-/* Example!
+resource "rancher2_catalog_v2" "cloud-provider-openstack" {
+  cluster_id = "${rancher2_cluster_v2.cluster.cluster_v1_id}"
+  name = "cloud-provider-openstack"
+  url = "https://kubernetes.github.io/cloud-provider-openstack"
+  depends_on = [
+    rancher2_cluster_sync.cluster,
+  ]
+}
+resource "rancher2_catalog_v2" "ingress-nginx" {
+  cluster_id = "${rancher2_cluster_v2.cluster.cluster_v1_id}"
+  name = "ingress-nginx"
+  url = "https://kubernetes.github.io/ingress-nginx"
+  depends_on = [
+    rancher2_cluster_sync.cluster,
+  ]
+}
+
+resource "rancher2_catalog_v2" "jetstack" {
+  cluster_id = "${rancher2_cluster_v2.cluster.cluster_v1_id}"
+  name = "jetstack"
+  url = "https://charts.jetstack.io"
+  depends_on = [
+    rancher2_cluster_sync.cluster,
+  ]
+}
+
+resource "rancher2_app_v2" "cert-manager" {
+  cluster_id = "${rancher2_cluster_v2.cluster.cluster_v1_id}"
+  name = "cert-manager"
+  namespace = "cert-manager"
+  repo_name = "jetstack"
+  chart_name = "cert-manager"
+  values = file("${path.module}/config/cert_manager-values.yaml")
+  depends_on = [
+    rancher2_catalog_v2.jetstack,
+  ]
+}
+
+resource "rancher2_app_v2" "openstack-cloud-controller-manager" {
+  cluster_id = "${rancher2_cluster_v2.cluster.cluster_v1_id}"
+  name = "openstack-cloud-controller-manager"
+  namespace = "kube-system"
+  repo_name = "cloud-provider-openstack"
+  chart_name = "openstack-cloud-controller-manager"
+  values = templatefile("${path.module}/templates/openstack-ccm_values.tftpl", { floating_network_id = (data.openstack_networking_network_v2.external-network.id), subnet_id = openstack_networking_subnet_v2.subnet.id, endpoint = "https://${lower(data.openstack_identity_auth_scope_v3.scope.region)}.citycloud.com:5000", username = (data.openstack_identity_auth_scope_v3.scope.user_name), password = var.os_password, tenant_name = (data.openstack_identity_auth_scope_v3.scope.project_name), domain_name = (data.openstack_identity_auth_scope_v3.scope.user_domain_name), region = (data.openstack_identity_auth_scope_v3.scope.region), router_id = "${openstack_networking_router_v2.router.id}", cluster_name = rancher2_cluster_v2.cluster.name })
+  # For some reason, that I can't really find, the cloud-provider-openstack helm chart does not create the neccesary roles and rolebindings.
+  # So, solving it like this for now.
+  provisioner "local-exec" {
+    command = <<-EOT
+      kubectl --kubeconfig kubeconfig.yml apply -f https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/master/manifests/controller-manager/cloud-controller-manager-roles.yaml
+      kubectl --kubeconfig kubeconfig.yml apply -f https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/master/manifests/controller-manager/cloud-controller-manager-role-bindings.yaml
+    EOT
+  }
+  depends_on = [
+    rancher2_catalog_v2.cloud-provider-openstack,
+    local_sensitive_file.kube_config,
+  ]
+}
+
+resource "rancher2_app_v2" "openstack-cinder-csi" {
+  cluster_id = "${rancher2_cluster_v2.cluster.cluster_v1_id}"
+  name = "openstack-cinder-csi"
+  namespace = "kube-system"
+  repo_name = "cloud-provider-openstack"
+  chart_name = "openstack-cinder-csi"
+  values = templatefile("${path.module}/templates/csi-values.tftpl", { endpoint = "https://${lower(data.openstack_identity_auth_scope_v3.scope.region)}.citycloud.com:5000/v3/", username = (data.openstack_identity_auth_scope_v3.scope.user_name), password = var.os_password, tenant_name = (data.openstack_identity_auth_scope_v3.scope.project_name), domain_name = (data.openstack_identity_auth_scope_v3.scope.user_domain_name), subnet_name = openstack_networking_subnet_v2.subnet.name, cluster_id = "${rancher2_cluster_v2.cluster.cluster_v1_id}" })
+  depends_on = [
+    rancher2_catalog_v2.cloud-provider-openstack,
+  ]
+  # Terraform wants to apply changes to the chart values.yaml on every apply, so ignoring any changes pending a proper fix.
+  lifecycle {
+    ignore_changes = [
+      values,
+    ]
+  }
+}
+
+resource "rancher2_app_v2" "rancher-monitoring" {
+  cluster_id = "${rancher2_cluster_v2.cluster.cluster_v1_id}"
+  name = "rancher-monitoring"
+  namespace = "cattle-monitoring-system"
+  repo_name = "rancher-charts"
+  chart_name = "rancher-monitoring"
+  values = file("./config/rancher-monitoring_values.yaml")
+  depends_on = [
+    rancher2_cluster_sync.cluster,
+  ]
+}
 
-resource "rancher2_cluster_sync" "active_cluster" {
-  cluster_id =  rancher2_cluster.cluster.id
+resource "rancher2_app_v2" "ingress-nginx" {
+  cluster_id = "${rancher2_cluster_v2.cluster.cluster_v1_id}"
+  name = "ingress-nginx"
+  namespace = "ingress-nginx"
+  repo_name = "ingress-nginx"
+  chart_name = "ingress-nginx"
+  values = file("./config/ingress-nginx_values.yaml")
+  depends_on = [
+    rancher2_catalog_v2.ingress-nginx,
+    rancher2_app_v2.openstack-cloud-controller-manager,
+  ]
 }
 
-module "wordpress" {
-  source = "./modules/wordpress"
-  domain = var.caas_domain
-  cluster = rancher2_cluster_sync.active_cluster
+resource "rancher2_secret_v2" "rancher-backup-secret" {
+  cluster_id = "${rancher2_cluster_v2.cluster.cluster_v1_id}"
+  name = "s3-credentials"
+  namespace = "cattle-resources-system"
+  data = {
+      accessKey = var.s3_access_key
+      secretKey = var.s3_secret_key
+  }
+  depends_on = [
+    rancher2_cluster_sync.cluster,
+  ]
 }
 
-*/
+resource "rancher2_app_v2" "rancher-backup" {
+  cluster_id = "${rancher2_cluster_v2.cluster.cluster_v1_id}"
+  name = "rancher-backup"
+  namespace = "cattle-resources-system"
+  repo_name = "rancher-charts"
+  chart_name = "rancher-backup"
+  values = templatefile("${path.module}/templates/rancher-backup_values.tftpl", { region = (data.openstack_identity_auth_scope_v3.scope.region), bucket_name = var.prefix, s3_endpoint = var.s3_endpoint,  })
+  provisioner "local-exec" {
+    command = <<-EOT
+      kubectl --kubeconfig kubeconfig.yml apply -f ./config/rancher-backup_midnight.yaml
+    EOT
+  }
+  depends_on = [
+    rancher2_cluster_sync.cluster,
+  ]
+}

clusters/raas-citycloud/cluster.tf

@@ -1,44 +1,58 @@
-
-resource "rancher2_cluster" "cluster" {
+resource "rancher2_cluster_v2" "cluster" {
   name = var.prefix
+  kubernetes_version = "v1.24.8+rke2r1"
+  enable_network_policy = false
+  default_cluster_role_for_project_members = "user"
   rke_config {
-    network {
-      plugin = "canal"
+    machine_pools {
+      name = "master"
+      cloud_credential_secret_name = data.rancher2_cloud_credential.rancher2_cloud_credential.id
+      control_plane_role = true
+      etcd_role = true
+      worker_role = false
+      quantity = var.no_master_nodes
+      machine_config {
+        kind = rancher2_machine_config_v2.openstack.kind
+        name = rancher2_machine_config_v2.openstack.name
+      }
     }
-
-    cloud_provider {
-      openstack_cloud_provider {
-        global {
-          auth_url = "https://${lower(data.openstack_identity_auth_scope_v3.scope.region)}.citycloud.com:5000/v3/"
-          password = var.os_password
-          username = data.openstack_identity_auth_scope_v3.scope.user_name
-          domain_id = data.openstack_identity_auth_scope_v3.scope.project_domain_id
-          tenant_id = data.openstack_identity_auth_scope_v3.scope.project_id
-        }
-        block_storage {
-          ignore_volume_az = true
-          trust_device_path = false
-        }
-        metadata {
-          request_timeout = 0
-        }
-        load_balancer {
-          subnet_id = openstack_networking_subnet_v2.subnet.id
-          floating_network_id = data.openstack_networking_network_v2.external-network.id
-          use_octavia = true
-        }
+    machine_pools {
+      name = "worker"
+      cloud_credential_secret_name = data.rancher2_cloud_credential.rancher2_cloud_credential.id
+      control_plane_role = false
+      etcd_role = false
+      worker_role = true
+      quantity = var.no_worker_nodes
+      machine_config {
+        kind = rancher2_machine_config_v2.openstack.kind
+        name = rancher2_machine_config_v2.openstack.name
+      }
+    }
+    machine_selector_config {
+      config = {
+        disable = "rke2-ingress-nginx"
       }
     }
+}
+  lifecycle {
+    ignore_changes = [
+      cloud_credential_secret_name,
+    ]
   }
+  depends_on = [
+    openstack_networking_subnet_v2.subnet,
+    rancher2_cloud_credential.cloud_credential_name,
+  ]
 }
 
-resource "rancher2_node_pool" "node_pool" {
-  cluster_id = rancher2_cluster.cluster.id
-  name = var.prefix
-  hostname_prefix = var.prefix
-  node_template_id = rancher2_node_template.citycloud.id
-  quantity = 3
-  control_plane = true
-  etcd = true
-  worker = true
+resource "rancher2_cluster_sync" "cluster" {
+  cluster_id = "${rancher2_cluster_v2.cluster.cluster_v1_id}"
+  depends_on = [
+    rancher2_cluster_v2.cluster,
+  ]
+}
+
+resource "local_sensitive_file" "kube_config" {
+  filename = "kubeconfig.yml"
+  content = rancher2_cluster_v2.cluster.kube_config
 }

clusters/raas-citycloud/config/cert_manager-values.yaml

@@ -0,0 +1 @@
+installCRDs: true

clusters/raas-citycloud/config/ingress-nginx_values.yaml

@@ -0,0 +1,2 @@
+controller:
+  kind: DaemonSet

clusters/raas-citycloud/config/rancher-backup_midnight.yaml

@@ -0,0 +1,9 @@
+apiVersion: resources.cattle.io/v1
+kind: Backup
+metadata:
+  name: daily
+  namespace: cattle-resources-system
+spec:
+  resourceSetName: rancher-resource-set
+  retentionCount: 10
+  schedule: '@midnight'

clusters/raas-citycloud/config/rancher-monitoring_values.yaml

@@ -0,0 +1,28 @@
+alertmanager:
+  enabled: true
+grafana:
+  enabled: true
+  persistence:
+    accessModes:
+      - ReadWriteOnce
+    enabled: true
+    finalizers:
+      - kubernetes.io/pvc-protection
+    inMemory:
+      enabled: false
+    size: 5Gi
+    type: pvc
+    storageClassName: csi-cinder-sc-delete
+prometheus:
+  enabled: true
+  prometheusSpec:
+    storageSpec:
+      volumeClaimTemplate:
+        spec:
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: 50Gi
+          storageClassName: csi-cinder-sc-delete
+          volumeMode: Filesystem

clusters/raas-citycloud/data.tf

@@ -5,3 +5,13 @@ data "openstack_networking_network_v2" "external-network" {
 data "openstack_identity_auth_scope_v3" "scope" {
   name = "scope"
 }
+
+# https://github.com/rancher/terraform-provider-rancher2/issues/835
+data "rancher2_cloud_credential" "rancher2_cloud_credential" {
+  name = "${var.prefix}"
+}
+
+data "rancher2_project" "system" {
+  name = "System"
+  cluster_id = "${rancher2_cluster_v2.cluster.cluster_v1_id}"
+}

clusters/raas-citycloud/networks.tf

@@ -17,3 +17,8 @@ resource "openstack_networking_router_interface_v2" "router-if" {
   router_id = openstack_networking_router_v2.router.id
   subnet_id = openstack_networking_subnet_v2.subnet.id
 }
+
+resource "openstack_compute_keypair_v2" "keypair" {
+  name = var.vm_keypair_name
+  public_key = file(pathexpand(var.ssh_public_key))
+}

clusters/raas-citycloud/node_template.tf

@@ -1,30 +1,32 @@
 
-resource "rancher2_cloud_credential" "cred" {
-  name = "${var.prefix}-cred"
+resource "rancher2_cloud_credential" "cloud_credential_name" {
+  name = "${var.prefix}"
   openstack_credential_config {
     password = var.os_password
   }
 }
 
-resource "rancher2_node_template" "citycloud" {
-  name = "${lower(data.openstack_identity_auth_scope_v3.scope.project_name)}-${data.openstack_identity_auth_scope_v3.scope.region}-${var.node_flavor}"
-  cloud_credential_id = rancher2_cloud_credential.cred.id
-  engine_install_url= "https://releases.rancher.com/install-docker/${var.docker_version}.sh"
+resource "rancher2_machine_config_v2" "openstack" {
+  generate_name = "openstack"
   openstack_config {
-    availability_zone = ""
-    auth_url = "https://${lower(data.openstack_identity_auth_scope_v3.scope.region)}.citycloud.com:5000/v3"
+    availability_zone = "nova"
+    auth_url = "https://${lower(data.openstack_identity_auth_scope_v3.scope.region)}.citycloud.com:5000/v3/"
     region = data.openstack_identity_auth_scope_v3.scope.region
     username = data.openstack_identity_auth_scope_v3.scope.user_name
-    // Do NOT use domain_name as it does not work with Openstack ATM
     domain_id = data.openstack_identity_auth_scope_v3.scope.project_domain_id
-    // Do NOT use tenant_name as it does not work with Openstack ATM
-    tenant_id = data.openstack_identity_auth_scope_v3.scope.project_id
-    // Do NOT use net_name as it does not work with Openstack ATM
-    net_id = openstack_networking_network_v2.custom_network.id
     endpoint_type = "publicURL"
-    floating_ip_pool = data.openstack_networking_network_v2.external-network.name
     flavor_name = var.node_flavor
     image_name = var.vm_image
+    keypair_name = var.vm_keypair_name
+    private_key_file = "${var.ssh_private_key}"
+    net_id = openstack_networking_network_v2.custom_network.id
+    password = var.os_password
     ssh_user = var.vm_ssh_user
+    tenant_id = data.openstack_identity_auth_scope_v3.scope.project_id
+  }
+  lifecycle {
+    ignore_changes = [
+      openstack_config.0.volume_size,
+    ]
   }
 }