diff --git a/.werft/aks-installer-tests.yaml b/.werft/aks-installer-tests.yaml new file mode 100644 index 00000000000000..ae4e956c8073d1 --- /dev/null +++ b/.werft/aks-installer-tests.yaml @@ -0,0 +1,94 @@ +# debug using `werft run github -f -s .werft/installer-tests.ts -j .werft/aks-installer-tests.yaml -a debug=true` +pod: + serviceAccount: werft + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: dev/workload + operator: In + values: + - "builds" + securityContext: + runAsUser: 0 + volumes: + - name: sh-playground-sa-perm + secret: + secretName: sh-playground-sa-perm + - name: sh-playground-dns-perm + secret: + secretName: sh-playground-dns-perm + - name: sh-aks-perm + secret: + secretName: aks-credentials + containers: + - name: nightly-test + image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:cw-werft-cred.0 + workingDir: /workspace + imagePullPolicy: Always + volumeMounts: + - name: sh-playground-sa-perm + mountPath: /mnt/secrets/sh-playground-sa-perm + - name: sh-aks-perm + mountPath: /mnt/secrets/sh-aks-perm + - name: sh-playground-dns-perm # this sa is used for the DNS management + mountPath: /mnt/secrets/sh-playground-dns-perm + env: + - name: ARM_SUBSCRIPTION_ID + valueFrom: + secretKeyRef: + name: aks-credentials + key: subscriptionid + - name: ARM_TENANT_ID + valueFrom: + secretKeyRef: + name: aks-credentials + key: tenantid + - name: ARM_CLIENT_ID + valueFrom: + secretKeyRef: + name: aks-credentials + key: clientid + - name: ARM_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: aks-credentials + key: clientsecret + - name: WERFT_HOST + value: "werft.werft.svc.cluster.local:7777" + - name: GOOGLE_APPLICATION_CREDENTIALS + value: "/mnt/secrets/sh-playground-sa-perm/sh-sa.json" + - name: WERFT_K8S_NAMESPACE + value: "werft" + - name: WERFT_K8S_LABEL + value: "component=werft" + - name: TF_VAR_sa_creds + value: "/mnt/secrets/sh-playground-sa-perm/sh-sa.json" + - name: TF_VAR_dns_sa_creds + value: "/mnt/secrets/sh-playground-dns-perm/sh-dns-sa.json" + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + command: + - bash + - -c + - | + sleep 1 + set -Eeuo pipefail + + sudo chown -R gitpod:gitpod /workspace + sudo apt update && apt install gettext-base + + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + + export TF_VAR_TEST_ID=$(echo $RANDOM | md5sum | head -c 5; echo) + + (cd .werft && yarn install && mv node_modules ..) | werft log slice prep + printf '{{ toJson . }}' > context.json + + npx ts-node .werft/installer-tests.ts "STANDARD_AKS_TEST" +# The bit below makes this a cron job +# plugins: +# cron: "15 3 * * *" diff --git a/.werft/installer-tests.ts b/.werft/installer-tests.ts index 74d1abab27db8e..591c8371d46c17 100644 --- a/.werft/installer-tests.ts +++ b/.werft/installer-tests.ts @@ -86,6 +86,23 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = { "RESULTS", ], }, + STANDARD_AKS_TEST: { + CLOUD: "azure", + DESCRIPTION: "Deploy Gitpod on AKS, with managed DNS, and run integration tests", + PHASES: [ + "STANDARD_AKS_CLUSTER", + "CERT_MANAGER", + "AZURE_ISSUER", + "AZURE_EXTERNALDNS", + "ADD_NS_RECORD", + "GENERATE_KOTS_CONFIG", + "INSTALL_GITPOD", + "RESULTS", + "CHECK_INSTALLATION", + "RUN_INTEGRATION_TESTS", + "DESTROY", + ], + }, }; const config: TestConfig = TEST_CONFIGURATIONS[testConfig]; @@ -106,6 +123,11 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = { makeTarget: "k3s-standard-cluster", description: "Creating a k3s cluster on GCP with 1 node", }, + STANDARD_AKS_CLUSTER: { + phase: "create-std-aks-cluster", + makeTarget: "aks-standard-cluster", + description: "Creating an aks cluster(azure)", + }, CERT_MANAGER: { phase: "setup-cert-manager", makeTarget: "cert-manager", @@ -124,6 +146,21 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = { )} db=${randomize("db", cloud)}`, description: `Generate KOTS Config file`, }, + AZURE_ISSUER: { + phase: "setup-azure-cluster-issuer", + makeTarget: "azure-issuer", + description: "Deploys ClusterIssuer for azure", + }, + AZURE_EXTERNALDNS: { + phase: "azure-external-dns", + makeTarget: "azure-external-dns", + description: "Deploys external-dns with azure provider", + }, + ADD_NS_RECORD: { + phase: "add-ns-record", + makeTarget: "add-ns-record", + description: "Adds NS record for subdomain under gitpod-self-hosted.com", + }, INSTALL_GITPOD_IGNORE_PREFLIGHTS: { phase: "install-gitpod-without-preflights", makeTarget: `kots-install channel=${channel} version=${version} preflights=false`, // this is a bit of a hack, for now we pass params like this diff --git a/install/infra/terraform/aks/README.md b/install/infra/terraform/aks/README.md new file mode 100644 index 00000000000000..e03e1cc8289b3b --- /dev/null +++ b/install/infra/terraform/aks/README.md @@ -0,0 +1,97 @@ +# Azure + +Azure provider for Gitpod testing + + + +- [Terraform Documentation](#terraform-documentation) + * [Requirements](#requirements) + * [Providers](#providers) + * [Modules](#modules) + * [Resources](#resources) + * [Inputs](#inputs) + * [Outputs](#outputs) + + + +# Terraform Documentation + + +## Requirements + +| Name | Version | +|------|---------| +| [azurerm](#requirement\_azurerm) | >= 3.0.0, < 4.0.0 | + +## Providers + +| Name | Version | +|------|---------| +| [azurerm](#provider\_azurerm) | >= 3.0.0, < 4.0.0 | +| [random](#provider\_random) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [azurerm_container_registry.registry](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/container_registry) | resource | +| [azurerm_dns_zone.dns](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/dns_zone) | resource | +| [azurerm_kubernetes_cluster.k8s](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster) | resource | +| [azurerm_kubernetes_cluster_node_pool.pools](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster_node_pool) | resource | +| [azurerm_log_analytics_solution.monitoring](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/log_analytics_solution) | resource | +| [azurerm_log_analytics_workspace.monitoring](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/log_analytics_workspace) | resource | +| [azurerm_mysql_database.db](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/mysql_database) | resource | +| [azurerm_mysql_firewall_rule.db](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/mysql_firewall_rule) | resource | +| [azurerm_mysql_server.db](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/mysql_server) | resource | +| [azurerm_network_security_rule.k8s](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_security_rule) | resource | +| [azurerm_resource_group.gitpod](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource | +| [azurerm_role_assignment.k8s](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | +| [azurerm_role_assignment.registry](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | +| [azurerm_storage_account.storage](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_account) | resource | +| [azurerm_subnet.network](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/subnet) | resource | +| [azurerm_virtual_network.network](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/virtual_network) | resource | +| [random_integer.db](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/integer) | resource | +| [random_integer.registry](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/integer) | resource | +| [random_integer.storage](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/integer) | resource | +| [random_password.db](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | +| [azurerm_client_config.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/client_config) | data source | +| [azurerm_kubernetes_service_versions.k8s](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/kubernetes_service_versions) | data source | +| [azurerm_resources.k8s](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resources) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [dns\_enabled](#input\_dns\_enabled) | Common variables | `any` | n/a | yes | +| [domain\_name](#input\_domain\_name) | n/a | `any` | n/a | yes | +| [enable\_airgapped](#input\_enable\_airgapped) | n/a | `any` | n/a | yes | +| [enable\_external\_database](#input\_enable\_external\_database) | n/a | `any` | n/a | yes | +| [enable\_external\_registry](#input\_enable\_external\_registry) | n/a | `any` | n/a | yes | +| [enable\_external\_storage](#input\_enable\_external\_storage) | n/a | `any` | n/a | yes | +| [labels](#input\_labels) | n/a | `any` | n/a | yes | +| [location](#input\_location) | Azure-specific variables | `any` | n/a | yes | +| [name\_format](#input\_name\_format) | n/a | `any` | n/a | yes | +| [name\_format\_global](#input\_name\_format\_global) | n/a | `any` | n/a | yes | +| [workspace\_name](#input\_workspace\_name) | n/a | `any` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [cert\_manager\_issuer](#output\_cert\_manager\_issuer) | n/a | +| [cert\_manager\_secret](#output\_cert\_manager\_secret) | n/a | +| [cluster\_name](#output\_cluster\_name) | n/a | +| [database](#output\_database) | n/a | +| [domain\_nameservers](#output\_domain\_nameservers) | n/a | +| [external\_dns\_secrets](#output\_external\_dns\_secrets) | n/a | +| [external\_dns\_settings](#output\_external\_dns\_settings) | n/a | +| [k8s\_connection](#output\_k8s\_connection) | n/a | +| [kubeconfig](#output\_kubeconfig) | n/a | +| [region](#output\_region) | n/a | +| [registry](#output\_registry) | n/a | +| [storage](#output\_storage) | n/a | + diff --git a/install/infra/terraform/aks/database.tf b/install/infra/terraform/aks/database.tf new file mode 100644 index 00000000000000..50a8a21e3fa9e2 --- /dev/null +++ b/install/infra/terraform/aks/database.tf @@ -0,0 +1,50 @@ +resource "random_integer" "db" { + count = var.enable_external_database ? 1 : 0 + + min = 10000 + max = 99999 +} + +resource "random_password" "db" { + count = var.enable_external_database ? 1 : 0 + + length = 32 +} + +resource "azurerm_mysql_server" "db" { + count = var.enable_external_database ? 1 : 0 + + name = "gitpod-${random_integer.db[count.index].result}" + location = azurerm_resource_group.gitpod.location + resource_group_name = azurerm_resource_group.gitpod.name + + sku_name = local.db + storage_mb = 20480 + ssl_enforcement_enabled = false + ssl_minimal_tls_version_enforced = "TLSEnforcementDisabled" + version = "5.7" + + auto_grow_enabled = true + administrator_login = "gitpod" + administrator_login_password = random_password.db[count.index].result +} + +resource "azurerm_mysql_firewall_rule" "db" { + count = var.enable_external_database ? 1 : 0 + + name = "Azure_Resource" + resource_group_name = azurerm_resource_group.gitpod.name + server_name = azurerm_mysql_server.db[count.index].name + start_ip_address = "0.0.0.0" + end_ip_address = "0.0.0.0" +} + +resource "azurerm_mysql_database" "db" { + count = var.enable_external_database ? 1 : 0 + + name = "gitpod" + resource_group_name = azurerm_resource_group.gitpod.name + server_name = azurerm_mysql_server.db[count.index].name + charset = "utf8" + collation = "utf8_unicode_ci" +} diff --git a/install/infra/terraform/aks/kubernetes.tf b/install/infra/terraform/aks/kubernetes.tf new file mode 100644 index 00000000000000..16629d7c3bfc53 --- /dev/null +++ b/install/infra/terraform/aks/kubernetes.tf @@ -0,0 +1,122 @@ +data "azurerm_kubernetes_service_versions" "k8s" { + location = azurerm_resource_group.gitpod.location + include_preview = false +} + +resource "azurerm_role_assignment" "k8s" { + count = var.dns_enabled ? 1 : 0 + + principal_id = azurerm_kubernetes_cluster.k8s.kubelet_identity[count.index].object_id + role_definition_name = "DNS Zone Contributor" + scope = azurerm_dns_zone.dns[count.index].id +} + +resource "azurerm_role_assignment" "k8s_reader" { + count = var.dns_enabled ? 1 : 0 + + principal_id = azurerm_kubernetes_cluster.k8s.kubelet_identity[count.index].object_id + role_definition_name = "Reader" + scope = azurerm_dns_zone.dns[count.index].id +} + +resource "azurerm_kubernetes_cluster" "k8s" { + name = format(local.name_format, local.location, "primary") + location = azurerm_resource_group.gitpod.location + resource_group_name = azurerm_resource_group.gitpod.name + dns_prefix = "gitpod" + tags = {} + api_server_authorized_ip_ranges = [] + + kubernetes_version = data.azurerm_kubernetes_service_versions.k8s.latest_version + http_application_routing_enabled = false + + default_node_pool { + name = local.nodes.0.name + vm_size = local.machine + + + node_taints = [] + tags = {} + zones = [] + + enable_auto_scaling = true + min_count = 2 + max_count = 10 + orchestrator_version = data.azurerm_kubernetes_service_versions.k8s.latest_version + node_labels = local.nodes.0.labels + + type = "VirtualMachineScaleSets" + vnet_subnet_id = azurerm_subnet.network.id + } + + identity { + type = "SystemAssigned" + identity_ids = [] + } + + network_profile { + network_plugin = "kubenet" + network_policy = "calico" + } + + oms_agent { + log_analytics_workspace_id = azurerm_log_analytics_workspace.monitoring.id + } +} + +resource "azurerm_kubernetes_cluster_node_pool" "pools" { + count = length(local.nodes) - 1 + + kubernetes_cluster_id = azurerm_kubernetes_cluster.k8s.id + name = local.nodes[count.index + 1].name + vm_size = local.machine + + enable_auto_scaling = true + min_count = 2 + max_count = 10 + orchestrator_version = data.azurerm_kubernetes_service_versions.k8s.latest_version + node_labels = local.nodes[count.index + 1].labels + vnet_subnet_id = azurerm_subnet.network.id +} + +data "azurerm_resources" "k8s" { + count = var.enable_airgapped ? 1 : 0 + + resource_group_name = azurerm_kubernetes_cluster.k8s.node_resource_group + type = "Microsoft.Network/networkSecurityGroups" + + depends_on = [ + azurerm_kubernetes_cluster.k8s, + azurerm_kubernetes_cluster_node_pool.pools + ] +} + +resource "azurerm_network_security_rule" "k8s" { + count = length(local.network_security_rules) + + resource_group_name = azurerm_kubernetes_cluster.k8s.node_resource_group + network_security_group_name = data.azurerm_resources.k8s.0.resources.0.name + + priority = lookup(local.network_security_rules[count.index], "priority", sum([100, count.index])) + name = local.network_security_rules[count.index].name + access = local.network_security_rules[count.index].access + direction = local.network_security_rules[count.index].direction + protocol = local.network_security_rules[count.index].protocol + + description = lookup(local.network_security_rules[count.index], "description", null) + source_port_range = lookup(local.network_security_rules[count.index], "source_port_range", null) + destination_port_range = lookup(local.network_security_rules[count.index], "destination_port_range", null) + source_address_prefix = lookup(local.network_security_rules[count.index], "source_address_prefix", null) + destination_address_prefix = lookup(local.network_security_rules[count.index], "destination_address_prefix", null) +} + +resource "local_file" "kubeconfig" { + depends_on = [ + resource.azurerm_kubernetes_cluster_node_pool.pools, + ] + filename = var.kubeconfig + content = azurerm_kubernetes_cluster.k8s.kube_config_raw + lifecycle { + create_before_destroy = true + } +} diff --git a/install/infra/terraform/aks/local.tf b/install/infra/terraform/aks/local.tf new file mode 100644 index 00000000000000..e11d87159206cc --- /dev/null +++ b/install/infra/terraform/aks/local.tf @@ -0,0 +1,95 @@ +locals { + labels = tomap({ + workload_meta : "gitpod.io/workload_meta" + workload_ide : "gitpod.io/workload_ide" + workspace_services : "gitpod.io/workload_workspace_services" + workspace_regular : "gitpod.io/workload_workspace_regular" + workspace_headless : "gitpod.io/workload_workspace_headless" + }) + dns_enabled = var.domain_name != null + name_format = join("-", [ + "gitpod", + "%s", # region + "%s", # name + local.workspace_name + ]) + name_format_global = join("-", [ + "gitpod", + "%s", # name + local.workspace_name + ]) + workspace_name = replace(terraform.workspace, "/[\\W\\-]/", "") # alphanumeric workspace name + db = "GP_Gen5_2" + location = substr(var.location, 0, 3) # Short code for location + machine = "Standard_D4_v3" + network_security_rules = var.enable_airgapped ? [ + { + name = "AllowContainerRegistry" + description = "Allow outgoing traffic to the container registry" + direction = "Outbound" + access = "Allow" + protocol = "*" + source_port_range = "*" + destination_port_range = "*" + source_address_prefix = "*" + destination_address_prefix = "AzureContainerRegistry" + }, + { + name = "AllowDatabase" + description = "Allow outgoing traffic to the database" + direction = "Outbound" + access = "Allow" + protocol = "*" + source_port_range = "*" + destination_port_range = "*" + source_address_prefix = "*" + destination_address_prefix = "Sql" + }, + { + name = "AllowStorage" + description = "Allow outgoing traffic to the storage" + direction = "Outbound" + access = "Allow" + protocol = "*" + source_port_range = "*" + destination_port_range = "*" + source_address_prefix = "*" + destination_address_prefix = "Storage" + }, + { + name = "AllowAzureCloud" + description = "Allow outgoing traffic to the Azure cloud" + direction = "Outbound" + access = "Allow" + protocol = "*" + source_port_range = "*" + destination_port_range = "*" + source_address_prefix = "*" + destination_address_prefix = "AzureCloud" + }, + { + name = "DenyInternetOutBound" + description = "Deny outgoing traffic to the public internet" + direction = "Outbound" + access = "Deny" + protocol = "*" + source_port_range = "*" + destination_port_range = "*" + source_address_prefix = "*" + destination_address_prefix = "Internet" + priority = 4096 + } + ] : [] + nodes = [ + { + name = "services" + labels = { + lookup(local.labels, "workload_meta") = true + lookup(local.labels, "workload_ide") = true + lookup(local.labels, "workspace_services") = true + lookup(local.labels, "workspace_regular") = true + lookup(local.labels, "workspace_headless") = true + } + } + ] +} diff --git a/install/infra/terraform/aks/main.tf b/install/infra/terraform/aks/main.tf new file mode 100644 index 00000000000000..ceff581fdf624a --- /dev/null +++ b/install/infra/terraform/aks/main.tf @@ -0,0 +1,19 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = ">= 3.0.0, < 4.0.0" + } + } +} + +provider "azurerm" { + features {} +} + +data "azurerm_client_config" "current" {} + +resource "azurerm_resource_group" "gitpod" { + name = format(local.name_format_global, local.location) + location = var.location +} diff --git a/install/infra/terraform/aks/monitoring.tf b/install/infra/terraform/aks/monitoring.tf new file mode 100644 index 00000000000000..d02844ca3f9547 --- /dev/null +++ b/install/infra/terraform/aks/monitoring.tf @@ -0,0 +1,19 @@ +resource "azurerm_log_analytics_workspace" "monitoring" { + name = format(local.name_format, var.location, "monitoring") + location = azurerm_resource_group.gitpod.location + resource_group_name = azurerm_resource_group.gitpod.name + sku = "PerGB2018" +} + +resource "azurerm_log_analytics_solution" "monitoring" { + solution_name = "ContainerInsights" + location = azurerm_resource_group.gitpod.location + resource_group_name = azurerm_resource_group.gitpod.name + workspace_name = azurerm_log_analytics_workspace.monitoring.name + workspace_resource_id = azurerm_log_analytics_workspace.monitoring.id + + plan { + publisher = "Microsoft" + product = "OMSGallery/ContainerInsights" + } +} diff --git a/install/infra/terraform/aks/networks.tf b/install/infra/terraform/aks/networks.tf new file mode 100644 index 00000000000000..1534593f8b6532 --- /dev/null +++ b/install/infra/terraform/aks/networks.tf @@ -0,0 +1,20 @@ +resource "azurerm_virtual_network" "network" { + name = format(local.name_format, local.location, "network") + location = azurerm_resource_group.gitpod.location + resource_group_name = azurerm_resource_group.gitpod.name + address_space = ["10.2.0.0/16"] +} + +resource "azurerm_subnet" "network" { + name = format(local.name_format, local.location, "network") + resource_group_name = azurerm_resource_group.gitpod.name + virtual_network_name = azurerm_virtual_network.network.name + address_prefixes = ["10.2.1.0/24"] +} + +resource "azurerm_dns_zone" "dns" { + count = var.dns_enabled ? 1 : 0 + + name = var.domain_name + resource_group_name = azurerm_resource_group.gitpod.name +} diff --git a/install/infra/terraform/aks/output.tf b/install/infra/terraform/aks/output.tf new file mode 100644 index 00000000000000..0ea0d4bf2f8b0e --- /dev/null +++ b/install/infra/terraform/aks/output.tf @@ -0,0 +1,85 @@ +output "cert_manager_issuer" { + value = try({ + subscriptionID = data.azurerm_client_config.current.subscription_id + resourceGroupName = azurerm_resource_group.gitpod.name + hostedZoneName = azurerm_dns_zone.dns.0.name + managedIdentity = { + clientID = azurerm_kubernetes_cluster.k8s.kubelet_identity.0.client_id + } + }, {}) +} + +output "cert_manager_secret" { + value = {} +} + +output "cluster_name" { + value = azurerm_kubernetes_cluster.k8s.name +} + +output "database" { + sensitive = true + value = try({ + host = "${azurerm_mysql_server.db.0.name}.mysql.database.azure.com" + password = azurerm_mysql_server.db.0.administrator_login_password + port = 3306 + username = "${azurerm_mysql_server.db.0.administrator_login}@${azurerm_mysql_server.db.0.name}" + }, {}) +} + +output "domain_nameservers" { + value = try(azurerm_dns_zone.dns.0.name_servers, null) +} + +output "external_dns_secrets" { + value = {} +} + +output "external_dns_settings" { + value = { + provider = "azure" + "azure.resourceGroup" = azurerm_resource_group.gitpod.name + "azure.subscriptionId" = data.azurerm_client_config.current.subscription_id + "azure.tenantId" = data.azurerm_client_config.current.tenant_id + "azure.useManagedIdentityExtension" = true + "azure.userAssignedIdentityID" = azurerm_kubernetes_cluster.k8s.kubelet_identity.0.client_id + } +} + +output "k8s_connection" { + sensitive = true + value = { + host = azurerm_kubernetes_cluster.k8s.kube_config.0.host + username = azurerm_kubernetes_cluster.k8s.kube_config.0.username + password = azurerm_kubernetes_cluster.k8s.kube_config.0.password + client_certificate = base64decode(azurerm_kubernetes_cluster.k8s.kube_config.0.client_certificate) + client_key = base64decode(azurerm_kubernetes_cluster.k8s.kube_config.0.client_key) + cluster_ca_certificate = base64decode(azurerm_kubernetes_cluster.k8s.kube_config.0.cluster_ca_certificate) + } +} + +output "kubeconfig" { + sensitive = true + value = azurerm_kubernetes_cluster.k8s.kube_config_raw +} + +output "region" { + value = var.location +} + +output "registry" { + sensitive = true + value = try({ + server = azurerm_container_registry.registry.0.login_server + password = azurerm_container_registry.registry.0.admin_password + username = azurerm_container_registry.registry.0.admin_username + }, {}) +} + +output "storage" { + sensitive = true + value = try({ + username = azurerm_storage_account.storage.0.name + password = azurerm_storage_account.storage.0.primary_access_key + }, {}) +} diff --git a/install/infra/terraform/aks/registry.tf b/install/infra/terraform/aks/registry.tf new file mode 100644 index 00000000000000..87ec560c6b5441 --- /dev/null +++ b/install/infra/terraform/aks/registry.tf @@ -0,0 +1,25 @@ +resource "random_integer" "registry" { + count = var.enable_external_registry ? 1 : 0 + + min = 10000 + max = 99999 +} + +resource "azurerm_container_registry" "registry" { + count = var.enable_external_registry ? 1 : 0 + + name = "gitpod${random_integer.registry[count.index].result}" + resource_group_name = azurerm_resource_group.gitpod.name + location = azurerm_resource_group.gitpod.location + admin_enabled = true + sku = "Premium" +} + +resource "azurerm_role_assignment" "registry" { + count = var.enable_external_registry ? 1 : 0 + + principal_id = azurerm_kubernetes_cluster.k8s.kubelet_identity[0].object_id + role_definition_name = "AcrPush" + scope = azurerm_container_registry.registry[count.index].id + skip_service_principal_aad_check = true +} diff --git a/install/infra/terraform/aks/storage.tf b/install/infra/terraform/aks/storage.tf new file mode 100644 index 00000000000000..ca4ef115dcf924 --- /dev/null +++ b/install/infra/terraform/aks/storage.tf @@ -0,0 +1,16 @@ +resource "random_integer" "storage" { + count = var.enable_external_storage ? 1 : 0 + + min = 10000 + max = 99999 +} + +resource "azurerm_storage_account" "storage" { + count = var.enable_external_storage ? 1 : 0 + + name = "gitpod${random_integer.storage[count.index].result}" + resource_group_name = azurerm_resource_group.gitpod.name + location = azurerm_resource_group.gitpod.location + account_tier = "Standard" + account_replication_type = "LRS" +} diff --git a/install/infra/terraform/aks/variables.tf b/install/infra/terraform/aks/variables.tf new file mode 100644 index 00000000000000..94dabc09e6f98d --- /dev/null +++ b/install/infra/terraform/aks/variables.tf @@ -0,0 +1,19 @@ +// Common variables +variable "kubeconfig" { + default = "./kubeconfig" + +} +variable "dns_enabled" {} +variable "domain_name" {} +variable "enable_airgapped" {} +variable "enable_external_database" {} +variable "enable_external_registry" {} +variable "enable_external_storage" {} +variable "workspace_name" { +} + +// Azure-specific variables +variable "location" { + default = "northeurope" + +} diff --git a/install/infra/terraform/tools/azure-external-dns/main.tf b/install/infra/terraform/tools/azure-external-dns/main.tf new file mode 100644 index 00000000000000..61c3017c53073a --- /dev/null +++ b/install/infra/terraform/tools/azure-external-dns/main.tf @@ -0,0 +1,61 @@ +variable settings {} +variable domain_name { default = "test"} +variable kubeconfig { default = "conf"} + +provider "helm" { + kubernetes { + config_path = var.kubeconfig + } +} + +# External DNS Deployment using Helm +resource "helm_release" "external_dns" { + name = "external-dns" + repository = "https://charts.bitnami.com" + chart = "external-dns" + namespace = "external-dns" + create_namespace = true + + set { + name = "domainFilters[0]" + value = var.domain_name + } + + set { + name = "provider" + value = "azure" + } + set { + name = "azure.userAssignedIdentityID" + value = var.settings["azure.userAssignedIdentityID"] + } + + set { + name = "azure.useManagedIdentityExtension" + value = var.settings["azure.useManagedIdentityExtension"] + } + + set { + name = "azure.tenantId" + value = var.settings["azure.tenantId"] + } + + set { + name = "azure.subscriptionId" + value = var.settings["azure.subscriptionId"] + } + + set { + name = "azure.resourceGroup" + value = var.settings["azure.resourceGroup"] + } + + # TODO Add tags using dynamic block + # https://github.com/hashicorp/terraform/issues/22340 + # dynamic "set" { + # for_each = var.tags + # iterator = "tag" + # name = "podLabels[${index(var.tags, tag.key)}]" + # value = tag.value + # } +} diff --git a/install/infra/terraform/tools/cert-manager/main.tf b/install/infra/terraform/tools/cert-manager/main.tf index 232e4026b9493e..c19f364cb6ccea 100644 --- a/install/infra/terraform/tools/cert-manager/main.tf +++ b/install/infra/terraform/tools/cert-manager/main.tf @@ -1,20 +1,9 @@ -provider "kubernetes" { - config_path = var.kubeconfig -} - provider "helm" { kubernetes { config_path = var.kubeconfig } } -#create namespace for cert mananger -resource "kubernetes_namespace" "cert" { - metadata { - name = "cert-manager" - } -} - variable "extraArgs" { description = "List of additional arguments for cert-manager" type = list(any) @@ -27,10 +16,11 @@ variable "extraArgs" { #deploy cert manager resource "helm_release" "cert" { name = "cert-manager" - namespace = kubernetes_namespace.cert.metadata[0].name + namespace = "cert-manager" + create_namespace = true repository = "https://charts.jetstack.io" chart = "cert-manager" - depends_on = [kubernetes_namespace.cert] + wait = true set { name = "version" value = "v1.8.0" @@ -44,6 +34,10 @@ resource "helm_release" "cert" { name = "extraArgs" value = "{${join(",", var.extraArgs)}}" } + + provisioner "local-exec" { + command = "echo 'Waiting for cert-manager validating webhook to get its CA injected, so we can start to apply custom resources ...' && sleep 60" + } } # the following is only for GCP managed DNS setup @@ -53,6 +47,10 @@ data local_file "gcp_credentials" { filename = var.credentials } +provider "kubernetes" { + config_path = var.kubeconfig +} + resource "kubernetes_secret" "dns_solver" { count = var.credentials == null ? 0 : 1 depends_on = [ diff --git a/install/infra/terraform/tools/cloud-dns-ns/main.tf b/install/infra/terraform/tools/cloud-dns-ns/main.tf new file mode 100644 index 00000000000000..65c2650909e31d --- /dev/null +++ b/install/infra/terraform/tools/cloud-dns-ns/main.tf @@ -0,0 +1,19 @@ +variable credentials {} +variable nameservers {} +variable domain_name {} +variable managed_dns_zone {} +variable dns_project {} + +provider "google" { + credentials = var.credentials +} + +resource "google_dns_record_set" "gitpod-dns-3" { + name = "${var.domain_name}." + managed_zone = var.managed_dns_zone + project = var.dns_project + type = "NS" + ttl = 5 + + rrdatas = var.nameservers +} diff --git a/install/infra/terraform/tools/issuer/azure/main.tf b/install/infra/terraform/tools/issuer/azure/main.tf new file mode 100644 index 00000000000000..d2ad4fd71ce4fe --- /dev/null +++ b/install/infra/terraform/tools/issuer/azure/main.tf @@ -0,0 +1,28 @@ +provider "kubernetes" { + config_path = var.kubeconfig +} + +resource "kubernetes_manifest" "clusterissuer_gitpod" { + manifest = { + "apiVersion" = "cert-manager.io/v1" + "kind" = "ClusterIssuer" + "metadata" = { + "name" = "gitpod-issuer" + } + "spec" = { + "acme" = { + "privateKeySecretRef" = { + "name" = "issuer-account-key" + } + "server" = "https://acme-v02.api.letsencrypt.org/directory" + "solvers" = [ + { + "dns01" = { + "azureDNS" = var.cert_manager_issuer + } + } + ] + } + } + } +} diff --git a/install/infra/terraform/tools/issuer/azure/variables.tf b/install/infra/terraform/tools/issuer/azure/variables.tf new file mode 100644 index 00000000000000..2960f621b2a55e --- /dev/null +++ b/install/infra/terraform/tools/issuer/azure/variables.tf @@ -0,0 +1,8 @@ +variable "kubeconfig" { + description = "Path to the KUBECONFIG file to connect to the cluster" + default = "./kubeconfig" +} + +variable "cert_manager_issuer" { + default = null +} diff --git a/install/tests/Makefile b/install/tests/Makefile index c6c8702f60c631..f49336d2815a96 100644 --- a/install/tests/Makefile +++ b/install/tests/Makefile @@ -28,6 +28,34 @@ gke-standard-cluster: terraform workspace new $(TF_VAR_TEST_ID) || terraform workspace select $(TF_VAR_TEST_ID) && \ terraform apply -target=module.gke -var kubeconfig=${KUBECONFIG} --auto-approve +.PHONY: +## aks-standard-cluster: Creates an AKS cluster +aks-standard-cluster: + terraform init --upgrade && \ + terraform workspace new $(TF_VAR_TEST_ID) || terraform workspace select $(TF_VAR_TEST_ID) && \ + terraform apply -target=module.aks -var kubeconfig=${KUBECONFIG} --auto-approve + +.PHONY: +## azure-external-dns: Sets up external-dns with azure provider +azure-external-dns: + terraform init --upgrade && \ + terraform workspace new $(TF_VAR_TEST_ID) || terraform workspace select $(TF_VAR_TEST_ID) && \ + terraform apply -target=module.azure-externaldns -var kubeconfig=${KUBECONFIG} --auto-approve + +.PHONY: +## add-ns-record: Adds NS record for subdomain under gitpod-selfhosted.com +add-ns-record: + terraform init --upgrade && \ + terraform workspace new $(TF_VAR_TEST_ID) || terraform workspace select $(TF_VAR_TEST_ID) && \ + terraform apply -target=module.add_gcp_nameservers -var kubeconfig=${KUBECONFIG} --auto-approve + +.PHONY: +## azure-issuer: Creates a cluster issuer with AD access +azure-issuer: + terraform init --upgrade && \ + terraform workspace new $(TF_VAR_TEST_ID) || terraform workspace select $(TF_VAR_TEST_ID) && \ + terraform apply -target=module.azure-issuer -var kubeconfig=${KUBECONFIG} --auto-approve + .PHONY: ## k3s-standard-cluster: Creates a K3S cluster on GCP with one master and 1 worker node k3s-standard-cluster: @@ -41,14 +69,14 @@ CLUSTER_ISSUER_CLOUD_DNS := "./manifests/gcp-issuer.yaml" ## cert-manager: Installs cert-manager, optionally create secret for cloud-dns access cert-manager: terraform workspace select $(TF_VAR_TEST_ID) && \ - terraform apply -target=module.certmanager -var kubeconfig=${KUBECONFIG} --auto-approve && \ - kubectl --kubeconfig=${KUBECONFIG} apply -f ${CLUSTER_ISSUER_CLOUD_DNS} + terraform apply -target=module.certmanager -var kubeconfig=${KUBECONFIG} --auto-approve .PHONY: ## managed-dns: Installs external-dns, and setup up CloudDNS access managed-dns: check-env-sub-domain terraform workspace select $(TF_VAR_TEST_ID) && \ - terraform apply -target=module.externaldns -var kubeconfig=${KUBECONFIG} --auto-approve + terraform apply -target=module.externaldns -var kubeconfig=${KUBECONFIG} --auto-approve && \ + kubectl --kubeconfig=${KUBECONFIG} apply -f ${CLUSTER_ISSUER_CLOUD_DNS} .PHONY: ## get-kubeconfig: Returns KUBECONFIG of a just created cluster @@ -87,7 +115,7 @@ kots-install: install-kots-cli --config-values tmp_config.yml delete-cm-setup: - sleep 120 && kubectl --kubeconfig=${KUBECONFIG} delete pods --all -n cert-manager && sleep 300; + sleep 300 && kubectl --kubeconfig=${KUBECONFIG} delete pods --all -n cert-manager && sleep 600; check-kots-app: kubectl kots get --kubeconfig=${KUBECONFIG} app gitpod -n gitpod | grep gitpod | awk '{print $$2}' | grep "ready" || { echo "Gitpod is not ready"; exit 1; } @@ -103,13 +131,35 @@ kots-upgrade: @echo "Upgrade gitpod KOTS app to latest" kubectl kots upstream upgrade --kubeconfig=${KUBECONFIG} gitpod -n gitpod --deploy -cleanup: +cleanup: destroy-gcp-externaldns destroy-gcpns destroy-aks-edns destroy-aks-issuer destroy-certmanager destroy-k3s destroy-gke destroy-aks + +select-workspace: terraform workspace select $(TF_VAR_TEST_ID) - which ${KUBECONFIG} && terraform destroy -target=module.externaldns -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" - which ${KUBECONFIG} && terraform destroy -target=module.certmanager -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" + +destroy-gcp-externaldns: select-workspace + ls ${KUBECONFIG} && terraform destroy -target=module.externaldns -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" + +destroy-certmanager: select-workspace + ls ${KUBECONFIG} && terraform destroy -target=module.certmanager -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" + +destroy-gcpns: select-workspace + ls ${KUBECONFIG} && terraform destroy -target=module.add_gcp_nameservers -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" + +destroy-aks-edns: select-workspace + ls ${KUBECONFIG} && terraform destroy -target=module.azure-externaldns -var kubeconfig=${KUBECONFIG} --auto-approve + +destroy-aks-issuer: select-workspace + ls ${KUBECONFIG} && terraform destroy -target=module.azure-issuer -var kubeconfig=${KUBECONFIG} --auto-approve + +destroy-gke: select-workspace terraform destroy -target=module.gke -var kubeconfig=${KUBECONFIG} --auto-approve + +destroy-k3s: select-workspace terraform destroy -target=module.k3s -var kubeconfig=${KUBECONFIG} --auto-approve +destroy-aks: select-workspace + terraform destroy -target=module.aks -var kubeconfig=${KUBECONFIG} --auto-approve + get-results: @echo "If you have gotten this far, it means your setup succeeded" @echo "The IP address of you setup is "https://$(TF_VAR_TEST_ID).gitpod-self-hosted.com"" diff --git a/install/tests/main.tf b/install/tests/main.tf index 8e2e4949b2d503..bc2b5e08a550bf 100644 --- a/install/tests/main.tf +++ b/install/tests/main.tf @@ -1,13 +1,7 @@ -variable "kubeconfig" {} -variable "TEST_ID" { - default = "nightly" -} -variable "project" { - default = "sh-automated-tests" -} -variable "sa_creds" {} -variable "dns_sa_creds" {} +variable "kubeconfig" { } +variable "TEST_ID" { default = "nightly" } +# We store the state always in a GCS bucket terraform { backend "gcs" { bucket = "nightly-tests" @@ -15,6 +9,10 @@ terraform { } } +variable "project" { default = "sh-automated-tests" } +variable "sa_creds" { default = null } +variable "dns_sa_creds" {default = null } + module "gke" { # source = "github.com/gitpod-io/gitpod//install/infra/terraform/gke?ref=main" # we can later use tags here source = "../infra/terraform/gke" # we can later use tags here @@ -41,6 +39,19 @@ module "k3s" { domain_name = "${var.TEST_ID}.gitpod-self-hosted.com" } +module "aks" { + # source = "github.com/gitpod-io/gitpod//install/infra/terraform/aks?ref=main" # we can later use tags here + source = "../infra/terraform/aks" + + domain_name = "${var.TEST_ID}.gitpod-self-hosted.com" + enable_airgapped = false + enable_external_database = false + enable_external_registry = false + enable_external_storage = false + dns_enabled = true + workspace_name = var.TEST_ID +} + module "certmanager" { # source = "github.com/gitpod-io/gitpod//install/infra/terraform/tools/cert-manager?ref=main" source = "../infra/terraform/tools/cert-manager" @@ -52,8 +63,30 @@ module "certmanager" { module "externaldns" { # source = "github.com/gitpod-io/gitpod//install/infra/terraform/tools/external-dns?ref=main" source = "../infra/terraform/tools/external-dns" - kubeconfig = var.kubeconfig credentials = var.dns_sa_creds txt_owner_id = var.TEST_ID } + +module "azure-externaldns" { + source = "../infra/terraform/tools/azure-external-dns" + kubeconfig = var.kubeconfig + settings = module.aks.external_dns_settings + domain_name = "${var.TEST_ID}.gitpod-self-hosted.com" +} + +module "azure-issuer" { + source = "../infra/terraform/tools/issuer/azure" + kubeconfig = var.kubeconfig + cert_manager_issuer = module.aks.cert_manager_issuer +} + +module "add_gcp_nameservers" { + # source = "github.com/gitpod-io/gitpod//install/infra/terraform/tools/cloud-dns-ns?ref=main" + source = "../infra/terraform/tools/cloud-dns-ns" + credentials = var.dns_sa_creds + nameservers = module.aks.domain_nameservers + dns_project = "dns-for-playgrounds" + managed_dns_zone = "gitpod-self-hosted-com" + domain_name = "${var.TEST_ID}.gitpod-self-hosted.com" +}