Skip to content

Commit

Permalink
[installer-tests] Add tests for k8s versions and ubuntu version
Browse files Browse the repository at this point in the history
  • Loading branch information
@nandajavarma @mustard-mh
nandajavarma authored and mustard-mh committed Aug 4, 2022
1 parent fbf2e6f commit 42eae5f
Show file tree
Hide file tree
Showing 12 changed files with 224 additions and 126 deletions.
81 changes: 60 additions & 21 deletions .werft/installer-tests.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,15 @@ interface TestConfig {
CLOUD: string;
}

const k8s_version: string = randK8sVersion(testConfig)
const os_version: string = randOsVersion() // applicable only for k3s

// Each of the TEST_CONFIGURATIONS define an integration test end-to-end
// It should be a combination of multiple INFRA_PHASES, order of PHASES slice is important
const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
STANDARD_GKE_TEST: {
CLOUD: "gcp",
DESCRIPTION: "Deploy Gitpod on GKE, with managed DNS, and run integration tests",
DESCRIPTION: `Deploy Gitpod on GKE(version ${k8s_version})`,
PHASES: [
"STANDARD_GKE_CLUSTER",
"CERT_MANAGER",
Expand All @@ -52,8 +55,7 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
STANDARD_K3S_TEST: {
CLOUD: "gcp", // the cloud provider is still GCP
DESCRIPTION:
"Deploy Gitpod on a K3s cluster, created on a GCP instance," +
" with managed DNS and run integrations tests",
`Deploy Gitpod on a K3s cluster(version ${k8s_version}), on a GCP instance with ubuntu ${os_version}`,
PHASES: [
"STANDARD_K3S_CLUSTER_ON_GCP",
"CERT_MANAGER",
Expand All @@ -65,7 +67,7 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
},
STANDARD_AKS_TEST: {
CLOUD: "azure",
DESCRIPTION: "Deploy Gitpod on AKS, with managed DNS, and run integration tests",
DESCRIPTION: `Deploy Gitpod on AKS(version ${k8s_version})`,
PHASES: [
"STANDARD_AKS_CLUSTER",
"CERT_MANAGER",
Expand All @@ -79,7 +81,7 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
},
STANDARD_EKS_TEST: {
CLOUD: "aws",
DESCRIPTION: "Create an EKS cluster",
DESCRIPTION: `Create an EKS cluster(version ${k8s_version})`,
PHASES: [
"STANDARD_EKS_CLUSTER",
"CERT_MANAGER",
Expand Down Expand Up @@ -110,23 +112,23 @@ const cloud: string = config.CLOUD;
const INFRA_PHASES: { [name: string]: InfraConfig } = {
STANDARD_GKE_CLUSTER: {
phase: "create-std-gke-cluster",
makeTarget: "gke-standard-cluster",
description: "Creating a GKE cluster with 1 nodepool each for workspace and server",
makeTarget: `gke-standard-cluster`,
description: `Creating a GCP GKE cluster(version: ${k8s_version}) with 1 nodepool each for workspace and server`,
},
STANDARD_K3S_CLUSTER_ON_GCP: {
phase: "create-std-k3s-cluster",
makeTarget: "k3s-standard-cluster",
description: "Creating a k3s cluster on GCP with 1 node",
makeTarget: `k3s-standard-cluster os_version=${os_version}`,
description: `Creating a k3s(version: ${k8s_version}) cluster on GCP with 1 node`,
},
STANDARD_AKS_CLUSTER: {
phase: "create-std-aks-cluster",
makeTarget: "aks-standard-cluster",
description: "Creating an aks cluster(azure)",
makeTarget: `aks-standard-cluster`,
description: `Creating an Azure AKS cluster(version: ${k8s_version})`,
},
STANDARD_EKS_CLUSTER: {
phase: "create-std-eks-cluster",
makeTarget: "eks-standard-cluster",
description: "Creating a EKS cluster with 1 nodepool each for workspace and server",
makeTarget: `eks-standard-cluster`,
description: `Creating a AWS EKS cluster(version: ${k8s_version}) with 1 nodepool each for workspace and server`,
},
CERT_MANAGER: {
phase: "setup-cert-manager",
Expand All @@ -140,7 +142,7 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = {
},
GENERATE_KOTS_CONFIG: {
phase: "generate-kots-config",
makeTarget: `generate-kots-config storage=${randomize()} registry=${randomize()} db=${randomize()}`,
makeTarget: `generate-kots-config storage=${randDeps()} registry=${randDeps()} db=${randDeps()}`,
description: `Generate KOTS Config file`,
},
CLUSTER_ISSUER: {
Expand Down Expand Up @@ -335,7 +337,7 @@ function callMakeTargets(phase: string, description: string, makeTarget: string,
werft.log(phase, `Calling ${makeTarget}`);

// exporting cloud env var is important for the make targets
const response = exec(`export cloud=${cloud} && make -C ${makefilePath} ${makeTarget}`, {
const response = exec(`export TF_VAR_cluster_version=${k8s_version} cloud=${cloud} && make -C ${makefilePath} ${makeTarget}`, {
slice: phase,
dontCheckRc: true,
});
Expand All @@ -347,24 +349,61 @@ function callMakeTargets(phase: string, description: string, makeTarget: string,
werft.fail(phase, "Operation failed");
return response.code;
}
werft.log(phase, `Phase failed`);
werft.log(phase, `'${description}' failed`);
} else {
werft.log(phase, `Phase succeeded`);
werft.log(phase, `'${description}' succeeded`);
werft.done(phase);
}

return response.code;
}

function randomize(): string {
// in the follow-up PR we will add `${platform}-${resource}` as an option here to
// test against resource dependencies(storage, db, registry) for each cloud platform
function randomize(options: string[]): string {
return options[Math.floor(Math.random() * options.length)];
}

function randDeps(): string {
var depOptions: string[] = ["incluster", "external"]

if(deps && depOptions.includes(deps)) {
return deps
}

return depOptions[Math.floor(Math.random() * depOptions.length)];
return randomize(depOptions)
}

function randK8sVersion(config: string): string {
var options: string[] = []
switch(config) {
case "STANDARD_GKE_TEST": {
options = ["1.21", "1.22", "1.23"]
break;
}
case "STANDARD_AKS_TEST": {
options = ["1.21", "1.22", "1.23"]
break;
}
case "STANDARD_EKS_TEST": {
options = ["1.20", "1.21", "1.22"]
break;
}
case "STANDARD_K3S_TEST": {
options = ["v1.22.12+k3s1", "v1.23.9+k3s1", "v1.24.3+k3s1"]
break;
}
}
// in the follow-up PR we will add `${platform}-${resource}` as an option here to
// test against resource dependencies(storage, db, registry) for each cloud platform

return randomize(options)
}

function randOsVersion(): string {
// in the follow-up PR we will add `${platform}-${resource}` as an option here to
// test against resource dependencies(storage, db, registry) for each cloud platform
var options: string[] = ["2204", "2004", "1804"]

return randomize(options)
}

function cleanup() {
Expand Down
2 changes: 1 addition & 1 deletion .werft/jobs/build/self-hosted-upgrade-tests.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ export async function triggerUpgradeTests(werft: Werft, config: JobConfig, usern

werft.phase(upgradeConfig.phase, upgradeConfig.description);

annotation = `${annotation} -a cluster=${phase}`
annotation = `${annotation} -a cluster=${phase} -a updateGitHubStatus=gitpod-io/gitpod`

const testFile: string = ".werft/self-hosted-installer-tests.yaml";

Expand Down
33 changes: 14 additions & 19 deletions install/infra/terraform/aks/kubernetes.tf
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,3 @@
data "azurerm_kubernetes_service_versions" "k8s" {
location = azurerm_resource_group.gitpod.location
include_preview = false
}

resource "azurerm_role_assignment" "k8s" {
count = var.dns_enabled ? 1 : 0

Expand All @@ -20,37 +15,37 @@ resource "azurerm_role_assignment" "k8s_reader" {
}

resource "azurerm_kubernetes_cluster" "k8s" {
name = format(local.name_format, local.location, "primary")
location = azurerm_resource_group.gitpod.location
resource_group_name = azurerm_resource_group.gitpod.name
dns_prefix = "gitpod"
tags = {}
api_server_authorized_ip_ranges = []

kubernetes_version = data.azurerm_kubernetes_service_versions.k8s.latest_version
name = format(local.name_format, local.location, "primary")
location = azurerm_resource_group.gitpod.location
resource_group_name = azurerm_resource_group.gitpod.name
dns_prefix = "gitpod"
tags = {}
api_server_authorized_ip_ranges = []

kubernetes_version = var.cluster_version
http_application_routing_enabled = false

default_node_pool {
name = local.nodes.0.name
vm_size = local.machine


node_taints = []
tags = {}
zones = []
node_taints = []
tags = {}
zones = []

enable_auto_scaling = true
min_count = 1
max_count = 10
orchestrator_version = data.azurerm_kubernetes_service_versions.k8s.latest_version
orchestrator_version = var.cluster_version
node_labels = local.nodes.0.labels

type = "VirtualMachineScaleSets"
vnet_subnet_id = azurerm_subnet.network.id
}

identity {
type = "SystemAssigned"
type = "SystemAssigned"
identity_ids = []
}

Expand All @@ -74,7 +69,7 @@ resource "azurerm_kubernetes_cluster_node_pool" "pools" {
enable_auto_scaling = true
min_count = 1
max_count = 10
orchestrator_version = data.azurerm_kubernetes_service_versions.k8s.latest_version
orchestrator_version = var.cluster_version
node_labels = local.nodes[count.index + 1].labels
vnet_subnet_id = azurerm_subnet.network.id
}
Expand Down
9 changes: 6 additions & 3 deletions install/infra/terraform/aks/variables.tf
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,12 @@
// Common variables
variable "kubeconfig" {
default = "./kubeconfig"
default = "./kubeconfig"
}

variable "cluster_version" {
description = "kubernetes version of to create the cluster with"
}

variable "dns_enabled" {}
variable "domain_name" {}
variable "enable_airgapped" {}
Expand All @@ -14,6 +18,5 @@ variable "workspace_name" {

// Azure-specific variables
variable "location" {
default = "northeurope"

default = "northeurope"
}
36 changes: 18 additions & 18 deletions install/infra/terraform/eks/kubernetes.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,30 +2,30 @@ module "vpc" {
source = "terraform-aws-modules/vpc/aws"
version = "3.12.0"

name = "vpc-${var.cluster_name}"
cidr = var.vpc_cidr
azs = var.vpc_availability_zones
private_subnets = [var.private_primary_subnet_cidr, var.private_secondary_subnet_cidr]
public_subnets = [var.public_primary_subnet_cidr, var.public_secondary_subnet_cidr, var.public_db_subnet_cidr_1, var.public_db_subnet_cidr_2]
enable_nat_gateway = true
name = "vpc-${var.cluster_name}"
cidr = var.vpc_cidr
azs = var.vpc_availability_zones
private_subnets = [var.private_primary_subnet_cidr, var.private_secondary_subnet_cidr]
public_subnets = [var.public_primary_subnet_cidr, var.public_secondary_subnet_cidr, var.public_db_subnet_cidr_1, var.public_db_subnet_cidr_2]
enable_nat_gateway = true
enable_dns_hostnames = true
}

resource "aws_security_group" "nodes" {
name = "nodes-sg-${var.cluster_name}"
name = "nodes-sg-${var.cluster_name}"
vpc_id = module.vpc.vpc_id

ingress {
from_port = 0
to_port = 0
protocol = "-1"
from_port = 0
to_port = 0
protocol = "-1"
cidr_blocks = ["0.0.0.0/0"]
}

egress {
from_port = 0
to_port = 0
protocol = "-1"
from_port = 0
to_port = 0
protocol = "-1"
cidr_blocks = ["0.0.0.0/0"]
}
}
Expand All @@ -34,10 +34,10 @@ module "eks" {
source = "terraform-aws-modules/eks/aws"
version = "18.8.1"

cluster_name = var.cluster_name
cluster_version = "1.22"
cluster_name = var.cluster_name
cluster_version = var.cluster_version

cluster_endpoint_public_access = true
cluster_endpoint_public_access = true

vpc_id = module.vpc.vpc_id
subnet_ids = module.vpc.public_subnets
Expand All @@ -58,7 +58,7 @@ module "eks" {
iam_role_attach_cni_policy = true
ami_id = var.image_id
enable_bootstrap_user_data = true
vpc_security_group_ids = [aws_security_group.nodes.id]
vpc_security_group_ids = [aws_security_group.nodes.id]
}

eks_managed_node_groups = {
Expand Down Expand Up @@ -144,7 +144,7 @@ module "vpc_cni_irsa" {
}

resource "null_resource" "kubeconfig" {
depends_on = [ module.eks ]
depends_on = [module.eks]
provisioner "local-exec" {
command = "aws eks update-kubeconfig --region ${var.region} --name ${var.cluster_name} --kubeconfig ${var.kubeconfig}"
}
Expand Down
21 changes: 14 additions & 7 deletions install/infra/terraform/eks/variables.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,38 +2,45 @@ variable "cluster_name" {
type = string
description = "EKS cluster name."
}

variable "cluster_version" {
type = string
description = "Kubernetes version to create the cluster with"
default = "1.22"
}

variable "kubeconfig" {
type = string
description = "Path to the kubeconfig file"
default = "kubeconfig"
}

variable "image_id" {
type = string
type = string
description = "AMI Image ID specific to the region"
// latest ubuntu image for 1.22 k8s for eu-west-1 region, refer https://cloud-images.ubuntu.com/docs/aws/eks/
default = "ami-0793b4124359a6ad7"
}

variable "service_machine_type" {
type = string
type = string
description = "Machine type for service workload node pool"
default = "m6i.xlarge"
default = "m6i.xlarge"
}

variable "workspace_machine_type" {
type = string
type = string
description = "Machine type for workspace workload node pool"
default = "m6i.2xlarge"
default = "m6i.2xlarge"
}

variable "region" {
type = string
type = string
default = "eu-west-1"
}

variable "vpc_availability_zones" {
type = list(string)
type = list(string)
default = ["eu-west-1c", "eu-west-1b"]
}

Expand Down
Loading

0 comments on commit 42eae5f

Please sign in to comment.