From e4e66358103841e12a08ca96bc5c81af035002df Mon Sep 17 00:00:00 2001 From: Shaunak Joshi Date: Tue, 29 Nov 2022 13:16:00 +0300 Subject: [PATCH] e2e test for MachinePool and AWSManagedMachinePool (#334) * Added missing configuration for MachinePool creation * Added docs for AWSMachinePool * Refactor docs * Adding common e2e manifest for MachineDeployment and MachinePool * Minor syntax fix * Removing extra spaces * Adding assert for MachinePool * Debugging for e2e tests * Adding env variables in manifest * Using envsubst to populate the e2e manifest * Resolving errors in e2e tests * Minor syntax fix * Docs changes --- docs/gen2_Tutorial.md | 137 ++++++++++++++++++-- pkg/basecluster/data.go | 8 ++ testing/capi-quickstart-e2e-template.yaml | 149 ++++++++++++++++++++++ testing/e2e/00-deploy/02-assert.yaml | 14 ++ testing/e2e_setup.sh | 21 +-- 5 files changed, 307 insertions(+), 22 deletions(-) create mode 100644 testing/capi-quickstart-e2e-template.yaml diff --git a/docs/gen2_Tutorial.md b/docs/gen2_Tutorial.md index 94ae93e0..37afb392 100644 --- a/docs/gen2_Tutorial.md +++ b/docs/gen2_Tutorial.md @@ -16,10 +16,10 @@ Note: The CAPA version used here is v2.0 and the manifests created here are in a Refer the [compatibility matrix for Cluster API provider and CAPA versions](https://github.com/kubernetes-sigs/cluster-api-provider-aws#compatibility-with-cluster-api-and-kubernetes-versions) for supported versions. -### MachineDeployment - Before deploying a EKS cluster, make sure to setup the AWS Environment as stated in the [quickstart giude for CAPI](https://cluster-api.sigs.k8s.io/user/quick-start.html) +### MachineDeployment + Here is an example of a manifest file that we can use to create a *base cluster*. This manifest file helps in deploying an EKS cluster with 'machine deployment' component from the cluster API (CAPI). This file has been generated by the following command @@ -101,7 +101,7 @@ spec: template: spec: iamInstanceProfile: nodes.cluster-api-provider-aws.sigs.k8s.io - instanceType: t2.medium + instanceType: {INSTANCE_TYPE} sshKeyName: {SSH_KEYNAME} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 @@ -115,9 +115,7 @@ spec: ### AWSManagedMachinePool -Before deploying a EKS managed node group cluster using AWSManagedMachinePool, make sure to setup the AWS Environment as described [here](https://cluster-api.sigs.k8s.io/user/quick-start.html). - -Then initialize the environment for AWSManagedMachinePool as stated [here](https://cluster-api-aws.sigs.k8s.io/topics/machinepools.html#awsmanagedmachinepool) +Initialize the environment for AWSManagedMachinePool as stated [here](https://cluster-api-aws.sigs.k8s.io/topics/machinepools.html#awsmanagedmachinepool) Before deploying an EKS cluster, make sure that the MachinePool feature gate is enabled. To do so, run this command: @@ -207,7 +205,118 @@ metadata: spec: {} ``` -### gen2 cluster creation using Arlon +### AWSMachinePool + +An AWSMachinePool corresponds to an AWS AutoScaling Groups, which provides the cloud provider specific resource for orchestrating a group of EC2 machines. + +Initialize the environment for AWSMachinePool as stated [here]() + +Before deploying an EKS cluster, make sure that the AWSMachinePool feature gate is enabled. To do so, run this command: + +```shell +kubectl describe deployment capa-controller-manager -n capa-system +``` + +In the output, in the feature gates section of the deployment, MachinePool must be set to true. + +```shell +> kubectl describe deployment capa-controller-manager -n capa-system +.......... +.......... +--featuregates=EKS=true,EKSEnableIAM=false,EKSAllowAddRoles=false,EKSFargate=true,MachinePool=true,EventBridgeInstanceState=false, +AutoControllerIdentityCreator=true,BootstrapFormatIgnition=false,ExternalResourceGC=false +.......... +.......... +``` + +This manifest file helps in deploying an EKS cluster with 'AWSManagedMachinePool' component from the cluster API (CAPI). This file has been generated by the following command + +```shell +clusterctl generate cluster awsmanaged-cluster --kubernetes-version v1.22.0 --flavor eks-machinepool > manifest.yaml +``` + +```yaml +# YAML +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: awsmanaged-cluster + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: AWSManagedControlPlane + name: awsmanaged-cluster-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSManagedCluster + name: awsmanaged-cluster +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSManagedCluster +metadata: + name: awsmanaged-cluster + namespace: default +spec: {} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: AWSManagedControlPlane +metadata: + name: awsmanaged-cluster-control-plane + namespace: default +spec: + region: {REGION} + sshKeyName: {SSH_KEYNAME} + version: v1.22.0 +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachinePool +metadata: + name: awsmanaged-cluster-mp-0 + namespace: default +spec: + clusterName: awsmanaged-cluster + replicas: 1 + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: EKSConfig + name: awsmanaged-cluster-mp-0 + clusterName: awsmanaged-cluster + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSMachinePool + name: awsmanaged-cluster-mp-0 + version: v1.22.0 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSMachinePool +metadata: + name: awsmanaged-cluster-mp-0 + namespace: default +spec: + awsLaunchTemplate: + iamInstanceProfile: nodes.cluster-api-provider-aws.sigs.k8s.io + instanceType: {INSTANCE_TYPE} + sshKeyName: {SSH_KEYNAME} + maxSize: 10 + minSize: 1 +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: EKSConfig +metadata: + name: awsmanaged-cluster-mp-0 + namespace: default +spec: {} +``` + +## gen2 cluster creation using Arlon This manifest file needs to be pushed to the workspace repository before the manifest directory is prepped and then validated. @@ -215,7 +324,7 @@ Before a manifest directory can be used as a base cluster, it must first be "pre by Arlon. The "prep" phase makes minor changes to the directory and manifest to help Arlon deploy multiple copies of the cluster without naming conflicts. -### manifest directory preparation +## manifest directory preparation To prepare a git directory to serve as base cluster, use the `basecluster preparegit` command: @@ -229,7 +338,7 @@ arlon basecluster preparegit --repo-path [--repo-revision revi arlon basecluster preparegit --repo-alias prod --repo-path [--repo-revision revision] ``` -### manifest directory validation +## manifest directory validation Post the successful preparation of the basecluster manifest directory using `basecluster preparegit`, the basecluster manifest directory needs to be validated before the basecluster is created. @@ -245,7 +354,7 @@ arlon basecluster validategit --repo-path [--repo-revision rev arlon basecluster validategit --repo-alias prod --repo-path [--repo-revision revision] ``` -### gen2 cluster creation +## gen2 cluster creation To create a gen2 workload cluster from the base cluster: @@ -259,7 +368,7 @@ arlon cluster create --cluster-name --repo-path arlon cluster create --cluster-name --repo-alias prod --repo-path [--output-yaml] [--profile ] [--repo-revision ] ``` -### gen2 cluster update +## gen2 cluster update To update the profiles of a gen2 workload cluster: @@ -276,7 +385,7 @@ to the existing cluster which will create profile app in argocd along with bundl An existing profile can be deleted from the cluster as well using the above command. Executing this command will delete the profile app and all the bundles associated with the profile in argocd. -### gen2 cluster deletion +## gen2 cluster deletion To destroy a gen2 workload cluster: @@ -288,7 +397,7 @@ Arlon creates between 2 and 3 ArgoCD application resources to compose a gen2 clu an optional profile is specified at cluster creation time). When you destroy a gen2 cluster, Arlon will find all related ArgoCD applications and clean them up. -### Known issues and limitations +## Known issues and limitations Gen2 clusters are powerful because the base cluster can be arbitrarily complex and feature rich. Since they are fairly new and still evolving, gen2 clusters have several known limitations relative to gen1. @@ -311,6 +420,6 @@ Examples of immutable properties: * Most fields of AWSMachineTemplate (instance type, labels, etc...) -### For more information +## For more information For more details on gen2 clusters, refer to the [design document](baseclusters.md). diff --git a/pkg/basecluster/data.go b/pkg/basecluster/data.go index 2681dc54..8471144a 100644 --- a/pkg/basecluster/data.go +++ b/pkg/basecluster/data.go @@ -67,6 +67,8 @@ nameReference: fieldSpecs: - path: spec/bootstrap/configRef/name kind: Machine + - path: spec/template/spec/bootstrap/configRef/name + kind: MachinePool - kind: AWSMachineTemplate group: infrastructure.cluster.x-k8s.io version: v1beta2 @@ -145,4 +147,10 @@ nameReference: fieldSpecs: - path: spec/template/spec/infrastructureRef/name kind: MachinePool +- kind: EKSConfig + group: bootstrap.cluster.x-k8s.io + version: v1beta2 + fieldSpecs: + - path: spec/template/spec/bootstrap/configRef/name + kind: MachinePool ` diff --git a/testing/capi-quickstart-e2e-template.yaml b/testing/capi-quickstart-e2e-template.yaml new file mode 100644 index 00000000..47d6e0cf --- /dev/null +++ b/testing/capi-quickstart-e2e-template.yaml @@ -0,0 +1,149 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: capi-quickstart + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: AWSManagedControlPlane + name: capi-quickstart-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSManagedCluster + name: capi-quickstart +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSManagedCluster +metadata: + name: capi-quickstart + namespace: default +spec: {} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: AWSManagedControlPlane +metadata: + name: capi-quickstart-control-plane + namespace: default +spec: + region: ${AWS_REGION} + sshKeyName: ${AWS_SSH_KEY_NAME} + version: v1.23.10 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSManagedMachinePool +metadata: + name: capi-quickstart-pool-0 + namespace: default +spec: + scaling: + minSize: 0 + maxSize: 10 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSMachinePool +metadata: + name: capi-quickstart-pool-1 + namespace: default +spec: + awsLaunchTemplate: + iamInstanceProfile: nodes.cluster-api-provider-aws.sigs.k8s.io + instanceType: ${AWS_NODE_MACHINE_TYPE} + sshKeyName: ${AWS_SSH_KEY_NAME} + maxSize: 10 + minSize: 1 +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachinePool +metadata: + name: capi-quickstart-pool-1 + namespace: default +spec: + clusterName: capi-quickstart + replicas: 1 + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: EKSConfig + name: capi-quickstart-pool-1 + clusterName: capi-quickstart + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSMachinePool + name: capi-quickstart-pool-1 + version: v1.23.10 +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachinePool +metadata: + name: capi-quickstart-pool-0 + namespace: default +spec: + clusterName: capi-quickstart + replicas: 1 + template: + spec: + bootstrap: + dataSecretName: "" + clusterName: capi-quickstart + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSManagedMachinePool + name: capi-quickstart-pool-0 +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: EKSConfig +metadata: + name: capi-quickstart-pool-1 + namespace: default +spec: {} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: capi-quickstart-md-0 + namespace: default +spec: + clusterName: capi-quickstart + replicas: 2 + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: EKSConfigTemplate + name: capi-quickstart-md-0 + clusterName: capi-quickstart + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSMachineTemplate + name: capi-quickstart-md-0 + version: v1.23.10 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSMachineTemplate +metadata: + name: capi-quickstart-md-0 + namespace: default +spec: + template: + spec: + iamInstanceProfile: nodes.cluster-api-provider-aws.sigs.k8s.io + instanceType: ${AWS_NODE_MACHINE_TYPE} + sshKeyName: ${AWS_SSH_KEY_NAME} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: EKSConfigTemplate +metadata: + name: capi-quickstart-md-0 + namespace: default +spec: + template: {} diff --git a/testing/e2e/00-deploy/02-assert.yaml b/testing/e2e/00-deploy/02-assert.yaml index c2e69f3d..fe96ade7 100644 --- a/testing/e2e/00-deploy/02-assert.yaml +++ b/testing/e2e/00-deploy/02-assert.yaml @@ -24,3 +24,17 @@ metadata: namespace: ec2-cluster status: ready: true +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSMachinePool +metadata: + namespace: ec2-cluster +status: + ready: true +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSManagedMachinePool +metadata: + namespace: ec2-cluster +status: + ready: true \ No newline at end of file diff --git a/testing/e2e_setup.sh b/testing/e2e_setup.sh index 3684c333..c98ed280 100755 --- a/testing/e2e_setup.sh +++ b/testing/e2e_setup.sh @@ -310,9 +310,21 @@ if ! which kubectl-kuttl &>/dev/null; then chmod +x "${HOME}/.local/bin/kubectl-kuttl" fi +if ! which envsubst &> /dev/null; then + echo Downloading envsubst plugin to run e2e tests + curl -Lo "${HOME}/.local/bin/envsubst" https://github.com/a8m/envsubst/releases/download/v1.2.0/envsubst-`uname -s`-`uname -m` + chmod +x "${HOME}/.local/bin/envsubst" +fi + +# Substituting the environment variables in e2e test manifest file +e2eTemplate=${arlon_repo}/testing/capi-quickstart-e2e-template.yaml +manifestfile=${arlon_repo}/testing/capi-quickstart-e2e-test.yaml +envsubst < ${e2eTemplate} > ${manifestfile} + # not needed for us... #clusterawsadm bootstrap iam create-cloudformation-stack - +echo "Enabling MachinePool Feature Gate for running e2e tests" +export EXP_MACHINE_POOL=true export AWS_B64ENCODED_CREDENTIALS=$(clusterawsadm bootstrap credentials encode-as-profile) clusterctl init --infrastructure aws --wait-providers @@ -322,13 +334,6 @@ echo "Run the following command to use kubectl, argocd, clusterctl, clusterawsad echo 'PATH=$PATH:$HOME/.local/bin' echo Installation successfull -clusterctl generate cluster capi-quickstart --flavor eks \ - --kubernetes-version v1.23.10 \ - --control-plane-machine-count=3 \ - --worker-machine-count=2 \ - --infrastructure aws \ - >${arlon_repo}/testing/capi-quickstart-e2e-test.yaml - repodir="${GIT_CLONE_ROOT}/myrepo" repopath=basecluster/ec2-cluster