diff --git a/cmd/clusterawsadm/api/bootstrap/v1alpha1/defaults.go b/cmd/clusterawsadm/api/bootstrap/v1alpha1/defaults.go index 5b624a7a2b..e65e05a632 100644 --- a/cmd/clusterawsadm/api/bootstrap/v1alpha1/defaults.go +++ b/cmd/clusterawsadm/api/bootstrap/v1alpha1/defaults.go @@ -58,6 +58,11 @@ func SetDefaults_AWSIAMConfigurationSpec(obj *AWSIAMConfigurationSpec) { //nolin } else if obj.EKS.Enable { obj.Nodes.EC2ContainerRegistryReadOnly = true } + if obj.EKS.ManagedMachinePool == nil { + obj.EKS.ManagedMachinePool = &AWSIAMRoleSpec{ + Disable: true, + } + } } // SetDefaults_AWSIAMConfiguration is used by defaulter-gen diff --git a/cmd/clusterawsadm/api/bootstrap/v1alpha1/types.go b/cmd/clusterawsadm/api/bootstrap/v1alpha1/types.go index e50d485405..10f5d5ba8b 100644 --- a/cmd/clusterawsadm/api/bootstrap/v1alpha1/types.go +++ b/cmd/clusterawsadm/api/bootstrap/v1alpha1/types.go @@ -102,6 +102,9 @@ type EKSConfig struct { // no role is included in the spec and automatic creation of the role // isn't enabled DefaultControlPlaneRole AWSIAMRoleSpec `json:"defaultControlPlaneRole,omitempty"` + // ManagedMachinePool controls the configuration of the AWS IAM role for + // used by EKS managed machine pools. + ManagedMachinePool *AWSIAMRoleSpec `json:"managedMachinePool,omitempty"` } // ClusterAPIControllers controls the configuration of the AWS IAM role for diff --git a/cmd/clusterawsadm/api/bootstrap/v1alpha1/zz_generated.deepcopy.go b/cmd/clusterawsadm/api/bootstrap/v1alpha1/zz_generated.deepcopy.go index 62f2ed804a..39b30da7e7 100644 --- a/cmd/clusterawsadm/api/bootstrap/v1alpha1/zz_generated.deepcopy.go +++ b/cmd/clusterawsadm/api/bootstrap/v1alpha1/zz_generated.deepcopy.go @@ -201,6 +201,11 @@ func (in *ControlPlane) DeepCopy() *ControlPlane { func (in *EKSConfig) DeepCopyInto(out *EKSConfig) { *out = *in in.DefaultControlPlaneRole.DeepCopyInto(&out.DefaultControlPlaneRole) + if in.ManagedMachinePool != nil { + in, out := &in.ManagedMachinePool, &out.ManagedMachinePool + *out = new(AWSIAMRoleSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EKSConfig. diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/cluster_api_controller.go b/cmd/clusterawsadm/cloudformation/bootstrap/cluster_api_controller.go index be008a9111..cb300525f0 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/cluster_api_controller.go +++ b/cmd/clusterawsadm/cloudformation/bootstrap/cluster_api_controller.go @@ -199,7 +199,7 @@ func (t Template) controllersPolicy() *iamv1.PolicyDocument { "iam:GetPolicy", }, Resource: iamv1.Resources{ - "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", + EKSClusterPolicy, }, Effect: iamv1.EffectAllow, }, { diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_default_roles.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_default_roles.yaml index df7dde669b..9581fe261a 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_default_roles.yaml +++ b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_default_roles.yaml @@ -304,6 +304,23 @@ Resources: - arn:aws:iam::aws:policy/AmazonEKSClusterPolicy RoleName: eks-controlplane.cluster-api-provider-aws.sigs.k8s.io Type: AWS::IAM::Role + AWSIAMRoleEKSNodegroup: + Properties: + AssumeRolePolicyDocument: + Statement: + - Action: + - sts:AssumeRole + Effect: Allow + Principal: + Service: + - eks.amazonaws.com + Version: 2012-10-17 + ManagedPolicyArns: + - arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy + - arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy + - arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly + RoleName: eks-nodegroup.cluster-api-provider-aws.sigs.k8s.io + Type: AWS::IAM::Role AWSIAMRoleNodes: Properties: AssumeRolePolicyDocument: diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/managed_nodegroup.go b/cmd/clusterawsadm/cloudformation/bootstrap/managed_nodegroup.go new file mode 100644 index 0000000000..7ab4387c75 --- /dev/null +++ b/cmd/clusterawsadm/cloudformation/bootstrap/managed_nodegroup.go @@ -0,0 +1,30 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/eks" + +func (t Template) eksMachinePoolPolicies() []string { + policies := eks.NodegroupRolePolicies() + if t.Spec.EKS.ManagedMachinePool.ExtraPolicyAttachments != nil { + for _, policy := range t.Spec.EKS.ManagedMachinePool.ExtraPolicyAttachments { + policies = append(policies, policy) + } + } + + return policies +} diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/template.go b/cmd/clusterawsadm/cloudformation/bootstrap/template.go index e1be2bd4d6..f7f02a361c 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/template.go +++ b/cmd/clusterawsadm/cloudformation/bootstrap/template.go @@ -26,6 +26,7 @@ import ( iamv1 "sigs.k8s.io/cluster-api-provider-aws/cmd/clusterawsadm/api/iam/v1alpha1" "sigs.k8s.io/cluster-api-provider-aws/cmd/clusterawsadm/converters" ekscontrolplanev1 "sigs.k8s.io/cluster-api-provider-aws/controlplane/eks/api/v1alpha3" + infrav1exp "sigs.k8s.io/cluster-api-provider-aws/exp/api/v1alpha3" ) const ( @@ -37,6 +38,7 @@ const ( AWSIAMRoleControlPlane = "AWSIAMRoleControlPlane" AWSIAMRoleNodes = "AWSIAMRoleNodes" AWSIAMRoleEKSControlPlane = "AWSIAMRoleEKSControlPlane" + AWSIAMRoleEKSNodegroup = "AWSIAMRoleEKSNodegroup" AWSIAMUserBootstrapper = "AWSIAMUserBootstrapper" ControllersPolicy PolicyName = "AWSIAMManagedPolicyControllers" ControlPlanePolicy PolicyName = "AWSIAMManagedPolicyCloudProviderControlPlane" @@ -167,6 +169,15 @@ func (t Template) RenderCloudFormation() *cloudformation.Template { } } + if !t.Spec.EKS.ManagedMachinePool.Disable { + template.Resources[AWSIAMRoleEKSNodegroup] = &cfn_iam.Role{ + RoleName: infrav1exp.DefaultEKSNodegroupRole, + AssumeRolePolicyDocument: eksAssumeRolePolicy(), + ManagedPolicyArns: t.eksMachinePoolPolicies(), + Tags: converters.MapToCloudFormationTags(t.Spec.EKS.ManagedMachinePool.Tags), + } + } + return template } diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/template_test.go b/cmd/clusterawsadm/cloudformation/bootstrap/template_test.go index ae61820b6a..86c6d68409 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/template_test.go +++ b/cmd/clusterawsadm/cloudformation/bootstrap/template_test.go @@ -80,6 +80,7 @@ func Test_RenderCloudformation(t *testing.T) { t.Spec.EKS.Enable = true t.Spec.Nodes.EC2ContainerRegistryReadOnly = true t.Spec.EKS.DefaultControlPlaneRole.Disable = false + t.Spec.EKS.ManagedMachinePool.Disable = false return t }, }, diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmanagedmachinepools.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmanagedmachinepools.yaml new file mode 100644 index 0000000000..72db159e6b --- /dev/null +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmanagedmachinepools.yaml @@ -0,0 +1,173 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.9 + creationTimestamp: null + name: awsmanagedmachinepools.infrastructure.cluster.x-k8s.io +spec: + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AWSManagedMachinePool + listKind: AWSManagedMachinePoolList + plural: awsmanagedmachinepools + singular: awsmanagedmachinepool + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: MachinePool ready status + jsonPath: .status.ready + name: Ready + type: string + - description: Number of replicas + jsonPath: .status.replicas + name: Replicas + type: integer + name: v1alpha3 + schema: + openAPIV3Schema: + description: AWSManagedMachinePool is the Schema for the awsmanagedmachinepools API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AWSManagedMachinePoolSpec defines the desired state of AWSManagedMachinePool + properties: + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to AWS resources managed by the AWS provider, in addition to the ones added by default. + type: object + amiType: + default: AL2_x86_64 + description: AMIType defines the AMI type + enum: + - AL2_x86_64 + - AL2_x86_64_GPU + - AL2_ARM_64 + type: string + amiVersion: + description: AMIVersion defines the desired AMI release version. If no version number is supplied then the latest version for the Kubernetes version will be used + minLength: 2 + type: string + diskSize: + description: DiskSize specifies the root disk size + format: int32 + type: integer + eksNodegroupName: + description: EKSNodegroupName specifies the name of the nodegroup in AWS corresponding to this MachinePool. If you don't specify a name then a default name will be created based on the namespace and name of the managed machine pool. + type: string + instanceType: + description: InstanceType specifies the AWS instance type + type: string + labels: + additionalProperties: + type: string + description: Labels specifies labels for the Kubernetes node objects + type: object + providerIDList: + description: ProviderIDList are the provider IDs of instances in the autoscaling group corresponding to the nodegroup represented by this machine pool + items: + type: string + type: array + remoteAccess: + description: RemoteAccess specifies how machines can be accessed remotely + properties: + sourceSecurityGroups: + description: SourceSecurityGroups specifies which security groups are allowed access An empty array opens port 22 to the public internet + items: + type: string + type: array + sshKeyName: + description: SSHKeyName specifies which EC2 SSH key can be used to access machines + type: string + type: object + roleName: + description: RoleName specifies the name of IAM role for the node group. If the role is pre-existing we will treat it as unmanaged and not delete it on deletion. If the EKSEnableIAM feature flag is true and no name is supplied then a role is created. + type: string + scaling: + description: Scaling specifies scaling for the ASG behind this pool + properties: + maxSize: + format: int32 + type: integer + minSize: + format: int32 + type: integer + type: object + subnetIDs: + description: SubnetIDs specifies which subnets are used for the auto scaling group of this nodegroup + items: + type: string + type: array + type: object + status: + description: AWSManagedMachinePoolStatus defines the observed state of AWSManagedMachinePool + properties: + conditions: + description: Conditions defines current service state of the managed machine pool + items: + description: Condition defines an observation of a Cluster API resource operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition in CamelCase. The specific API may choose whether or not this field is considered a guaranteed API. This field may not be empty. + type: string + severity: + description: Severity provides an explicit classification of Reason code, so the users or machines can immediately understand the current situation and act accordingly. The Severity field MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. + type: string + required: + - status + - type + type: object + type: array + failureMessage: + description: "FailureMessage will be set in the event that there is a terminal problem reconciling the MachinePool and will contain a more verbose string suitable for logging and human consumption. \n This field should not be set for transitive errors that a controller faces that are expected to be fixed automatically over time (like service outages), but instead indicate that something is fundamentally wrong with the MachinePool's spec or the configuration of the controller, and that manual intervention is required. Examples of terminal errors would be invalid combinations of settings in the spec, values that are unsupported by the controller, or the responsible controller itself being critically misconfigured. \n Any transient errors that occur during the reconciliation of MachinePools can be added as events to the MachinePool object and/or logged in the controller's output." + type: string + failureReason: + description: "FailureReason will be set in the event that there is a terminal problem reconciling the MachinePool and will contain a succinct value suitable for machine interpretation. \n This field should not be set for transitive errors that a controller faces that are expected to be fixed automatically over time (like service outages), but instead indicate that something is fundamentally wrong with the Machine's spec or the configuration of the controller, and that manual intervention is required. Examples of terminal errors would be invalid combinations of settings in the spec, values that are unsupported by the controller, or the responsible controller itself being critically misconfigured. \n Any transient errors that occur during the reconciliation of MachinePools can be added as events to the MachinePool object and/or logged in the controller's output." + type: string + ready: + default: false + description: Ready denotes that the AWSManagedMachinePool nodegroup has joined the cluster + type: boolean + replicas: + description: Replicas is the most recently observed number of replicas. + format: int32 + type: integer + required: + - ready + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 5b09b484fe..67465d6107 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -10,6 +10,7 @@ resources: - bases/infrastructure.cluster.x-k8s.io_awsmachinetemplates.yaml - bases/infrastructure.cluster.x-k8s.io_awsmanagedclusters.yaml - bases/infrastructure.cluster.x-k8s.io_awsmachinepools.yaml +- bases/infrastructure.cluster.x-k8s.io_awsmanagedmachinepools.yaml # +kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index e9cfd7a4f2..6192cef164 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -19,7 +19,7 @@ spec: containers: - args: - --enable-leader-election - - "--feature-gates=EKS=${EXP_EKS:=false}" + - "--feature-gates=EKS=${EXP_EKS:=false},EKSEnableIAM=${EXP_EKS_IAM:=false},EKSAllowAddRoles=${EXP_EKS_ADD_ROLES:=false}" image: controller:latest imagePullPolicy: Always name: manager diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 55310f2f15..fe1ba87c93 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -56,6 +56,16 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - get + - list + - patch + - watch - apiGroups: - exp.cluster.x-k8s.io resources: @@ -145,3 +155,23 @@ rules: - get - patch - update +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - awsmanagedmachinepools + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - awsmanagedmachinepools/status + verbs: + - get + - patch + - update diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index 06376cb7d7..d08dfa1a8c 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -26,6 +26,26 @@ webhooks: resources: - awsclusters sideEffects: None +- clientConfig: + caBundle: Cg== + service: + name: webhook-service + namespace: system + path: /mutate-infrastructure-cluster-x-k8s-io-v1alpha3-awsmanagedmachinepool + failurePolicy: Fail + matchPolicy: Equivalent + name: default.awsmanagedmachinepool.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1alpha3 + operations: + - CREATE + - UPDATE + resources: + - awsmanagedmachinepools + sideEffects: None --- apiVersion: admissionregistration.k8s.io/v1beta1 @@ -94,3 +114,23 @@ webhooks: resources: - awsmachinetemplates sideEffects: None +- clientConfig: + caBundle: Cg== + service: + name: webhook-service + namespace: system + path: /validate-infrastructure-cluster-x-k8s-io-v1alpha3-awsmanagedmachinepool + failurePolicy: Fail + matchPolicy: Equivalent + name: validation.awsmanagedmachinepool.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1alpha3 + operations: + - CREATE + - UPDATE + resources: + - awsmanagedmachinepools + sideEffects: None diff --git a/controlplane/eks/api/v1alpha3/awsmanagedcontrolplane_webhook.go b/controlplane/eks/api/v1alpha3/awsmanagedcontrolplane_webhook.go index 4c30f86903..27b9fca2ec 100644 --- a/controlplane/eks/api/v1alpha3/awsmanagedcontrolplane_webhook.go +++ b/controlplane/eks/api/v1alpha3/awsmanagedcontrolplane_webhook.go @@ -18,7 +18,6 @@ package v1alpha3 import ( "fmt" - "strings" "github.com/pkg/errors" @@ -32,14 +31,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook" infrav1 "sigs.k8s.io/cluster-api-provider-aws/api/v1alpha3" - "sigs.k8s.io/cluster-api-provider-aws/pkg/hash" -) - -const ( - // maxCharsName maximum number of characters for the name - maxCharsName = 100 - - clusterPrefix = "capa_" + "sigs.k8s.io/cluster-api-provider-aws/pkg/eks" ) // log is for logging in this package. @@ -188,7 +180,7 @@ func (r *AWSManagedControlPlane) Default() { if r.Spec.EKSClusterName == "" { mcpLog.Info("EKSClusterName is empty, generating name") - name, err := generateEKSName(r.Name, r.Namespace) + name, err := eks.GenerateEKSName(r.Name, r.Namespace) if err != nil { mcpLog.Error(err, "failed to create EKS cluster name") return @@ -211,21 +203,3 @@ func (r *AWSManagedControlPlane) Default() { infrav1.SetDefaults_Bastion(&r.Spec.Bastion) infrav1.SetDefaults_NetworkSpec(&r.Spec.NetworkSpec) } - -// generateEKSName generates a name of the EKS cluster -func generateEKSName(clusterName, namespace string) (string, error) { - escapedName := strings.Replace(clusterName, ".", "_", -1) - eksName := fmt.Sprintf("%s_%s", namespace, escapedName) - - if len(eksName) < maxCharsName { - return eksName, nil - } - - hashLength := 32 - len(clusterPrefix) - hashedName, err := hash.Base36TruncatedHash(eksName, hashLength) - if err != nil { - return "", fmt.Errorf("creating hash from cluster name: %w", err) - } - - return fmt.Sprintf("%s%s", clusterPrefix, hashedName), nil -} diff --git a/exp/api/v1alpha3/awsmanagedmachinepool_types.go b/exp/api/v1alpha3/awsmanagedmachinepool_types.go new file mode 100644 index 0000000000..f39518b4ac --- /dev/null +++ b/exp/api/v1alpha3/awsmanagedmachinepool_types.go @@ -0,0 +1,224 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha3 + +import ( + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + infrav1 "sigs.k8s.io/cluster-api-provider-aws/api/v1alpha3" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" + "sigs.k8s.io/cluster-api/errors" +) + +const ( + // ManagedMachinePoolFinalizer allows the controller to clean up resources on delete + ManagedMachinePoolFinalizer = "awsmanagedmachinepools.infrastructure.cluster.x-k8s.io" +) + +// ManagedMachineAMIType specifies which AWS AMI to use for a managed MachinePool +type ManagedMachineAMIType string + +const ( + // Al2x86_64 is the default AMI type + Al2x86_64 ManagedMachineAMIType = "AL2_x86_64" + // Al2x86_64GPU is the x86-64 GPU AMI type + Al2x86_64GPU ManagedMachineAMIType = "AL2_x86_64_GPU" + // Al2Arm64 is the Arm AMI type + Al2Arm64 ManagedMachineAMIType = "AL2_ARM_64" +) + +var ( + // DefaultEKSNodegroupRole is the name of the default IAM role to use for EKS nodegroups + // if no other role is supplied in the spec and if iam role creation is not enabled. The default + // can be created using clusterawsadm or created manually + DefaultEKSNodegroupRole = fmt.Sprintf("eks-nodegroup%s", infrav1.DefaultNameSuffix) +) + +// AWSManagedMachinePoolSpec defines the desired state of AWSManagedMachinePool +type AWSManagedMachinePoolSpec struct { + // EKSNodegroupName specifies the name of the nodegroup in AWS + // corresponding to this MachinePool. If you don't specify a name + // then a default name will be created based on the namespace and + // name of the managed machine pool. + // +optional + EKSNodegroupName string `json:"eksNodegroupName,omitempty"` + + // SubnetIDs specifies which subnets are used for the + // auto scaling group of this nodegroup + // +optional + SubnetIDs []string `json:"subnetIDs,omitempty"` + + // AdditionalTags is an optional set of tags to add to AWS resources managed by the AWS provider, in addition to the + // ones added by default. + // +optional + AdditionalTags infrav1.Tags `json:"additionalTags,omitempty"` + + // RoleName specifies the name of IAM role for the node group. + // If the role is pre-existing we will treat it as unmanaged + // and not delete it on deletion. If the EKSEnableIAM feature + // flag is true and no name is supplied then a role is created. + // +optional + RoleName string `json:"roleName,omitempty"` + + // AMIVersion defines the desired AMI release version. If no version number + // is supplied then the latest version for the Kubernetes version + // will be used + // +kubebuilder:validation:MinLength:=2 + // +optional + AMIVersion *string `json:"amiVersion,omitempty"` + + // AMIType defines the AMI type + // +kubebuilder:validation:Enum:=AL2_x86_64;AL2_x86_64_GPU;AL2_ARM_64 + // +kubebuilder:default:=AL2_x86_64 + // +optional + AMIType *ManagedMachineAMIType `json:"amiType,omitempty"` + + // Labels specifies labels for the Kubernetes node objects + // +optional + Labels map[string]string `json:"labels,omitempty"` + + // DiskSize specifies the root disk size + // +optional + DiskSize *int32 `json:"diskSize,omitempty"` + + // InstanceType specifies the AWS instance type + // +optional + InstanceType *string `json:"instanceType,omitempty"` + + // Scaling specifies scaling for the ASG behind this pool + // +optional + Scaling *ManagedMachinePoolScaling `json:"scaling,omitempty"` + + // RemoteAccess specifies how machines can be accessed remotely + // +optional + RemoteAccess *ManagedRemoteAccess `json:"remoteAccess,omitempty"` + + // ProviderIDList are the provider IDs of instances in the + // autoscaling group corresponding to the nodegroup represented by this + // machine pool + // +optional + ProviderIDList []string `json:"providerIDList,omitempty"` +} + +// ManagedMachinePoolScaling specifies scaling options +type ManagedMachinePoolScaling struct { + MinSize *int32 `json:"minSize,omitempty"` + MaxSize *int32 `json:"maxSize,omitempty"` +} + +// ManagedRemoteAccess specifies remote access settings for EC2 instances +type ManagedRemoteAccess struct { + // SSHKeyName specifies which EC2 SSH key can be used to access machines + SSHKeyName *string `json:"sshKeyName,omitempty"` + + // SourceSecurityGroups specifies which security groups are allowed access + // An empty array opens port 22 to the public internet + SourceSecurityGroups []string `json:"sourceSecurityGroups,omitempty"` +} + +// AWSManagedMachinePoolStatus defines the observed state of AWSManagedMachinePool +type AWSManagedMachinePoolStatus struct { + // Ready denotes that the AWSManagedMachinePool nodegroup has joined + // the cluster + // +kubebuilder:default=false + Ready bool `json:"ready"` + + // Replicas is the most recently observed number of replicas. + // +optional + Replicas int32 `json:"replicas"` + + // FailureReason will be set in the event that there is a terminal problem + // reconciling the MachinePool and will contain a succinct value suitable + // for machine interpretation. + // + // This field should not be set for transitive errors that a controller + // faces that are expected to be fixed automatically over + // time (like service outages), but instead indicate that something is + // fundamentally wrong with the Machine's spec or the configuration of + // the controller, and that manual intervention is required. Examples + // of terminal errors would be invalid combinations of settings in the + // spec, values that are unsupported by the controller, or the + // responsible controller itself being critically misconfigured. + // + // Any transient errors that occur during the reconciliation of MachinePools + // can be added as events to the MachinePool object and/or logged in the + // controller's output. + // +optional + FailureReason *errors.MachineStatusError `json:"failureReason,omitempty"` + + // FailureMessage will be set in the event that there is a terminal problem + // reconciling the MachinePool and will contain a more verbose string suitable + // for logging and human consumption. + // + // This field should not be set for transitive errors that a controller + // faces that are expected to be fixed automatically over + // time (like service outages), but instead indicate that something is + // fundamentally wrong with the MachinePool's spec or the configuration of + // the controller, and that manual intervention is required. Examples + // of terminal errors would be invalid combinations of settings in the + // spec, values that are unsupported by the controller, or the + // responsible controller itself being critically misconfigured. + // + // Any transient errors that occur during the reconciliation of MachinePools + // can be added as events to the MachinePool object and/or logged in the + // controller's output. + // +optional + FailureMessage *string `json:"failureMessage,omitempty"` + + // Conditions defines current service state of the managed machine pool + // +optional + Conditions clusterv1.Conditions `json:"conditions,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:path=awsmanagedmachinepools,scope=Namespaced,categories=cluster-api +// +kubebuilder:storageversion +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.ready",description="MachinePool ready status" +// +kubebuilder:printcolumn:name="Replicas",type="integer",JSONPath=".status.replicas",description="Number of replicas" + +// AWSManagedMachinePool is the Schema for the awsmanagedmachinepools API +type AWSManagedMachinePool struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec AWSManagedMachinePoolSpec `json:"spec,omitempty"` + Status AWSManagedMachinePoolStatus `json:"status,omitempty"` +} + +func (r *AWSManagedMachinePool) GetConditions() clusterv1.Conditions { + return r.Status.Conditions +} + +func (r *AWSManagedMachinePool) SetConditions(conditions clusterv1.Conditions) { + r.Status.Conditions = conditions +} + +// +kubebuilder:object:root=true + +// AWSManagedMachinePoolList contains a list of AWSManagedMachinePools +type AWSManagedMachinePoolList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []AWSManagedMachinePool `json:"items"` +} + +func init() { + SchemeBuilder.Register(&AWSManagedMachinePool{}, &AWSManagedMachinePoolList{}) +} diff --git a/exp/api/v1alpha3/awsmanagedmachinepool_webhook.go b/exp/api/v1alpha3/awsmanagedmachinepool_webhook.go new file mode 100644 index 0000000000..e146901baf --- /dev/null +++ b/exp/api/v1alpha3/awsmanagedmachinepool_webhook.go @@ -0,0 +1,181 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha3 + +import ( + "fmt" + "reflect" + + "github.com/pkg/errors" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation/field" + + ctrl "sigs.k8s.io/controller-runtime" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + "sigs.k8s.io/cluster-api-provider-aws/pkg/eks" +) + +// log is for logging in this package. +var mmpLog = logf.Log.WithName("awsmanagedmachinepool-resource") + +// SetupWebhookWithManager will setup the webhooks for the AWSManagedMachinePool +func (r *AWSManagedMachinePool) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(r). + Complete() +} + +// +kubebuilder:webhook:verbs=create;update,path=/validate-infrastructure-cluster-x-k8s-io-v1alpha3-awsmanagedmachinepool,mutating=false,failurePolicy=fail,matchPolicy=Equivalent,groups=infrastructure.cluster.x-k8s.io,resources=awsmanagedmachinepools,versions=v1alpha3,name=validation.awsmanagedmachinepool.infrastructure.cluster.x-k8s.io,sideEffects=None +// +kubebuilder:webhook:verbs=create;update,path=/mutate-infrastructure-cluster-x-k8s-io-v1alpha3-awsmanagedmachinepool,mutating=true,failurePolicy=fail,matchPolicy=Equivalent,groups=infrastructure.cluster.x-k8s.io,resources=awsmanagedmachinepools,versions=v1alpha3,name=default.awsmanagedmachinepool.infrastructure.cluster.x-k8s.io,sideEffects=None + +var _ webhook.Defaulter = &AWSManagedMachinePool{} +var _ webhook.Validator = &AWSManagedMachinePool{} + +func (r *AWSManagedMachinePool) validateScaling() field.ErrorList { + var allErrs field.ErrorList + if r.Spec.Scaling != nil { // nolint:nestif + minField := field.NewPath("spec", "scaling", "minSize") + maxField := field.NewPath("spec", "scaling", "maxSize") + min := r.Spec.Scaling.MinSize + max := r.Spec.Scaling.MaxSize + if min != nil { + if *min < 0 { + allErrs = append(allErrs, field.Invalid(minField, *min, "must be greater than zero")) + } + if max != nil && *max < *min { + allErrs = append(allErrs, field.Invalid(maxField, *max, fmt.Sprintf("must be greater than field %s", minField.String()))) + } + } + if max != nil && *max < 0 { + allErrs = append(allErrs, field.Invalid(maxField, *max, "must be greater than zero")) + } + } + if len(allErrs) == 0 { + return nil + } + return allErrs +} + +// ValidateCreate will do any extra validation when creating a AWSManagedMachinePool +func (r *AWSManagedMachinePool) ValidateCreate() error { + mmpLog.Info("AWSManagedMachinePool validate create", "name", r.Name) + + var allErrs field.ErrorList + + if r.Spec.EKSNodegroupName == "" { + allErrs = append(allErrs, field.Required(field.NewPath("spec.eksNodegroupName"), "eksNodegroupName is required")) + } + if errs := r.validateScaling(); errs != nil || len(errs) == 0 { + allErrs = append(allErrs, errs...) + } + + if len(allErrs) == 0 { + return nil + } + + return apierrors.NewInvalid( + r.GroupVersionKind().GroupKind(), + r.Name, + allErrs, + ) +} + +// ValidateUpdate will do any extra validation when updating a AWSManagedMachinePool +func (r *AWSManagedMachinePool) ValidateUpdate(old runtime.Object) error { + mmpLog.Info("AWSManagedMachinePool validate update", "name", r.Name) + oldPool, ok := old.(*AWSManagedMachinePool) + if !ok { + return apierrors.NewInvalid(GroupVersion.WithKind("AWSManagedMachinePool").GroupKind(), r.Name, field.ErrorList{ + field.InternalError(nil, errors.New("failed to convert old AWSManagedMachinePool to object")), + }) + } + + var allErrs field.ErrorList + allErrs = append(allErrs, r.validateImmutable(oldPool)...) + + if errs := r.validateScaling(); errs != nil || len(errs) == 0 { + allErrs = append(allErrs, errs...) + } + + if len(allErrs) == 0 { + return nil + } + + return apierrors.NewInvalid( + r.GroupVersionKind().GroupKind(), + r.Name, + allErrs, + ) +} + +// ValidateDelete allows you to add any extra validation when deleting +func (r *AWSManagedMachinePool) ValidateDelete() error { + mmpLog.Info("AWSManagedMachinePool validate delete", "name", r.Name) + + return nil +} + +func (r *AWSManagedMachinePool) validateImmutable(old *AWSManagedMachinePool) field.ErrorList { + var allErrs field.ErrorList + + appendErrorIfMutated := func(old, update interface{}, name string) { + if !reflect.DeepEqual(old, update) { + allErrs = append( + allErrs, + field.Invalid(field.NewPath("spec", name), update, "field is immutable"), + ) + } + } + appendErrorIfSetAndMutated := func(old, update interface{}, name string) { + if !reflect.ValueOf(old).IsZero() && !reflect.DeepEqual(old, update) { + allErrs = append( + allErrs, + field.Invalid(field.NewPath("spec", name), update, "field is immutable"), + ) + } + } + + appendErrorIfMutated(old.Spec.EKSNodegroupName, r.Spec.EKSNodegroupName, "eksNodegroupName") + appendErrorIfMutated(old.Spec.SubnetIDs, r.Spec.SubnetIDs, "subnetIDs") + appendErrorIfSetAndMutated(old.Spec.RoleName, r.Spec.RoleName, "roleName") + appendErrorIfMutated(old.Spec.DiskSize, r.Spec.DiskSize, "diskSize") + appendErrorIfMutated(old.Spec.AMIType, r.Spec.AMIType, "amiType") + appendErrorIfMutated(old.Spec.RemoteAccess, r.Spec.RemoteAccess, "remoteAccess") + + return allErrs +} + +// Default will set default values for the AWSManagedMachinePool +func (r *AWSManagedMachinePool) Default() { + mmpLog.Info("AWSManagedMachinePool setting defaults", "name", r.Name) + + if r.Spec.EKSNodegroupName == "" { + mmpLog.Info("EKSNodegroupName is empty, generating name") + name, err := eks.GenerateEKSName(r.Name, r.Namespace) + if err != nil { + mmpLog.Error(err, "failed to create EKS nodegroup name") + return + } + + mmpLog.Info("Generated EKSNodegroupName", "nodegroup-name", name) + r.Spec.EKSNodegroupName = name + } +} diff --git a/exp/api/v1alpha3/conditions_consts.go b/exp/api/v1alpha3/conditions_consts.go index b3812572cb..69e59e8b15 100644 --- a/exp/api/v1alpha3/conditions_consts.go +++ b/exp/api/v1alpha3/conditions_consts.go @@ -35,3 +35,22 @@ const ( // LaunchTemplateCreateFailedReason used for failures during Launch Template creation LaunchTemplateCreateFailedReason = "LaunchTemplateCreateFailed" ) + +const ( + // EKSNodegroupReadyCondition condition reports on the successful reconciliation of eks control plane. + EKSNodegroupReadyCondition clusterv1.ConditionType = "EKSNodegroupReady" + // EKSNodegroupReconciliationFailedReason used to report failures while reconciling EKS control plane + EKSNodegroupReconciliationFailedReason = "EKSNodegroupReconciliationFailed" + // WaitingForEKSControlPlaneReason used when the machine pool is waiting for + // EKS control plane infrastructure to be ready before proceeding. + WaitingForEKSControlPlaneReason = "WaitingForEKSControlPlane" +) + +const ( + // IAMNodegroupRolesReadyCondition condition reports on the successful + // reconciliation of EKS nodegroup iam roles. + IAMNodegroupRolesReadyCondition clusterv1.ConditionType = "IAMNodegroupRolesReady" + // IAMNodegroupRolesReconciliationFailedReason used to report failures while + // reconciling EKS nodegroup iam roles + IAMNodegroupRolesReconciliationFailedReason = "IAMNodegroupRolesReconciliationFailed" +) diff --git a/exp/api/v1alpha3/zz_generated.deepcopy.go b/exp/api/v1alpha3/zz_generated.deepcopy.go index 8179ebe4a1..8455d8aadb 100644 --- a/exp/api/v1alpha3/zz_generated.deepcopy.go +++ b/exp/api/v1alpha3/zz_generated.deepcopy.go @@ -21,7 +21,7 @@ limitations under the License. package v1alpha3 import ( - runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime" apiv1alpha3 "sigs.k8s.io/cluster-api-provider-aws/api/v1alpha3" cluster_apiapiv1alpha3 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/cluster-api/errors" @@ -303,6 +303,166 @@ func (in *AWSManagedClusterStatus) DeepCopy() *AWSManagedClusterStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AWSManagedMachinePool) DeepCopyInto(out *AWSManagedMachinePool) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AWSManagedMachinePool. +func (in *AWSManagedMachinePool) DeepCopy() *AWSManagedMachinePool { + if in == nil { + return nil + } + out := new(AWSManagedMachinePool) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *AWSManagedMachinePool) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AWSManagedMachinePoolList) DeepCopyInto(out *AWSManagedMachinePoolList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]AWSManagedMachinePool, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AWSManagedMachinePoolList. +func (in *AWSManagedMachinePoolList) DeepCopy() *AWSManagedMachinePoolList { + if in == nil { + return nil + } + out := new(AWSManagedMachinePoolList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *AWSManagedMachinePoolList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AWSManagedMachinePoolSpec) DeepCopyInto(out *AWSManagedMachinePoolSpec) { + *out = *in + if in.SubnetIDs != nil { + in, out := &in.SubnetIDs, &out.SubnetIDs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.AdditionalTags != nil { + in, out := &in.AdditionalTags, &out.AdditionalTags + *out = make(apiv1alpha3.Tags, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.AMIVersion != nil { + in, out := &in.AMIVersion, &out.AMIVersion + *out = new(string) + **out = **in + } + if in.AMIType != nil { + in, out := &in.AMIType, &out.AMIType + *out = new(ManagedMachineAMIType) + **out = **in + } + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.DiskSize != nil { + in, out := &in.DiskSize, &out.DiskSize + *out = new(int32) + **out = **in + } + if in.InstanceType != nil { + in, out := &in.InstanceType, &out.InstanceType + *out = new(string) + **out = **in + } + if in.Scaling != nil { + in, out := &in.Scaling, &out.Scaling + *out = new(ManagedMachinePoolScaling) + (*in).DeepCopyInto(*out) + } + if in.RemoteAccess != nil { + in, out := &in.RemoteAccess, &out.RemoteAccess + *out = new(ManagedRemoteAccess) + (*in).DeepCopyInto(*out) + } + if in.ProviderIDList != nil { + in, out := &in.ProviderIDList, &out.ProviderIDList + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AWSManagedMachinePoolSpec. +func (in *AWSManagedMachinePoolSpec) DeepCopy() *AWSManagedMachinePoolSpec { + if in == nil { + return nil + } + out := new(AWSManagedMachinePoolSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AWSManagedMachinePoolStatus) DeepCopyInto(out *AWSManagedMachinePoolStatus) { + *out = *in + if in.FailureReason != nil { + in, out := &in.FailureReason, &out.FailureReason + *out = new(errors.MachineStatusError) + **out = **in + } + if in.FailureMessage != nil { + in, out := &in.FailureMessage, &out.FailureMessage + *out = new(string) + **out = **in + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(cluster_apiapiv1alpha3.Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AWSManagedMachinePoolStatus. +func (in *AWSManagedMachinePoolStatus) DeepCopy() *AWSManagedMachinePoolStatus { + if in == nil { + return nil + } + out := new(AWSManagedMachinePoolStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AutoScalingGroup) DeepCopyInto(out *AutoScalingGroup) { *out = *in @@ -403,6 +563,56 @@ func (in *InstancesDistribution) DeepCopy() *InstancesDistribution { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ManagedMachinePoolScaling) DeepCopyInto(out *ManagedMachinePoolScaling) { + *out = *in + if in.MinSize != nil { + in, out := &in.MinSize, &out.MinSize + *out = new(int32) + **out = **in + } + if in.MaxSize != nil { + in, out := &in.MaxSize, &out.MaxSize + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ManagedMachinePoolScaling. +func (in *ManagedMachinePoolScaling) DeepCopy() *ManagedMachinePoolScaling { + if in == nil { + return nil + } + out := new(ManagedMachinePoolScaling) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ManagedRemoteAccess) DeepCopyInto(out *ManagedRemoteAccess) { + *out = *in + if in.SSHKeyName != nil { + in, out := &in.SSHKeyName, &out.SSHKeyName + *out = new(string) + **out = **in + } + if in.SourceSecurityGroups != nil { + in, out := &in.SourceSecurityGroups, &out.SourceSecurityGroups + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ManagedRemoteAccess. +func (in *ManagedRemoteAccess) DeepCopy() *ManagedRemoteAccess { + if in == nil { + return nil + } + out := new(ManagedRemoteAccess) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MixedInstancesPolicy) DeepCopyInto(out *MixedInstancesPolicy) { *out = *in diff --git a/exp/controllers/awsmanagedmachinepool_controller.go b/exp/controllers/awsmanagedmachinepool_controller.go new file mode 100644 index 0000000000..a670cdf29b --- /dev/null +++ b/exp/controllers/awsmanagedmachinepool_controller.go @@ -0,0 +1,273 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + + "github.com/go-logr/logr" + "github.com/pkg/errors" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/tools/record" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" + capiv1exp "sigs.k8s.io/cluster-api/exp/api/v1alpha3" + "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/cluster-api/util/conditions" + "sigs.k8s.io/cluster-api/util/predicates" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + controlplanev1 "sigs.k8s.io/cluster-api-provider-aws/controlplane/eks/api/v1alpha3" + infrav1exp "sigs.k8s.io/cluster-api-provider-aws/exp/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/scope" + "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/eks" +) + +// AWSManagedMachinePoolReconciler reconciles a AWSManagedMachinePool object +type AWSManagedMachinePoolReconciler struct { + client.Client + Log logr.Logger + Recorder record.EventRecorder + Endpoints []scope.ServiceEndpoint + + EnableIAM bool +} + +// SetupWithManager is used to setup the controller +func (r *AWSManagedMachinePoolReconciler) SetupWithManager(mgr ctrl.Manager, options controller.Options) error { + gvk, err := apiutil.GVKForObject(new(infrav1exp.AWSManagedMachinePool), mgr.GetScheme()) + if err != nil { + return errors.Wrapf(err, "failed to find GVK for AWSManagedMachinePool") + } + managedControlPlaneToManagedMachinePoolMap := managedControlPlaneToManagedMachinePoolMapFunc(r.Client, gvk, r.Log) + return ctrl.NewControllerManagedBy(mgr). + For(&infrav1exp.AWSManagedMachinePool{}). + WithOptions(options). + WithEventFilter(predicates.ResourceNotPaused(r.Log)). + Watches( + &source.Kind{Type: &capiv1exp.MachinePool{}}, + &handler.EnqueueRequestsFromMapFunc{ + ToRequests: machinePoolToInfrastructureMapFunc(gvk), + }, + ). + Watches( + &source.Kind{Type: &controlplanev1.AWSManagedControlPlane{}}, + &handler.EnqueueRequestsFromMapFunc{ + ToRequests: managedControlPlaneToManagedMachinePoolMap, + }, + ). + Complete(r) +} + +// +kubebuilder:rbac:groups=exp.cluster.x-k8s.io,resources=machinepools;machinepools/status,verbs=get;list;watch +// +kubebuilder:rbac:groups=core,resources=events,verbs=get;list;watch;create;patch +// +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=awsmanagedcontrolplanes;awsmanagedcontrolplanes/status,verbs=get;list;watch +// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=awsmanagedmachinepools,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=awsmanagedmachinepools/status,verbs=get;update;patch + +// Reconcile reconciles AWSManagedMachinePools +func (r *AWSManagedMachinePoolReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, reterr error) { + logger := r.Log.WithValues("namespace", req.Namespace, "AWSManagedMachinePool", req.Name) + ctx := context.Background() + + awsPool := &infrav1exp.AWSManagedMachinePool{} + if err := r.Get(ctx, req.NamespacedName, awsPool); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{Requeue: true}, nil + } + + machinePool, err := getOwnerMachinePool(ctx, r.Client, awsPool.ObjectMeta) + if err != nil { + logger.Error(err, "Failed to retrieve owner MachinePool from the API Server") + return ctrl.Result{}, err + } + if machinePool == nil { + logger.Info("MachinePool Controller has not yet set OwnerRef") + return ctrl.Result{}, nil + } + + logger = logger.WithValues("MachinePool", machinePool.Name) + + cluster, err := util.GetClusterFromMetadata(ctx, r.Client, machinePool.ObjectMeta) + if err != nil { + logger.Info("Failed to retrieve Cluster from MachinePool") + return reconcile.Result{}, nil + } + + logger = logger.WithValues("Cluster", cluster.Name) + + controlPlaneKey := client.ObjectKey{ + Namespace: awsPool.Namespace, + Name: cluster.Spec.ControlPlaneRef.Name, + } + controlPlane := &controlplanev1.AWSManagedControlPlane{} + if err := r.Client.Get(ctx, controlPlaneKey, controlPlane); err != nil { + logger.Info("Failed to retrieve ControlPlane from MachinePool") + return reconcile.Result{}, nil + } + + logger = logger.WithValues("AWSManagedControlPlane", controlPlane.Name) + + if !controlPlane.Status.Ready { + logger.Info("Control plane is not ready yet") + conditions.MarkFalse(awsPool, infrav1exp.EKSNodegroupReadyCondition, infrav1exp.WaitingForEKSControlPlaneReason, clusterv1.ConditionSeverityInfo, "") + return ctrl.Result{}, nil + } + + machinePoolScope, err := scope.NewManagedMachinePoolScope(scope.ManagedMachinePoolScopeParams{ + Logger: logger, + Client: r.Client, + ControllerName: "awsmanagedmachinepool", + ControlPlane: controlPlane, + MachinePool: machinePool, + ManagedMachinePool: awsPool, + EnableIAM: r.EnableIAM, + Endpoints: r.Endpoints, + }) + if err != nil { + return ctrl.Result{}, errors.Wrap(err, "failed to create scope") + } + + defer func() { + applicableConditions := []clusterv1.ConditionType{ + infrav1exp.EKSNodegroupReadyCondition, + infrav1exp.IAMNodegroupRolesReadyCondition, + } + + conditions.SetSummary(machinePoolScope.ManagedMachinePool, conditions.WithConditions(applicableConditions...), conditions.WithStepCounter()) + + if err := machinePoolScope.Close(); err != nil && reterr == nil { + reterr = err + } + }() + + if !awsPool.ObjectMeta.DeletionTimestamp.IsZero() { + return r.reconcileDelete(ctx, machinePoolScope) + } + + return r.reconcileNormal(ctx, machinePoolScope) +} + +func (r *AWSManagedMachinePoolReconciler) reconcileNormal( + _ context.Context, + machinePoolScope *scope.ManagedMachinePoolScope, +) (ctrl.Result, error) { + machinePoolScope.Info("Reconciling AWSManagedMachinePool") + + controllerutil.AddFinalizer(machinePoolScope.ManagedMachinePool, infrav1exp.ManagedMachinePoolFinalizer) + if err := machinePoolScope.PatchObject(); err != nil { + return ctrl.Result{}, err + } + + ekssvc := eks.NewNodegroupService(machinePoolScope) + + if err := ekssvc.ReconcilePool(); err != nil { + return reconcile.Result{}, errors.Wrapf(err, "failed to reconcile machine pool for AWSManagedMachinePool %s/%s", machinePoolScope.ManagedMachinePool.Namespace, machinePoolScope.ManagedMachinePool.Name) + } + + return ctrl.Result{}, nil +} + +func (r *AWSManagedMachinePoolReconciler) reconcileDelete( + _ context.Context, + machinePoolScope *scope.ManagedMachinePoolScope, +) (ctrl.Result, error) { + machinePoolScope.Info("Reconciling deletion of AWSManagedMachinePool") + + ekssvc := eks.NewNodegroupService(machinePoolScope) + + if err := ekssvc.ReconcilePoolDelete(); err != nil { + return reconcile.Result{}, errors.Wrapf(err, "failed to reconcile machine pool deletion for AWSManagedMachinePool %s/%s", machinePoolScope.ManagedMachinePool.Namespace, machinePoolScope.ManagedMachinePool.Name) + } + + controllerutil.RemoveFinalizer(machinePoolScope.ManagedMachinePool, infrav1exp.ManagedMachinePoolFinalizer) + + return reconcile.Result{}, nil +} + +// GetOwnerClusterKey returns only the Cluster name and namespace +func GetOwnerClusterKey(obj metav1.ObjectMeta) (*client.ObjectKey, error) { + for _, ref := range obj.OwnerReferences { + if ref.Kind != "Cluster" { + continue + } + gv, err := schema.ParseGroupVersion(ref.APIVersion) + if err != nil { + return nil, errors.WithStack(err) + } + if gv.Group == clusterv1.GroupVersion.Group { + return &client.ObjectKey{ + Namespace: obj.Namespace, + Name: ref.Name, + }, nil + } + } + return nil, nil +} + +func managedControlPlaneToManagedMachinePoolMapFunc(c client.Client, gvk schema.GroupVersionKind, log logr.Logger) handler.ToRequestsFunc { + return func(o handler.MapObject) []reconcile.Request { + ctx := context.Background() + awsControlPlane, ok := o.Object.(*controlplanev1.AWSManagedControlPlane) + if !ok { + return nil + } + if !awsControlPlane.ObjectMeta.DeletionTimestamp.IsZero() { + return nil + } + + clusterKey, err := GetOwnerClusterKey(awsControlPlane.ObjectMeta) + if err != nil { + log.Error(err, "couldn't get AWS control plane owner ObjectKey") + return nil + } + if clusterKey == nil { + return nil + } + + managedPoolForClusterList := capiv1exp.MachinePoolList{} + if err := c.List( + ctx, &managedPoolForClusterList, client.InNamespace(clusterKey.Namespace), client.MatchingLabels{clusterv1.ClusterLabelName: clusterKey.Name}, + ); err != nil { + log.Error(err, "couldn't list pools for cluster") + return nil + } + + mapFunc := machinePoolToInfrastructureMapFunc(gvk) + + var results []ctrl.Request + for i := range managedPoolForClusterList.Items { + managedPool := mapFunc.Map(handler.MapObject{ + Object: &managedPoolForClusterList.Items[i], + }) + results = append(results, managedPool...) + } + + return results + } +} diff --git a/main.go b/main.go index f97b668331..10636b9a98 100644 --- a/main.go +++ b/main.go @@ -17,6 +17,7 @@ limitations under the License. package main import ( + "errors" "flag" "fmt" "math/rand" @@ -78,6 +79,7 @@ var ( webhookPort int healthAddr string serviceEndpoints string + errEKSInvalidFlags = errors.New("invalid EKS flag combination") ) func main() { @@ -159,6 +161,24 @@ func main() { if feature.Gates.Enabled(feature.EKS) { setupLog.Info("enabling EKS controllers") + enableIAM := feature.Gates.Enabled(feature.EKSEnableIAM) + allowAddRoles := feature.Gates.Enabled(feature.EKSAllowAddRoles) + + if allowAddRoles && !enableIAM { + setupLog.Error(errEKSInvalidFlags, "cannot use EKSAllowAddRoles flag without EKSEnableIAM") + os.Exit(1) + } + + if err = (&controllersexp.AWSManagedMachinePoolReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("AWSManagedMachinePool"), + Recorder: mgr.GetEventRecorderFor("awsmanagedmachinepool-reconciler"), + EnableIAM: enableIAM, + Endpoints: AWSServiceEndpoints, + }).SetupWithManager(mgr, controller.Options{}); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "AWSManagedMachinePool") + os.Exit(1) + } if err = (&controllersexp.AWSManagedClusterReconciler{ Client: mgr.GetClient(), Log: ctrl.Log.WithName("controllers").WithName("AWSManagedCluster"), @@ -202,6 +222,13 @@ func main() { setupLog.Error(err, "unable to create webhook", "webhook", "AWSClusterList") os.Exit(1) } + if feature.Gates.Enabled(feature.EKS) { + setupLog.Info("enabling EKS webhooks") + if err = (&infrav1alpha3exp.AWSManagedMachinePool{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "AWSManagedMachinePool") + os.Exit(1) + } + } } // +kubebuilder:scaffold:builder diff --git a/pkg/cloud/scope/clients.go b/pkg/cloud/scope/clients.go index 2c78a9cc82..a862e684ec 100644 --- a/pkg/cloud/scope/clients.go +++ b/pkg/cloud/scope/clients.go @@ -46,7 +46,7 @@ import ( "sigs.k8s.io/cluster-api-provider-aws/version" ) -// NewEC2Client creates a new EC2 API client for a given session +// NewASGClient creates a new ASG API client for a given session func NewASGClient(scopeUser cloud.ScopeUsage, session cloud.Session, target runtime.Object) autoscalingiface.AutoScalingAPI { asgClient := autoscaling.New(session.Session()) asgClient.Handlers.Build.PushFrontNamed(getUserAgentHandler()) diff --git a/pkg/cloud/scope/managednodegroup.go b/pkg/cloud/scope/managednodegroup.go new file mode 100644 index 0000000000..a64c46a04d --- /dev/null +++ b/pkg/cloud/scope/managednodegroup.go @@ -0,0 +1,225 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scope + +import ( + "context" + + awsclient "github.com/aws/aws-sdk-go/aws/client" + "github.com/go-logr/logr" + "github.com/pkg/errors" + "k8s.io/klog/klogr" + + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" + clusterv1exp "sigs.k8s.io/cluster-api/exp/api/v1alpha3" + "sigs.k8s.io/cluster-api/util/conditions" + "sigs.k8s.io/cluster-api/util/patch" + "sigs.k8s.io/controller-runtime/pkg/client" + + infrav1 "sigs.k8s.io/cluster-api-provider-aws/api/v1alpha3" + controlplanev1exp "sigs.k8s.io/cluster-api-provider-aws/controlplane/eks/api/v1alpha3" + infrav1exp "sigs.k8s.io/cluster-api-provider-aws/exp/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud" +) + +// ManagedMachinePoolScopeParams defines the input parameters used to create a new Scope. +type ManagedMachinePoolScopeParams struct { + Client client.Client + Logger logr.Logger + ControlPlane *controlplanev1exp.AWSManagedControlPlane + ManagedMachinePool *infrav1exp.AWSManagedMachinePool + MachinePool *clusterv1exp.MachinePool + ControllerName string + Endpoints []ServiceEndpoint + Session awsclient.ConfigProvider + + EnableIAM bool +} + +// NewManagedMachinePoolScope creates a new Scope from the supplied parameters. +// This is meant to be called for each reconcile iteration. +func NewManagedMachinePoolScope(params ManagedMachinePoolScopeParams) (*ManagedMachinePoolScope, error) { + if params.ControlPlane == nil { + return nil, errors.New("failed to generate new scope from nil AWSManagedMachinePool") + } + if params.MachinePool == nil { + return nil, errors.New("failed to generate new scope from nil MachinePool") + } + if params.Logger == nil { + params.Logger = klogr.New() + } + + session, err := sessionForRegion(params.ControlPlane.Spec.Region, params.Endpoints) + if err != nil { + return nil, errors.Errorf("failed to create aws session: %v", err) + } + + helper, err := patch.NewHelper(params.ManagedMachinePool, params.Client) + if err != nil { + return nil, errors.Wrap(err, "failed to init patch helper") + } + + return &ManagedMachinePoolScope{ + Logger: params.Logger, + Client: params.Client, + ControlPlane: params.ControlPlane, + ManagedMachinePool: params.ManagedMachinePool, + MachinePool: params.MachinePool, + patchHelper: helper, + session: session, + controllerName: params.ControllerName, + enableIAM: params.EnableIAM, + }, nil +} + +// ManagedMachinePoolScope defines the basic context for an actuator to operate upon. +type ManagedMachinePoolScope struct { + logr.Logger + Client client.Client + patchHelper *patch.Helper + + ControlPlane *controlplanev1exp.AWSManagedControlPlane + ManagedMachinePool *infrav1exp.AWSManagedMachinePool + MachinePool *clusterv1exp.MachinePool + + session awsclient.ConfigProvider + controllerName string + + enableIAM bool +} + +// Name returns the machine pool name. +func (s *ManagedMachinePoolScope) Name() string { + return s.ManagedMachinePool.Name +} + +// EnableIAM indicates that reconciliation should create IAM roles +func (s *ManagedMachinePoolScope) EnableIAM() bool { + return s.enableIAM +} + +// AdditionalTags returns AdditionalTags from the scope's ManagedMachinePool +// The returned value will never be nil. +func (s *ManagedMachinePoolScope) AdditionalTags() infrav1.Tags { + if s.ManagedMachinePool.Spec.AdditionalTags == nil { + s.ManagedMachinePool.Spec.AdditionalTags = infrav1.Tags{} + } + + return s.ManagedMachinePool.Spec.AdditionalTags.DeepCopy() +} + +// RoleName returns the node group role name +func (s *ManagedMachinePoolScope) RoleName() string { + return s.ManagedMachinePool.Spec.RoleName +} + +// Version returns the nodegroup Kubernetes version +func (s *ManagedMachinePoolScope) Version() *string { + return s.MachinePool.Spec.Template.Spec.Version +} + +// ControlPlaneSubnets returns the control plane subnets. +func (s *ManagedMachinePoolScope) ControlPlaneSubnets() infrav1.Subnets { + return s.ControlPlane.Spec.NetworkSpec.Subnets +} + +// SubnetIDs returns the machine pool subnet IDs. +func (s *ManagedMachinePoolScope) SubnetIDs() []string { + return s.ManagedMachinePool.Spec.SubnetIDs +} + +// NodegroupReadyFalse marks the ready condition false using warning if error isn't +// empty +func (s *ManagedMachinePoolScope) NodegroupReadyFalse(reason string, err string) error { + severity := clusterv1.ConditionSeverityWarning + if err == "" { + severity = clusterv1.ConditionSeverityInfo + } + conditions.MarkFalse( + s.ManagedMachinePool, + infrav1exp.EKSNodegroupReadyCondition, + reason, + severity, + err, + ) + if err := s.PatchObject(); err != nil { + return errors.Wrap(err, "failed to mark nodegroup not ready") + } + return nil +} + +// IAMReadyFalse marks the ready condition false using warning if error isn't +// empty +func (s *ManagedMachinePoolScope) IAMReadyFalse(reason string, err string) error { + severity := clusterv1.ConditionSeverityWarning + if err == "" { + severity = clusterv1.ConditionSeverityInfo + } + conditions.MarkFalse( + s.ManagedMachinePool, + infrav1exp.IAMNodegroupRolesReadyCondition, + reason, + severity, + err, + ) + if err := s.PatchObject(); err != nil { + return errors.Wrap(err, "failed to mark nodegroup role not ready") + } + return nil +} + +// PatchObject persists the control plane configuration and status. +func (s *ManagedMachinePoolScope) PatchObject() error { + return s.patchHelper.Patch( + context.TODO(), + s.ManagedMachinePool, + patch.WithOwnedConditions{Conditions: []clusterv1.ConditionType{ + infrav1exp.EKSNodegroupReadyCondition, + infrav1exp.IAMNodegroupRolesReadyCondition, + }}) +} + +// Close closes the current scope persisting the control plane configuration and status. +func (s *ManagedMachinePoolScope) Close() error { + return s.PatchObject() +} + +// InfraCluster returns the AWS infrastructure cluster or control plane object. +func (s *ManagedMachinePoolScope) InfraCluster() cloud.ClusterObject { + return s.ControlPlane +} + +// Session returns the AWS SDK session. Used for creating clients +func (s *ManagedMachinePoolScope) Session() awsclient.ConfigProvider { + return s.session +} + +// ControllerName returns the name of the controller that +// created the ManagedMachinePool. +func (s *ManagedMachinePoolScope) ControllerName() string { + return s.controllerName +} + +// KubernetesClusterName is the name of the EKS cluster name. +func (s *ManagedMachinePoolScope) KubernetesClusterName() string { + return s.ControlPlane.Spec.EKSClusterName +} + +// NodegroupName is the name of the EKS nodegroup +func (s *ManagedMachinePoolScope) NodegroupName() string { + return s.ManagedMachinePool.Spec.EKSNodegroupName +} diff --git a/pkg/cloud/services/eks/cluster.go b/pkg/cloud/services/eks/cluster.go index a212a2d602..69b628d11e 100644 --- a/pkg/cloud/services/eks/cluster.go +++ b/pkg/cloud/services/eks/cluster.go @@ -316,7 +316,7 @@ func (s *Service) createCluster(eksClusterName string) (*eks.Cluster, error) { tags[k] = &tagValue } - role, err := s.getIAMRole(*s.scope.ControlPlane.Spec.RoleName) + role, err := s.GetIAMRole(*s.scope.ControlPlane.Spec.RoleName) if err != nil { return nil, errors.Wrapf(err, "error getting control plane iam role: %s", *s.scope.ControlPlane.Spec.RoleName) } diff --git a/pkg/cloud/services/eks/eks.go b/pkg/cloud/services/eks/eks.go index cfe763da0e..2ecb6ba23f 100644 --- a/pkg/cloud/services/eks/eks.go +++ b/pkg/cloud/services/eks/eks.go @@ -19,10 +19,14 @@ package eks import ( "context" + "github.com/pkg/errors" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/cluster-api/util/conditions" ekscontrolplanev1 "sigs.k8s.io/cluster-api-provider-aws/controlplane/eks/api/v1alpha3" + infrav1exp "sigs.k8s.io/cluster-api-provider-aws/exp/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/awserrors" + "sigs.k8s.io/cluster-api-provider-aws/pkg/record" ) // ReconcileControlPlane reconciles a EKS control plane @@ -64,3 +68,66 @@ func (s *Service) DeleteControlPlane() (err error) { s.scope.V(2).Info("Delete EKS control plane completed successfully") return nil } + +// ReconcilePool is the entrypoint for ManagedMachinePool reconciliation +func (s *NodegroupService) ReconcilePool() error { + s.scope.V(2).Info("Reconciling EKS nodegroup") + + if err := s.reconcileNodegroupIAMRole(); err != nil { + conditions.MarkFalse( + s.scope.ManagedMachinePool, + infrav1exp.IAMNodegroupRolesReadyCondition, + infrav1exp.IAMNodegroupRolesReconciliationFailedReason, + clusterv1.ConditionSeverityError, + err.Error(), + ) + return err + } + conditions.MarkTrue(s.scope.ManagedMachinePool, infrav1exp.IAMNodegroupRolesReadyCondition) + + if err := s.reconcileNodegroup(); err != nil { + conditions.MarkFalse( + s.scope.ManagedMachinePool, + infrav1exp.EKSNodegroupReadyCondition, + infrav1exp.EKSNodegroupReconciliationFailedReason, + clusterv1.ConditionSeverityError, + err.Error(), + ) + return err + } + conditions.MarkTrue(s.scope.ManagedMachinePool, infrav1exp.EKSNodegroupReadyCondition) + + return nil +} + +// ReconcilePoolDelete is the entrypoint for ManagedMachinePool deletion +// reconciliation +func (s *NodegroupService) ReconcilePoolDelete() error { + s.scope.V(2).Info("Reconciling deletion of EKS nodegroup") + + eksNodegroupName := s.scope.NodegroupName() + + ng, err := s.describeNodegroup() + if err != nil { + if awserrors.IsNotFound(err) { + s.scope.V(4).Info("EKS nodegroup does not exist") + return nil + } + return errors.Wrap(err, "failed to describe EKS nodegroup") + } + if ng == nil { + return nil + } + + if err := s.deleteNodegroupAndWait(); err != nil { + return errors.Wrap(err, "failed to delete nodegroup") + } + + if err := s.deleteNodegroupIAMRole(); err != nil { + return errors.Wrap(err, "failed to delete nodegroup IAM role") + } + + record.Eventf(s.scope.ManagedMachinePool, "SuccessfulDeleteEKSNodegroup", "Deleted EKS nodegroup %s", eksNodegroupName) + + return nil +} diff --git a/pkg/cloud/services/eks/errors.go b/pkg/cloud/services/eks/errors.go index 2477ba7e36..592360025d 100644 --- a/pkg/cloud/services/eks/errors.go +++ b/pkg/cloud/services/eks/errors.go @@ -28,6 +28,8 @@ var ( ErrClusterRoleNameMissing = errors.New("a cluster role name must be specified") // ErrClusterRoleNotFound is an error if the specified role couldn't be founbd in AWS ErrClusterRoleNotFound = errors.New("the specified cluster role couldn't be found") + // ErrNodegroupRoleNotFound is an error if the specified role couldn't be founbd in AWS + ErrNodegroupRoleNotFound = errors.New("the specified nodegroup role couldn't be found") // ErrCannotUseAdditionalRoles is an error if the spec contains additional role and the // EKSAllowAddRoles feature flag isn't enabled ErrCannotUseAdditionalRoles = errors.New("additional rules cannot be added as this has been disabled") diff --git a/pkg/cloud/services/eks/iam/iam.go b/pkg/cloud/services/eks/iam/iam.go new file mode 100644 index 0000000000..c1ffa074f8 --- /dev/null +++ b/pkg/cloud/services/eks/iam/iam.go @@ -0,0 +1,350 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package iam + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/iam" + "github.com/aws/aws-sdk-go/service/iam/iamiface" + "github.com/go-logr/logr" + "github.com/pkg/errors" + + infrav1 "sigs.k8s.io/cluster-api-provider-aws/api/v1alpha3" + apiiam "sigs.k8s.io/cluster-api-provider-aws/cmd/clusterawsadm/api/iam/v1alpha1" + "sigs.k8s.io/cluster-api-provider-aws/cmd/clusterawsadm/converters" +) + +type IAMService struct { + logr.Logger + IAMClient iamiface.IAMAPI +} + +func (s *IAMService) GetIAMRole(name string) (*iam.Role, error) { + input := &iam.GetRoleInput{ + RoleName: aws.String(name), + } + + out, err := s.IAMClient.GetRole(input) + if err != nil { + return nil, err + } + + return out.Role, nil +} + +func (s *IAMService) getIAMPolicy(policyArn string) (*iam.Policy, error) { + input := &iam.GetPolicyInput{ + PolicyArn: &policyArn, + } + + out, err := s.IAMClient.GetPolicy(input) + if err != nil { + return nil, err + } + + return out.Policy, nil +} + +func (s *IAMService) getIAMRolePolicies(roleName string) ([]*string, error) { + input := &iam.ListAttachedRolePoliciesInput{ + RoleName: &roleName, + } + + out, err := s.IAMClient.ListAttachedRolePolicies(input) + if err != nil { + return nil, errors.Wrapf(err, "error listing role polices for %s", roleName) + } + + policies := []*string{} + for _, policy := range out.AttachedPolicies { + policies = append(policies, policy.PolicyArn) + } + + return policies, nil +} + +func (s *IAMService) detachIAMRolePolicy(roleName string, policyARN string) error { + input := &iam.DetachRolePolicyInput{ + RoleName: aws.String(roleName), + PolicyArn: aws.String(policyARN), + } + + _, err := s.IAMClient.DetachRolePolicy(input) + if err != nil { + return errors.Wrapf(err, "error detaching policy %s from role %s", policyARN, roleName) + } + + return nil +} + +func (s *IAMService) attachIAMRolePolicy(roleName string, policyARN string) error { + input := &iam.AttachRolePolicyInput{ + RoleName: aws.String(roleName), + PolicyArn: aws.String(policyARN), + } + + _, err := s.IAMClient.AttachRolePolicy(input) + if err != nil { + return errors.Wrapf(err, "error attaching policy %s to role %s", policyARN, roleName) + } + + return nil +} + +func (s *IAMService) EnsurePoliciesAttached(role *iam.Role, policies []*string) error { + s.V(2).Info("Ensuring Polices are attached to role") + existingPolices, err := s.getIAMRolePolicies(*role.RoleName) + if err != nil { + return err + } + + // Remove polices that aren't in the list + for _, existingPolicy := range existingPolices { + found := findStringInSlice(policies, *existingPolicy) + if !found { + err = s.detachIAMRolePolicy(*role.RoleName, *existingPolicy) + if err != nil { + return err + } + s.V(2).Info("Detached policy from role", "role", role.RoleName, "policy", existingPolicy) + } + } + + // Add any policies that aren't currently attached + for _, policy := range policies { + found := findStringInSlice(existingPolices, *policy) + if !found { + // Make sure policy exists before attaching + _, err := s.getIAMPolicy(*policy) + if err != nil { + return errors.Wrapf(err, "error getting policy %s", *policy) + } + + err = s.attachIAMRolePolicy(*role.RoleName, *policy) + if err != nil { + return err + } + s.V(2).Info("Attached policy to role", "role", role.RoleName, "policy", *policy) + } + } + + return nil +} + +func RoleTags(key string, additionalTags infrav1.Tags) []*iam.Tag { + additionalTags[infrav1.ClusterAWSCloudProviderTagKey(key)] = string(infrav1.ResourceLifecycleOwned) + tags := []*iam.Tag{} + for k, v := range additionalTags { + tags = append(tags, &iam.Tag{ + Key: aws.String(k), + Value: aws.String(v), + }) + } + return tags +} + +func (s *IAMService) CreateRole( + roleName string, + key string, + trustRelationship *apiiam.PolicyDocument, + additionalTags infrav1.Tags, +) (*iam.Role, error) { + tags := RoleTags(key, additionalTags) + + trustRelationshipJSON, err := converters.IAMPolicyDocumentToJSON(*trustRelationship) + if err != nil { + return nil, errors.Wrap(err, "error converting trust relationship to json") + } + + input := &iam.CreateRoleInput{ + RoleName: aws.String(roleName), + Tags: tags, + AssumeRolePolicyDocument: aws.String(trustRelationshipJSON), + } + + out, err := s.IAMClient.CreateRole(input) + if err != nil { + return nil, err + } + + return out.Role, nil +} + +func (s *IAMService) EnsureTagsAndPolicy( + role *iam.Role, + key string, + trustRelationship *apiiam.PolicyDocument, + additionalTags infrav1.Tags, +) error { + s.V(2).Info("Ensuring tags and AssumeRolePolicyDocument are set on role") + trustRelationshipJSON, err := converters.IAMPolicyDocumentToJSON(*trustRelationship) + if err != nil { + return errors.Wrap(err, "error converting trust relationship to json") + } + + if trustRelationshipJSON != *role.AssumeRolePolicyDocument { + policyInput := &iam.UpdateAssumeRolePolicyInput{ + RoleName: role.RoleName, + PolicyDocument: aws.String(trustRelationshipJSON), + } + _, err := s.IAMClient.UpdateAssumeRolePolicy(policyInput) + if err != nil { + return err + } + + } + + tagInput := &iam.TagRoleInput{ + RoleName: role.RoleName, + } + untagInput := &iam.UntagRoleInput{ + RoleName: role.RoleName, + } + currentTags := make(map[string]string) + for _, tag := range role.Tags { + currentTags[*tag.Key] = *tag.Value + if *tag.Key == infrav1.ClusterAWSCloudProviderTagKey(key) { + continue + } + if _, ok := additionalTags[*tag.Key]; !ok { + untagInput.TagKeys = append(untagInput.TagKeys, tag.Key) + } + } + for key, value := range additionalTags { + if currentV, ok := currentTags[key]; !ok || value != currentV { + tagInput.Tags = append(tagInput.Tags, &iam.Tag{ + Key: aws.String(key), + Value: aws.String(value), + }) + } + } + + if len(tagInput.Tags) > 0 { + _, err = s.IAMClient.TagRole(tagInput) + if err != nil { + return err + } + } + + if len(untagInput.TagKeys) > 0 { + _, err = s.IAMClient.UntagRole(untagInput) + if err != nil { + return err + } + } + + return nil +} + +func (s *IAMService) detachAllPoliciesForRole(name string) error { + s.V(3).Info("Detaching all policies for role", "role", name) + input := &iam.ListAttachedRolePoliciesInput{ + RoleName: &name, + } + policies, err := s.IAMClient.ListAttachedRolePolicies(input) + if err != nil { + return errors.Wrapf(err, "error fetching policies for role %s", name) + } + for _, p := range policies.AttachedPolicies { + s.V(2).Info("Detaching policy", "policy", *p) + if err := s.detachIAMRolePolicy(name, *p.PolicyArn); err != nil { + return err + } + } + return nil +} + +func (s *IAMService) DeleteRole(name string) error { + if err := s.detachAllPoliciesForRole(name); err != nil { + return errors.Wrapf(err, "error detaching policies for role %s", name) + } + + input := &iam.DeleteRoleInput{ + RoleName: aws.String(name), + } + + _, err := s.IAMClient.DeleteRole(input) + if err != nil { + return errors.Wrapf(err, "error deleting role %s", name) + } + + return nil +} + +func (s *IAMService) IsUnmanaged(role *iam.Role, key string) bool { + keyToFind := infrav1.ClusterAWSCloudProviderTagKey(key) + for _, tag := range role.Tags { + if *tag.Key == keyToFind && *tag.Value == string(infrav1.ResourceLifecycleOwned) { + return false + } + } + + return true +} + +func ControlPlaneTrustRelationship(enableFargate bool) *apiiam.PolicyDocument { + principal := make(apiiam.Principals) + principal["Service"] = []string{"eks.amazonaws.com"} + if enableFargate { + principal["Service"] = append(principal["Service"], "eks-fargate-pods.amazonaws.com") + } + + policy := &apiiam.PolicyDocument{ + Version: "2012-10-17", + Statement: []apiiam.StatementEntry{ + { + Effect: "Allow", + Action: []string{ + "sts:AssumeRole", + }, + Principal: principal, + }, + }, + } + + return policy +} + +func NodegroupTrustRelationship() *apiiam.PolicyDocument { + principal := make(apiiam.Principals) + principal["Service"] = []string{"ec2.amazonaws.com"} + + policy := &apiiam.PolicyDocument{ + Version: "2012-10-17", + Statement: []apiiam.StatementEntry{ + { + Effect: "Allow", + Action: []string{ + "sts:AssumeRole", + }, + Principal: principal, + }, + }, + } + + return policy +} + +func findStringInSlice(slice []*string, toFind string) bool { + for _, item := range slice { + if *item == toFind { + return true + } + } + + return false +} diff --git a/pkg/cloud/services/eks/nodegroup.go b/pkg/cloud/services/eks/nodegroup.go new file mode 100644 index 0000000000..6e8d52d72a --- /dev/null +++ b/pkg/cloud/services/eks/nodegroup.go @@ -0,0 +1,450 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package eks + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/autoscaling" + "github.com/aws/aws-sdk-go/service/eks" + "github.com/aws/aws-sdk-go/service/iam" + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/util/version" + + infrav1 "sigs.k8s.io/cluster-api-provider-aws/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/wait" + "sigs.k8s.io/cluster-api-provider-aws/pkg/record" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" + "sigs.k8s.io/cluster-api/controllers/noderefutil" +) + +func (s *NodegroupService) describeNodegroup() (*eks.Nodegroup, error) { + eksClusterName := s.scope.KubernetesClusterName() + nodegroupName := s.scope.NodegroupName() + input := &eks.DescribeNodegroupInput{ + ClusterName: aws.String(eksClusterName), + NodegroupName: aws.String(nodegroupName), + } + + out, err := s.EKSClient.DescribeNodegroup(input) + if err != nil { + if aerr, ok := err.(awserr.Error); ok { + switch aerr.Code() { + case eks.ErrCodeResourceNotFoundException: + return nil, nil + default: + return nil, errors.Wrap(err, "failed to describe nodegroup") + } + } else { + return nil, errors.Wrap(err, "failed to describe nodegroup") + } + } + + return out.Nodegroup, nil +} + +func (s *NodegroupService) scalingConfig() *eks.NodegroupScalingConfig { + var replicas int32 = 1 + if s.scope.MachinePool.Spec.Replicas != nil { + replicas = *s.scope.MachinePool.Spec.Replicas + } + cfg := eks.NodegroupScalingConfig{ + DesiredSize: aws.Int64(int64(replicas)), + } + scaling := s.scope.ManagedMachinePool.Spec.Scaling + if scaling.MaxSize != nil { + cfg.MaxSize = aws.Int64(int64(*scaling.MaxSize)) + } + if scaling.MaxSize != nil { + cfg.MinSize = aws.Int64(int64(*scaling.MinSize)) + } + return &cfg +} + +func (s *NodegroupService) subnets() []string { + subnetIDs := s.scope.SubnetIDs() + // If not specified, use all + if len(subnetIDs) == 0 { + subnetIDs := []string{} + for _, subnet := range s.scope.ControlPlaneSubnets() { + subnetIDs = append(subnetIDs, subnet.ID) + } + return subnetIDs + } + return subnetIDs +} + +func (s *NodegroupService) roleArn() (*string, error) { + var role *iam.Role + if s.scope.RoleName() != "" { + var err error + role, err = s.GetIAMRole(s.scope.RoleName()) + if err != nil { + return nil, errors.Wrapf(err, "error getting node group IAM role: %s", s.scope.RoleName()) + } + } + return role.Arn, nil +} + +func ngTags(key string, additionalTags infrav1.Tags) map[string]string { + tags := additionalTags.DeepCopy() + tags[infrav1.ClusterAWSCloudProviderTagKey(key)] = string(infrav1.ResourceLifecycleOwned) + return tags +} + +func (s *NodegroupService) createNodegroup() (*eks.Nodegroup, error) { + eksClusterName := s.scope.KubernetesClusterName() + nodegroupName := s.scope.NodegroupName() + additionalTags := s.scope.AdditionalTags() + roleArn, err := s.roleArn() + if err != nil { + return nil, err + } + managedPool := s.scope.ManagedMachinePool.Spec + tags := ngTags(s.scope.Name(), additionalTags) + input := &eks.CreateNodegroupInput{ + ScalingConfig: s.scalingConfig(), + ClusterName: aws.String(eksClusterName), + NodegroupName: aws.String(nodegroupName), + Subnets: aws.StringSlice(s.subnets()), + NodeRole: roleArn, + Labels: aws.StringMap(managedPool.Labels), + Tags: aws.StringMap(tags), + } + if managedPool.AMIType != nil { + input.AmiType = aws.String(string(*managedPool.AMIType)) + } + if managedPool.DiskSize != nil { + input.DiskSize = aws.Int64(int64(*managedPool.DiskSize)) + } + if managedPool.InstanceType != nil { + input.InstanceTypes = []*string{managedPool.InstanceType} + } + if managedPool.RemoteAccess != nil { + input.RemoteAccess = &eks.RemoteAccessConfig{ + Ec2SshKey: managedPool.RemoteAccess.SSHKeyName, + SourceSecurityGroups: aws.StringSlice(managedPool.RemoteAccess.SourceSecurityGroups), + } + } + if err := input.Validate(); err != nil { + return nil, errors.Wrap(err, "created invalid CreateNodegroupInput") + } + + out, err := s.EKSClient.CreateNodegroup(input) + if err != nil { + if aerr, ok := err.(awserr.Error); ok { + switch aerr.Code() { + // TODO + case eks.ErrCodeResourceNotFoundException: + return nil, nil + default: + return nil, errors.Wrap(err, "failed to create nodegroup") + } + } else { + return nil, errors.Wrap(err, "failed to create nodegroup") + } + } + + return out.Nodegroup, nil +} + +func (s *NodegroupService) deleteNodegroupAndWait() (reterr error) { + eksClusterName := s.scope.KubernetesClusterName() + nodegroupName := s.scope.NodegroupName() + if err := s.scope.NodegroupReadyFalse(clusterv1.DeletingReason, ""); err != nil { + return err + } + defer func() { + if reterr != nil { + record.Warnf( + s.scope.ManagedMachinePool, "FailedDeleteEKSNodegroup", "Failed to delete EKS nodegroup %s: %v", s.scope.NodegroupName(), reterr, + ) + if err := s.scope.NodegroupReadyFalse("DeletingFailed", reterr.Error()); err != nil { + reterr = err + } + } else if err := s.scope.NodegroupReadyFalse(clusterv1.DeletedReason, ""); err != nil { + reterr = err + } + }() + input := &eks.DeleteNodegroupInput{ + ClusterName: aws.String(eksClusterName), + NodegroupName: aws.String(nodegroupName), + } + if err := input.Validate(); err != nil { + return errors.Wrap(err, "created invalid DeleteNodegroupInput") + } + + _, err := s.EKSClient.DeleteNodegroup(input) + if err != nil { + if aerr, ok := err.(awserr.Error); ok { + switch aerr.Code() { + // TODO + case eks.ErrCodeResourceNotFoundException: + return nil + default: + return errors.Wrap(err, "failed to delete nodegroup") + } + } else { + return errors.Wrap(err, "failed to delete nodegroup") + } + } + + waitInput := &eks.DescribeNodegroupInput{ + ClusterName: aws.String(eksClusterName), + NodegroupName: aws.String(nodegroupName), + } + err = s.EKSClient.WaitUntilNodegroupDeleted(waitInput) + if err != nil { + return errors.Wrapf(err, "failed waiting for EKS nodegroup %s to delete", nodegroupName) + } + + return nil +} + +func (s *NodegroupService) reconcileNodegroupVersion(ng *eks.Nodegroup) error { + var specVersion *version.Version + if s.scope.Version() != nil { + specVersion = parseEKSVersion(*s.scope.Version()) + } + ngVersion := version.MustParseGeneric(*ng.Version) + specAMI := s.scope.ManagedMachinePool.Spec.AMIVersion + ngAMI := *ng.ReleaseVersion + + eksClusterName := s.scope.KubernetesClusterName() + if (specVersion != nil && ngVersion.LessThan(specVersion)) || (specAMI != nil && *specAMI != ngAMI) { + input := &eks.UpdateNodegroupVersionInput{ + ClusterName: aws.String(eksClusterName), + NodegroupName: aws.String(s.scope.NodegroupName()), + } + + var updateMsg string + // Either update k8s version or AMI version + if specVersion != nil && ngVersion.LessThan(specVersion) { + // NOTE: you can only upgrade increments of minor versions. If you want to upgrade 1.14 to 1.16 we + // need to go 1.14-> 1.15 and then 1.15 -> 1.16. + input.Version = aws.String(versionToEKS(ngVersion.WithMinor(ngVersion.Minor() + 1))) + updateMsg = fmt.Sprintf("to version %s", *input.Version) + } else if specAMI != nil && *specAMI != ngAMI { + input.ReleaseVersion = specAMI + updateMsg = fmt.Sprintf("to AMI version %s", *input.ReleaseVersion) + } + + if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (bool, error) { + if _, err := s.EKSClient.UpdateNodegroupVersion(input); err != nil { + if aerr, ok := err.(awserr.Error); ok { + return false, aerr + } + return false, err + } + record.Eventf(s.scope.ManagedMachinePool, "SuccessfulUpdateEKSNodegroup", "Updated EKS nodegroup %s %s", eksClusterName, updateMsg) + return true, nil + }); err != nil { + record.Warnf(s.scope.ManagedMachinePool, "FailedUpdateEKSNodegroup", "failed to update the EKS nodegroup %s %s: %v", eksClusterName, updateMsg, err) + return errors.Wrapf(err, "failed to update EKS nodegroup") + } + } + return nil +} + +func createLabelUpdate(specLabels map[string]string, ng *eks.Nodegroup) *eks.UpdateLabelsPayload { + current := ng.Labels + payload := eks.UpdateLabelsPayload{} + for k, v := range specLabels { + if currentV, ok := current[k]; !ok || currentV == nil || v != *currentV { + payload.AddOrUpdateLabels[k] = aws.String(v) + } + } + for k := range current { + if _, ok := specLabels[k]; !ok { + payload.RemoveLabels = append(payload.RemoveLabels, aws.String(k)) + } + } + if len(payload.AddOrUpdateLabels) > 0 || len(payload.RemoveLabels) > 0 { + return &payload + } + return nil +} + +func (s *NodegroupService) reconcileNodegroupConfig(ng *eks.Nodegroup) error { + eksClusterName := s.scope.KubernetesClusterName() + machinePool := s.scope.MachinePool.Spec + managedPool := s.scope.ManagedMachinePool.Spec + input := &eks.UpdateNodegroupConfigInput{ + ClusterName: aws.String(eksClusterName), + NodegroupName: aws.String(managedPool.EKSNodegroupName), + } + var needsUpdate bool + if labelPayload := createLabelUpdate(managedPool.Labels, ng); labelPayload != nil { + input.Labels = labelPayload + needsUpdate = true + } + if machinePool.Replicas == nil { + if ng.ScalingConfig.DesiredSize != nil && *ng.ScalingConfig.DesiredSize != 1 { + input.ScalingConfig = s.scalingConfig() + needsUpdate = true + } + } else if ng.ScalingConfig.DesiredSize == nil || int64(*machinePool.Replicas) != *ng.ScalingConfig.DesiredSize { + input.ScalingConfig = s.scalingConfig() + needsUpdate = true + } + if !needsUpdate { + return nil + } + if err := input.Validate(); err != nil { + return errors.Wrap(err, "created invalid UpdateNodegroupConfigInput") + } + + _, err := s.EKSClient.UpdateNodegroupConfig(input) + if err != nil { + return errors.Wrap(err, "failed to update nodegroup config") + } + + return nil +} + +func (s *NodegroupService) reconcileNodegroup() error { + eksClusterName := s.scope.KubernetesClusterName() + eksNodegroupName := s.scope.NodegroupName() + + ng, err := s.describeNodegroup() + if err != nil { + return errors.Wrap(err, "failed to describe nodegroup") + } + + if ng == nil { + ng, err = s.createNodegroup() + if err != nil { + return errors.Wrap(err, "failed to create nodegroup") + } + s.scope.Info("Created EKS nodegroup in AWS", "cluster-name", eksClusterName, "nodegroup-name", eksNodegroupName) + } else { + tagKey := infrav1.ClusterAWSCloudProviderTagKey(s.scope.Name()) + ownedTag := ng.Tags[tagKey] + if ownedTag == nil { + return errors.Wrapf(err, "owner of %s mismatch: %s", eksNodegroupName, s.scope.Name()) + } + s.scope.V(2).Info("Found owned EKS nodegroup in AWS", "cluster-name", eksClusterName, "nodegroup-name", eksNodegroupName) + } + + if err := s.setStatus(ng); err != nil { + return errors.Wrap(err, "failed to set status") + } + + switch *ng.Status { + case eks.NodegroupStatusCreating, eks.NodegroupStatusUpdating: + ng, err = s.waitForNodegroupActive() + default: + break + } + + if err != nil { + return errors.Wrap(err, "failed to wait for nodegroup to be active") + } + + if err := s.reconcileNodegroupVersion(ng); err != nil { + return errors.Wrap(err, "failed to reconcile nodegroup version") + } + + if err := s.reconcileNodegroupConfig(ng); err != nil { + return errors.Wrap(err, "failed to reconcile nodegroup config") + } + + if err := s.reconcileTags(ng); err != nil { + return errors.Wrapf(err, "failed to reconcile nodegroup tags") + } + + return nil +} + +func (s *NodegroupService) setStatus(ng *eks.Nodegroup) error { + managedPool := s.scope.ManagedMachinePool + switch *ng.Status { + case eks.NodegroupStatusDeleting: + managedPool.Status.Ready = false + case eks.NodegroupStatusCreateFailed, eks.NodegroupStatusDeleteFailed: + managedPool.Status.Ready = false + // TODO FailureReason + failureMsg := fmt.Sprintf("EKS nodegroup in failed %s status", *ng.Status) + managedPool.Status.FailureMessage = &failureMsg + case eks.NodegroupStatusActive: + managedPool.Status.Ready = true + managedPool.Status.FailureMessage = nil + // TODO FailureReason + case eks.NodegroupStatusCreating: + managedPool.Status.Ready = false + case eks.NodegroupStatusUpdating: + managedPool.Status.Ready = true + default: + return errors.Errorf("unexpected EKS nodegroup status %s", *ng.Status) + } + if managedPool.Status.Ready && ng.Resources != nil && len(ng.Resources.AutoScalingGroups) > 0 { + req := autoscaling.DescribeAutoScalingGroupsInput{} + for _, asg := range ng.Resources.AutoScalingGroups { + req.AutoScalingGroupNames = append(req.AutoScalingGroupNames, asg.Name) + } + groups, err := s.AutoscalingClient.DescribeAutoScalingGroups(&req) + if err != nil { + return errors.Wrap(err, "failed to describe AutoScalingGroup for nodegroup") + } + + var replicas int32 + var providerIDList []string + for _, group := range groups.AutoScalingGroups { + replicas += int32(len(group.Instances)) + for _, instance := range group.Instances { + id, err := noderefutil.NewProviderID(fmt.Sprintf("aws://%s/%s", *instance.AvailabilityZone, *instance.InstanceId)) + if err != nil { + s.Error(err, "couldn't create provider ID for instance", "id", *instance.InstanceId) + continue + } + providerIDList = append(providerIDList, id.String()) + } + } + managedPool.Spec.ProviderIDList = providerIDList + managedPool.Status.Replicas = replicas + } + if err := s.scope.PatchObject(); err != nil { + return errors.Wrap(err, "failed to update nodegroup") + } + return nil +} + +func (s *NodegroupService) waitForNodegroupActive() (*eks.Nodegroup, error) { + eksClusterName := s.scope.KubernetesClusterName() + eksNodegroupName := s.scope.NodegroupName() + req := eks.DescribeNodegroupInput{ + ClusterName: aws.String(eksClusterName), + NodegroupName: aws.String(eksNodegroupName), + } + if err := s.EKSClient.WaitUntilNodegroupActive(&req); err != nil { + return nil, errors.Wrapf(err, "failed to wait for EKS nodegroup %q", *req.NodegroupName) + } + + s.scope.Info("EKS nodegroup is now available", "nodegroup-name", eksNodegroupName) + + ng, err := s.describeNodegroup() + if err != nil { + return nil, errors.Wrap(err, "failed to describe EKS nodegroup") + } + if err := s.setStatus(ng); err != nil { + return nil, errors.Wrap(err, "failed to set status") + } + + return ng, nil +} diff --git a/pkg/cloud/services/eks/roles.go b/pkg/cloud/services/eks/roles.go index c60add142e..1f1fe918ab 100644 --- a/pkg/cloud/services/eks/roles.go +++ b/pkg/cloud/services/eks/roles.go @@ -17,7 +17,6 @@ limitations under the License. package eks import ( - "encoding/json" "fmt" "github.com/aws/aws-sdk-go/aws" @@ -25,39 +24,26 @@ import ( "github.com/aws/aws-sdk-go/service/iam" "github.com/pkg/errors" - infrav1 "sigs.k8s.io/cluster-api-provider-aws/api/v1alpha3" ekscontrolplanev1 "sigs.k8s.io/cluster-api-provider-aws/controlplane/eks/api/v1alpha3" + infrav1exp "sigs.k8s.io/cluster-api-provider-aws/exp/api/v1alpha3" + eksiam "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/eks/iam" "sigs.k8s.io/cluster-api-provider-aws/pkg/record" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" ) -// TrustRelationshipPolicyDocument represesnts an IAM policy docyment -type TrustRelationshipPolicyDocument struct { - Version string - Statement []StatementEntry -} - -// ToJSONString converts the document to a JSON string -func (d *TrustRelationshipPolicyDocument) ToJSONString() (string, error) { - b, err := json.Marshal(d) - if err != nil { - return "", err +// NodegroupRolePolicies gives the policies required for a nodegroup role +func NodegroupRolePolicies() []string { + return []string{ + "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", + "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", //TODO: Can remove when CAPA supports provisioning of OIDC web identity federation with service account token volume projection + "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", } - - return string(b), nil -} - -// StatementEntry represents a statement within an IAM policy document -type StatementEntry struct { - Effect string - Action []string - Principal map[string][]string } func (s *Service) reconcileControlPlaneIAMRole() error { s.scope.V(2).Info("Reconciling EKS Control Plane IAM Role") if s.scope.ControlPlane.Spec.RoleName == nil { - //TODO (richardcase): in the future use a default role created by clusterawsadm if !s.scope.EnableIAM() { s.scope.Info("no eks control plane role specified, using default eks control plane role") s.scope.ControlPlane.Spec.RoleName = &ekscontrolplanev1.DefaultEKSControlPlaneRole @@ -65,11 +51,10 @@ func (s *Service) reconcileControlPlaneIAMRole() error { s.scope.Info("no eks control plane role specified, using role based on cluster name") s.scope.ControlPlane.Spec.RoleName = aws.String(fmt.Sprintf("%s-iam-service-role", s.scope.Name())) } - } s.scope.Info("using eks control plane role", "role-name", *s.scope.ControlPlane.Spec.RoleName) - role, err := s.getIAMRole(*s.scope.ControlPlane.Spec.RoleName) + role, err := s.GetIAMRole(*s.scope.ControlPlane.Spec.RoleName) if err != nil { if !isNotFound(err) { return err @@ -80,7 +65,7 @@ func (s *Service) reconcileControlPlaneIAMRole() error { return fmt.Errorf("getting role %s: %w", *s.scope.ControlPlane.Spec.RoleName, ErrClusterRoleNotFound) } - role, err = s.createRole(*s.scope.ControlPlane.Spec.RoleName) + role, err = s.CreateRole(*s.scope.ControlPlane.Spec.RoleName, s.scope.Name(), eksiam.ControlPlaneTrustRelationship(false), s.scope.AdditionalTags()) if err != nil { record.Warnf(s.scope.ControlPlane, "FailedIAMRoleCreation", "Failed to create control plane IAM role %q: %v", *s.scope.ControlPlane.Spec.RoleName, err) @@ -89,7 +74,7 @@ func (s *Service) reconcileControlPlaneIAMRole() error { record.Eventf(s.scope.ControlPlane, "SucessfulIAMRoleCreation", "Created control plane IAM role %q", *s.scope.ControlPlane.Spec.RoleName) } - if s.isUnmanaged(role) { + if s.IsUnmanaged(role, s.scope.Name()) { s.scope.V(2).Info("Skipping, EKS control plane role policy assignment as role is unamanged") return nil } @@ -109,7 +94,7 @@ func (s *Service) reconcileControlPlaneIAMRole() error { policies = append(policies, &additionalPolicy) } } - err = s.ensurePoliciesAttached(role, policies) + err = s.EnsurePoliciesAttached(role, policies) if err != nil { return errors.Wrapf(err, "error ensuring policies are attached: %v", policies) } @@ -117,252 +102,145 @@ func (s *Service) reconcileControlPlaneIAMRole() error { return nil } -func (s *Service) getIAMRole(name string) (*iam.Role, error) { - input := &iam.GetRoleInput{ - RoleName: aws.String(name), - } - - out, err := s.IAMClient.GetRole(input) - if err != nil { - return nil, err - } - - return out.Role, nil -} - -func (s *Service) getIAMPolicy(policyArn string) (*iam.Policy, error) { - input := &iam.GetPolicyInput{ - PolicyArn: &policyArn, +func (s *Service) deleteControlPlaneIAMRole() error { + if s.scope.ControlPlane.Spec.RoleName == nil { + return nil } - - out, err := s.IAMClient.GetPolicy(input) - if err != nil { - return nil, err + roleName := *s.scope.ControlPlane.Spec.RoleName + if !s.scope.EnableIAM() { + s.scope.V(2).Info("EKS IAM disabled, skipping deleting EKS Control Plane IAM Role") + return nil } - return out.Policy, nil -} - -func (s *Service) getIAMRolePolicies(roleName string) ([]*string, error) { - input := &iam.ListAttachedRolePoliciesInput{ - RoleName: &roleName, - } + s.scope.V(2).Info("Deleting EKS Control Plane IAM Role") - out, err := s.IAMClient.ListAttachedRolePolicies(input) + role, err := s.GetIAMRole(roleName) if err != nil { - return nil, errors.Wrapf(err, "error listing role polices for %s", roleName) - } + if isNotFound(err) { + s.V(2).Info("EKS Control Plane IAM Role already deleted") + return nil + } - policies := []*string{} - for _, policy := range out.AttachedPolicies { - policies = append(policies, policy.PolicyArn) + return errors.Wrap(err, "getting eks control plane iam role") } - return policies, nil -} - -func (s *Service) detachIAMRolePolicy(roleName string, policyARN string) error { - input := &iam.DetachRolePolicyInput{ - RoleName: aws.String(roleName), - PolicyArn: aws.String(policyARN), + if s.IsUnmanaged(role, s.scope.Name()) { + s.V(2).Info("Skipping, EKS control plane iam role deletion as role is unamanged") + return nil } - _, err := s.IAMClient.DetachRolePolicy(input) + err = s.DeleteRole(*s.scope.ControlPlane.Spec.RoleName) if err != nil { - return errors.Wrapf(err, "error detaching policy %s from role %s", policyARN, roleName) + record.Eventf(s.scope.ControlPlane, "FailedIAMRoleDeletion", "Failed to delete control Plane IAM role %q: %v", *s.scope.ControlPlane.Spec.RoleName, err) + return err } + record.Eventf(s.scope.ControlPlane, "SucessfulIAMRoleDeletion", "Deleted Control Plane IAM role %q", *s.scope.ControlPlane.Spec.RoleName) return nil } -func (s *Service) attachIAMRolePolicy(roleName string, policyARN string) error { - input := &iam.AttachRolePolicyInput{ - RoleName: aws.String(roleName), - PolicyArn: aws.String(policyARN), - } +func (s *NodegroupService) reconcileNodegroupIAMRole() error { + s.scope.V(2).Info("Reconciling EKS Nodegroup IAM Role") - _, err := s.IAMClient.AttachRolePolicy(input) - if err != nil { - return errors.Wrapf(err, "error attaching policy %s to role %s", policyARN, roleName) + if s.scope.RoleName() == "" { + var roleName string + if !s.scope.EnableIAM() { + s.scope.Info("no EKS nodegroup role specified, using default EKS nodegroup role") + roleName = infrav1exp.DefaultEKSNodegroupRole + } else { + s.scope.Info("no EKS nodegroup role specified, using role based on nodegroup name") + roleName = fmt.Sprintf("%s-nodegroup-iam-service-role", s.scope.Name()) + } + s.scope.ManagedMachinePool.Spec.RoleName = roleName } - return nil -} - -func (s *Service) ensurePoliciesAttached(role *iam.Role, policies []*string) error { - s.scope.V(2).Info("Ensuring Polices are attached to EKS Control Plane IAM Role") - existingPolices, err := s.getIAMRolePolicies(*role.RoleName) + role, err := s.GetIAMRole(s.scope.RoleName()) if err != nil { - return err - } - - // Remove polices that aren't in the list - for _, existingPolicy := range existingPolices { - found := findStringInSlice(policies, *existingPolicy) - if !found { - err = s.detachIAMRolePolicy(*role.RoleName, *existingPolicy) - if err != nil { - return err - } - s.scope.V(2).Info("Detached policy from role", "role", role.RoleName, "policy", existingPolicy) + if !isNotFound(err) { + return err } - } - // Add any policies that aren't currently attached - for _, policy := range policies { - found := findStringInSlice(existingPolices, *policy) - if !found { - // Make sure policy exists before attaching - _, err := s.getIAMPolicy(*policy) - if err != nil { - return errors.Wrapf(err, "error getting policy %s", *policy) - } - - err = s.attachIAMRolePolicy(*role.RoleName, *policy) - if err != nil { - return err - } - s.scope.V(2).Info("Attached policy to role", "role", role.RoleName, "policy", *policy) + // If the disable IAM flag is used then the role must exist + if !s.scope.EnableIAM() { + return ErrNodegroupRoleNotFound } - } - - return nil -} - -func (s *Service) createRole(name string) (*iam.Role, error) { - //TODO: tags also needs a separate sync - additionalTags := s.scope.AdditionalTags() - additionalTags[infrav1.ClusterAWSCloudProviderTagKey(s.scope.Name())] = string(infrav1.ResourceLifecycleOwned) - tags := []*iam.Tag{} - for k, v := range additionalTags { - tags = append(tags, &iam.Tag{ - Key: aws.String(k), - Value: aws.String(v), - }) - } - - trustRelationship := s.controlPlaneTrustRelationship(false) - trustRelationShipJSON, err := trustRelationship.ToJSONString() - if err != nil { - return nil, errors.Wrap(err, "error converting trust relationship to json") - } - - input := &iam.CreateRoleInput{ - RoleName: aws.String(name), - Tags: tags, - AssumeRolePolicyDocument: aws.String(trustRelationShipJSON), - } - - out, err := s.IAMClient.CreateRole(input) - if err != nil { - return nil, err - } - - return out.Role, nil -} -func (s *Service) detachAllPoliciesForRole(name string) error { - s.scope.V(3).Info("Detaching all policies for role", "role", name) - input := &iam.ListAttachedRolePoliciesInput{ - RoleName: &name, - } - policies, err := s.IAMClient.ListAttachedRolePolicies(input) - if err != nil { - return errors.Wrapf(err, "error fetching policies for role %s", name) - } - for _, p := range policies.AttachedPolicies { - s.scope.V(2).Info("Detaching policy", "policy", *p) - if err := s.detachIAMRolePolicy(name, *p.PolicyArn); err != nil { + role, err = s.CreateRole(s.scope.ManagedMachinePool.Spec.RoleName, s.scope.Name(), eksiam.NodegroupTrustRelationship(), s.scope.AdditionalTags()) + if err != nil { + record.Warnf(s.scope.ManagedMachinePool, "FailedIAMRoleCreation", "Failed to create nodegroup IAM role %q: %v", s.scope.RoleName(), err) return err } + record.Eventf(s.scope.ManagedMachinePool, "SucessfulIAMRoleCreation", "Created nodegroup IAM role %q", s.scope.RoleName()) } - return nil -} -func (s *Service) deleteRole(name string) error { - if err := s.detachAllPoliciesForRole(name); err != nil { - return errors.Wrapf(err, "error detaching policies for role %s", name) + if s.IsUnmanaged(role, s.scope.Name()) { + s.scope.V(2).Info("Skipping, EKS nodegroup role policy assignment as role is unamanged") + return nil } - input := &iam.DeleteRoleInput{ - RoleName: aws.String(name), + err = s.EnsureTagsAndPolicy(role, s.scope.Name(), eksiam.NodegroupTrustRelationship(), s.scope.AdditionalTags()) + if err != nil { + return errors.Wrapf(err, "error ensuring tags and policy document are set on node role") } - _, err := s.IAMClient.DeleteRole(input) + policies := NodegroupRolePolicies() + err = s.EnsurePoliciesAttached(role, aws.StringSlice(policies)) if err != nil { - return errors.Wrapf(err, "error deleting role %s", name) + return errors.Wrapf(err, "error ensuring policies are attached: %v", policies) } return nil } -func (s *Service) deleteControlPlaneIAMRole() error { +func (s *NodegroupService) deleteNodegroupIAMRole() (reterr error) { + if err := s.scope.IAMReadyFalse(clusterv1.DeletingReason, ""); err != nil { + return err + } + defer func() { + if reterr != nil { + record.Warnf( + s.scope.ManagedMachinePool, "FailedDeleteIAMNodegroupRole", "Failed to delete EKS nodegroup role %s: %v", s.scope.ManagedMachinePool.Spec.RoleName, reterr, + ) + if err := s.scope.IAMReadyFalse("DeletingFailed", reterr.Error()); err != nil { + reterr = err + } + } else if err := s.scope.IAMReadyFalse(clusterv1.DeletedReason, ""); err != nil { + reterr = err + } + }() + roleName := s.scope.RoleName() if !s.scope.EnableIAM() { - s.scope.V(2).Info("EKS IAM disabled, skipping deleting EKS Control Plane IAM Role") + s.scope.V(2).Info("EKS IAM disabled, skipping deleting EKS Nodegroup IAM Role") return nil } - s.scope.V(2).Info("Deleting EKS Control Plane IAM Role") + s.scope.V(2).Info("Deleting EKS Nodegroup IAM Role") - role, err := s.getIAMRole(*s.scope.ControlPlane.Spec.RoleName) + role, err := s.GetIAMRole(roleName) if err != nil { if isNotFound(err) { - s.scope.V(2).Info("EKS Control Plane IAM Role already deleted") + s.V(2).Info("EKS Nodegroup IAM Role already deleted") return nil } - return errors.Wrap(err, "getting eks control plane iam role") + return errors.Wrap(err, "getting EKS nodegroup iam role") } - if s.isUnmanaged(role) { - s.scope.V(2).Info("Skipping, EKS control plane iam role deletion as role is unamanged") + if s.IsUnmanaged(role, s.scope.Name()) { + s.V(2).Info("Skipping, EKS Nodegroup iam role deletion as role is unamanged") return nil } - err = s.deleteRole(*s.scope.ControlPlane.Spec.RoleName) + err = s.DeleteRole(s.scope.RoleName()) if err != nil { - record.Eventf(s.scope.ControlPlane, "FailedIAMRoleDeletion", "Failed to delete control Plane IAM role %q: %v", *s.scope.ControlPlane.Spec.RoleName, err) + record.Eventf(s.scope.ManagedMachinePool, "FailedIAMRoleDeletion", "Failed to delete Nodegroup IAM role %q: %v", s.scope.ManagedMachinePool.Spec.RoleName, err) return err } - record.Eventf(s.scope.ControlPlane, "SucessfulIAMRoleDeletion", "Deleted Control Plane IAM role %q", *s.scope.ControlPlane.Spec.RoleName) + record.Eventf(s.scope.ManagedMachinePool, "SucessfulIAMRoleDeletion", "Deleted Nodegroup IAM role %q", s.scope.ManagedMachinePool.Spec.RoleName) return nil } -func (s *Service) isUnmanaged(role *iam.Role) bool { - keyToFind := infrav1.ClusterAWSCloudProviderTagKey(s.scope.Name()) - for _, tag := range role.Tags { - if *tag.Key == keyToFind && *tag.Value == string(infrav1.ResourceLifecycleOwned) { - return false - } - } - - return true -} - -func (s *Service) controlPlaneTrustRelationship(enableFargate bool) *TrustRelationshipPolicyDocument { - principal := make(map[string][]string) - principal["Service"] = []string{"eks.amazonaws.com"} - if enableFargate { - principal["Service"] = append(principal["Service"], "eks-fargate-pods.amazonaws.com") - } - - policy := &TrustRelationshipPolicyDocument{ - Version: "2012-10-17", - Statement: []StatementEntry{ - { - Effect: "Allow", - Action: []string{ - "sts:AssumeRole", - }, - Principal: principal, - }, - }, - } - - return policy -} - func isNotFound(err error) bool { if aerr, ok := err.(awserr.Error); ok { switch aerr.Code() { @@ -375,13 +253,3 @@ func isNotFound(err error) bool { return false } - -func findStringInSlice(slice []*string, toFind string) bool { - for _, item := range slice { - if *item == toFind { - return true - } - } - - return false -} diff --git a/pkg/cloud/services/eks/service.go b/pkg/cloud/services/eks/service.go index fd3c822f42..1a55819fc5 100644 --- a/pkg/cloud/services/eks/service.go +++ b/pkg/cloud/services/eks/service.go @@ -17,12 +17,13 @@ limitations under the License. package eks import ( + "github.com/aws/aws-sdk-go/service/autoscaling/autoscalingiface" "github.com/aws/aws-sdk-go/service/ec2/ec2iface" "github.com/aws/aws-sdk-go/service/eks/eksiface" - "github.com/aws/aws-sdk-go/service/iam/iamiface" "github.com/aws/aws-sdk-go/service/sts/stsiface" "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/scope" + "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/eks/iam" ) // Service holds a collection of interfaces. @@ -32,7 +33,7 @@ type Service struct { scope *scope.ManagedControlPlaneScope EC2Client ec2iface.EC2API EKSClient eksiface.EKSAPI - IAMClient iamiface.IAMAPI + iam.IAMService STSClient stsiface.STSAPI } @@ -42,7 +43,35 @@ func NewService(controlPlaneScope *scope.ManagedControlPlaneScope) *Service { scope: controlPlaneScope, EC2Client: scope.NewEC2Client(controlPlaneScope, controlPlaneScope, controlPlaneScope.ControlPlane), EKSClient: scope.NewEKSClient(controlPlaneScope, controlPlaneScope, controlPlaneScope.ControlPlane), - IAMClient: scope.NewIAMClient(controlPlaneScope, controlPlaneScope, controlPlaneScope.ControlPlane), + IAMService: iam.IAMService{ + Logger: controlPlaneScope.Logger, + IAMClient: scope.NewIAMClient(controlPlaneScope, controlPlaneScope, controlPlaneScope.ControlPlane), + }, STSClient: scope.NewSTSClient(controlPlaneScope, controlPlaneScope, controlPlaneScope.ControlPlane), } } + +// NodegroupService holds a collection of interfaces. +// The interfaces are broken down like this to group functions together. +// One alternative is to have a large list of functions from the ec2 client. +type NodegroupService struct { + scope *scope.ManagedMachinePoolScope + AutoscalingClient autoscalingiface.AutoScalingAPI + EKSClient eksiface.EKSAPI + iam.IAMService + STSClient stsiface.STSAPI +} + +// NewNodegroupService returns a new service given the api clients. +func NewNodegroupService(machinePoolScope *scope.ManagedMachinePoolScope) *NodegroupService { + return &NodegroupService{ + scope: machinePoolScope, + AutoscalingClient: scope.NewASGClient(machinePoolScope, machinePoolScope, machinePoolScope.ManagedMachinePool), + EKSClient: scope.NewEKSClient(machinePoolScope, machinePoolScope, machinePoolScope.ManagedMachinePool), + IAMService: iam.IAMService{ + Logger: machinePoolScope.Logger, + IAMClient: scope.NewIAMClient(machinePoolScope, machinePoolScope, machinePoolScope.ManagedMachinePool), + }, + STSClient: scope.NewSTSClient(machinePoolScope, machinePoolScope, machinePoolScope.ManagedMachinePool), + } +} diff --git a/pkg/cloud/services/eks/tags.go b/pkg/cloud/services/eks/tags.go index d8e24406d8..4fe637edbf 100644 --- a/pkg/cloud/services/eks/tags.go +++ b/pkg/cloud/services/eks/tags.go @@ -50,3 +50,49 @@ func (s *Service) getEKSTagParams(id string) *infrav1.BuildParams { Additional: s.scope.AdditionalTags(), } } + +func getTagUpdates(currentTags map[string]string, tags map[string]string) (untagKeys []string, newTags map[string]string) { + untagKeys = []string{} + newTags = make(map[string]string) + for key := range currentTags { + if _, ok := tags[key]; !ok { + untagKeys = append(untagKeys, key) + } + } + for key, value := range tags { + if currentV, ok := currentTags[key]; !ok || value != currentV { + newTags[key] = value + } + } + return untagKeys, newTags +} + +func (s *NodegroupService) reconcileTags(ng *eks.Nodegroup) error { + tags := ngTags(s.scope.Name(), s.scope.AdditionalTags()) + + untagKeys, newTags := getTagUpdates(aws.StringValueMap(ng.Tags), tags) + + if len(newTags) > 0 { + tagInput := &eks.TagResourceInput{ + ResourceArn: ng.NodegroupArn, + Tags: aws.StringMap(newTags), + } + _, err := s.EKSClient.TagResource(tagInput) + if err != nil { + return err + } + } + + if len(untagKeys) > 0 { + untagInput := &eks.UntagResourceInput{ + ResourceArn: ng.NodegroupArn, + TagKeys: aws.StringSlice(untagKeys), + } + _, err := s.EKSClient.UntagResource(untagInput) + if err != nil { + return err + } + } + + return nil +} diff --git a/pkg/cloud/services/eks/tags_test.go b/pkg/cloud/services/eks/tags_test.go new file mode 100644 index 0000000000..e59f3a8175 --- /dev/null +++ b/pkg/cloud/services/eks/tags_test.go @@ -0,0 +1,54 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package eks + +import ( + "strconv" + "testing" + + . "github.com/onsi/gomega" +) + +func TestGetTagUpdates(t *testing.T) { + testCases := []struct { + current map[string]string + next map[string]string + expectUntag []string + expectTag map[string]string + }{ + { + current: map[string]string{ + "x": "1", + }, + next: map[string]string{ + "x": "2", + }, + expectUntag: []string{}, + expectTag: map[string]string{ + "x": "2", + }, + }, + } + for i, tc := range testCases { + t.Run(strconv.Itoa(i), func(t *testing.T) { + g := NewWithT(t) + untag, tag := getTagUpdates(tc.current, tc.next) + g.Expect(untag).To(Equal(tc.expectUntag)) + g.Expect(tag).To(Equal(tc.expectTag)) + }) + } +} diff --git a/pkg/eks/eks.go b/pkg/eks/eks.go new file mode 100644 index 0000000000..cae82162df --- /dev/null +++ b/pkg/eks/eks.go @@ -0,0 +1,49 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package eks + +import ( + "fmt" + "strings" + + "sigs.k8s.io/cluster-api-provider-aws/pkg/hash" +) + +const ( + // maxCharsName maximum number of characters for the name + maxCharsName = 100 + + clusterPrefix = "capa_" +) + +// GenerateEKSName generates a name of an EKS cluster or nodegroup +func GenerateEKSName(clusterName, namespace string) (string, error) { + escapedName := strings.Replace(clusterName, ".", "_", -1) + eksName := fmt.Sprintf("%s_%s", namespace, escapedName) + + if len(eksName) < maxCharsName { + return eksName, nil + } + + hashLength := 32 - len(clusterPrefix) + hashedName, err := hash.Base36TruncatedHash(eksName, hashLength) + if err != nil { + return "", fmt.Errorf("creating hash from cluster name: %w", err) + } + + return fmt.Sprintf("%s%s", clusterPrefix, hashedName), nil +}