feat: allow resolving EKS AMIs by version/slug for better readability… (

#351) * feat: allow resolving EKS AMIs by version/slug for better readability and portability Signed-off-by: Jonah Back <[email protected]> * address review comments Signed-off-by: Jonah Back <[email protected]>
keikoproj · Mar 7, 2022 · 4c9eccf · 4c9eccf
1 parent 798d06d
commit 4c9eccf
Show file tree

Hide file tree

Showing 9 changed files with 183 additions and 52 deletions.
diff --git a/api/v1alpha1/instancegroup_types.go b/api/v1alpha1/instancegroup_types.go
@@ -79,6 +79,7 @@ const (
 	DedicatedPlacementTenancyType = "dedicated"
 
 	ImageLatestValue = "latest"
+	ImageSSMPrefix   = "ssm://"
 )
 
 type ContainerRuntime string

diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/controllers/providers/aws/ssm.go b/controllers/providers/aws/ssm.go
@@ -15,12 +15,12 @@ import (
 type architectureMap map[string]string
 
 const (
-	EksOptimisedAmiPath           = "/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id"
-	EksOptimisedAmazonLinux2Arm64 = "/aws/service/eks/optimized-ami/%s/amazon-linux-2-arm64/recommended/image_id"
-	EksOptimisedBottlerocket      = "/aws/service/bottlerocket/aws-k8s-%s/x86_64/latest/image_id"
-	EksOptimisedBottlerocketArm64 = "/aws/service/bottlerocket/aws-k8s-%s/arm64/latest/image_id"
-	EksOptimisedWindowsCore       = "/aws/service/ami-windows-latest/Windows_Server-2019-English-Core-EKS_Optimized-%s/image_id"
-	EksOptimisedWindowsFull       = "/aws/service/ami-windows-latest/Windows_Server-2019-English-Full-EKS_Optimized-%s/image_id"
+	EksOptimisedAmiPath           = "/aws/service/eks/optimized-ami/%s/amazon-linux-2/%s/image_id"
+	EksOptimisedAmazonLinux2Arm64 = "/aws/service/eks/optimized-ami/%s/amazon-linux-2-arm64/%s/image_id"
+	EksOptimisedBottlerocket      = "/aws/service/bottlerocket/aws-k8s-%s/x86_64/%s/image_id"
+	EksOptimisedBottlerocketArm64 = "/aws/service/bottlerocket/aws-k8s-%s/arm64/%s/image_id"
+	EksOptimisedWindowsCore       = "/aws/service/ami-windows-%s/Windows_Server-2019-English-Core-EKS_Optimized-%s/image_id"
+	EksOptimisedWindowsFull       = "/aws/service/ami-windows-%s/Windows_Server-2019-English-Full-EKS_Optimized-%s/image_id"
 )
 
 var (
@@ -37,6 +37,11 @@ var (
 			"x86_64": EksOptimisedWindowsCore,
 		},
 	}
+	LatestIdentifiers = map[string]string{
+		"bottlerocket": "latest",
+		"amazonlinux2": "recommended",
+		"windows":      "latest",
+	}
 )
 
 func GetAwsSsmClient(region string, cacheCfg *cache.Config, maxRetries int, collector *common.MetricsCollector) ssmiface.SSMAPI {
@@ -60,8 +65,16 @@ func GetAwsSsmClient(region string, cacheCfg *cache.Config, maxRetries int, coll
 }
 
 func (w *AwsWorker) GetEksLatestAmi(OSFamily string, arch string, kubernetesVersion string) (string, error) {
+	return w.GetEksSsmAmi(OSFamily, arch, kubernetesVersion, LatestIdentifiers[OSFamily])
+}
+
+func (w *AwsWorker) GetEksSsmAmi(OSFamily string, arch string, kubernetesVersion string, ssmId string) (string, error) {
+	var inputString = aws.String(fmt.Sprintf(EksAmis[OSFamily][arch], kubernetesVersion, ssmId))
+	if OSFamily == "windows" {
+		inputString = aws.String(fmt.Sprintf(EksAmis[OSFamily][arch], ssmId, kubernetesVersion))
+	}
 	input := &ssm.GetParameterInput{
-		Name: aws.String(fmt.Sprintf(EksAmis[OSFamily][arch], kubernetesVersion)),
+		Name: inputString,
 	}
 
 	output, err := w.SsmClient.GetParameter(input)

diff --git a/controllers/provisioners/eks/cloud.go b/controllers/provisioners/eks/cloud.go
@@ -186,6 +186,16 @@ func (ctx *EksInstanceGroupContext) CloudDiscovery() error {
 		ctx.Log.V(4).Info("Updating Image ID with latest", "ami_id", latestAmiId)
 	}
 
+	if strings.HasPrefix(configuration.Image, v1alpha1.ImageSSMPrefix) {
+		ssmKey := strings.TrimPrefix(configuration.Image, v1alpha1.ImageSSMPrefix)
+		amiId, err := ctx.GetEksSsmAmi(ssmKey)
+		if err != nil {
+			return errors.Wrap(err, "failed to discover ami")
+		}
+		configuration.Image = amiId
+		ctx.Log.V(4).Info("Updating Image ID with ami", "ami_id", amiId)
+	}
+
 	// All information needed to creating the scaling group must happen before this line.
 	// find all owned scaling groups
 	ownedScalingGroups := ctx.findOwnedScalingGroups(scalingGroups)

diff --git a/controllers/provisioners/eks/create_test.go b/controllers/provisioners/eks/create_test.go
@@ -342,6 +342,11 @@ func TestCreateAutoScalingGroupNegative(t *testing.T) {
 	g.Expect(ctx.GetState()).To(gomega.Equal(v1alpha1.ReconcileModifying))
 }
 
+type AMITest struct {
+	igImage     string
+	expectedAmi string
+}
+
 func TestCreateLatestAMI(t *testing.T) {
 	var (
 		g       = gomega.NewGomegaWithT(t)
@@ -355,6 +360,7 @@ func TestCreateLatestAMI(t *testing.T) {
 	)
 
 	testLatestAmiID := "ami-12345678"
+	testCustomAmi := "ami-98765432"
 	w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock)
 	ctx := MockContext(ig, k, w)
 
@@ -366,10 +372,11 @@ func TestCreateLatestAMI(t *testing.T) {
 		RoleName: aws.String("some-role"),
 	}
 
-	// Setup Latest AMI
-	ig.GetEKSConfiguration().Image = "latest"
-	ssmMock.latestAMI = testLatestAmiID
-
+	// Setup mock SSM parameters
+	ssmMock.parameterMap = map[string]string{
+		"/aws/service/eks/optimized-ami/1.18/amazon-linux-2/recommended/image_id":                    testLatestAmiID,
+		"/aws/service/eks/optimized-ami/1.18/amazon-linux-2/amazon-eks-node-1.18-v20220226/image_id": testCustomAmi,
+	}
 	ec2Mock.InstanceTypes = []*ec2.InstanceTypeInfo{
 		&ec2.InstanceTypeInfo{
 			InstanceType: aws.String("m5.large"),
@@ -379,19 +386,33 @@ func TestCreateLatestAMI(t *testing.T) {
 		},
 	}
 
-	err := ctx.CloudDiscovery()
-	g.Expect(err).NotTo(gomega.HaveOccurred())
-	// Must happen after ctx.CloudDiscover()
-	ctx.GetDiscoveredState().SetInstanceTypeInfo([]*ec2.InstanceTypeInfo{
+	testAmis := []AMITest{
 		{
-			InstanceType: aws.String("m5.large"),
-			ProcessorInfo: &ec2.ProcessorInfo{
-				SupportedArchitectures: []*string{aws.String("x86_64")},
-			},
+			igImage:     "latest",
+			expectedAmi: testLatestAmiID,
 		},
-	})
+		{
+			igImage:     "ssm://amazon-eks-node-1.18-v20220226",
+			expectedAmi: testCustomAmi,
+		},
+	}
 
-	err = ctx.Create()
-	g.Expect(err).NotTo(gomega.HaveOccurred())
-	g.Expect(ctx.GetInstanceGroup().Spec.EKSSpec.EKSConfiguration.Image).To(gomega.Equal(testLatestAmiID))
+	for _, tc := range testAmis {
+		ig.GetEKSConfiguration().Image = tc.igImage
+		err := ctx.CloudDiscovery()
+		g.Expect(err).NotTo(gomega.HaveOccurred())
+		// Must happen after ctx.CloudDiscover()
+		ctx.GetDiscoveredState().SetInstanceTypeInfo([]*ec2.InstanceTypeInfo{
+			{
+				InstanceType: aws.String("m5.large"),
+				ProcessorInfo: &ec2.ProcessorInfo{
+					SupportedArchitectures: []*string{aws.String("x86_64")},
+				},
+			},
+		})
+
+		err = ctx.Create()
+		g.Expect(err).NotTo(gomega.HaveOccurred())
+		g.Expect(ctx.GetInstanceGroup().Spec.EKSSpec.EKSConfiguration.Image).To(gomega.Equal(tc.expectedAmi))
+	}
 }
diff --git a/controllers/provisioners/eks/eks_test.go b/controllers/provisioners/eks/eks_test.go
@@ -865,13 +865,13 @@ func (i *MockIamClient) WaitUntilInstanceProfileExists(input *iam.GetInstancePro
 
 type MockSsmClient struct {
 	ssmiface.SSMAPI
-	latestAMI string
+	parameterMap map[string]string
 }
 
 func (i *MockSsmClient) GetParameter(input *ssm.GetParameterInput) (*ssm.GetParameterOutput, error) {
 	return &ssm.GetParameterOutput{
 		Parameter: &ssm.Parameter{
-			Value: aws.String(i.latestAMI),
+			Value: aws.String(i.parameterMap[*input.Name]),
 		},
 	}, nil
 }
diff --git a/controllers/provisioners/eks/helpers.go b/controllers/provisioners/eks/helpers.go
@@ -1240,3 +1240,21 @@ func (ctx *EksInstanceGroupContext) GetEksLatestAmi() (string, error) {
 
 	return ctx.AwsWorker.GetEksLatestAmi(OSFamily, arch, clusterVersion)
 }
+
+func (ctx *EksInstanceGroupContext) GetEksSsmAmi(id string) (string, error) {
+	var (
+		instanceGroup = ctx.GetInstanceGroup()
+		state         = ctx.GetDiscoveredState()
+		osFamily      = ctx.GetOsFamily()
+		configuration = instanceGroup.GetEKSConfiguration()
+	)
+	clusterVersion := state.GetClusterVersion()
+
+	supportedArchitectures := awsprovider.GetInstanceTypeArchitectures(state.GetInstanceTypeInfo(), configuration.InstanceType)
+	arch := FilterSupportedArch(supportedArchitectures)
+	if arch == "" {
+		return "", fmt.Errorf("No supported CPU architecture found for instance type %s", configuration.InstanceType)
+	}
+
+	return ctx.AwsWorker.GetEksSsmAmi(osFamily, arch, clusterVersion, id)
+}
diff --git a/controllers/provisioners/eks/update_test.go b/controllers/provisioners/eks/update_test.go
@@ -701,6 +701,12 @@ func TestUpdateManagedPolicies(t *testing.T) {
 	}
 }
 
+type amiTest struct {
+	igImage     string
+	expectedAmi string
+	osFamily    string
+}
+
 func TestUpdateWithLatestAmiID(t *testing.T) {
 	var (
 		g       = gomega.NewGomegaWithT(t)
@@ -714,6 +720,10 @@ func TestUpdateWithLatestAmiID(t *testing.T) {
 	)
 
 	testLatestAmiID := "ami-12345678"
+	testCustomAmi := "ami-98765432"
+	testBottlerocketAmi := "ami-56789101"
+	testWindowsAmi := "ami-56789201"
+
 	w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock)
 	ctx := MockContext(ig, k, w)
 
@@ -738,9 +748,12 @@ func TestUpdateWithLatestAmiID(t *testing.T) {
 	}
 
 	// Setup Latest AMI
-	ig.GetEKSConfiguration().Image = "latest"
-	ssmMock.latestAMI = testLatestAmiID
-
+	ssmMock.parameterMap = map[string]string{
+		"/aws/service/eks/optimized-ami/1.18/amazon-linux-2/recommended/image_id":                      testLatestAmiID,
+		"/aws/service/eks/optimized-ami/1.18/amazon-linux-2/amazon-eks-node-1.18-v20220226/image_id":   testCustomAmi,
+		"/aws/service/bottlerocket/aws-k8s-1.18/x86_64/1.6.1/image_id":                                 testBottlerocketAmi,
+		"/aws/service/ami-windows-latest/Windows_Server-2019-English-Core-EKS_Optimized-1.18/image_id": testWindowsAmi,
+	}
 	ec2Mock.InstanceTypes = []*ec2.InstanceTypeInfo{
 		&ec2.InstanceTypeInfo{
 			InstanceType: aws.String("m5.large"),
@@ -750,33 +763,61 @@ func TestUpdateWithLatestAmiID(t *testing.T) {
 		},
 	}
 
-	err := ctx.CloudDiscovery()
-	g.Expect(err).NotTo(gomega.HaveOccurred())
-
-	ctx.SetDiscoveredState(&DiscoveredState{
-		Publisher: kubeprovider.EventPublisher{
-			Client: k.Kubernetes,
+	testAmis := []amiTest{
+		{
+			igImage:     "latest",
+			expectedAmi: testLatestAmiID,
+			osFamily:    "amazonlinux2",
 		},
-		ScalingGroup: mockScalingGroup,
-		ScalingConfiguration: &scaling.LaunchConfiguration{
-			AwsWorker: w,
+		{
+			igImage:     "ssm://amazon-eks-node-1.18-v20220226",
+			expectedAmi: testCustomAmi,
+			osFamily:    "amazonlinux2",
 		},
-		InstanceProfile: &iam.InstanceProfile{
-			Arn: aws.String("some-instance-arn"),
+		{
+			igImage:     "ssm://1.6.1",
+			expectedAmi: testBottlerocketAmi,
+			osFamily:    "bottlerocket",
 		},
-		ClusterNodes: nil,
-		Cluster:      nil,
-		InstanceTypeInfo: []*ec2.InstanceTypeInfo{
-			{
-				InstanceType: aws.String("m5.large"),
-				ProcessorInfo: &ec2.ProcessorInfo{
-					SupportedArchitectures: []*string{aws.String("x86_64")},
+		{
+			igImage:     "ssm://latest",
+			expectedAmi: testWindowsAmi,
+			osFamily:    "windows",
+		},
+	}
+
+	for _, tc := range testAmis {
+		ig.GetEKSConfiguration().Image = tc.igImage
+		ig.Annotations[OsFamilyAnnotation] = tc.osFamily
+		err := ctx.CloudDiscovery()
+		g.Expect(err).NotTo(gomega.HaveOccurred())
+
+		ctx.SetDiscoveredState(&DiscoveredState{
+			Publisher: kubeprovider.EventPublisher{
+				Client: k.Kubernetes,
+			},
+			ScalingGroup: mockScalingGroup,
+			ScalingConfiguration: &scaling.LaunchConfiguration{
+				AwsWorker: w,
+			},
+			InstanceProfile: &iam.InstanceProfile{
+				Arn: aws.String("some-instance-arn"),
+			},
+			ClusterNodes: nil,
+			Cluster:      nil,
+			InstanceTypeInfo: []*ec2.InstanceTypeInfo{
+				{
+					InstanceType: aws.String("m5.large"),
+					ProcessorInfo: &ec2.ProcessorInfo{
+						SupportedArchitectures: []*string{aws.String("x86_64")},
+					},
 				},
 			},
-		},
-	})
+		})
+
+		err = ctx.Update()
+		g.Expect(err).NotTo(gomega.HaveOccurred())
+		g.Expect(ctx.GetInstanceGroup().Spec.EKSSpec.EKSConfiguration.Image).To(gomega.Equal(tc.expectedAmi))
+	}
 
-	err = ctx.Update()
-	g.Expect(err).NotTo(gomega.HaveOccurred())
-	g.Expect(ctx.GetInstanceGroup().Spec.EKSSpec.EKSConfiguration.Image).To(gomega.Equal(testLatestAmiID))
 }
diff --git a/docs/examples/EKS.md b/docs/examples/EKS.md
@@ -202,4 +202,30 @@ spec:
   eks:
     configuration:
       image: latest
-```
+```
+
+
+## SSM-based versioned AMI
+
+It might not be acceptable to pin your `InstanceGroup` to the latest AMI. If you want to pin to a specific version,
+you can specify the relevant version information using syntax like `ssm://version`. An example for the EKS-optimized Linux AMI might look like:
+
+```yaml
+apiVersion: instancemgr.keikoproj.io/v1alpha1
+kind: InstanceGroup
+metadata:
+  name: hello-world
+  namespace: instance-manager
+  annotations:
+    instancemgr.keikoproj.io/os-family: "amazonlinux2" #Default if not provided
+spec:
+  strategy: <...>
+  provisioner: eks
+  eks:
+    configuration:
+      image: ssm://amazon-eks-node-1.18-v20220226 # resolves to value in /aws/service/eks/optimized-ami/1.18/amazon-linux-2/amazon-eks-node-1.18-v20220226/image_id
+```
+
+Using this type of image reference over a raw ami ID has a few benefits:
+- The same image reference can be used across regions, since it's not a direct reference to the AMI.
+- The image is more readable - for users leveraging GitOps, the changes will be much easier to review.