Resolve flakiness in IPAMD warm target tests; update test go.sum

test/framework/resources/k8s/utils/daemonset.go

@@ -86,7 +86,7 @@ func updateDaemonsetEnvVarsAndWait(f *framework.Framework, dsName string, dsName
 }
 
 func getDaemonSet(f *framework.Framework, dsName string, dsNamespace string) *v1.DaemonSet {
-	By(fmt.Sprintf("getting the %s daemon set in namesapce %s", dsName, dsNamespace))
+	By(fmt.Sprintf("getting the %s daemon set in namespace %s", dsName, dsNamespace))
 	ds, err := f.K8sResourceManagers.
 		DaemonSetManager().
 		GetDaemonSet(dsNamespace, dsName)

test/go.sum

@@ -553,6 +553,7 @@ github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85n
 github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
 github.com/gobuffalo/logger v1.0.3 h1:YaXOTHNPCvkqqA7w05A4v0k2tCdpr+sgFlgINbQ6gqc=
 github.com/gobuffalo/logger v1.0.3/go.mod h1:SoeejUwldiS7ZsyCBphOGURmWdwUFXs0J7TCjEhjKxM=
@@ -672,6 +673,7 @@ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=

test/integration/ipamd/ipamd_suite_test.go

@@ -14,17 +14,23 @@
 package ipamd
 
 import (
+	"fmt"
 	"testing"
 	"time"
 
 	"github.com/aws/amazon-vpc-cni-k8s/test/framework"
 	k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils"
 	"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
+	"github.com/aws/aws-sdk-go/service/ec2"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	v1 "k8s.io/api/apps/v1"
 )
 
+var primaryInstance *ec2.Instance
+var f *framework.Framework
+var err error
+
 const (
 	CoreDNSDeploymentName      = "coredns"
 	CoreDNSDeploymentNameSpace = "kube-system"
@@ -51,13 +57,14 @@ var _ = BeforeSuite(func() {
 	numOfNodes = len(nodeList.Items)
 	Expect(numOfNodes).Should(BeNumerically(">", 1))
 
-	// Nominate the first node as the 'primary' node and force coredns deployment on this
+	// Nominate the first node as the one to run coredns deployment against
 	By("adding nodeSelector in coredns deployment to be scheduled on single node")
-	instanceID := k8sUtils.GetInstanceIDFromNode(nodeList.Items[0])
+	primaryNode := nodeList.Items[0]
+	fmt.Fprintf(GinkgoWriter, "coredns node is %s\n", primaryNode.Name)
+	instanceID := k8sUtils.GetInstanceIDFromNode(primaryNode)
 	primaryInstance, err = f.CloudServices.EC2().DescribeInstance(instanceID)
 	Expect(err).ToNot(HaveOccurred())
 
-	// Add nodeSelector label to coredns deployment so coredns pods are scheduled on 'primary' node
 	By("getting node with no pods scheduled to run tests")
 	coreDNSDeployment, err := f.K8sResourceManagers.DeploymentManager().GetDeployment(CoreDNSDeploymentName,
 		CoreDNSDeploymentNameSpace)
@@ -66,16 +73,18 @@ var _ = BeforeSuite(func() {
 	// Copy the deployment to restore later
 	coreDNSDeploymentCopy = coreDNSDeployment.DeepCopy()
 
+	// Add nodeSelector label to coredns deployment so coredns pods are scheduled on 'primary' node
 	coreDNSDeployment.Spec.Template.Spec.NodeSelector = map[string]string{
 		"kubernetes.io/hostname": *primaryInstance.PrivateDnsName,
 	}
-
 	err = f.K8sResourceManagers.DeploymentManager().UpdateAndWaitTillDeploymentIsReady(coreDNSDeployment,
 		utils.DefaultDeploymentReadyTimeout)
 	Expect(err).ToNot(HaveOccurred())
 
-	// Nominate primaryInstance to node without coredns pods
-	instanceID = k8sUtils.GetInstanceIDFromNode(nodeList.Items[1])
+	// Redefine primary node as node without coredns pods. Note that this node may have previously had coredns pods.
+	primaryNode = nodeList.Items[1]
+	fmt.Fprintf(GinkgoWriter, "primary node is %s\n", primaryNode.Name)
+	instanceID = k8sUtils.GetInstanceIDFromNode(primaryNode)
 	primaryInstance, err = f.CloudServices.EC2().DescribeInstance(instanceID)
 	Expect(err).ToNot(HaveOccurred())
 
@@ -90,7 +99,7 @@ var _ = BeforeSuite(func() {
 		})
 
 	// Allow reconciler to free up ENIs if any
-	time.Sleep(utils.PollIntervalLong)
+	time.Sleep(utils.PollIntervalLong * 2)
 })
 
 var _ = AfterSuite(func() {
@@ -103,3 +112,28 @@ var _ = AfterSuite(func() {
 	f.K8sResourceManagers.NamespaceManager().
 		DeleteAndWaitTillNamespaceDeleted(utils.DefaultTestNamespace)
 })
+
+func ceil(x, y int) int {
+	return (x + y - 1) / y
+}
+
+func Max(x, y int) int {
+	if x < y {
+		return y
+	}
+	return x
+}
+
+// MinIgnoreZero returns smaller of two number, if any number is zero returns the other number
+func MinIgnoreZero(x, y int) int {
+	if x == 0 {
+		return y
+	}
+	if y == 0 {
+		return x
+	}
+	if x < y {
+		return x
+	}
+	return y
+}

test/integration/ipamd/warm_target_test.go

@@ -41,13 +41,14 @@ var _ = Describe("test warm target variables", func() {
 					"MAX_ENI":         strconv.Itoa(maxENI),
 				})
 
-			// Allow for IPAMD to reconcile it's state
+			// Allow for IPAMD to reconcile its state
 			time.Sleep(utils.PollIntervalLong * 5)
 
 			primaryInstance, err = f.CloudServices.
 				EC2().DescribeInstance(*primaryInstance.InstanceId)
 			Expect(err).ToNot(HaveOccurred())
 
+			// Validate number of allocated ENIs
 			Expect(len(primaryInstance.NetworkInterfaces)).
 				Should(Equal(MinIgnoreZero(warmENITarget, maxENI)))
 		})
@@ -58,13 +59,13 @@ var _ = Describe("test warm target variables", func() {
 				map[string]struct{}{"WARM_ENI_TARGET": {}, "MAX_ENI": {}})
 		})
 
-		Context("when WARM_ENI_TARGET = 2 and MAX_ENI = 1", func() {
+		Context("when WARM_ENI_TARGET = 3 and MAX_ENI = 2", func() {
 			BeforeEach(func() {
-				warmENITarget = 2
-				maxENI = 1
+				warmENITarget = 3
+				maxENI = 2
 			})
 
-			It("instance should have only 1 ENI", func() {})
+			It("instance should have only 2 ENIs", func() {})
 		})
 
 		Context("when WARM_ENI_TARGET = 3", func() {
@@ -76,13 +77,13 @@ var _ = Describe("test warm target variables", func() {
 			It("instance should have only 3 ENIs", func() {})
 		})
 
-		Context("when WARM_ENI_TARGET = 1", func() {
+		Context("when WARM_ENI_TARGET = 2", func() {
 			BeforeEach(func() {
-				warmENITarget = 1
+				warmENITarget = 2
 				maxENI = 0
 			})
 
-			It("instance should have only 1 ENI", func() {})
+			It("instance should have only 2 ENIs", func() {})
 		})
 
 	})

test/integration/ipamd/warm_target_test_PD_enabled.go

@@ -17,20 +17,13 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/aws/amazon-vpc-cni-k8s/test/framework"
-	"github.com/aws/aws-sdk-go/service/ec2"
-
 	k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils"
 	"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
 
-var primaryInstance *ec2.Instance
-var f *framework.Framework
-var err error
-
 // IMPORTANT: THE NODEGROUP TO RUN THE TEST MUST NOT HAVE ANY POD
 // Ideally we should drain the node, but drain from go client is non trivial
 // IMPORTANT: Only support nodes that can have 16+ Secondary IPV4s across at least 3 ENI
@@ -113,7 +106,7 @@ var _ = Describe("test warm target variables", func() {
 				minIPTarget = 16
 			})
 
-			It("should have 1 prefixe", func() {})
+			It("should have 1 prefix", func() {})
 		})
 
 		Context("when MINIMUM_IP_TARGET = 6 and WARM_IP_TARGET = 10", func() {
@@ -258,7 +251,7 @@ var _ = Describe("test warm target variables", func() {
 				warmPrefixTarget = 0
 			})
 
-			It("should have 1 prefixe", func() {})
+			It("should have 1 prefix", func() {})
 		})
 
 		Context("when MINIMUM_IP_TARGET = 6, WARM_IP_TARGET = 10 and WARM_PREFIX_TARGET = 1", func() {
@@ -272,28 +265,3 @@ var _ = Describe("test warm target variables", func() {
 		})
 	})
 })
-
-func ceil(x, y int) int {
-	return (x + y - 1) / y
-}
-
-func Max(x, y int) int {
-	if x < y {
-		return y
-	}
-	return x
-}
-
-// MinIgnoreZero returns smaller of two number, if any number is zero returns the other number
-func MinIgnoreZero(x, y int) int {
-	if x == 0 {
-		return y
-	}
-	if y == 0 {
-		return x
-	}
-	if x < y {
-		return x
-	}
-	return y
-}