Feature: ENIConfig set by custom annotation or label names

README.md

@@ -93,6 +93,17 @@ Type: Boolean
 Default: `false`  
 Specifies that your pods may use subnets and security groups that are independent of your worker node's VPC configuration\. By default, pods share the same subnet and security groups as the worker node's primary interface\. Setting this variable to `true` causes `ipamD` to use the security groups and VPC subnet in a worker node's `ENIConfig` for elastic network interface allocation\. You must create an `ENIConfig` custom resource definition for each subnet that your pods will reside in, and then annotate each worker node to use a specific `ENIConfig` \(multiple worker nodes can be annotated with the same `ENIConfig`\)\. Worker nodes can only be annotated with a single `ENIConfig` at a time, and the subnet in the `ENIConfig` must belong to the same Availability Zone that the worker node resides in\. For more information, see [https://github.com/aws/amazon-vpc-cni-k8s/pull/165](https://github.com/aws/amazon-vpc-cni-k8s/pull/165)\.
 
+`ENI_CONFIG_ANNOTATION_DEF`
+Type: String
+Default: k8s.amazonaws.com/eniConfig
+Specifies node annotation key name. This should be used when AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG=true. Custom annotation value will be used to set eniConfig name.
+
+`ENI_CONFIG_LABEL_DEF`
+Type: String
+Default: k8s.amazonaws.com/eniConfig
+Specifies node label key name. This should be used when AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG=true. Custom annotation value will be used to set eniConfig name.
+For example if you set ENI_CONFIG_LABEL_DEF=failure-domain.beta.kubernetes.io/zone, the node that is deployed in `us-east-1a` would have it set to that zone and CNI would use eniConfig named `us-east-1a`.
+
 `AWS_VPC_K8S_CNI_EXTERNALSNAT`  
 Type: Boolean  
 Default: `false`  

pkg/eniconfig/eniconfig.go

@@ -35,8 +35,19 @@ import (
 )
 
 const (
-	eniConfigAnnotationDef = "k8s.amazonaws.com/eniConfig"
-	eniConfigDefault       = "default"
+	defaultEniConfigAnnotationDef = "k8s.amazonaws.com/eniConfig"
+	defaultEniConfigLabelDef      = "k8s.amazonaws.com/eniConfig"
+	eniConfigDefault              = "default"
+
+	// when "ENI_CONFIG_LABEL_DEF is defined, ENIConfigController will use that label key to
+	// search if is setting value for eniConfigLabelDef
+	// Example:
+	//   Node has set label k8s.amazonaws.com/eniConfigCustom=customConfig
+	//   We can get that value in controller by setting environmental variable ENI_CONFIG_LABEL_DEF
+	//   ENI_CONFIG_LABEL_DEF=k8s.amazonaws.com/eniConfigOverride
+	//   This will set eniConfigLabelDef to eniConfigOverride
+	envEniConfigAnnotationDef = "ENI_CONFIG_ANNOTATION_DEF"
+	envEniConfigLabelDef      = "ENI_CONFIG_LABEL_DEF"
 )
 
 type ENIConfig interface {
@@ -48,10 +59,12 @@ var ErrNoENIConfig = errors.New("eniconfig: eniconfig is not available")
 
 // ENIConfigController defines global context for ENIConfig controller
 type ENIConfigController struct {
-	eni        map[string]*v1alpha1.ENIConfigSpec
-	myENI      string
-	eniLock    sync.RWMutex
-	myNodeName string
+	eni                    map[string]*v1alpha1.ENIConfigSpec
+	myENI                  string
+	eniLock                sync.RWMutex
+	myNodeName             string
+	eniConfigAnnotationDef string
+	eniConfigLabelDef      string
 }
 
 // ENIConfigInfo returns locally cached ENIConfigs
@@ -66,6 +79,8 @@ func NewENIConfigController() *ENIConfigController {
 		myNodeName: os.Getenv("MY_NODE_NAME"),
 		eni:        make(map[string]*v1alpha1.ENIConfigSpec),
 		myENI:      eniConfigDefault,
+		eniConfigAnnotationDef: getEniConfigAnnotationDef(),
+		eniConfigLabelDef:      getEniConfigLabelDef(),
 	}
 }
 
@@ -104,21 +119,34 @@ func (h *Handler) Handle(ctx context.Context, event sdk.Event) error {
 
 	case *corev1.Node:
 
-		log.Infof("Handle corev1.Node: %s, %v", o.GetName(), o.GetAnnotations())
+		log.Infof("Handle corev1.Node: %s, %v, %v", o.GetName(), o.GetAnnotations(), o.GetLabels())
+		// Get annotations if not found get labels if not found fallback use default
 		if h.controller.myNodeName == o.GetName() {
 			annotation := o.GetAnnotations()
 
-			val, ok := annotation[eniConfigAnnotationDef]
+			val, ok := annotation[h.controller.eniConfigAnnotationDef]
 			if ok {
 				h.controller.eniLock.Lock()
 				defer h.controller.eniLock.Unlock()
 				h.controller.myENI = val
 				log.Infof(" Setting myENI to: %s", val)
 			} else {
-				h.controller.eniLock.Lock()
-				defer h.controller.eniLock.Unlock()
-				h.controller.myENI = eniConfigDefault
-				log.Infof(" Setting myENI to: %s", eniConfigDefault)
+
+				label := o.GetLabels()
+
+				val, ok := label[h.controller.eniConfigLabelDef]
+				if ok {
+					h.controller.eniLock.Lock()
+					defer h.controller.eniLock.Unlock()
+					h.controller.myENI = val
+					log.Infof(" Setting myENI to: %s", val)
+				} else {
+
+					h.controller.eniLock.Lock()
+					defer h.controller.eniLock.Unlock()
+					h.controller.myENI = eniConfigDefault
+					log.Infof(" Setting myENI to: %s", eniConfigDefault)
+				}
 			}
 		}
 	}
@@ -182,3 +210,33 @@ func (eniCfg *ENIConfigController) MyENIConfig() (*v1alpha1.ENIConfigSpec, error
 	}
 	return nil, ErrNoENIConfig
 }
+
+// getEniConfigAnnotationDef returns eniConfigAnnotation
+func getEniConfigAnnotationDef() string {
+	inputStr, found := os.LookupEnv(envEniConfigAnnotationDef)
+
+	if !found {
+		return defaultEniConfigAnnotationDef
+	}
+	if len(inputStr) > 0 {
+		log.Debugf("Using ENI_CONFIG_ANNOTATION_DEF %v", inputStr)
+		return inputStr
+	}
+
+	return defaultEniConfigAnnotationDef
+}
+
+// getEniConfigLabelDef returns eniConfigLabel name
+func getEniConfigLabelDef() string {
+	inputStr, found := os.LookupEnv(envEniConfigLabelDef)
+
+	if !found {
+		return defaultEniConfigLabelDef
+	}
+	if len(inputStr) > 0 {
+		log.Debugf("Using ENI_CONFIG_LABEL_DEF %v", inputStr)
+		return inputStr
+	}
+
+	return defaultEniConfigLabelDef
+}

pkg/eniconfig/eniconfig_test.go

@@ -60,6 +60,7 @@ func updateNodeAnnotation(hdlr sdk.Handler, nodeName string, configName string,
 		Deleted: toDelete,
 	}
 	eniAnnotations := make(map[string]string)
+	eniConfigAnnotationDef := getEniConfigAnnotationDef()
 
 	if !toDelete {
 		eniAnnotations[eniConfigAnnotationDef] = configName
@@ -68,6 +69,34 @@ func updateNodeAnnotation(hdlr sdk.Handler, nodeName string, configName string,
 	hdlr.Handle(nil, event)
 }
 
+func updateNodeLabel(hdlr sdk.Handler, nodeName string, configName string, toDelete bool) {
+
+	node := corev1.Node{
+		TypeMeta: metav1.TypeMeta{APIVersion: corev1.SchemeGroupVersion.String()},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: nodeName,
+		},
+	}
+	accessor, err := meta.Accessor(&node)
+
+	if err != nil {
+		fmt.Printf("Failed to call meta.Access %v", err)
+	}
+
+	event := sdk.Event{
+		Object:  &node,
+		Deleted: toDelete,
+	}
+	eniLabels := make(map[string]string)
+	eniConfigLabelDef := getEniConfigLabelDef()
+
+	if !toDelete {
+		eniLabels[eniConfigLabelDef] = configName
+	}
+	accessor.SetLabels(eniLabels)
+	hdlr.Handle(nil, event)
+}
+
 func TestENIConfig(t *testing.T) {
 
 	testENIConfigController := NewENIConfigController()
@@ -105,7 +134,7 @@ func TestENIConfig(t *testing.T) {
 }
 
 func TestNodeENIConfig(t *testing.T) {
-	myNodeName := "testMyNode"
+	myNodeName := "testMyNodeWithAnnotation"
 	myENIConfig := "testMyENIConfig"
 	os.Setenv("MY_NODE_NAME", myNodeName)
 	testENIConfigController := NewENIConfigController()
@@ -144,3 +173,68 @@ func TestNodeENIConfig(t *testing.T) {
 	assert.Equal(t, defaultCfg, *outputCfg)
 
 }
+
+func TestNodeENIConfigLabel(t *testing.T) {
+	myNodeName := "testMyNodeWithLabel"
+	myENIConfig := "testMyENIConfig"
+	os.Setenv("MY_NODE_NAME", myNodeName)
+	testENIConfigController := NewENIConfigController()
+
+	testHandler := NewHandler(testENIConfigController)
+	updateNodeLabel(testHandler, myNodeName, myENIConfig, false)
+
+	// If there is no ENI config
+	_, err := testENIConfigController.MyENIConfig()
+	assert.Error(t, err)
+
+	// Add eniconfig for myENIConfig
+	group1Cfg := v1alpha1.ENIConfigSpec{
+		SecurityGroups: []string{"sg21-id", "sg22-id"},
+		Subnet:         "subnet21"}
+	updateENIConfig(testHandler, myENIConfig, group1Cfg, false)
+	outputCfg, err := testENIConfigController.MyENIConfig()
+	assert.NoError(t, err)
+	assert.Equal(t, group1Cfg, *outputCfg)
+
+	// Add default config
+	defaultSGs := []string{"sg1-id", "sg2-id"}
+	defaultSubnet := "subnet1"
+	defaultCfg := v1alpha1.ENIConfigSpec{
+		SecurityGroups: defaultSGs,
+		Subnet:         defaultSubnet}
+	updateENIConfig(testHandler, eniConfigDefault, defaultCfg, false)
+	outputCfg, err = testENIConfigController.MyENIConfig()
+	assert.NoError(t, err)
+	assert.Equal(t, group1Cfg, *outputCfg)
+
+	// Delete node's myENIConfig annotation, then the value should fallback to default
+	updateNodeLabel(testHandler, myNodeName, myENIConfig, true)
+	outputCfg, err = testENIConfigController.MyENIConfig()
+	assert.NoError(t, err)
+	assert.Equal(t, defaultCfg, *outputCfg)
+
+}
+
+func TestGetEniConfigAnnotationDefDefault(t *testing.T) {
+	os.Unsetenv(envEniConfigAnnotationDef)
+	eniConfigAnnotationDef := getEniConfigAnnotationDef()
+	assert.Equal(t, eniConfigAnnotationDef, defaultEniConfigAnnotationDef)
+}
+
+func TestGetEniConfigAnnotationlDefCustom(t *testing.T) {
+	os.Setenv(envEniConfigAnnotationDef, "k8s.amazonaws.com/eniConfigCustom")
+	eniConfigAnnotationDef := getEniConfigAnnotationDef()
+	assert.Equal(t, eniConfigAnnotationDef, "k8s.amazonaws.com/eniConfigCustom")
+}
+
+func TestGetEniConfigLabelDefDefault(t *testing.T) {
+	os.Unsetenv(envEniConfigLabelDef)
+	eniConfigLabelDef := getEniConfigLabelDef()
+	assert.Equal(t, eniConfigLabelDef, defaultEniConfigLabelDef)
+}
+
+func TestGetEniConfigLabelDefCustom(t *testing.T) {
+	os.Setenv(envEniConfigLabelDef, "k8s.amazonaws.com/eniConfigCustom")
+	eniConfigLabelDef := getEniConfigLabelDef()
+	assert.Equal(t, eniConfigLabelDef, "k8s.amazonaws.com/eniConfigCustom")
+}