Add support for AWS Network Load Balancers

Makefile

@@ -48,23 +48,11 @@ vendor:
 	patch -p1 < 0002-Sort-machines-before-syncing.patch
 
 .PHONY: generate
-generate: gendeepcopy generate-mocks
+generate: 
+	go generate ./pkg/... ./cmd/...
 
 .PHONY: test
-test: generate-mocks unit
-
-.PHONY: gendeepcopy
-gendeepcopy:
-	go build -o $$GOPATH/bin/deepcopy-gen sigs.k8s.io/cluster-api-provider-aws/vendor/k8s.io/code-generator/cmd/deepcopy-gen
-	deepcopy-gen \
-	  -i ./pkg/cloud/aws/providerconfig,./pkg/cloud/aws/providerconfig/v1alpha1 \
-	  -O zz_generated.deepcopy \
-	  -h boilerplate.go.txt
-
-.PHONY: generate-mocks
-generate-mocks:
-	go build -o $$GOPATH/bin/mockgen sigs.k8s.io/cluster-api-provider-aws/vendor/github.com/golang/mock/mockgen/
-	go generate ./pkg/cloud/aws/client/
+test: unit
 
 bin:
 	@mkdir $@

hack/go-lint.sh

@@ -7,8 +7,8 @@ if [ "$IS_CONTAINER" != "" ]; then
 else
   docker run --rm \
     --env IS_CONTAINER=TRUE \
-    --volume "${PWD}:/go/src/github.com/openshift/${REPO_NAME}:z" \
-    --workdir "/go/src/github.com/openshift/${REPO_NAME}" \
+    --volume "${PWD}:/go/src/sigs.k8s.io/${REPO_NAME}:z" \
+    --workdir "/go/src/sigs.k8s.io/${REPO_NAME}" \
     openshift/origin-release:golang-1.10 \
     ./hack/go-lint.sh "${@}"
 fi

pkg/apis/awsproviderconfig/v1alpha1/awsmachineproviderconfig_types.go

@@ -133,9 +133,9 @@ type AWSMachineProviderConfig struct {
 	// Placement specifies where to create the instance in AWS
 	Placement Placement `json:"placement"`
 
-	// LoadBalancerNames is the names of the load balancers to which the new instance
+	// LoadBalancers is the set of load balancers to which the new instance
 	// should be added once it is created.
-	LoadBalancerNames []string `json:"loadBalancerIds"`
+	LoadBalancers []LoadBalancerReference `json:"loadBalancers"`
 }
 
 // AWSResourceReference is a reference to a specific AWS resource by ID, ARN, or filters.
@@ -188,6 +188,23 @@ type AWSMachineProviderConfigList struct {
 	Items           []AWSMachineProviderConfig `json:"items"`
 }
 
+// LoadBalancerReference is a reference to a load balancer on AWS.
+type LoadBalancerReference struct {
+	Name string              `json:"name"`
+	Type AWSLoadBalancerType `json:"type"`
+}
+
+// AWSLoadBalancerType is the type of LoadBalancer to use when registering
+// an instance with load balancers specified in LoadBalancerNames
+type AWSLoadBalancerType string
+
+// Possible values for AWSLoadBalancerType. Add to this list as other types
+// of load balancer are supported by the actuator.
+const (
+	ClassicLoadBalancerType AWSLoadBalancerType = "classic" // AWS classic ELB
+	NetworkLoadBalancerType AWSLoadBalancerType = "network" // AWS Network Load Balancer (NLB)
+)
+
 func init() {
 	SchemeBuilder.Register(&AWSMachineProviderConfig{}, &AWSMachineProviderConfigList{}, &AWSMachineProviderStatus{})
 }

pkg/cloud/aws/actuators/machine/actuator.go

@@ -19,6 +19,7 @@ package machine
 import (
 	"encoding/base64"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/golang/glog"
@@ -27,6 +28,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/equality"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	errorutil "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/client-go/kubernetes"
 
 	providerconfigv1 "sigs.k8s.io/cluster-api-provider-aws/pkg/apis/awsproviderconfig/v1alpha1"
@@ -36,6 +38,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/elb"
+	"github.com/aws/aws-sdk-go/service/elbv2"
 
 	"k8s.io/apimachinery/pkg/runtime"
 	awsclient "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/aws/client"
@@ -547,6 +550,7 @@ func (a *Actuator) Update(cluster *clusterv1.Cluster, machine *clusterv1.Machine
 
 	err = a.UpdateLoadBalancers(client, machineProviderConfig, newestInstance)
 	if err != nil {
+		glog.Errorf("error updating load balancers: %v", err)
 		return err
 	}
 
@@ -614,12 +618,48 @@ func (a *Actuator) getMachineInstances(cluster *clusterv1.Cluster, machine *clus
 
 // UpdateLoadBalancers adds a given machine instance to the load balancers specified in its provider config
 func (a *Actuator) UpdateLoadBalancers(client awsclient.Client, providerConfig *providerconfigv1.AWSMachineProviderConfig, instance *ec2.Instance) error {
-	if len(providerConfig.LoadBalancerNames) == 0 {
+	if len(providerConfig.LoadBalancers) == 0 {
+		glog.V(4).Infof("Instance %q has no load balancers configured. Skipping", *instance.InstanceId)
 		return nil
 	}
+	errs := []error{}
+	classicLoadBalancerNames := []string{}
+	networkLoadBalancerNames := []string{}
+	for _, loadBalancerRef := range providerConfig.LoadBalancers {
+		switch loadBalancerRef.Type {
+		case providerconfigv1.NetworkLoadBalancerType:
+			networkLoadBalancerNames = append(networkLoadBalancerNames, loadBalancerRef.Name)
+		case providerconfigv1.ClassicLoadBalancerType:
+			classicLoadBalancerNames = append(classicLoadBalancerNames, loadBalancerRef.Name)
+		}
+	}
+
+	var err error
+	if len(classicLoadBalancerNames) > 0 {
+		err := a.registerWithClassicLoadBalancers(client, classicLoadBalancerNames, instance)
+		if err != nil {
+			glog.Errorf("failed to register classic load balancers: %v", err)
+			errs = append(errs, err)
+		}
+	}
+	if len(networkLoadBalancerNames) > 0 {
+		err = a.registerWithNetworkLoadBalancers(client, networkLoadBalancerNames, instance)
+		if err != nil {
+			glog.Errorf("failed to register network load balancers: %v", err)
+			errs = append(errs, err)
+		}
+	}
+	if len(errs) > 0 {
+		return errorutil.NewAggregate(errs)
+	}
+	return nil
+}
+
+func (a *Actuator) registerWithClassicLoadBalancers(client awsclient.Client, names []string, instance *ec2.Instance) error {
+	glog.V(4).Infof("Updating classic load balancer registration for %q", *instance.InstanceId)
 	elbInstance := &elb.Instance{InstanceId: instance.InstanceId}
 	var errs []error
-	for _, elbName := range providerConfig.LoadBalancerNames {
+	for _, elbName := range names {
 		req := &elb.RegisterInstancesWithLoadBalancerInput{
 			Instances:        []*elb.Instance{elbInstance},
 			LoadBalancerName: aws.String(elbName),
@@ -631,7 +671,73 @@ func (a *Actuator) UpdateLoadBalancers(client awsclient.Client, providerConfig *
 	}
 
 	if len(errs) > 0 {
-		return fmt.Errorf("failed to register instances with elbs: %v", errs)
+		return errorutil.NewAggregate(errs)
+	}
+	return nil
+}
+
+func (a *Actuator) registerWithNetworkLoadBalancers(client awsclient.Client, names []string, instance *ec2.Instance) error {
+	glog.V(4).Infof("Updating network load balancer registration for %q", *instance.InstanceId)
+	lbNames := make([]*string, len(names))
+	for i, name := range names {
+		lbNames[i] = aws.String(name)
+	}
+	lbsRequest := &elbv2.DescribeLoadBalancersInput{
+		Names: lbNames,
+	}
+	lbsResponse, err := client.ELBv2DescribeLoadBalancers(lbsRequest)
+	if err != nil {
+		glog.Errorf("failed to describe load balancers %v: %v", names, err)
+		return err
+	}
+	// Use a map for target groups to get unique target group entries across load balancers
+	targetGroups := map[string]*elbv2.TargetGroup{}
+	for _, loadBalancer := range lbsResponse.LoadBalancers {
+		glog.V(4).Infof("retrieving target groups for load balancer %q", *loadBalancer.LoadBalancerName)
+		targetGroupsInput := &elbv2.DescribeTargetGroupsInput{
+			LoadBalancerArn: loadBalancer.LoadBalancerArn,
+		}
+		targetGroupsOutput, err := client.ELBv2DescribeTargetGroups(targetGroupsInput)
+		if err != nil {
+			glog.Errorf("failed to retrieve load balancer target groups for %q: %v", *loadBalancer.LoadBalancerName, err)
+			return err
+		}
+		for _, targetGroup := range targetGroupsOutput.TargetGroups {
+			targetGroups[*targetGroup.TargetGroupArn] = targetGroup
+		}
+	}
+	if glog.V(4) {
+		targetGroupArns := make([]string, 0, len(targetGroups))
+		for arn := range targetGroups {
+			targetGroupArns = append(targetGroupArns, fmt.Sprintf("%q", arn))
+		}
+		glog.Infof("registering instance %q with target groups: %v", *instance.InstanceId, strings.Join(targetGroupArns, ","))
+	}
+	errs := []error{}
+	for _, targetGroup := range targetGroups {
+		var target *elbv2.TargetDescription
+		switch *targetGroup.TargetType {
+		case elbv2.TargetTypeEnumInstance:
+			target = &elbv2.TargetDescription{
+				Id: instance.InstanceId,
+			}
+		case elbv2.TargetTypeEnumIp:
+			target = &elbv2.TargetDescription{
+				Id: instance.PrivateIpAddress,
+			}
+		}
+		registerTargetsInput := &elbv2.RegisterTargetsInput{
+			TargetGroupArn: targetGroup.TargetGroupArn,
+			Targets:        []*elbv2.TargetDescription{target},
+		}
+		_, err := client.ELBv2RegisterTargets(registerTargetsInput)
+		if err != nil {
+			glog.Errorf("failed to register instance %q with target group %q: %v", *instance.InstanceId, *targetGroup.TargetGroupArn, err)
+			errs = append(errs, fmt.Errorf("%s: %v", *targetGroup.TargetGroupArn, err))
+		}
+	}
+	if len(errs) > 0 {
+		return errorutil.NewAggregate(errs)
 	}
 	return nil
 }

pkg/cloud/aws/actuators/machine/actuator_test.go

@@ -111,10 +111,19 @@ func testMachineAPIResources(clusterID string) (*clusterv1.Machine, *clusterv1.C
 			{ID: aws.String("sg-08b1ffd32874d59a2")}, // aws-actuator_infra_k8s
 		},
 		PublicIP: aws.Bool(true),
-		LoadBalancerNames: []string{
-			"cluster-con",
-			"cluster-ext",
-			"cluster-int",
+		LoadBalancers: []providerconfigv1.LoadBalancerReference{
+			{
+				Name: "cluster-con",
+				Type: providerconfigv1.ClassicLoadBalancerType,
+			},
+			{
+				Name: "cluster-ext",
+				Type: providerconfigv1.ClassicLoadBalancerType,
+			},
+			{
+				Name: "cluster-int",
+				Type: providerconfigv1.ClassicLoadBalancerType,
+			},
 		},
 	}
 

pkg/cloud/aws/client/client.go

@@ -29,6 +29,8 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
 	"github.com/aws/aws-sdk-go/service/elb"
 	"github.com/aws/aws-sdk-go/service/elb/elbiface"
+	"github.com/aws/aws-sdk-go/service/elbv2"
+	"github.com/aws/aws-sdk-go/service/elbv2/elbv2iface"
 )
 
 //go:generate mockgen -source=./client.go -destination=./mock/client_generated.go -package=mock
@@ -54,11 +56,15 @@ type Client interface {
 	TerminateInstances(*ec2.TerminateInstancesInput) (*ec2.TerminateInstancesOutput, error)
 
 	RegisterInstancesWithLoadBalancer(*elb.RegisterInstancesWithLoadBalancerInput) (*elb.RegisterInstancesWithLoadBalancerOutput, error)
+	ELBv2DescribeLoadBalancers(*elbv2.DescribeLoadBalancersInput) (*elbv2.DescribeLoadBalancersOutput, error)
+	ELBv2DescribeTargetGroups(*elbv2.DescribeTargetGroupsInput) (*elbv2.DescribeTargetGroupsOutput, error)
+	ELBv2RegisterTargets(*elbv2.RegisterTargetsInput) (*elbv2.RegisterTargetsOutput, error)
 }
 
 type awsClient struct {
-	ec2Client ec2iface.EC2API
-	elbClient elbiface.ELBAPI
+	ec2Client   ec2iface.EC2API
+	elbClient   elbiface.ELBAPI
+	elbv2Client elbv2iface.ELBV2API
 }
 
 func (c *awsClient) DescribeImages(input *ec2.DescribeImagesInput) (*ec2.DescribeImagesOutput, error) {
@@ -93,6 +99,18 @@ func (c *awsClient) RegisterInstancesWithLoadBalancer(input *elb.RegisterInstanc
 	return c.elbClient.RegisterInstancesWithLoadBalancer(input)
 }
 
+func (c *awsClient) ELBv2DescribeLoadBalancers(input *elbv2.DescribeLoadBalancersInput) (*elbv2.DescribeLoadBalancersOutput, error) {
+	return c.elbv2Client.DescribeLoadBalancers(input)
+}
+
+func (c *awsClient) ELBv2DescribeTargetGroups(input *elbv2.DescribeTargetGroupsInput) (*elbv2.DescribeTargetGroupsOutput, error) {
+	return c.elbv2Client.DescribeTargetGroups(input)
+}
+
+func (c *awsClient) ELBv2RegisterTargets(input *elbv2.RegisterTargetsInput) (*elbv2.RegisterTargetsOutput, error) {
+	return c.elbv2Client.RegisterTargets(input)
+}
+
 // NewClient creates our client wrapper object for the actual AWS clients we use.
 // For authentication the underlying clients will use either the cluster AWS credentials
 // secret if defined (i.e. in the root cluster),
@@ -127,7 +145,8 @@ func NewClient(kubeClient kubernetes.Interface, secretName, namespace, region st
 	}
 
 	return &awsClient{
-		ec2Client: ec2.New(s),
-		elbClient: elb.New(s),
+		ec2Client:   ec2.New(s),
+		elbClient:   elb.New(s),
+		elbv2Client: elbv2.New(s),
 	}, nil
 }

pkg/cloud/aws/client/fake/fake.go

@@ -6,6 +6,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/elb"
+	"github.com/aws/aws-sdk-go/service/elbv2"
 	"k8s.io/client-go/kubernetes"
 	"sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/aws/client"
 )
@@ -97,6 +98,21 @@ func (c *awsClient) RegisterInstancesWithLoadBalancer(input *elb.RegisterInstanc
 	return &elb.RegisterInstancesWithLoadBalancerOutput{}, nil
 }
 
+func (c *awsClient) ELBv2DescribeLoadBalancers(*elbv2.DescribeLoadBalancersInput) (*elbv2.DescribeLoadBalancersOutput, error) {
+	// Feel free to extend the returned values
+	return &elbv2.DescribeLoadBalancersOutput{}, nil
+}
+
+func (c *awsClient) ELBv2DescribeTargetGroups(*elbv2.DescribeTargetGroupsInput) (*elbv2.DescribeTargetGroupsOutput, error) {
+	// Feel free to extend the returned values
+	return &elbv2.DescribeTargetGroupsOutput{}, nil
+}
+
+func (c *awsClient) ELBv2RegisterTargets(*elbv2.RegisterTargetsInput) (*elbv2.RegisterTargetsOutput, error) {
+	// Feel free to extend the returned values
+	return &elbv2.RegisterTargetsOutput{}, nil
+}
+
 // NewClient creates our client wrapper object for the actual AWS clients we use.
 // For authentication the underlying clients will use either the cluster AWS credentials
 // secret if defined (i.e. in the root cluster),