Allow additional CIDRs to be excluded from SNAT

.travis.yml

@@ -23,6 +23,7 @@ install:
 
 # Tests to run
 script:
+  - GO111MODULE=on make check-format
   - GO111MODULE=on make build-linux
   - GO111MODULE=on make lint
   - GO111MODULE=on make vet

Makefile

@@ -12,7 +12,7 @@
 # language governing permissions and limitations under the License.
 #
 
-.PHONY: all build-linux clean docker docker-build lint unit-test vet download-portmap build-docker-test build-metrics docker-metrics metrics-unit-test docker-metrics-test docker-vet
+.PHONY: all build-linux clean format check-format docker docker-build lint unit-test vet download-portmap build-docker-test build-metrics docker-metrics metrics-unit-test docker-metrics-test docker-vet
 
 IMAGE   ?= amazon/amazon-k8s-cni
 VERSION ?= $(shell git describe --tags --always --dirty)
@@ -123,3 +123,19 @@ clean:
 	rm -f aws-cni
 	rm -f cni-metrics-helper/cni-metrics-helper
 	rm -f portmap
+
+files := $(shell find . -path ./vendor -prune -or -not -name 'mock_publisher.go' -name '*.go' -print)
+unformatted = $(shell goimports -l $(files))
+
+format :
+	@echo "== format"
+	@goimports -w $(files)
+	@sync
+
+check-format :
+	@echo "== check formatting"
+ifneq "$(unformatted)" ""
+	@echo "needs formatting: $(unformatted)"
+	@echo "run 'make format'"
+	@exit 1
+endif

README.md

@@ -151,6 +151,12 @@ Disable (`none`) this functionality if you rely on sequential port allocation fo
 
 *Note*: Any options other than `none` will cause outbound connections to be assigned a source port that's not necessarily part of the ephemeral port range set at the OS level (/proc/sys/net/ipv4/ip_local_port_range). This is relevant for any customers that might have NACLs restricting traffic based on the port range found in ip_local_port_range
 
+`AWS_VPC_K8S_CNI_EXCLUDE_SNAT_CIDRS`
+Type: String
+Default: empty
+Specify a comma separated list of IPv4 CIDRs to exclude from SNAT. For every item in the list an `iptables` rule and off\-VPC
+IP rule will be applied. If an item is not a valid ipv4 range it will be skipped. This should be used when `AWS_VPC_K8S_CNI_EXTERNALSNAT=false`.
+
 `WARM_ENI_TARGET`
 Type: Integer
 Default: `1`

ipamd/ipamd.go

@@ -672,7 +672,7 @@ func (c *IPAMContext) tryAssignIPs() (increasedPool bool, err error) {
 		log.Debugf("Found ENI %s that has less than the maximum number of IP addresses allocated: cur=%d, max=%d", eni.ID, currentNumberOfAllocatedIPs, c.maxIPsPerENI)
 		// Try to allocate all available IPs for this ENI
 		// TODO: Retry with back-off, trying with half the number of IPs each time
-		err = c.awsClient.AllocIPAddresses(eni.ID, c.maxIPsPerENI- currentNumberOfAllocatedIPs)
+		err = c.awsClient.AllocIPAddresses(eni.ID, c.maxIPsPerENI-currentNumberOfAllocatedIPs)
 		if err != nil {
 			log.Warnf("failed to allocate all available IP addresses on ENI %s, err: %v", eni.ID, err)
 			// Try to just get one more IP
@@ -692,7 +692,7 @@ func (c *IPAMContext) tryAssignIPs() (increasedPool bool, err error) {
 		return true, nil
 	}
 	return false, nil
-} 
+}
 
 // setupENI does following:
 // 1) add ENI to datastore

ipamd/ipamd_test.go

@@ -24,13 +24,13 @@ import (
 	"github.com/aws/amazon-vpc-cni-k8s/ipamd/datastore"
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1"
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils"
-	"github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils/mocks"
+	mock_awsutils "github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils/mocks"
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/docker"
-	"github.com/aws/amazon-vpc-cni-k8s/pkg/docker/mocks"
-	"github.com/aws/amazon-vpc-cni-k8s/pkg/eniconfig/mocks"
+	mock_docker "github.com/aws/amazon-vpc-cni-k8s/pkg/docker/mocks"
+	mock_eniconfig "github.com/aws/amazon-vpc-cni-k8s/pkg/eniconfig/mocks"
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/k8sapi"
-	"github.com/aws/amazon-vpc-cni-k8s/pkg/k8sapi/mocks"
-	"github.com/aws/amazon-vpc-cni-k8s/pkg/networkutils/mocks"
+	mock_k8sapi "github.com/aws/amazon-vpc-cni-k8s/pkg/k8sapi/mocks"
+	mock_networkutils "github.com/aws/amazon-vpc-cni-k8s/pkg/networkutils/mocks"
 
 	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/assert"
@@ -237,9 +237,9 @@ func testIncreaseIPPool(t *testing.T, useENIConfig bool) {
 
 	mockAWS.EXPECT().DescribeENI(eni2).Return(
 		[]*ec2.NetworkInterfacePrivateIpAddress{
-			{ PrivateIpAddress: &testAddr11, Primary: &primary },
-			{ PrivateIpAddress: &testAddr12, Primary: &notPrimary },
-			{ PrivateIpAddress: &testAddr12, Primary: &notPrimary },
+			{PrivateIpAddress: &testAddr11, Primary: &primary},
+			{PrivateIpAddress: &testAddr12, Primary: &notPrimary},
+			{PrivateIpAddress: &testAddr12, Primary: &notPrimary},
 		},
 		&attachmentID, nil)
 
@@ -302,9 +302,9 @@ func TestTryAddIPToENI(t *testing.T) {
 	mockAWS.EXPECT().GetPrimaryENI().Return(primaryENIid)
 	mockAWS.EXPECT().DescribeENI(secENIid).Return(
 		[]*ec2.NetworkInterfacePrivateIpAddress{
-			{ PrivateIpAddress: &testAddr11, Primary: &primary },
-			{ PrivateIpAddress: &testAddr12, Primary: &notPrimary },
-			{ PrivateIpAddress: &testAddr12, Primary: &notPrimary },
+			{PrivateIpAddress: &testAddr11, Primary: &primary},
+			{PrivateIpAddress: &testAddr12, Primary: &notPrimary},
+			{PrivateIpAddress: &testAddr12, Primary: &notPrimary},
 		},
 		&attachmentID, nil)
 

ipamd/rpc_handler.go

@@ -54,12 +54,20 @@ func (s *server) AddNetwork(ctx context.Context, in *pb.AddNetworkRequest) (*pb.
 		pbVPCcidrs = append(pbVPCcidrs, *cidr)
 	}
 
+	useExternalSNAT := s.ipamContext.networkClient.UseExternalSNAT()
+	if !useExternalSNAT {
+		for _, cidr := range s.ipamContext.networkClient.GetExcludeSNATCIDRs() {
+			log.Debugf("CIDR SNAT Exclusion %s", cidr)
+			pbVPCcidrs = append(pbVPCcidrs, cidr)
+		}
+	}
+
 	resp := pb.AddNetworkReply{
 		Success:         err == nil,
 		IPv4Addr:        addr,
 		IPv4Subnet:      "",
 		DeviceNumber:    int32(deviceNumber),
-		UseExternalSNAT: s.ipamContext.networkClient.UseExternalSNAT(),
+		UseExternalSNAT: useExternalSNAT,
 		VPCcidrs:        pbVPCcidrs,
 	}
 

ipamd/rpc_handler_test.go

@@ -0,0 +1,93 @@
+// Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//     http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package ipamd
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/amazon-vpc-cni-k8s/ipamd/datastore"
+	"github.com/aws/aws-sdk-go/aws"
+
+	pb "github.com/aws/amazon-vpc-cni-k8s/rpc"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestServer_AddNetwork(t *testing.T) {
+	ctrl, mockAWS, mockK8S, mockDocker, mockNetwork, _ := setup(t)
+	defer ctrl.Finish()
+
+	mockContext := &IPAMContext{
+		awsClient:     mockAWS,
+		k8sClient:     mockK8S,
+		maxIPsPerENI:  14,
+		maxENI:        4,
+		warmENITarget: 1,
+		warmIPTarget:  3,
+		dockerClient:  mockDocker,
+		networkClient: mockNetwork,
+		dataStore:     datastore.NewDataStore(),
+	}
+
+	rpcServer := server{ipamContext: mockContext}
+
+	addNetworkRequest := &pb.AddNetworkRequest{
+		Netns:                      "netns",
+		K8S_POD_NAME:               "pod",
+		K8S_POD_NAMESPACE:          "ns",
+		K8S_POD_INFRA_CONTAINER_ID: "cid",
+		IfName:                     "eni",
+	}
+
+	vpcCIDRs := []*string{aws.String(vpcCIDR)}
+	testCases := []struct {
+		name               string
+		useExternalSNAT    bool
+		vpcCIDRs           []*string
+		snatExclusionCIDRs []string
+	}{
+		{
+			"VPC CIDRs",
+			true,
+			vpcCIDRs,
+			nil,
+		},
+		{
+			"SNAT Exclusion CIDRs",
+			false,
+			vpcCIDRs,
+			[]string{"10.12.0.0/16", "10.13.0.0/16"},
+		},
+	}
+	for _, tc := range testCases {
+		mockAWS.EXPECT().GetVPCIPv4CIDRs().Return(tc.vpcCIDRs)
+		mockNetwork.EXPECT().UseExternalSNAT().Return(tc.useExternalSNAT)
+		if !tc.useExternalSNAT {
+			mockNetwork.EXPECT().GetExcludeSNATCIDRs().Return(tc.snatExclusionCIDRs)
+		}
+
+		addNetworkReply, err := rpcServer.AddNetwork(context.TODO(), addNetworkRequest)
+		assert.NoError(t, err, tc.name)
+
+		assert.Equal(t, tc.useExternalSNAT, addNetworkReply.UseExternalSNAT, tc.name)
+
+		var expectedCIDRs []string
+		for _, cidr := range tc.vpcCIDRs {
+			expectedCIDRs = append(expectedCIDRs, *cidr)
+		}
+		expectedCIDRs = append([]string{vpcCIDR}, tc.snatExclusionCIDRs...)
+		assert.Equal(t, expectedCIDRs, addNetworkReply.VPCcidrs, tc.name)
+	}
+}

pkg/awsutils/awsutils_test.go

@@ -24,8 +24,8 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 
-	"github.com/aws/amazon-vpc-cni-k8s/pkg/ec2metadata/mocks"
-	"github.com/aws/amazon-vpc-cni-k8s/pkg/ec2wrapper/mocks"
+	mock_ec2metadata "github.com/aws/amazon-vpc-cni-k8s/pkg/ec2metadata/mocks"
+	mock_ec2wrapper "github.com/aws/amazon-vpc-cni-k8s/pkg/ec2wrapper/mocks"
 )
 
 const (

pkg/ec2metadatawrapper/ec2metadatawrapper_test.go

@@ -1,10 +1,11 @@
 package ec2metadatawrapper
 
 import (
-	mockec2metadatawrapper "github.com/aws/amazon-vpc-cni-k8s/pkg/ec2metadatawrapper/mocks"
 	"testing"
 	"time"
 
+	mockec2metadatawrapper "github.com/aws/amazon-vpc-cni-k8s/pkg/ec2metadatawrapper/mocks"
+
 	"github.com/aws/aws-sdk-go/aws/ec2metadata"
 
 	"github.com/golang/mock/gomock"

pkg/k8sapi/discovery.go

@@ -15,7 +15,7 @@ import (
 	clientset "k8s.io/client-go/kubernetes"
 
 	"github.com/operator-framework/operator-sdk/pkg/k8sclient"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"

pkg/netlinkwrapper/netlink.go

@@ -14,8 +14,9 @@
 package netlinkwrapper
 
 import (
-	"github.com/vishvananda/netlink"
 	"syscall"
+
+	"github.com/vishvananda/netlink"
 )
 
 // NetLink wraps methods used from the vishvananda/netlink package