Cilium ENI ipam mode support (#3556)

* unmanage interfaces * add permissions for cilium in ENI mode * add permissions for cilium in ENI mode * add permissions for cilium in ENI mode * add permissions for cilium in ENI mode * add permissions for cilium in ENI mode * fix changelog * fix golden files * ignore CVE * disable kubernertes ip pool on eni mode * Ensure calico image is at least 3.22.3 before upgrading to cilium in `eni` IPAM mode. * Ensure calico image is at least 3.22.3 before upgrading to cilium in `eni` IPAM mode. * disable calico upgrade * remove calico upgrade before cilium switch * Only run calico and kube-proxy on old nodes during migration to cilium. * rebase * rebase * rebase * Only run aws-node, calico and kube-proxy on old nodes during migration to cilium. * Only run aws-node, calico and kube-proxy on old nodes during migration to cilium. --------- Co-authored-by: Christian Bianchi <[email protected]>
giantswarm · Sep 1, 2023 · e66125a · e66125a
1 parent 9fced78
commit e66125a
Show file tree

Hide file tree

Showing 13 changed files with 116 additions and 9 deletions.
diff --git a/.nancy-ignore b/.nancy-ignore
@@ -31,3 +31,4 @@ CVE-2023-28642
 CVE-2023-26125
 CVE-2023-29401
 CVE-2023-32731
+CVE-2023-3978
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -14,6 +14,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Allow newer flatcar releases for node pools as provided by AWS release.
 - Add `sigs.k8s.io/cluster-api-provider-aws/role` tag to all subnets as preparation for migration to CAPI.
 
+### Changed
+
+- Unmanage interfaces for CNI eth[1-9] on workers eth[2-9] on masters
+- [cilium eni mode] Only run aws-node, calico and kube-proxy on old nodes during migration to cilium.
+
 ## [14.19.2] - 2023-08-03
 
 ### Fixed

diff --git a/service/controller/cluster.go b/service/controller/cluster.go
@@ -374,7 +374,8 @@ func newClusterResources(config ClusterConfig) ([]resource.Interface, error) {
 	var restrictAwsNodeDaemonsetResource resource.Interface
 	{
 		c := restrictawsnodedaemonset.Config{
-			Logger: config.Logger,
+			CtrlClient: config.K8sClient.CtrlClient(),
+			Logger:     config.Logger,
 		}
 
 		restrictAwsNodeDaemonsetResource, err = restrictawsnodedaemonset.New(c)

diff --git a/service/controller/key/cluster.go b/service/controller/key/cluster.go
@@ -228,11 +228,25 @@ func IsAWSCNINeeded(cluster apiv1beta1.Cluster) bool {
 		return true
 	}
 
+	if IsCiliumEniModeEnabled(cluster) {
+		return true
+	}
+
 	_, needed := cluster.Annotations[annotation.CiliumPodCidr]
 
 	return needed
 }
 
+func IsCiliumEniModeEnabled(cluster apiv1beta1.Cluster) bool {
+	mode, found := cluster.Annotations[annotation.CiliumIpamModeAnnotation]
+	if !found {
+		// we default to 'kubernetes' mode
+		return false
+	}
+
+	return mode == annotation.CiliumIpamModeENI
+}
+
 func MasterAvailabilityZone(cluster infrastructurev1alpha3.AWSCluster) string {
 	return cluster.Spec.Provider.Master.AvailabilityZone
 }

diff --git a/service/controller/resource/restrictawsnodedaemonset/create.go b/service/controller/resource/restrictawsnodedaemonset/create.go
@@ -7,6 +7,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	apiv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/giantswarm/aws-operator/v14/pkg/label"
@@ -16,9 +17,10 @@ import (
 )
 
 const (
-	dsNamespace     = "kube-system"
-	awsNodeDsName   = "aws-node"
-	KubeProxyDsName = "kube-proxy"
+	dsNamespace      = "kube-system"
+	awsNodeDsName    = "aws-node"
+	calicoNodeDsName = "calico-node"
+	kubeProxyDsName  = "kube-proxy"
 )
 
 func (r *Resource) EnsureCreated(ctx context.Context, obj interface{}) error {
@@ -55,7 +57,20 @@ func (r *Resource) EnsureCreated(ctx context.Context, obj interface{}) error {
 
 	ctrlClient := cc.Client.TenantCluster.K8s.CtrlClient()
 
-	for _, dsName := range []string{awsNodeDsName} {
+	// Only run this if the Cluster CR has the cilium pod annotation
+	cluster := apiv1beta1.Cluster{}
+	err = r.ctrlClient.Get(ctx, client.ObjectKey{Namespace: cr.Namespace, Name: key.ClusterID(&cr)}, &cluster)
+	if err != nil {
+		return microerror.Mask(err)
+	}
+
+	daemonsetNames := []string{awsNodeDsName}
+	if key.IsCiliumEniModeEnabled(cluster) {
+		daemonsetNames = append(daemonsetNames, kubeProxyDsName)
+		daemonsetNames = append(daemonsetNames, calicoNodeDsName)
+	}
+
+	for _, dsName := range daemonsetNames {
 		ds := appsv1.DaemonSet{}
 		err = ctrlClient.Get(ctx, client.ObjectKey{
 			Namespace: dsNamespace,

diff --git a/service/controller/resource/restrictawsnodedaemonset/resource.go b/service/controller/resource/restrictawsnodedaemonset/resource.go
@@ -3,28 +3,35 @@ package restrictawsnodedaemonset
 import (
 	"github.com/giantswarm/microerror"
 	"github.com/giantswarm/micrologger"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const (
 	Name = "restrictawsnodedaemonset"
 )
 
 type Config struct {
-	Logger micrologger.Logger
+	CtrlClient client.Client
+	Logger     micrologger.Logger
 }
 
 // Resource that ensures the `aws-node` daemonset in the WC only creates pods in old nodes, not upgraded ones.
 type Resource struct {
-	logger micrologger.Logger
+	ctrlClient client.Client
+	logger     micrologger.Logger
 }
 
 func New(config Config) (*Resource, error) {
+	if config.CtrlClient == nil {
+		return nil, microerror.Maskf(invalidConfigError, "%T.CtrlClient must not be empty", config)
+	}
 	if config.Logger == nil {
 		return nil, microerror.Maskf(invalidConfigError, "%T.Logger must not be empty", config)
 	}
 
 	r := &Resource{
-		logger: config.Logger,
+		ctrlClient: config.CtrlClient,
+		logger:     config.Logger,
 	}
 
 	return r, nil

diff --git a/service/controller/resource/tcnp/create.go b/service/controller/resource/tcnp/create.go
@@ -489,6 +489,7 @@ func (r *Resource) newIAMPolicies(ctx context.Context, cr infrastructurev1alpha3
 			},
 			EC2ServiceDomain: key.EC2ServiceDomain(cc.Status.TenantCluster.AWS.Region),
 			EnableAWSCNI:     key.IsAWSCNINeeded(cluster),
+			CiliumENIMode:    key.IsCiliumEniModeEnabled(cluster),
 			KMSKeyARN:        ek,
 			NodePool: template.ParamsMainIAMPoliciesNodePool{
 				ID: key.MachineDeploymentID(&cr),

diff --git a/service/controller/resource/tcnp/template/params_main_iam_policies.go b/service/controller/resource/tcnp/template/params_main_iam_policies.go
@@ -4,6 +4,7 @@ type ParamsMainIAMPolicies struct {
 	Cluster          ParamsMainIAMPoliciesCluster
 	EC2ServiceDomain string
 	EnableAWSCNI     bool
+	CiliumENIMode    bool
 	KMSKeyARN        string
 	NodePool         ParamsMainIAMPoliciesNodePool
 	RegionARN        string

diff --git a/service/controller/resource/tcnp/template/template_main_iam_policies.go b/service/controller/resource/tcnp/template/template_main_iam_policies.go
@@ -82,6 +82,27 @@ const TemplateMainIAMPolicies = `
               - arn:{{ .IAMPolicies.RegionARN }}:ec2:*:*:network-interface/*
           {{- end }}
 
+          {{- if .IAMPolicies.CiliumENIMode }}
+          # Following rules are required to make the Cilium in AWS ENI mode work. See also
+          # https://docs.cilium.io/en/v1.13/network/concepts/ipam/eni/#required-privileges
+          - Effect: Allow
+            Action:
+              - ec2:DescribeInstances
+              - ec2:CreateTags
+              - ec2:ModifyNetworkInterfaceAttribute
+              - ec2:DeleteNetworkInterface
+              - ec2:AssignPrivateIpAddresses
+              - ec2:DescribeSecurityGroups
+              - ec2:CreateNetworkInterface
+              - ec2:DescribeNetworkInterfaces
+              - ec2:DescribeVpcs
+              - ec2:DescribeInstanceTypes
+              - ec2:AttachNetworkInterface
+              - ec2:UnassignPrivateIpAddresses
+              - ec2:DescribeSubnets
+            Resource: "*"
+          {{- end }}
+
           # Following rules are required for EBS snapshots.
           - Effect: Allow
             Action:

diff --git a/service/internal/cloudconfig/tccpn.go b/service/internal/cloudconfig/tccpn.go
@@ -391,7 +391,7 @@ func (t *TCCPN) newTemplate(ctx context.Context, obj interface{}, mapping hamast
 	}
 
 	var controllerManagerExtraArgs []string
-	if hasCilium {
+	if hasCilium && !key.IsCiliumEniModeEnabled(cluster) {
 		controllerManagerExtraArgs = append(controllerManagerExtraArgs, "--allocate-node-cidrs=true")
 		controllerManagerExtraArgs = append(controllerManagerExtraArgs, "--cluster-cidr="+podCidr)
 		controllerManagerExtraArgs = append(controllerManagerExtraArgs, "--node-cidr-mask-size=25")

diff --git a/service/internal/cloudconfig/tccpn_extension.go b/service/internal/cloudconfig/tccpn_extension.go
@@ -3,6 +3,7 @@ package cloudconfig
 import (
 	"context"
 	"encoding/base64"
+	"fmt"
 
 	"github.com/blang/semver"
 	infrastructurev1alpha3 "github.com/giantswarm/apiextensions/v6/pkg/apis/infrastructure/v1alpha3"
@@ -167,6 +168,21 @@ func (e *TCCPNExtension) Files() ([]k8scloudconfig.FileAsset, error) {
 			},
 			Permissions: 0644,
 		},
+		{
+			// on worker NIC eth0  is used for machine and all other eth interfaces are for aws cni
+			// add configuration for systemd-network to ignore aws cni interfaces
+			AssetContent: fmt.Sprintf(template.NetworkdIgnoreAWSCNiInterfaces, "eth[2-9]*"),
+			Path:         "/etc/systemd/network/00-ignore-aws-cni-interfaces.network",
+			Owner: k8scloudconfig.Owner{
+				Group: k8scloudconfig.Group{
+					Name: FileOwnerGroupName,
+				},
+				User: k8scloudconfig.User{
+					Name: FileOwnerUserName,
+				},
+			},
+			Permissions: 0644,
+		},
 	}
 
 	// TODO we install etcd-cluster-migrator in every case of HA masters. The etcd-cluster-migrator app

diff --git a/service/internal/cloudconfig/tcnp_extension.go b/service/internal/cloudconfig/tcnp_extension.go
@@ -3,6 +3,7 @@ package cloudconfig
 import (
 	"context"
 	"encoding/base64"
+	"fmt"
 
 	infrastructurev1alpha3 "github.com/giantswarm/apiextensions/v6/pkg/apis/infrastructure/v1alpha3"
 	g8sv1alpha1 "github.com/giantswarm/apiextensions/v6/pkg/apis/provider/v1alpha1"
@@ -61,6 +62,21 @@ func (e *TCNPExtension) Files() ([]k8scloudconfig.FileAsset, error) {
 			},
 			Permissions: 0700,
 		},
+		{
+			// on worker NIC eth0  is used for machine and all other eth interfaces are for aws cni
+			// add configuration for systemd-network to ignore aws cni interfaces
+			AssetContent: fmt.Sprintf(template.NetworkdIgnoreAWSCNiInterfaces, "eth[1-9]*"),
+			Path:         "/etc/systemd/network/00-ignore-aws-cni-interfaces.network",
+			Owner: k8scloudconfig.Owner{
+				Group: k8scloudconfig.Group{
+					Name: FileOwnerGroupName,
+				},
+				User: k8scloudconfig.User{
+					Name: FileOwnerUserName,
+				},
+			},
+			Permissions: 0644,
+		},
 	}
 
 	certsMeta := []k8scloudconfig.FileMetadata{}

diff --git a/service/internal/cloudconfig/template/netwokd-ignore-aws-cni.go b/service/internal/cloudconfig/template/netwokd-ignore-aws-cni.go
@@ -0,0 +1,9 @@
+package template
+
+const NetworkdIgnoreAWSCNiInterfaces = `
+[Match]
+Name=%s
+
+[Link]
+Unmanaged=yes
+`