resolved conflict aws-k8s-cni-*.yaml

Add VlanId in the cmdAdd Result struct
This VlanId will appear in the prevResult during cmdDel request

CleanUp Pod Network using vlanId from prevResult in CNI itself
No need to call ipamd

Log formatting changes

Added hostNetworking Setup test for pods using security groups

Updated cleanUpPodENI method

Skip processing Delete request if prevResult is nil
Add Logging vlanId to ipamd

Add support to test with containerd nodegroup in pod-eni test

Add check for empty Netns() in cni

Manifests and Readme updates (#1732)

* Manifests and Readme updates

* update manifest.jsonnet

Readme updates (#1735)

Updates to troubleshooting doc (#1737)

* Updates to troubleshooting doc

* updates to troubleshooting doc

imdsv2 changes (#1743)

fix flaky canary test (#1742)

add CODEOWNERS (#1747)

Snat tests: [agent is already updated] (#1513)

* resolved conflicts with go.sum

* Updated test agent image

* Removed redundant files

* Addressed PR comments

Fixed go.sum in root folder

Changed DescribeInstanceWithFilter to DescribeInstances
Moved GetPrimaryInstanceId from ec2 interface
Added GinkgoWriter

Updated Readme for Snat test

Rearranged snat_test logic
Updated Readme for test/e2e

* Minor change to logging

Fix compilation errors (#1751)

add support for running canary script in different regions (#1752)

Regenerate pod eni values for new instance types (#1754)

* Regenerate pod eni values for new instance types

Co-authored-by: Senthil Kumaran <[email protected]>

Minor change to container runtime argument

Check for Empty NetNs() first
Fallback to older method if prevResult is nil

Closed issue message (#1761)

* closed issue message

* update message

fix typo in upload script (#1763)

Update calico file path

Use an unique s3 bucket name (#1760)

Update region

Workflow to build arm and x86 images (#1764)

DataStore.GetStats() refactoring to simplify adding new fields (#1704)

* DataStore.GetStats() refactoring to simplify adding new fields

* cleanup

* cleanup

* cleanup

* goimports

* rename test to TestGetStatsV4

* address comments

* fix typo

* update

* update "IP pool is too low" logging

* GetStats() -> GetIpStats()

* GetStats() -> GetIpStats() in tests and comments

* update test

* cleanup test

* add logPoolStats comment

Fix KOPS_STATE_STORE (#1770)

Automation script for running IT  (#1759)

Update issue template

Update issue template with email address

Update issue template

Update go.mod for integration folder (#1741)

* Update go.mod for integration folder

- Update go.mod for integration folder

* Change integration test to use new K8s test framework

* Modify server pod image

* Switch to Nginx port 80 for server pod

* Switch server port in client test

* Remove custom command directive for Nginx pod

* Added ping command for host checks

README: mention arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy (#1768)

Co-authored-by: Shreya027 <[email protected]>

Add dl1.24xlarge to ENILimits override list (#1777)

Chart and Manifest updates (#1771)

* Chart and Manifest updates

* Update probe timeout values

Change workflow to use git install (#1785)

- Change workflow to use git install as the go get command was
  altering go.mod file without updating go.sum file

cmd/routed-eni-cni-plugin/cni.go

@@ -22,6 +22,7 @@ import (
 	"net"
 	"os"
 	"runtime"
+	"strconv"
 	"strings"
 
 	"github.com/containernetworking/cni/pkg/skel"
@@ -43,6 +44,7 @@ import (
 )
 
 const ipamdAddress = "127.0.0.1:50051"
+const vlanInterfaceName = "vlanId"
 
 var version string
 
@@ -95,6 +97,12 @@ func LoadNetConf(bytes []byte) (*NetConf, logger.Logger, error) {
 		return nil, nil, errors.Wrap(err, "add cmd: error loading config from args")
 	}
 
+	if conf.RawPrevResult != nil {
+		if err := cniSpecVersion.ParsePrevResult(&conf.NetConf); err != nil {
+			return nil, nil, fmt.Errorf("could not parse prevResult: %v", err)
+		}
+	}
+
 	logConfig := logger.Configuration{
 		LogLevel:    conf.PluginLogLevel,
 		LogLocation: conf.PluginLogFile,
@@ -122,6 +130,8 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
 	log.Infof("Received CNI add request: ContainerID(%s) Netns(%s) IfName(%s) Args(%s) Path(%s) argsStdinData(%s)",
 		args.ContainerID, args.Netns, args.IfName, args.Args, args.Path, args.StdinData)
 
+	log.Infof("Prev Result: %v\n", conf.PrevResult)
+
 	var k8sArgs K8sArgs
 	if err := cniTypes.LoadArgs(args.Args, &k8sArgs); err != nil {
 		log.Errorf("Failed to load k8s config from arg: %v", err)
@@ -194,14 +204,12 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
 	var hostVethName string
 	if r.PodVlanId != 0 {
 		hostVethName = generateHostVethName("vlan", string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
-
 		err = driverClient.SetupPodENINetwork(hostVethName, args.IfName, args.Netns, v4Addr, v6Addr, int(r.PodVlanId), r.PodENIMAC,
 			r.PodENISubnetGW, int(r.ParentIfIndex), mtu, log)
 	} else {
 		// build hostVethName
 		// Note: the maximum length for linux interface name is 15
 		hostVethName = generateHostVethName(conf.VethPrefix, string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
-
 		err = driverClient.SetupNS(hostVethName, args.IfName, args.Netns, v4Addr, v6Addr, int(r.DeviceNumber), r.VPCv4CIDRs, r.UseExternalSNAT, mtu, log)
 	}
 
@@ -241,12 +249,15 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
 
 	hostInterface := &current.Interface{Name: hostVethName}
 	containerInterface := &current.Interface{Name: args.IfName, Sandbox: args.Netns}
+	vlanInterface := &current.Interface{Name: vlanInterfaceName, Mac: fmt.Sprint(r.PodVlanId)}
+	log.Infof("Using vlanInterface: %v", vlanInterface)
 
 	result := &current.Result{
 		IPs: ips,
 		Interfaces: []*current.Interface{
 			hostInterface,
 			containerInterface,
+			vlanInterface,
 		},
 	}
 
@@ -270,6 +281,8 @@ func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
 	driverClient driver.NetworkAPIs) error {
 
 	conf, log, err := LoadNetConf(args.StdinData)
+	log.Infof("Prev Result: %v\n", conf.PrevResult)
+
 	if err != nil {
 		return errors.Wrap(err, "add cmd: error loading config from args")
 	}
@@ -283,6 +296,39 @@ func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
 		return errors.Wrap(err, "del cmd: failed to load k8s config from args")
 	}
 
+	if args.Netns == "" {
+		log.Info("Netns() is empty, so network already cleanedup. Nothing to do")
+		return nil
+	}
+	prevResult, ok := conf.PrevResult.(*current.Result)
+
+	// Try to use prevResult if available
+	// prevResult might not be availabe, if we are still using older cni spec < 0.4.0.
+	// So we should fallback to the old clean up method
+	if ok {
+		for _, iface := range prevResult.Interfaces {
+			if iface.Name == vlanInterfaceName {
+				podVlanId, err := strconv.Atoi(iface.Mac)
+				if err != nil {
+					return errors.Wrap(err, "Failed to parse vlanId from prevResult")
+				}
+				// podVlanId == 0 means pod is not using branch ENI
+				// then fallback to existing cleanup
+				if podVlanId == 0 {
+					break
+				}
+				// if podVlanId != 0 means pod is using branch ENI
+				err = cleanUpPodENI(podVlanId, log, args.ContainerID, driverClient)
+				if err != nil {
+					return err
+				}
+				log.Infof("Received del network response for pod %s namespace %s sandbox %s with vlanId: %v", string(k8sArgs.K8S_POD_NAME),
+					string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID), podVlanId)
+				return nil
+			}
+		}
+	}
+
 	// notify local IP address manager to free secondary IP
 	// Set up a connection to the server.
 	conn, err := grpcClient.Dial(ipamdAddress, grpc.WithInsecure())
@@ -362,6 +408,16 @@ func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
 	return nil
 }
 
+func cleanUpPodENI(podVlanId int, log logger.Logger, containerId string, driverClient driver.NetworkAPIs) error {
+	err := driverClient.TeardownPodENINetwork(podVlanId, log)
+	if err != nil {
+		log.Errorf("Failed on TeardownPodNetwork for container ID %s: %v",
+			containerId, err)
+		return errors.Wrap(err, "del cmd: failed on tear down pod network")
+	}
+	return nil
+}
+
 func main() {
 	log := logger.DefaultLogger()
 	about := fmt.Sprintf("AWS CNI %s", version)

misc/10-aws.conflist

@@ -1,5 +1,5 @@
 {
-  "cniVersion": "0.3.1",
+  "cniVersion": "0.4.0",
   "name": "aws-cni",
   "plugins": [
     {

pkg/ipamd/rpc_handler.go

@@ -110,6 +110,8 @@ func (s *server) AddNetwork(ctx context.Context, in *rpc.AddNetworkRequest) (*rp
 					ipv4Addr = firstENI.PrivateIP
 					branchENIMAC = firstENI.IfAddress
 					vlanID = firstENI.VlanID
+					log.Infof("Pod vlandId: %d", vlanID)
+
 					if ipv4Addr == "" || branchENIMAC == "" || vlanID == 0 {
 						log.Errorf("Failed to parse pod-ENI annotation: %s", val)
 						return &failureResponse, nil

test/e2e/pod-eni/securiy_group_per_pod_suite_test.go

@@ -23,8 +23,9 @@ import (
 	awsUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/aws/utils"
 	k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils"
 	"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
-
 	"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/aws/vpc"
+	v1 "k8s.io/api/core/v1"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -52,6 +53,8 @@ var (
 	clusterRoleName string
 	// NodeSecurityGroupId for Node-Node communication
 	nodeSecurityGroupID string
+
+	node v1.Node
 )
 
 func TestSecurityGroupForPods(t *testing.T) {
@@ -116,7 +119,7 @@ var _ = BeforeSuite(func() {
 
 	// Get ref to any node from newly created nodegroup
 	By("Getting providerID of the node")
-	node := nodeList.Items[0]
+	node = nodeList.Items[0]
 	providerID := node.Spec.ProviderID
 	Expect(len(providerID)).To(BeNumerically(">", 0))
 

test/e2e/pod-eni/securiy_group_per_pod_test.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/aws/amazon-vpc-cni-k8s/test/agent/pkg/input"
 	"github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/agent"
 	"github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest"
 	k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils"
@@ -32,13 +33,21 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
+type TestType int
+
+const (
+	NetworkingTearDownSucceeds TestType = iota
+	NetworkingSetupSucceeds
+)
+
 var _ = Describe("Security Group for Pods Test", func() {
 	var (
 		// The Pod labels for client and server in order to retrieve the
 		// client and server Pods belonging to a Deployment/Jobs
-		labelKey          = "app"
-		serverPodLabelVal = "server-pod"
-		clientPodLabelVal = "client-pod"
+		labelKey           = "app"
+		serverPodLabelVal  = "server-pod"
+		clientPodLabelVal  = "client-pod"
+		busyboxPodLabelVal = "busybox-pod"
 		// The Security Group Policy take list of Pod Label Value and if the
 		// Pod has any label in the list, it should get Branch ENI
 		branchPodLabelVal       []string
@@ -264,8 +273,107 @@ var _ = Describe("Security Group for Pods Test", func() {
 			It("TCP liveness probe will succeed", func() {})
 		})
 	})
+
+	Context("Verify HostNetworking", func() {
+		BeforeEach(func() {
+			// BusyBox Pods will get Branch ENI
+			branchPodLabelVal = []string{busyboxPodLabelVal}
+		})
+		It("Deploy BusyBox Pods with branch ENI and verify HostNetworking", func() {
+			deployment := manifest.NewBusyBoxDeploymentBuilder().
+				Replicas(totalBranchInterface/asgSize).
+				PodLabel(labelKey, busyboxPodLabelVal).
+				NodeName(node.Name).
+				Build()
+
+			By("creating a deployment to launch pod using Branch ENI")
+			_, err = f.K8sResourceManagers.DeploymentManager().
+				CreateAndWaitTillDeploymentIsReady(deployment, utils.DefaultDeploymentReadyTimeout)
+			Expect(err).ToNot(HaveOccurred())
+
+			By("getting the list of pods using BranchENI")
+			podList, err := f.K8sResourceManagers.
+				PodManager().
+				GetPodsWithLabelSelector(labelKey, busyboxPodLabelVal)
+			Expect(err).ToNot(HaveOccurred())
+
+			By("generating the pod networking validation input to be passed to tester")
+			input, err := GetPodNetworkingValidationInput(podList).Serialize()
+			Expect(err).NotTo(HaveOccurred())
+
+			By("validating host networking setup is setup correctly")
+			ValidateHostNetworking(NetworkingSetupSucceeds, input)
+
+			By("deleting the deployment to test teardown")
+			err = f.K8sResourceManagers.DeploymentManager().
+				DeleteAndWaitTillDeploymentIsDeleted(deployment)
+			Expect(err).ToNot(HaveOccurred())
+
+			By("waiting to allow CNI to tear down networking for terminated pods")
+			time.Sleep(time.Second * 60)
+
+			By("validating host networking is teared down correctly")
+			ValidateHostNetworking(NetworkingTearDownSucceeds, input)
+		})
+	})
 })
 
+func GetPodNetworkingValidationInput(podList v1.PodList) input.PodNetworkingValidationInput {
+	ip := input.PodNetworkingValidationInput{
+		VethPrefix:  "vlan",
+		PodList:     []input.Pod{},
+		ValidateMTU: true,
+		MTU:         9001,
+	}
+
+	for _, pod := range podList.Items {
+		ip.PodList = append(ip.PodList, input.Pod{
+			PodName:        pod.Name,
+			PodNamespace:   pod.Namespace,
+			PodIPv4Address: pod.Status.PodIP,
+		})
+	}
+	return ip
+}
+
+func ValidateHostNetworking(testType TestType, podValidationInputString string) {
+	testerArgs := []string{fmt.Sprintf("-pod-networking-validation-input=%s",
+		podValidationInputString)}
+
+	if NetworkingSetupSucceeds == testType {
+		testerArgs = append(testerArgs, "-test-setup=true", "-test-ppsg=true")
+	} else if NetworkingTearDownSucceeds == testType {
+		testerArgs = append(testerArgs, "-test-cleanup=true", "-test-ppsg=true")
+	}
+
+	testContainer := manifest.NewTestHelperContainer().
+		Command([]string{"./networking"}).
+		Args(testerArgs).
+		Build()
+
+	testPod := manifest.NewDefaultPodBuilder().
+		Container(testContainer).
+		NodeName(node.Name).
+		HostNetwork(true).
+		Build()
+
+	By("creating pod to test host networking setup")
+	testPod, err := f.K8sResourceManagers.PodManager().
+		CreateAndWaitTillPodCompleted(testPod)
+	Expect(err).ToNot(HaveOccurred())
+
+	logs, errLogs := f.K8sResourceManagers.PodManager().
+		PodLogs(testPod.Namespace, testPod.Name)
+	Expect(errLogs).ToNot(HaveOccurred())
+
+	fmt.Fprintln(GinkgoWriter, logs)
+
+	By("deleting the host networking setup pod")
+	err = f.K8sResourceManagers.PodManager().
+		DeleteAndWaitTillPodDeleted(testPod)
+	Expect(err).ToNot(HaveOccurred())
+}
+
 func ValidatePodsHaveBranchENI(podList v1.PodList) error {
 	for _, pod := range podList.Items {
 		if val, ok := pod.Annotations["vpc.amazonaws.com/pod-eni"]; ok {

test/framework/resources/aws/utils/nodegroup.go

@@ -30,6 +30,11 @@ import (
 
 const CreateNodeGroupCFNTemplateURL = "https://raw.githubusercontent.com/awslabs/amazon-eks-ami/master/amazon-eks-nodegroup.yaml"
 
+// Docker will be default, if not specified
+const (
+	CONTAINERD = "containerd"
+)
+
 type NodeGroupProperties struct {
 	// Required to verify the node is up and ready
 	NgLabelKey string
@@ -44,6 +49,9 @@ type NodeGroupProperties struct {
 	Subnet       []string
 	InstanceType string
 	KeyPairName  string
+
+	// optional: specify container runtime
+	ContainerRuntime string
 }
 
 type ClusterVPCConfig struct {
@@ -95,6 +103,11 @@ func CreateAndWaitTillSelfManagedNGReady(f *framework.Framework, properties Node
 		kubeletExtraArgs += fmt.Sprintf(" --max-pods=%d", maxPods)
 	}
 
+	containerRuntime := properties.ContainerRuntime
+	if containerRuntime != "" {
+		bootstrapArgs += fmt.Sprintf(" --container-runtime %s", containerRuntime)
+	}
+
 	asgSizeString := strconv.Itoa(properties.AsgSize)
 
 	createNgStackParams := []*cloudformation.Parameter{

test/framework/resources/k8s/utils/daemonset.go

@@ -76,7 +76,7 @@ func updateDaemonsetEnvVarsAndWait(f *framework.Framework, dsName string, dsName
 	if dsName != utils.MultusNodeName {
 		_, err := f.K8sResourceManagers.DaemonSetManager().GetDaemonSet(dsNamespace, utils.MultusNodeName)
 		if err == nil {
-			By("Restarting Multus daemonset")
+			By("Restarting Multus daemonset to use the update aws-node changes")
 			td := time.Now()
 			updateDaemonsetEnvVarsAndWait(f, utils.MultusNodeName, dsNamespace, utils.MultusContainerName, map[string]string{
 				"forceUpdatedAt": td.String(),