Fix deletion of hostVeth rule for pods using security group

cmd/routed-eni-cni-plugin/driver/driver.go

@@ -313,14 +313,28 @@ func (os *linuxNetwork) SetupPodENINetwork(hostVethName string, contVethName str
 	vlanTableID := vlanID + 100
 	vlanLink := buildVlanLink(vlanID, parentIfIndex, eniMAC)
 
-	// 1. clean up if vlan already exists (necessary when trunk ENI changes).
+	// 1a. clean up if vlan already exists (necessary when trunk ENI changes).
 	if oldVlan, err := os.netLink.LinkByName(vlanLink.Name); err == nil {
 		if err = os.netLink.LinkDel(oldVlan); err != nil {
 			return errors.Wrapf(err, "SetupPodENINetwork: failed to delete old vlan %s", vlanLink.Name)
 		}
 		log.Debugf("Cleaned up old vlan: %s", vlanLink.Name)
 	}
 
+	// 1b. clean up any previous hostVeth ip rule
+	oldVlanRule := os.netLink.NewRule()
+	oldVlanRule.IifName = hostVethName
+	oldVlanRule.Priority = vlanRulePriority
+	// loop is required to clean up all existing rules created on the host (when pod with same name are recreated multiple times)
+	for {
+		if err := os.netLink.RuleDel(oldVlanRule); err != nil {
+			if !containsNoSuchRule(err) {
+				return errors.Wrapf(err, "SetupPodENINetwork: failed to delete hostveth rule for %s", hostVeth.Attrs().Name)
+			}
+			break
+		}
+	}
+
 	// 2. add new vlan link
 	err = os.netLink.LinkAdd(vlanLink)
 	if err != nil {

cmd/routed-eni-cni-plugin/driver/driver_test.go

@@ -514,6 +514,15 @@ func (m *testMocks) mockSetupPodENINetworkWithFailureAt(t *testing.T, addr *net.
 	m.netlink.EXPECT().LinkByName(testVlanName).Return(nil,
 		errors.New("link not found"))
 
+	actualRule := &netlink.Rule{}
+	m.netlink.EXPECT().NewRule().Return(actualRule)
+
+	oldVethRule := &netlink.Rule{
+		IifName:  testHostVethName,
+		Priority: vlanRulePriority,
+	}
+	m.netlink.EXPECT().RuleDel(gomock.Eq(oldVethRule)).Return(syscall.ENOENT)
+
 	vlanLink := buildVlanLink(1, 2, "eniMacAddress")
 	// add the link
 	m.netlink.EXPECT().LinkAdd(gomock.Eq(vlanLink)).Return(nil)
@@ -544,7 +553,6 @@ func (m *testMocks) mockSetupPodENINetworkWithFailureAt(t *testing.T, addr *net.
 	}
 	m.netlink.EXPECT().RouteReplace(gomock.Eq(&route)).Return(nil)
 
-	actualRule := &netlink.Rule{}
 	m.netlink.EXPECT().NewRule().Return(actualRule)
 
 	// add two ip rules based on iff interfaces