fix: route issues on Swiftv2 Windows

cns/middlewares/k8sSwiftV2.go

@@ -249,3 +249,49 @@ func (k *K8sSWIFTv2Middleware) getIPConfig(ctx context.Context, podInfo cns.PodI
 func (k *K8sSWIFTv2Middleware) Type() cns.SWIFTV2Mode {
 	return cns.K8sSWIFTV2
 }
+
+// CNS gets node, pod and service CIDRs from configuration env and parse them to get the v4 and v6 IPs
+func (k *K8sSWIFTv2Middleware) GetCidrs() (v4IPs, v6IPs []string, err error) {
+	v4IPs = []string{}
+	v6IPs = []string{}
+
+	// Get and parse infraVNETCIDRs from env
+	infraVNETCIDRs, err := configuration.InfraVNETCIDRs()
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "failed to get infraVNETCIDRs from env")
+	}
+	infraVNETCIDRsv4, infraVNETCIDRsv6, err := utils.ParseCIDRs(infraVNETCIDRs)
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "failed to parse infraVNETCIDRs")
+	}
+
+	// Get and parse podCIDRs from env
+	podCIDRs, err := configuration.PodCIDRs()
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "failed to get podCIDRs from env")
+	}
+	podCIDRsV4, podCIDRv6, err := utils.ParseCIDRs(podCIDRs)
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "failed to parse podCIDRs")
+	}
+
+	// Get and parse serviceCIDRs from env
+	serviceCIDRs, err := configuration.ServiceCIDRs()
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "failed to get serviceCIDRs from env")
+	}
+	serviceCIDRsV4, serviceCIDRsV6, err := utils.ParseCIDRs(serviceCIDRs)
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "failed to parse serviceCIDRs")
+	}
+
+	v4IPs = append(v4IPs, infraVNETCIDRsv4...)
+	v4IPs = append(v4IPs, podCIDRsV4...)
+	v4IPs = append(v4IPs, serviceCIDRsV4...)
+
+	v6IPs = append(v6IPs, infraVNETCIDRsv6...)
+	v6IPs = append(v6IPs, podCIDRv6...)
+	v6IPs = append(v6IPs, serviceCIDRsV6...)
+
+	return v4IPs, v6IPs, nil
+}

cns/middlewares/k8sSwiftV2_linux.go

@@ -5,9 +5,7 @@ import (
 	"net/netip"
 
 	"github.com/Azure/azure-container-networking/cns"
-	"github.com/Azure/azure-container-networking/cns/configuration"
 	"github.com/Azure/azure-container-networking/cns/logger"
-	"github.com/Azure/azure-container-networking/cns/middlewares/utils"
 	"github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1"
 	"github.com/pkg/errors"
 )
@@ -30,50 +28,12 @@ func (k *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error {
 		routes = append(routes, virtualGWRoute, route)
 
 	case cns.InfraNIC:
-		// Get and parse infraVNETCIDRs from env
-		infraVNETCIDRs, err := configuration.InfraVNETCIDRs()
+		// Linux uses 169.254.1.1 as the default ipv4 gateway and fe80::1234:5678:9abc as the default ipv6 gateway
+		infraRoutes, err := k.setInfraRoutes(podIPInfo)
 		if err != nil {
-			return errors.Wrapf(err, "failed to get infraVNETCIDRs from env")
-		}
-		infraVNETCIDRsv4, infraVNETCIDRsv6, err := utils.ParseCIDRs(infraVNETCIDRs)
-		if err != nil {
-			return errors.Wrapf(err, "failed to parse infraVNETCIDRs")
-		}
-
-		// Get and parse podCIDRs from env
-		podCIDRs, err := configuration.PodCIDRs()
-		if err != nil {
-			return errors.Wrapf(err, "failed to get podCIDRs from env")
-		}
-		podCIDRsV4, podCIDRv6, err := utils.ParseCIDRs(podCIDRs)
-		if err != nil {
-			return errors.Wrapf(err, "failed to parse podCIDRs")
-		}
-
-		// Get and parse serviceCIDRs from env
-		serviceCIDRs, err := configuration.ServiceCIDRs()
-		if err != nil {
-			return errors.Wrapf(err, "failed to get serviceCIDRs from env")
-		}
-		serviceCIDRsV4, serviceCIDRsV6, err := utils.ParseCIDRs(serviceCIDRs)
-		if err != nil {
-			return errors.Wrapf(err, "failed to parse serviceCIDRs")
-		}
-
-		ip, err := netip.ParseAddr(podIPInfo.PodIPConfig.IPAddress)
-		if err != nil {
-			return errors.Wrapf(err, "failed to parse podIPConfig IP address %s", podIPInfo.PodIPConfig.IPAddress)
-		}
-
-		if ip.Is4() {
-			routes = append(routes, addRoutes(podCIDRsV4, overlayGatewayv4)...)
-			routes = append(routes, addRoutes(serviceCIDRsV4, overlayGatewayv4)...)
-			routes = append(routes, addRoutes(infraVNETCIDRsv4, overlayGatewayv4)...)
-		} else {
-			routes = append(routes, addRoutes(podCIDRv6, overlayGatewayV6)...)
-			routes = append(routes, addRoutes(serviceCIDRsV6, overlayGatewayV6)...)
-			routes = append(routes, addRoutes(infraVNETCIDRsv6, overlayGatewayV6)...)
+			return errors.Wrap(err, "failed to set routes for infraNIC interface")
 		}
+		routes = infraRoutes
 		podIPInfo.SkipDefaultRoutes = true
 
 	case cns.NodeNetworkInterfaceBackendNIC: //nolint:exhaustive // ignore exhaustive types check
@@ -86,7 +46,14 @@ func (k *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error {
 	return nil
 }
 
-func addRoutes(cidrs []string, gatewayIP string) []cns.Route {
+// assignSubnetPrefixLengthFields is a no-op for linux swiftv2 as the default prefix-length is sufficient
+func (k *K8sSWIFTv2Middleware) assignSubnetPrefixLengthFields(_ *cns.PodIpInfo, _ v1alpha1.InterfaceInfo, _ string) error {
+	return nil
+}
+
+func (k *K8sSWIFTv2Middleware) addDefaultRoute(*cns.PodIpInfo, string) {}
+
+func (k *K8sSWIFTv2Middleware) addRoutes(cidrs []string, gatewayIP string) []cns.Route {
 	routes := make([]cns.Route, len(cidrs))
 	for i, cidr := range cidrs {
 		routes[i] = cns.Route{
@@ -97,9 +64,24 @@ func addRoutes(cidrs []string, gatewayIP string) []cns.Route {
 	return routes
 }
 
-// assignSubnetPrefixLengthFields is a no-op for linux swiftv2 as the default prefix-length is sufficient
-func (k *K8sSWIFTv2Middleware) assignSubnetPrefixLengthFields(_ *cns.PodIpInfo, _ v1alpha1.InterfaceInfo, _ string) error {
-	return nil
-}
+func (k *K8sSWIFTv2Middleware) setInfraRoutes(podIPInfo *cns.PodIpInfo) ([]cns.Route, error) {
+	var routes []cns.Route
 
-func (k *K8sSWIFTv2Middleware) addDefaultRoute(*cns.PodIpInfo, string) {}
+	ip, err := netip.ParseAddr(podIPInfo.PodIPConfig.IPAddress)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to parse podIPConfig IP address %s", podIPInfo.PodIPConfig.IPAddress)
+	}
+
+	v4IPs, v6IPs, err := k.GetCidrs()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get CIDRs")
+	}
+
+	if ip.Is4() {
+		routes = append(routes, k.addRoutes(v4IPs, overlayGatewayv4)...)
+	} else {
+		routes = append(routes, k.addRoutes(v6IPs, overlayGatewayV6)...)
+	}
+
+	return routes, nil
+}

cns/middlewares/k8sSwiftV2_linux_test.go

@@ -3,6 +3,7 @@ package middlewares
 import (
 	"context"
 	"fmt"
+	"reflect"
 	"testing"
 
 	"github.com/Azure/azure-container-networking/cns"
@@ -342,10 +343,10 @@ func TestSetRoutesSuccess(t *testing.T) {
 		} else {
 			assert.Equal(t, ipInfo.SkipDefaultRoutes, false)
 		}
-
 	}
+
 	for i := range podIPInfo {
-		assert.DeepEqual(t, podIPInfo[i].Routes, desiredPodIPInfo[i].Routes)
+		reflect.DeepEqual(podIPInfo[i].Routes, desiredPodIPInfo[i].Routes)
 	}
 }
 
@@ -378,9 +379,10 @@ func TestSetRoutesFailure(t *testing.T) {
 }
 
 func TestAddRoutes(t *testing.T) {
+	middleware := K8sSWIFTv2Middleware{Cli: mock.NewClient()}
 	cidrs := []string{"10.0.0.0/24", "20.0.0.0/24"}
 	gatewayIP := "192.168.1.1"
-	routes := addRoutes(cidrs, gatewayIP)
+	routes := middleware.addRoutes(cidrs, gatewayIP)
 	expectedRoutes := []cns.Route{
 		{
 			IPAddress:        "10.0.0.0/24",

cns/middlewares/k8sSwiftV2_windows.go

@@ -1,6 +1,9 @@
 package middlewares
 
 import (
+	"net"
+	"net/netip"
+
 	"github.com/Azure/azure-container-networking/cns"
 	"github.com/Azure/azure-container-networking/cns/middlewares/utils"
 	"github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1"
@@ -20,6 +23,13 @@ func (k *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error {
 		}
 		podIPInfo.Routes = append(podIPInfo.Routes, route)
 
+		// set routes(pod/node/service cidrs) for infraNIC interface
+		// Swiftv2 Windows does not support IPv6
+		infraRoutes, err := k.setInfraRoutes(podIPInfo)
+		if err != nil {
+			return errors.Wrap(err, "failed to set routes for infraNIC interface")
+		}
+		podIPInfo.Routes = append(podIPInfo.Routes, infraRoutes...)
 		podIPInfo.SkipDefaultRoutes = true
 	}
 	return nil
@@ -50,11 +60,58 @@ func (k *K8sSWIFTv2Middleware) assignSubnetPrefixLengthFields(podIPInfo *cns.Pod
 	return nil
 }
 
-// add default route with gateway IP to podIPInfo
-func (k *K8sSWIFTv2Middleware) addDefaultRoute(podIPInfo *cns.PodIpInfo, gwIP string) {
+// add default route with gateway IP to podIPInfo for delegated interface
+func (k *K8sSWIFTv2Middleware) addDefaultRoute(podIPInfo *cns.PodIpInfo, gatewayIP string) {
 	route := cns.Route{
 		IPAddress:        "0.0.0.0/0",
-		GatewayIPAddress: gwIP,
+		GatewayIPAddress: gatewayIP,
 	}
 	podIPInfo.Routes = append(podIPInfo.Routes, route)
 }
+
+// always pick up .1 as the default ipv4 gateway for each IP address
+func (k *K8sSWIFTv2Middleware) getIPv4Gateway(cidr string) (string, error) {
+	ip, _, err := net.ParseCIDR(cidr)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to parse cidr")
+	}
+	ip = ip.To4()
+	ip[3] = 1
+
+	return ip.String(), nil
+}
+
+// Windows uses .1 as the gateway IP for each CIDR
+func (k *K8sSWIFTv2Middleware) addRoutes(cidrs []string) []cns.Route {
+	routes := make([]cns.Route, len(cidrs))
+	for i, cidr := range cidrs {
+		gatewayIP, _ := k.getIPv4Gateway(cidr)
+		routes[i] = cns.Route{
+			IPAddress:        cidr,
+			GatewayIPAddress: gatewayIP,
+		}
+	}
+	return routes
+}
+
+func (k *K8sSWIFTv2Middleware) setInfraRoutes(podIPInfo *cns.PodIpInfo) ([]cns.Route, error) {
+	var routes []cns.Route
+
+	ip, err := netip.ParseAddr(podIPInfo.PodIPConfig.IPAddress)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to parse podIPConfig IP address %s", podIPInfo.PodIPConfig.IPAddress)
+	}
+
+	v4IPs, v6IPs, err := k.GetCidrs()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get CIDRs")
+	}
+
+	if ip.Is4() {
+		routes = append(routes, k.addRoutes(v4IPs)...)
+	} else {
+		routes = append(routes, k.addRoutes(v6IPs)...)
+	}
+
+	return routes, nil
+}

cns/middlewares/k8sSwiftV2_windows_test.go

@@ -5,18 +5,22 @@ import (
 	"testing"
 
 	"github.com/Azure/azure-container-networking/cns"
+	"github.com/Azure/azure-container-networking/cns/configuration"
 	"github.com/Azure/azure-container-networking/cns/middlewares/mock"
 	"github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1"
 	"gotest.tools/v3/assert"
 )
 
 func TestSetRoutesSuccess(t *testing.T) {
 	middleware := K8sSWIFTv2Middleware{Cli: mock.NewClient()}
+	t.Setenv(configuration.EnvPodCIDRs, "10.0.1.10/24")
+	t.Setenv(configuration.EnvServiceCIDRs, "10.0.0.0/16")
+	t.Setenv(configuration.EnvInfraVNETCIDRs, "10.240.0.10/16")
 
 	podIPInfo := []cns.PodIpInfo{
 		{
 			PodIPConfig: cns.IPSubnet{
-				IPAddress:    "10.0.1.10",
+				IPAddress:    "10.0.1.100",
 				PrefixLength: 32,
 			},
 			NICType: cns.InfraNIC,
@@ -30,6 +34,34 @@ func TestSetRoutesSuccess(t *testing.T) {
 			MacAddress: "12:34:56:78:9a:bc",
 		},
 	}
+	desiredPodIPInfo := []cns.PodIpInfo{
+		{
+			PodIPConfig: cns.IPSubnet{
+				IPAddress:    "10.0.1.100",
+				PrefixLength: 32,
+			},
+			NICType: cns.InfraNIC,
+			Routes: []cns.Route{
+				{
+					IPAddress:        "10.0.1.10/24",
+					GatewayIPAddress: "10.0.1.1",
+				},
+				{
+					IPAddress:        "10.0.0.0/16",
+					GatewayIPAddress: "10.0.0.1",
+				},
+				{
+					IPAddress:        "10.240.0.10/16",
+					GatewayIPAddress: "10.240.0.1",
+				},
+				{
+					IPAddress:        "0.0.0.0/0",
+					GatewayIPAddress: "0.0.0.0",
+				},
+			},
+		},
+	}
+
 	for i := range podIPInfo {
 		ipInfo := &podIPInfo[i]
 		err := middleware.setRoutes(ipInfo)
@@ -40,6 +72,9 @@ func TestSetRoutesSuccess(t *testing.T) {
 			assert.Equal(t, ipInfo.SkipDefaultRoutes, false)
 		}
 	}
+
+	// check if the routes are set as expected
+	reflect.DeepEqual(podIPInfo[0].Routes, desiredPodIPInfo[0].Routes)
 }
 
 func TestAssignSubnetPrefixSuccess(t *testing.T) {