From 3e9153639e3fc7504000098d3e0dd90de75af7f7 Mon Sep 17 00:00:00 2001 From: Lucas Mundim Date: Sun, 10 Feb 2019 02:49:15 -0200 Subject: [PATCH 1/2] Introduces the option --full-overlay, to always generate IPIP tunnels regardless of node subnets --- docs/user-guide.md | 3 +- .../routing/network_routes_controller.go | 59 ++++++++----------- pkg/options/options.go | 11 +++- 3 files changed, 36 insertions(+), 37 deletions(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index 0a3d4255c7..78d912fd46 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -49,9 +49,10 @@ Usage of kube-router: --disable-source-dest-check Disable the source-dest-check attribute for AWS EC2 instances. When this option is false, it must be set some other way. (default true) --enable-cni Enable CNI plugin. Disable if you want to use kube-router features alongside another CNI plugin. (default true) --enable-ibgp Enables peering with nodes with the same ASN, if disabled will only peer with external BGP peers (default true) - --enable-overlay When enable-overlay set to true, IP-in-IP tunneling is used for pod-to-pod networking across nodes in different subnets. When set to false no tunneling is used and routing infrastrcture is expected to route traffic for pod-to-pod networking across nodes in different subnets (default true) + --enable-overlay When enable-overlay is set to true, IP-in-IP tunneling is used for pod-to-pod networking across nodes in different subnets. When set to false no tunneling is used and routing infrastructure is expected to route traffic for pod-to-pod networking across nodes in different subnets (default true) --enable-pod-egress SNAT traffic from Pods to destinations outside the cluster. (default true) --enable-pprof Enables pprof for debugging performance and memory leak issues. + --full-overlay When full-overlay is set to true, it changes "--enable-overlay=true" default behavior so that IP-in-IP tunneling is used for pod-to-pod networking across nodes regardless of the subnet the nodes are in. When set to false, the default, default "--enable-overlay=true" behavior is used --hairpin-mode Add iptables rules for every Service Endpoint to support hairpin traffic. --health-port uint16 Health check port, 0 = Disabled (default 20244) -h, --help Print usage information. diff --git a/pkg/controllers/routing/network_routes_controller.go b/pkg/controllers/routing/network_routes_controller.go index 85b30b37ce..872d01379d 100644 --- a/pkg/controllers/routing/network_routes_controller.go +++ b/pkg/controllers/routing/network_routes_controller.go @@ -91,6 +91,7 @@ type NetworkRoutingController struct { bgpGracefulRestart bool ipSetHandler *utils.IPSet enableOverlays bool + fullOverlay bool peerMultihopTTL uint8 MetricsEnabled bool bgpServerStarted bool @@ -410,50 +411,41 @@ func (nrc *NetworkRoutingController) injectRoute(path *table.Path) error { dst, _ := netlink.ParseIPNet(nlri.String()) var route *netlink.Route - // check if the neighbour is in same subnet. If node is not in same subnet and --override-nexthop=false - // only then create IPIP tunnels - if !nrc.nodeSubnet.Contains(nexthop) && !nrc.overrideNextHop { - tunnelName := generateTunnelName(nexthop.String()) - glog.Infof("Found node: " + nexthop.String() + " to be in different subnet.") - - // if overlay is not enabled then skip creating tunnels and adding route - if !nrc.enableOverlays { - glog.Infof("Found node: " + nexthop.String() + " to be in different subnet but overlays are " + - "disabled so not creating any tunnel and injecting route for the node's pod CIDR.") - - glog.Infof("Cleaning up old routes if there are any") - routes, err := netlink.RouteListFiltered(nl.FAMILY_ALL, &netlink.Route{ - Dst: dst, Protocol: 0x11, - }, netlink.RT_FILTER_DST|netlink.RT_FILTER_PROTOCOL) - if err != nil { - glog.Errorf("Failed to get routes from netlink") - } - for i, r := range routes { - glog.V(2).Infof("Found route to remove: %s", r.String()) - err := netlink.RouteDel(&routes[i]) - if err != nil { - glog.Errorf("Failed to remove route due to " + err.Error()) - } - } + tunnelName := generateTunnelName(nexthop.String()) + sameSubnet := nrc.nodeSubnet.Contains(nexthop) - glog.Infof("Cleaning up if there is any existing tunnel interface for the node") - link, err := netlink.LinkByName(tunnelName) - if err != nil { - return nil + // cleanup route and tunnel if overlay is disabled or node is in same subnet and full overlay is disabled + if !nrc.enableOverlays || (sameSubnet && !nrc.fullOverlay) { + glog.Infof("Cleaning up old routes if there are any") + routes, err := netlink.RouteListFiltered(nl.FAMILY_ALL, &netlink.Route{ + Dst: dst, Protocol: 0x11, + }, netlink.RT_FILTER_DST|netlink.RT_FILTER_PROTOCOL) + if err != nil { + glog.Errorf("Failed to get routes from netlink") + } + for i, r := range routes { + glog.V(2).Infof("Found route to remove: %s", r.String()) + if err := netlink.RouteDel(&routes[i]); err != nil { + glog.Errorf("Failed to remove route due to " + err.Error()) } - err = netlink.LinkDel(link) - if err != nil { + } + + glog.Infof("Cleaning up if there is any existing tunnel interface for the node") + if link, err := netlink.LinkByName(tunnelName); err == nil { + if err = netlink.LinkDel(link); err != nil { glog.Errorf("Failed to delete tunnel link for the node due to " + err.Error()) } - return nil } + } + // create IPIP tunnels only when node is not in same subnet or full overlay is true + // prevent creation when --override-nexthop=true as well + if (!sameSubnet || nrc.fullOverlay) && !nrc.overrideNextHop { // create ip-in-ip tunnel and inject route as overlay is enabled var link netlink.Link var err error link, err = netlink.LinkByName(tunnelName) if err != nil { - glog.Infof("Found node: " + nexthop.String() + " to be in different subnet. Creating tunnel: " + tunnelName) out, err := exec.Command("ip", "tunnel", "add", tunnelName, "mode", "ipip", "local", nrc.nodeIP.String(), "remote", nexthop.String(), "dev", nrc.nodeInterface).CombinedOutput() if err != nil { @@ -938,6 +930,7 @@ func NewNetworkRoutingController(clientset kubernetes.Interface, nrc.advertiseLoadBalancerIP = kubeRouterConfig.AdvertiseLoadBalancerIp nrc.advertisePodCidr = kubeRouterConfig.AdvertiseNodePodCidr nrc.enableOverlays = kubeRouterConfig.EnableOverlay + nrc.fullOverlay = kubeRouterConfig.FullOverlay nrc.bgpPort = kubeRouterConfig.BGPPort diff --git a/pkg/options/options.go b/pkg/options/options.go index 9e2473914d..099651c972 100644 --- a/pkg/options/options.go +++ b/pkg/options/options.go @@ -4,8 +4,9 @@ import ( "net" "time" - "github.com/spf13/pflag" "strconv" + + "github.com/spf13/pflag" ) const DEFAULT_BGP_PORT = 179 @@ -28,6 +29,7 @@ type KubeRouterConfig struct { EnablePodEgress bool EnablePprof bool FullMeshMode bool + FullOverlay bool GlobalHairpinMode bool HealthPort uint16 HelpRequested bool @@ -134,8 +136,11 @@ func (s *KubeRouterConfig) AddFlags(fs *pflag.FlagSet) { fs.BoolVar(&s.NodePortBindOnAllIp, "nodeport-bindon-all-ip", false, "For service of NodePort type create IPVS service that listens on all IP's of the node.") fs.BoolVar(&s.EnableOverlay, "enable-overlay", true, - "When enable-overlay set to true, IP-in-IP tunneling is used for pod-to-pod networking across nodes in different subnets. "+ - "When set to false no tunneling is used and routing infrastrcture is expected to route traffic for pod-to-pod networking across nodes in different subnets") + "When enable-overlay is set to true, IP-in-IP tunneling is used for pod-to-pod networking across nodes in different subnets. "+ + "When set to false no tunneling is used and routing infrastructure is expected to route traffic for pod-to-pod networking across nodes in different subnets") + fs.BoolVar(&s.FullOverlay, "full-overlay", false, + "When full-overlay is set to true, it changes \"--enable-overlay=true\" default behavior so that IP-in-IP tunneling is used for pod-to-pod networking across nodes regardless of the subnet the nodes are in. "+ + "When set to false, the default, default \"--enable-overlay=true\" behavior is used") fs.StringSliceVar(&s.PeerPasswords, "peer-router-passwords", s.PeerPasswords, "Password for authenticating against the BGP peer defined with \"--peer-router-ips\".") fs.BoolVar(&s.EnablePprof, "enable-pprof", false, From d9a636783fe0e23ad32928cffd5b5a86ccdf1c02 Mon Sep 17 00:00:00 2001 From: Lucas Mundim Date: Fri, 15 Mar 2019 02:35:46 -0300 Subject: [PATCH 2/2] Use --overlay-type={subnet,full} instead of --full-overlay={true,false} --- docs/user-guide.md | 2 +- pkg/controllers/routing/network_routes_controller.go | 12 ++++++------ pkg/options/options.go | 10 ++++++---- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index 78d912fd46..b599703d3e 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -52,7 +52,6 @@ Usage of kube-router: --enable-overlay When enable-overlay is set to true, IP-in-IP tunneling is used for pod-to-pod networking across nodes in different subnets. When set to false no tunneling is used and routing infrastructure is expected to route traffic for pod-to-pod networking across nodes in different subnets (default true) --enable-pod-egress SNAT traffic from Pods to destinations outside the cluster. (default true) --enable-pprof Enables pprof for debugging performance and memory leak issues. - --full-overlay When full-overlay is set to true, it changes "--enable-overlay=true" default behavior so that IP-in-IP tunneling is used for pod-to-pod networking across nodes regardless of the subnet the nodes are in. When set to false, the default, default "--enable-overlay=true" behavior is used --hairpin-mode Add iptables rules for every Service Endpoint to support hairpin traffic. --health-port uint16 Health check port, 0 = Disabled (default 20244) -h, --help Print usage information. @@ -66,6 +65,7 @@ Usage of kube-router: --metrics-port uint16 Prometheus metrics port, (Default 0, Disabled) --nodeport-bindon-all-ip For service of NodePort type create IPVS service that listens on all IP's of the node. --nodes-full-mesh Each node in the cluster will setup BGP peering with rest of the nodes. (default true) + --overlay-type string Possible values: subnet,full - When set to "subnet", the default, default "--enable-overlay=true" behavior is used. When set to "full", it changes "--enable-overlay=true" default behavior so that IP-in-IP tunneling is used for pod-to-pod networking across nodes regardless of the subnet the nodes are in. (default "subnet") --override-nexthop Override the next-hop in bgp routes sent to peers with the local ip. --peer-router-asns uints ASN numbers of the BGP peer to which cluster nodes will advertise cluster ip and node's pod cidr. (default []) --peer-router-ips ipSlice The ip address of the external router to which all nodes will peer and advertise the cluster ip and pod cidr's. (default []) diff --git a/pkg/controllers/routing/network_routes_controller.go b/pkg/controllers/routing/network_routes_controller.go index 872d01379d..a2440ce2fa 100644 --- a/pkg/controllers/routing/network_routes_controller.go +++ b/pkg/controllers/routing/network_routes_controller.go @@ -91,7 +91,7 @@ type NetworkRoutingController struct { bgpGracefulRestart bool ipSetHandler *utils.IPSet enableOverlays bool - fullOverlay bool + overlayType string peerMultihopTTL uint8 MetricsEnabled bool bgpServerStarted bool @@ -414,8 +414,8 @@ func (nrc *NetworkRoutingController) injectRoute(path *table.Path) error { tunnelName := generateTunnelName(nexthop.String()) sameSubnet := nrc.nodeSubnet.Contains(nexthop) - // cleanup route and tunnel if overlay is disabled or node is in same subnet and full overlay is disabled - if !nrc.enableOverlays || (sameSubnet && !nrc.fullOverlay) { + // cleanup route and tunnel if overlay is disabled or node is in same subnet and overlay-type is set to 'subnet' + if !nrc.enableOverlays || (sameSubnet && nrc.overlayType == "subnet") { glog.Infof("Cleaning up old routes if there are any") routes, err := netlink.RouteListFiltered(nl.FAMILY_ALL, &netlink.Route{ Dst: dst, Protocol: 0x11, @@ -438,9 +438,9 @@ func (nrc *NetworkRoutingController) injectRoute(path *table.Path) error { } } - // create IPIP tunnels only when node is not in same subnet or full overlay is true + // create IPIP tunnels only when node is not in same subnet or overlay-type is set to 'full' // prevent creation when --override-nexthop=true as well - if (!sameSubnet || nrc.fullOverlay) && !nrc.overrideNextHop { + if (!sameSubnet || nrc.overlayType == "full") && !nrc.overrideNextHop { // create ip-in-ip tunnel and inject route as overlay is enabled var link netlink.Link var err error @@ -930,7 +930,7 @@ func NewNetworkRoutingController(clientset kubernetes.Interface, nrc.advertiseLoadBalancerIP = kubeRouterConfig.AdvertiseLoadBalancerIp nrc.advertisePodCidr = kubeRouterConfig.AdvertiseNodePodCidr nrc.enableOverlays = kubeRouterConfig.EnableOverlay - nrc.fullOverlay = kubeRouterConfig.FullOverlay + nrc.overlayType = kubeRouterConfig.OverlayType nrc.bgpPort = kubeRouterConfig.BGPPort diff --git a/pkg/options/options.go b/pkg/options/options.go index 099651c972..333f208875 100644 --- a/pkg/options/options.go +++ b/pkg/options/options.go @@ -29,7 +29,7 @@ type KubeRouterConfig struct { EnablePodEgress bool EnablePprof bool FullMeshMode bool - FullOverlay bool + OverlayType string GlobalHairpinMode bool HealthPort uint16 HelpRequested bool @@ -66,6 +66,7 @@ func NewKubeRouterConfig() *KubeRouterConfig { IPTablesSyncPeriod: 5 * time.Minute, RoutesSyncPeriod: 5 * time.Minute, EnableOverlay: true, + OverlayType: "subnet", } } @@ -138,9 +139,10 @@ func (s *KubeRouterConfig) AddFlags(fs *pflag.FlagSet) { fs.BoolVar(&s.EnableOverlay, "enable-overlay", true, "When enable-overlay is set to true, IP-in-IP tunneling is used for pod-to-pod networking across nodes in different subnets. "+ "When set to false no tunneling is used and routing infrastructure is expected to route traffic for pod-to-pod networking across nodes in different subnets") - fs.BoolVar(&s.FullOverlay, "full-overlay", false, - "When full-overlay is set to true, it changes \"--enable-overlay=true\" default behavior so that IP-in-IP tunneling is used for pod-to-pod networking across nodes regardless of the subnet the nodes are in. "+ - "When set to false, the default, default \"--enable-overlay=true\" behavior is used") + fs.StringVar(&s.OverlayType, "overlay-type", s.OverlayType, + "Possible values: subnet,full - "+ + "When set to \"subnet\", the default, default \"--enable-overlay=true\" behavior is used. "+ + "When set to \"full\", it changes \"--enable-overlay=true\" default behavior so that IP-in-IP tunneling is used for pod-to-pod networking across nodes regardless of the subnet the nodes are in.") fs.StringSliceVar(&s.PeerPasswords, "peer-router-passwords", s.PeerPasswords, "Password for authenticating against the BGP peer defined with \"--peer-router-ips\".") fs.BoolVar(&s.EnablePprof, "enable-pprof", false,