Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Introduces the option --overlay-type={subnet,full}, to be able to always generate IPIP tunnels regardless of node subnets #666

Merged
merged 2 commits into from
Mar 23, 2019
Merged

Introduces the option --overlay-type={subnet,full}, to be able to always generate IPIP tunnels regardless of node subnets #666

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3e91536
Introduces the option --full-overlay, to always generate IPIP tunnels…
lucasmundim Feb 10, 2019
d9a6367
Use --overlay-type={subnet,full} instead of --full-overlay={true,false}
lucasmundim Mar 15, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion docs/user-guide.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ Usage of kube-router:
--disable-source-dest-check Disable the source-dest-check attribute for AWS EC2 instances. When this option is false, it must be set some other way. (default true)
--enable-cni Enable CNI plugin. Disable if you want to use kube-router features alongside another CNI plugin. (default true)
--enable-ibgp Enables peering with nodes with the same ASN, if disabled will only peer with external BGP peers (default true)
--enable-overlay When enable-overlay set to true, IP-in-IP tunneling is used for pod-to-pod networking across nodes in different subnets. When set to false no tunneling is used and routing infrastrcture is expected to route traffic for pod-to-pod networking across nodes in different subnets (default true)
--enable-overlay When enable-overlay is set to true, IP-in-IP tunneling is used for pod-to-pod networking across nodes in different subnets. When set to false no tunneling is used and routing infrastructure is expected to route traffic for pod-to-pod networking across nodes in different subnets (default true)
--enable-pod-egress SNAT traffic from Pods to destinations outside the cluster. (default true)
--enable-pprof Enables pprof for debugging performance and memory leak issues.
--hairpin-mode Add iptables rules for every Service Endpoint to support hairpin traffic.
Expand All @@ -65,6 +65,7 @@ Usage of kube-router:
--metrics-port uint16 Prometheus metrics port, (Default 0, Disabled)
--nodeport-bindon-all-ip For service of NodePort type create IPVS service that listens on all IP's of the node.
--nodes-full-mesh Each node in the cluster will setup BGP peering with rest of the nodes. (default true)
--overlay-type string Possible values: subnet,full - When set to "subnet", the default, default "--enable-overlay=true" behavior is used. When set to "full", it changes "--enable-overlay=true" default behavior so that IP-in-IP tunneling is used for pod-to-pod networking across nodes regardless of the subnet the nodes are in. (default "subnet")
--override-nexthop Override the next-hop in bgp routes sent to peers with the local ip.
--peer-router-asns uints ASN numbers of the BGP peer to which cluster nodes will advertise cluster ip and node's pod cidr. (default [])
--peer-router-ips ipSlice The ip address of the external router to which all nodes will peer and advertise the cluster ip and pod cidr's. (default [])
Expand Down
59 changes: 26 additions & 33 deletions pkg/controllers/routing/network_routes_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ type NetworkRoutingController struct {
bgpGracefulRestart bool
ipSetHandler *utils.IPSet
enableOverlays bool
overlayType string
peerMultihopTTL uint8
MetricsEnabled bool
bgpServerStarted bool
Expand Down Expand Up @@ -410,50 +411,41 @@ func (nrc *NetworkRoutingController) injectRoute(path *table.Path) error {
dst, _ := netlink.ParseIPNet(nlri.String())
var route *netlink.Route

// check if the neighbour is in same subnet. If node is not in same subnet and --override-nexthop=false
// only then create IPIP tunnels
if !nrc.nodeSubnet.Contains(nexthop) && !nrc.overrideNextHop {
tunnelName := generateTunnelName(nexthop.String())
glog.Infof("Found node: " + nexthop.String() + " to be in different subnet.")

// if overlay is not enabled then skip creating tunnels and adding route
if !nrc.enableOverlays {
glog.Infof("Found node: " + nexthop.String() + " to be in different subnet but overlays are " +
"disabled so not creating any tunnel and injecting route for the node's pod CIDR.")

glog.Infof("Cleaning up old routes if there are any")
routes, err := netlink.RouteListFiltered(nl.FAMILY_ALL, &netlink.Route{
Dst: dst, Protocol: 0x11,
}, netlink.RT_FILTER_DST|netlink.RT_FILTER_PROTOCOL)
if err != nil {
glog.Errorf("Failed to get routes from netlink")
}
for i, r := range routes {
glog.V(2).Infof("Found route to remove: %s", r.String())
err := netlink.RouteDel(&routes[i])
if err != nil {
glog.Errorf("Failed to remove route due to " + err.Error())
}
}
tunnelName := generateTunnelName(nexthop.String())
sameSubnet := nrc.nodeSubnet.Contains(nexthop)

glog.Infof("Cleaning up if there is any existing tunnel interface for the node")
link, err := netlink.LinkByName(tunnelName)
if err != nil {
return nil
// cleanup route and tunnel if overlay is disabled or node is in same subnet and overlay-type is set to 'subnet'
if !nrc.enableOverlays || (sameSubnet && nrc.overlayType == "subnet") {
glog.Infof("Cleaning up old routes if there are any")
routes, err := netlink.RouteListFiltered(nl.FAMILY_ALL, &netlink.Route{
Dst: dst, Protocol: 0x11,
}, netlink.RT_FILTER_DST|netlink.RT_FILTER_PROTOCOL)
if err != nil {
glog.Errorf("Failed to get routes from netlink")
}
for i, r := range routes {
glog.V(2).Infof("Found route to remove: %s", r.String())
if err := netlink.RouteDel(&routes[i]); err != nil {
glog.Errorf("Failed to remove route due to " + err.Error())
}
err = netlink.LinkDel(link)
if err != nil {
}

glog.Infof("Cleaning up if there is any existing tunnel interface for the node")
if link, err := netlink.LinkByName(tunnelName); err == nil {
if err = netlink.LinkDel(link); err != nil {
glog.Errorf("Failed to delete tunnel link for the node due to " + err.Error())
}
return nil
}
}

// create IPIP tunnels only when node is not in same subnet or overlay-type is set to 'full'
// prevent creation when --override-nexthop=true as well
if (!sameSubnet || nrc.overlayType == "full") && !nrc.overrideNextHop {
// create ip-in-ip tunnel and inject route as overlay is enabled
var link netlink.Link
var err error
link, err = netlink.LinkByName(tunnelName)
if err != nil {
glog.Infof("Found node: " + nexthop.String() + " to be in different subnet. Creating tunnel: " + tunnelName)
out, err := exec.Command("ip", "tunnel", "add", tunnelName, "mode", "ipip", "local", nrc.nodeIP.String(),
"remote", nexthop.String(), "dev", nrc.nodeInterface).CombinedOutput()
if err != nil {
Expand Down Expand Up @@ -938,6 +930,7 @@ func NewNetworkRoutingController(clientset kubernetes.Interface,
nrc.advertiseLoadBalancerIP = kubeRouterConfig.AdvertiseLoadBalancerIp
nrc.advertisePodCidr = kubeRouterConfig.AdvertiseNodePodCidr
nrc.enableOverlays = kubeRouterConfig.EnableOverlay
nrc.overlayType = kubeRouterConfig.OverlayType

nrc.bgpPort = kubeRouterConfig.BGPPort

Expand Down
13 changes: 10 additions & 3 deletions pkg/options/options.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,9 @@ import (
"net"
"time"

"github.com/spf13/pflag"
"strconv"

"github.com/spf13/pflag"
)

const DEFAULT_BGP_PORT = 179
Expand All @@ -28,6 +29,7 @@ type KubeRouterConfig struct {
EnablePodEgress bool
EnablePprof bool
FullMeshMode bool
OverlayType string
GlobalHairpinMode bool
HealthPort uint16
HelpRequested bool
Expand Down Expand Up @@ -64,6 +66,7 @@ func NewKubeRouterConfig() *KubeRouterConfig {
IPTablesSyncPeriod: 5 * time.Minute,
RoutesSyncPeriod: 5 * time.Minute,
EnableOverlay: true,
OverlayType: "subnet",
}
}

Expand Down Expand Up @@ -134,8 +137,12 @@ func (s *KubeRouterConfig) AddFlags(fs *pflag.FlagSet) {
fs.BoolVar(&s.NodePortBindOnAllIp, "nodeport-bindon-all-ip", false,
"For service of NodePort type create IPVS service that listens on all IP's of the node.")
fs.BoolVar(&s.EnableOverlay, "enable-overlay", true,
"When enable-overlay set to true, IP-in-IP tunneling is used for pod-to-pod networking across nodes in different subnets. "+
"When set to false no tunneling is used and routing infrastrcture is expected to route traffic for pod-to-pod networking across nodes in different subnets")
"When enable-overlay is set to true, IP-in-IP tunneling is used for pod-to-pod networking across nodes in different subnets. "+
"When set to false no tunneling is used and routing infrastructure is expected to route traffic for pod-to-pod networking across nodes in different subnets")
fs.StringVar(&s.OverlayType, "overlay-type", s.OverlayType,
"Possible values: subnet,full - "+
"When set to \"subnet\", the default, default \"--enable-overlay=true\" behavior is used. "+
"When set to \"full\", it changes \"--enable-overlay=true\" default behavior so that IP-in-IP tunneling is used for pod-to-pod networking across nodes regardless of the subnet the nodes are in.")
fs.StringSliceVar(&s.PeerPasswords, "peer-router-passwords", s.PeerPasswords,
"Password for authenticating against the BGP peer defined with \"--peer-router-ips\".")
fs.BoolVar(&s.EnablePprof, "enable-pprof", false,
Expand Down