diff --git a/README.md b/README.md index 06d8d54d75..97012488d0 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,7 @@ that has been thoroughly tested and optimized. - [Architecture](./docs/architecture.md) - [See Kube-router in action](./docs/see-it-in-action.md) - [User Guide](./docs/user-guide.md) +- [Operations Guide](./docs/operations-guide.md) - [Developer Guide](./docs/developing.md) ## Project status diff --git a/cmd/kube-router/kube-router_test.go b/cmd/kube-router/kube-router_test.go deleted file mode 100644 index d198d96aee..0000000000 --- a/cmd/kube-router/kube-router_test.go +++ /dev/null @@ -1,50 +0,0 @@ -package main - -import ( - "bytes" - "io" - "os" - "sync" - "testing" -) - -func TestMainHelp(t *testing.T) { - origStderr := os.Stderr - stderrR, stderrW, _ := os.Pipe() - os.Stderr = stderrW - defer func() { os.Stderr = origStderr }() - - stderrBuf := bytes.NewBuffer(nil) - wg := &sync.WaitGroup{} - wg.Add(1) - go func() { - io.Copy(stderrBuf, stderrR) - wg.Done() - }() - - origArgs := os.Args - os.Args = []string{"kube-router", "--help"} - defer func() { os.Args = origArgs }() - - if err := Main(); err != nil { - t.Fatalf("kube-router exited with error: %s\n", err) - } - stderrW.Close() - wg.Wait() - - docF, err := os.Open("../../docs/user-guide.md") - if err != nil { - t.Fatalf("could not open docs/user-guide.md: %s\n", err) - } - docBuf := bytes.NewBuffer(nil) - docBuf.ReadFrom(docF) - docF.Close() - - exp := append([]byte("```\n"), stderrBuf.Bytes()...) - exp = append(exp, []byte("```\n")...) - - if !bytes.Contains(docBuf.Bytes(), exp) { - t.Errorf("docs/README.md 'command line options' section does not match `kube-router --help`.\nExpected:\n%s", exp) - t.Errorf("\nGot:\n%s", docBuf.Bytes()) - } -} diff --git a/daemonset/kube-router-daemonset-advertise-cluster-subnet.yaml b/daemonset/kube-router-daemonset-advertise-cluster-subnet.yaml new file mode 100644 index 0000000000..8663a170a1 --- /dev/null +++ b/daemonset/kube-router-daemonset-advertise-cluster-subnet.yaml @@ -0,0 +1,134 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-router-cfg + namespace: kube-system + labels: + tier: node + k8s-app: kube-router +data: + cni-conf.json: | + { + "cniVersion":"0.3.0", + "name":"mynet", + "plugins":[ + { + "name":"kubernetes", + "type":"bridge", + "bridge":"kube-bridge", + "isDefaultGateway":true, + "ipam":{ + "type":"host-local" + } + } + ] + } +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kube-router + namespace: kube-system + labels: + k8s-app: kube-router +spec: + selector: + matchLabels: + k8s-app: kube-router + template: + metadata: + labels: + k8s-app: kube-router + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + containers: + - name: kube-router + image: docker.io/cloudnativelabs/kube-router + args: + - "--run-router=true" + - "--run-firewall=true" + - "--run-service-proxy=true" + - "--kubeconfig=/var/lib/kube-router/kubeconfig" + - "--advertise-cluster-ip=true" + - "--advertise-cluster-subnet=172.30.0.0/16" + - "--enable-ibgp=false" + - "--enable-overlay=false" + - "--enable-pod-egress=false" + - "--advertise-pod-cidr=true" + - "--nodes-full-mesh=true" + - "--cluster-asn=64558" + - "--peer-router-ips=192.168.110.1" + - "--peer-router-asns=64558" + securityContext: + privileged: true + imagePullPolicy: Always + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: KUBE_ROUTER_CNI_CONF_FILE + value: /etc/cni/net.d/10-kuberouter.conflist + livenessProbe: + httpGet: + path: /healthz + port: 20244 + initialDelaySeconds: 10 + periodSeconds: 3 + volumeMounts: + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: cni-conf-dir + mountPath: /etc/cni/net.d + - name: kubeconfig + mountPath: /var/lib/kube-router/kubeconfig + readOnly: true + initContainers: + - name: install-cni + image: busybox + imagePullPolicy: Always + command: + - /bin/sh + - -c + - set -e -x; + if [ ! -f /etc/cni/net.d/10-kuberouter.conflist ]; then + if [ -f /etc/cni/net.d/*.conf ]; then + rm -f /etc/cni/net.d/*.conf; + fi; + TMP=/etc/cni/net.d/.tmp-kuberouter-cfg; + cp /etc/kube-router/cni-conf.json ${TMP}; + mv ${TMP} /etc/cni/net.d/10-kuberouter.conflist; + fi + volumeMounts: + - name: cni-conf-dir + mountPath: /etc/cni/net.d + - name: kube-router-cfg + mountPath: /etc/kube-router + hostNetwork: true + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + volumes: + - name: lib-modules + hostPath: + path: /lib/modules + - name: cni-conf-dir + hostPath: + path: /etc/cni/net.d + - name: kube-router-cfg + configMap: + name: kube-router-cfg + - name: kubeconfig + hostPath: + path: /var/lib/kube-router/kubeconfig + + + diff --git a/docs/img/large-networks01.jpg b/docs/img/large-networks01.jpg new file mode 100644 index 0000000000..2ce21e0138 Binary files /dev/null and b/docs/img/large-networks01.jpg differ diff --git a/docs/img/large-networks02.jpg b/docs/img/large-networks02.jpg new file mode 100644 index 0000000000..5682a34e32 Binary files /dev/null and b/docs/img/large-networks02.jpg differ diff --git a/docs/img/large-networks03.jpg b/docs/img/large-networks03.jpg new file mode 100644 index 0000000000..a30325969d Binary files /dev/null and b/docs/img/large-networks03.jpg differ diff --git a/docs/img/large-networks04.jpg b/docs/img/large-networks04.jpg new file mode 100644 index 0000000000..d277742580 Binary files /dev/null and b/docs/img/large-networks04.jpg differ diff --git a/docs/img/large-networks05.jpg b/docs/img/large-networks05.jpg new file mode 100644 index 0000000000..1514fa501b Binary files /dev/null and b/docs/img/large-networks05.jpg differ diff --git a/docs/img/large-networks06.jpg b/docs/img/large-networks06.jpg new file mode 100644 index 0000000000..ef12cb6eae Binary files /dev/null and b/docs/img/large-networks06.jpg differ diff --git a/docs/img/large-networks07.jpg b/docs/img/large-networks07.jpg new file mode 100644 index 0000000000..58043b5dde Binary files /dev/null and b/docs/img/large-networks07.jpg differ diff --git a/docs/img/large-networks08.jpg b/docs/img/large-networks08.jpg new file mode 100644 index 0000000000..0bd7527394 Binary files /dev/null and b/docs/img/large-networks08.jpg differ diff --git a/docs/img/large-networks09.jpg b/docs/img/large-networks09.jpg new file mode 100644 index 0000000000..66203a94ba Binary files /dev/null and b/docs/img/large-networks09.jpg differ diff --git a/docs/large-networks01.md b/docs/large-networks01.md new file mode 100644 index 0000000000..302736a248 --- /dev/null +++ b/docs/large-networks01.md @@ -0,0 +1,34 @@ +**How to optimize the neighborhood relationship and reduce the number of routes through kube-router in order to support the BGP network of large kubernetes cluster?** +
+
+
+ +**Introduction of cluster BGP network environment**
+ + + + + + + + +
**Item****Content**
Version of kube-routerModified based on version 0.3.2
The number of kubernetes nodeAbout 3000
Pod subnet of kubernetes cluster172.31.0.0/16
Service subnet of kubernetes cluster172.30.0.0/16
The number of Router Device About 100
Router-id of Uplink Router Device192.168.110.1(router1)、
192.168.120.1(router2)、
192.168.130.1(router3)、
192.168.140.1(router4)
......
+ +The network topology is as follows: +![avatar](../docs/img/large-networks01.jpg) + +
+**Optimization scheme of large k8s cluster BGP network**
+In order for your architecture to support a larger network, you need to do the following two things: +
+        (1) Set the parameter "--enable-ibgp=false", do not let kubernetes node directly establish BGP neighbors with each other. Let your kubernetes node only build BGP neighbors with the on-line router device. See this link for details (large-networks02 documentation). +
+        (2) You'd better turn on the BGP ECMP function of the kubernetes node on-line router device. The effect of this method is that when the user's access traffic enters the router device, it is first balanced to the kubernetes node of the back end through ECMP load balancing, and then to the final pod through IPVS load balancing. When devices, links and nodes in the network are down, traffic can be automatically switched to other healthy devices, links and nodes. In this way, the availability, high performance and scalability of the network are realized.See this link for details (large-networks04 documentation). +
+        (3) You need to set both "--advertise-cluster-IP=true" and "--advertise-cluster-subnet=subnet" parameters.Let k8s node only notify k8s service aggregate routes to the upstream routers, reducing the service routing entries of the upstream routers.See this link for details (large-networks03 documentation). +
+
+**YAML source files used in this document**
+For the YAML source files used in this document, please click on this link to view them. +
+
diff --git a/docs/large-networks02.md b/docs/large-networks02.md new file mode 100644 index 0000000000..3e0c3b06cb --- /dev/null +++ b/docs/large-networks02.md @@ -0,0 +1,308 @@ + +**Method of reducing BGP neighbors between each kubernetes node in AS** +
+
+
+ + + +**The number growth of BGP neighbors before optimization**
+ +When "--enable-ibgp=true" is set by default by kube-router, all kubernetes nodes establish BGP neighborhood relationships with each other. Suppose there are eight k8s-nodes in our network. Their topological relationships are as follows: + +![avatar](../docs/img/large-networks02.jpg) + +We can login to each kubernetes node to see the local neighborhood. +
+View on k8s-node-1: +
+[k8s-node-1 ~]# gobgp neighbor
+Peer AS Up/Down State |#Received Accepted
+192.168.110.12 64558 17:17:28 Establ | 0 0
+192.168.120.13 64558 17:17:28 Establ | 0 0
+192.168.120.14 64558 17:17:28 Establ | 0 0
+192.168.130.15 64558 17:17:28 Establ | 0 0
+192.168.130.16 64558 17:17:28 Establ | 0 0
+192.168.140.17 64558 17:17:28 Establ | 0 0
+192.168.140.18 64558 17:17:28 Establ | 0 0
+[k8s-node-1 ~]# +
+ +View on k8s-node-2: +
+[k8s-node-2 ~]# gobgp neighbor
+Peer AS Up/Down State |#Received Accepted
+192.168.110.11 64558 17:17:28 Establ | 0 0
+192.168.120.13 64558 17:17:28 Establ | 0 0
+192.168.120.14 64558 17:17:28 Establ | 0 0
+192.168.130.15 64558 17:17:28 Establ | 0 0
+192.168.130.16 64558 17:17:28 Establ | 0 0
+192.168.140.17 64558 17:17:28 Establ | 0 0
+192.168.140.18 64558 17:17:28 Establ | 0 0
+[k8s-node-2 ~]# +
+ +View on k8s-node-3: +
+[k8s-node-3 ~]# gobgp neighbor
+Peer AS Up/Down State |#Received Accepted
+192.168.110.11 64558 17:17:28 Establ | 0 0
+192.168.110.12 64558 17:17:28 Establ | 0 0
+192.168.120.14 64558 17:17:28 Establ | 0 0
+192.168.130.15 64558 17:17:28 Establ | 0 0
+192.168.130.16 64558 17:17:28 Establ | 0 0
+192.168.140.17 64558 17:17:28 Establ | 0 0
+192.168.140.18 64558 17:17:28 Establ | 0 0
+[k8s-node-3 ~]# +
+ +View on k8s-node-4: +
+[k8s-node-4 ~]# gobgp neighbor
+Peer AS Up/Down State |#Received Accepted
+192.168.110.11 64558 17:17:28 Establ | 0 0
+192.168.110.12 64558 17:17:28 Establ | 0 0
+192.168.120.13 64558 17:17:28 Establ | 0 0
+192.168.130.15 64558 17:17:28 Establ | 0 0
+192.168.130.16 64558 17:17:28 Establ | 0 0
+192.168.140.17 64558 17:17:28 Establ | 0 0
+192.168.140.18 64558 17:17:28 Establ | 0 0
+[k8s-node-4 ~]# +
+ +View on k8s-node-5: +
+[k8s-node-5 ~]# gobgp neighbor
+Peer AS Up/Down State |#Received Accepted
+192.168.110.11 64558 17:17:28 Establ | 0 0
+192.168.110.12 64558 17:17:28 Establ | 0 0
+192.168.120.13 64558 17:17:28 Establ | 0 0
+192.168.120.14 64558 17:17:28 Establ | 0 0
+192.168.130.16 64558 17:17:28 Establ | 0 0
+192.168.140.17 64558 17:17:28 Establ | 0 0
+192.168.140.18 64558 17:17:28 Establ | 0 0
+[k8s-node-5 ~]# +
+ +View on k8s-node-6: +
+[k8s-node-6 ~]# gobgp neighbor
+Peer AS Up/Down State |#Received Accepted
+192.168.110.11 64558 17:17:28 Establ | 0 0
+192.168.110.12 64558 17:17:28 Establ | 0 0
+192.168.120.13 64558 17:17:28 Establ | 0 0
+192.168.120.14 64558 17:17:28 Establ | 0 0
+192.168.130.15 64558 17:17:28 Establ | 0 0
+192.168.140.17 64558 17:17:28 Establ | 0 0
+192.168.140.18 64558 17:17:28 Establ | 0 0
+[k8s-node-6 ~]# +
+ +View on k8s-node-7: +
+[k8s-node-7 ~]# gobgp neighbor
+Peer AS Up/Down State |#Received Accepted
+192.168.110.11 64558 17:17:28 Establ | 0 0
+192.168.110.12 64558 17:17:28 Establ | 0 0
+192.168.120.13 64558 17:17:28 Establ | 0 0
+192.168.120.14 64558 17:17:28 Establ | 0 0
+192.168.130.15 64558 17:17:28 Establ | 0 0
+192.168.130.16 64558 17:17:28 Establ | 0 0
+192.168.140.18 64558 17:17:28 Establ | 0 0
+[k8s-node-7 ~]# +
+ +View on k8s-node-8: +
+[k8s-node-8 ~]# gobgp neighbor
+Peer AS Up/Down State |#Received Accepted
+192.168.110.11 64558 17:17:28 Establ | 0 0
+192.168.110.12 64558 17:17:28 Establ | 0 0
+192.168.120.13 64558 17:17:28 Establ | 0 0
+192.168.120.14 64558 17:17:28 Establ | 0 0
+192.168.130.15 64558 17:17:28 Establ | 0 0
+192.168.130.16 64558 17:17:28 Establ | 0 0
+192.168.140.17 64558 17:17:28 Establ | 0 0
+ [k8s-node-8 ~]# +
+ +We present the neighborhood relationships between all kubernetes nodes in red dotted lines as follows: + +![avatar](../docs/img/large-networks03.jpg) + +You will find that the number of BGP peers per kubernetes node is n-1 (n is the total number of all nodes in the k8s cluster). If we had 2000 node servers in our k8s cluster, there would be 1999 BGP peers on each node. With the increasing number of kubernetes nodes in k8s cluster, the number of BGP peers per k8s-node will also increase, which is undoubtedly not a small overhead for node servers. With the increase of cluster size, the pressure of node server will increase. +
+ + + +
+
+**How to optimize the BGP neighborhood of kubernetes node?**
+ +To solve this problem, you need to turn on the BGP function of the router device on the node server. Moreover, hardware router devices have special chips to handle BGP routing forwarding, and its performance is better. At the same time, you need to set the parameter "--enable-ibgp=false" of kube-router. +
+
+For example： +
+You can see the white font parameters in the configuration example below. +
+ - name: kube-router
+ image: cloudnativelabs/kube-router:0.3.2
+ imagePullPolicy: IfNotPresent
+ args:
+ - --run-router=true
+ - --run-firewall=true
+ - --run-service-proxy=true
+ - --enable-overlay=false
+ - --advertise-pod-cidr=true
+ - --advertise-cluster-ip=true
+ - --bgp-graceful-restart=true
+ - --enable-ibgp=false
+ - --nodes-full-mesh=true
+ - --cluster-asn=64558
+ - --peer-router-ips=192.168.110.1
+ - --peer-router-asns=64558
+ ......
+
+ +When you set this parameter, all kubernetes nodes will not establish BGP neighborhood relationships with each other. Each kubernetes node establishes a BGP peer only between the router it is connected to. +
+We can log in to each kubernetes node and router to see the local neighborhood. +
+View on router1: +
+dis bgp peer
+ BGP local router ID : 192.168.110.1
+ Local AS number : 64558
+ Total number of peers : 2
+ Peers in established state : 2
+ + Peer V AS MsgRcvd MsgSent OutQ Up/Down State PrefRcv
+ 192.168.110.11 4 64558 2203 2524 0 17:57:52 Established 1
+ 192.168.110.12 4 64558 2203 2524 0 17:57:57 Established 1
+ +
+ +View on router2: +
+dis bgp peer
+ BGP local router ID : 192.168.120.1
+ Local AS number : 64558
+ Total number of peers : 2
+ Peers in established state : 2
+ + Peer V AS MsgRcvd MsgSent OutQ Up/Down State PrefRcv
+ 192.168.120.13 4 64558 2203 2524 0 17:57:52 Established 1
+ 192.168.120.14 4 64558 2203 2524 0 17:57:57 Established 1
+ +
+ +View on router3: +
+dis bgp peer
+ BGP local router ID : 192.168.130.1
+ Local AS number : 64558
+ Total number of peers : 2
+ Peers in established state : 2
+ + Peer V AS MsgRcvd MsgSent OutQ Up/Down State PrefRcv
+ 192.168.130.15 4 64558 2203 2524 0 17:57:52 Established 1
+ 192.168.130.16 4 64558 2203 2524 0 17:57:57 Established 1
+ +
+ +View on router4: +
+dis bgp peer
+ BGP local router ID : 192.168.140.1
+ Local AS number : 64558
+ Total number of peers : 2
+ Peers in established state : 2
+ + Peer V AS MsgRcvd MsgSent OutQ Up/Down State PrefRcv
+ 192.168.140.17 4 64558 2203 2524 0 17:57:52 Established 1
+ 192.168.140.18 4 64558 2203 2524 0 17:57:57 Established 1
+ +
+ + +View on k8s-node-1: +
+[k8s-node-1 ~]# gobgp neighbor
+Peer AS Up/Down State |#Received Accepted
+192.168.110.1 64558 17:17:28 Establ | 0 0
+[k8s-node-1 ~]# +
+ +View on k8s-node-2: +
+[k8s-node-2 ~]# gobgp neighbor
+Peer AS Up/Down State |#Received Accepted
+192.168.110.1 64558 17:17:28 Establ | 0 0
+[k8s-node-2 ~]# +
+ +View on k8s-node-3: +
+[k8s-node-3 ~]# gobgp neighbor
+Peer AS Up/Down State |#Received Accepted
+192.168.120.1 64558 17:17:28 Establ | 0 0
+[k8s-node-3~]# +
+ +View on k8s-node-4: +
+[k8s-node-4 ~]# gobgp neighbor
+Peer AS Up/Down State |#Received Accepted
+192.168.120.1 64558 17:17:28 Establ | 0 0
+[k8s-node-4 ~]# +
+ +View on k8s-node-5: +
+[k8s-node-5 ~]# gobgp neighbor
+Peer AS Up/Down State |#Received Accepted
+192.168.130.1 64558 17:17:28 Establ | 0 0
+[k8s-node-5 ~]# +
+ +View on k8s-node-6: +
+[k8s-node-6 ~]# gobgp neighbor
+Peer AS Up/Down State |#Received Accepted
+192.168.130.1 64558 17:17:28 Establ | 0 0
+[k8s-node-6 ~]# +
+ +View on k8s-node-7: +
+[k8s-node-7 ~]# gobgp neighbor
+Peer AS Up/Down State |#Received Accepted
+192.168.140.1 64558 17:17:28 Establ | 0 0
+[k8s-node-7 ~]# +
+ +View on k8s-node-8: +
+[k8s-node-8 ~]# gobgp neighbor
+Peer AS Up/Down State |#Received Accepted
+192.168.140.1 64558 17:17:28 Establ | 0 0
+[k8s-node-8 ~]# +
+ + +We present the neighborhood relationships between all kubernetes nodes in red dotted lines as follows: + +![avatar](../docs/img/large-networks04.jpg) + +You will find that no matter how many kubernetes nodes you have in your k8s cluster, there is only one BGP neighbor on each kubernetes node. Each router has only BGP neighbors to the downlink kubernetes node and a small number of BGP neighbors to the downlink. In the actual production environment, due to the number of interfaces of router devices, the number of kubernetes nodes that can be connected to each router is very limited, generally less than 100. +
+By setting the parameter "--enable-ibgp=false", we achieve the effect of BGP neighbor control on each kubernetes node in the same AS. This reduces the pressure on the kubernetes node server. +
+
+ + + + + + diff --git a/docs/large-networks03.md b/docs/large-networks03.md new file mode 100644 index 0000000000..6c54580dfa --- /dev/null +++ b/docs/large-networks03.md @@ -0,0 +1,1593 @@ +**Each node announces only the method of aggregating BGP routes to the upstream router device** +
+
+
+ + + +**1 Introduction of cluster BGP network environment**
+ + + + + + + + +
**Item****Content**
Version of kube-routerModified based on version 0.3.2
The number of kubernetes nodeAbout 3000(46 kubernetes nodes are listed as examples in this document)
Pod subnet of kubernetes cluster172.31.0.0/16
Service subnet of kubernetes cluster172.30.0.0/16
The number of Router Device About 100(Four router devices are listed as examples in this document.)
Router-id of Uplink Router Device192.168.110.1(Router1)、
192.168.120.1(Router2)、
192.168.130.1(Router3)、
192.168.140.1(Router4)
......
+ + +The network topology is as follows: +![avatar](../docs/img/large-networks05.jpg) + +Look at all nodes in the cluster as follows: +
+[root@k8s-master ~]# kubectl get node
+NAME STATUS ROLES AGE VERSION
+192.168.120.10 Ready 282d v1.14.0
+192.168.120.11 Ready 282d v1.14.0
+192.168.120.12 Ready 282d v1.14.0
+192.168.120.13 Ready 282d v1.14.0
+192.168.120.14 Ready 282d v1.14.0
+192.168.120.15 Ready 282d v1.14.0
+192.168.120.16 Ready 282d v1.14.0
+192.168.120.17 Ready 282d v1.14.0
+192.168.120.18 Ready 282d v1.14.0
+192.168.120.19 Ready 282d v1.14.0
+192.168.120.20 Ready 282d v1.14.0
+192.168.120.21 Ready 282d v1.14.0
+192.168.120.22 Ready 282d v1.14.0
+192.168.120.23 Ready 282d v1.14.0
+192.168.120.24 Ready 282d v1.14.0
+192.168.120.25 Ready 282d v1.14.0
+192.168.120.26 Ready 282d v1.14.0
+192.168.110.50 Ready 282d v1.14.0
+192.168.110.51 Ready 282d v1.14.0
+192.168.110.52 Ready 49d v1.14.0
+192.168.110.53 Ready 282d v1.14.0
+192.168.110.54 Ready 282d v1.14.0
+192.168.110.55 Ready 49d v1.14.0
+192.168.110.56 Ready 282d v1.14.0
+192.168.110.57 Ready 282d v1.14.0
+192.168.110.58 Ready 49d v1.14.0
+192.168.110.59 Ready 282d v1.14.0
+192.168.110.60 Ready 282d v1.14.0
+ [root@k8s-master ~]#
+
+ +Look at all the services in the cluster as follows: +
+[root@k8s-master ~]# kubectl get service --all-namespaces
+NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
+ci jenkins ClusterIP 172.30.2.11 80/TCP,50000/TCP 34d
+db data-comp ClusterIP 172.30.5.27 80/TCP 43d
+db redis ClusterIP 172.30.47.53 6379/TCP 145d
+coredns corednsdev ClusterIP 172.30.0.2 53/UDP,53/TCP,9153/TCP 93d
+default guestbook-ui ClusterIP 172.30.12.213 80/TCP 145d
+default kubernetes ClusterIP 172.30.0.1 443/TCP 145d
+default nginx01 ClusterIP 172.30.28.176 80/TCP 143d
+ [root@k8s-master ~]# +
+ + +
+
+**2 Bad conditions before optimization using the "advertise-cluster-subnet" parameter**
+ +**2.1 A Brief Introduction to Bad Conditions**
+ +
+The configuration parameters of kube-router are as follows: +
+ - name: kube-router
+ image: cloudnativelabs/kube-router:0.3.2
+ imagePullPolicy: IfNotPresent
+ args:
+ - --run-router=true
+ - --run-firewall=true
+ - --run-service-proxy=true
+ - --enable-overlay=false
+ - --advertise-pod-cidr=true
+ - --advertise-cluster-ip=true
+ - --bgp-graceful-restart=true
+ - --enable-ibgp=false
+ - --nodes-full-mesh=true
+ - --cluster-asn=64558
+ - --peer-router-ips=192.168.110.1
+ - --peer-router-asns=64558
+ ......
+
+ +Create kube-router daemonset using "kubectl create": +
+[root@k8s-master kube-router]# kubectl create -f kube-router-daemonset-old.yaml
+daemonset.extensions/kube-system-kube-router-old created
+[root@k8s-master kube-router]#
+
+ +After creation, use "kubectl get" to view kube-router pod: + +
+[root@k8s-master kube-router]# kubectl get pod -o wide -n kube-system
+NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
+kube-system-kube-router-old-7f2qm 1/1 Running 0 73s 192.168.120.10 192.168.120.10
+kube-system-kube-router-old-9tz2b 1/1 Running 0 73s 192.168.120.11 192.168.120.11
+kube-system-kube-router-old-7f2qq 1/1 Running 0 73s 192.168.120.12 192.168.120.12
+kube-system-kube-router-old-9tz2w 1/1 Running 0 73s 192.168.120.13 192.168.120.13
+kube-system-kube-router-old-7f2qe 1/1 Running 0 73s 192.168.120.14 192.168.120.14
+kube-system-kube-router-old-9tz2r 1/1 Running 0 73s 192.168.120.15 192.168.120.15
+kube-system-kube-router-old-7f2qt 1/1 Running 0 73s 192.168.120.16 192.168.120.16
+kube-system-kube-router-old-9tz2y 1/1 Running 0 73s 192.168.120.17 192.168.120.17
+kube-system-kube-router-old-7f2qu 1/1 Running 0 73s 192.168.120.18 192.168.120.18
+kube-system-kube-router-old-9tz2i 1/1 Running 0 73s 192.168.120.19 192.168.120.19
+kube-system-kube-router-old-7f2qo 1/1 Running 0 73s 192.168.120.20 192.168.120.20
+kube-system-kube-router-old-9tz2p 1/1 Running 0 73s 192.168.120.21 192.168.120.21
+kube-system-kube-router-old-7f2qa 1/1 Running 0 73s 192.168.120.22 192.168.120.22
+kube-system-kube-router-old-9tz2s 1/1 Running 0 73s 192.168.120.23 192.168.120.23
+kube-system-kube-router-old-7f2qd 1/1 Running 0 73s 192.168.120.24 192.168.120.24
+kube-system-kube-router-old-9tz2f 1/1 Running 0 73s 192.168.120.25 192.168.120.25
+kube-system-kube-router-old-7f2qg 1/1 Running 0 73s 192.168.120.26 192.168.120.26
+kube-system-kube-router-old-9tz2h 1/1 Running 0 73s 192.168.110.50 192.168.110.50
+kube-system-kube-router-old-9tz2j 1/1 Running 0 73s 192.168.110.51 192.168.110.51
+kube-system-kube-router-old-9tz2k 1/1 Running 0 73s 192.168.110.52 192.168.110.52
+kube-system-kube-router-old-9tz2l 1/1 Running 0 73s 192.168.110.53 192.168.110.53
+kube-system-kube-router-old-9tz2z 1/1 Running 0 73s 192.168.110.54 192.168.110.54
+kube-system-kube-router-old-9tzxx 1/1 Running 0 73s 192.168.110.55 192.168.110.55
+kube-system-kube-router-old-9tz2c 1/1 Running 0 73s 192.168.110.56 192.168.110.56
+kube-system-kube-router-old-9tz2v 1/1 Running 0 73s 192.168.110.57 192.168.110.57
+kube-system-kube-router-old-9tz2n 1/1 Running 0 73s 192.168.110.58 192.168.110.58
+kube-system-kube-router-old-9tzkk 1/1 Running 0 73s 192.168.110.59 192.168.110.59
+kube-system-kube-router-old-9tzsd 1/1 Running 0 73s 192.168.110.60 192.168.110.60
+[root@k8s-master kube-router]#
+
+ + +View the content of the routing table on Router1: + +
+[Router1] display ip route
+Proto: Protocol Pre: Preference +Route Flags: R - relay, D - download to fib, T - to vpn-instance, B - black hole route
+------------------------------------------------------------------------------
+Routing Table : _public_
+ +Destination/Mask Proto Pre Cost Flags NextHop Interface
+……
+172.30.0.1/32 IBGP 255 0 RD 192.168.110.50 Vlanif110
+                IBGP 255 0 RD 192.168.110.51 Vlanif110
+                IBGP 255 0 RD 192.168.110.52 Vlanif110
+                IBGP 255 0 RD 192.168.110.53 Vlanif110
+                IBGP 255 0 RD 192.168.110.54 Vlanif110
+                IBGP 255 0 RD 192.168.110.55 Vlanif110
+                IBGP 255 0 RD 192.168.110.56 Vlanif110
+                IBGP 255 0 RD 192.168.110.57 Vlanif110
+                IBGP 255 0 RD 192.168.110.58 Vlanif110
+                IBGP 255 0 RD 192.168.110.59 Vlanif110
+                IBGP 255 0 RD 192.168.110.60 Vlanif110
+172.30.0.2/32 IBGP 255 0 RD 192.168.110.50 Vlanif110
+                IBGP 255 0 RD 192.168.110.51 Vlanif110
+                IBGP 255 0 RD 192.168.110.52 Vlanif110
+                IBGP 255 0 RD 192.168.110.53 Vlanif110
+                IBGP 255 0 RD 192.168.110.54 Vlanif110
+                IBGP 255 0 RD 192.168.110.55 Vlanif110
+                IBGP 255 0 RD 192.168.110.56 Vlanif110
+                IBGP 255 0 RD 192.168.110.57 Vlanif110
+                IBGP 255 0 RD 192.168.110.58 Vlanif110
+                IBGP 255 0 RD 192.168.110.59 Vlanif110
+                IBGP 255 0 RD 192.168.110.60 Vlanif110
+172.30.0.9/32 IBGP 255 0 RD 192.168.110.50 Vlanif110
+                IBGP 255 0 RD 192.168.110.51 Vlanif110
+                IBGP 255 0 RD 192.168.110.52 Vlanif110
+                IBGP 255 0 RD 192.168.110.53 Vlanif110
+                IBGP 255 0 RD 192.168.110.54 Vlanif110
+                IBGP 255 0 RD 192.168.110.55 Vlanif110
+                IBGP 255 0 RD 192.168.110.56 Vlanif110
+                IBGP 255 0 RD 192.168.110.57 Vlanif110
+                IBGP 255 0 RD 192.168.110.58 Vlanif110
+                IBGP 255 0 RD 192.168.110.59 Vlanif110
+                IBGP 255 0 RD 192.168.110.60 Vlanif110
+172.30.5.27/32 IBGP 255 0 RD 192.168.110.50 Vlanif110
+                IBGP 255 0 RD 192.168.110.51 Vlanif110
+                IBGP 255 0 RD 192.168.110.52 Vlanif110
+                IBGP 255 0 RD 192.168.110.53 Vlanif110
+                IBGP 255 0 RD 192.168.110.54 Vlanif110
+                IBGP 255 0 RD 192.168.110.55 Vlanif110
+                IBGP 255 0 RD 192.168.110.56 Vlanif110
+                IBGP 255 0 RD 192.168.110.57 Vlanif110
+                IBGP 255 0 RD 192.168.110.58 Vlanif110
+                IBGP 255 0 RD 192.168.110.59 Vlanif110
+                IBGP 255 0 RD 192.168.110.60 Vlanif110
+172.30.12.213/32 IBGP 255 0 RD 192.168.110.50 Vlanif110
+                IBGP 255 0 RD 192.168.110.51 Vlanif110
+                IBGP 255 0 RD 192.168.110.52 Vlanif110
+                IBGP 255 0 RD 192.168.110.53 Vlanif110
+                IBGP 255 0 RD 192.168.110.54 Vlanif110
+                IBGP 255 0 RD 192.168.110.55 Vlanif110
+                IBGP 255 0 RD 192.168.110.56 Vlanif110
+                IBGP 255 0 RD 192.168.110.57 Vlanif110
+                IBGP 255 0 RD 192.168.110.58 Vlanif110
+                IBGP 255 0 RD 192.168.110.59 Vlanif110
+                IBGP 255 0 RD 192.168.110.60 Vlanif110
+172.30.28.176/32 IBGP 255 0 RD 192.168.110.50 Vlanif110
+                IBGP 255 0 RD 192.168.110.51 Vlanif110
+                IBGP 255 0 RD 192.168.110.52 Vlanif110
+                IBGP 255 0 RD 192.168.110.53 Vlanif110
+                IBGP 255 0 RD 192.168.110.54 Vlanif110
+                IBGP 255 0 RD 192.168.110.55 Vlanif110
+                IBGP 255 0 RD 192.168.110.56 Vlanif110
+                IBGP 255 0 RD 192.168.110.57 Vlanif110
+                IBGP 255 0 RD 192.168.110.58 Vlanif110
+                IBGP 255 0 RD 192.168.110.59 Vlanif110
+                IBGP 255 0 RD 192.168.110.60 Vlanif110
+172.30.47.53/32 IBGP 255 0 RD 192.168.110.50 Vlanif110
+                IBGP 255 0 RD 192.168.110.51 Vlanif110
+                IBGP 255 0 RD 192.168.110.52 Vlanif110
+                IBGP 255 0 RD 192.168.110.53 Vlanif110
+                IBGP 255 0 RD 192.168.110.54 Vlanif110
+                IBGP 255 0 RD 192.168.110.55 Vlanif110
+                IBGP 255 0 RD 192.168.110.56 Vlanif110
+                IBGP 255 0 RD 192.168.110.57 Vlanif110
+                IBGP 255 0 RD 192.168.110.58 Vlanif110
+                IBGP 255 0 RD 192.168.110.59 Vlanif110
+                IBGP 255 0 RD 192.168.110.60 Vlanif110
+172.31.0.64/26 IBGP 255 0 RD 192.168.110.50 Vlanif110
+172.31.3.128/26 IBGP 255 0 RD 192.168.110.51 Vlanif110
+172.31.5.192/26 IBGP 255 0 RD 192.168.110.52 Vlanif110
+172.31.6.0/26 IBGP 255 0 RD 192.168.110.53 Vlanif110
+172.31.6.64/26 IBGP 255 0 RD 192.168.110.54 Vlanif110
+172.31.7.0/26 IBGP 255 0 RD 192.168.110.55 Vlanif110
+172.31.7.192/26 IBGP 255 0 RD 192.168.110.56 Vlanif110
+172.31.8.64/26 IBGP 255 0 RD 192.168.110.57 Vlanif110
+172.31.9.64/26 IBGP 255 0 RD 192.168.110.58 Vlanif110
+172.31.9.128/26 IBGP 255 0 RD 192.168.110.59 Vlanif110
+172.31.16.64/26 IBGP 255 0 RD 192.168.110.60 Vlanif110
+……
+255.255.255.255/32 Direct 0 0 D 127.0.0.1 InLoopBack0
+[Router1]
+
+ +As can be seen from the routing table on Router1, there are 77 service IP 32-bit host routes. + + + +
Calculation formulaDescribe
7x11=77Seven k8s services correspond to seven cluster ips, each cluster IP balances its load to 11 next hop routes through the ECMP function.
+ + +View the content of the routing table on Router2: +
+[Router2] display ip route
+Proto: Protocol Pre: Preference
+Route Flags: R - relay, D - download to fib, T - to vpn-instance, B - black hole route
+------------------------------------------------------------------------------
+Routing Table : _public_
+ +Destination/Mask Proto Pre Cost Flags NextHop Interface
+……
+172.30.0.1/32 IBGP 255 0 RD 192.168.120.10 Vlanif120
+                IBGP 255 0 RD 192.168.120.11 Vlanif120
+                IBGP 255 0 RD 192.168.120.12 Vlanif120
+                IBGP 255 0 RD 192.168.120.13 Vlanif120
+                IBGP 255 0 RD 192.168.120.14 Vlanif120
+                IBGP 255 0 RD 192.168.120.15 Vlanif120
+                IBGP 255 0 RD 192.168.120.16 Vlanif120
+                IBGP 255 0 RD 192.168.120.17 Vlanif120
+                IBGP 255 0 RD 192.168.120.18 Vlanif120
+                IBGP 255 0 RD 192.168.120.19 Vlanif120
+                IBGP 255 0 RD 192.168.120.20 Vlanif120
+                IBGP 255 0 RD 192.168.120.21 Vlanif120
+                IBGP 255 0 RD 192.168.120.22 Vlanif120
+                IBGP 255 0 RD 192.168.120.23 Vlanif120
+                IBGP 255 0 RD 192.168.120.24 Vlanif120
+                IBGP 255 0 RD 192.168.120.25 Vlanif120
+                IBGP 255 0 RD 192.168.120.26 Vlanif120
+172.30.0.2/32 IBGP 255 0 RD 192.168.120.10 Vlanif120
+                IBGP 255 0 RD 192.168.120.11 Vlanif120
+                IBGP 255 0 RD 192.168.120.12 Vlanif120
+                IBGP 255 0 RD 192.168.120.13 Vlanif120
+                IBGP 255 0 RD 192.168.120.14 Vlanif120
+                IBGP 255 0 RD 192.168.120.15 Vlanif120
+                IBGP 255 0 RD 192.168.120.16 Vlanif120
+                IBGP 255 0 RD 192.168.120.17 Vlanif120
+                IBGP 255 0 RD 192.168.120.18 Vlanif120
+                IBGP 255 0 RD 192.168.120.19 Vlanif120
+                IBGP 255 0 RD 192.168.120.20 Vlanif120
+                IBGP 255 0 RD 192.168.120.21 Vlanif120
+                IBGP 255 0 RD 192.168.120.22 Vlanif120
+                IBGP 255 0 RD 192.168.120.23 Vlanif120
+                IBGP 255 0 RD 192.168.120.24 Vlanif120
+                IBGP 255 0 RD 192.168.120.25 Vlanif120
+                IBGP 255 0 RD 192.168.120.26 Vlanif120
+172.30.0.9/32 IBGP 255 0 RD 192.168.120.10 Vlanif120
+                IBGP 255 0 RD 192.168.120.11 Vlanif120
+                IBGP 255 0 RD 192.168.120.12 Vlanif120
+                IBGP 255 0 RD 192.168.120.13 Vlanif120
+                IBGP 255 0 RD 192.168.120.14 Vlanif120
+                IBGP 255 0 RD 192.168.120.15 Vlanif120
+                IBGP 255 0 RD 192.168.120.16 Vlanif120
+                IBGP 255 0 RD 192.168.120.17 Vlanif120
+                IBGP 255 0 RD 192.168.120.18 Vlanif120
+                IBGP 255 0 RD 192.168.120.19 Vlanif120
+                IBGP 255 0 RD 192.168.120.20 Vlanif120
+                IBGP 255 0 RD 192.168.120.21 Vlanif120
+                IBGP 255 0 RD 192.168.120.22 Vlanif120
+                IBGP 255 0 RD 192.168.120.23 Vlanif120
+                IBGP 255 0 RD 192.168.120.24 Vlanif120
+                IBGP 255 0 RD 192.168.120.25 Vlanif120
+                IBGP 255 0 RD 192.168.120.26 Vlanif120
+172.30.5.27/32 IBGP 255 0 RD 192.168.120.10 Vlanif120
+                IBGP 255 0 RD 192.168.120.11 Vlanif120
+                IBGP 255 0 RD 192.168.120.12 Vlanif120
+                IBGP 255 0 RD 192.168.120.13 Vlanif120
+                IBGP 255 0 RD 192.168.120.14 Vlanif120
+                IBGP 255 0 RD 192.168.120.15 Vlanif120
+                IBGP 255 0 RD 192.168.120.16 Vlanif120
+                IBGP 255 0 RD 192.168.120.17 Vlanif120
+                IBGP 255 0 RD 192.168.120.18 Vlanif120
+                IBGP 255 0 RD 192.168.120.19 Vlanif120
+                IBGP 255 0 RD 192.168.120.20 Vlanif120
+                IBGP 255 0 RD 192.168.120.21 Vlanif120
+                IBGP 255 0 RD 192.168.120.22 Vlanif120
+                IBGP 255 0 RD 192.168.120.23 Vlanif120
+                IBGP 255 0 RD 192.168.120.24 Vlanif120
+                IBGP 255 0 RD 192.168.120.25 Vlanif120
+                IBGP 255 0 RD 192.168.120.26 Vlanif120
+172.30.12.213/32 IBGP 255 0 RD 192.168.120.10 Vlanif120
+                IBGP 255 0 RD 192.168.120.11 Vlanif120
+                IBGP 255 0 RD 192.168.120.12 Vlanif120
+                IBGP 255 0 RD 192.168.120.13 Vlanif120
+                IBGP 255 0 RD 192.168.120.14 Vlanif120
+                IBGP 255 0 RD 192.168.120.15 Vlanif120
+                IBGP 255 0 RD 192.168.120.16 Vlanif120
+                IBGP 255 0 RD 192.168.120.17 Vlanif120
+                IBGP 255 0 RD 192.168.120.18 Vlanif120
+                IBGP 255 0 RD 192.168.120.19 Vlanif120
+                IBGP 255 0 RD 192.168.120.20 Vlanif120
+                IBGP 255 0 RD 192.168.120.21 Vlanif120
+                IBGP 255 0 RD 192.168.120.22 Vlanif120
+                IBGP 255 0 RD 192.168.120.23 Vlanif120
+                IBGP 255 0 RD 192.168.120.24 Vlanif120
+                IBGP 255 0 RD 192.168.120.25 Vlanif120
+                IBGP 255 0 RD 192.168.120.26 Vlanif120
+172.30.28.176/32 IBGP 255 0 RD 192.168.120.10 Vlanif120
+                IBGP 255 0 RD 192.168.120.11 Vlanif120
+                IBGP 255 0 RD 192.168.120.12 Vlanif120
+                IBGP 255 0 RD 192.168.120.13 Vlanif120
+                IBGP 255 0 RD 192.168.120.14 Vlanif120
+                IBGP 255 0 RD 192.168.120.15 Vlanif120
+                IBGP 255 0 RD 192.168.120.16 Vlanif120
+                IBGP 255 0 RD 192.168.120.17 Vlanif120
+                IBGP 255 0 RD 192.168.120.18 Vlanif120
+                IBGP 255 0 RD 192.168.120.19 Vlanif120
+                IBGP 255 0 RD 192.168.120.20 Vlanif120
+                IBGP 255 0 RD 192.168.120.21 Vlanif120
+                IBGP 255 0 RD 192.168.120.22 Vlanif120
+                IBGP 255 0 RD 192.168.120.23 Vlanif120
+                IBGP 255 0 RD 192.168.120.24 Vlanif120
+                IBGP 255 0 RD 192.168.120.25 Vlanif120
+                IBGP 255 0 RD 192.168.120.26 Vlanif120
+172.30.47.53/32 IBGP 255 0 RD 192.168.120.10 Vlanif120
+                IBGP 255 0 RD 192.168.120.11 Vlanif120
+                IBGP 255 0 RD 192.168.120.12 Vlanif120
+                IBGP 255 0 RD 192.168.120.13 Vlanif120
+                IBGP 255 0 RD 192.168.120.14 Vlanif120
+                IBGP 255 0 RD 192.168.120.15 Vlanif120
+                IBGP 255 0 RD 192.168.120.16 Vlanif120
+                IBGP 255 0 RD 192.168.120.17 Vlanif120
+                IBGP 255 0 RD 192.168.120.18 Vlanif120
+                IBGP 255 0 RD 192.168.120.19 Vlanif120
+                IBGP 255 0 RD 192.168.120.20 Vlanif120
+                IBGP 255 0 RD 192.168.120.21 Vlanif120
+                IBGP 255 0 RD 192.168.120.22 Vlanif120
+                IBGP 255 0 RD 192.168.120.23 Vlanif120
+                IBGP 255 0 RD 192.168.120.24 Vlanif120
+                IBGP 255 0 RD 192.168.120.25 Vlanif120
+                IBGP 255 0 RD 192.168.120.26 Vlanif120
+172.31.0.0/26 IBGP 255 0 RD 192.168.120.10 Vlanif120
+172.31.2.64/26 IBGP 255 0 RD 192.168.120.11 Vlanif120
+172.31.3.0/26 IBGP 255 0 RD 192.168.120.12 Vlanif120
+172.31.4.192/26 IBGP 255 0 RD 192.168.120.13 Vlanif120
+172.31.5.64/26 IBGP 255 0 RD 192.168.120.14 Vlanif120
+172.31.6.128/26 IBGP 255 0 RD 192.168.120.15 Vlanif120
+172.31.6.192/26 IBGP 255 0 RD 192.168.120.16 Vlanif120
+172.31.7.64/26 IBGP 255 0 RD 192.168.120.17 Vlanif120
+172.31.7.128/26 IBGP 255 0 RD 192.168.120.18 Vlanif120
+172.31.9.0/26 IBGP 255 0 RD 192.168.120.19 Vlanif120
+172.31.10.0/26 IBGP 255 0 RD 192.168.120.20 Vlanif120
+172.31.10.64/26 IBGP 255 0 RD 192.168.120.21 Vlanif120
+172.31.11.0/26 IBGP 255 0 RD 192.168.120.22 Vlanif120
+172.31.11.192/26 IBGP 255 0 RD 192.168.120.23 Vlanif120
+172.31.12.64/26 IBGP 255 0 RD 192.168.120.24 Vlanif120
+172.31.13.128/26 IBGP 255 0 RD 192.168.120.25 Vlanif120
+172.31.14.0/26 IBGP 255 0 RD 192.168.120.26 Vlanif120
+…… +255.255.255.255/32 Direct 0 0 D 127.0.0.1 InLoopBack0
+[Router2]
+
+ + +As can be seen from the routing table on Router2, there are 77 service IP 32-bit host routes. + + + +
Calculation formulaDescribe
7x11=77Seven k8s services correspond to seven cluster ips, each cluster IP balances its load to 11 next hop routes through the ECMP function.
+ + +Similarly, we look at the kubernetes service routing on Router3 and Router4, and the results are the same as those on Router1 and Router2. + + + +**2.2 Problems after adding kubernetes node nodes：**
+ +At this time, we will add three kubernetes nodes to Router1, Router2 and Router3 devices respectively. +![avatar](../docs/img/large-networks06.jpg) + +Using the "kubectl get node" command, you can see three new kubernetes nodes in the white font section: +
+[root@k8s-master ~]# kubectl get node
+NAME STATUS ROLES AGE VERSION
+192.168.120.10 Ready 282d v1.14.0
+192.168.120.11 Ready 282d v1.14.0
+192.168.120.12 Ready 282d v1.14.0
+192.168.120.13 Ready 282d v1.14.0
+192.168.120.14 Ready 282d v1.14.0
+192.168.120.15 Ready 282d v1.14.0
+192.168.120.16 Ready 282d v1.14.0
+192.168.120.17 Ready 282d v1.14.0
+192.168.120.18 Ready 282d v1.14.0
+192.168.120.19 Ready 282d v1.14.0
+192.168.120.20 Ready 282d v1.14.0
+192.168.120.21 Ready 282d v1.14.0
+192.168.120.22 Ready 282d v1.14.0
+192.168.120.23 Ready 282d v1.14.0
+192.168.120.24 Ready 282d v1.14.0
+192.168.120.25 Ready 282d v1.14.0
+192.168.120.26 Ready 282d v1.14.0
+192.168.110.50 Ready 282d v1.14.0
+192.168.110.51 Ready 282d v1.14.0
+192.168.110.52 Ready 49d v1.14.0
+192.168.110.53 Ready 282d v1.14.0
+192.168.110.54 Ready 282d v1.14.0
+192.168.110.55 Ready 49d v1.14.0
+192.168.110.56 Ready 282d v1.14.0
+192.168.110.57 Ready 282d v1.14.0
+192.168.110.58 Ready 49d v1.14.0
+192.168.110.59 Ready 282d v1.14.0
+192.168.110.60 Ready 282d v1.14.0
+ +192.168.120.81 Ready 282d v1.14.0
+192.168.120.82 Ready 282d v1.14.0
+192.168.120.83 Ready 282d v1.14.0
+192.168.110.81 Ready 49d v1.14.0
+192.168.110.82 Ready 282d v1.14.0
+192.168.110.83 Ready 282d v1.14.0
+ + [root@k8s-master ~]# +
+ +Using the "kubectl get pod" command, you can see that the white font part is the three new kube-router pods: +
+[root@k8s-master kube-router]# kubectl get pod -o wide -n kube-system
+NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
+kube-system-kube-router-old-7f2qm 1/1 Running 0 73s 192.168.120.10 192.168.120.10
+kube-system-kube-router-old-9tz2b 1/1 Running 0 73s 192.168.120.11 192.168.120.11
+kube-system-kube-router-old-7f2qq 1/1 Running 0 73s 192.168.120.12 192.168.120.12
+kube-system-kube-router-old-9tz2w 1/1 Running 0 73s 192.168.120.13 192.168.120.13
+kube-system-kube-router-old-7f2qe 1/1 Running 0 73s 192.168.120.14 192.168.120.14
+kube-system-kube-router-old-9tz2r 1/1 Running 0 73s 192.168.120.15 192.168.120.15
+kube-system-kube-router-old-7f2qt 1/1 Running 0 73s 192.168.120.16 192.168.120.16
+kube-system-kube-router-old-9tz2y 1/1 Running 0 73s 192.168.120.17 192.168.120.17
+kube-system-kube-router-old-7f2qu 1/1 Running 0 73s 192.168.120.18 192.168.120.18
+kube-system-kube-router-old-9tz2i 1/1 Running 0 73s 192.168.120.19 192.168.120.19
+kube-system-kube-router-old-7f2qo 1/1 Running 0 73s 192.168.120.20 192.168.120.20
+kube-system-kube-router-old-9tz2p 1/1 Running 0 73s 192.168.120.21 192.168.120.21
+kube-system-kube-router-old-7f2qa 1/1 Running 0 73s 192.168.120.22 192.168.120.22
+kube-system-kube-router-old-9tz2s 1/1 Running 0 73s 192.168.120.23 192.168.120.23
+kube-system-kube-router-old-7f2qd 1/1 Running 0 73s 192.168.120.24 192.168.120.24
+kube-system-kube-router-old-9tz2f 1/1 Running 0 73s 192.168.120.25 192.168.120.25
+kube-system-kube-router-old-7f2qg 1/1 Running 0 73s 192.168.120.26 192.168.120.26
+kube-system-kube-router-old-9tz2h 1/1 Running 0 73s 192.168.110.50 192.168.110.50
+kube-system-kube-router-old-9tz2j 1/1 Running 0 73s 192.168.110.51 192.168.110.51
+kube-system-kube-router-old-9tz2k 1/1 Running 0 73s 192.168.110.52 192.168.110.52
+kube-system-kube-router-old-9tz2l 1/1 Running 0 73s 192.168.110.53 192.168.110.53
+kube-system-kube-router-old-9tz2z 1/1 Running 0 73s 192.168.110.54 192.168.110.54
+kube-system-kube-router-old-9tzxx 1/1 Running 0 73s 192.168.110.55 192.168.110.55
+kube-system-kube-router-old-9tz2c 1/1 Running 0 73s 192.168.110.56 192.168.110.56
+kube-system-kube-router-old-9tz2v 1/1 Running 0 73s 192.168.110.57 192.168.110.57
+kube-system-kube-router-old-9tz2n 1/1 Running 0 73s 192.168.110.58 192.168.110.58
+kube-system-kube-router-old-9tzkk 1/1 Running 0 73s 192.168.110.59 192.168.110.59
+kube-system-kube-router-old-9tzsd 1/1 Running 0 73s 192.168.110.60 192.168.110.60
+ +kube-system-kube-router-old-qf2q1 1/1 Running 0 73s 192.168.120.81 192.168.120.81
+kube-system-kube-router-old-wtz22 1/1 Running 0 73s 192.168.120.82 192.168.120.82
+kube-system-kube-router-old-ef2qe 1/1 Running 0 73s 192.168.120.83 192.168.120.83
+kube-system-kube-router-old-rtz2g 1/1 Running 0 73s 192.168.110.81 192.168.110.81
+kube-system-kube-router-old-ttz2p 1/1 Running 0 73s 192.168.110.82 192.168.110.82
+kube-system-kube-router-old-ytz2o 1/1 Running 0 73s 192.168.110.83 192.168.110.83
+ +[root@k8s-master kube-router]#
+
+ + + + +Look at Router 1's routing table again: +
+[Router1] display ip route
+Proto: Protocol Pre: Preference
+Route Flags: R - relay, D - download to fib, T - to vpn-instance, B - black hole route
+------------------------------------------------------------------------------
+Routing Table : _public_
+ +Destination/Mask Proto Pre Cost Flags NextHop Interface
+……
+172.30.0.1/32 IBGP 255 0 RD 192.168.110.50 Vlanif110
+                IBGP 255 0 RD 192.168.110.51 Vlanif110
+                IBGP 255 0 RD 192.168.110.52 Vlanif110
+                IBGP 255 0 RD 192.168.110.53 Vlanif110
+                IBGP 255 0 RD 192.168.110.54 Vlanif110
+                IBGP 255 0 RD 192.168.110.55 Vlanif110
+                IBGP 255 0 RD 192.168.110.56 Vlanif110
+                IBGP 255 0 RD 192.168.110.57 Vlanif110
+                IBGP 255 0 RD 192.168.110.58 Vlanif110
+                IBGP 255 0 RD 192.168.110.59 Vlanif110
+                IBGP 255 0 RD 192.168.110.60 Vlanif110
+ +                IBGP 255 0 RD 192.168.110.81 Vlanif110
+                IBGP 255 0 RD 192.168.110.82 Vlanif110
+                IBGP 255 0 RD 192.168.110.83 Vlanif110
+ +172.30.0.2/32 IBGP 255 0 RD 192.168.110.50 Vlanif110
+                IBGP 255 0 RD 192.168.110.51 Vlanif110
+                IBGP 255 0 RD 192.168.110.52 Vlanif110
+                IBGP 255 0 RD 192.168.110.53 Vlanif110
+                IBGP 255 0 RD 192.168.110.54 Vlanif110
+                IBGP 255 0 RD 192.168.110.55 Vlanif110
+                IBGP 255 0 RD 192.168.110.56 Vlanif110
+                IBGP 255 0 RD 192.168.110.57 Vlanif110
+                IBGP 255 0 RD 192.168.110.58 Vlanif110
+                IBGP 255 0 RD 192.168.110.59 Vlanif110
+                IBGP 255 0 RD 192.168.110.60 Vlanif110
+ +                IBGP 255 0 RD 192.168.110.81 Vlanif110
+                IBGP 255 0 RD 192.168.110.82 Vlanif110
+                IBGP 255 0 RD 192.168.110.83 Vlanif110
+ +172.30.0.9/32 IBGP 255 0 RD 192.168.110.50 Vlanif110
+                IBGP 255 0 RD 192.168.110.51 Vlanif110
+                IBGP 255 0 RD 192.168.110.52 Vlanif110
+                IBGP 255 0 RD 192.168.110.53 Vlanif110
+                IBGP 255 0 RD 192.168.110.54 Vlanif110
+                IBGP 255 0 RD 192.168.110.55 Vlanif110
+                IBGP 255 0 RD 192.168.110.56 Vlanif110
+                IBGP 255 0 RD 192.168.110.57 Vlanif110
+                IBGP 255 0 RD 192.168.110.58 Vlanif110
+                IBGP 255 0 RD 192.168.110.59 Vlanif110
+                IBGP 255 0 RD 192.168.110.60 Vlanif110
+ +                IBGP 255 0 RD 192.168.110.81 Vlanif110
+                IBGP 255 0 RD 192.168.110.82 Vlanif110
+                IBGP 255 0 RD 192.168.110.83 Vlanif110
+ +172.30.5.27/32 IBGP 255 0 RD 192.168.110.50 Vlanif110
+                IBGP 255 0 RD 192.168.110.51 Vlanif110
+                IBGP 255 0 RD 192.168.110.52 Vlanif110
+                IBGP 255 0 RD 192.168.110.53 Vlanif110
+                IBGP 255 0 RD 192.168.110.54 Vlanif110
+                IBGP 255 0 RD 192.168.110.55 Vlanif110
+                IBGP 255 0 RD 192.168.110.56 Vlanif110
+                IBGP 255 0 RD 192.168.110.57 Vlanif110
+                IBGP 255 0 RD 192.168.110.58 Vlanif110
+                IBGP 255 0 RD 192.168.110.59 Vlanif110
+                IBGP 255 0 RD 192.168.110.60 Vlanif110
+ +                IBGP 255 0 RD 192.168.110.81 Vlanif110
+                IBGP 255 0 RD 192.168.110.82 Vlanif110
+                IBGP 255 0 RD 192.168.110.83 Vlanif110
+ +172.30.12.213/32 IBGP 255 0 RD 192.168.110.50 Vlanif110
+                IBGP 255 0 RD 192.168.110.51 Vlanif110
+                IBGP 255 0 RD 192.168.110.52 Vlanif110
+                IBGP 255 0 RD 192.168.110.53 Vlanif110
+                IBGP 255 0 RD 192.168.110.54 Vlanif110
+                IBGP 255 0 RD 192.168.110.55 Vlanif110
+                IBGP 255 0 RD 192.168.110.56 Vlanif110
+                IBGP 255 0 RD 192.168.110.57 Vlanif110
+                IBGP 255 0 RD 192.168.110.58 Vlanif110
+                IBGP 255 0 RD 192.168.110.59 Vlanif110
+                IBGP 255 0 RD 192.168.110.60 Vlanif110
+ +                IBGP 255 0 RD 192.168.110.81 Vlanif110
+                IBGP 255 0 RD 192.168.110.82 Vlanif110
+                IBGP 255 0 RD 192.168.110.83 Vlanif110
+ +172.30.28.176/32 IBGP 255 0 RD 192.168.110.50 Vlanif110
+                IBGP 255 0 RD 192.168.110.51 Vlanif110
+                IBGP 255 0 RD 192.168.110.52 Vlanif110
+                IBGP 255 0 RD 192.168.110.53 Vlanif110
+                IBGP 255 0 RD 192.168.110.54 Vlanif110
+                IBGP 255 0 RD 192.168.110.55 Vlanif110
+                IBGP 255 0 RD 192.168.110.56 Vlanif110
+                IBGP 255 0 RD 192.168.110.57 Vlanif110
+                IBGP 255 0 RD 192.168.110.58 Vlanif110
+                IBGP 255 0 RD 192.168.110.59 Vlanif110
+                IBGP 255 0 RD 192.168.110.60 Vlanif110
+ +                IBGP 255 0 RD 192.168.110.81 Vlanif110
+                IBGP 255 0 RD 192.168.110.82 Vlanif110
+                IBGP 255 0 RD 192.168.110.83 Vlanif110
+ +172.30.47.53/32 IBGP 255 0 RD 192.168.110.50 Vlanif110
+                IBGP 255 0 RD 192.168.110.51 Vlanif110
+                IBGP 255 0 RD 192.168.110.52 Vlanif110
+                IBGP 255 0 RD 192.168.110.53 Vlanif110
+                IBGP 255 0 RD 192.168.110.54 Vlanif110
+                IBGP 255 0 RD 192.168.110.55 Vlanif110
+                IBGP 255 0 RD 192.168.110.56 Vlanif110
+                IBGP 255 0 RD 192.168.110.57 Vlanif110
+                IBGP 255 0 RD 192.168.110.58 Vlanif110
+                IBGP 255 0 RD 192.168.110.59 Vlanif110
+                IBGP 255 0 RD 192.168.110.60 Vlanif110
+ +                IBGP 255 0 RD 192.168.110.81 Vlanif110
+                IBGP 255 0 RD 192.168.110.82 Vlanif110
+                IBGP 255 0 RD 192.168.110.83 Vlanif110
+ +172.31.0.64/26 IBGP 255 0 RD 192.168.110.50 Vlanif110
+172.31.3.128/26 IBGP 255 0 RD 192.168.110.51 Vlanif110
+172.31.5.192/26 IBGP 255 0 RD 192.168.110.52 Vlanif110
+172.31.6.0/26 IBGP 255 0 RD 192.168.110.53 Vlanif110
+172.31.6.64/26 IBGP 255 0 RD 192.168.110.54 Vlanif110
+172.31.7.0/26 IBGP 255 0 RD 192.168.110.55 Vlanif110
+172.31.7.192/26 IBGP 255 0 RD 192.168.110.56 Vlanif110
+172.31.8.64/26 IBGP 255 0 RD 192.168.110.57 Vlanif110
+172.31.9.64/26 IBGP 255 0 RD 192.168.110.58 Vlanif110
+172.31.9.128/26 IBGP 255 0 RD 192.168.110.59 Vlanif110
+172.31.16.64/26 IBGP 255 0 RD 192.168.110.60 Vlanif110
+72.31.17.64/26 IBGP 255 0 RD 192.168.110.81 Vlanif110
+172.31.17.128/26 IBGP 255 0 RD 192.168.110.82 Vlanif110
+172.31.17.64/26 IBGP 255 0 RD 192.168.110.83 Vlanif110
+……
+255.255.255.255/32 Direct 0 0 D 127.0.0.1 InLoopBack0
+[Router1]
+
+ + +As can be seen from the routing table on Router1, there are 98 service IP 32-bit host routes.On the original basis, 21 routes have been added. + + + +
Calculation formulaDescribe
7x14=98Seven k8s services correspond to seven cluster ips, each cluster IP has 14 routes for the next hop.
+ + + + +Look at Router2's routing table again: +
+[Router2] display ip route
+Proto: Protocol Pre: Preference
+Route Flags: R - relay, D - download to fib, T - to vpn-instance, B - black hole route
+------------------------------------------------------------------------------
+Routing Table : _public_
+ +Destination/Mask Proto Pre Cost Flags NextHop Interface
+……
+172.30.0.1/32 IBGP 255 0 RD 192.168.120.10 Vlanif120
+                IBGP 255 0 RD 192.168.120.11 Vlanif120
+                IBGP 255 0 RD 192.168.120.12 Vlanif120
+                IBGP 255 0 RD 192.168.120.13 Vlanif120
+                IBGP 255 0 RD 192.168.120.14 Vlanif120
+                IBGP 255 0 RD 192.168.120.15 Vlanif120
+                IBGP 255 0 RD 192.168.120.16 Vlanif120
+                IBGP 255 0 RD 192.168.120.17 Vlanif120
+                IBGP 255 0 RD 192.168.120.18 Vlanif120
+                IBGP 255 0 RD 192.168.120.19 Vlanif120
+                IBGP 255 0 RD 192.168.120.20 Vlanif120
+                IBGP 255 0 RD 192.168.120.21 Vlanif120
+                IBGP 255 0 RD 192.168.120.22 Vlanif120
+                IBGP 255 0 RD 192.168.120.23 Vlanif120
+                IBGP 255 0 RD 192.168.120.24 Vlanif120
+                IBGP 255 0 RD 192.168.120.25 Vlanif120
+                IBGP 255 0 RD 192.168.120.26 Vlanif120
+ +                IBGP 255 0 RD 192.168.120.81 Vlanif120
+                IBGP 255 0 RD 192.168.120.82 Vlanif120
+                IBGP 255 0 RD 192.168.120.83 Vlanif120
+ +172.30.0.2/32 IBGP 255 0 RD 192.168.120.10 Vlanif120
+                IBGP 255 0 RD 192.168.120.11 Vlanif120
+                IBGP 255 0 RD 192.168.120.12 Vlanif120
+                IBGP 255 0 RD 192.168.120.13 Vlanif120
+                IBGP 255 0 RD 192.168.120.14 Vlanif120
+                IBGP 255 0 RD 192.168.120.15 Vlanif120
+                IBGP 255 0 RD 192.168.120.16 Vlanif120
+                IBGP 255 0 RD 192.168.120.17 Vlanif120
+                IBGP 255 0 RD 192.168.120.18 Vlanif120
+                IBGP 255 0 RD 192.168.120.19 Vlanif120
+                IBGP 255 0 RD 192.168.120.20 Vlanif120
+                IBGP 255 0 RD 192.168.120.21 Vlanif120
+                IBGP 255 0 RD 192.168.120.22 Vlanif120
+                IBGP 255 0 RD 192.168.120.23 Vlanif120
+                IBGP 255 0 RD 192.168.120.24 Vlanif120
+                IBGP 255 0 RD 192.168.120.25 Vlanif120
+                IBGP 255 0 RD 192.168.120.26 Vlanif120
+ +                IBGP 255 0 RD 192.168.120.81 Vlanif120
+                IBGP 255 0 RD 192.168.120.82 Vlanif120
+                IBGP 255 0 RD 192.168.120.83 Vlanif120
+ +172.30.0.9/32 IBGP 255 0 RD 192.168.120.10 Vlanif120
+                IBGP 255 0 RD 192.168.120.11 Vlanif120
+                IBGP 255 0 RD 192.168.120.12 Vlanif120
+                IBGP 255 0 RD 192.168.120.13 Vlanif120
+                IBGP 255 0 RD 192.168.120.14 Vlanif120
+                IBGP 255 0 RD 192.168.120.15 Vlanif120
+                IBGP 255 0 RD 192.168.120.16 Vlanif120
+                IBGP 255 0 RD 192.168.120.17 Vlanif120
+                IBGP 255 0 RD 192.168.120.18 Vlanif120
+                IBGP 255 0 RD 192.168.120.19 Vlanif120
+                IBGP 255 0 RD 192.168.120.20 Vlanif120
+                IBGP 255 0 RD 192.168.120.21 Vlanif120
+                IBGP 255 0 RD 192.168.120.22 Vlanif120
+                IBGP 255 0 RD 192.168.120.23 Vlanif120
+                IBGP 255 0 RD 192.168.120.24 Vlanif120
+                IBGP 255 0 RD 192.168.120.25 Vlanif120
+                IBGP 255 0 RD 192.168.120.26 Vlanif120
+ +                IBGP 255 0 RD 192.168.120.81 Vlanif120
+                IBGP 255 0 RD 192.168.120.82 Vlanif120
+                IBGP 255 0 RD 192.168.120.83 Vlanif120
+ +172.30.5.27/32 IBGP 255 0 RD 192.168.120.10 Vlanif120
+                IBGP 255 0 RD 192.168.120.11 Vlanif120
+                IBGP 255 0 RD 192.168.120.12 Vlanif120
+                IBGP 255 0 RD 192.168.120.13 Vlanif120
+                IBGP 255 0 RD 192.168.120.14 Vlanif120
+                IBGP 255 0 RD 192.168.120.15 Vlanif120
+                IBGP 255 0 RD 192.168.120.16 Vlanif120
+                IBGP 255 0 RD 192.168.120.17 Vlanif120
+                IBGP 255 0 RD 192.168.120.18 Vlanif120
+                IBGP 255 0 RD 192.168.120.19 Vlanif120
+                IBGP 255 0 RD 192.168.120.20 Vlanif120
+                IBGP 255 0 RD 192.168.120.21 Vlanif120
+                IBGP 255 0 RD 192.168.120.22 Vlanif120
+                IBGP 255 0 RD 192.168.120.23 Vlanif120
+                IBGP 255 0 RD 192.168.120.24 Vlanif120
+                IBGP 255 0 RD 192.168.120.25 Vlanif120
+                IBGP 255 0 RD 192.168.120.26 Vlanif120
+ +                IBGP 255 0 RD 192.168.120.81 Vlanif120
+                IBGP 255 0 RD 192.168.120.82 Vlanif120
+                IBGP 255 0 RD 192.168.120.83 Vlanif120
+ +172.30.12.213/32 IBGP 255 0 RD 192.168.120.10 Vlanif120
+                IBGP 255 0 RD 192.168.120.11 Vlanif120
+                IBGP 255 0 RD 192.168.120.12 Vlanif120
+                IBGP 255 0 RD 192.168.120.13 Vlanif120
+                IBGP 255 0 RD 192.168.120.14 Vlanif120
+                IBGP 255 0 RD 192.168.120.15 Vlanif120
+                IBGP 255 0 RD 192.168.120.16 Vlanif120
+                IBGP 255 0 RD 192.168.120.17 Vlanif120
+                IBGP 255 0 RD 192.168.120.18 Vlanif120
+                IBGP 255 0 RD 192.168.120.19 Vlanif120
+                IBGP 255 0 RD 192.168.120.20 Vlanif120
+                IBGP 255 0 RD 192.168.120.21 Vlanif120
+                IBGP 255 0 RD 192.168.120.22 Vlanif120
+                IBGP 255 0 RD 192.168.120.23 Vlanif120
+                IBGP 255 0 RD 192.168.120.24 Vlanif120
+                IBGP 255 0 RD 192.168.120.25 Vlanif120
+                IBGP 255 0 RD 192.168.120.26 Vlanif120
+ +                IBGP 255 0 RD 192.168.120.81 Vlanif120
+                IBGP 255 0 RD 192.168.120.82 Vlanif120
+                IBGP 255 0 RD 192.168.120.83 Vlanif120
+ +172.30.28.176/32 IBGP 255 0 RD 192.168.120.10 Vlanif120
+                IBGP 255 0 RD 192.168.120.11 Vlanif120
+                IBGP 255 0 RD 192.168.120.12 Vlanif120
+                IBGP 255 0 RD 192.168.120.13 Vlanif120
+                IBGP 255 0 RD 192.168.120.14 Vlanif120
+                IBGP 255 0 RD 192.168.120.15 Vlanif120
+                IBGP 255 0 RD 192.168.120.16 Vlanif120
+                IBGP 255 0 RD 192.168.120.17 Vlanif120
+                IBGP 255 0 RD 192.168.120.18 Vlanif120
+                IBGP 255 0 RD 192.168.120.19 Vlanif120
+                IBGP 255 0 RD 192.168.120.20 Vlanif120
+                IBGP 255 0 RD 192.168.120.21 Vlanif120
+                IBGP 255 0 RD 192.168.120.22 Vlanif120
+                IBGP 255 0 RD 192.168.120.23 Vlanif120
+                IBGP 255 0 RD 192.168.120.24 Vlanif120
+                IBGP 255 0 RD 192.168.120.25 Vlanif120
+                IBGP 255 0 RD 192.168.120.26 Vlanif120
+ +                IBGP 255 0 RD 192.168.120.81 Vlanif120
+                IBGP 255 0 RD 192.168.120.82 Vlanif120
+                IBGP 255 0 RD 192.168.120.83 Vlanif120
+ +172.30.47.53/32 IBGP 255 0 RD 192.168.120.10 Vlanif120
+                IBGP 255 0 RD 192.168.120.11 Vlanif120
+                IBGP 255 0 RD 192.168.120.12 Vlanif120
+                IBGP 255 0 RD 192.168.120.13 Vlanif120
+                IBGP 255 0 RD 192.168.120.14 Vlanif120
+                IBGP 255 0 RD 192.168.120.15 Vlanif120
+                IBGP 255 0 RD 192.168.120.16 Vlanif120
+                IBGP 255 0 RD 192.168.120.17 Vlanif120
+                IBGP 255 0 RD 192.168.120.18 Vlanif120
+                IBGP 255 0 RD 192.168.120.19 Vlanif120
+                IBGP 255 0 RD 192.168.120.20 Vlanif120
+                IBGP 255 0 RD 192.168.120.21 Vlanif120
+                IBGP 255 0 RD 192.168.120.22 Vlanif120
+                IBGP 255 0 RD 192.168.120.23 Vlanif120
+                IBGP 255 0 RD 192.168.120.24 Vlanif120
+                IBGP 255 0 RD 192.168.120.25 Vlanif120
+                IBGP 255 0 RD 192.168.120.26 Vlanif120
+ +                IBGP 255 0 RD 192.168.120.81 Vlanif120
+                IBGP 255 0 RD 192.168.120.82 Vlanif120
+                IBGP 255 0 RD 192.168.120.83 Vlanif120
+ +172.31.0.0/26 IBGP 255 0 RD 192.168.120.10 Vlanif120
+172.31.2.64/26 IBGP 255 0 RD 192.168.120.11 Vlanif120
+172.31.3.0/26 IBGP 255 0 RD 192.168.120.12 Vlanif120
+172.31.4.192/26 IBGP 255 0 RD 192.168.120.13 Vlanif120
+172.31.5.64/26 IBGP 255 0 RD 192.168.120.14 Vlanif120
+172.31.6.128/26 IBGP 255 0 RD 192.168.120.15 Vlanif120
+172.31.6.192/26 IBGP 255 0 RD 192.168.120.16 Vlanif120
+172.31.7.64/26 IBGP 255 0 RD 192.168.120.17 Vlanif120
+172.31.7.128/26 IBGP 255 0 RD 192.168.120.18 Vlanif120
+172.31.9.0/26 IBGP 255 0 RD 192.168.120.19 Vlanif120
+172.31.10.0/26 IBGP 255 0 RD 192.168.120.20 Vlanif120
+172.31.10.64/26 IBGP 255 0 RD 192.168.120.21 Vlanif120
+172.31.11.0/26 IBGP 255 0 RD 192.168.120.22 Vlanif120
+172.31.11.192/26 IBGP 255 0 RD 192.168.120.23 Vlanif120
+172.31.12.64/26 IBGP 255 0 RD 192.168.120.24 Vlanif120
+172.31.13.128/26 IBGP 255 0 RD 192.168.120.25 Vlanif120
+172.31.14.0/26 IBGP 255 0 RD 192.168.120.26 Vlanif120
+……
+255.255.255.255/32 Direct 0 0 D 127.0.0.1 InLoopBack0
+[Router2]
+
+ + + +As can be seen from the routing table on Router2, there are 140 service IP 32-bit host routes.On the original basis, 21 routes have been added. + + + +
Calculation formulaDescribe
7x20=140Seven k8s services correspond to seven cluster ips, each cluster IP has 20 routes for the next hop.
+ + +Similarly, we look at the kubernetes service routing on Router3, and the results are the same as those on Router1 and Router2. + + + + + + + +**2.3 Questions after adding kubernetes service nodes:**
+ + +Let's add three more kubernetes services to the k8s cluster to see what's wrong. +![avatar](../docs/img/large-networks07.jpg) + +Using "kubectl get service", you can see that the white font part is the newly added test-1 (172.30.99.97), test-1 (172.30.99.97), test-1 (172.30.99.97), and TEST-1 (172.99.97). The three kubernetes services are also "CLUSTER-IP". +
+[root@k8s-master ~]# kubectl get service --all-namespaces
+NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
+ci jenkins ClusterIP 172.30.2.11 80/TCP,50000/TCP 34d
+db data-comp ClusterIP 172.30.5.27 80/TCP 43d
+db redis ClusterIP 172.30.47.53 6379/TCP 145d
+coredns corednsdev ClusterIP 172.30.0.2 53/UDP,53/TCP,9153/TCP 93d
+default guestbook-ui ClusterIP 172.30.12.213 80/TCP 145d
+default kubernetes ClusterIP 172.30.0.1 443/TCP 145d
+default nginx01 ClusterIP 172.30.28.176 80/TCP 143d
+ +default test-1 ClusterIP 172.30.99.97 80/TCP 1h
+default test-2 ClusterIP 172.30.99.98 80/TCP 1h
+default test-3 ClusterIP 172.30.99.99 80/TCP 1h
+ + [root@k8s-master ~]#
+
+ + + + + +Next,Look at Router1's routing table again: + +【Router1】 +
+[Router1] display ip route
+Proto: Protocol Pre: Preference
+Route Flags: R - relay, D - download to fib, T - to vpn-instance, B - black hole route
+------------------------------------------------------------------------------
+Routing Table : _public_
+ +Destination/Mask Proto Pre Cost Flags NextHop Interface
+……
+172.30.0.1/32 IBGP 255 0 RD 192.168.110.50 Vlanif110
+                IBGP 255 0 RD 192.168.110.51 Vlanif110
+                IBGP 255 0 RD 192.168.110.52 Vlanif110
+                IBGP 255 0 RD 192.168.110.53 Vlanif110
+                IBGP 255 0 RD 192.168.110.54 Vlanif110
+                IBGP 255 0 RD 192.168.110.55 Vlanif110
+                IBGP 255 0 RD 192.168.110.56 Vlanif110
+                IBGP 255 0 RD 192.168.110.57 Vlanif110
+                IBGP 255 0 RD 192.168.110.58 Vlanif110
+                IBGP 255 0 RD 192.168.110.59 Vlanif110
+                IBGP 255 0 RD 192.168.110.60 Vlanif110
+                IBGP 255 0 RD 192.168.110.81 Vlanif110
+                IBGP 255 0 RD 192.168.110.82 Vlanif110
+                IBGP 255 0 RD 192.168.110.83 Vlanif110
+172.30.0.2/32 IBGP 255 0 RD 192.168.110.50 Vlanif110
+                IBGP 255 0 RD 192.168.110.51 Vlanif110
+                IBGP 255 0 RD 192.168.110.52 Vlanif110
+                IBGP 255 0 RD 192.168.110.53 Vlanif110
+                IBGP 255 0 RD 192.168.110.54 Vlanif110
+                IBGP 255 0 RD 192.168.110.55 Vlanif110
+                IBGP 255 0 RD 192.168.110.56 Vlanif110
+                IBGP 255 0 RD 192.168.110.57 Vlanif110
+                IBGP 255 0 RD 192.168.110.58 Vlanif110
+                IBGP 255 0 RD 192.168.110.59 Vlanif110
+                IBGP 255 0 RD 192.168.110.60 Vlanif110
+                IBGP 255 0 RD 192.168.110.81 Vlanif110
+                IBGP 255 0 RD 192.168.110.82 Vlanif110
+                IBGP 255 0 RD 192.168.110.83 Vlanif110
+172.30.0.9/32 IBGP 255 0 RD 192.168.110.50 Vlanif110
+                IBGP 255 0 RD 192.168.110.51 Vlanif110
+                IBGP 255 0 RD 192.168.110.52 Vlanif110
+                IBGP 255 0 RD 192.168.110.53 Vlanif110
+                IBGP 255 0 RD 192.168.110.54 Vlanif110
+                IBGP 255 0 RD 192.168.110.55 Vlanif110
+                IBGP 255 0 RD 192.168.110.56 Vlanif110
+                IBGP 255 0 RD 192.168.110.57 Vlanif110
+                IBGP 255 0 RD 192.168.110.58 Vlanif110
+                IBGP 255 0 RD 192.168.110.59 Vlanif110
+                IBGP 255 0 RD 192.168.110.60 Vlanif110
+                IBGP 255 0 RD 192.168.110.81 Vlanif110
+                IBGP 255 0 RD 192.168.110.82 Vlanif110
+                IBGP 255 0 RD 192.168.110.83 Vlanif110
+172.30.5.27/32 IBGP 255 0 RD 192.168.110.50 Vlanif110
+                IBGP 255 0 RD 192.168.110.51 Vlanif110
+                IBGP 255 0 RD 192.168.110.52 Vlanif110
+                IBGP 255 0 RD 192.168.110.53 Vlanif110
+                IBGP 255 0 RD 192.168.110.54 Vlanif110
+                IBGP 255 0 RD 192.168.110.55 Vlanif110
+                IBGP 255 0 RD 192.168.110.56 Vlanif110
+                IBGP 255 0 RD 192.168.110.57 Vlanif110
+                IBGP 255 0 RD 192.168.110.58 Vlanif110
+                IBGP 255 0 RD 192.168.110.59 Vlanif110
+                IBGP 255 0 RD 192.168.110.60 Vlanif110
+                IBGP 255 0 RD 192.168.110.81 Vlanif110
+                IBGP 255 0 RD 192.168.110.82 Vlanif110
+                IBGP 255 0 RD 192.168.110.83 Vlanif110
+172.30.12.213/32 IBGP 255 0 RD 192.168.110.50 Vlanif110
+                IBGP 255 0 RD 192.168.110.51 Vlanif110
+                IBGP 255 0 RD 192.168.110.52 Vlanif110
+                IBGP 255 0 RD 192.168.110.53 Vlanif110
+                IBGP 255 0 RD 192.168.110.54 Vlanif110
+                IBGP 255 0 RD 192.168.110.55 Vlanif110
+                IBGP 255 0 RD 192.168.110.56 Vlanif110
+                IBGP 255 0 RD 192.168.110.57 Vlanif110
+                IBGP 255 0 RD 192.168.110.58 Vlanif110
+                IBGP 255 0 RD 192.168.110.59 Vlanif110
+                IBGP 255 0 RD 192.168.110.60 Vlanif110
+                IBGP 255 0 RD 192.168.110.81 Vlanif110
+                IBGP 255 0 RD 192.168.110.82 Vlanif110
+                IBGP 255 0 RD 192.168.110.83 Vlanif110
+172.30.28.176/32 IBGP 255 0 RD 192.168.110.50 Vlanif110
+                IBGP 255 0 RD 192.168.110.51 Vlanif110
+                IBGP 255 0 RD 192.168.110.52 Vlanif110
+                IBGP 255 0 RD 192.168.110.53 Vlanif110
+                IBGP 255 0 RD 192.168.110.54 Vlanif110
+                IBGP 255 0 RD 192.168.110.55 Vlanif110
+                IBGP 255 0 RD 192.168.110.56 Vlanif110
+                IBGP 255 0 RD 192.168.110.57 Vlanif110
+                IBGP 255 0 RD 192.168.110.58 Vlanif110
+                IBGP 255 0 RD 192.168.110.59 Vlanif110
+                IBGP 255 0 RD 192.168.110.60 Vlanif110
+                IBGP 255 0 RD 192.168.110.81 Vlanif110
+                IBGP 255 0 RD 192.168.110.82 Vlanif110
+                IBGP 255 0 RD 192.168.110.83 Vlanif110
+172.30.47.53/32 IBGP 255 0 RD 192.168.110.50 Vlanif110
+                IBGP 255 0 RD 192.168.110.51 Vlanif110
+                IBGP 255 0 RD 192.168.110.52 Vlanif110
+                IBGP 255 0 RD 192.168.110.53 Vlanif110
+                IBGP 255 0 RD 192.168.110.54 Vlanif110
+                IBGP 255 0 RD 192.168.110.55 Vlanif110
+                IBGP 255 0 RD 192.168.110.56 Vlanif110
+                IBGP 255 0 RD 192.168.110.57 Vlanif110
+                IBGP 255 0 RD 192.168.110.58 Vlanif110
+                IBGP 255 0 RD 192.168.110.59 Vlanif110
+                IBGP 255 0 RD 192.168.110.60 Vlanif110
+                IBGP 255 0 RD 192.168.110.81 Vlanif110
+                IBGP 255 0 RD 192.168.110.82 Vlanif110
+                IBGP 255 0 RD 192.168.110.83 Vlanif110
+ +172.30.99.97/32 IBGP 255 0 RD 192.168.110.50 Vlanif110
+                IBGP 255 0 RD 192.168.110.51 Vlanif110
+                IBGP 255 0 RD 192.168.110.52 Vlanif110
+                IBGP 255 0 RD 192.168.110.53 Vlanif110
+                IBGP 255 0 RD 192.168.110.54 Vlanif110
+                IBGP 255 0 RD 192.168.110.55 Vlanif110
+                IBGP 255 0 RD 192.168.110.56 Vlanif110
+                IBGP 255 0 RD 192.168.110.57 Vlanif110
+                IBGP 255 0 RD 192.168.110.58 Vlanif110
+                IBGP 255 0 RD 192.168.110.59 Vlanif110
+                IBGP 255 0 RD 192.168.110.60 Vlanif110
+                IBGP 255 0 RD 192.168.110.81 Vlanif110
+                IBGP 255 0 RD 192.168.110.82 Vlanif110
+                IBGP 255 0 RD 192.168.110.83 Vlanif110
+172.30.99.98/32 IBGP 255 0 RD 192.168.110.50 Vlanif110
+                IBGP 255 0 RD 192.168.110.51 Vlanif110
+                IBGP 255 0 RD 192.168.110.52 Vlanif110
+                IBGP 255 0 RD 192.168.110.53 Vlanif110
+                IBGP 255 0 RD 192.168.110.54 Vlanif110
+                IBGP 255 0 RD 192.168.110.55 Vlanif110
+                IBGP 255 0 RD 192.168.110.56 Vlanif110
+                IBGP 255 0 RD 192.168.110.57 Vlanif110
+                IBGP 255 0 RD 192.168.110.58 Vlanif110
+                IBGP 255 0 RD 192.168.110.59 Vlanif110
+                IBGP 255 0 RD 192.168.110.60 Vlanif110
+                IBGP 255 0 RD 192.168.110.81 Vlanif110
+                IBGP 255 0 RD 192.168.110.82 Vlanif110
+                IBGP 255 0 RD 192.168.110.83 Vlanif110
+172.30.99.99/32 IBGP 255 0 RD 192.168.110.50 Vlanif110
+                IBGP 255 0 RD 192.168.110.51 Vlanif110
+                IBGP 255 0 RD 192.168.110.52 Vlanif110
+                IBGP 255 0 RD 192.168.110.53 Vlanif110
+                IBGP 255 0 RD 192.168.110.54 Vlanif110
+                IBGP 255 0 RD 192.168.110.55 Vlanif110
+                IBGP 255 0 RD 192.168.110.56 Vlanif110
+                IBGP 255 0 RD 192.168.110.57 Vlanif110
+                IBGP 255 0 RD 192.168.110.58 Vlanif110
+                IBGP 255 0 RD 192.168.110.59 Vlanif110
+                IBGP 255 0 RD 192.168.110.60 Vlanif110
+                IBGP 255 0 RD 192.168.110.81 Vlanif110
+                IBGP 255 0 RD 192.168.110.82 Vlanif110
+                IBGP 255 0 RD 192.168.110.83 Vlanif110
+ +172.31.0.64/26 IBGP 255 0 RD 192.168.110.50 Vlanif110
+172.31.3.128/26 IBGP 255 0 RD 192.168.110.51 Vlanif110
+172.31.5.192/26 IBGP 255 0 RD 192.168.110.52 Vlanif110
+172.31.6.0/26 IBGP 255 0 RD 192.168.110.53 Vlanif110
+172.31.6.64/26 IBGP 255 0 RD 192.168.110.54 Vlanif110
+172.31.7.0/26 IBGP 255 0 RD 192.168.110.55 Vlanif110
+172.31.7.192/26 IBGP 255 0 RD 192.168.110.56 Vlanif110
+172.31.8.64/26 IBGP 255 0 RD 192.168.110.57 Vlanif110
+172.31.9.64/26 IBGP 255 0 RD 192.168.110.58 Vlanif110
+172.31.9.128/26 IBGP 255 0 RD 192.168.110.59 Vlanif110
+172.31.16.64/26 IBGP 255 0 RD 192.168.110.60 Vlanif110
+72.31.17.64/26 IBGP 255 0 RD 192.168.110.81 Vlanif110
+172.31.17.128/26 IBGP 255 0 RD 192.168.110.82 Vlanif110
+172.31.17.64/26 IBGP 255 0 RD 192.168.110.83 Vlanif110
+……
+255.255.255.255/32 Direct 0 0 D 127.0.0.1 InLoopBack0
+[Router1]
+
+ + +As can be seen from the routing table on Router1, there are 140 service IP 32-bit host routes.On the original basis, 42 routes have been added. + + + +
Calculation formulaDescribe
10x14=14010 k8s services correspond to seven cluster ips, each cluster IP has 14 next hop routes.
+ + + +Next,Look at Router2's routing table again: + +

+[Router2] display ip route
+Proto: Protocol Pre: Preference
+Route Flags: R - relay, D - download to fib, T - to vpn-instance, B - black hole route
+------------------------------------------------------------------------------
+Routing Table : _public_
+ +Destination/Mask Proto Pre Cost Flags NextHop Interface
+……
+172.30.0.1/32 IBGP 255 0 RD 192.168.120.10 Vlanif120
+                IBGP 255 0 RD 192.168.120.11 Vlanif120
+                IBGP 255 0 RD 192.168.120.12 Vlanif120
+                IBGP 255 0 RD 192.168.120.13 Vlanif120
+                IBGP 255 0 RD 192.168.120.14 Vlanif120
+                IBGP 255 0 RD 192.168.120.15 Vlanif120
+                IBGP 255 0 RD 192.168.120.16 Vlanif120
+                IBGP 255 0 RD 192.168.120.17 Vlanif120
+                IBGP 255 0 RD 192.168.120.18 Vlanif120
+                IBGP 255 0 RD 192.168.120.19 Vlanif120
+                IBGP 255 0 RD 192.168.120.20 Vlanif120
+                IBGP 255 0 RD 192.168.120.21 Vlanif120
+                IBGP 255 0 RD 192.168.120.22 Vlanif120
+                IBGP 255 0 RD 192.168.120.23 Vlanif120
+                IBGP 255 0 RD 192.168.120.24 Vlanif120
+                IBGP 255 0 RD 192.168.120.25 Vlanif120
+                IBGP 255 0 RD 192.168.120.26 Vlanif120
+                IBGP 255 0 RD 192.168.120.81 Vlanif120
+                IBGP 255 0 RD 192.168.120.82 Vlanif120
+                IBGP 255 0 RD 192.168.120.83 Vlanif120
+172.30.0.2/32 IBGP 255 0 RD 192.168.120.10 Vlanif120
+                IBGP 255 0 RD 192.168.120.11 Vlanif120
+                IBGP 255 0 RD 192.168.120.12 Vlanif120
+                IBGP 255 0 RD 192.168.120.13 Vlanif120
+                IBGP 255 0 RD 192.168.120.14 Vlanif120
+                IBGP 255 0 RD 192.168.120.15 Vlanif120
+                IBGP 255 0 RD 192.168.120.16 Vlanif120
+                IBGP 255 0 RD 192.168.120.17 Vlanif120
+                IBGP 255 0 RD 192.168.120.18 Vlanif120
+                IBGP 255 0 RD 192.168.120.19 Vlanif120
+                IBGP 255 0 RD 192.168.120.20 Vlanif120
+                IBGP 255 0 RD 192.168.120.21 Vlanif120
+                IBGP 255 0 RD 192.168.120.22 Vlanif120
+                IBGP 255 0 RD 192.168.120.23 Vlanif120
+                IBGP 255 0 RD 192.168.120.24 Vlanif120
+                IBGP 255 0 RD 192.168.120.25 Vlanif120
+                IBGP 255 0 RD 192.168.120.26 Vlanif120
+                IBGP 255 0 RD 192.168.120.81 Vlanif120
+                IBGP 255 0 RD 192.168.120.82 Vlanif120
+                IBGP 255 0 RD 192.168.120.83 Vlanif120
+172.30.0.9/32 IBGP 255 0 RD 192.168.120.10 Vlanif120
+                IBGP 255 0 RD 192.168.120.11 Vlanif120
+                IBGP 255 0 RD 192.168.120.12 Vlanif120
+                IBGP 255 0 RD 192.168.120.13 Vlanif120
+                IBGP 255 0 RD 192.168.120.14 Vlanif120
+                IBGP 255 0 RD 192.168.120.15 Vlanif120
+                IBGP 255 0 RD 192.168.120.16 Vlanif120
+                IBGP 255 0 RD 192.168.120.17 Vlanif120
+                IBGP 255 0 RD 192.168.120.18 Vlanif120
+                IBGP 255 0 RD 192.168.120.19 Vlanif120
+                IBGP 255 0 RD 192.168.120.20 Vlanif120
+                IBGP 255 0 RD 192.168.120.21 Vlanif120
+                IBGP 255 0 RD 192.168.120.22 Vlanif120
+                IBGP 255 0 RD 192.168.120.23 Vlanif120
+                IBGP 255 0 RD 192.168.120.24 Vlanif120
+                IBGP 255 0 RD 192.168.120.25 Vlanif120
+                IBGP 255 0 RD 192.168.120.26 Vlanif120
+                IBGP 255 0 RD 192.168.120.81 Vlanif120
+                IBGP 255 0 RD 192.168.120.82 Vlanif120
+                IBGP 255 0 RD 192.168.120.83 Vlanif120
+172.30.5.27/32 IBGP 255 0 RD 192.168.120.10 Vlanif120
+                IBGP 255 0 RD 192.168.120.11 Vlanif120
+                IBGP 255 0 RD 192.168.120.12 Vlanif120
+                IBGP 255 0 RD 192.168.120.13 Vlanif120
+                IBGP 255 0 RD 192.168.120.14 Vlanif120
+                IBGP 255 0 RD 192.168.120.15 Vlanif120
+                IBGP 255 0 RD 192.168.120.16 Vlanif120
+                IBGP 255 0 RD 192.168.120.17 Vlanif120
+                IBGP 255 0 RD 192.168.120.18 Vlanif120
+                IBGP 255 0 RD 192.168.120.19 Vlanif120
+                IBGP 255 0 RD 192.168.120.20 Vlanif120
+                IBGP 255 0 RD 192.168.120.21 Vlanif120
+                IBGP 255 0 RD 192.168.120.22 Vlanif120
+                IBGP 255 0 RD 192.168.120.23 Vlanif120
+                IBGP 255 0 RD 192.168.120.24 Vlanif120
+                IBGP 255 0 RD 192.168.120.25 Vlanif120
+                IBGP 255 0 RD 192.168.120.26 Vlanif120
+                IBGP 255 0 RD 192.168.120.81 Vlanif120
+                IBGP 255 0 RD 192.168.120.82 Vlanif120
+                IBGP 255 0 RD 192.168.120.83 Vlanif120
+172.30.12.213/32 IBGP 255 0 RD 192.168.120.10 Vlanif120
+                IBGP 255 0 RD 192.168.120.11 Vlanif120
+                IBGP 255 0 RD 192.168.120.12 Vlanif120
+                IBGP 255 0 RD 192.168.120.13 Vlanif120
+                IBGP 255 0 RD 192.168.120.14 Vlanif120
+                IBGP 255 0 RD 192.168.120.15 Vlanif120
+                IBGP 255 0 RD 192.168.120.16 Vlanif120
+                IBGP 255 0 RD 192.168.120.17 Vlanif120
+                IBGP 255 0 RD 192.168.120.18 Vlanif120
+                IBGP 255 0 RD 192.168.120.19 Vlanif120
+                IBGP 255 0 RD 192.168.120.20 Vlanif120
+                IBGP 255 0 RD 192.168.120.21 Vlanif120
+                IBGP 255 0 RD 192.168.120.22 Vlanif120
+                IBGP 255 0 RD 192.168.120.23 Vlanif120
+                IBGP 255 0 RD 192.168.120.24 Vlanif120
+                IBGP 255 0 RD 192.168.120.25 Vlanif120
+                IBGP 255 0 RD 192.168.120.26 Vlanif120
+                IBGP 255 0 RD 192.168.120.81 Vlanif120
+                IBGP 255 0 RD 192.168.120.82 Vlanif120
+                IBGP 255 0 RD 192.168.120.83 Vlanif120
+172.30.28.176/32 IBGP 255 0 RD 192.168.120.10 Vlanif120
+                IBGP 255 0 RD 192.168.120.11 Vlanif120
+                IBGP 255 0 RD 192.168.120.12 Vlanif120
+                IBGP 255 0 RD 192.168.120.13 Vlanif120
+                IBGP 255 0 RD 192.168.120.14 Vlanif120
+                IBGP 255 0 RD 192.168.120.15 Vlanif120
+                IBGP 255 0 RD 192.168.120.16 Vlanif120
+                IBGP 255 0 RD 192.168.120.17 Vlanif120
+                IBGP 255 0 RD 192.168.120.18 Vlanif120
+                IBGP 255 0 RD 192.168.120.19 Vlanif120
+                IBGP 255 0 RD 192.168.120.20 Vlanif120
+                IBGP 255 0 RD 192.168.120.21 Vlanif120
+                IBGP 255 0 RD 192.168.120.22 Vlanif120
+                IBGP 255 0 RD 192.168.120.23 Vlanif120
+                IBGP 255 0 RD 192.168.120.24 Vlanif120
+                IBGP 255 0 RD 192.168.120.25 Vlanif120
+                IBGP 255 0 RD 192.168.120.26 Vlanif120
+                IBGP 255 0 RD 192.168.120.81 Vlanif120
+                IBGP 255 0 RD 192.168.120.82 Vlanif120
+                IBGP 255 0 RD 192.168.120.83 Vlanif120
+172.30.47.53/32 IBGP 255 0 RD 192.168.120.10 Vlanif120
+                IBGP 255 0 RD 192.168.120.11 Vlanif120
+                IBGP 255 0 RD 192.168.120.12 Vlanif120
+                IBGP 255 0 RD 192.168.120.13 Vlanif120
+                IBGP 255 0 RD 192.168.120.14 Vlanif120
+                IBGP 255 0 RD 192.168.120.15 Vlanif120
+                IBGP 255 0 RD 192.168.120.16 Vlanif120
+                IBGP 255 0 RD 192.168.120.17 Vlanif120
+                IBGP 255 0 RD 192.168.120.18 Vlanif120
+                IBGP 255 0 RD 192.168.120.19 Vlanif120
+                IBGP 255 0 RD 192.168.120.20 Vlanif120
+                IBGP 255 0 RD 192.168.120.21 Vlanif120
+                IBGP 255 0 RD 192.168.120.22 Vlanif120
+                IBGP 255 0 RD 192.168.120.23 Vlanif120
+                IBGP 255 0 RD 192.168.120.24 Vlanif120
+                IBGP 255 0 RD 192.168.120.25 Vlanif120
+                IBGP 255 0 RD 192.168.120.26 Vlanif120
+                IBGP 255 0 RD 192.168.120.81 Vlanif120
+                IBGP 255 0 RD 192.168.120.82 Vlanif120
+                IBGP 255 0 RD 192.168.120.83 Vlanif120
+ +172.30.99.97/32 IBGP 255 0 RD 192.168.120.10 Vlanif120
+                IBGP 255 0 RD 192.168.120.11 Vlanif120
+                IBGP 255 0 RD 192.168.120.12 Vlanif120
+                IBGP 255 0 RD 192.168.120.13 Vlanif120
+                IBGP 255 0 RD 192.168.120.14 Vlanif120
+                IBGP 255 0 RD 192.168.120.15 Vlanif120
+                IBGP 255 0 RD 192.168.120.16 Vlanif120
+                IBGP 255 0 RD 192.168.120.17 Vlanif120
+                IBGP 255 0 RD 192.168.120.18 Vlanif120
+                IBGP 255 0 RD 192.168.120.19 Vlanif120
+                IBGP 255 0 RD 192.168.120.20 Vlanif120
+                IBGP 255 0 RD 192.168.120.21 Vlanif120
+                IBGP 255 0 RD 192.168.120.22 Vlanif120
+                IBGP 255 0 RD 192.168.120.23 Vlanif120
+                IBGP 255 0 RD 192.168.120.24 Vlanif120
+                IBGP 255 0 RD 192.168.120.25 Vlanif120
+                IBGP 255 0 RD 192.168.120.26 Vlanif120
+                IBGP 255 0 RD 192.168.120.81 Vlanif120
+                IBGP 255 0 RD 192.168.120.82 Vlanif120
+                IBGP 255 0 RD 192.168.120.83 Vlanif120
+172.30.99.98/32 IBGP 255 0 RD 192.168.120.10 Vlanif120
+                IBGP 255 0 RD 192.168.120.11 Vlanif120
+                IBGP 255 0 RD 192.168.120.12 Vlanif120
+                IBGP 255 0 RD 192.168.120.13 Vlanif120
+                IBGP 255 0 RD 192.168.120.14 Vlanif120
+                IBGP 255 0 RD 192.168.120.15 Vlanif120
+                IBGP 255 0 RD 192.168.120.16 Vlanif120
+                IBGP 255 0 RD 192.168.120.17 Vlanif120
+                IBGP 255 0 RD 192.168.120.18 Vlanif120
+                IBGP 255 0 RD 192.168.120.19 Vlanif120
+                IBGP 255 0 RD 192.168.120.20 Vlanif120
+                IBGP 255 0 RD 192.168.120.21 Vlanif120
+                IBGP 255 0 RD 192.168.120.22 Vlanif120
+                IBGP 255 0 RD 192.168.120.23 Vlanif120
+                IBGP 255 0 RD 192.168.120.24 Vlanif120
+                IBGP 255 0 RD 192.168.120.25 Vlanif120
+                IBGP 255 0 RD 192.168.120.26 Vlanif120
+                IBGP 255 0 RD 192.168.120.81 Vlanif120
+                IBGP 255 0 RD 192.168.120.82 Vlanif120
+                IBGP 255 0 RD 192.168.120.83 Vlanif120
+172.30.99.99/32 IBGP 255 0 RD 192.168.120.10 Vlanif120
+                IBGP 255 0 RD 192.168.120.11 Vlanif120
+                IBGP 255 0 RD 192.168.120.12 Vlanif120
+                IBGP 255 0 RD 192.168.120.13 Vlanif120
+                IBGP 255 0 RD 192.168.120.14 Vlanif120
+                IBGP 255 0 RD 192.168.120.15 Vlanif120
+                IBGP 255 0 RD 192.168.120.16 Vlanif120
+                IBGP 255 0 RD 192.168.120.17 Vlanif120
+                IBGP 255 0 RD 192.168.120.18 Vlanif120
+                IBGP 255 0 RD 192.168.120.19 Vlanif120
+                IBGP 255 0 RD 192.168.120.20 Vlanif120
+                IBGP 255 0 RD 192.168.120.21 Vlanif120
+                IBGP 255 0 RD 192.168.120.22 Vlanif120
+                IBGP 255 0 RD 192.168.120.23 Vlanif120
+                IBGP 255 0 RD 192.168.120.24 Vlanif120
+                IBGP 255 0 RD 192.168.120.25 Vlanif120
+                IBGP 255 0 RD 192.168.120.26 Vlanif120
+                IBGP 255 0 RD 192.168.120.81 Vlanif120
+                IBGP 255 0 RD 192.168.120.82 Vlanif120
+                IBGP 255 0 RD 192.168.120.83 Vlanif120
+ +172.31.0.0/26 IBGP 255 0 RD 192.168.120.10 Vlanif120
+172.31.2.64/26 IBGP 255 0 RD 192.168.120.11 Vlanif120
+172.31.3.0/26 IBGP 255 0 RD 192.168.120.12 Vlanif120
+172.31.4.192/26 IBGP 255 0 RD 192.168.120.13 Vlanif120
+172.31.5.64/26 IBGP 255 0 RD 192.168.120.14 Vlanif120
+172.31.6.128/26 IBGP 255 0 RD 192.168.120.15 Vlanif120
+172.31.6.192/26 IBGP 255 0 RD 192.168.120.16 Vlanif120
+172.31.7.64/26 IBGP 255 0 RD 192.168.120.17 Vlanif120
+172.31.7.128/26 IBGP 255 0 RD 192.168.120.18 Vlanif120
+172.31.9.0/26 IBGP 255 0 RD 192.168.120.19 Vlanif120
+172.31.10.0/26 IBGP 255 0 RD 192.168.120.20 Vlanif120
+172.31.10.64/26 IBGP 255 0 RD 192.168.120.21 Vlanif120
+172.31.11.0/26 IBGP 255 0 RD 192.168.120.22 Vlanif120
+172.31.11.192/26 IBGP 255 0 RD 192.168.120.23 Vlanif120
+172.31.12.64/26 IBGP 255 0 RD 192.168.120.24 Vlanif120
+172.31.13.128/26 IBGP 255 0 RD 192.168.120.25 Vlanif120
+172.31.14.0/26 IBGP 255 0 RD 192.168.120.26 Vlanif120
+……
+255.255.255.255/32 Direct 0 0 D 127.0.0.1 InLoopBack0
+[Router2]
+
+ + +As can be seen from the routing table on Router1, there are 200 service IP 32-bit host routes.On the original basis, 60 routes have been added. + + + +
Calculation formulaDescribe
10x20=20010 k8s services correspond to seven cluster ips, each cluster IP has 20 next hop routes.
+ + + +Similarly, we look at the kubernetes service routing on Router3 and Router4, and the results are the same as those on Router1 and Router2. + + + + + + + +**2.4 Summary**
+ + +At first, there were only seven services in the kubernetes cluster:
+Router 1 has only 11 k8s-nodes and Router 1 has 7 X11 = 77 service 32-bit host routes.
+There are only 11 k8s-nodes on Router 2, and 7 x17 = 119 service 32-bit host routes on Router 1.
+ + +By adding nodes and services, the kubernetes cluster grew to 10 services:
+Router 1 has 14 k8s-nodes, Router 1 has 12-bit host routes with 10 x14 = 140 services, and 63 routes have been added.
+Router 2 is increased to 20 k8s-nodes, Router 1 has 12-bit host routes with 10 x 20 = 200 services, and 81 routes are added.
+ +By analogy, if our kubernetes cluster grows to 1000 services and k8s-nodes on each router grow to 40, there will be 400,000 32-bit host routes on each router.
+ +Although in the actual production environment, because of the number of interfaces of router devices, each router will not access too many kubernetes node servers, but with the growth of service, the number of router entries will multiply. The end result is that the load pressure on the router becomes enormous until it goes down.
+ + + + +
+
+**3 Adjusting "advertise-cluster-subnet" parameters to optimize network routing**
+ +**3.1 How to set "advertise-cluster-subnet" parameter correctly**
+ +You need to set both "--advertise-cluster-IP=true" and "--advertise-cluster-subnet=subnet" parameters.Let k8s node only notify k8s service aggregate routes to the upstream routers, reducing the service routing entries of the upstream routers.Please continue reading the following. + + + +**3.2 Configure kube-router run parameters**
+ +
+You can set the following parameters in the white font section: +
+ - name: kube-router
+ image: cloudnativelabs/kube-router:0.3.2
+ imagePullPolicy: IfNotPresent
+ args:
+ - --run-router=true
+ - --run-firewall=true
+ - --run-service-proxy=true
+ - --enable-overlay=false
+ - --advertise-pod-cidr=true
+ - --advertise-cluster-ip=true
+ - --advertise-cluster-subnet=172.30.0.0/16
+ - --bgp-graceful-restart=true
+ - --enable-ibgp=false
+ - --nodes-full-mesh=true
+ - --cluster-asn=64558
+ - --peer-router-ips=192.168.110.1
+ - --peer-router-asns=64558
+ ......
+
+Note: Look at the two parameters in the white font section above - - advertise - cluster - IP = true and - - advertise - cluster - subnet = subnet. +
+ +Update kube-router daemonset using kubectl apply: + +
+[root@k8s-master kube-router]# kubectl apply -f kube-router-daemonset-new.yaml
+daemonset.extensions/kube-system-kube-router-new created
+[root@k8s-master kube-router]#
+
+ +Use "kubectl get pod" to see that kube-router pod is working properly: +
+[root@k8s-master kube-router]# kubectl get pod -o wide -n kube-system
+NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
+kube-system-kube-router-new-7f2qm 1/1 Running 0 73s 192.168.120.10 192.168.120.10
+kube-system-kube-router-new-9tz2b 1/1 Running 0 73s 192.168.120.11 192.168.120.11
+kube-system-kube-router-new-7f2qq 1/1 Running 0 73s 192.168.120.12 192.168.120.12
+kube-system-kube-router-new-9tz2w 1/1 Running 0 73s 192.168.120.13 192.168.120.13
+kube-system-kube-router-new-7f2qe 1/1 Running 0 73s 192.168.120.14 192.168.120.14
+kube-system-kube-router-new-9tz2r 1/1 Running 0 73s 192.168.120.15 192.168.120.15
+kube-system-kube-router-new-7f2qt 1/1 Running 0 73s 192.168.120.16 192.168.120.16
+kube-system-kube-router-new-9tz2y 1/1 Running 0 73s 192.168.120.17 192.168.120.17
+kube-system-kube-router-new-7f2qu 1/1 Running 0 73s 192.168.120.18 192.168.120.18
+kube-system-kube-router-new-9tz2i 1/1 Running 0 73s 192.168.120.19 192.168.120.19
+kube-system-kube-router-new-7f2qo 1/1 Running 0 73s 192.168.120.20 192.168.120.20
+kube-system-kube-router-new-9tz2p 1/1 Running 0 73s 192.168.120.21 192.168.120.21
+kube-system-kube-router-new-7f2qa 1/1 Running 0 73s 192.168.120.22 192.168.120.22
+kube-system-kube-router-new-9tz2s 1/1 Running 0 73s 192.168.120.23 192.168.120.23
+kube-system-kube-router-new-7f2qd 1/1 Running 0 73s 192.168.120.24 192.168.120.24
+kube-system-kube-router-new-9tz2f 1/1 Running 0 73s 192.168.120.25 192.168.120.25
+kube-system-kube-router-new-7f2qg 1/1 Running 0 73s 192.168.120.26 192.168.120.26
+kube-system-kube-router-new-qf2q1 1/1 Running 0 73s 192.168.120.81 192.168.120.81
+kube-system-kube-router-new-wtz22 1/1 Running 0 73s 192.168.120.82 192.168.120.82
+kube-system-kube-router-new-ef2qe 1/1 Running 0 73s 192.168.120.83 192.168.120.83
+kube-system-kube-router-new-9tz2h 1/1 Running 0 73s 192.168.110.50 192.168.110.50
+kube-system-kube-router-new-9tz2j 1/1 Running 0 73s 192.168.110.51 192.168.110.51
+kube-system-kube-router-new-9tz2k 1/1 Running 0 73s 192.168.110.52 192.168.110.52
+kube-system-kube-router-new-9tz2l 1/1 Running 0 73s 192.168.110.53 192.168.110.53
+kube-system-kube-router-new-9tz2z 1/1 Running 0 73s 192.168.110.54 192.168.110.54
+kube-system-kube-router-new-9tzxx 1/1 Running 0 73s 192.168.110.55 192.168.110.55
+kube-system-kube-router-new-9tz2c 1/1 Running 0 73s 192.168.110.56 192.168.110.56
+kube-system-kube-router-new-9tz2v 1/1 Running 0 73s 192.168.110.57 192.168.110.57
+kube-system-kube-router-new-9tz2n 1/1 Running 0 73s 192.168.110.58 192.168.110.58
+kube-system-kube-router-new-9tzkk 1/1 Running 0 73s 192.168.110.59 192.168.110.59
+kube-system-kube-router-new-9tzsd 1/1 Running 0 73s 192.168.110.60 192.168.110.60
+kube-system-kube-router-new-rtz2g 1/1 Running 0 73s 192.168.110.81 192.168.110.81
+kube-system-kube-router-new-ttz2p 1/1 Running 0 73s 192.168.110.82 192.168.110.82
+kube-system-kube-router-new-ytz2o 1/1 Running 0 73s 192.168.110.83 192.168.110.83
+[root@k8s-master kube-router]#
+
+ + + + +**3.3 View the optimized results**
+ +Look at Router1's routing table again: +【Router1】 +
+[Router1] display ip route
+Proto: Protocol Pre: Preference
+Route Flags: R - relay, D - download to fib, T - to vpn-instance, B - black hole route
+------------------------------------------------------------------------------
+Routing Table : _public_
+ +Destination/Mask Proto Pre Cost Flags NextHop Interface
+……
+ +172.30.0.0/16 IBGP 255 0 RD 192.168.110.50 Vlanif110
+                IBGP 255 0 RD 192.168.110.51 Vlanif110
+                IBGP 255 0 RD 192.168.110.52 Vlanif110
+                IBGP 255 0 RD 192.168.110.53 Vlanif110
+                IBGP 255 0 RD 192.168.110.54 Vlanif110
+                IBGP 255 0 RD 192.168.110.55 Vlanif110
+                IBGP 255 0 RD 192.168.110.56 Vlanif110
+                IBGP 255 0 RD 192.168.110.57 Vlanif110
+                IBGP 255 0 RD 192.168.110.58 Vlanif110
+                IBGP 255 0 RD 192.168.110.59 Vlanif110
+                IBGP 255 0 RD 192.168.110.60 Vlanif110
+                IBGP 255 0 RD 192.168.110.81 Vlanif110
+                IBGP 255 0 RD 192.168.110.82 Vlanif110
+                IBGP 255 0 RD 192.168.110.83 Vlanif110
+ +172.31.0.64/26 IBGP 255 0 RD 192.168.110.50 Vlanif110
+172.31.3.128/26 IBGP 255 0 RD 192.168.110.51 Vlanif110
+172.31.5.192/26 IBGP 255 0 RD 192.168.110.52 Vlanif110
+172.31.6.0/26 IBGP 255 0 RD 192.168.110.53 Vlanif110
+172.31.6.64/26 IBGP 255 0 RD 192.168.110.54 Vlanif110
+172.31.7.0/26 IBGP 255 0 RD 192.168.110.55 Vlanif110
+172.31.7.192/26 IBGP 255 0 RD 192.168.110.56 Vlanif110
+172.31.8.64/26 IBGP 255 0 RD 192.168.110.57 Vlanif110
+172.31.9.64/26 IBGP 255 0 RD 192.168.110.58 Vlanif110
+172.31.9.128/26 IBGP 255 0 RD 192.168.110.59 Vlanif110
+172.31.16.64/26 IBGP 255 0 RD 192.168.110.60 Vlanif110
+72.31.17.64/26 IBGP 255 0 RD 192.168.110.81 Vlanif110
+172.31.17.128/26 IBGP 255 0 RD 192.168.110.82 Vlanif110
+172.31.17.64/26 IBGP 255 0 RD 192.168.110.83 Vlanif110
+……
+255.255.255.255/32 Direct 0 0 D 127.0.0.1 InLoopBack0
+[Router1]
+
+ + + +From the routing table on Router1, it can be seen that there are 14 subnetwork routes of service.Routing entries are 90% less than before. + + + +
Calculation formulaDescribe
1x14=14Ten K8S services are aggregated into a aggregate route of "172.30.0.0/16". +This aggregate route balances the load of the ECMP function to 14 next hop routes.
+ + + +Next,Look at Router2's routing table again: + +
+[Router2] display ip route
+Proto: Protocol Pre: Preference
+Route Flags: R - relay, D - download to fib, T - to vpn-instance, B - black hole route
+------------------------------------------------------------------------------
+Routing Table : _public_
+ +Destination/Mask Proto Pre Cost Flags NextHop Interface
+……
+ +172.30.0.0/16 IBGP 255 0 RD 192.168.120.10 Vlanif120
+                IBGP 255 0 RD 192.168.120.11 Vlanif120
+                IBGP 255 0 RD 192.168.120.12 Vlanif120
+                IBGP 255 0 RD 192.168.120.13 Vlanif120
+                IBGP 255 0 RD 192.168.120.14 Vlanif120
+                IBGP 255 0 RD 192.168.120.15 Vlanif120
+                IBGP 255 0 RD 192.168.120.16 Vlanif120
+                IBGP 255 0 RD 192.168.120.17 Vlanif120
+                IBGP 255 0 RD 192.168.120.18 Vlanif120
+                IBGP 255 0 RD 192.168.120.19 Vlanif120
+                IBGP 255 0 RD 192.168.120.20 Vlanif120
+                IBGP 255 0 RD 192.168.120.21 Vlanif120
+                IBGP 255 0 RD 192.168.120.22 Vlanif120
+                IBGP 255 0 RD 192.168.120.23 Vlanif120
+                IBGP 255 0 RD 192.168.120.24 Vlanif120
+                IBGP 255 0 RD 192.168.120.25 Vlanif120
+                IBGP 255 0 RD 192.168.120.26 Vlanif120
+                IBGP 255 0 RD 192.168.120.81 Vlanif120
+                IBGP 255 0 RD 192.168.120.82 Vlanif120
+                IBGP 255 0 RD 192.168.120.83 Vlanif120
+ +172.31.0.0/26 IBGP 255 0 RD 192.168.120.10 Vlanif120
+172.31.2.64/26 IBGP 255 0 RD 192.168.120.11 Vlanif120
+172.31.3.0/26 IBGP 255 0 RD 192.168.120.12 Vlanif120
+172.31.4.192/26 IBGP 255 0 RD 192.168.120.13 Vlanif120
+172.31.5.64/26 IBGP 255 0 RD 192.168.120.14 Vlanif120
+172.31.6.128/26 IBGP 255 0 RD 192.168.120.15 Vlanif120
+172.31.6.192/26 IBGP 255 0 RD 192.168.120.16 Vlanif120
+172.31.7.64/26 IBGP 255 0 RD 192.168.120.17 Vlanif120
+172.31.7.128/26 IBGP 255 0 RD 192.168.120.18 Vlanif120
+172.31.9.0/26 IBGP 255 0 RD 192.168.120.19 Vlanif120
+172.31.10.0/26 IBGP 255 0 RD 192.168.120.20 Vlanif120
+172.31.10.64/26 IBGP 255 0 RD 192.168.120.21 Vlanif120
+172.31.11.0/26 IBGP 255 0 RD 192.168.120.22 Vlanif120
+172.31.11.192/26 IBGP 255 0 RD 192.168.120.23 Vlanif120
+172.31.12.64/26 IBGP 255 0 RD 192.168.120.24 Vlanif120
+172.31.13.128/26 IBGP 255 0 RD 192.168.120.25 Vlanif120
+172.31.14.0/26 IBGP 255 0 RD 192.168.120.26 Vlanif120
+……
+255.255.255.255/32 Direct 0 0 D 127.0.0.1 InLoopBack0
+[Router2]
+
+ + +From the routing table on Router2, it can be seen that there are 10 subnetwork routes of service.Routing entries are 90% less than before. + + + +
Calculation formulaDescribe
1x20=20Ten K8S services are aggregated into a aggregate route of "172.30.0.0/16". +This aggregate route balances the load of the ECMP function to 20 next hop routes.
+ + + +Similarly, we look at the kubernetes service routing on Router3, and the results are the same as those on Router1 and Router2. + + + +**3.4 Summary**
+ + +Through the above optimization, we can know that the number of routes has been reduced by 90%. The comparison before and after optimization is as follows: + + + + +
ConditionBefore optimizationAfter optimization
There are 10 services in the kubernetes cluster, and 14 k8s-nodes are connected under Router1.Router1 has 140 service aggregate network routes.There are only 14 service aggregate network routes on Router1.
There are 10 services in the kubernetes cluster, and 20 k8s-nodes are connected under Router2.Router2 has 200 service aggregate network routes.There are only 20 service aggregate network routes on Router2.
+ +After optimization, no matter how many services you add to your kubernetes cluster, the number of service routes on each router will only be equal to the number of kubernetes nodes it connects to. In the actual production environment, due to the limitation of the number of interfaces of router devices, the kubernetes node that can be connected on each router is very limited. Usually less than 100. Therefore, if it is a BGP network with a large kubernetes cluster, after the above optimization, the number of kubernetes service routing entries of Node-Connected router devices will be reduced from millions to tens of orders of magnitude. +
+In this way, your kubernetes cluster network can support a larger network concurrent access scenario. +
+
+
+**4 YAML source files used in this document**
+For the YAML source files used in this document, please click on this link to view them. +
+
+
+ + + + + + + diff --git a/docs/large-networks04.md b/docs/large-networks04.md new file mode 100644 index 0000000000..7ac4f3f86f --- /dev/null +++ b/docs/large-networks04.md @@ -0,0 +1,62 @@ +**How to optimize the neighborhood relationship and reduce the number of routes through kube-router in order to support the BGP network of large kubernetes cluster?** +
+
+
+ + + +**1 Contrast before and after enabling ECMP function of network equipment** +
+
+**1.1 Network traffic flow before ECMP is enabled**
+Before optimization, network traffic will choose a path to reach its destination. +
+The network topology is as follows: +![avatar](../docs/img/large-networks08.jpg) + + +**1.2 Network traffic sharing after ECMP enabled**
+After optimization, network traffic will be balanced to all network devices and nodes. +
+The network topology is as follows: +![avatar](../docs/img/large-networks09.jpg) + +
+ +**2 How to enable BGP ECMP for network devices?**
+ +Taking the router equipment of the mainstream network manufacturer as an example, this paper briefly introduces how the router equipment can turn on the equivalent routing load balancing of BGP ECMP. + +**2.1 Cisco Network Equipment Configuration BGP ECMP**
+可以使用下面的命令来开启EBGP ECMP： +
+[config-Router] config terminal
+[config-Router] bgp 64558
+[config-Router] maximum-paths 32
+
+ +可以使用下面的命令来开启IBGP ECMP： +
+[config-Router] config terminal
+[config-Router] bgp 64558
+[config-Router] maximum-paths ibgp 32
+
+ +**2.2 Huawei Network Equipment Configuration BGP ECMP**
+可以使用下面的命令来开启EBGP ECMP： +
+[Router] system view
+[Router] bgp 64558
+[Router-bgp] ipv4-family unicast
+[Router-bgp-af-ipv4] maximum load-balancing ebgp 32
+
+ +可以使用下面的命令来开启IBGP ECMP： +
+[Router] system view
+[Router] bgp 64558
+[Router-bgp] ipv4-family unicast
+[Router-bgp-af-ipv4] maximum load-balancing ibgp 32
+
+
+
diff --git a/pkg/controllers/routing/bgp_policies.go b/pkg/controllers/routing/bgp_policies.go index a900520c21..7773470b52 100644 --- a/pkg/controllers/routing/bgp_policies.go +++ b/pkg/controllers/routing/bgp_policies.go @@ -41,9 +41,17 @@ func (nrc *NetworkRoutingController) AddPolicies() error { // creates prefix set to represent all the advertisable IP associated with the services advIPPrefixList := make([]config.Prefix, 0) advIps, _, _ := nrc.getAllVIPs() - for _, ip := range advIps { - advIPPrefixList = append(advIPPrefixList, config.Prefix{IpPrefix: ip + "/32"}) - } + + //If the value of advertise-svc-cidr parameter is not empty, then the value of advertise-svc-cidr parameter is put into RIB, otherwise it will be done according to the original rules. + if len(nrc.advertiseClusterSubnet) != 0 { + advIPPrefixList = append(advIPPrefixList, config.Prefix{IpPrefix: nrc.advertiseClusterSubnet}) + } else { + for _, ip := range advIps { + //housj add + advIPPrefixList = append(advIPPrefixList, config.Prefix{IpPrefix: ip + "/32"}) + } + } + clusterIPPrefixSet, err := table.NewPrefixSet(config.PrefixSet{ PrefixSetName: "clusteripprefixset", PrefixList: advIPPrefixList, @@ -53,10 +61,10 @@ func (nrc *NetworkRoutingController) AddPolicies() error { nrc.bgpServer.AddDefinedSet(clusterIPPrefixSet) } - iBGPPeers := make([]string, 0) if nrc.bgpEnableInternal { // Get the current list of the nodes from the local cache nodes := nrc.nodeLister.List() + iBGPPeers := make([]string, 0) for _, node := range nodes { nodeObj := node.(*v1core.Node) nodeIP, err := utils.GetNodeIP(nodeObj) @@ -97,17 +105,6 @@ func (nrc *NetworkRoutingController) AddPolicies() error { } } - // a slice of all peers is used as a match condition for reject statement of clusteripprefixset import polcy - allBgpPeers := append(externalBgpPeers, iBGPPeers...) - ns, _ := table.NewNeighborSet(config.NeighborSet{ - NeighborSetName: "allpeerset", - NeighborInfoList: allBgpPeers, - }) - err = nrc.bgpServer.ReplaceDefinedSet(ns) - if err != nil { - nrc.bgpServer.AddDefinedSet(ns) - } - err = nrc.addExportPolicies() if err != nil { return err @@ -269,7 +266,7 @@ func (nrc *NetworkRoutingController) addExportPolicies() error { } // BGP import policies are added so that the following conditions are met: -// - do not import Service VIPs advertised from any peers, instead each kube-router originates and injects Service VIPs into local rib. +// - do not import Service VIPs at all, instead traffic to service VIPs should be sent to the gateway and ECMPed from there func (nrc *NetworkRoutingController) addImportPolicies() error { statements := make([]config.Statement, 0) @@ -278,9 +275,6 @@ func (nrc *NetworkRoutingController) addImportPolicies() error { MatchPrefixSet: config.MatchPrefixSet{ PrefixSet: "clusteripprefixset", }, - MatchNeighborSet: config.MatchNeighborSet{ - NeighborSet: "allpeerset", - }, }, Actions: config.Actions{ RouteDisposition: config.ROUTE_DISPOSITION_REJECT_ROUTE, diff --git a/pkg/controllers/routing/ecmp_vip.go b/pkg/controllers/routing/ecmp_vip.go index 6522118cc0..2e76a6692d 100644 --- a/pkg/controllers/routing/ecmp_vip.go +++ b/pkg/controllers/routing/ecmp_vip.go @@ -6,13 +6,12 @@ import ( "strconv" "time" - "strings" - "github.com/golang/glog" "github.com/osrg/gobgp/packet/bgp" "github.com/osrg/gobgp/table" v1core "k8s.io/api/core/v1" "k8s.io/client-go/tools/cache" + "strings" ) // bgpAdvertiseVIP advertises the service vip (cluster ip or load balancer ip or external IP) the configured peers @@ -23,10 +22,17 @@ func (nrc *NetworkRoutingController) bgpAdvertiseVIP(vip string) error { bgp.NewPathAttributeNextHop(nrc.nodeIP.String()), } - glog.V(2).Infof("Advertising route: '%s/%s via %s' to peers", vip, strconv.Itoa(32), nrc.nodeIP.String()) - - _, err := nrc.bgpServer.AddPath("", []*table.Path{table.NewPath(nil, bgp.NewIPAddrPrefix(uint8(32), - vip), false, attrs, time.Now(), false)}) + //If the value of advertise-svc-cidr parameter is not empty, then the value of advertise-svc-cidr parameter is put into RIB, otherwise it will be done according to the original rules. + var svcSubnet = vip + var svcCidrLen = 32 + if len(nrc.advertiseClusterSubnet) != 0 { + svcCidrStr := strings.Split(nrc.advertiseClusterSubnet, "/") + svcSubnet = svcCidrStr[0] + svcCidrLen, _ = strconv.Atoi(svcCidrStr[1]) + } + glog.V(2).Infof("Advertising route: '%s/%s via %s' to peers", svcSubnet, strconv.Itoa(svcCidrLen), nrc.nodeIP.String()) + _, err := nrc.bgpServer.AddPath("", []*table.Path{table.NewPath(nil, bgp.NewIPAddrPrefix(uint8(svcCidrLen), + svcSubnet), false, attrs, time.Now(), false)}) return err } @@ -35,10 +41,16 @@ func (nrc *NetworkRoutingController) bgpAdvertiseVIP(vip string) error { func (nrc *NetworkRoutingController) bgpWithdrawVIP(vip string) error { glog.V(2).Infof("Withdrawing route: '%s/%s via %s' to peers", vip, strconv.Itoa(32), nrc.nodeIP.String()) - pathList := []*table.Path{table.NewPath(nil, bgp.NewIPAddrPrefix(uint8(32), - vip), true, nil, time.Now(), false)} - - err := nrc.bgpServer.DeletePath([]byte(nil), 0, "", pathList) + //If the value of the advertise-svc-cidr parameter is not empty, no operation will be performed, otherwise the original rules will be followed. + var err error + if len(nrc.advertiseClusterSubnet) == 0 { + glog.V(2).Infof("Withdrawing route: '%s/%s via %s' to peers", vip, strconv.Itoa(32), nrc.nodeIP.String()) + pathList := []*table.Path{table.NewPath(nil, bgp.NewIPAddrPrefix(uint8(32), + vip), true, nil, time.Now(), false)} + err = nrc.bgpServer.DeletePath([]byte(nil), 0, "", pathList) + } else { + err = nil + } return err } @@ -104,22 +116,6 @@ func (nrc *NetworkRoutingController) handleServiceUpdate(svc *v1core.Service) { nrc.withdrawVIPs(toWithdraw) } -func (nrc *NetworkRoutingController) handleServiceDelete(svc *v1core.Service) { - - if !nrc.bgpServerStarted { - glog.V(3).Infof("Skipping update to service: %s/%s, controller still performing bootup full-sync", svc.Namespace, svc.Name) - return - } - - err := nrc.AddPolicies() - if err != nil { - glog.Errorf("Error adding BGP policies: %s", err.Error()) - } - - nrc.withdrawVIPs(nrc.getAllVIPsForService(svc)) - -} - func (nrc *NetworkRoutingController) tryHandleServiceUpdate(obj interface{}, logMsgFormat string) { if svc := getServiceObject(obj); svc != nil { glog.V(1).Infof(logMsgFormat, svc.Namespace, svc.Name) @@ -127,13 +123,6 @@ func (nrc *NetworkRoutingController) tryHandleServiceUpdate(obj interface{}, log } } -func (nrc *NetworkRoutingController) tryHandleServiceDelete(obj interface{}, logMsgFormat string) { - if svc := getServiceObject(obj); svc != nil { - glog.V(1).Infof(logMsgFormat, svc.Namespace, svc.Name) - nrc.handleServiceDelete(svc) - } -} - // OnServiceCreate handles new service create event from the kubernetes API server func (nrc *NetworkRoutingController) OnServiceCreate(obj interface{}) { nrc.tryHandleServiceUpdate(obj, "Received new service: %s/%s from watch API") @@ -165,7 +154,7 @@ func getMissingPrevGen(old, new []string) (withdrawIPs []string) { // OnServiceDelete handles the service delete updates from the kubernetes API server func (nrc *NetworkRoutingController) OnServiceDelete(obj interface{}) { - nrc.tryHandleServiceDelete(obj, "Received event to delete service: %s/%s from watch API") + nrc.tryHandleServiceUpdate(obj, "Received event to delete service: %s/%s from watch API") } func (nrc *NetworkRoutingController) newEndpointsEventHandler() cache.ResourceEventHandler { @@ -330,38 +319,24 @@ func (nrc *NetworkRoutingController) shouldAdvertiseService(svc *v1core.Service, } func (nrc *NetworkRoutingController) getVIPsForService(svc *v1core.Service, onlyActiveEndpoints bool) ([]string, []string, error) { - - advertise := true - - _, hasLocalAnnotation := svc.Annotations[svcLocalAnnotation] - hasLocalTrafficPolicy := svc.Spec.ExternalTrafficPolicy == v1core.ServiceExternalTrafficPolicyTypeLocal - isLocal := hasLocalAnnotation || hasLocalTrafficPolicy - - if onlyActiveEndpoints && isLocal { - var err error - advertise, err = nrc.nodeHasEndpointsForService(svc) - if err != nil { - return nil, nil, err + ipList := make([]string, 0) + var err error + + nodeHasEndpoints := true + if onlyActiveEndpoints { + _, isLocal := svc.Annotations[svcLocalAnnotation] + if isLocal || svc.Spec.ExternalTrafficPolicy == v1core.ServiceExternalTrafficPolicyTypeLocal { + nodeHasEndpoints, err = nrc.nodeHasEndpointsForService(svc) + if err != nil { + return nil, nil, err + } } } - ipList := nrc.getAllVIPsForService(svc) - - if !advertise { - return nil, ipList, nil - } - - return ipList, nil, nil -} - -func (nrc *NetworkRoutingController) getAllVIPsForService(svc *v1core.Service) []string { - - ipList := make([]string, 0) - if nrc.shouldAdvertiseService(svc, svcAdvertiseClusterAnnotation, nrc.advertiseClusterIP) { - clusterIP := nrc.getClusterIp(svc) - if clusterIP != "" { - ipList = append(ipList, clusterIP) + clusterIp := nrc.getClusterIp(svc) + if clusterIp != "" { + ipList = append(ipList, clusterIp) } } @@ -376,8 +351,11 @@ func (nrc *NetworkRoutingController) getAllVIPsForService(svc *v1core.Service) [ ipList = append(ipList, nrc.getLoadBalancerIps(svc)...) } - return ipList + if !nodeHasEndpoints { + return nil, ipList, nil + } + return ipList, nil, nil } func isEndpointsForLeaderElection(ep *v1core.Endpoints) bool { diff --git a/pkg/controllers/routing/network_routes_controller.go b/pkg/controllers/routing/network_routes_controller.go index 590f4d2ff5..f154c46ee4 100644 --- a/pkg/controllers/routing/network_routes_controller.go +++ b/pkg/controllers/routing/network_routes_controller.go @@ -63,52 +63,52 @@ const ( // NetworkRoutingController is struct to hold necessary information required by controller type NetworkRoutingController struct { - nodeIP net.IP - nodeName string - nodeSubnet net.IPNet - nodeInterface string - routerId string - isIpv6 bool - activeNodes map[string]bool - mu sync.Mutex - clientset kubernetes.Interface - bgpServer *gobgp.BgpServer - syncPeriod time.Duration - clusterCIDR string - enablePodEgress bool - hostnameOverride string - advertiseClusterIP bool - advertiseExternalIP bool - advertiseLoadBalancerIP bool - advertisePodCidr bool - defaultNodeAsnNumber uint32 - nodeAsnNumber uint32 - globalPeerRouters []*config.Neighbor - nodePeerRouters []string - enableCNI bool - bgpFullMeshMode bool - bgpEnableInternal bool - bgpGracefulRestart bool - bgpGracefulRestartDeferralTime time.Duration - ipSetHandler *utils.IPSet - enableOverlays bool - overlayType string - peerMultihopTTL uint8 - MetricsEnabled bool - bgpServerStarted bool - bgpPort uint16 - bgpRRClient bool - bgpRRServer bool - bgpClusterID uint32 - cniConfFile string - disableSrcDstCheck bool - initSrcDstCheckDone bool - ec2IamAuthorized bool - pathPrependAS string - pathPrependCount uint8 - pathPrepend bool - localAddressList []string - overrideNextHop bool + nodeIP net.IP + nodeName string + nodeSubnet net.IPNet + nodeInterface string + routerId string + isIpv6 bool + activeNodes map[string]bool + mu sync.Mutex + clientset kubernetes.Interface + bgpServer *gobgp.BgpServer + syncPeriod time.Duration + clusterCIDR string + enablePodEgress bool + hostnameOverride string + advertiseClusterIP bool + advertiseClusterSubnet string + advertiseExternalIP bool + advertiseLoadBalancerIP bool + advertisePodCidr bool + defaultNodeAsnNumber uint32 + nodeAsnNumber uint32 + globalPeerRouters []*config.Neighbor + nodePeerRouters []string + enableCNI bool + bgpFullMeshMode bool + bgpEnableInternal bool + bgpGracefulRestart bool + ipSetHandler *utils.IPSet + enableOverlays bool + overlayType string + peerMultihopTTL uint8 + MetricsEnabled bool + bgpServerStarted bool + bgpPort uint16 + bgpRRClient bool + bgpRRServer bool + bgpClusterID uint32 + cniConfFile string + disableSrcDstCheck bool + initSrcDstCheckDone bool + ec2IamAuthorized bool + pathPrependAS string + pathPrependCount uint8 + pathPrepend bool + localAddressList []string + overrideNextHop bool nodeLister cache.Indexer svcLister cache.Indexer @@ -822,7 +822,7 @@ func (nrc *NetworkRoutingController) startBgpServer() error { } if len(nrc.globalPeerRouters) != 0 { - err := connectToExternalBGPPeers(nrc.bgpServer, nrc.globalPeerRouters, nrc.bgpGracefulRestart, nrc.bgpGracefulRestartDeferralTime, nrc.peerMultihopTTL) + err := connectToExternalBGPPeers(nrc.bgpServer, nrc.globalPeerRouters, nrc.bgpGracefulRestart, nrc.peerMultihopTTL) if err != nil { nrc.bgpServer.Stop() return fmt.Errorf("Failed to peer with Global Peer Router(s): %s", @@ -859,7 +859,6 @@ func NewNetworkRoutingController(clientset kubernetes.Interface, nrc.enableCNI = kubeRouterConfig.EnableCNI nrc.bgpEnableInternal = kubeRouterConfig.EnableiBGP nrc.bgpGracefulRestart = kubeRouterConfig.BGPGracefulRestart - nrc.bgpGracefulRestartDeferralTime = kubeRouterConfig.BGPGracefulRestartDeferralTime nrc.peerMultihopTTL = kubeRouterConfig.PeerMultihopTtl nrc.enablePodEgress = kubeRouterConfig.EnablePodEgress nrc.syncPeriod = kubeRouterConfig.RoutesSyncPeriod @@ -871,6 +870,7 @@ func NewNetworkRoutingController(clientset kubernetes.Interface, nrc.bgpServerStarted = false nrc.disableSrcDstCheck = kubeRouterConfig.DisableSrcDstCheck nrc.initSrcDstCheckDone = false + nrc.advertiseClusterSubnet = kubeRouterConfig.AdvertiseClusterSubnet nrc.hostnameOverride = kubeRouterConfig.HostnameOverride node, err := utils.GetNodeObject(clientset, nrc.hostnameOverride) diff --git a/pkg/controllers/routing/network_routes_controller_test.go b/pkg/controllers/routing/network_routes_controller_test.go deleted file mode 100644 index 8a3ed9091b..0000000000 --- a/pkg/controllers/routing/network_routes_controller_test.go +++ /dev/null @@ -1,2522 +0,0 @@ -package routing - -import ( - "errors" - "net" - "os" - "reflect" - "testing" - "time" - - "github.com/osrg/gobgp/config" - gobgp "github.com/osrg/gobgp/server" - "github.com/osrg/gobgp/table" - - v1core "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/kubernetes/fake" - "k8s.io/client-go/tools/cache" -) - -func Test_advertiseClusterIPs(t *testing.T) { - testcases := []struct { - name string - nrc *NetworkRoutingController - existingServices []*v1core.Service - // the key is the subnet from the watch event - watchEvents map[string]bool - }{ - { - "add bgp path for service with ClusterIP", - &NetworkRoutingController{ - bgpServer: gobgp.NewBgpServer(), - }, - []*v1core.Service{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-1", - }, - Spec: v1core.ServiceSpec{ - Type: "ClusterIP", - ClusterIP: "10.0.0.1", - }, - }, - }, - map[string]bool{ - "10.0.0.1/32": true, - }, - }, - { - "add bgp path for service with ClusterIP/NodePort/LoadBalancer", - &NetworkRoutingController{ - bgpServer: gobgp.NewBgpServer(), - }, - []*v1core.Service{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-1", - }, - Spec: v1core.ServiceSpec{ - Type: "ClusterIP", - ClusterIP: "10.0.0.1", - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-2", - }, - Spec: v1core.ServiceSpec{ - Type: "LoadBalancer", - ClusterIP: "10.0.0.2", - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-3", - }, - Spec: v1core.ServiceSpec{ - Type: "NodePort", - ClusterIP: "10.0.0.3", - }, - }, - }, - map[string]bool{ - "10.0.0.1/32": true, - "10.0.0.2/32": true, - "10.0.0.3/32": true, - }, - }, - { - "add bgp path for invalid service type", - &NetworkRoutingController{ - bgpServer: gobgp.NewBgpServer(), - }, - []*v1core.Service{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-1", - }, - Spec: v1core.ServiceSpec{ - Type: "ClusterIP", - ClusterIP: "10.0.0.1", - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-2", - }, - Spec: v1core.ServiceSpec{ - Type: "AnotherType", - ClusterIP: "10.0.0.2", - }, - }, - }, - map[string]bool{ - "10.0.0.1/32": true, - }, - }, - { - "add bgp path for headless service", - &NetworkRoutingController{ - bgpServer: gobgp.NewBgpServer(), - }, - []*v1core.Service{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-1", - }, - Spec: v1core.ServiceSpec{ - Type: "ClusterIP", - ClusterIP: "10.0.0.1", - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-2", - }, - Spec: v1core.ServiceSpec{ - Type: "ClusterIP", - ClusterIP: "None", - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-3", - }, - Spec: v1core.ServiceSpec{ - Type: "ClusterIP", - ClusterIP: "", - }, - }, - }, - map[string]bool{ - "10.0.0.1/32": true, - }, - }, - } - - for _, testcase := range testcases { - t.Run(testcase.name, func(t *testing.T) { - go testcase.nrc.bgpServer.Serve() - err := testcase.nrc.bgpServer.Start(&config.Global{ - Config: config.GlobalConfig{ - As: 1, - RouterId: "10.0.0.0", - Port: 10000, - }, - }) - if err != nil { - t.Fatalf("failed to start BGP server: %v", err) - } - defer testcase.nrc.bgpServer.Stop() - w := testcase.nrc.bgpServer.Watch(gobgp.WatchBestPath(false)) - - clientset := fake.NewSimpleClientset() - startInformersForRoutes(testcase.nrc, clientset) - - err = createServices(clientset, testcase.existingServices) - if err != nil { - t.Fatalf("failed to create existing services: %v", err) - } - - waitForListerWithTimeout(testcase.nrc.svcLister, time.Second*10, t) - - // ClusterIPs - testcase.nrc.advertiseClusterIP = true - testcase.nrc.advertiseExternalIP = false - testcase.nrc.advertiseLoadBalancerIP = false - - toAdvertise, toWithdraw, _ := testcase.nrc.getActiveVIPs() - testcase.nrc.advertiseVIPs(toAdvertise) - testcase.nrc.withdrawVIPs(toWithdraw) - - watchEvents := waitForBGPWatchEventWithTimeout(time.Second*10, len(testcase.watchEvents), w, t) - for _, watchEvent := range watchEvents { - for _, path := range watchEvent.PathList { - if _, ok := testcase.watchEvents[path.GetNlri().String()]; ok { - continue - } else { - t.Errorf("got unexpected path: %v", path.GetNlri().String()) - } - } - } - }) - } -} - -func Test_advertiseExternalIPs(t *testing.T) { - testcases := []struct { - name string - nrc *NetworkRoutingController - existingServices []*v1core.Service - // the key is the subnet from the watch event - watchEvents map[string]bool - }{ - { - "add bgp path for service with external IPs", - &NetworkRoutingController{ - bgpServer: gobgp.NewBgpServer(), - }, - []*v1core.Service{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-1", - }, - Spec: v1core.ServiceSpec{ - Type: "ClusterIP", - ClusterIP: "10.0.0.1", - ExternalIPs: []string{"1.1.1.1", "2.2.2.2"}, - }, - }, - }, - map[string]bool{ - "1.1.1.1/32": true, - "2.2.2.2/32": true, - }, - }, - { - "add bgp path for services with external IPs of type ClusterIP/NodePort/LoadBalancer", - &NetworkRoutingController{ - bgpServer: gobgp.NewBgpServer(), - }, - []*v1core.Service{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-1", - }, - Spec: v1core.ServiceSpec{ - Type: "ClusterIP", - ClusterIP: "10.0.0.1", - ExternalIPs: []string{"1.1.1.1"}, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-2", - }, - Spec: v1core.ServiceSpec{ - Type: "LoadBalancer", - ClusterIP: "10.0.0.2", - // ignored since LoadBalancer services don't - // advertise external IPs. - ExternalIPs: []string{"2.2.2.2"}, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-3", - }, - Spec: v1core.ServiceSpec{ - Type: "NodePort", - ClusterIP: "10.0.0.3", - ExternalIPs: []string{"3.3.3.3", "4.4.4.4"}, - }, - }, - }, - map[string]bool{ - "1.1.1.1/32": true, - "3.3.3.3/32": true, - "4.4.4.4/32": true, - }, - }, - { - "add bgp path for invalid service type", - &NetworkRoutingController{ - bgpServer: gobgp.NewBgpServer(), - }, - []*v1core.Service{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-1", - }, - Spec: v1core.ServiceSpec{ - Type: "ClusterIP", - ClusterIP: "10.0.0.1", - ExternalIPs: []string{"1.1.1.1"}, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-2", - }, - Spec: v1core.ServiceSpec{ - Type: "AnotherType", - ClusterIP: "10.0.0.2", - ExternalIPs: []string{"2.2.2.2"}, - }, - }, - }, - map[string]bool{ - "1.1.1.1/32": true, - }, - }, - { - "add bgp path for headless service", - &NetworkRoutingController{ - bgpServer: gobgp.NewBgpServer(), - }, - []*v1core.Service{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-1", - }, - Spec: v1core.ServiceSpec{ - Type: "ClusterIP", - ClusterIP: "10.0.0.1", - ExternalIPs: []string{"1.1.1.1"}, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-2", - }, - Spec: v1core.ServiceSpec{ - Type: "ClusterIP", - ClusterIP: "None", - ExternalIPs: []string{"2.2.2.2"}, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-3", - }, - Spec: v1core.ServiceSpec{ - Type: "ClusterIP", - ClusterIP: "", - ExternalIPs: []string{"3.3.3.3"}, - }, - }, - }, - map[string]bool{ - "1.1.1.1/32": true, - }, - }, - { - "skip bgp path to loadbalancerIP for service without LoadBalancer IP", - &NetworkRoutingController{ - bgpServer: gobgp.NewBgpServer(), - }, - []*v1core.Service{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-1", - }, - Spec: v1core.ServiceSpec{ - Type: "LoadBalancer", - ClusterIP: "10.0.0.1", - }, - Status: v1core.ServiceStatus{ - LoadBalancer: v1core.LoadBalancerStatus{ - Ingress: []v1core.LoadBalancerIngress{ - { - Hostname: "foo-bar.zone.elb.example.com", - }, - }, - }, - }, - }, - }, - map[string]bool{}, - }, - { - "add bgp path to loadbalancerIP for service with LoadBalancer IP", - &NetworkRoutingController{ - bgpServer: gobgp.NewBgpServer(), - }, - []*v1core.Service{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-1", - }, - Spec: v1core.ServiceSpec{ - Type: "LoadBalancer", - ClusterIP: "10.0.0.1", - }, - Status: v1core.ServiceStatus{ - LoadBalancer: v1core.LoadBalancerStatus{ - Ingress: []v1core.LoadBalancerIngress{ - { - IP: "10.0.255.1", - }, - { - IP: "10.0.255.2", - }, - }, - }, - }, - }, - }, - map[string]bool{ - "10.0.255.1/32": true, - "10.0.255.2/32": true, - }, - }, - { - "no bgp path to nil loadbalancerIPs for service with LoadBalancer", - &NetworkRoutingController{ - bgpServer: gobgp.NewBgpServer(), - }, - []*v1core.Service{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-1", - }, - Spec: v1core.ServiceSpec{ - Type: "LoadBalancer", - ClusterIP: "10.0.0.1", - }, - Status: v1core.ServiceStatus{ - LoadBalancer: v1core.LoadBalancerStatus{ - Ingress: []v1core.LoadBalancerIngress{}, - }, - }, - }, - }, - map[string]bool{}, - }, - { - "no bgp path to loadbalancerIPs for service with LoadBalancer and skiplbips annotation", - &NetworkRoutingController{ - bgpServer: gobgp.NewBgpServer(), - }, - []*v1core.Service{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-1", - Annotations: map[string]string{ - svcSkipLbIpsAnnotation: "true", - }, - }, - Spec: v1core.ServiceSpec{ - Type: "LoadBalancer", - ClusterIP: "10.0.0.1", - }, - Status: v1core.ServiceStatus{ - LoadBalancer: v1core.LoadBalancerStatus{ - Ingress: []v1core.LoadBalancerIngress{ - { - IP: "10.0.255.1", - }, - { - IP: "10.0.255.2", - }, - }, - }, - }, - }, - }, - map[string]bool{}, - }, - } - - for _, testcase := range testcases { - t.Run(testcase.name, func(t *testing.T) { - go testcase.nrc.bgpServer.Serve() - err := testcase.nrc.bgpServer.Start(&config.Global{ - Config: config.GlobalConfig{ - As: 1, - RouterId: "10.0.0.0", - Port: 10000, - }, - }) - if err != nil { - t.Fatalf("failed to start BGP server: %v", err) - } - defer testcase.nrc.bgpServer.Stop() - w := testcase.nrc.bgpServer.Watch(gobgp.WatchBestPath(false)) - - clientset := fake.NewSimpleClientset() - startInformersForRoutes(testcase.nrc, clientset) - - err = createServices(clientset, testcase.existingServices) - if err != nil { - t.Fatalf("failed to create existing services: %v", err) - } - - waitForListerWithTimeout(testcase.nrc.svcLister, time.Second*10, t) - - // ExternalIPs - testcase.nrc.advertiseClusterIP = false - testcase.nrc.advertiseExternalIP = true - testcase.nrc.advertiseLoadBalancerIP = true - - toAdvertise, toWithdraw, _ := testcase.nrc.getActiveVIPs() - testcase.nrc.advertiseVIPs(toAdvertise) - testcase.nrc.withdrawVIPs(toWithdraw) - - watchEvents := waitForBGPWatchEventWithTimeout(time.Second*10, len(testcase.watchEvents), w, t) - for _, watchEvent := range watchEvents { - for _, path := range watchEvent.PathList { - if _, ok := testcase.watchEvents[path.GetNlri().String()]; ok { - continue - } else { - t.Errorf("got unexpected path: %v", path.GetNlri().String()) - } - } - } - }) - } -} - -func Test_advertiseAnnotationOptOut(t *testing.T) { - testcases := []struct { - name string - nrc *NetworkRoutingController - existingServices []*v1core.Service - // the key is the subnet from the watch event - watchEvents map[string]bool - }{ - { - "add bgp paths for all service IPs", - &NetworkRoutingController{ - bgpServer: gobgp.NewBgpServer(), - }, - []*v1core.Service{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-1", - }, - Spec: v1core.ServiceSpec{ - Type: "ClusterIP", - ClusterIP: "10.0.0.1", - ExternalIPs: []string{"1.1.1.1"}, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-2", - }, - Spec: v1core.ServiceSpec{ - Type: "NodePort", - ClusterIP: "10.0.0.2", - ExternalIPs: []string{"2.2.2.2", "3.3.3.3"}, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-3", - }, - Spec: v1core.ServiceSpec{ - Type: "LoadBalancer", - ClusterIP: "10.0.0.3", - // ignored since LoadBalancer services don't - // advertise external IPs. - ExternalIPs: []string{"4.4.4.4"}, - }, - Status: v1core.ServiceStatus{ - LoadBalancer: v1core.LoadBalancerStatus{ - Ingress: []v1core.LoadBalancerIngress{ - { - IP: "10.0.255.1", - }, - { - IP: "10.0.255.2", - }, - }, - }, - }, - }, - }, - map[string]bool{ - "10.0.0.1/32": true, - "10.0.0.2/32": true, - "10.0.0.3/32": true, - "1.1.1.1/32": true, - "2.2.2.2/32": true, - "3.3.3.3/32": true, - "10.0.255.1/32": true, - "10.0.255.2/32": true, - }, - }, - { - "opt out to advertise any IPs via annotations", - &NetworkRoutingController{ - bgpServer: gobgp.NewBgpServer(), - }, - []*v1core.Service{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-1", - Annotations: map[string]string{ - svcAdvertiseClusterAnnotation: "false", - svcAdvertiseExternalAnnotation: "false", - svcAdvertiseLoadBalancerAnnotation: "false", - }, - }, - Spec: v1core.ServiceSpec{ - Type: "LoadBalancer", - ClusterIP: "10.0.0.1", - ExternalIPs: []string{"1.1.1.1", "2.2.2.2"}, - }, - Status: v1core.ServiceStatus{ - LoadBalancer: v1core.LoadBalancerStatus{ - Ingress: []v1core.LoadBalancerIngress{ - { - IP: "10.0.255.1", - }, - { - IP: "10.0.255.2", - }, - }, - }, - }, - }, - }, - map[string]bool{}, - }, - } - - for _, testcase := range testcases { - t.Run(testcase.name, func(t *testing.T) { - go testcase.nrc.bgpServer.Serve() - err := testcase.nrc.bgpServer.Start(&config.Global{ - Config: config.GlobalConfig{ - As: 1, - RouterId: "10.0.0.0", - Port: 10000, - }, - }) - if err != nil { - t.Fatalf("failed to start BGP server: %v", err) - } - defer testcase.nrc.bgpServer.Stop() - w := testcase.nrc.bgpServer.Watch(gobgp.WatchBestPath(false)) - - clientset := fake.NewSimpleClientset() - startInformersForRoutes(testcase.nrc, clientset) - - err = createServices(clientset, testcase.existingServices) - if err != nil { - t.Fatalf("failed to create existing services: %v", err) - } - - waitForListerWithTimeout(testcase.nrc.svcLister, time.Second*10, t) - - // By default advertise all IPs - testcase.nrc.advertiseClusterIP = true - testcase.nrc.advertiseExternalIP = true - testcase.nrc.advertiseLoadBalancerIP = true - - toAdvertise, toWithdraw, _ := testcase.nrc.getActiveVIPs() - testcase.nrc.advertiseVIPs(toAdvertise) - testcase.nrc.withdrawVIPs(toWithdraw) - - watchEvents := waitForBGPWatchEventWithTimeout(time.Second*10, len(testcase.watchEvents), w, t) - for _, watchEvent := range watchEvents { - for _, path := range watchEvent.PathList { - if _, ok := testcase.watchEvents[path.GetNlri().String()]; ok { - continue - } else { - t.Errorf("got unexpected path: %v", path.GetNlri().String()) - } - } - } - }) - } -} - -func Test_advertiseAnnotationOptIn(t *testing.T) { - testcases := []struct { - name string - nrc *NetworkRoutingController - existingServices []*v1core.Service - // the key is the subnet from the watch event - watchEvents map[string]bool - }{ - { - "no bgp paths for any service IPs", - &NetworkRoutingController{ - bgpServer: gobgp.NewBgpServer(), - }, - []*v1core.Service{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-1", - }, - Spec: v1core.ServiceSpec{ - Type: "ClusterIP", - ClusterIP: "10.0.0.1", - ExternalIPs: []string{"1.1.1.1"}, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-2", - }, - Spec: v1core.ServiceSpec{ - Type: "NodePort", - ClusterIP: "10.0.0.2", - ExternalIPs: []string{"2.2.2.2", "3.3.3.3"}, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-3", - }, - Spec: v1core.ServiceSpec{ - Type: "LoadBalancer", - ClusterIP: "10.0.0.3", - // ignored since LoadBalancer services don't - // advertise external IPs. - ExternalIPs: []string{"4.4.4.4"}, - }, - Status: v1core.ServiceStatus{ - LoadBalancer: v1core.LoadBalancerStatus{ - Ingress: []v1core.LoadBalancerIngress{ - { - IP: "10.0.255.1", - }, - { - IP: "10.0.255.2", - }, - }, - }, - }, - }, - }, - map[string]bool{}, - }, - { - "opt in to advertise all IPs via annotations", - &NetworkRoutingController{ - bgpServer: gobgp.NewBgpServer(), - }, - []*v1core.Service{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-1", - Annotations: map[string]string{ - svcAdvertiseClusterAnnotation: "true", - svcAdvertiseExternalAnnotation: "true", - svcAdvertiseLoadBalancerAnnotation: "true", - }, - }, - Spec: v1core.ServiceSpec{ - Type: "ClusterIP", - ClusterIP: "10.0.0.1", - ExternalIPs: []string{"1.1.1.1"}, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-2", - Annotations: map[string]string{ - svcAdvertiseClusterAnnotation: "true", - svcAdvertiseExternalAnnotation: "true", - svcAdvertiseLoadBalancerAnnotation: "true", - }, - }, - Spec: v1core.ServiceSpec{ - Type: "NodePort", - ClusterIP: "10.0.0.2", - ExternalIPs: []string{"2.2.2.2", "3.3.3.3"}, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-3", - Annotations: map[string]string{ - svcAdvertiseClusterAnnotation: "true", - svcAdvertiseExternalAnnotation: "true", - svcAdvertiseLoadBalancerAnnotation: "true", - }, - }, - Spec: v1core.ServiceSpec{ - Type: "LoadBalancer", - ClusterIP: "10.0.0.3", - // ignored since LoadBalancer services don't - // advertise external IPs. - ExternalIPs: []string{"4.4.4.4"}, - }, - Status: v1core.ServiceStatus{ - LoadBalancer: v1core.LoadBalancerStatus{ - Ingress: []v1core.LoadBalancerIngress{ - { - IP: "10.0.255.1", - }, - { - IP: "10.0.255.2", - }, - }, - }, - }, - }, - }, - map[string]bool{ - "10.0.0.1/32": true, - "10.0.0.2/32": true, - "10.0.0.3/32": true, - "1.1.1.1/32": true, - "2.2.2.2/32": true, - "3.3.3.3/32": true, - "10.0.255.1/32": true, - "10.0.255.2/32": true, - }, - }, - } - - for _, testcase := range testcases { - t.Run(testcase.name, func(t *testing.T) { - go testcase.nrc.bgpServer.Serve() - err := testcase.nrc.bgpServer.Start(&config.Global{ - Config: config.GlobalConfig{ - As: 1, - RouterId: "10.0.0.0", - Port: 10000, - }, - }) - if err != nil { - t.Fatalf("failed to start BGP server: %v", err) - } - defer testcase.nrc.bgpServer.Stop() - w := testcase.nrc.bgpServer.Watch(gobgp.WatchBestPath(false)) - - clientset := fake.NewSimpleClientset() - startInformersForRoutes(testcase.nrc, clientset) - - err = createServices(clientset, testcase.existingServices) - if err != nil { - t.Fatalf("failed to create existing services: %v", err) - } - - waitForListerWithTimeout(testcase.nrc.svcLister, time.Second*10, t) - - // By default do not advertise any IPs - testcase.nrc.advertiseClusterIP = false - testcase.nrc.advertiseExternalIP = false - testcase.nrc.advertiseLoadBalancerIP = false - - toAdvertise, toWithdraw, _ := testcase.nrc.getActiveVIPs() - testcase.nrc.advertiseVIPs(toAdvertise) - testcase.nrc.withdrawVIPs(toWithdraw) - - watchEvents := waitForBGPWatchEventWithTimeout(time.Second*10, len(testcase.watchEvents), w, t) - for _, watchEvent := range watchEvents { - for _, path := range watchEvent.PathList { - if _, ok := testcase.watchEvents[path.GetNlri().String()]; ok { - continue - } else { - t.Errorf("got unexpected path: %v", path.GetNlri().String()) - } - } - } - }) - } -} - -func Test_nodeHasEndpointsForService(t *testing.T) { - testcases := []struct { - name string - nrc *NetworkRoutingController - existingService *v1core.Service - existingEndpoint *v1core.Endpoints - nodeHasEndpoints bool - err error - }{ - { - "node has endpoints for service", - &NetworkRoutingController{ - nodeName: "node-1", - }, - &v1core.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-1", - Namespace: "default", - }, - Spec: v1core.ServiceSpec{ - Type: "ClusterIP", - ClusterIP: "10.0.0.1", - ExternalIPs: []string{"1.1.1.1", "2.2.2.2"}, - }, - }, - &v1core.Endpoints{ - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-1", - Namespace: "default", - }, - Subsets: []v1core.EndpointSubset{ - { - Addresses: []v1core.EndpointAddress{ - { - IP: "172.20.1.1", - NodeName: ptrToString("node-1"), - }, - { - IP: "172.20.1.2", - NodeName: ptrToString("node-2"), - }, - }, - }, - }, - }, - true, - nil, - }, - { - "node has no endpoints for service", - &NetworkRoutingController{ - nodeName: "node-1", - }, - &v1core.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-1", - Namespace: "default", - }, - Spec: v1core.ServiceSpec{ - Type: "ClusterIP", - ClusterIP: "10.0.0.1", - ExternalIPs: []string{"1.1.1.1", "2.2.2.2"}, - }, - }, - &v1core.Endpoints{ - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-1", - Namespace: "default", - }, - Subsets: []v1core.EndpointSubset{ - { - Addresses: []v1core.EndpointAddress{ - { - IP: "172.20.1.1", - NodeName: ptrToString("node-2"), - }, - { - IP: "172.20.1.2", - NodeName: ptrToString("node-3"), - }, - }, - }, - }, - }, - false, - nil, - }, - } - - for _, testcase := range testcases { - t.Run(testcase.name, func(t *testing.T) { - clientset := fake.NewSimpleClientset() - startInformersForRoutes(testcase.nrc, clientset) - - _, err := clientset.CoreV1().Endpoints("default").Create(testcase.existingEndpoint) - if err != nil { - t.Fatalf("failed to create existing endpoints: %v", err) - } - - _, err = clientset.CoreV1().Services("default").Create(testcase.existingService) - if err != nil { - t.Fatalf("failed to create existing services: %v", err) - } - - waitForListerWithTimeout(testcase.nrc.svcLister, time.Second*10, t) - waitForListerWithTimeout(testcase.nrc.epLister, time.Second*10, t) - - nodeHasEndpoints, err := testcase.nrc.nodeHasEndpointsForService(testcase.existingService) - if !reflect.DeepEqual(err, testcase.err) { - t.Logf("actual err: %v", err) - t.Logf("expected err: %v", testcase.err) - t.Error("unexpected error") - } - if nodeHasEndpoints != testcase.nodeHasEndpoints { - t.Logf("expected nodeHasEndpoints: %v", testcase.nodeHasEndpoints) - t.Logf("actual nodeHasEndpoints: %v", nodeHasEndpoints) - t.Error("unexpected nodeHasEndpoints") - } - - }) - } -} - -func Test_advertisePodRoute(t *testing.T) { - testcases := []struct { - name string - nrc *NetworkRoutingController - envNodeName string - node *v1core.Node - // the key is the subnet from the watch event - watchEvents map[string]bool - err error - }{ - { - "add bgp path for pod cidr using NODE_NAME", - &NetworkRoutingController{ - bgpServer: gobgp.NewBgpServer(), - }, - "node-1", - &v1core.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node-1", - }, - Spec: v1core.NodeSpec{ - PodCIDR: "172.20.0.0/24", - }, - }, - map[string]bool{ - "172.20.0.0/24": true, - }, - nil, - }, - { - "add bgp path for pod cidr using hostname override", - &NetworkRoutingController{ - bgpServer: gobgp.NewBgpServer(), - hostnameOverride: "node-1", - }, - "", - &v1core.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node-1", - }, - Spec: v1core.NodeSpec{ - PodCIDR: "172.20.0.0/24", - }, - }, - map[string]bool{ - "172.20.0.0/24": true, - }, - nil, - }, - { - "add bgp path for pod cidr without NODE_NAME or hostname override", - &NetworkRoutingController{ - bgpServer: gobgp.NewBgpServer(), - }, - "", - &v1core.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node-1", - }, - Spec: v1core.NodeSpec{ - PodCIDR: "172.20.0.0/24", - }, - }, - map[string]bool{}, - errors.New("Failed to get pod CIDR allocated for the node due to: Failed to identify the node by NODE_NAME, hostname or --hostname-override"), - }, - { - "node does not have pod cidr set", - &NetworkRoutingController{ - bgpServer: gobgp.NewBgpServer(), - }, - "node-1", - &v1core.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node-1", - }, - Spec: v1core.NodeSpec{ - PodCIDR: "", - }, - }, - map[string]bool{}, - errors.New("node.Spec.PodCIDR not set for node: node-1"), - }, - } - - for _, testcase := range testcases { - t.Run(testcase.name, func(t *testing.T) { - go testcase.nrc.bgpServer.Serve() - err := testcase.nrc.bgpServer.Start(&config.Global{ - Config: config.GlobalConfig{ - As: 1, - RouterId: "10.0.0.0", - Port: 10000, - }, - }) - if err != nil { - t.Fatalf("failed to start BGP server: %v", err) - } - defer testcase.nrc.bgpServer.Stop() - w := testcase.nrc.bgpServer.Watch(gobgp.WatchBestPath(false)) - - clientset := fake.NewSimpleClientset() - _, err = clientset.CoreV1().Nodes().Create(testcase.node) - if err != nil { - t.Fatalf("failed to create node: %v", err) - } - testcase.nrc.clientset = clientset - - os.Setenv("NODE_NAME", testcase.envNodeName) - defer os.Unsetenv("NODE_NAME") - - err = testcase.nrc.advertisePodRoute() - if !reflect.DeepEqual(err, testcase.err) { - t.Logf("actual error: %v", err) - t.Logf("expected error: %v", testcase.err) - t.Error("did not get expected error") - } - - watchEvents := waitForBGPWatchEventWithTimeout(time.Second*10, len(testcase.watchEvents), w, t) - for _, watchEvent := range watchEvents { - for _, path := range watchEvent.PathList { - if _, ok := testcase.watchEvents[path.GetNlri().String()]; ok { - continue - } else { - t.Errorf("got unexpected path: %v", path.GetNlri().String()) - } - } - } - }) - } -} - -func Test_syncInternalPeers(t *testing.T) { - testcases := []struct { - name string - nrc *NetworkRoutingController - existingNodes []*v1core.Node - neighbors map[string]bool - }{ - { - "sync 1 peer", - &NetworkRoutingController{ - bgpFullMeshMode: true, - clientset: fake.NewSimpleClientset(), - nodeIP: net.ParseIP("10.0.0.0"), - bgpServer: gobgp.NewBgpServer(), - activeNodes: make(map[string]bool), - }, - []*v1core.Node{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "node-1", - }, - Status: v1core.NodeStatus{ - Addresses: []v1core.NodeAddress{ - { - Type: v1core.NodeInternalIP, - Address: "10.0.0.1", - }, - }, - }, - }, - }, - map[string]bool{ - "10.0.0.1": true, - }, - }, - { - "sync multiple peers", - &NetworkRoutingController{ - bgpFullMeshMode: true, - clientset: fake.NewSimpleClientset(), - nodeIP: net.ParseIP("10.0.0.0"), - bgpServer: gobgp.NewBgpServer(), - activeNodes: make(map[string]bool), - }, - []*v1core.Node{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "node-1", - }, - Status: v1core.NodeStatus{ - Addresses: []v1core.NodeAddress{ - { - Type: v1core.NodeInternalIP, - Address: "10.0.0.1", - }, - }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "node-2", - }, - Status: v1core.NodeStatus{ - Addresses: []v1core.NodeAddress{ - { - Type: v1core.NodeInternalIP, - Address: "10.0.0.2", - }, - }, - }, - }, - }, - map[string]bool{ - "10.0.0.1": true, - "10.0.0.2": true, - }, - }, - { - "sync peer with removed nodes", - &NetworkRoutingController{ - bgpFullMeshMode: true, - clientset: fake.NewSimpleClientset(), - nodeIP: net.ParseIP("10.0.0.0"), - bgpServer: gobgp.NewBgpServer(), - activeNodes: map[string]bool{ - "10.0.0.2": true, - }, - }, - []*v1core.Node{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "node-1", - }, - Status: v1core.NodeStatus{ - Addresses: []v1core.NodeAddress{ - { - Type: v1core.NodeInternalIP, - Address: "10.0.0.1", - }, - }, - }, - }, - }, - map[string]bool{ - "10.0.0.1": true, - }, - }, - { - "sync multiple peers with full mesh disabled", - &NetworkRoutingController{ - bgpFullMeshMode: false, - clientset: fake.NewSimpleClientset(), - nodeIP: net.ParseIP("10.0.0.0"), - bgpServer: gobgp.NewBgpServer(), - activeNodes: make(map[string]bool), - nodeAsnNumber: 100, - }, - []*v1core.Node{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "node-1", - Annotations: map[string]string{ - "kube-router.io/node.asn": "100", - }, - }, - Status: v1core.NodeStatus{ - Addresses: []v1core.NodeAddress{ - { - Type: v1core.NodeInternalIP, - Address: "10.0.0.1", - }, - }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "node-2", - }, - Status: v1core.NodeStatus{ - Addresses: []v1core.NodeAddress{ - { - Type: v1core.NodeInternalIP, - Address: "10.0.0.2", - }, - }, - }, - }, - }, - map[string]bool{ - "10.0.0.1": true, - }, - }, - } - - for _, testcase := range testcases { - t.Run(testcase.name, func(t *testing.T) { - go testcase.nrc.bgpServer.Serve() - err := testcase.nrc.bgpServer.Start(&config.Global{ - Config: config.GlobalConfig{ - As: 1, - RouterId: "10.0.0.0", - Port: 10000, - }, - }) - if err != nil { - t.Fatalf("failed to start BGP server: %v", err) - } - defer testcase.nrc.bgpServer.Stop() - - if err = createNodes(testcase.nrc.clientset, testcase.existingNodes); err != nil { - t.Errorf("failed to create existing nodes: %v", err) - } - - testcase.nrc.syncInternalPeers() - - neighbors := testcase.nrc.bgpServer.GetNeighbor("", false) - for _, neighbor := range neighbors { - _, exists := testcase.neighbors[neighbor.Config.NeighborAddress] - if !exists { - t.Errorf("expected neighbor: %v doesn't exist", neighbor.Config.NeighborAddress) - } - } - - if !reflect.DeepEqual(testcase.nrc.activeNodes, testcase.neighbors) { - t.Logf("actual active nodes: %v", testcase.nrc.activeNodes) - t.Logf("expected active nodes: %v", testcase.neighbors) - t.Errorf("did not get expected activeNodes") - } - }) - } -} - -/* Disabling test for now. OnNodeUpdate() behaviour is changed. test needs to be adopted. -func Test_OnNodeUpdate(t *testing.T) { - testcases := []struct { - name string - nrc *NetworkRoutingController - nodeEvents []*watchers.NodeUpdate - activeNodes map[string]bool - }{ - { - "node add event", - &NetworkRoutingController{ - activeNodes: make(map[string]bool), - bgpServer: gobgp.NewBgpServer(), - defaultNodeAsnNumber: 1, - clientset: fake.NewSimpleClientset(), - }, - []*watchers.NodeUpdate{ - { - Node: &v1core.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node-1", - }, - Status: v1core.NodeStatus{ - Addresses: []v1core.NodeAddress{ - { - Type: v1core.NodeInternalIP, - Address: "10.0.0.1", - }, - }, - }, - }, - Op: watchers.ADD, - }, - }, - map[string]bool{ - "10.0.0.1": true, - }, - }, - { - "add multiple nodes", - &NetworkRoutingController{ - activeNodes: make(map[string]bool), - bgpServer: gobgp.NewBgpServer(), - defaultNodeAsnNumber: 1, - clientset: fake.NewSimpleClientset(), - }, - []*watchers.NodeUpdate{ - { - Node: &v1core.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node-1", - }, - Status: v1core.NodeStatus{ - Addresses: []v1core.NodeAddress{ - { - Type: v1core.NodeInternalIP, - Address: "10.0.0.1", - }, - }, - }, - }, - Op: watchers.ADD, - }, - { - Node: &v1core.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node-2", - }, - Status: v1core.NodeStatus{ - Addresses: []v1core.NodeAddress{ - { - Type: v1core.NodeExternalIP, - Address: "1.1.1.1", - }, - }, - }, - }, - Op: watchers.ADD, - }, - }, - map[string]bool{ - "10.0.0.1": true, - "1.1.1.1": true, - }, - }, - { - "add and then delete nodes", - &NetworkRoutingController{ - activeNodes: make(map[string]bool), - bgpServer: gobgp.NewBgpServer(), - defaultNodeAsnNumber: 1, - clientset: fake.NewSimpleClientset(), - }, - []*watchers.NodeUpdate{ - { - Node: &v1core.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node-1", - }, - Status: v1core.NodeStatus{ - Addresses: []v1core.NodeAddress{ - { - Type: v1core.NodeInternalIP, - Address: "10.0.0.1", - }, - }, - }, - }, - Op: watchers.ADD, - }, - { - Node: &v1core.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node-1", - }, - Status: v1core.NodeStatus{ - Addresses: []v1core.NodeAddress{ - { - Type: v1core.NodeInternalIP, - Address: "10.0.0.1", - }, - }, - }, - }, - Op: watchers.REMOVE, - }, - }, - map[string]bool{}, - }, - } - - for _, testcase := range testcases { - t.Run(testcase.name, func(t *testing.T) { - go testcase.nrc.bgpServer.Serve() - err := testcase.nrc.bgpServer.Start(&config.Global{ - Config: config.GlobalConfig{ - As: 1, - RouterId: "10.0.0.0", - Port: 10000, - }, - }) - testcase.nrc.bgpServerStarted = true - if err != nil { - t.Fatalf("failed to start BGP server: %v", err) - } - defer testcase.nrc.bgpServer.Stop() - - for _, nodeEvent := range testcase.nodeEvents { - testcase.nrc.OnNodeUpdate(nodeEvent) - } - - neighbors := testcase.nrc.bgpServer.GetNeighbor("", false) - for _, neighbor := range neighbors { - _, exists := testcase.activeNodes[neighbor.Config.NeighborAddress] - if !exists { - t.Errorf("expected neighbor: %v doesn't exist", neighbor.Config.NeighborAddress) - } - } - - if !reflect.DeepEqual(testcase.nrc.activeNodes, testcase.activeNodes) { - t.Logf("actual active nodes: %v", testcase.nrc.activeNodes) - t.Logf("expected active nodes: %v", testcase.activeNodes) - t.Errorf("did not get expected activeNodes") - } - }) - } -} -*/ - -type PolicyTestCase struct { - name string - nrc *NetworkRoutingController - existingNodes []*v1core.Node - existingServices []*v1core.Service - podDefinedSet *config.DefinedSets - clusterIPDefinedSet *config.DefinedSets - externalPeerDefinedSet *config.DefinedSets - allPeerDefinedSet *config.DefinedSets - exportPolicyStatements []*config.Statement - importPolicyStatements []*config.Statement - err error -} - -func Test_AddPolicies(t *testing.T) { - testcases := []PolicyTestCase{ - { - "has nodes and services", - &NetworkRoutingController{ - clientset: fake.NewSimpleClientset(), - hostnameOverride: "node-1", - bgpFullMeshMode: false, - bgpEnableInternal: true, - bgpServer: gobgp.NewBgpServer(), - activeNodes: make(map[string]bool), - nodeAsnNumber: 100, - }, - []*v1core.Node{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "node-1", - Annotations: map[string]string{ - "kube-router.io/node.asn": "100", - }, - }, - Status: v1core.NodeStatus{ - Addresses: []v1core.NodeAddress{ - { - Type: v1core.NodeInternalIP, - Address: "10.0.0.1", - }, - }, - }, - Spec: v1core.NodeSpec{ - PodCIDR: "172.20.0.0/24", - }, - }, - }, - []*v1core.Service{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-1", - }, - Spec: v1core.ServiceSpec{ - Type: "ClusterIP", - ClusterIP: "10.0.0.1", - ExternalIPs: []string{"1.1.1.1"}, - }, - }, - }, - &config.DefinedSets{ - PrefixSets: []config.PrefixSet{ - { - PrefixSetName: "podcidrprefixset", - PrefixList: []config.Prefix{ - { - IpPrefix: "172.20.0.0/24", - MasklengthRange: "24..24", - }, - }, - }, - }, - NeighborSets: []config.NeighborSet{}, - TagSets: []config.TagSet{}, - BgpDefinedSets: config.BgpDefinedSets{}, - }, - &config.DefinedSets{ - PrefixSets: []config.PrefixSet{ - { - PrefixSetName: "clusteripprefixset", - PrefixList: []config.Prefix{ - { - IpPrefix: "1.1.1.1/32", - MasklengthRange: "32..32", - }, - { - IpPrefix: "10.0.0.1/32", - MasklengthRange: "32..32", - }, - }, - }, - }, - NeighborSets: []config.NeighborSet{}, - TagSets: []config.TagSet{}, - BgpDefinedSets: config.BgpDefinedSets{}, - }, - &config.DefinedSets{}, - &config.DefinedSets{ - PrefixSets: []config.PrefixSet{}, - NeighborSets: []config.NeighborSet{ - { - NeighborSetName: "allpeerset", - NeighborInfoList: []string{}, - }, - }, - TagSets: []config.TagSet{}, - BgpDefinedSets: config.BgpDefinedSets{}, - }, - []*config.Statement{ - { - Name: "kube_router_export_stmt0", - Conditions: config.Conditions{ - MatchPrefixSet: config.MatchPrefixSet{ - PrefixSet: "podcidrprefixset", - MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, - }, - MatchNeighborSet: config.MatchNeighborSet{ - NeighborSet: "iBGPpeerset", - MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, - }, - }, - Actions: config.Actions{ - RouteDisposition: config.ROUTE_DISPOSITION_ACCEPT_ROUTE, - }, - }, - }, - []*config.Statement{ - { - Name: "kube_router_import_stmt0", - Conditions: config.Conditions{ - MatchPrefixSet: config.MatchPrefixSet{ - PrefixSet: "clusteripprefixset", - MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, - }, - MatchNeighborSet: config.MatchNeighborSet{ - NeighborSet: "allpeerset", - MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, - }, - }, - Actions: config.Actions{ - RouteDisposition: config.ROUTE_DISPOSITION_REJECT_ROUTE, - }, - }, - }, - nil, - }, - { - "has nodes, services with external peers", - &NetworkRoutingController{ - clientset: fake.NewSimpleClientset(), - hostnameOverride: "node-1", - bgpFullMeshMode: false, - bgpEnableInternal: true, - bgpServer: gobgp.NewBgpServer(), - activeNodes: make(map[string]bool), - globalPeerRouters: []*config.Neighbor{ - { - Config: config.NeighborConfig{NeighborAddress: "10.10.0.1"}, - }, - { - Config: config.NeighborConfig{NeighborAddress: "10.10.0.2"}, - }, - }, - nodeAsnNumber: 100, - }, - []*v1core.Node{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "node-1", - Annotations: map[string]string{ - "kube-router.io/node.asn": "100", - }, - }, - Status: v1core.NodeStatus{ - Addresses: []v1core.NodeAddress{ - { - Type: v1core.NodeInternalIP, - Address: "10.0.0.1", - }, - }, - }, - Spec: v1core.NodeSpec{ - PodCIDR: "172.20.0.0/24", - }, - }, - }, - []*v1core.Service{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-1", - }, - Spec: v1core.ServiceSpec{ - Type: "ClusterIP", - ClusterIP: "10.0.0.1", - ExternalIPs: []string{"1.1.1.1"}, - }, - }, - }, - &config.DefinedSets{ - PrefixSets: []config.PrefixSet{ - { - PrefixSetName: "podcidrprefixset", - PrefixList: []config.Prefix{ - { - IpPrefix: "172.20.0.0/24", - MasklengthRange: "24..24", - }, - }, - }, - }, - NeighborSets: []config.NeighborSet{}, - TagSets: []config.TagSet{}, - BgpDefinedSets: config.BgpDefinedSets{}, - }, - &config.DefinedSets{ - PrefixSets: []config.PrefixSet{ - { - PrefixSetName: "clusteripprefixset", - PrefixList: []config.Prefix{ - { - IpPrefix: "1.1.1.1/32", - MasklengthRange: "32..32", - }, - { - IpPrefix: "10.0.0.1/32", - MasklengthRange: "32..32", - }, - }, - }, - }, - NeighborSets: []config.NeighborSet{}, - TagSets: []config.TagSet{}, - BgpDefinedSets: config.BgpDefinedSets{}, - }, - &config.DefinedSets{ - PrefixSets: []config.PrefixSet{}, - NeighborSets: []config.NeighborSet{ - { - NeighborSetName: "externalpeerset", - NeighborInfoList: []string{"10.10.0.1/32", "10.10.0.2/32"}, - }, - }, - TagSets: []config.TagSet{}, - BgpDefinedSets: config.BgpDefinedSets{}, - }, - &config.DefinedSets{ - PrefixSets: []config.PrefixSet{}, - NeighborSets: []config.NeighborSet{ - { - NeighborSetName: "allpeerset", - NeighborInfoList: []string{"10.10.0.1/32", "10.10.0.2/32"}, - }, - }, - TagSets: []config.TagSet{}, - BgpDefinedSets: config.BgpDefinedSets{}, - }, - []*config.Statement{ - { - Name: "kube_router_export_stmt0", - Conditions: config.Conditions{ - MatchPrefixSet: config.MatchPrefixSet{ - PrefixSet: "podcidrprefixset", - MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, - }, - MatchNeighborSet: config.MatchNeighborSet{ - NeighborSet: "iBGPpeerset", - MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, - }, - }, - Actions: config.Actions{ - RouteDisposition: config.ROUTE_DISPOSITION_ACCEPT_ROUTE, - }, - }, - { - Name: "kube_router_export_stmt1", - Conditions: config.Conditions{ - MatchPrefixSet: config.MatchPrefixSet{ - PrefixSet: "clusteripprefixset", - MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, - }, - MatchNeighborSet: config.MatchNeighborSet{ - NeighborSet: "externalpeerset", - MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, - }, - }, - Actions: config.Actions{ - RouteDisposition: config.ROUTE_DISPOSITION_ACCEPT_ROUTE, - }, - }, - }, - []*config.Statement{ - { - Name: "kube_router_import_stmt0", - Conditions: config.Conditions{ - MatchPrefixSet: config.MatchPrefixSet{ - PrefixSet: "clusteripprefixset", - MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, - }, - MatchNeighborSet: config.MatchNeighborSet{ - NeighborSet: "allpeerset", - MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, - }, - }, - Actions: config.Actions{ - RouteDisposition: config.ROUTE_DISPOSITION_REJECT_ROUTE, - }, - }, - }, - nil, - }, - { - "has nodes, services with external peers and iBGP disabled", - &NetworkRoutingController{ - clientset: fake.NewSimpleClientset(), - hostnameOverride: "node-1", - bgpFullMeshMode: false, - bgpEnableInternal: false, - bgpServer: gobgp.NewBgpServer(), - activeNodes: make(map[string]bool), - globalPeerRouters: []*config.Neighbor{ - { - Config: config.NeighborConfig{NeighborAddress: "10.10.0.1"}, - }, - { - Config: config.NeighborConfig{NeighborAddress: "10.10.0.2"}, - }, - }, - nodeAsnNumber: 100, - }, - []*v1core.Node{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "node-1", - Annotations: map[string]string{ - "kube-router.io/node.asn": "100", - }, - }, - Status: v1core.NodeStatus{ - Addresses: []v1core.NodeAddress{ - { - Type: v1core.NodeInternalIP, - Address: "10.0.0.1", - }, - }, - }, - Spec: v1core.NodeSpec{ - PodCIDR: "172.20.0.0/24", - }, - }, - }, - []*v1core.Service{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-1", - }, - Spec: v1core.ServiceSpec{ - Type: "ClusterIP", - ClusterIP: "10.0.0.1", - ExternalIPs: []string{"1.1.1.1"}, - }, - }, - }, - &config.DefinedSets{ - PrefixSets: []config.PrefixSet{ - { - PrefixSetName: "podcidrprefixset", - PrefixList: []config.Prefix{ - { - IpPrefix: "172.20.0.0/24", - MasklengthRange: "24..24", - }, - }, - }, - }, - NeighborSets: []config.NeighborSet{}, - TagSets: []config.TagSet{}, - BgpDefinedSets: config.BgpDefinedSets{}, - }, - &config.DefinedSets{ - PrefixSets: []config.PrefixSet{ - { - PrefixSetName: "clusteripprefixset", - PrefixList: []config.Prefix{ - { - IpPrefix: "1.1.1.1/32", - MasklengthRange: "32..32", - }, - { - IpPrefix: "10.0.0.1/32", - MasklengthRange: "32..32", - }, - }, - }, - }, - NeighborSets: []config.NeighborSet{}, - TagSets: []config.TagSet{}, - BgpDefinedSets: config.BgpDefinedSets{}, - }, - &config.DefinedSets{ - PrefixSets: []config.PrefixSet{}, - NeighborSets: []config.NeighborSet{ - { - NeighborSetName: "externalpeerset", - NeighborInfoList: []string{"10.10.0.1/32", "10.10.0.2/32"}, - }, - }, - TagSets: []config.TagSet{}, - BgpDefinedSets: config.BgpDefinedSets{}, - }, - &config.DefinedSets{ - PrefixSets: []config.PrefixSet{}, - NeighborSets: []config.NeighborSet{ - { - NeighborSetName: "allpeerset", - NeighborInfoList: []string{"10.10.0.1/32", "10.10.0.2/32"}, - }, - }, - TagSets: []config.TagSet{}, - BgpDefinedSets: config.BgpDefinedSets{}, - }, - []*config.Statement{ - { - Name: "kube_router_export_stmt0", - Conditions: config.Conditions{ - MatchPrefixSet: config.MatchPrefixSet{ - PrefixSet: "clusteripprefixset", - MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, - }, - MatchNeighborSet: config.MatchNeighborSet{ - NeighborSet: "externalpeerset", - MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, - }, - }, - Actions: config.Actions{ - RouteDisposition: config.ROUTE_DISPOSITION_ACCEPT_ROUTE, - }, - }, - }, - []*config.Statement{ - { - Name: "kube_router_import_stmt0", - Conditions: config.Conditions{ - MatchPrefixSet: config.MatchPrefixSet{ - PrefixSet: "clusteripprefixset", - MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, - }, - MatchNeighborSet: config.MatchNeighborSet{ - NeighborSet: "allpeerset", - MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, - }, - }, - Actions: config.Actions{ - RouteDisposition: config.ROUTE_DISPOSITION_REJECT_ROUTE, - }, - }, - }, - nil, - }, - { - "prepends AS with external peers", - &NetworkRoutingController{ - clientset: fake.NewSimpleClientset(), - hostnameOverride: "node-1", - bgpEnableInternal: true, - bgpFullMeshMode: false, - pathPrepend: true, - pathPrependCount: 5, - pathPrependAS: "65100", - bgpServer: gobgp.NewBgpServer(), - activeNodes: make(map[string]bool), - globalPeerRouters: []*config.Neighbor{ - { - Config: config.NeighborConfig{NeighborAddress: "10.10.0.1"}, - }, - { - Config: config.NeighborConfig{NeighborAddress: "10.10.0.2"}, - }, - }, - nodeAsnNumber: 100, - }, - []*v1core.Node{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "node-1", - Annotations: map[string]string{ - "kube-router.io/node.asn": "100", - }, - }, - Status: v1core.NodeStatus{ - Addresses: []v1core.NodeAddress{ - { - Type: v1core.NodeInternalIP, - Address: "10.0.0.1", - }, - }, - }, - Spec: v1core.NodeSpec{ - PodCIDR: "172.20.0.0/24", - }, - }, - }, - []*v1core.Service{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-1", - }, - Spec: v1core.ServiceSpec{ - Type: "ClusterIP", - ClusterIP: "10.0.0.1", - ExternalIPs: []string{"1.1.1.1"}, - }, - }, - }, - &config.DefinedSets{ - PrefixSets: []config.PrefixSet{ - { - PrefixSetName: "podcidrprefixset", - PrefixList: []config.Prefix{ - { - IpPrefix: "172.20.0.0/24", - MasklengthRange: "24..24", - }, - }, - }, - }, - NeighborSets: []config.NeighborSet{}, - TagSets: []config.TagSet{}, - BgpDefinedSets: config.BgpDefinedSets{}, - }, - &config.DefinedSets{ - PrefixSets: []config.PrefixSet{ - { - PrefixSetName: "clusteripprefixset", - PrefixList: []config.Prefix{ - { - IpPrefix: "1.1.1.1/32", - MasklengthRange: "32..32", - }, - { - IpPrefix: "10.0.0.1/32", - MasklengthRange: "32..32", - }, - }, - }, - }, - NeighborSets: []config.NeighborSet{}, - TagSets: []config.TagSet{}, - BgpDefinedSets: config.BgpDefinedSets{}, - }, - &config.DefinedSets{ - PrefixSets: []config.PrefixSet{}, - NeighborSets: []config.NeighborSet{ - { - NeighborSetName: "externalpeerset", - NeighborInfoList: []string{"10.10.0.1/32", "10.10.0.2/32"}, - }, - }, - TagSets: []config.TagSet{}, - BgpDefinedSets: config.BgpDefinedSets{}, - }, - &config.DefinedSets{ - PrefixSets: []config.PrefixSet{}, - NeighborSets: []config.NeighborSet{ - { - NeighborSetName: "allpeerset", - NeighborInfoList: []string{"10.10.0.1/32", "10.10.0.2/32"}, - }, - }, - TagSets: []config.TagSet{}, - BgpDefinedSets: config.BgpDefinedSets{}, - }, - []*config.Statement{ - { - Name: "kube_router_export_stmt0", - Conditions: config.Conditions{ - MatchPrefixSet: config.MatchPrefixSet{ - PrefixSet: "podcidrprefixset", - MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, - }, - MatchNeighborSet: config.MatchNeighborSet{ - NeighborSet: "iBGPpeerset", - MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, - }, - }, - Actions: config.Actions{ - RouteDisposition: config.ROUTE_DISPOSITION_ACCEPT_ROUTE, - }, - }, - { - Name: "kube_router_export_stmt1", - Conditions: config.Conditions{ - MatchPrefixSet: config.MatchPrefixSet{ - PrefixSet: "clusteripprefixset", - MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, - }, - MatchNeighborSet: config.MatchNeighborSet{ - NeighborSet: "externalpeerset", - MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, - }, - }, - Actions: config.Actions{ - RouteDisposition: config.ROUTE_DISPOSITION_ACCEPT_ROUTE, - BgpActions: config.BgpActions{ - SetAsPathPrepend: config.SetAsPathPrepend{ - As: "65100", - RepeatN: 5, - }, - }, - }, - }, - }, - []*config.Statement{ - { - Name: "kube_router_import_stmt0", - Conditions: config.Conditions{ - MatchPrefixSet: config.MatchPrefixSet{ - PrefixSet: "clusteripprefixset", - MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, - }, - MatchNeighborSet: config.MatchNeighborSet{ - NeighborSet: "allpeerset", - MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, - }, - }, - Actions: config.Actions{ - RouteDisposition: config.ROUTE_DISPOSITION_REJECT_ROUTE, - }, - }, - }, - nil, - }, - { - "only prepends AS when both node annotations are present", - &NetworkRoutingController{ - clientset: fake.NewSimpleClientset(), - hostnameOverride: "node-1", - bgpEnableInternal: true, - bgpFullMeshMode: false, - pathPrepend: false, - pathPrependAS: "65100", - bgpServer: gobgp.NewBgpServer(), - activeNodes: make(map[string]bool), - globalPeerRouters: []*config.Neighbor{ - { - Config: config.NeighborConfig{NeighborAddress: "10.10.0.1"}, - }, - { - Config: config.NeighborConfig{NeighborAddress: "10.10.0.2"}, - }, - }, - nodeAsnNumber: 100, - }, - []*v1core.Node{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "node-1", - Annotations: map[string]string{ - "kube-router.io/node.asn": "100", - }, - }, - Status: v1core.NodeStatus{ - Addresses: []v1core.NodeAddress{ - { - Type: v1core.NodeInternalIP, - Address: "10.0.0.1", - }, - }, - }, - Spec: v1core.NodeSpec{ - PodCIDR: "172.20.0.0/24", - }, - }, - }, - []*v1core.Service{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "svc-1", - }, - Spec: v1core.ServiceSpec{ - Type: "ClusterIP", - ClusterIP: "10.0.0.1", - ExternalIPs: []string{"1.1.1.1"}, - }, - }, - }, - &config.DefinedSets{ - PrefixSets: []config.PrefixSet{ - { - PrefixSetName: "podcidrprefixset", - PrefixList: []config.Prefix{ - { - IpPrefix: "172.20.0.0/24", - MasklengthRange: "24..24", - }, - }, - }, - }, - NeighborSets: []config.NeighborSet{}, - TagSets: []config.TagSet{}, - BgpDefinedSets: config.BgpDefinedSets{}, - }, - &config.DefinedSets{ - PrefixSets: []config.PrefixSet{ - { - PrefixSetName: "clusteripprefixset", - PrefixList: []config.Prefix{ - { - IpPrefix: "1.1.1.1/32", - MasklengthRange: "32..32", - }, - { - IpPrefix: "10.0.0.1/32", - MasklengthRange: "32..32", - }, - }, - }, - }, - NeighborSets: []config.NeighborSet{}, - TagSets: []config.TagSet{}, - BgpDefinedSets: config.BgpDefinedSets{}, - }, - &config.DefinedSets{ - PrefixSets: []config.PrefixSet{}, - NeighborSets: []config.NeighborSet{ - { - NeighborSetName: "externalpeerset", - NeighborInfoList: []string{"10.10.0.1/32", "10.10.0.2/32"}, - }, - }, - TagSets: []config.TagSet{}, - BgpDefinedSets: config.BgpDefinedSets{}, - }, - &config.DefinedSets{ - PrefixSets: []config.PrefixSet{}, - NeighborSets: []config.NeighborSet{ - { - NeighborSetName: "allpeerset", - NeighborInfoList: []string{"10.10.0.1/32", "10.10.0.2/32"}, - }, - }, - TagSets: []config.TagSet{}, - BgpDefinedSets: config.BgpDefinedSets{}, - }, - []*config.Statement{ - { - Name: "kube_router_export_stmt0", - Conditions: config.Conditions{ - MatchPrefixSet: config.MatchPrefixSet{ - PrefixSet: "podcidrprefixset", - MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, - }, - MatchNeighborSet: config.MatchNeighborSet{ - NeighborSet: "iBGPpeerset", - MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, - }, - }, - Actions: config.Actions{ - RouteDisposition: config.ROUTE_DISPOSITION_ACCEPT_ROUTE, - }, - }, - { - Name: "kube_router_export_stmt1", - Conditions: config.Conditions{ - MatchPrefixSet: config.MatchPrefixSet{ - PrefixSet: "clusteripprefixset", - MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, - }, - MatchNeighborSet: config.MatchNeighborSet{ - NeighborSet: "externalpeerset", - MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, - }, - }, - Actions: config.Actions{ - RouteDisposition: config.ROUTE_DISPOSITION_ACCEPT_ROUTE, - }, - }, - }, - []*config.Statement{ - { - Name: "kube_router_import_stmt0", - Conditions: config.Conditions{ - MatchPrefixSet: config.MatchPrefixSet{ - PrefixSet: "clusteripprefixset", - MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, - }, - MatchNeighborSet: config.MatchNeighborSet{ - NeighborSet: "allpeerset", - MatchSetOptions: config.MATCH_SET_OPTIONS_RESTRICTED_TYPE_ANY, - }, - }, - Actions: config.Actions{ - RouteDisposition: config.ROUTE_DISPOSITION_REJECT_ROUTE, - }, - }, - }, - nil, - }, - } - - for _, testcase := range testcases { - t.Run(testcase.name, func(t *testing.T) { - go testcase.nrc.bgpServer.Serve() - err := testcase.nrc.bgpServer.Start(&config.Global{ - Config: config.GlobalConfig{ - As: 1, - RouterId: "10.0.0.0", - Port: 10000, - }, - }) - if err != nil { - t.Fatalf("failed to start BGP server: %v", err) - } - defer testcase.nrc.bgpServer.Stop() - - startInformersForRoutes(testcase.nrc, testcase.nrc.clientset) - - if err = createNodes(testcase.nrc.clientset, testcase.existingNodes); err != nil { - t.Errorf("failed to create existing nodes: %v", err) - } - - if err = createServices(testcase.nrc.clientset, testcase.existingServices); err != nil { - t.Errorf("failed to create existing nodes: %v", err) - } - - // ClusterIPs and ExternalIPs - waitForListerWithTimeout(testcase.nrc.svcLister, time.Second*10, t) - - testcase.nrc.advertiseClusterIP = true - testcase.nrc.advertiseExternalIP = true - testcase.nrc.advertiseLoadBalancerIP = false - - informerFactory := informers.NewSharedInformerFactory(testcase.nrc.clientset, 0) - nodeInformer := informerFactory.Core().V1().Nodes().Informer() - testcase.nrc.nodeLister = nodeInformer.GetIndexer() - err = testcase.nrc.AddPolicies() - if !reflect.DeepEqual(err, testcase.err) { - t.Logf("expected err %v", testcase.err) - t.Logf("actual err %v", err) - t.Error("unexpected error") - } - - podDefinedSet, err := testcase.nrc.bgpServer.GetDefinedSet(table.DEFINED_TYPE_PREFIX, "podcidrprefixset") - if err != nil { - t.Fatalf("error validating defined sets: %v", err) - } - - if !podDefinedSet.Equal(testcase.podDefinedSet) { - t.Logf("expected pod defined set: %+v", testcase.podDefinedSet.PrefixSets) - t.Logf("actual pod defined set: %+v", podDefinedSet.PrefixSets) - t.Error("unexpected pod defined set") - } - - clusterIPDefinedSet, err := testcase.nrc.bgpServer.GetDefinedSet(table.DEFINED_TYPE_PREFIX, "clusteripprefixset") - if err != nil { - t.Fatalf("error validating defined sets: %v", err) - } - - if !clusterIPDefinedSet.Equal(testcase.clusterIPDefinedSet) { - t.Logf("expected cluster ip defined set: %+v", testcase.clusterIPDefinedSet.PrefixSets) - t.Logf("actual cluster ip defined set: %+v", clusterIPDefinedSet.PrefixSets) - t.Error("unexpected cluster ip defined set") - } - - externalPeerDefinedSet, err := testcase.nrc.bgpServer.GetDefinedSet(table.DEFINED_TYPE_NEIGHBOR, "externalpeerset") - if err != nil { - t.Fatalf("error validating defined sets: %v", err) - } - - if !externalPeerDefinedSet.Equal(testcase.externalPeerDefinedSet) { - t.Logf("expected external peer defined set: %+v", testcase.externalPeerDefinedSet.NeighborSets) - t.Logf("actual external peer defined set: %+v", externalPeerDefinedSet.NeighborSets) - t.Error("unexpected external peer defined set") - } - - allPeerDefinedSet, err := testcase.nrc.bgpServer.GetDefinedSet(table.DEFINED_TYPE_NEIGHBOR, "allpeerset") - if err != nil { - t.Fatalf("error validating defined sets: %v", err) - } - - if !allPeerDefinedSet.Equal(testcase.allPeerDefinedSet) { - t.Logf("expected all peer defined set: %+v", testcase.allPeerDefinedSet.NeighborSets) - t.Logf("actual all peer defined set: %+v", allPeerDefinedSet.NeighborSets) - t.Error("unexpected all peer defined set") - } - - checkPolicies(t, testcase, table.POLICY_DIRECTION_EXPORT, table.ROUTE_TYPE_REJECT, testcase.exportPolicyStatements) - checkPolicies(t, testcase, table.POLICY_DIRECTION_IMPORT, table.ROUTE_TYPE_ACCEPT, testcase.importPolicyStatements) - }) - } -} - -func checkPolicies(t *testing.T, testcase PolicyTestCase, direction table.PolicyDirection, defaultPolicy table.RouteType, - policyStatements []*config.Statement) { - policies := testcase.nrc.bgpServer.GetPolicy() - policyExists := false - for _, policy := range policies { - if policy.Name == "kube_router_"+direction.String() { - policyExists = true - break - } - } - if !policyExists { - t.Errorf("policy 'kube_router_%v' was not added", direction) - } - - routeType, policyAssignments, err := testcase.nrc.bgpServer.GetPolicyAssignment("", direction) - if routeType != defaultPolicy { - t.Errorf("expected route type '%v' for %v policy assignment, but got %v", defaultPolicy, direction, routeType) - } - if err != nil { - t.Fatalf("failed to get policy assignments: %v", err) - } - - policyAssignmentExists := false - for _, policyAssignment := range policyAssignments { - if policyAssignment.Name == "kube_router_"+direction.String() { - policyAssignmentExists = true - } - } - - if !policyAssignmentExists { - t.Errorf("export policy assignment 'kube_router_%v' was not added", direction) - } - - statements := testcase.nrc.bgpServer.GetStatement() - for _, expectedStatement := range policyStatements { - found := false - for _, statement := range statements { - if reflect.DeepEqual(statement, expectedStatement) { - found = true - } - } - - if !found { - t.Errorf("statement %v not found", expectedStatement) - } - } -} - -func Test_generateTunnelName(t *testing.T) { - testcases := []struct { - name string - nodeIP string - tunnelName string - }{ - { - "IP less than 12 characters after removing '.'", - "10.0.0.1", - "tun-10001", - }, - { - "IP has 12 characters after removing '.'", - "100.200.300.400", - "tun100200300400", - }, - } - - for _, testcase := range testcases { - t.Run(testcase.name, func(t *testing.T) { - tunnelName := generateTunnelName(testcase.nodeIP) - if tunnelName != testcase.tunnelName { - t.Logf("actual tunnel interface name: %s", tunnelName) - t.Logf("expected tunnel interface name: %s", testcase.tunnelName) - t.Error("did not get expected tunnel interface name") - } - }) - } -} - -func createServices(clientset kubernetes.Interface, svcs []*v1core.Service) error { - for _, svc := range svcs { - _, err := clientset.CoreV1().Services("default").Create(svc) - if err != nil { - return err - } - } - - return nil -} - -func createNodes(clientset kubernetes.Interface, nodes []*v1core.Node) error { - for _, node := range nodes { - _, err := clientset.CoreV1().Nodes().Create(node) - if err != nil { - return err - } - } - - return nil -} - -func startInformersForRoutes(nrc *NetworkRoutingController, clientset kubernetes.Interface) { - informerFactory := informers.NewSharedInformerFactory(clientset, 0) - svcInformer := informerFactory.Core().V1().Services().Informer() - epInformer := informerFactory.Core().V1().Endpoints().Informer() - - go informerFactory.Start(nil) - informerFactory.WaitForCacheSync(nil) - - nrc.svcLister = svcInformer.GetIndexer() - nrc.epLister = epInformer.GetIndexer() -} - -func waitForListerWithTimeout(lister cache.Indexer, timeout time.Duration, t *testing.T) { - tick := time.Tick(100 * time.Millisecond) - timeoutCh := time.After(timeout) - for { - select { - case <-timeoutCh: - t.Fatal("timeout exceeded waiting for service lister to fill cache") - case <-tick: - if len(lister.List()) != 0 { - return - } - } - } -} - -func waitForBGPWatchEventWithTimeout(timeout time.Duration, expectedNumEvents int, w *gobgp.Watcher, t *testing.T) []*gobgp.WatchEventBestPath { - timeoutCh := time.After(timeout) - var events []*gobgp.WatchEventBestPath - for { - select { - case <-timeoutCh: - t.Fatalf("timeout exceeded waiting for %d watch events, got %d", expectedNumEvents, len(events)) - case event := <-w.Event(): - events = append(events, event.(*gobgp.WatchEventBestPath)) - default: - if len(events) == expectedNumEvents { - return events - } - } - } -} - -func ptrToString(str string) *string { - return &str -} diff --git a/pkg/options/options.go b/pkg/options/options.go index bd3cf5184b..a5f355dbd1 100644 --- a/pkg/options/options.go +++ b/pkg/options/options.go @@ -12,66 +12,65 @@ import ( const DEFAULT_BGP_PORT = 179 type KubeRouterConfig struct { - AdvertiseClusterIp bool - AdvertiseExternalIp bool - AdvertiseNodePodCidr bool - AdvertiseLoadBalancerIp bool - BGPGracefulRestart bool - BGPGracefulRestartDeferralTime time.Duration - BGPPort uint16 - CacheSyncTimeout time.Duration - CleanupConfig bool - ClusterAsn uint - ClusterCIDR string - DisableSrcDstCheck bool - EnableCNI bool - EnableiBGP bool - EnableOverlay bool - EnablePodEgress bool - EnablePprof bool - FullMeshMode bool - OverlayType string - GlobalHairpinMode bool - HealthPort uint16 - HelpRequested bool - HostnameOverride string - IPTablesSyncPeriod time.Duration - IpvsSyncPeriod time.Duration - IpvsGracefulPeriod time.Duration - IpvsGracefulTermination bool - Kubeconfig string - MasqueradeAll bool - Master string - MetricsEnabled bool - MetricsPath string - MetricsPort uint16 - NodePortBindOnAllIp bool - OverrideNextHop bool - PeerASNs []uint - PeerMultihopTtl uint8 - PeerPasswords []string - PeerPorts []uint - PeerRouters []net.IP - RouterId string - RoutesSyncPeriod time.Duration - RunFirewall bool - RunRouter bool - RunServiceProxy bool - Version bool - VLevel string + AdvertiseClusterIp bool + AdvertiseClusterSubnet string + AdvertiseExternalIp bool + AdvertiseNodePodCidr bool + AdvertiseLoadBalancerIp bool + BGPGracefulRestart bool + BGPPort uint16 + CacheSyncTimeout time.Duration + CleanupConfig bool + ClusterAsn uint + ClusterCIDR string + DisableSrcDstCheck bool + EnableCNI bool + EnableiBGP bool + EnableOverlay bool + EnablePodEgress bool + EnablePprof bool + FullMeshMode bool + OverlayType string + GlobalHairpinMode bool + HealthPort uint16 + HelpRequested bool + HostnameOverride string + IPTablesSyncPeriod time.Duration + IpvsSyncPeriod time.Duration + IpvsGracefulPeriod time.Duration + IpvsGracefulTermination bool + Kubeconfig string + MasqueradeAll bool + Master string + MetricsEnabled bool + MetricsPath string + MetricsPort uint16 + NodePortBindOnAllIp bool + OverrideNextHop bool + PeerASNs []uint + PeerMultihopTtl uint8 + PeerPasswords []string + PeerPorts []uint + PeerRouters []net.IP + RouterId string + RoutesSyncPeriod time.Duration + RunFirewall bool + RunRouter bool + RunServiceProxy bool + Version bool + VLevel string // FullMeshPassword string } func NewKubeRouterConfig() *KubeRouterConfig { return &KubeRouterConfig{ - CacheSyncTimeout: 1 * time.Minute, - IpvsSyncPeriod: 5 * time.Minute, - IPTablesSyncPeriod: 5 * time.Minute, - IpvsGracefulPeriod: 30 * time.Second, - RoutesSyncPeriod: 5 * time.Minute, - BGPGracefulRestartDeferralTime: 360 * time.Second, - EnableOverlay: true, - OverlayType: "subnet", + CacheSyncTimeout: 1 * time.Minute, + IpvsSyncPeriod: 5 * time.Minute, + IPTablesSyncPeriod: 5 * time.Minute, + IpvsGracefulPeriod: 30 * time.Second, + RoutesSyncPeriod: 5 * time.Minute, + EnableOverlay: true, + OverlayType: "subnet", } } @@ -112,6 +111,8 @@ func (s *KubeRouterConfig) AddFlags(fs *pflag.FlagSet) { "The delay between route updates and advertisements (e.g. '5s', '1m', '2h22m'). Must be greater than 0.") fs.BoolVar(&s.AdvertiseClusterIp, "advertise-cluster-ip", false, "Add Cluster IP of the service to the RIB so that it gets advertises to the BGP peers.") + fs.StringVar(&s.AdvertiseClusterSubnet, "advertise-cluster-subnet", s.AdvertiseClusterSubnet, + "If this parameter is set, the cluster IP of service will not be put into RIB, but the value set by this parameter will be put into RIB. The main purpose of this parameter is to reduce the number of service routes sent by kube-router to the upper BGP network devices.") fs.BoolVar(&s.AdvertiseExternalIp, "advertise-external-ip", false, "Add External IP of service to the RIB so that it gets advertised to the BGP peers.") fs.BoolVar(&s.AdvertiseLoadBalancerIp, "advertise-loadbalancer-ip", false, @@ -132,8 +133,6 @@ func (s *KubeRouterConfig) AddFlags(fs *pflag.FlagSet) { "Each node in the cluster will setup BGP peering with rest of the nodes.") fs.BoolVar(&s.BGPGracefulRestart, "bgp-graceful-restart", false, "Enables the BGP Graceful Restart capability so that routes are preserved on unexpected restarts") - fs.DurationVar(&s.BGPGracefulRestartDeferralTime, "bgp-graceful-restart-deferral-time", s.BGPGracefulRestartDeferralTime, - "BGP Graceful restart deferral time according to RFC4724 4.1, maximum 18h.") fs.Uint16Var(&s.BGPPort, "bgp-port", DEFAULT_BGP_PORT, "The port open for incoming BGP connections and to use for connecting with other BGP peers.") fs.StringVar(&s.RouterId, "router-id", "", "BGP router-id. Must be specified in a ipv6 only cluster.")