Deploy Calico cni with k8s-1.17 (#256)

* Deploy Calico pod network plugin on k8s-1.17 Flannel has compatability issues with k8s-1.17 flannel-io/flannel#1245. deploy calico plugin instead also for better proformance. calico.yaml file is copied from Calico's documantation and no change should be done to it. Signed-off-by: Or Mergi <[email protected]> * CNI manifest file names and kubernetes versions map This map will corrolate between k8s version and the plugin we would like to deploy. Signed-off-by: Or Mergi <[email protected]> * Separate cni selection logic from provision scripts. cli.sh, create /tmp/scripts directory in the VM and copy cni-map.sh . cnis-map.sh map between k8s version and cni manifest file name to use. node01.sh provision.sh, use cnis-map.sh to resolve the right cni manifest to use. Signed-off-by: Or Mergi <[email protected]>
kubevirt · Feb 3, 2020 · 7b63bc6 · 7b63bc6
1 parent 0c39117
commit 7b63bc6
Show file tree

Hide file tree

Showing 6 changed files with 831 additions and 20 deletions.
diff --git a/cluster-provision/cli/cli b/cluster-provision/cli/cli
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-set -e
+set -ex
 
 WITH_REGISTRY=false
 NODES=1
@@ -176,7 +176,6 @@ fi
 DNSMASQ_CID=$(docker run -d ${PORTS} -e NUM_NODES=${NODES} --name ${PREFIX}dnsmasq --privileged ${BASE} /bin/bash -c /dnsmasq.sh)
 CONTAINERS=${DNSMASQ_CID}
 
-
 if [ "$COMMAND" == "provision" ] ; then
   VM_CID=$(docker run -d --privileged --net=container:${DNSMASQ_CID} --name ${PREFIX}node01 ${BASE} /bin/bash -c "/vm.sh ${MEMORY} ${CPU} ${QEMU_ARGS}")
   CONTAINERS="${CONTAINERS} ${VM_CID}"
@@ -190,6 +189,8 @@ if [ "$COMMAND" == "provision" ] ; then
   docker exec ${VM_CID} /bin/bash -c "scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i vagrant.key -P 22 /manifests/ceph/*.yaml [email protected]:/tmp/ceph"
   docker exec ${VM_CID} /bin/bash -c "ssh.sh mkdir -p /tmp/cna"
   docker exec ${VM_CID} /bin/bash -c "scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i vagrant.key -P 22 /manifests/cna/*.yaml [email protected]:/tmp/cna"
+  docker exec ${VM_CID} /bin/bash -c "ssh.sh mkdir -p /tmp/scripts"
+  docker exec ${VM_CID} /bin/bash -c "scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i vagrant.key -P 22 /scripts/cnis-map.sh [email protected]:/tmp/scripts/cnis-map.sh"
   docker exec ${VM_CID} /bin/bash -c "ssh.sh sudo version=${K8S_VERSION} /bin/bash -s ${CRIO} < /scripts/provision.sh"
   docker exec ${VM_CID} ssh.sh "sudo shutdown -h"
   docker exec ${VM_CID} /bin/bash -c "rm /usr/local/bin/ssh.sh"

diff --git a/cluster-provision/k8s/scripts/cnis-map.sh b/cluster-provision/k8s/scripts/cnis-map.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -e
+
+declare -A CNI_MANIFESTS
+CNI_MANIFESTS[1.17.0]="calico.yaml"
+CNI_MANIFESTS[1.16.2]="flannel-ge-16.yaml"
+CNI_MANIFESTS[1.15.1]="flannel-ge-16.yaml"
+CNI_MANIFESTS[1.14.6]="flannel-ge-16.yaml"
+CNI_MANIFESTS[1.13.3]="flannel-ge-12.yaml"
+CNI_MANIFESTS[1.12.0]="flannel-ge-12.yaml"
+CNI_MANIFESTS[1.11.0]="flannel.yaml"
+CNI_MANIFESTS[1.10.11]="flannel.yaml"
+export CNI_MANIFESTS
diff --git a/cluster-provision/k8s/scripts/node01.sh b/cluster-provision/k8s/scripts/node01.sh
@@ -2,6 +2,12 @@
 
 set -ex
 
+source /tmp/scripts/cnis-map.sh
+
+version=`kubectl version --short --client | cut -d":" -f2 |sed  's/ //g' | cut -c2- `
+minor_version=`echo $version | cut -d"." -f2 `
+cni_manifest="/tmp/${CNI_MANIFESTS[$version]}"
+
 # Wait for docker, else network might not be ready yet
 while [[ `systemctl status docker | grep active | wc -l` -eq 0 ]]
 do
@@ -10,14 +16,8 @@ done
 
 kubeadm init --config /etc/kubernetes/kubeadm.conf
 
-version=`kubectl version --short --client | cut -d":" -f2 |sed  's/ //g' | cut -c2- | cut -d"." -f2`
-flannel_manifest="/tmp/flannel.yaml"
-if [[ $version -ge "16" ]]; then
-    flannel_manifest="/tmp/flannel-ge-16.yaml"
-elif [[ $version -ge "12" ]]; then
-    flannel_manifest="/tmp/flannel-ge-12.yaml"
-fi
-kubectl --kubeconfig=/etc/kubernetes/admin.conf create -f "$flannel_manifest"
+# cni manifest is already configured at provision stage.
+kubectl --kubeconfig=/etc/kubernetes/admin.conf create -f "$cni_manifest"
 
 kubectl --kubeconfig=/etc/kubernetes/admin.conf taint nodes node01 node-role.kubernetes.io/master:NoSchedule-
 
@@ -34,7 +34,7 @@ while [[ $retry_counter -lt 20 && $kubectl_rc -ne 0 ]]; do
 done
 
 local_volume_manifest="/tmp/local-volume.yaml"
-if [[ $version -ge "16" ]]; then
+if [[ $minor_version -ge "16" ]]; then
     local_volume_manifest="/tmp/local-volume-ge-16.yaml"
 fi
 kubectl --kubeconfig=/etc/kubernetes/admin.conf create -f "$local_volume_manifest"
diff --git a/cluster-provision/k8s/scripts/provision.sh b/cluster-provision/k8s/scripts/provision.sh
@@ -2,11 +2,15 @@
 
 set -ex
 
+source /tmp/scripts/cnis-map.sh
+
 function get_minor_version() {
     [[ $1 =~ \.([0-9]+) ]]
     echo ${BASH_REMATCH[1]}
 }
 
+cni_manifest="/tmp/${CNI_MANIFESTS[$version]}"
+
 minor_version=$(get_minor_version $version)
 
 setenforce 0
@@ -104,14 +108,12 @@ sysctl --system
 echo bridge >> /etc/modules
 echo br_netfilter >> /etc/modules
 
-kubeadm init --pod-network-cidr=10.244.0.0/16 --kubernetes-version v${version} --token abcdef.1234567890123456
-flannel_manifest="/tmp/flannel.yaml"
-if [[ $minor_version -ge "16" ]]; then
-    flannel_manifest="/tmp/flannel-ge-16.yaml"
-elif [[ $minor_version -ge "12" ]]; then
-    flannel_manifest="/tmp/flannel-ge-12.yaml"
-fi
-kubectl --kubeconfig=/etc/kubernetes/admin.conf create -f "$flannel_manifest"
+default_cidr="192.168.0.0/16"
+pod_cidr="10.244.0.0/16"
+kubeadm init --pod-network-cidr=$pod_cidr --kubernetes-version v${version} --token abcdef.1234567890123456
+
+sed -i -e "s?$default_cidr?$pod_cidr?g" $cni_manifest 
+kubectl --kubeconfig=/etc/kubernetes/admin.conf create -f "$cni_manifest"
 
 # Wait at least for 7 pods
 while [[ "$(kubectl --kubeconfig=/etc/kubernetes/admin.conf get pods -n kube-system --no-headers | wc -l)" -lt 7 ]]; do