From f53cee54b886e571209a5e68e09554fb666c6a0b Mon Sep 17 00:00:00 2001 From: song jiang Date: Tue, 5 Mar 2019 15:07:49 +0000 Subject: [PATCH] Update calico-daemonset.yaml --- ...bernetesmasteraddons-calico-daemonset.yaml | 929 +++++++++--------- ...bernetesmasteraddons-calico-daemonset.yaml | 929 +++++++++--------- ...bernetesmasteraddons-calico-daemonset.yaml | 929 +++++++++--------- ...bernetesmasteraddons-calico-daemonset.yaml | 929 +++++++++--------- ...bernetesmasteraddons-calico-daemonset.yaml | 929 +++++++++--------- ...bernetesmasteraddons-calico-daemonset.yaml | 927 ++++++++--------- parts/k8s/kubernetesinstalls.sh | 2 +- 7 files changed, 2869 insertions(+), 2705 deletions(-) diff --git a/parts/k8s/addons/1.10/kubernetesmasteraddons-calico-daemonset.yaml b/parts/k8s/addons/1.10/kubernetesmasteraddons-calico-daemonset.yaml index 29d80fabf38..f7889a7d98f 100644 --- a/parts/k8s/addons/1.10/kubernetesmasteraddons-calico-daemonset.yaml +++ b/parts/k8s/addons/1.10/kubernetesmasteraddons-calico-daemonset.yaml @@ -1,108 +1,8 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: calico-node - namespace: kube-system - labels: - addonmanager.kubernetes.io/mode: "EnsureExists" ---- - -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: calico-node - labels: - addonmanager.kubernetes.io/mode: "EnsureExists" -rules: - - apiGroups: [""] - resources: - - namespaces - - serviceaccounts - verbs: - - get - - list - - watch - - apiGroups: [""] - resources: - - pods/status - verbs: - - patch - - apiGroups: [""] - resources: - - pods - verbs: - - get - - list - - watch - - apiGroups: [""] - resources: - - services - verbs: - - get - - apiGroups: [""] - resources: - - endpoints - verbs: - - get - - apiGroups: [""] - resources: - - nodes - verbs: - - get - - list - - update - - watch - - apiGroups: ["extensions"] - resources: - - networkpolicies - verbs: - - get - - list - - watch - - apiGroups: ["networking.k8s.io"] - resources: - - networkpolicies - verbs: - - watch - - list - - apiGroups: ["crd.projectcalico.org"] - resources: - - globalfelixconfigs - - felixconfigurations - - bgppeers - - globalbgpconfigs - - bgpconfigurations - - ippools - - globalnetworkpolicies - - globalnetworksets - - networkpolicies - - clusterinformations - - hostendpoints - verbs: - - create - - get - - list - - update - - watch - ---- - -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: calico-node - labels: - addonmanager.kubernetes.io/mode: "EnsureExists" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-node -subjects: -- kind: ServiceAccount - name: calico-node - namespace: kube-system - ---- +# Calico Version v3.5.0 +# https://docs.projectcalico.org/v3.5/releases#v3.5.0 +# This manifest includes the following component versions: +# calico/node:v3.5.0 +# calico/cni:v3.5.0 # This ConfigMap is used to configure a self-hosted Calico installation. kind: ConfigMap @@ -113,9 +13,11 @@ metadata: labels: addonmanager.kubernetes.io/mode: "EnsureExists" data: + # You must set a non-zero value for Typha replicas below. typha_service_name: "calico-typha" - # The CNI network configuration to install on each node. + # The CNI network configuration to install on each node. The special + # values in this config will be automatically populated. cni_network_config: |- { "name": "k8s-pod-network", @@ -158,13 +60,12 @@ metadata: addonmanager.kubernetes.io/mode: "EnsureExists" spec: ports: - - port: 5473 - protocol: TCP - targetPort: calico-typha - name: calico-typha + - port: 5473 + protocol: TCP + targetPort: calico-typha + name: calico-typha selector: k8s-app: calico-typha - --- # This manifest creates a Deployment of Typha to back the above service. @@ -184,228 +85,77 @@ spec: # We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is essential # (when using the Kubernetes datastore). Use one replica for every 100-200 nodes. In # production, we recommend running at least 3 replicas to reduce the impact of rolling upgrade. - replicas: 0 + replicas: 1 revisionHistoryLimit: 2 template: metadata: labels: k8s-app: calico-typha - addonmanager.kubernetes.io/mode: "EnsureExists" annotations: # This, along with the CriticalAddonsOnly toleration below, marks the pod as a critical # add-on, ensuring it gets priority scheduling and that its resources are reserved # if it ever gets evicted. scheduler.alpha.kubernetes.io/critical-pod: '' + cluster-autoscaler.kubernetes.io/safe-to-evict: 'true' spec: - hostNetwork: true nodeSelector: beta.kubernetes.io/os: linux + hostNetwork: true tolerations: + # Mark the pod as a critical add-on for rescheduling. - key: CriticalAddonsOnly operator: Exists # Since Calico can't network a pod until Typha is up, we need to run Typha itself # as a host-networked pod. serviceAccountName: calico-node containers: - - image: quay.io/calico/typha:v3.3.1 + - image: calico/typha:v3.5.0 name: calico-typha ports: - containerPort: 5473 name: calico-typha protocol: TCP env: - # Enable "info" logging by default. Can be set to "debug" to increase verbosity. - - name: TYPHA_LOGSEVERITYSCREEN - value: "info" - # Disable logging to file and syslog since those don't make sense in Kubernetes. - - name: TYPHA_LOGFILEPATH - value: "none" - - name: TYPHA_LOGSEVERITYSYS - value: "none" - # Monitor the Kubernetes API to find the number of running instances and rebalance - # connections. - - name: TYPHA_CONNECTIONREBALANCINGMODE - value: "kubernetes" - - name: TYPHA_DATASTORETYPE - value: "kubernetes" - - name: TYPHA_HEALTHENABLED - value: "true" - # Uncomment these lines to enable prometheus metrics. Since Typha is host-networked, - # this opens a port on the host, which may need to be secured. - #- name: TYPHA_PROMETHEUSMETRICSENABLED - # value: "true" - #- name: TYPHA_PROMETHEUSMETRICSPORT - # value: "9093" + # Enable "info" logging by default. Can be set to "debug" to increase verbosity. + - name: TYPHA_LOGSEVERITYSCREEN + value: "info" + # Disable logging to file and syslog since those don't make sense in Kubernetes. + - name: TYPHA_LOGFILEPATH + value: "none" + - name: TYPHA_LOGSEVERITYSYS + value: "none" + # Monitor the Kubernetes API to find the number of running instances and rebalance + # connections. + - name: TYPHA_CONNECTIONREBALANCINGMODE + value: "kubernetes" + - name: TYPHA_DATASTORETYPE + value: "kubernetes" + - name: TYPHA_HEALTHENABLED + value: "true" + - name: FELIX_INTERFACEPREFIX + value: "azv" + # Uncomment these lines to enable prometheus metrics. Since Typha is host-networked, + # this opens a port on the host, which may need to be secured. + #- name: TYPHA_PROMETHEUSMETRICSENABLED + # value: "true" + #- name: TYPHA_PROMETHEUSMETRICSPORT + # value: "9093" livenessProbe: - httpGet: - path: /liveness - port: 9098 - host: localhost + exec: + command: + - calico-typha + - check + - liveness periodSeconds: 30 initialDelaySeconds: 30 readinessProbe: - httpGet: - path: /readiness - port: 9098 - host: localhost + exec: + command: + - calico-typha + - check + - readiness periodSeconds: 10 - ---- - -# Typha Horizontal Autoscaler Cluster Role -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: typha-cpha - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["list"] - ---- - -# Typha Horizontal Autoscaler Cluster Role Binding -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: typha-cpha - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: typha-cpha -subjects: - - kind: ServiceAccount - name: typha-cpha - namespace: kube-system - --- - -# Typha Horizontal Autoscaler ConfigMap -kind: ConfigMap -apiVersion: v1 -metadata: - name: calico-typha-horizontal-autoscaler - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -data: - ladder: |- - { - "coresToReplicas": [], - "nodesToReplicas": - [ - [1, 1], - [10, 2], - [100, 3], - [250, 4], - [500, 5], - [1000, 6], - [1500, 7], - [2000, 8] - ] - } - ---- - -# Typha Horizontal Autoscaler Deployment -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: calico-typha-horizontal-autoscaler - namespace: kube-system - labels: - k8s-app: calico-typha-autoscaler - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -spec: - replicas: 1 - template: - metadata: - labels: - k8s-app: calico-typha-autoscaler - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' - spec: - priorityClassName: system-node-critical - securityContext: - supplementalGroups: [ 65534 ] - fsGroup: 65534 - containers: - - image: k8s.gcr.io/cluster-proportional-autoscaler-amd64:1.1.2-r2 - name: autoscaler - command: - - /cluster-proportional-autoscaler - - --namespace=kube-system - - --configmap=calico-typha-horizontal-autoscaler - - --target=deployment/calico-typha - - --logtostderr=true - - --v=2 - resources: - requests: - cpu: 10m - limits: - cpu: 10m - serviceAccountName: typha-cpha - ---- - -# Typha Horizontal Autoscaler Role -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: typha-cpha - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -rules: - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get"] - - apiGroups: ["extensions"] - resources: ["deployments/scale"] - verbs: ["get", "update"] - ---- - -# Typha Horizontal Autoscaler Role Binding -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: typha-cpha - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: typha-cpha -subjects: - - kind: ServiceAccount - name: typha-cpha - namespace: kube-system - ---- - -# Typha Horizontal Autoscaler Service Account -apiVersion: v1 -kind: ServiceAccount -metadata: - name: typha-cpha - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - ---- - # This manifest installs the calico/node container, as well # as the Calico CNI plugins and network config on # each master and worker node in a Kubernetes cluster. @@ -416,7 +166,6 @@ metadata: namespace: kube-system labels: k8s-app: calico-node - # github.com/kubernetes/kubernetes/tree/master/cluster/addons/addon-manager addonmanager.kubernetes.io/mode: "EnsureExists" spec: selector: @@ -435,157 +184,173 @@ spec: # marks the pod as a critical add-on, ensuring it gets # priority scheduling and that its resources are reserved # if it ever gets evicted. - # Deprecated in 1.10, Removed in 1.11. kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods scheduler.alpha.kubernetes.io/critical-pod: '' spec: + nodeSelector: + beta.kubernetes.io/os: linux hostNetwork: true tolerations: - # Allow the pod to run on the master. This is required for - # the master to communicate with pods. - - key: node-role.kubernetes.io/master - operator: Equal - value: "true" - effect: NoSchedule - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - operator: Exists + # Make sure calico-node gets scheduled on all nodes. + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists serviceAccountName: calico-node # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. terminationGracePeriodSeconds: 0 + initContainers: + # This container installs the Calico CNI binaries + # and CNI network config file on each node. + - name: install-cni + image: calico/cni:v3.5.0 + command: ["/install-cni.sh"] + env: + # Name of the CNI config file to create. + - name: CNI_CONF_NAME + value: "10-calico.conflist" + # The CNI network config to install on each node. + - name: CNI_NETWORK_CONFIG + valueFrom: + configMapKeyRef: + name: calico-config + key: cni_network_config + # Set the hostname based on the k8s node name. + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # Prevents the container from sleeping forever. + - name: SLEEP + value: "false" + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + - mountPath: /host/etc/cni/net.d + name: cni-net-dir containers: # Runs calico/node container on each Kubernetes node. This # container programs network policy and routes on each # host. - - name: calico-node - image: quay.io/calico/node:v3.3.1 - env: - # Use Kubernetes API as the backing datastore. - - name: DATASTORE_TYPE - value: "kubernetes" - # Enable felix info logging. - - name: FELIX_LOGSEVERITYSCREEN - value: "info" - # Don't enable BGP. - - name: CALICO_NETWORKING_BACKEND - value: "none" - # Cluster type to identify the deployment type - - name: CLUSTER_TYPE - value: "k8s" - # Disable file logging so `kubectl logs` works. - - name: CALICO_DISABLE_FILE_LOGGING - value: "true" - # Set Felix endpoint to host default action to ACCEPT. - - name: FELIX_DEFAULTENDPOINTTOHOSTACTION - value: "ACCEPT" - # Disable IPV6 on Kubernetes. - - name: FELIX_IPV6SUPPORT - value: "false" - # Wait for the datastore. - - name: WAIT_FOR_DATASTORE - value: "true" - # The Calico IPv4 pool to use. This should match `--cluster-cidr` - - name: CALICO_IPV4POOL_CIDR - value: "" - # Enable IPIP - - name: CALICO_IPV4POOL_IPIP - value: "off" - - name: FELIX_IPTABLESREFRESHINTERVAL - value: "60" - # Typha support: controlled by the ConfigMap. - - name: FELIX_TYPHAK8SSERVICENAME - valueFrom: - configMapKeyRef: - name: calico-config - key: typha_service_name - # Set based on the k8s node name. - - name: NODENAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: FELIX_HEALTHENABLED - value: "true" - - name: FELIX_IPINIPENABLED - value: "false" - securityContext: - privileged: true - resources: - requests: - cpu: 250m - livenessProbe: - httpGet: - path: /liveness - port: 9099 - host: localhost - periodSeconds: 10 - initialDelaySeconds: 10 - failureThreshold: 6 - readinessProbe: - httpGet: - path: /readiness - port: 9099 - host: localhost - periodSeconds: 10 - volumeMounts: - - mountPath: /lib/modules - name: lib-modules - readOnly: true - - mountPath: /var/run/calico - name: var-run-calico - readOnly: false - - mountPath: /var/lib/calico - name: var-lib-calico - readOnly: false - # This container installs the Calico CNI binaries - # and CNI network config file on each node. - - name: install-cni - image: quay.io/calico/cni:v3.4.0-0.dev-34-g83daff2 - command: ["/install-cni.sh"] - env: - # Name of the CNI config file to create. - - name: CNI_CONF_NAME - value: "10-calico.conflist" - # The CNI network config to install on each node. - - name: CNI_NETWORK_CONFIG - valueFrom: - configMapKeyRef: - name: calico-config - key: cni_network_config - # Set the hostname based on the k8s node name. - - name: KUBERNETES_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - volumeMounts: - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - - mountPath: /host/etc/cni/net.d - name: cni-net-dir + - name: calico-node + image: calico/node:v3.5.0 + env: + # Use Kubernetes API as the backing datastore. + - name: DATASTORE_TYPE + value: "kubernetes" + # Typha support: controlled by the ConfigMap. + - name: FELIX_TYPHAK8SSERVICENAME + valueFrom: + configMapKeyRef: + name: calico-config + key: typha_service_name + # Wait for the datastore. + - name: WAIT_FOR_DATASTORE + value: "true" + # Set based on the k8s node name. + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # Don't enable BGP. + - name: CALICO_NETWORKING_BACKEND + value: "none" + # Cluster type to identify the deployment type + - name: CLUSTER_TYPE + value: "k8s" + # The default IPv4 pool to create on startup if none exists. Pod IPs will be + # chosen from this range. Changing this value after installation will have + # no effect. This should fall within `--cluster-cidr`. + - name: CALICO_IPV4POOL_CIDR + value: "" + # Disable file logging so `kubectl logs` works. + - name: CALICO_DISABLE_FILE_LOGGING + value: "true" + # Set Felix endpoint to host default action to ACCEPT. + - name: FELIX_DEFAULTENDPOINTTOHOSTACTION + value: "ACCEPT" + # Disable IPv6 on Kubernetes. + - name: FELIX_IPV6SUPPORT + value: "false" + # Set Felix logging to "info" + - name: FELIX_LOGSEVERITYSCREEN + value: "info" + - name: FELIX_HEALTHENABLED + value: "true" + - name: CALICO_IPV4POOL_IPIP + value: "off" + - name: FELIX_INTERFACEPREFIX + value: "azv" + securityContext: + privileged: true + resources: + requests: + cpu: 250m + livenessProbe: + httpGet: + path: /liveness + port: 9099 + host: localhost + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + readinessProbe: + exec: + command: + - /bin/calico-node + - -felix-ready + periodSeconds: 10 + volumeMounts: + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - mountPath: /var/run/calico + name: var-run-calico + readOnly: false + - mountPath: /var/lib/calico + name: var-lib-calico + readOnly: false volumes: - # Used by calico/node. - - name: lib-modules - hostPath: - path: /lib/modules - - name: var-run-calico - hostPath: - path: /var/run/calico - - name: var-lib-calico - hostPath: - path: /var/lib/calico - # Used to install CNI. - - name: cni-bin-dir - hostPath: - path: /opt/cni/bin - - name: cni-net-dir - hostPath: - path: /etc/cni/net.d + # Used by calico/node. + - name: lib-modules + hostPath: + path: /lib/modules + - name: var-run-calico + hostPath: + path: /var/run/calico + - name: var-lib-calico + hostPath: + path: /var/lib/calico + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + # Used to install CNI. + - name: cni-bin-dir + hostPath: + path: /opt/cni/bin + - name: cni-net-dir + hostPath: + path: /etc/cni/net.d +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-node + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: "EnsureExists" +--- # Create all the CustomResourceDefinitions needed for # Calico policy-only mode. ---- -# Calico Felix Configuration + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -600,9 +365,8 @@ spec: kind: FelixConfiguration plural: felixconfigurations singular: felixconfiguration - --- -# Calico BGP Configuration + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -617,9 +381,8 @@ spec: kind: BGPConfiguration plural: bgpconfigurations singular: bgpconfiguration - --- -# Calico IP Pools + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -634,9 +397,8 @@ spec: kind: IPPool plural: ippools singular: ippool - --- -# Calico Host Endpoints + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -651,9 +413,8 @@ spec: kind: HostEndpoint plural: hostendpoints singular: hostendpoint - --- -# Calico Cluster Information + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -668,9 +429,8 @@ spec: kind: ClusterInformation plural: clusterinformations singular: clusterinformation - --- -# Calico Global Network Policies + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -685,9 +445,8 @@ spec: kind: GlobalNetworkPolicy plural: globalnetworkpolicies singular: globalnetworkpolicy - --- -# Calico Global Network Sets + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -702,9 +461,8 @@ spec: kind: GlobalNetworkSet plural: globalnetworksets singular: globalnetworkset - --- -# Calico Network Policies + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -719,3 +477,272 @@ spec: kind: NetworkPolicy plural: networkpolicies singular: networkpolicy +--- + +# Include a clusterrole for the calico-node DaemonSet, +# and bind it to the calico-node serviceaccount. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: calico-node + labels: + addonmanager.kubernetes.io/mode: "EnsureExists" +rules: +# The CNI plugin needs to get pods, nodes, and namespaces. +- apiGroups: [""] + resources: + - pods + - nodes + - namespaces + verbs: + - get +- apiGroups: [""] + resources: + - endpoints + - services + verbs: + # Used to discover service IPs for advertisement. + - watch + - list + # Used to discover Typhas. + - get +- apiGroups: [""] + resources: + - nodes/status + verbs: + # Needed for clearing NodeNetworkUnavailable flag. + - patch + # Calico stores some configuration information in node annotations. + - update +# Watch for changes to Kubernetes NetworkPolicies. +- apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: + - watch + - list +# Used by Calico for policy information. +- apiGroups: [""] + resources: + - pods + - namespaces + - serviceaccounts + verbs: + - list + - watch +# The CNI plugin patches pods/status. +- apiGroups: [""] + resources: + - pods/status + verbs: + - patch +# Calico monitors various CRDs for config. +- apiGroups: ["crd.projectcalico.org"] + resources: + - globalfelixconfigs + - felixconfigurations + - bgppeers + - globalbgpconfigs + - bgpconfigurations + - ippools + - globalnetworkpolicies + - globalnetworksets + - networkpolicies + - clusterinformations + - hostendpoints + verbs: + - get + - list + - watch +# Calico must create and update some CRDs on startup. +- apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + - felixconfigurations + - clusterinformations + verbs: + - create + - update +# Calico stores some configuration information on the node. +- apiGroups: [""] + resources: + - nodes + verbs: + - get + - list + - watch +# These permissions are only requried for upgrade from v2.6, and can +# be removed after upgrade or on fresh installations. +- apiGroups: ["crd.projectcalico.org"] + resources: + - bgpconfigurations + - bgppeers + verbs: + - create + - update +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: calico-node + labels: + addonmanager.kubernetes.io/mode: "EnsureExists" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-node +subjects: +- kind: ServiceAccount + name: calico-node + namespace: kube-system +--- + +# Typha Horizontal Autoscaler ConfigMap +kind: ConfigMap +apiVersion: v1 +metadata: + name: calico-typha-horizontal-autoscaler + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +data: + ladder: |- + { + "coresToReplicas": [], + "nodesToReplicas": + [ + [1, 1], + [10, 2], + [100, 3], + [250, 4], + [500, 5], + [1000, 6], + [1500, 7], + [2000, 8] + ] + } + +--- + +# Typha Horizontal Autoscaler Deployment +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: calico-typha-horizontal-autoscaler + namespace: kube-system + labels: + k8s-app: calico-typha-autoscaler + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +spec: + replicas: 1 + template: + metadata: + labels: + k8s-app: calico-typha-autoscaler + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + priorityClassName: system-cluster-critical + securityContext: + supplementalGroups: [65534] + fsGroup: 65534 + containers: + - image: k8s.gcr.io/cluster-proportional-autoscaler-amd64:1.1.2-r2 + name: autoscaler + command: + - /cluster-proportional-autoscaler + - --namespace=kube-system + - --configmap=calico-typha-horizontal-autoscaler + - --target=deployment/calico-typha + - --logtostderr=true + - --v=2 + resources: + requests: + cpu: 10m + limits: + cpu: 10m + serviceAccountName: typha-cpha +--- + +# Typha Horizontal Autoscaler Cluster Role +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: typha-cpha + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +rules: +- apiGroups: [""] + resources: ["nodes"] + verbs: ["list"] + +--- + +# Typha Horizontal Autoscaler Cluster Role Binding +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: typha-cpha + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: typha-cpha +subjects: +- kind: ServiceAccount + name: typha-cpha + namespace: kube-system +--- + +# Typha Horizontal Autoscaler Role +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: typha-cpha + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] +- apiGroups: ["extensions"] + resources: ["deployments/scale"] + verbs: ["get", "update"] + +--- + +# Typha Horizontal Autoscaler Role Binding +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: typha-cpha + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: typha-cpha +subjects: +- kind: ServiceAccount + name: typha-cpha + namespace: kube-system +--- + +# Typha Horizontal Autoscaler Service Account +apiVersion: v1 +kind: ServiceAccount +metadata: + name: typha-cpha + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + diff --git a/parts/k8s/addons/1.11/kubernetesmasteraddons-calico-daemonset.yaml b/parts/k8s/addons/1.11/kubernetesmasteraddons-calico-daemonset.yaml index 29d80fabf38..f7889a7d98f 100644 --- a/parts/k8s/addons/1.11/kubernetesmasteraddons-calico-daemonset.yaml +++ b/parts/k8s/addons/1.11/kubernetesmasteraddons-calico-daemonset.yaml @@ -1,108 +1,8 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: calico-node - namespace: kube-system - labels: - addonmanager.kubernetes.io/mode: "EnsureExists" ---- - -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: calico-node - labels: - addonmanager.kubernetes.io/mode: "EnsureExists" -rules: - - apiGroups: [""] - resources: - - namespaces - - serviceaccounts - verbs: - - get - - list - - watch - - apiGroups: [""] - resources: - - pods/status - verbs: - - patch - - apiGroups: [""] - resources: - - pods - verbs: - - get - - list - - watch - - apiGroups: [""] - resources: - - services - verbs: - - get - - apiGroups: [""] - resources: - - endpoints - verbs: - - get - - apiGroups: [""] - resources: - - nodes - verbs: - - get - - list - - update - - watch - - apiGroups: ["extensions"] - resources: - - networkpolicies - verbs: - - get - - list - - watch - - apiGroups: ["networking.k8s.io"] - resources: - - networkpolicies - verbs: - - watch - - list - - apiGroups: ["crd.projectcalico.org"] - resources: - - globalfelixconfigs - - felixconfigurations - - bgppeers - - globalbgpconfigs - - bgpconfigurations - - ippools - - globalnetworkpolicies - - globalnetworksets - - networkpolicies - - clusterinformations - - hostendpoints - verbs: - - create - - get - - list - - update - - watch - ---- - -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: calico-node - labels: - addonmanager.kubernetes.io/mode: "EnsureExists" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-node -subjects: -- kind: ServiceAccount - name: calico-node - namespace: kube-system - ---- +# Calico Version v3.5.0 +# https://docs.projectcalico.org/v3.5/releases#v3.5.0 +# This manifest includes the following component versions: +# calico/node:v3.5.0 +# calico/cni:v3.5.0 # This ConfigMap is used to configure a self-hosted Calico installation. kind: ConfigMap @@ -113,9 +13,11 @@ metadata: labels: addonmanager.kubernetes.io/mode: "EnsureExists" data: + # You must set a non-zero value for Typha replicas below. typha_service_name: "calico-typha" - # The CNI network configuration to install on each node. + # The CNI network configuration to install on each node. The special + # values in this config will be automatically populated. cni_network_config: |- { "name": "k8s-pod-network", @@ -158,13 +60,12 @@ metadata: addonmanager.kubernetes.io/mode: "EnsureExists" spec: ports: - - port: 5473 - protocol: TCP - targetPort: calico-typha - name: calico-typha + - port: 5473 + protocol: TCP + targetPort: calico-typha + name: calico-typha selector: k8s-app: calico-typha - --- # This manifest creates a Deployment of Typha to back the above service. @@ -184,228 +85,77 @@ spec: # We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is essential # (when using the Kubernetes datastore). Use one replica for every 100-200 nodes. In # production, we recommend running at least 3 replicas to reduce the impact of rolling upgrade. - replicas: 0 + replicas: 1 revisionHistoryLimit: 2 template: metadata: labels: k8s-app: calico-typha - addonmanager.kubernetes.io/mode: "EnsureExists" annotations: # This, along with the CriticalAddonsOnly toleration below, marks the pod as a critical # add-on, ensuring it gets priority scheduling and that its resources are reserved # if it ever gets evicted. scheduler.alpha.kubernetes.io/critical-pod: '' + cluster-autoscaler.kubernetes.io/safe-to-evict: 'true' spec: - hostNetwork: true nodeSelector: beta.kubernetes.io/os: linux + hostNetwork: true tolerations: + # Mark the pod as a critical add-on for rescheduling. - key: CriticalAddonsOnly operator: Exists # Since Calico can't network a pod until Typha is up, we need to run Typha itself # as a host-networked pod. serviceAccountName: calico-node containers: - - image: quay.io/calico/typha:v3.3.1 + - image: calico/typha:v3.5.0 name: calico-typha ports: - containerPort: 5473 name: calico-typha protocol: TCP env: - # Enable "info" logging by default. Can be set to "debug" to increase verbosity. - - name: TYPHA_LOGSEVERITYSCREEN - value: "info" - # Disable logging to file and syslog since those don't make sense in Kubernetes. - - name: TYPHA_LOGFILEPATH - value: "none" - - name: TYPHA_LOGSEVERITYSYS - value: "none" - # Monitor the Kubernetes API to find the number of running instances and rebalance - # connections. - - name: TYPHA_CONNECTIONREBALANCINGMODE - value: "kubernetes" - - name: TYPHA_DATASTORETYPE - value: "kubernetes" - - name: TYPHA_HEALTHENABLED - value: "true" - # Uncomment these lines to enable prometheus metrics. Since Typha is host-networked, - # this opens a port on the host, which may need to be secured. - #- name: TYPHA_PROMETHEUSMETRICSENABLED - # value: "true" - #- name: TYPHA_PROMETHEUSMETRICSPORT - # value: "9093" + # Enable "info" logging by default. Can be set to "debug" to increase verbosity. + - name: TYPHA_LOGSEVERITYSCREEN + value: "info" + # Disable logging to file and syslog since those don't make sense in Kubernetes. + - name: TYPHA_LOGFILEPATH + value: "none" + - name: TYPHA_LOGSEVERITYSYS + value: "none" + # Monitor the Kubernetes API to find the number of running instances and rebalance + # connections. + - name: TYPHA_CONNECTIONREBALANCINGMODE + value: "kubernetes" + - name: TYPHA_DATASTORETYPE + value: "kubernetes" + - name: TYPHA_HEALTHENABLED + value: "true" + - name: FELIX_INTERFACEPREFIX + value: "azv" + # Uncomment these lines to enable prometheus metrics. Since Typha is host-networked, + # this opens a port on the host, which may need to be secured. + #- name: TYPHA_PROMETHEUSMETRICSENABLED + # value: "true" + #- name: TYPHA_PROMETHEUSMETRICSPORT + # value: "9093" livenessProbe: - httpGet: - path: /liveness - port: 9098 - host: localhost + exec: + command: + - calico-typha + - check + - liveness periodSeconds: 30 initialDelaySeconds: 30 readinessProbe: - httpGet: - path: /readiness - port: 9098 - host: localhost + exec: + command: + - calico-typha + - check + - readiness periodSeconds: 10 - ---- - -# Typha Horizontal Autoscaler Cluster Role -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: typha-cpha - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["list"] - ---- - -# Typha Horizontal Autoscaler Cluster Role Binding -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: typha-cpha - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: typha-cpha -subjects: - - kind: ServiceAccount - name: typha-cpha - namespace: kube-system - --- - -# Typha Horizontal Autoscaler ConfigMap -kind: ConfigMap -apiVersion: v1 -metadata: - name: calico-typha-horizontal-autoscaler - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -data: - ladder: |- - { - "coresToReplicas": [], - "nodesToReplicas": - [ - [1, 1], - [10, 2], - [100, 3], - [250, 4], - [500, 5], - [1000, 6], - [1500, 7], - [2000, 8] - ] - } - ---- - -# Typha Horizontal Autoscaler Deployment -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: calico-typha-horizontal-autoscaler - namespace: kube-system - labels: - k8s-app: calico-typha-autoscaler - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -spec: - replicas: 1 - template: - metadata: - labels: - k8s-app: calico-typha-autoscaler - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' - spec: - priorityClassName: system-node-critical - securityContext: - supplementalGroups: [ 65534 ] - fsGroup: 65534 - containers: - - image: k8s.gcr.io/cluster-proportional-autoscaler-amd64:1.1.2-r2 - name: autoscaler - command: - - /cluster-proportional-autoscaler - - --namespace=kube-system - - --configmap=calico-typha-horizontal-autoscaler - - --target=deployment/calico-typha - - --logtostderr=true - - --v=2 - resources: - requests: - cpu: 10m - limits: - cpu: 10m - serviceAccountName: typha-cpha - ---- - -# Typha Horizontal Autoscaler Role -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: typha-cpha - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -rules: - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get"] - - apiGroups: ["extensions"] - resources: ["deployments/scale"] - verbs: ["get", "update"] - ---- - -# Typha Horizontal Autoscaler Role Binding -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: typha-cpha - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: typha-cpha -subjects: - - kind: ServiceAccount - name: typha-cpha - namespace: kube-system - ---- - -# Typha Horizontal Autoscaler Service Account -apiVersion: v1 -kind: ServiceAccount -metadata: - name: typha-cpha - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - ---- - # This manifest installs the calico/node container, as well # as the Calico CNI plugins and network config on # each master and worker node in a Kubernetes cluster. @@ -416,7 +166,6 @@ metadata: namespace: kube-system labels: k8s-app: calico-node - # github.com/kubernetes/kubernetes/tree/master/cluster/addons/addon-manager addonmanager.kubernetes.io/mode: "EnsureExists" spec: selector: @@ -435,157 +184,173 @@ spec: # marks the pod as a critical add-on, ensuring it gets # priority scheduling and that its resources are reserved # if it ever gets evicted. - # Deprecated in 1.10, Removed in 1.11. kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods scheduler.alpha.kubernetes.io/critical-pod: '' spec: + nodeSelector: + beta.kubernetes.io/os: linux hostNetwork: true tolerations: - # Allow the pod to run on the master. This is required for - # the master to communicate with pods. - - key: node-role.kubernetes.io/master - operator: Equal - value: "true" - effect: NoSchedule - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - operator: Exists + # Make sure calico-node gets scheduled on all nodes. + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists serviceAccountName: calico-node # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. terminationGracePeriodSeconds: 0 + initContainers: + # This container installs the Calico CNI binaries + # and CNI network config file on each node. + - name: install-cni + image: calico/cni:v3.5.0 + command: ["/install-cni.sh"] + env: + # Name of the CNI config file to create. + - name: CNI_CONF_NAME + value: "10-calico.conflist" + # The CNI network config to install on each node. + - name: CNI_NETWORK_CONFIG + valueFrom: + configMapKeyRef: + name: calico-config + key: cni_network_config + # Set the hostname based on the k8s node name. + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # Prevents the container from sleeping forever. + - name: SLEEP + value: "false" + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + - mountPath: /host/etc/cni/net.d + name: cni-net-dir containers: # Runs calico/node container on each Kubernetes node. This # container programs network policy and routes on each # host. - - name: calico-node - image: quay.io/calico/node:v3.3.1 - env: - # Use Kubernetes API as the backing datastore. - - name: DATASTORE_TYPE - value: "kubernetes" - # Enable felix info logging. - - name: FELIX_LOGSEVERITYSCREEN - value: "info" - # Don't enable BGP. - - name: CALICO_NETWORKING_BACKEND - value: "none" - # Cluster type to identify the deployment type - - name: CLUSTER_TYPE - value: "k8s" - # Disable file logging so `kubectl logs` works. - - name: CALICO_DISABLE_FILE_LOGGING - value: "true" - # Set Felix endpoint to host default action to ACCEPT. - - name: FELIX_DEFAULTENDPOINTTOHOSTACTION - value: "ACCEPT" - # Disable IPV6 on Kubernetes. - - name: FELIX_IPV6SUPPORT - value: "false" - # Wait for the datastore. - - name: WAIT_FOR_DATASTORE - value: "true" - # The Calico IPv4 pool to use. This should match `--cluster-cidr` - - name: CALICO_IPV4POOL_CIDR - value: "" - # Enable IPIP - - name: CALICO_IPV4POOL_IPIP - value: "off" - - name: FELIX_IPTABLESREFRESHINTERVAL - value: "60" - # Typha support: controlled by the ConfigMap. - - name: FELIX_TYPHAK8SSERVICENAME - valueFrom: - configMapKeyRef: - name: calico-config - key: typha_service_name - # Set based on the k8s node name. - - name: NODENAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: FELIX_HEALTHENABLED - value: "true" - - name: FELIX_IPINIPENABLED - value: "false" - securityContext: - privileged: true - resources: - requests: - cpu: 250m - livenessProbe: - httpGet: - path: /liveness - port: 9099 - host: localhost - periodSeconds: 10 - initialDelaySeconds: 10 - failureThreshold: 6 - readinessProbe: - httpGet: - path: /readiness - port: 9099 - host: localhost - periodSeconds: 10 - volumeMounts: - - mountPath: /lib/modules - name: lib-modules - readOnly: true - - mountPath: /var/run/calico - name: var-run-calico - readOnly: false - - mountPath: /var/lib/calico - name: var-lib-calico - readOnly: false - # This container installs the Calico CNI binaries - # and CNI network config file on each node. - - name: install-cni - image: quay.io/calico/cni:v3.4.0-0.dev-34-g83daff2 - command: ["/install-cni.sh"] - env: - # Name of the CNI config file to create. - - name: CNI_CONF_NAME - value: "10-calico.conflist" - # The CNI network config to install on each node. - - name: CNI_NETWORK_CONFIG - valueFrom: - configMapKeyRef: - name: calico-config - key: cni_network_config - # Set the hostname based on the k8s node name. - - name: KUBERNETES_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - volumeMounts: - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - - mountPath: /host/etc/cni/net.d - name: cni-net-dir + - name: calico-node + image: calico/node:v3.5.0 + env: + # Use Kubernetes API as the backing datastore. + - name: DATASTORE_TYPE + value: "kubernetes" + # Typha support: controlled by the ConfigMap. + - name: FELIX_TYPHAK8SSERVICENAME + valueFrom: + configMapKeyRef: + name: calico-config + key: typha_service_name + # Wait for the datastore. + - name: WAIT_FOR_DATASTORE + value: "true" + # Set based on the k8s node name. + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # Don't enable BGP. + - name: CALICO_NETWORKING_BACKEND + value: "none" + # Cluster type to identify the deployment type + - name: CLUSTER_TYPE + value: "k8s" + # The default IPv4 pool to create on startup if none exists. Pod IPs will be + # chosen from this range. Changing this value after installation will have + # no effect. This should fall within `--cluster-cidr`. + - name: CALICO_IPV4POOL_CIDR + value: "" + # Disable file logging so `kubectl logs` works. + - name: CALICO_DISABLE_FILE_LOGGING + value: "true" + # Set Felix endpoint to host default action to ACCEPT. + - name: FELIX_DEFAULTENDPOINTTOHOSTACTION + value: "ACCEPT" + # Disable IPv6 on Kubernetes. + - name: FELIX_IPV6SUPPORT + value: "false" + # Set Felix logging to "info" + - name: FELIX_LOGSEVERITYSCREEN + value: "info" + - name: FELIX_HEALTHENABLED + value: "true" + - name: CALICO_IPV4POOL_IPIP + value: "off" + - name: FELIX_INTERFACEPREFIX + value: "azv" + securityContext: + privileged: true + resources: + requests: + cpu: 250m + livenessProbe: + httpGet: + path: /liveness + port: 9099 + host: localhost + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + readinessProbe: + exec: + command: + - /bin/calico-node + - -felix-ready + periodSeconds: 10 + volumeMounts: + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - mountPath: /var/run/calico + name: var-run-calico + readOnly: false + - mountPath: /var/lib/calico + name: var-lib-calico + readOnly: false volumes: - # Used by calico/node. - - name: lib-modules - hostPath: - path: /lib/modules - - name: var-run-calico - hostPath: - path: /var/run/calico - - name: var-lib-calico - hostPath: - path: /var/lib/calico - # Used to install CNI. - - name: cni-bin-dir - hostPath: - path: /opt/cni/bin - - name: cni-net-dir - hostPath: - path: /etc/cni/net.d + # Used by calico/node. + - name: lib-modules + hostPath: + path: /lib/modules + - name: var-run-calico + hostPath: + path: /var/run/calico + - name: var-lib-calico + hostPath: + path: /var/lib/calico + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + # Used to install CNI. + - name: cni-bin-dir + hostPath: + path: /opt/cni/bin + - name: cni-net-dir + hostPath: + path: /etc/cni/net.d +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-node + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: "EnsureExists" +--- # Create all the CustomResourceDefinitions needed for # Calico policy-only mode. ---- -# Calico Felix Configuration + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -600,9 +365,8 @@ spec: kind: FelixConfiguration plural: felixconfigurations singular: felixconfiguration - --- -# Calico BGP Configuration + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -617,9 +381,8 @@ spec: kind: BGPConfiguration plural: bgpconfigurations singular: bgpconfiguration - --- -# Calico IP Pools + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -634,9 +397,8 @@ spec: kind: IPPool plural: ippools singular: ippool - --- -# Calico Host Endpoints + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -651,9 +413,8 @@ spec: kind: HostEndpoint plural: hostendpoints singular: hostendpoint - --- -# Calico Cluster Information + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -668,9 +429,8 @@ spec: kind: ClusterInformation plural: clusterinformations singular: clusterinformation - --- -# Calico Global Network Policies + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -685,9 +445,8 @@ spec: kind: GlobalNetworkPolicy plural: globalnetworkpolicies singular: globalnetworkpolicy - --- -# Calico Global Network Sets + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -702,9 +461,8 @@ spec: kind: GlobalNetworkSet plural: globalnetworksets singular: globalnetworkset - --- -# Calico Network Policies + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -719,3 +477,272 @@ spec: kind: NetworkPolicy plural: networkpolicies singular: networkpolicy +--- + +# Include a clusterrole for the calico-node DaemonSet, +# and bind it to the calico-node serviceaccount. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: calico-node + labels: + addonmanager.kubernetes.io/mode: "EnsureExists" +rules: +# The CNI plugin needs to get pods, nodes, and namespaces. +- apiGroups: [""] + resources: + - pods + - nodes + - namespaces + verbs: + - get +- apiGroups: [""] + resources: + - endpoints + - services + verbs: + # Used to discover service IPs for advertisement. + - watch + - list + # Used to discover Typhas. + - get +- apiGroups: [""] + resources: + - nodes/status + verbs: + # Needed for clearing NodeNetworkUnavailable flag. + - patch + # Calico stores some configuration information in node annotations. + - update +# Watch for changes to Kubernetes NetworkPolicies. +- apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: + - watch + - list +# Used by Calico for policy information. +- apiGroups: [""] + resources: + - pods + - namespaces + - serviceaccounts + verbs: + - list + - watch +# The CNI plugin patches pods/status. +- apiGroups: [""] + resources: + - pods/status + verbs: + - patch +# Calico monitors various CRDs for config. +- apiGroups: ["crd.projectcalico.org"] + resources: + - globalfelixconfigs + - felixconfigurations + - bgppeers + - globalbgpconfigs + - bgpconfigurations + - ippools + - globalnetworkpolicies + - globalnetworksets + - networkpolicies + - clusterinformations + - hostendpoints + verbs: + - get + - list + - watch +# Calico must create and update some CRDs on startup. +- apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + - felixconfigurations + - clusterinformations + verbs: + - create + - update +# Calico stores some configuration information on the node. +- apiGroups: [""] + resources: + - nodes + verbs: + - get + - list + - watch +# These permissions are only requried for upgrade from v2.6, and can +# be removed after upgrade or on fresh installations. +- apiGroups: ["crd.projectcalico.org"] + resources: + - bgpconfigurations + - bgppeers + verbs: + - create + - update +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: calico-node + labels: + addonmanager.kubernetes.io/mode: "EnsureExists" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-node +subjects: +- kind: ServiceAccount + name: calico-node + namespace: kube-system +--- + +# Typha Horizontal Autoscaler ConfigMap +kind: ConfigMap +apiVersion: v1 +metadata: + name: calico-typha-horizontal-autoscaler + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +data: + ladder: |- + { + "coresToReplicas": [], + "nodesToReplicas": + [ + [1, 1], + [10, 2], + [100, 3], + [250, 4], + [500, 5], + [1000, 6], + [1500, 7], + [2000, 8] + ] + } + +--- + +# Typha Horizontal Autoscaler Deployment +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: calico-typha-horizontal-autoscaler + namespace: kube-system + labels: + k8s-app: calico-typha-autoscaler + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +spec: + replicas: 1 + template: + metadata: + labels: + k8s-app: calico-typha-autoscaler + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + priorityClassName: system-cluster-critical + securityContext: + supplementalGroups: [65534] + fsGroup: 65534 + containers: + - image: k8s.gcr.io/cluster-proportional-autoscaler-amd64:1.1.2-r2 + name: autoscaler + command: + - /cluster-proportional-autoscaler + - --namespace=kube-system + - --configmap=calico-typha-horizontal-autoscaler + - --target=deployment/calico-typha + - --logtostderr=true + - --v=2 + resources: + requests: + cpu: 10m + limits: + cpu: 10m + serviceAccountName: typha-cpha +--- + +# Typha Horizontal Autoscaler Cluster Role +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: typha-cpha + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +rules: +- apiGroups: [""] + resources: ["nodes"] + verbs: ["list"] + +--- + +# Typha Horizontal Autoscaler Cluster Role Binding +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: typha-cpha + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: typha-cpha +subjects: +- kind: ServiceAccount + name: typha-cpha + namespace: kube-system +--- + +# Typha Horizontal Autoscaler Role +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: typha-cpha + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] +- apiGroups: ["extensions"] + resources: ["deployments/scale"] + verbs: ["get", "update"] + +--- + +# Typha Horizontal Autoscaler Role Binding +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: typha-cpha + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: typha-cpha +subjects: +- kind: ServiceAccount + name: typha-cpha + namespace: kube-system +--- + +# Typha Horizontal Autoscaler Service Account +apiVersion: v1 +kind: ServiceAccount +metadata: + name: typha-cpha + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + diff --git a/parts/k8s/addons/1.7/kubernetesmasteraddons-calico-daemonset.yaml b/parts/k8s/addons/1.7/kubernetesmasteraddons-calico-daemonset.yaml index 29d80fabf38..f7889a7d98f 100644 --- a/parts/k8s/addons/1.7/kubernetesmasteraddons-calico-daemonset.yaml +++ b/parts/k8s/addons/1.7/kubernetesmasteraddons-calico-daemonset.yaml @@ -1,108 +1,8 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: calico-node - namespace: kube-system - labels: - addonmanager.kubernetes.io/mode: "EnsureExists" ---- - -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: calico-node - labels: - addonmanager.kubernetes.io/mode: "EnsureExists" -rules: - - apiGroups: [""] - resources: - - namespaces - - serviceaccounts - verbs: - - get - - list - - watch - - apiGroups: [""] - resources: - - pods/status - verbs: - - patch - - apiGroups: [""] - resources: - - pods - verbs: - - get - - list - - watch - - apiGroups: [""] - resources: - - services - verbs: - - get - - apiGroups: [""] - resources: - - endpoints - verbs: - - get - - apiGroups: [""] - resources: - - nodes - verbs: - - get - - list - - update - - watch - - apiGroups: ["extensions"] - resources: - - networkpolicies - verbs: - - get - - list - - watch - - apiGroups: ["networking.k8s.io"] - resources: - - networkpolicies - verbs: - - watch - - list - - apiGroups: ["crd.projectcalico.org"] - resources: - - globalfelixconfigs - - felixconfigurations - - bgppeers - - globalbgpconfigs - - bgpconfigurations - - ippools - - globalnetworkpolicies - - globalnetworksets - - networkpolicies - - clusterinformations - - hostendpoints - verbs: - - create - - get - - list - - update - - watch - ---- - -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: calico-node - labels: - addonmanager.kubernetes.io/mode: "EnsureExists" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-node -subjects: -- kind: ServiceAccount - name: calico-node - namespace: kube-system - ---- +# Calico Version v3.5.0 +# https://docs.projectcalico.org/v3.5/releases#v3.5.0 +# This manifest includes the following component versions: +# calico/node:v3.5.0 +# calico/cni:v3.5.0 # This ConfigMap is used to configure a self-hosted Calico installation. kind: ConfigMap @@ -113,9 +13,11 @@ metadata: labels: addonmanager.kubernetes.io/mode: "EnsureExists" data: + # You must set a non-zero value for Typha replicas below. typha_service_name: "calico-typha" - # The CNI network configuration to install on each node. + # The CNI network configuration to install on each node. The special + # values in this config will be automatically populated. cni_network_config: |- { "name": "k8s-pod-network", @@ -158,13 +60,12 @@ metadata: addonmanager.kubernetes.io/mode: "EnsureExists" spec: ports: - - port: 5473 - protocol: TCP - targetPort: calico-typha - name: calico-typha + - port: 5473 + protocol: TCP + targetPort: calico-typha + name: calico-typha selector: k8s-app: calico-typha - --- # This manifest creates a Deployment of Typha to back the above service. @@ -184,228 +85,77 @@ spec: # We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is essential # (when using the Kubernetes datastore). Use one replica for every 100-200 nodes. In # production, we recommend running at least 3 replicas to reduce the impact of rolling upgrade. - replicas: 0 + replicas: 1 revisionHistoryLimit: 2 template: metadata: labels: k8s-app: calico-typha - addonmanager.kubernetes.io/mode: "EnsureExists" annotations: # This, along with the CriticalAddonsOnly toleration below, marks the pod as a critical # add-on, ensuring it gets priority scheduling and that its resources are reserved # if it ever gets evicted. scheduler.alpha.kubernetes.io/critical-pod: '' + cluster-autoscaler.kubernetes.io/safe-to-evict: 'true' spec: - hostNetwork: true nodeSelector: beta.kubernetes.io/os: linux + hostNetwork: true tolerations: + # Mark the pod as a critical add-on for rescheduling. - key: CriticalAddonsOnly operator: Exists # Since Calico can't network a pod until Typha is up, we need to run Typha itself # as a host-networked pod. serviceAccountName: calico-node containers: - - image: quay.io/calico/typha:v3.3.1 + - image: calico/typha:v3.5.0 name: calico-typha ports: - containerPort: 5473 name: calico-typha protocol: TCP env: - # Enable "info" logging by default. Can be set to "debug" to increase verbosity. - - name: TYPHA_LOGSEVERITYSCREEN - value: "info" - # Disable logging to file and syslog since those don't make sense in Kubernetes. - - name: TYPHA_LOGFILEPATH - value: "none" - - name: TYPHA_LOGSEVERITYSYS - value: "none" - # Monitor the Kubernetes API to find the number of running instances and rebalance - # connections. - - name: TYPHA_CONNECTIONREBALANCINGMODE - value: "kubernetes" - - name: TYPHA_DATASTORETYPE - value: "kubernetes" - - name: TYPHA_HEALTHENABLED - value: "true" - # Uncomment these lines to enable prometheus metrics. Since Typha is host-networked, - # this opens a port on the host, which may need to be secured. - #- name: TYPHA_PROMETHEUSMETRICSENABLED - # value: "true" - #- name: TYPHA_PROMETHEUSMETRICSPORT - # value: "9093" + # Enable "info" logging by default. Can be set to "debug" to increase verbosity. + - name: TYPHA_LOGSEVERITYSCREEN + value: "info" + # Disable logging to file and syslog since those don't make sense in Kubernetes. + - name: TYPHA_LOGFILEPATH + value: "none" + - name: TYPHA_LOGSEVERITYSYS + value: "none" + # Monitor the Kubernetes API to find the number of running instances and rebalance + # connections. + - name: TYPHA_CONNECTIONREBALANCINGMODE + value: "kubernetes" + - name: TYPHA_DATASTORETYPE + value: "kubernetes" + - name: TYPHA_HEALTHENABLED + value: "true" + - name: FELIX_INTERFACEPREFIX + value: "azv" + # Uncomment these lines to enable prometheus metrics. Since Typha is host-networked, + # this opens a port on the host, which may need to be secured. + #- name: TYPHA_PROMETHEUSMETRICSENABLED + # value: "true" + #- name: TYPHA_PROMETHEUSMETRICSPORT + # value: "9093" livenessProbe: - httpGet: - path: /liveness - port: 9098 - host: localhost + exec: + command: + - calico-typha + - check + - liveness periodSeconds: 30 initialDelaySeconds: 30 readinessProbe: - httpGet: - path: /readiness - port: 9098 - host: localhost + exec: + command: + - calico-typha + - check + - readiness periodSeconds: 10 - ---- - -# Typha Horizontal Autoscaler Cluster Role -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: typha-cpha - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["list"] - ---- - -# Typha Horizontal Autoscaler Cluster Role Binding -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: typha-cpha - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: typha-cpha -subjects: - - kind: ServiceAccount - name: typha-cpha - namespace: kube-system - --- - -# Typha Horizontal Autoscaler ConfigMap -kind: ConfigMap -apiVersion: v1 -metadata: - name: calico-typha-horizontal-autoscaler - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -data: - ladder: |- - { - "coresToReplicas": [], - "nodesToReplicas": - [ - [1, 1], - [10, 2], - [100, 3], - [250, 4], - [500, 5], - [1000, 6], - [1500, 7], - [2000, 8] - ] - } - ---- - -# Typha Horizontal Autoscaler Deployment -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: calico-typha-horizontal-autoscaler - namespace: kube-system - labels: - k8s-app: calico-typha-autoscaler - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -spec: - replicas: 1 - template: - metadata: - labels: - k8s-app: calico-typha-autoscaler - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' - spec: - priorityClassName: system-node-critical - securityContext: - supplementalGroups: [ 65534 ] - fsGroup: 65534 - containers: - - image: k8s.gcr.io/cluster-proportional-autoscaler-amd64:1.1.2-r2 - name: autoscaler - command: - - /cluster-proportional-autoscaler - - --namespace=kube-system - - --configmap=calico-typha-horizontal-autoscaler - - --target=deployment/calico-typha - - --logtostderr=true - - --v=2 - resources: - requests: - cpu: 10m - limits: - cpu: 10m - serviceAccountName: typha-cpha - ---- - -# Typha Horizontal Autoscaler Role -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: typha-cpha - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -rules: - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get"] - - apiGroups: ["extensions"] - resources: ["deployments/scale"] - verbs: ["get", "update"] - ---- - -# Typha Horizontal Autoscaler Role Binding -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: typha-cpha - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: typha-cpha -subjects: - - kind: ServiceAccount - name: typha-cpha - namespace: kube-system - ---- - -# Typha Horizontal Autoscaler Service Account -apiVersion: v1 -kind: ServiceAccount -metadata: - name: typha-cpha - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - ---- - # This manifest installs the calico/node container, as well # as the Calico CNI plugins and network config on # each master and worker node in a Kubernetes cluster. @@ -416,7 +166,6 @@ metadata: namespace: kube-system labels: k8s-app: calico-node - # github.com/kubernetes/kubernetes/tree/master/cluster/addons/addon-manager addonmanager.kubernetes.io/mode: "EnsureExists" spec: selector: @@ -435,157 +184,173 @@ spec: # marks the pod as a critical add-on, ensuring it gets # priority scheduling and that its resources are reserved # if it ever gets evicted. - # Deprecated in 1.10, Removed in 1.11. kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods scheduler.alpha.kubernetes.io/critical-pod: '' spec: + nodeSelector: + beta.kubernetes.io/os: linux hostNetwork: true tolerations: - # Allow the pod to run on the master. This is required for - # the master to communicate with pods. - - key: node-role.kubernetes.io/master - operator: Equal - value: "true" - effect: NoSchedule - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - operator: Exists + # Make sure calico-node gets scheduled on all nodes. + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists serviceAccountName: calico-node # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. terminationGracePeriodSeconds: 0 + initContainers: + # This container installs the Calico CNI binaries + # and CNI network config file on each node. + - name: install-cni + image: calico/cni:v3.5.0 + command: ["/install-cni.sh"] + env: + # Name of the CNI config file to create. + - name: CNI_CONF_NAME + value: "10-calico.conflist" + # The CNI network config to install on each node. + - name: CNI_NETWORK_CONFIG + valueFrom: + configMapKeyRef: + name: calico-config + key: cni_network_config + # Set the hostname based on the k8s node name. + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # Prevents the container from sleeping forever. + - name: SLEEP + value: "false" + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + - mountPath: /host/etc/cni/net.d + name: cni-net-dir containers: # Runs calico/node container on each Kubernetes node. This # container programs network policy and routes on each # host. - - name: calico-node - image: quay.io/calico/node:v3.3.1 - env: - # Use Kubernetes API as the backing datastore. - - name: DATASTORE_TYPE - value: "kubernetes" - # Enable felix info logging. - - name: FELIX_LOGSEVERITYSCREEN - value: "info" - # Don't enable BGP. - - name: CALICO_NETWORKING_BACKEND - value: "none" - # Cluster type to identify the deployment type - - name: CLUSTER_TYPE - value: "k8s" - # Disable file logging so `kubectl logs` works. - - name: CALICO_DISABLE_FILE_LOGGING - value: "true" - # Set Felix endpoint to host default action to ACCEPT. - - name: FELIX_DEFAULTENDPOINTTOHOSTACTION - value: "ACCEPT" - # Disable IPV6 on Kubernetes. - - name: FELIX_IPV6SUPPORT - value: "false" - # Wait for the datastore. - - name: WAIT_FOR_DATASTORE - value: "true" - # The Calico IPv4 pool to use. This should match `--cluster-cidr` - - name: CALICO_IPV4POOL_CIDR - value: "" - # Enable IPIP - - name: CALICO_IPV4POOL_IPIP - value: "off" - - name: FELIX_IPTABLESREFRESHINTERVAL - value: "60" - # Typha support: controlled by the ConfigMap. - - name: FELIX_TYPHAK8SSERVICENAME - valueFrom: - configMapKeyRef: - name: calico-config - key: typha_service_name - # Set based on the k8s node name. - - name: NODENAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: FELIX_HEALTHENABLED - value: "true" - - name: FELIX_IPINIPENABLED - value: "false" - securityContext: - privileged: true - resources: - requests: - cpu: 250m - livenessProbe: - httpGet: - path: /liveness - port: 9099 - host: localhost - periodSeconds: 10 - initialDelaySeconds: 10 - failureThreshold: 6 - readinessProbe: - httpGet: - path: /readiness - port: 9099 - host: localhost - periodSeconds: 10 - volumeMounts: - - mountPath: /lib/modules - name: lib-modules - readOnly: true - - mountPath: /var/run/calico - name: var-run-calico - readOnly: false - - mountPath: /var/lib/calico - name: var-lib-calico - readOnly: false - # This container installs the Calico CNI binaries - # and CNI network config file on each node. - - name: install-cni - image: quay.io/calico/cni:v3.4.0-0.dev-34-g83daff2 - command: ["/install-cni.sh"] - env: - # Name of the CNI config file to create. - - name: CNI_CONF_NAME - value: "10-calico.conflist" - # The CNI network config to install on each node. - - name: CNI_NETWORK_CONFIG - valueFrom: - configMapKeyRef: - name: calico-config - key: cni_network_config - # Set the hostname based on the k8s node name. - - name: KUBERNETES_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - volumeMounts: - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - - mountPath: /host/etc/cni/net.d - name: cni-net-dir + - name: calico-node + image: calico/node:v3.5.0 + env: + # Use Kubernetes API as the backing datastore. + - name: DATASTORE_TYPE + value: "kubernetes" + # Typha support: controlled by the ConfigMap. + - name: FELIX_TYPHAK8SSERVICENAME + valueFrom: + configMapKeyRef: + name: calico-config + key: typha_service_name + # Wait for the datastore. + - name: WAIT_FOR_DATASTORE + value: "true" + # Set based on the k8s node name. + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # Don't enable BGP. + - name: CALICO_NETWORKING_BACKEND + value: "none" + # Cluster type to identify the deployment type + - name: CLUSTER_TYPE + value: "k8s" + # The default IPv4 pool to create on startup if none exists. Pod IPs will be + # chosen from this range. Changing this value after installation will have + # no effect. This should fall within `--cluster-cidr`. + - name: CALICO_IPV4POOL_CIDR + value: "" + # Disable file logging so `kubectl logs` works. + - name: CALICO_DISABLE_FILE_LOGGING + value: "true" + # Set Felix endpoint to host default action to ACCEPT. + - name: FELIX_DEFAULTENDPOINTTOHOSTACTION + value: "ACCEPT" + # Disable IPv6 on Kubernetes. + - name: FELIX_IPV6SUPPORT + value: "false" + # Set Felix logging to "info" + - name: FELIX_LOGSEVERITYSCREEN + value: "info" + - name: FELIX_HEALTHENABLED + value: "true" + - name: CALICO_IPV4POOL_IPIP + value: "off" + - name: FELIX_INTERFACEPREFIX + value: "azv" + securityContext: + privileged: true + resources: + requests: + cpu: 250m + livenessProbe: + httpGet: + path: /liveness + port: 9099 + host: localhost + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + readinessProbe: + exec: + command: + - /bin/calico-node + - -felix-ready + periodSeconds: 10 + volumeMounts: + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - mountPath: /var/run/calico + name: var-run-calico + readOnly: false + - mountPath: /var/lib/calico + name: var-lib-calico + readOnly: false volumes: - # Used by calico/node. - - name: lib-modules - hostPath: - path: /lib/modules - - name: var-run-calico - hostPath: - path: /var/run/calico - - name: var-lib-calico - hostPath: - path: /var/lib/calico - # Used to install CNI. - - name: cni-bin-dir - hostPath: - path: /opt/cni/bin - - name: cni-net-dir - hostPath: - path: /etc/cni/net.d + # Used by calico/node. + - name: lib-modules + hostPath: + path: /lib/modules + - name: var-run-calico + hostPath: + path: /var/run/calico + - name: var-lib-calico + hostPath: + path: /var/lib/calico + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + # Used to install CNI. + - name: cni-bin-dir + hostPath: + path: /opt/cni/bin + - name: cni-net-dir + hostPath: + path: /etc/cni/net.d +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-node + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: "EnsureExists" +--- # Create all the CustomResourceDefinitions needed for # Calico policy-only mode. ---- -# Calico Felix Configuration + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -600,9 +365,8 @@ spec: kind: FelixConfiguration plural: felixconfigurations singular: felixconfiguration - --- -# Calico BGP Configuration + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -617,9 +381,8 @@ spec: kind: BGPConfiguration plural: bgpconfigurations singular: bgpconfiguration - --- -# Calico IP Pools + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -634,9 +397,8 @@ spec: kind: IPPool plural: ippools singular: ippool - --- -# Calico Host Endpoints + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -651,9 +413,8 @@ spec: kind: HostEndpoint plural: hostendpoints singular: hostendpoint - --- -# Calico Cluster Information + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -668,9 +429,8 @@ spec: kind: ClusterInformation plural: clusterinformations singular: clusterinformation - --- -# Calico Global Network Policies + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -685,9 +445,8 @@ spec: kind: GlobalNetworkPolicy plural: globalnetworkpolicies singular: globalnetworkpolicy - --- -# Calico Global Network Sets + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -702,9 +461,8 @@ spec: kind: GlobalNetworkSet plural: globalnetworksets singular: globalnetworkset - --- -# Calico Network Policies + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -719,3 +477,272 @@ spec: kind: NetworkPolicy plural: networkpolicies singular: networkpolicy +--- + +# Include a clusterrole for the calico-node DaemonSet, +# and bind it to the calico-node serviceaccount. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: calico-node + labels: + addonmanager.kubernetes.io/mode: "EnsureExists" +rules: +# The CNI plugin needs to get pods, nodes, and namespaces. +- apiGroups: [""] + resources: + - pods + - nodes + - namespaces + verbs: + - get +- apiGroups: [""] + resources: + - endpoints + - services + verbs: + # Used to discover service IPs for advertisement. + - watch + - list + # Used to discover Typhas. + - get +- apiGroups: [""] + resources: + - nodes/status + verbs: + # Needed for clearing NodeNetworkUnavailable flag. + - patch + # Calico stores some configuration information in node annotations. + - update +# Watch for changes to Kubernetes NetworkPolicies. +- apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: + - watch + - list +# Used by Calico for policy information. +- apiGroups: [""] + resources: + - pods + - namespaces + - serviceaccounts + verbs: + - list + - watch +# The CNI plugin patches pods/status. +- apiGroups: [""] + resources: + - pods/status + verbs: + - patch +# Calico monitors various CRDs for config. +- apiGroups: ["crd.projectcalico.org"] + resources: + - globalfelixconfigs + - felixconfigurations + - bgppeers + - globalbgpconfigs + - bgpconfigurations + - ippools + - globalnetworkpolicies + - globalnetworksets + - networkpolicies + - clusterinformations + - hostendpoints + verbs: + - get + - list + - watch +# Calico must create and update some CRDs on startup. +- apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + - felixconfigurations + - clusterinformations + verbs: + - create + - update +# Calico stores some configuration information on the node. +- apiGroups: [""] + resources: + - nodes + verbs: + - get + - list + - watch +# These permissions are only requried for upgrade from v2.6, and can +# be removed after upgrade or on fresh installations. +- apiGroups: ["crd.projectcalico.org"] + resources: + - bgpconfigurations + - bgppeers + verbs: + - create + - update +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: calico-node + labels: + addonmanager.kubernetes.io/mode: "EnsureExists" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-node +subjects: +- kind: ServiceAccount + name: calico-node + namespace: kube-system +--- + +# Typha Horizontal Autoscaler ConfigMap +kind: ConfigMap +apiVersion: v1 +metadata: + name: calico-typha-horizontal-autoscaler + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +data: + ladder: |- + { + "coresToReplicas": [], + "nodesToReplicas": + [ + [1, 1], + [10, 2], + [100, 3], + [250, 4], + [500, 5], + [1000, 6], + [1500, 7], + [2000, 8] + ] + } + +--- + +# Typha Horizontal Autoscaler Deployment +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: calico-typha-horizontal-autoscaler + namespace: kube-system + labels: + k8s-app: calico-typha-autoscaler + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +spec: + replicas: 1 + template: + metadata: + labels: + k8s-app: calico-typha-autoscaler + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + priorityClassName: system-cluster-critical + securityContext: + supplementalGroups: [65534] + fsGroup: 65534 + containers: + - image: k8s.gcr.io/cluster-proportional-autoscaler-amd64:1.1.2-r2 + name: autoscaler + command: + - /cluster-proportional-autoscaler + - --namespace=kube-system + - --configmap=calico-typha-horizontal-autoscaler + - --target=deployment/calico-typha + - --logtostderr=true + - --v=2 + resources: + requests: + cpu: 10m + limits: + cpu: 10m + serviceAccountName: typha-cpha +--- + +# Typha Horizontal Autoscaler Cluster Role +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: typha-cpha + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +rules: +- apiGroups: [""] + resources: ["nodes"] + verbs: ["list"] + +--- + +# Typha Horizontal Autoscaler Cluster Role Binding +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: typha-cpha + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: typha-cpha +subjects: +- kind: ServiceAccount + name: typha-cpha + namespace: kube-system +--- + +# Typha Horizontal Autoscaler Role +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: typha-cpha + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] +- apiGroups: ["extensions"] + resources: ["deployments/scale"] + verbs: ["get", "update"] + +--- + +# Typha Horizontal Autoscaler Role Binding +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: typha-cpha + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: typha-cpha +subjects: +- kind: ServiceAccount + name: typha-cpha + namespace: kube-system +--- + +# Typha Horizontal Autoscaler Service Account +apiVersion: v1 +kind: ServiceAccount +metadata: + name: typha-cpha + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + diff --git a/parts/k8s/addons/1.8/kubernetesmasteraddons-calico-daemonset.yaml b/parts/k8s/addons/1.8/kubernetesmasteraddons-calico-daemonset.yaml index 29d80fabf38..f7889a7d98f 100644 --- a/parts/k8s/addons/1.8/kubernetesmasteraddons-calico-daemonset.yaml +++ b/parts/k8s/addons/1.8/kubernetesmasteraddons-calico-daemonset.yaml @@ -1,108 +1,8 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: calico-node - namespace: kube-system - labels: - addonmanager.kubernetes.io/mode: "EnsureExists" ---- - -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: calico-node - labels: - addonmanager.kubernetes.io/mode: "EnsureExists" -rules: - - apiGroups: [""] - resources: - - namespaces - - serviceaccounts - verbs: - - get - - list - - watch - - apiGroups: [""] - resources: - - pods/status - verbs: - - patch - - apiGroups: [""] - resources: - - pods - verbs: - - get - - list - - watch - - apiGroups: [""] - resources: - - services - verbs: - - get - - apiGroups: [""] - resources: - - endpoints - verbs: - - get - - apiGroups: [""] - resources: - - nodes - verbs: - - get - - list - - update - - watch - - apiGroups: ["extensions"] - resources: - - networkpolicies - verbs: - - get - - list - - watch - - apiGroups: ["networking.k8s.io"] - resources: - - networkpolicies - verbs: - - watch - - list - - apiGroups: ["crd.projectcalico.org"] - resources: - - globalfelixconfigs - - felixconfigurations - - bgppeers - - globalbgpconfigs - - bgpconfigurations - - ippools - - globalnetworkpolicies - - globalnetworksets - - networkpolicies - - clusterinformations - - hostendpoints - verbs: - - create - - get - - list - - update - - watch - ---- - -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: calico-node - labels: - addonmanager.kubernetes.io/mode: "EnsureExists" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-node -subjects: -- kind: ServiceAccount - name: calico-node - namespace: kube-system - ---- +# Calico Version v3.5.0 +# https://docs.projectcalico.org/v3.5/releases#v3.5.0 +# This manifest includes the following component versions: +# calico/node:v3.5.0 +# calico/cni:v3.5.0 # This ConfigMap is used to configure a self-hosted Calico installation. kind: ConfigMap @@ -113,9 +13,11 @@ metadata: labels: addonmanager.kubernetes.io/mode: "EnsureExists" data: + # You must set a non-zero value for Typha replicas below. typha_service_name: "calico-typha" - # The CNI network configuration to install on each node. + # The CNI network configuration to install on each node. The special + # values in this config will be automatically populated. cni_network_config: |- { "name": "k8s-pod-network", @@ -158,13 +60,12 @@ metadata: addonmanager.kubernetes.io/mode: "EnsureExists" spec: ports: - - port: 5473 - protocol: TCP - targetPort: calico-typha - name: calico-typha + - port: 5473 + protocol: TCP + targetPort: calico-typha + name: calico-typha selector: k8s-app: calico-typha - --- # This manifest creates a Deployment of Typha to back the above service. @@ -184,228 +85,77 @@ spec: # We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is essential # (when using the Kubernetes datastore). Use one replica for every 100-200 nodes. In # production, we recommend running at least 3 replicas to reduce the impact of rolling upgrade. - replicas: 0 + replicas: 1 revisionHistoryLimit: 2 template: metadata: labels: k8s-app: calico-typha - addonmanager.kubernetes.io/mode: "EnsureExists" annotations: # This, along with the CriticalAddonsOnly toleration below, marks the pod as a critical # add-on, ensuring it gets priority scheduling and that its resources are reserved # if it ever gets evicted. scheduler.alpha.kubernetes.io/critical-pod: '' + cluster-autoscaler.kubernetes.io/safe-to-evict: 'true' spec: - hostNetwork: true nodeSelector: beta.kubernetes.io/os: linux + hostNetwork: true tolerations: + # Mark the pod as a critical add-on for rescheduling. - key: CriticalAddonsOnly operator: Exists # Since Calico can't network a pod until Typha is up, we need to run Typha itself # as a host-networked pod. serviceAccountName: calico-node containers: - - image: quay.io/calico/typha:v3.3.1 + - image: calico/typha:v3.5.0 name: calico-typha ports: - containerPort: 5473 name: calico-typha protocol: TCP env: - # Enable "info" logging by default. Can be set to "debug" to increase verbosity. - - name: TYPHA_LOGSEVERITYSCREEN - value: "info" - # Disable logging to file and syslog since those don't make sense in Kubernetes. - - name: TYPHA_LOGFILEPATH - value: "none" - - name: TYPHA_LOGSEVERITYSYS - value: "none" - # Monitor the Kubernetes API to find the number of running instances and rebalance - # connections. - - name: TYPHA_CONNECTIONREBALANCINGMODE - value: "kubernetes" - - name: TYPHA_DATASTORETYPE - value: "kubernetes" - - name: TYPHA_HEALTHENABLED - value: "true" - # Uncomment these lines to enable prometheus metrics. Since Typha is host-networked, - # this opens a port on the host, which may need to be secured. - #- name: TYPHA_PROMETHEUSMETRICSENABLED - # value: "true" - #- name: TYPHA_PROMETHEUSMETRICSPORT - # value: "9093" + # Enable "info" logging by default. Can be set to "debug" to increase verbosity. + - name: TYPHA_LOGSEVERITYSCREEN + value: "info" + # Disable logging to file and syslog since those don't make sense in Kubernetes. + - name: TYPHA_LOGFILEPATH + value: "none" + - name: TYPHA_LOGSEVERITYSYS + value: "none" + # Monitor the Kubernetes API to find the number of running instances and rebalance + # connections. + - name: TYPHA_CONNECTIONREBALANCINGMODE + value: "kubernetes" + - name: TYPHA_DATASTORETYPE + value: "kubernetes" + - name: TYPHA_HEALTHENABLED + value: "true" + - name: FELIX_INTERFACEPREFIX + value: "azv" + # Uncomment these lines to enable prometheus metrics. Since Typha is host-networked, + # this opens a port on the host, which may need to be secured. + #- name: TYPHA_PROMETHEUSMETRICSENABLED + # value: "true" + #- name: TYPHA_PROMETHEUSMETRICSPORT + # value: "9093" livenessProbe: - httpGet: - path: /liveness - port: 9098 - host: localhost + exec: + command: + - calico-typha + - check + - liveness periodSeconds: 30 initialDelaySeconds: 30 readinessProbe: - httpGet: - path: /readiness - port: 9098 - host: localhost + exec: + command: + - calico-typha + - check + - readiness periodSeconds: 10 - ---- - -# Typha Horizontal Autoscaler Cluster Role -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: typha-cpha - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["list"] - ---- - -# Typha Horizontal Autoscaler Cluster Role Binding -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: typha-cpha - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: typha-cpha -subjects: - - kind: ServiceAccount - name: typha-cpha - namespace: kube-system - --- - -# Typha Horizontal Autoscaler ConfigMap -kind: ConfigMap -apiVersion: v1 -metadata: - name: calico-typha-horizontal-autoscaler - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -data: - ladder: |- - { - "coresToReplicas": [], - "nodesToReplicas": - [ - [1, 1], - [10, 2], - [100, 3], - [250, 4], - [500, 5], - [1000, 6], - [1500, 7], - [2000, 8] - ] - } - ---- - -# Typha Horizontal Autoscaler Deployment -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: calico-typha-horizontal-autoscaler - namespace: kube-system - labels: - k8s-app: calico-typha-autoscaler - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -spec: - replicas: 1 - template: - metadata: - labels: - k8s-app: calico-typha-autoscaler - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' - spec: - priorityClassName: system-node-critical - securityContext: - supplementalGroups: [ 65534 ] - fsGroup: 65534 - containers: - - image: k8s.gcr.io/cluster-proportional-autoscaler-amd64:1.1.2-r2 - name: autoscaler - command: - - /cluster-proportional-autoscaler - - --namespace=kube-system - - --configmap=calico-typha-horizontal-autoscaler - - --target=deployment/calico-typha - - --logtostderr=true - - --v=2 - resources: - requests: - cpu: 10m - limits: - cpu: 10m - serviceAccountName: typha-cpha - ---- - -# Typha Horizontal Autoscaler Role -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: typha-cpha - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -rules: - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get"] - - apiGroups: ["extensions"] - resources: ["deployments/scale"] - verbs: ["get", "update"] - ---- - -# Typha Horizontal Autoscaler Role Binding -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: typha-cpha - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: typha-cpha -subjects: - - kind: ServiceAccount - name: typha-cpha - namespace: kube-system - ---- - -# Typha Horizontal Autoscaler Service Account -apiVersion: v1 -kind: ServiceAccount -metadata: - name: typha-cpha - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - ---- - # This manifest installs the calico/node container, as well # as the Calico CNI plugins and network config on # each master and worker node in a Kubernetes cluster. @@ -416,7 +166,6 @@ metadata: namespace: kube-system labels: k8s-app: calico-node - # github.com/kubernetes/kubernetes/tree/master/cluster/addons/addon-manager addonmanager.kubernetes.io/mode: "EnsureExists" spec: selector: @@ -435,157 +184,173 @@ spec: # marks the pod as a critical add-on, ensuring it gets # priority scheduling and that its resources are reserved # if it ever gets evicted. - # Deprecated in 1.10, Removed in 1.11. kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods scheduler.alpha.kubernetes.io/critical-pod: '' spec: + nodeSelector: + beta.kubernetes.io/os: linux hostNetwork: true tolerations: - # Allow the pod to run on the master. This is required for - # the master to communicate with pods. - - key: node-role.kubernetes.io/master - operator: Equal - value: "true" - effect: NoSchedule - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - operator: Exists + # Make sure calico-node gets scheduled on all nodes. + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists serviceAccountName: calico-node # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. terminationGracePeriodSeconds: 0 + initContainers: + # This container installs the Calico CNI binaries + # and CNI network config file on each node. + - name: install-cni + image: calico/cni:v3.5.0 + command: ["/install-cni.sh"] + env: + # Name of the CNI config file to create. + - name: CNI_CONF_NAME + value: "10-calico.conflist" + # The CNI network config to install on each node. + - name: CNI_NETWORK_CONFIG + valueFrom: + configMapKeyRef: + name: calico-config + key: cni_network_config + # Set the hostname based on the k8s node name. + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # Prevents the container from sleeping forever. + - name: SLEEP + value: "false" + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + - mountPath: /host/etc/cni/net.d + name: cni-net-dir containers: # Runs calico/node container on each Kubernetes node. This # container programs network policy and routes on each # host. - - name: calico-node - image: quay.io/calico/node:v3.3.1 - env: - # Use Kubernetes API as the backing datastore. - - name: DATASTORE_TYPE - value: "kubernetes" - # Enable felix info logging. - - name: FELIX_LOGSEVERITYSCREEN - value: "info" - # Don't enable BGP. - - name: CALICO_NETWORKING_BACKEND - value: "none" - # Cluster type to identify the deployment type - - name: CLUSTER_TYPE - value: "k8s" - # Disable file logging so `kubectl logs` works. - - name: CALICO_DISABLE_FILE_LOGGING - value: "true" - # Set Felix endpoint to host default action to ACCEPT. - - name: FELIX_DEFAULTENDPOINTTOHOSTACTION - value: "ACCEPT" - # Disable IPV6 on Kubernetes. - - name: FELIX_IPV6SUPPORT - value: "false" - # Wait for the datastore. - - name: WAIT_FOR_DATASTORE - value: "true" - # The Calico IPv4 pool to use. This should match `--cluster-cidr` - - name: CALICO_IPV4POOL_CIDR - value: "" - # Enable IPIP - - name: CALICO_IPV4POOL_IPIP - value: "off" - - name: FELIX_IPTABLESREFRESHINTERVAL - value: "60" - # Typha support: controlled by the ConfigMap. - - name: FELIX_TYPHAK8SSERVICENAME - valueFrom: - configMapKeyRef: - name: calico-config - key: typha_service_name - # Set based on the k8s node name. - - name: NODENAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: FELIX_HEALTHENABLED - value: "true" - - name: FELIX_IPINIPENABLED - value: "false" - securityContext: - privileged: true - resources: - requests: - cpu: 250m - livenessProbe: - httpGet: - path: /liveness - port: 9099 - host: localhost - periodSeconds: 10 - initialDelaySeconds: 10 - failureThreshold: 6 - readinessProbe: - httpGet: - path: /readiness - port: 9099 - host: localhost - periodSeconds: 10 - volumeMounts: - - mountPath: /lib/modules - name: lib-modules - readOnly: true - - mountPath: /var/run/calico - name: var-run-calico - readOnly: false - - mountPath: /var/lib/calico - name: var-lib-calico - readOnly: false - # This container installs the Calico CNI binaries - # and CNI network config file on each node. - - name: install-cni - image: quay.io/calico/cni:v3.4.0-0.dev-34-g83daff2 - command: ["/install-cni.sh"] - env: - # Name of the CNI config file to create. - - name: CNI_CONF_NAME - value: "10-calico.conflist" - # The CNI network config to install on each node. - - name: CNI_NETWORK_CONFIG - valueFrom: - configMapKeyRef: - name: calico-config - key: cni_network_config - # Set the hostname based on the k8s node name. - - name: KUBERNETES_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - volumeMounts: - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - - mountPath: /host/etc/cni/net.d - name: cni-net-dir + - name: calico-node + image: calico/node:v3.5.0 + env: + # Use Kubernetes API as the backing datastore. + - name: DATASTORE_TYPE + value: "kubernetes" + # Typha support: controlled by the ConfigMap. + - name: FELIX_TYPHAK8SSERVICENAME + valueFrom: + configMapKeyRef: + name: calico-config + key: typha_service_name + # Wait for the datastore. + - name: WAIT_FOR_DATASTORE + value: "true" + # Set based on the k8s node name. + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # Don't enable BGP. + - name: CALICO_NETWORKING_BACKEND + value: "none" + # Cluster type to identify the deployment type + - name: CLUSTER_TYPE + value: "k8s" + # The default IPv4 pool to create on startup if none exists. Pod IPs will be + # chosen from this range. Changing this value after installation will have + # no effect. This should fall within `--cluster-cidr`. + - name: CALICO_IPV4POOL_CIDR + value: "" + # Disable file logging so `kubectl logs` works. + - name: CALICO_DISABLE_FILE_LOGGING + value: "true" + # Set Felix endpoint to host default action to ACCEPT. + - name: FELIX_DEFAULTENDPOINTTOHOSTACTION + value: "ACCEPT" + # Disable IPv6 on Kubernetes. + - name: FELIX_IPV6SUPPORT + value: "false" + # Set Felix logging to "info" + - name: FELIX_LOGSEVERITYSCREEN + value: "info" + - name: FELIX_HEALTHENABLED + value: "true" + - name: CALICO_IPV4POOL_IPIP + value: "off" + - name: FELIX_INTERFACEPREFIX + value: "azv" + securityContext: + privileged: true + resources: + requests: + cpu: 250m + livenessProbe: + httpGet: + path: /liveness + port: 9099 + host: localhost + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + readinessProbe: + exec: + command: + - /bin/calico-node + - -felix-ready + periodSeconds: 10 + volumeMounts: + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - mountPath: /var/run/calico + name: var-run-calico + readOnly: false + - mountPath: /var/lib/calico + name: var-lib-calico + readOnly: false volumes: - # Used by calico/node. - - name: lib-modules - hostPath: - path: /lib/modules - - name: var-run-calico - hostPath: - path: /var/run/calico - - name: var-lib-calico - hostPath: - path: /var/lib/calico - # Used to install CNI. - - name: cni-bin-dir - hostPath: - path: /opt/cni/bin - - name: cni-net-dir - hostPath: - path: /etc/cni/net.d + # Used by calico/node. + - name: lib-modules + hostPath: + path: /lib/modules + - name: var-run-calico + hostPath: + path: /var/run/calico + - name: var-lib-calico + hostPath: + path: /var/lib/calico + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + # Used to install CNI. + - name: cni-bin-dir + hostPath: + path: /opt/cni/bin + - name: cni-net-dir + hostPath: + path: /etc/cni/net.d +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-node + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: "EnsureExists" +--- # Create all the CustomResourceDefinitions needed for # Calico policy-only mode. ---- -# Calico Felix Configuration + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -600,9 +365,8 @@ spec: kind: FelixConfiguration plural: felixconfigurations singular: felixconfiguration - --- -# Calico BGP Configuration + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -617,9 +381,8 @@ spec: kind: BGPConfiguration plural: bgpconfigurations singular: bgpconfiguration - --- -# Calico IP Pools + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -634,9 +397,8 @@ spec: kind: IPPool plural: ippools singular: ippool - --- -# Calico Host Endpoints + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -651,9 +413,8 @@ spec: kind: HostEndpoint plural: hostendpoints singular: hostendpoint - --- -# Calico Cluster Information + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -668,9 +429,8 @@ spec: kind: ClusterInformation plural: clusterinformations singular: clusterinformation - --- -# Calico Global Network Policies + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -685,9 +445,8 @@ spec: kind: GlobalNetworkPolicy plural: globalnetworkpolicies singular: globalnetworkpolicy - --- -# Calico Global Network Sets + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -702,9 +461,8 @@ spec: kind: GlobalNetworkSet plural: globalnetworksets singular: globalnetworkset - --- -# Calico Network Policies + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -719,3 +477,272 @@ spec: kind: NetworkPolicy plural: networkpolicies singular: networkpolicy +--- + +# Include a clusterrole for the calico-node DaemonSet, +# and bind it to the calico-node serviceaccount. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: calico-node + labels: + addonmanager.kubernetes.io/mode: "EnsureExists" +rules: +# The CNI plugin needs to get pods, nodes, and namespaces. +- apiGroups: [""] + resources: + - pods + - nodes + - namespaces + verbs: + - get +- apiGroups: [""] + resources: + - endpoints + - services + verbs: + # Used to discover service IPs for advertisement. + - watch + - list + # Used to discover Typhas. + - get +- apiGroups: [""] + resources: + - nodes/status + verbs: + # Needed for clearing NodeNetworkUnavailable flag. + - patch + # Calico stores some configuration information in node annotations. + - update +# Watch for changes to Kubernetes NetworkPolicies. +- apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: + - watch + - list +# Used by Calico for policy information. +- apiGroups: [""] + resources: + - pods + - namespaces + - serviceaccounts + verbs: + - list + - watch +# The CNI plugin patches pods/status. +- apiGroups: [""] + resources: + - pods/status + verbs: + - patch +# Calico monitors various CRDs for config. +- apiGroups: ["crd.projectcalico.org"] + resources: + - globalfelixconfigs + - felixconfigurations + - bgppeers + - globalbgpconfigs + - bgpconfigurations + - ippools + - globalnetworkpolicies + - globalnetworksets + - networkpolicies + - clusterinformations + - hostendpoints + verbs: + - get + - list + - watch +# Calico must create and update some CRDs on startup. +- apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + - felixconfigurations + - clusterinformations + verbs: + - create + - update +# Calico stores some configuration information on the node. +- apiGroups: [""] + resources: + - nodes + verbs: + - get + - list + - watch +# These permissions are only requried for upgrade from v2.6, and can +# be removed after upgrade or on fresh installations. +- apiGroups: ["crd.projectcalico.org"] + resources: + - bgpconfigurations + - bgppeers + verbs: + - create + - update +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: calico-node + labels: + addonmanager.kubernetes.io/mode: "EnsureExists" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-node +subjects: +- kind: ServiceAccount + name: calico-node + namespace: kube-system +--- + +# Typha Horizontal Autoscaler ConfigMap +kind: ConfigMap +apiVersion: v1 +metadata: + name: calico-typha-horizontal-autoscaler + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +data: + ladder: |- + { + "coresToReplicas": [], + "nodesToReplicas": + [ + [1, 1], + [10, 2], + [100, 3], + [250, 4], + [500, 5], + [1000, 6], + [1500, 7], + [2000, 8] + ] + } + +--- + +# Typha Horizontal Autoscaler Deployment +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: calico-typha-horizontal-autoscaler + namespace: kube-system + labels: + k8s-app: calico-typha-autoscaler + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +spec: + replicas: 1 + template: + metadata: + labels: + k8s-app: calico-typha-autoscaler + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + priorityClassName: system-cluster-critical + securityContext: + supplementalGroups: [65534] + fsGroup: 65534 + containers: + - image: k8s.gcr.io/cluster-proportional-autoscaler-amd64:1.1.2-r2 + name: autoscaler + command: + - /cluster-proportional-autoscaler + - --namespace=kube-system + - --configmap=calico-typha-horizontal-autoscaler + - --target=deployment/calico-typha + - --logtostderr=true + - --v=2 + resources: + requests: + cpu: 10m + limits: + cpu: 10m + serviceAccountName: typha-cpha +--- + +# Typha Horizontal Autoscaler Cluster Role +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: typha-cpha + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +rules: +- apiGroups: [""] + resources: ["nodes"] + verbs: ["list"] + +--- + +# Typha Horizontal Autoscaler Cluster Role Binding +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: typha-cpha + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: typha-cpha +subjects: +- kind: ServiceAccount + name: typha-cpha + namespace: kube-system +--- + +# Typha Horizontal Autoscaler Role +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: typha-cpha + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] +- apiGroups: ["extensions"] + resources: ["deployments/scale"] + verbs: ["get", "update"] + +--- + +# Typha Horizontal Autoscaler Role Binding +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: typha-cpha + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: typha-cpha +subjects: +- kind: ServiceAccount + name: typha-cpha + namespace: kube-system +--- + +# Typha Horizontal Autoscaler Service Account +apiVersion: v1 +kind: ServiceAccount +metadata: + name: typha-cpha + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + diff --git a/parts/k8s/addons/1.9/kubernetesmasteraddons-calico-daemonset.yaml b/parts/k8s/addons/1.9/kubernetesmasteraddons-calico-daemonset.yaml index 29d80fabf38..f7889a7d98f 100644 --- a/parts/k8s/addons/1.9/kubernetesmasteraddons-calico-daemonset.yaml +++ b/parts/k8s/addons/1.9/kubernetesmasteraddons-calico-daemonset.yaml @@ -1,108 +1,8 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: calico-node - namespace: kube-system - labels: - addonmanager.kubernetes.io/mode: "EnsureExists" ---- - -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: calico-node - labels: - addonmanager.kubernetes.io/mode: "EnsureExists" -rules: - - apiGroups: [""] - resources: - - namespaces - - serviceaccounts - verbs: - - get - - list - - watch - - apiGroups: [""] - resources: - - pods/status - verbs: - - patch - - apiGroups: [""] - resources: - - pods - verbs: - - get - - list - - watch - - apiGroups: [""] - resources: - - services - verbs: - - get - - apiGroups: [""] - resources: - - endpoints - verbs: - - get - - apiGroups: [""] - resources: - - nodes - verbs: - - get - - list - - update - - watch - - apiGroups: ["extensions"] - resources: - - networkpolicies - verbs: - - get - - list - - watch - - apiGroups: ["networking.k8s.io"] - resources: - - networkpolicies - verbs: - - watch - - list - - apiGroups: ["crd.projectcalico.org"] - resources: - - globalfelixconfigs - - felixconfigurations - - bgppeers - - globalbgpconfigs - - bgpconfigurations - - ippools - - globalnetworkpolicies - - globalnetworksets - - networkpolicies - - clusterinformations - - hostendpoints - verbs: - - create - - get - - list - - update - - watch - ---- - -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: calico-node - labels: - addonmanager.kubernetes.io/mode: "EnsureExists" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-node -subjects: -- kind: ServiceAccount - name: calico-node - namespace: kube-system - ---- +# Calico Version v3.5.0 +# https://docs.projectcalico.org/v3.5/releases#v3.5.0 +# This manifest includes the following component versions: +# calico/node:v3.5.0 +# calico/cni:v3.5.0 # This ConfigMap is used to configure a self-hosted Calico installation. kind: ConfigMap @@ -113,9 +13,11 @@ metadata: labels: addonmanager.kubernetes.io/mode: "EnsureExists" data: + # You must set a non-zero value for Typha replicas below. typha_service_name: "calico-typha" - # The CNI network configuration to install on each node. + # The CNI network configuration to install on each node. The special + # values in this config will be automatically populated. cni_network_config: |- { "name": "k8s-pod-network", @@ -158,13 +60,12 @@ metadata: addonmanager.kubernetes.io/mode: "EnsureExists" spec: ports: - - port: 5473 - protocol: TCP - targetPort: calico-typha - name: calico-typha + - port: 5473 + protocol: TCP + targetPort: calico-typha + name: calico-typha selector: k8s-app: calico-typha - --- # This manifest creates a Deployment of Typha to back the above service. @@ -184,228 +85,77 @@ spec: # We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is essential # (when using the Kubernetes datastore). Use one replica for every 100-200 nodes. In # production, we recommend running at least 3 replicas to reduce the impact of rolling upgrade. - replicas: 0 + replicas: 1 revisionHistoryLimit: 2 template: metadata: labels: k8s-app: calico-typha - addonmanager.kubernetes.io/mode: "EnsureExists" annotations: # This, along with the CriticalAddonsOnly toleration below, marks the pod as a critical # add-on, ensuring it gets priority scheduling and that its resources are reserved # if it ever gets evicted. scheduler.alpha.kubernetes.io/critical-pod: '' + cluster-autoscaler.kubernetes.io/safe-to-evict: 'true' spec: - hostNetwork: true nodeSelector: beta.kubernetes.io/os: linux + hostNetwork: true tolerations: + # Mark the pod as a critical add-on for rescheduling. - key: CriticalAddonsOnly operator: Exists # Since Calico can't network a pod until Typha is up, we need to run Typha itself # as a host-networked pod. serviceAccountName: calico-node containers: - - image: quay.io/calico/typha:v3.3.1 + - image: calico/typha:v3.5.0 name: calico-typha ports: - containerPort: 5473 name: calico-typha protocol: TCP env: - # Enable "info" logging by default. Can be set to "debug" to increase verbosity. - - name: TYPHA_LOGSEVERITYSCREEN - value: "info" - # Disable logging to file and syslog since those don't make sense in Kubernetes. - - name: TYPHA_LOGFILEPATH - value: "none" - - name: TYPHA_LOGSEVERITYSYS - value: "none" - # Monitor the Kubernetes API to find the number of running instances and rebalance - # connections. - - name: TYPHA_CONNECTIONREBALANCINGMODE - value: "kubernetes" - - name: TYPHA_DATASTORETYPE - value: "kubernetes" - - name: TYPHA_HEALTHENABLED - value: "true" - # Uncomment these lines to enable prometheus metrics. Since Typha is host-networked, - # this opens a port on the host, which may need to be secured. - #- name: TYPHA_PROMETHEUSMETRICSENABLED - # value: "true" - #- name: TYPHA_PROMETHEUSMETRICSPORT - # value: "9093" + # Enable "info" logging by default. Can be set to "debug" to increase verbosity. + - name: TYPHA_LOGSEVERITYSCREEN + value: "info" + # Disable logging to file and syslog since those don't make sense in Kubernetes. + - name: TYPHA_LOGFILEPATH + value: "none" + - name: TYPHA_LOGSEVERITYSYS + value: "none" + # Monitor the Kubernetes API to find the number of running instances and rebalance + # connections. + - name: TYPHA_CONNECTIONREBALANCINGMODE + value: "kubernetes" + - name: TYPHA_DATASTORETYPE + value: "kubernetes" + - name: TYPHA_HEALTHENABLED + value: "true" + - name: FELIX_INTERFACEPREFIX + value: "azv" + # Uncomment these lines to enable prometheus metrics. Since Typha is host-networked, + # this opens a port on the host, which may need to be secured. + #- name: TYPHA_PROMETHEUSMETRICSENABLED + # value: "true" + #- name: TYPHA_PROMETHEUSMETRICSPORT + # value: "9093" livenessProbe: - httpGet: - path: /liveness - port: 9098 - host: localhost + exec: + command: + - calico-typha + - check + - liveness periodSeconds: 30 initialDelaySeconds: 30 readinessProbe: - httpGet: - path: /readiness - port: 9098 - host: localhost + exec: + command: + - calico-typha + - check + - readiness periodSeconds: 10 - ---- - -# Typha Horizontal Autoscaler Cluster Role -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: typha-cpha - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["list"] - ---- - -# Typha Horizontal Autoscaler Cluster Role Binding -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: typha-cpha - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: typha-cpha -subjects: - - kind: ServiceAccount - name: typha-cpha - namespace: kube-system - --- - -# Typha Horizontal Autoscaler ConfigMap -kind: ConfigMap -apiVersion: v1 -metadata: - name: calico-typha-horizontal-autoscaler - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -data: - ladder: |- - { - "coresToReplicas": [], - "nodesToReplicas": - [ - [1, 1], - [10, 2], - [100, 3], - [250, 4], - [500, 5], - [1000, 6], - [1500, 7], - [2000, 8] - ] - } - ---- - -# Typha Horizontal Autoscaler Deployment -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: calico-typha-horizontal-autoscaler - namespace: kube-system - labels: - k8s-app: calico-typha-autoscaler - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -spec: - replicas: 1 - template: - metadata: - labels: - k8s-app: calico-typha-autoscaler - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' - spec: - priorityClassName: system-node-critical - securityContext: - supplementalGroups: [ 65534 ] - fsGroup: 65534 - containers: - - image: k8s.gcr.io/cluster-proportional-autoscaler-amd64:1.1.2-r2 - name: autoscaler - command: - - /cluster-proportional-autoscaler - - --namespace=kube-system - - --configmap=calico-typha-horizontal-autoscaler - - --target=deployment/calico-typha - - --logtostderr=true - - --v=2 - resources: - requests: - cpu: 10m - limits: - cpu: 10m - serviceAccountName: typha-cpha - ---- - -# Typha Horizontal Autoscaler Role -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: typha-cpha - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -rules: - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get"] - - apiGroups: ["extensions"] - resources: ["deployments/scale"] - verbs: ["get", "update"] - ---- - -# Typha Horizontal Autoscaler Role Binding -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: typha-cpha - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: typha-cpha -subjects: - - kind: ServiceAccount - name: typha-cpha - namespace: kube-system - ---- - -# Typha Horizontal Autoscaler Service Account -apiVersion: v1 -kind: ServiceAccount -metadata: - name: typha-cpha - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - ---- - # This manifest installs the calico/node container, as well # as the Calico CNI plugins and network config on # each master and worker node in a Kubernetes cluster. @@ -416,7 +166,6 @@ metadata: namespace: kube-system labels: k8s-app: calico-node - # github.com/kubernetes/kubernetes/tree/master/cluster/addons/addon-manager addonmanager.kubernetes.io/mode: "EnsureExists" spec: selector: @@ -435,157 +184,173 @@ spec: # marks the pod as a critical add-on, ensuring it gets # priority scheduling and that its resources are reserved # if it ever gets evicted. - # Deprecated in 1.10, Removed in 1.11. kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods scheduler.alpha.kubernetes.io/critical-pod: '' spec: + nodeSelector: + beta.kubernetes.io/os: linux hostNetwork: true tolerations: - # Allow the pod to run on the master. This is required for - # the master to communicate with pods. - - key: node-role.kubernetes.io/master - operator: Equal - value: "true" - effect: NoSchedule - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - operator: Exists + # Make sure calico-node gets scheduled on all nodes. + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists serviceAccountName: calico-node # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. terminationGracePeriodSeconds: 0 + initContainers: + # This container installs the Calico CNI binaries + # and CNI network config file on each node. + - name: install-cni + image: calico/cni:v3.5.0 + command: ["/install-cni.sh"] + env: + # Name of the CNI config file to create. + - name: CNI_CONF_NAME + value: "10-calico.conflist" + # The CNI network config to install on each node. + - name: CNI_NETWORK_CONFIG + valueFrom: + configMapKeyRef: + name: calico-config + key: cni_network_config + # Set the hostname based on the k8s node name. + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # Prevents the container from sleeping forever. + - name: SLEEP + value: "false" + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + - mountPath: /host/etc/cni/net.d + name: cni-net-dir containers: # Runs calico/node container on each Kubernetes node. This # container programs network policy and routes on each # host. - - name: calico-node - image: quay.io/calico/node:v3.3.1 - env: - # Use Kubernetes API as the backing datastore. - - name: DATASTORE_TYPE - value: "kubernetes" - # Enable felix info logging. - - name: FELIX_LOGSEVERITYSCREEN - value: "info" - # Don't enable BGP. - - name: CALICO_NETWORKING_BACKEND - value: "none" - # Cluster type to identify the deployment type - - name: CLUSTER_TYPE - value: "k8s" - # Disable file logging so `kubectl logs` works. - - name: CALICO_DISABLE_FILE_LOGGING - value: "true" - # Set Felix endpoint to host default action to ACCEPT. - - name: FELIX_DEFAULTENDPOINTTOHOSTACTION - value: "ACCEPT" - # Disable IPV6 on Kubernetes. - - name: FELIX_IPV6SUPPORT - value: "false" - # Wait for the datastore. - - name: WAIT_FOR_DATASTORE - value: "true" - # The Calico IPv4 pool to use. This should match `--cluster-cidr` - - name: CALICO_IPV4POOL_CIDR - value: "" - # Enable IPIP - - name: CALICO_IPV4POOL_IPIP - value: "off" - - name: FELIX_IPTABLESREFRESHINTERVAL - value: "60" - # Typha support: controlled by the ConfigMap. - - name: FELIX_TYPHAK8SSERVICENAME - valueFrom: - configMapKeyRef: - name: calico-config - key: typha_service_name - # Set based on the k8s node name. - - name: NODENAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: FELIX_HEALTHENABLED - value: "true" - - name: FELIX_IPINIPENABLED - value: "false" - securityContext: - privileged: true - resources: - requests: - cpu: 250m - livenessProbe: - httpGet: - path: /liveness - port: 9099 - host: localhost - periodSeconds: 10 - initialDelaySeconds: 10 - failureThreshold: 6 - readinessProbe: - httpGet: - path: /readiness - port: 9099 - host: localhost - periodSeconds: 10 - volumeMounts: - - mountPath: /lib/modules - name: lib-modules - readOnly: true - - mountPath: /var/run/calico - name: var-run-calico - readOnly: false - - mountPath: /var/lib/calico - name: var-lib-calico - readOnly: false - # This container installs the Calico CNI binaries - # and CNI network config file on each node. - - name: install-cni - image: quay.io/calico/cni:v3.4.0-0.dev-34-g83daff2 - command: ["/install-cni.sh"] - env: - # Name of the CNI config file to create. - - name: CNI_CONF_NAME - value: "10-calico.conflist" - # The CNI network config to install on each node. - - name: CNI_NETWORK_CONFIG - valueFrom: - configMapKeyRef: - name: calico-config - key: cni_network_config - # Set the hostname based on the k8s node name. - - name: KUBERNETES_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - volumeMounts: - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - - mountPath: /host/etc/cni/net.d - name: cni-net-dir + - name: calico-node + image: calico/node:v3.5.0 + env: + # Use Kubernetes API as the backing datastore. + - name: DATASTORE_TYPE + value: "kubernetes" + # Typha support: controlled by the ConfigMap. + - name: FELIX_TYPHAK8SSERVICENAME + valueFrom: + configMapKeyRef: + name: calico-config + key: typha_service_name + # Wait for the datastore. + - name: WAIT_FOR_DATASTORE + value: "true" + # Set based on the k8s node name. + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # Don't enable BGP. + - name: CALICO_NETWORKING_BACKEND + value: "none" + # Cluster type to identify the deployment type + - name: CLUSTER_TYPE + value: "k8s" + # The default IPv4 pool to create on startup if none exists. Pod IPs will be + # chosen from this range. Changing this value after installation will have + # no effect. This should fall within `--cluster-cidr`. + - name: CALICO_IPV4POOL_CIDR + value: "" + # Disable file logging so `kubectl logs` works. + - name: CALICO_DISABLE_FILE_LOGGING + value: "true" + # Set Felix endpoint to host default action to ACCEPT. + - name: FELIX_DEFAULTENDPOINTTOHOSTACTION + value: "ACCEPT" + # Disable IPv6 on Kubernetes. + - name: FELIX_IPV6SUPPORT + value: "false" + # Set Felix logging to "info" + - name: FELIX_LOGSEVERITYSCREEN + value: "info" + - name: FELIX_HEALTHENABLED + value: "true" + - name: CALICO_IPV4POOL_IPIP + value: "off" + - name: FELIX_INTERFACEPREFIX + value: "azv" + securityContext: + privileged: true + resources: + requests: + cpu: 250m + livenessProbe: + httpGet: + path: /liveness + port: 9099 + host: localhost + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + readinessProbe: + exec: + command: + - /bin/calico-node + - -felix-ready + periodSeconds: 10 + volumeMounts: + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - mountPath: /var/run/calico + name: var-run-calico + readOnly: false + - mountPath: /var/lib/calico + name: var-lib-calico + readOnly: false volumes: - # Used by calico/node. - - name: lib-modules - hostPath: - path: /lib/modules - - name: var-run-calico - hostPath: - path: /var/run/calico - - name: var-lib-calico - hostPath: - path: /var/lib/calico - # Used to install CNI. - - name: cni-bin-dir - hostPath: - path: /opt/cni/bin - - name: cni-net-dir - hostPath: - path: /etc/cni/net.d + # Used by calico/node. + - name: lib-modules + hostPath: + path: /lib/modules + - name: var-run-calico + hostPath: + path: /var/run/calico + - name: var-lib-calico + hostPath: + path: /var/lib/calico + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + # Used to install CNI. + - name: cni-bin-dir + hostPath: + path: /opt/cni/bin + - name: cni-net-dir + hostPath: + path: /etc/cni/net.d +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-node + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: "EnsureExists" +--- # Create all the CustomResourceDefinitions needed for # Calico policy-only mode. ---- -# Calico Felix Configuration + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -600,9 +365,8 @@ spec: kind: FelixConfiguration plural: felixconfigurations singular: felixconfiguration - --- -# Calico BGP Configuration + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -617,9 +381,8 @@ spec: kind: BGPConfiguration plural: bgpconfigurations singular: bgpconfiguration - --- -# Calico IP Pools + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -634,9 +397,8 @@ spec: kind: IPPool plural: ippools singular: ippool - --- -# Calico Host Endpoints + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -651,9 +413,8 @@ spec: kind: HostEndpoint plural: hostendpoints singular: hostendpoint - --- -# Calico Cluster Information + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -668,9 +429,8 @@ spec: kind: ClusterInformation plural: clusterinformations singular: clusterinformation - --- -# Calico Global Network Policies + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -685,9 +445,8 @@ spec: kind: GlobalNetworkPolicy plural: globalnetworkpolicies singular: globalnetworkpolicy - --- -# Calico Global Network Sets + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -702,9 +461,8 @@ spec: kind: GlobalNetworkSet plural: globalnetworksets singular: globalnetworkset - --- -# Calico Network Policies + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -719,3 +477,272 @@ spec: kind: NetworkPolicy plural: networkpolicies singular: networkpolicy +--- + +# Include a clusterrole for the calico-node DaemonSet, +# and bind it to the calico-node serviceaccount. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: calico-node + labels: + addonmanager.kubernetes.io/mode: "EnsureExists" +rules: +# The CNI plugin needs to get pods, nodes, and namespaces. +- apiGroups: [""] + resources: + - pods + - nodes + - namespaces + verbs: + - get +- apiGroups: [""] + resources: + - endpoints + - services + verbs: + # Used to discover service IPs for advertisement. + - watch + - list + # Used to discover Typhas. + - get +- apiGroups: [""] + resources: + - nodes/status + verbs: + # Needed for clearing NodeNetworkUnavailable flag. + - patch + # Calico stores some configuration information in node annotations. + - update +# Watch for changes to Kubernetes NetworkPolicies. +- apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: + - watch + - list +# Used by Calico for policy information. +- apiGroups: [""] + resources: + - pods + - namespaces + - serviceaccounts + verbs: + - list + - watch +# The CNI plugin patches pods/status. +- apiGroups: [""] + resources: + - pods/status + verbs: + - patch +# Calico monitors various CRDs for config. +- apiGroups: ["crd.projectcalico.org"] + resources: + - globalfelixconfigs + - felixconfigurations + - bgppeers + - globalbgpconfigs + - bgpconfigurations + - ippools + - globalnetworkpolicies + - globalnetworksets + - networkpolicies + - clusterinformations + - hostendpoints + verbs: + - get + - list + - watch +# Calico must create and update some CRDs on startup. +- apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + - felixconfigurations + - clusterinformations + verbs: + - create + - update +# Calico stores some configuration information on the node. +- apiGroups: [""] + resources: + - nodes + verbs: + - get + - list + - watch +# These permissions are only requried for upgrade from v2.6, and can +# be removed after upgrade or on fresh installations. +- apiGroups: ["crd.projectcalico.org"] + resources: + - bgpconfigurations + - bgppeers + verbs: + - create + - update +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: calico-node + labels: + addonmanager.kubernetes.io/mode: "EnsureExists" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-node +subjects: +- kind: ServiceAccount + name: calico-node + namespace: kube-system +--- + +# Typha Horizontal Autoscaler ConfigMap +kind: ConfigMap +apiVersion: v1 +metadata: + name: calico-typha-horizontal-autoscaler + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +data: + ladder: |- + { + "coresToReplicas": [], + "nodesToReplicas": + [ + [1, 1], + [10, 2], + [100, 3], + [250, 4], + [500, 5], + [1000, 6], + [1500, 7], + [2000, 8] + ] + } + +--- + +# Typha Horizontal Autoscaler Deployment +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: calico-typha-horizontal-autoscaler + namespace: kube-system + labels: + k8s-app: calico-typha-autoscaler + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +spec: + replicas: 1 + template: + metadata: + labels: + k8s-app: calico-typha-autoscaler + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + priorityClassName: system-cluster-critical + securityContext: + supplementalGroups: [65534] + fsGroup: 65534 + containers: + - image: k8s.gcr.io/cluster-proportional-autoscaler-amd64:1.1.2-r2 + name: autoscaler + command: + - /cluster-proportional-autoscaler + - --namespace=kube-system + - --configmap=calico-typha-horizontal-autoscaler + - --target=deployment/calico-typha + - --logtostderr=true + - --v=2 + resources: + requests: + cpu: 10m + limits: + cpu: 10m + serviceAccountName: typha-cpha +--- + +# Typha Horizontal Autoscaler Cluster Role +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: typha-cpha + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +rules: +- apiGroups: [""] + resources: ["nodes"] + verbs: ["list"] + +--- + +# Typha Horizontal Autoscaler Cluster Role Binding +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: typha-cpha + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: typha-cpha +subjects: +- kind: ServiceAccount + name: typha-cpha + namespace: kube-system +--- + +# Typha Horizontal Autoscaler Role +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: typha-cpha + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] +- apiGroups: ["extensions"] + resources: ["deployments/scale"] + verbs: ["get", "update"] + +--- + +# Typha Horizontal Autoscaler Role Binding +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: typha-cpha + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: typha-cpha +subjects: +- kind: ServiceAccount + name: typha-cpha + namespace: kube-system +--- + +# Typha Horizontal Autoscaler Service Account +apiVersion: v1 +kind: ServiceAccount +metadata: + name: typha-cpha + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + diff --git a/parts/k8s/addons/kubernetesmasteraddons-calico-daemonset.yaml b/parts/k8s/addons/kubernetesmasteraddons-calico-daemonset.yaml index deeb5f8683f..f7889a7d98f 100644 --- a/parts/k8s/addons/kubernetesmasteraddons-calico-daemonset.yaml +++ b/parts/k8s/addons/kubernetesmasteraddons-calico-daemonset.yaml @@ -1,108 +1,8 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: calico-node - namespace: kube-system - labels: - addonmanager.kubernetes.io/mode: "EnsureExists" ---- - -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: calico-node - labels: - addonmanager.kubernetes.io/mode: "EnsureExists" -rules: - - apiGroups: [""] - resources: - - namespaces - - serviceaccounts - verbs: - - get - - list - - watch - - apiGroups: [""] - resources: - - pods/status - verbs: - - patch - - apiGroups: [""] - resources: - - pods - verbs: - - get - - list - - watch - - apiGroups: [""] - resources: - - services - verbs: - - get - - apiGroups: [""] - resources: - - endpoints - verbs: - - get - - apiGroups: [""] - resources: - - nodes - verbs: - - get - - list - - update - - watch - - apiGroups: ["extensions"] - resources: - - networkpolicies - verbs: - - get - - list - - watch - - apiGroups: ["networking.k8s.io"] - resources: - - networkpolicies - verbs: - - watch - - list - - apiGroups: ["crd.projectcalico.org"] - resources: - - globalfelixconfigs - - felixconfigurations - - bgppeers - - globalbgpconfigs - - bgpconfigurations - - ippools - - globalnetworkpolicies - - globalnetworksets - - networkpolicies - - clusterinformations - - hostendpoints - verbs: - - create - - get - - list - - update - - watch - ---- - -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: calico-node - labels: - addonmanager.kubernetes.io/mode: "EnsureExists" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-node -subjects: -- kind: ServiceAccount - name: calico-node - namespace: kube-system - ---- +# Calico Version v3.5.0 +# https://docs.projectcalico.org/v3.5/releases#v3.5.0 +# This manifest includes the following component versions: +# calico/node:v3.5.0 +# calico/cni:v3.5.0 # This ConfigMap is used to configure a self-hosted Calico installation. kind: ConfigMap @@ -113,9 +13,11 @@ metadata: labels: addonmanager.kubernetes.io/mode: "EnsureExists" data: + # You must set a non-zero value for Typha replicas below. typha_service_name: "calico-typha" - # The CNI network configuration to install on each node. + # The CNI network configuration to install on each node. The special + # values in this config will be automatically populated. cni_network_config: |- { "name": "k8s-pod-network", @@ -158,13 +60,12 @@ metadata: addonmanager.kubernetes.io/mode: "EnsureExists" spec: ports: - - port: 5473 - protocol: TCP - targetPort: calico-typha - name: calico-typha + - port: 5473 + protocol: TCP + targetPort: calico-typha + name: calico-typha selector: k8s-app: calico-typha - --- # This manifest creates a Deployment of Typha to back the above service. @@ -184,228 +85,77 @@ spec: # We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is essential # (when using the Kubernetes datastore). Use one replica for every 100-200 nodes. In # production, we recommend running at least 3 replicas to reduce the impact of rolling upgrade. - replicas: 0 + replicas: 1 revisionHistoryLimit: 2 template: metadata: labels: k8s-app: calico-typha - addonmanager.kubernetes.io/mode: "EnsureExists" annotations: # This, along with the CriticalAddonsOnly toleration below, marks the pod as a critical # add-on, ensuring it gets priority scheduling and that its resources are reserved # if it ever gets evicted. scheduler.alpha.kubernetes.io/critical-pod: '' + cluster-autoscaler.kubernetes.io/safe-to-evict: 'true' spec: - hostNetwork: true nodeSelector: beta.kubernetes.io/os: linux + hostNetwork: true tolerations: + # Mark the pod as a critical add-on for rescheduling. - key: CriticalAddonsOnly operator: Exists # Since Calico can't network a pod until Typha is up, we need to run Typha itself # as a host-networked pod. serviceAccountName: calico-node containers: - - image: quay.io/calico/typha:v3.3.1 + - image: calico/typha:v3.5.0 name: calico-typha ports: - containerPort: 5473 name: calico-typha protocol: TCP env: - # Enable "info" logging by default. Can be set to "debug" to increase verbosity. - - name: TYPHA_LOGSEVERITYSCREEN - value: "info" - # Disable logging to file and syslog since those don't make sense in Kubernetes. - - name: TYPHA_LOGFILEPATH - value: "none" - - name: TYPHA_LOGSEVERITYSYS - value: "none" - # Monitor the Kubernetes API to find the number of running instances and rebalance - # connections. - - name: TYPHA_CONNECTIONREBALANCINGMODE - value: "kubernetes" - - name: TYPHA_DATASTORETYPE - value: "kubernetes" - - name: TYPHA_HEALTHENABLED - value: "true" - # Uncomment these lines to enable prometheus metrics. Since Typha is host-networked, - # this opens a port on the host, which may need to be secured. - #- name: TYPHA_PROMETHEUSMETRICSENABLED - # value: "true" - #- name: TYPHA_PROMETHEUSMETRICSPORT - # value: "9093" + # Enable "info" logging by default. Can be set to "debug" to increase verbosity. + - name: TYPHA_LOGSEVERITYSCREEN + value: "info" + # Disable logging to file and syslog since those don't make sense in Kubernetes. + - name: TYPHA_LOGFILEPATH + value: "none" + - name: TYPHA_LOGSEVERITYSYS + value: "none" + # Monitor the Kubernetes API to find the number of running instances and rebalance + # connections. + - name: TYPHA_CONNECTIONREBALANCINGMODE + value: "kubernetes" + - name: TYPHA_DATASTORETYPE + value: "kubernetes" + - name: TYPHA_HEALTHENABLED + value: "true" + - name: FELIX_INTERFACEPREFIX + value: "azv" + # Uncomment these lines to enable prometheus metrics. Since Typha is host-networked, + # this opens a port on the host, which may need to be secured. + #- name: TYPHA_PROMETHEUSMETRICSENABLED + # value: "true" + #- name: TYPHA_PROMETHEUSMETRICSPORT + # value: "9093" livenessProbe: - httpGet: - path: /liveness - port: 9098 - host: localhost + exec: + command: + - calico-typha + - check + - liveness periodSeconds: 30 initialDelaySeconds: 30 readinessProbe: - httpGet: - path: /readiness - port: 9098 - host: localhost + exec: + command: + - calico-typha + - check + - readiness periodSeconds: 10 - --- - -# Typha Horizontal Autoscaler Cluster Role -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: typha-cpha - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["list"] - ---- - -# Typha Horizontal Autoscaler Cluster Role Binding -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: typha-cpha - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: typha-cpha -subjects: - - kind: ServiceAccount - name: typha-cpha - namespace: kube-system - ---- - -# Typha Horizontal Autoscaler ConfigMap -kind: ConfigMap -apiVersion: v1 -metadata: - name: calico-typha-horizontal-autoscaler - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -data: - ladder: |- - { - "coresToReplicas": [], - "nodesToReplicas": - [ - [1, 1], - [10, 2], - [100, 3], - [250, 4], - [500, 5], - [1000, 6], - [1500, 7], - [2000, 8] - ] - } - ---- - -# Typha Horizontal Autoscaler Deployment -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: calico-typha-horizontal-autoscaler - namespace: kube-system - labels: - k8s-app: calico-typha-autoscaler - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -spec: - replicas: 1 - template: - metadata: - labels: - k8s-app: calico-typha-autoscaler - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' - spec: - priorityClassName: system-node-critical - securityContext: - supplementalGroups: [ 65534 ] - fsGroup: 65534 - containers: - - image: k8s.gcr.io/cluster-proportional-autoscaler-amd64:1.1.2-r2 - name: autoscaler - command: - - /cluster-proportional-autoscaler - - --namespace=kube-system - - --configmap=calico-typha-horizontal-autoscaler - - --target=deployment/calico-typha - - --logtostderr=true - - --v=2 - resources: - requests: - cpu: 10m - limits: - cpu: 10m - serviceAccountName: typha-cpha - ---- - -# Typha Horizontal Autoscaler Role -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: typha-cpha - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -rules: - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get"] - - apiGroups: ["extensions"] - resources: ["deployments/scale"] - verbs: ["get", "update"] - ---- - -# Typha Horizontal Autoscaler Role Binding -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: typha-cpha - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: "EnsureExists" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: typha-cpha -subjects: - - kind: ServiceAccount - name: typha-cpha - namespace: kube-system - ---- - -# Typha Horizontal Autoscaler Service Account -apiVersion: v1 -kind: ServiceAccount -metadata: - name: typha-cpha - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - ---- - # This manifest installs the calico/node container, as well # as the Calico CNI plugins and network config on # each master and worker node in a Kubernetes cluster. @@ -416,7 +166,6 @@ metadata: namespace: kube-system labels: k8s-app: calico-node - # github.com/kubernetes/kubernetes/tree/master/cluster/addons/addon-manager addonmanager.kubernetes.io/mode: "EnsureExists" spec: selector: @@ -435,155 +184,173 @@ spec: # marks the pod as a critical add-on, ensuring it gets # priority scheduling and that its resources are reserved # if it ever gets evicted. - # Deprecated in 1.10, Removed in 1.11. kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods scheduler.alpha.kubernetes.io/critical-pod: '' spec: + nodeSelector: + beta.kubernetes.io/os: linux hostNetwork: true tolerations: - # Allow the pod to run on the master. This is required for - # the master to communicate with pods. - - effect: NoSchedule - operator: Exists - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - operator: Exists + # Make sure calico-node gets scheduled on all nodes. + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists serviceAccountName: calico-node # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. terminationGracePeriodSeconds: 0 + initContainers: + # This container installs the Calico CNI binaries + # and CNI network config file on each node. + - name: install-cni + image: calico/cni:v3.5.0 + command: ["/install-cni.sh"] + env: + # Name of the CNI config file to create. + - name: CNI_CONF_NAME + value: "10-calico.conflist" + # The CNI network config to install on each node. + - name: CNI_NETWORK_CONFIG + valueFrom: + configMapKeyRef: + name: calico-config + key: cni_network_config + # Set the hostname based on the k8s node name. + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # Prevents the container from sleeping forever. + - name: SLEEP + value: "false" + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + - mountPath: /host/etc/cni/net.d + name: cni-net-dir containers: # Runs calico/node container on each Kubernetes node. This # container programs network policy and routes on each # host. - - name: calico-node - image: quay.io/calico/node:v3.3.1 - env: - # Use Kubernetes API as the backing datastore. - - name: DATASTORE_TYPE - value: "kubernetes" - # Enable felix info logging. - - name: FELIX_LOGSEVERITYSCREEN - value: "info" - # Don't enable BGP. - - name: CALICO_NETWORKING_BACKEND - value: "none" - # Cluster type to identify the deployment type - - name: CLUSTER_TYPE - value: "k8s" - # Disable file logging so `kubectl logs` works. - - name: CALICO_DISABLE_FILE_LOGGING - value: "true" - # Set Felix endpoint to host default action to ACCEPT. - - name: FELIX_DEFAULTENDPOINTTOHOSTACTION - value: "ACCEPT" - # Disable IPV6 on Kubernetes. - - name: FELIX_IPV6SUPPORT - value: "false" - # Wait for the datastore. - - name: WAIT_FOR_DATASTORE - value: "true" - # The Calico IPv4 pool to use. This should match `--cluster-cidr` - - name: CALICO_IPV4POOL_CIDR - value: "" - # Enable IPIP - - name: CALICO_IPV4POOL_IPIP - value: "off" - - name: FELIX_IPTABLESREFRESHINTERVAL - value: "60" - # Typha support: controlled by the ConfigMap. - - name: FELIX_TYPHAK8SSERVICENAME - valueFrom: - configMapKeyRef: - name: calico-config - key: typha_service_name - # Set based on the k8s node name. - - name: NODENAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: FELIX_HEALTHENABLED - value: "true" - - name: FELIX_IPINIPENABLED - value: "false" - securityContext: - privileged: true - resources: - requests: - cpu: 250m - livenessProbe: - httpGet: - path: /liveness - port: 9099 - host: localhost - periodSeconds: 10 - initialDelaySeconds: 10 - failureThreshold: 6 - readinessProbe: - httpGet: - path: /readiness - port: 9099 - host: localhost - periodSeconds: 10 - volumeMounts: - - mountPath: /lib/modules - name: lib-modules - readOnly: true - - mountPath: /var/run/calico - name: var-run-calico - readOnly: false - - mountPath: /var/lib/calico - name: var-lib-calico - readOnly: false - # This container installs the Calico CNI binaries - # and CNI network config file on each node. - - name: install-cni - image: quay.io/calico/cni:v3.4.0-0.dev-34-g83daff2 - command: ["/install-cni.sh"] - env: - # Name of the CNI config file to create. - - name: CNI_CONF_NAME - value: "10-calico.conflist" - # The CNI network config to install on each node. - - name: CNI_NETWORK_CONFIG - valueFrom: - configMapKeyRef: - name: calico-config - key: cni_network_config - # Set the hostname based on the k8s node name. - - name: KUBERNETES_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - volumeMounts: - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - - mountPath: /host/etc/cni/net.d - name: cni-net-dir + - name: calico-node + image: calico/node:v3.5.0 + env: + # Use Kubernetes API as the backing datastore. + - name: DATASTORE_TYPE + value: "kubernetes" + # Typha support: controlled by the ConfigMap. + - name: FELIX_TYPHAK8SSERVICENAME + valueFrom: + configMapKeyRef: + name: calico-config + key: typha_service_name + # Wait for the datastore. + - name: WAIT_FOR_DATASTORE + value: "true" + # Set based on the k8s node name. + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # Don't enable BGP. + - name: CALICO_NETWORKING_BACKEND + value: "none" + # Cluster type to identify the deployment type + - name: CLUSTER_TYPE + value: "k8s" + # The default IPv4 pool to create on startup if none exists. Pod IPs will be + # chosen from this range. Changing this value after installation will have + # no effect. This should fall within `--cluster-cidr`. + - name: CALICO_IPV4POOL_CIDR + value: "" + # Disable file logging so `kubectl logs` works. + - name: CALICO_DISABLE_FILE_LOGGING + value: "true" + # Set Felix endpoint to host default action to ACCEPT. + - name: FELIX_DEFAULTENDPOINTTOHOSTACTION + value: "ACCEPT" + # Disable IPv6 on Kubernetes. + - name: FELIX_IPV6SUPPORT + value: "false" + # Set Felix logging to "info" + - name: FELIX_LOGSEVERITYSCREEN + value: "info" + - name: FELIX_HEALTHENABLED + value: "true" + - name: CALICO_IPV4POOL_IPIP + value: "off" + - name: FELIX_INTERFACEPREFIX + value: "azv" + securityContext: + privileged: true + resources: + requests: + cpu: 250m + livenessProbe: + httpGet: + path: /liveness + port: 9099 + host: localhost + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + readinessProbe: + exec: + command: + - /bin/calico-node + - -felix-ready + periodSeconds: 10 + volumeMounts: + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - mountPath: /var/run/calico + name: var-run-calico + readOnly: false + - mountPath: /var/lib/calico + name: var-lib-calico + readOnly: false volumes: - # Used by calico/node. - - name: lib-modules - hostPath: - path: /lib/modules - - name: var-run-calico - hostPath: - path: /var/run/calico - - name: var-lib-calico - hostPath: - path: /var/lib/calico - # Used to install CNI. - - name: cni-bin-dir - hostPath: - path: /opt/cni/bin - - name: cni-net-dir - hostPath: - path: /etc/cni/net.d + # Used by calico/node. + - name: lib-modules + hostPath: + path: /lib/modules + - name: var-run-calico + hostPath: + path: /var/run/calico + - name: var-lib-calico + hostPath: + path: /var/lib/calico + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + # Used to install CNI. + - name: cni-bin-dir + hostPath: + path: /opt/cni/bin + - name: cni-net-dir + hostPath: + path: /etc/cni/net.d +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-node + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: "EnsureExists" +--- # Create all the CustomResourceDefinitions needed for # Calico policy-only mode. ---- -# Calico Felix Configuration + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -598,9 +365,8 @@ spec: kind: FelixConfiguration plural: felixconfigurations singular: felixconfiguration - --- -# Calico BGP Configuration + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -615,9 +381,8 @@ spec: kind: BGPConfiguration plural: bgpconfigurations singular: bgpconfiguration - --- -# Calico IP Pools + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -632,9 +397,8 @@ spec: kind: IPPool plural: ippools singular: ippool - --- -# Calico Host Endpoints + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -649,9 +413,8 @@ spec: kind: HostEndpoint plural: hostendpoints singular: hostendpoint - --- -# Calico Cluster Information + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -666,9 +429,8 @@ spec: kind: ClusterInformation plural: clusterinformations singular: clusterinformation - --- -# Calico Global Network Policies + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -683,9 +445,8 @@ spec: kind: GlobalNetworkPolicy plural: globalnetworkpolicies singular: globalnetworkpolicy - --- -# Calico Global Network Sets + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -700,9 +461,8 @@ spec: kind: GlobalNetworkSet plural: globalnetworksets singular: globalnetworkset - --- -# Calico Network Policies + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -717,3 +477,272 @@ spec: kind: NetworkPolicy plural: networkpolicies singular: networkpolicy +--- + +# Include a clusterrole for the calico-node DaemonSet, +# and bind it to the calico-node serviceaccount. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: calico-node + labels: + addonmanager.kubernetes.io/mode: "EnsureExists" +rules: +# The CNI plugin needs to get pods, nodes, and namespaces. +- apiGroups: [""] + resources: + - pods + - nodes + - namespaces + verbs: + - get +- apiGroups: [""] + resources: + - endpoints + - services + verbs: + # Used to discover service IPs for advertisement. + - watch + - list + # Used to discover Typhas. + - get +- apiGroups: [""] + resources: + - nodes/status + verbs: + # Needed for clearing NodeNetworkUnavailable flag. + - patch + # Calico stores some configuration information in node annotations. + - update +# Watch for changes to Kubernetes NetworkPolicies. +- apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: + - watch + - list +# Used by Calico for policy information. +- apiGroups: [""] + resources: + - pods + - namespaces + - serviceaccounts + verbs: + - list + - watch +# The CNI plugin patches pods/status. +- apiGroups: [""] + resources: + - pods/status + verbs: + - patch +# Calico monitors various CRDs for config. +- apiGroups: ["crd.projectcalico.org"] + resources: + - globalfelixconfigs + - felixconfigurations + - bgppeers + - globalbgpconfigs + - bgpconfigurations + - ippools + - globalnetworkpolicies + - globalnetworksets + - networkpolicies + - clusterinformations + - hostendpoints + verbs: + - get + - list + - watch +# Calico must create and update some CRDs on startup. +- apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + - felixconfigurations + - clusterinformations + verbs: + - create + - update +# Calico stores some configuration information on the node. +- apiGroups: [""] + resources: + - nodes + verbs: + - get + - list + - watch +# These permissions are only requried for upgrade from v2.6, and can +# be removed after upgrade or on fresh installations. +- apiGroups: ["crd.projectcalico.org"] + resources: + - bgpconfigurations + - bgppeers + verbs: + - create + - update +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: calico-node + labels: + addonmanager.kubernetes.io/mode: "EnsureExists" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-node +subjects: +- kind: ServiceAccount + name: calico-node + namespace: kube-system +--- + +# Typha Horizontal Autoscaler ConfigMap +kind: ConfigMap +apiVersion: v1 +metadata: + name: calico-typha-horizontal-autoscaler + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +data: + ladder: |- + { + "coresToReplicas": [], + "nodesToReplicas": + [ + [1, 1], + [10, 2], + [100, 3], + [250, 4], + [500, 5], + [1000, 6], + [1500, 7], + [2000, 8] + ] + } + +--- + +# Typha Horizontal Autoscaler Deployment +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: calico-typha-horizontal-autoscaler + namespace: kube-system + labels: + k8s-app: calico-typha-autoscaler + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +spec: + replicas: 1 + template: + metadata: + labels: + k8s-app: calico-typha-autoscaler + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + priorityClassName: system-cluster-critical + securityContext: + supplementalGroups: [65534] + fsGroup: 65534 + containers: + - image: k8s.gcr.io/cluster-proportional-autoscaler-amd64:1.1.2-r2 + name: autoscaler + command: + - /cluster-proportional-autoscaler + - --namespace=kube-system + - --configmap=calico-typha-horizontal-autoscaler + - --target=deployment/calico-typha + - --logtostderr=true + - --v=2 + resources: + requests: + cpu: 10m + limits: + cpu: 10m + serviceAccountName: typha-cpha +--- + +# Typha Horizontal Autoscaler Cluster Role +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: typha-cpha + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +rules: +- apiGroups: [""] + resources: ["nodes"] + verbs: ["list"] + +--- + +# Typha Horizontal Autoscaler Cluster Role Binding +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: typha-cpha + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: typha-cpha +subjects: +- kind: ServiceAccount + name: typha-cpha + namespace: kube-system +--- + +# Typha Horizontal Autoscaler Role +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: typha-cpha + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] +- apiGroups: ["extensions"] + resources: ["deployments/scale"] + verbs: ["get", "update"] + +--- + +# Typha Horizontal Autoscaler Role Binding +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: typha-cpha + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: typha-cpha +subjects: +- kind: ServiceAccount + name: typha-cpha + namespace: kube-system +--- + +# Typha Horizontal Autoscaler Service Account +apiVersion: v1 +kind: ServiceAccount +metadata: + name: typha-cpha + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + diff --git a/parts/k8s/kubernetesinstalls.sh b/parts/k8s/kubernetesinstalls.sh index b250860a7da..669e272de88 100755 --- a/parts/k8s/kubernetesinstalls.sh +++ b/parts/k8s/kubernetesinstalls.sh @@ -191,7 +191,7 @@ installClearContainersRuntime() { } installNetworkPlugin() { - if [ "${NETWORK_PLUGIN}" = "azure" ]; then + if [[ "${NETWORK_PLUGIN}" = "azure" ]]; then installAzureCNI fi installCNI