Update to Kubernetes v1.16

Signed-off-by: Mikkel Oscar Lyderik Larsen <[email protected]>
zalando-incubator · Dec 19, 2019 · 7b2efba · 7b2efba
1 parent 3c78950
commit 7b2efba
Show file tree

Hide file tree

Showing 24 changed files with 363 additions and 90 deletions.
diff --git a/cluster/config-defaults.yaml b/cluster/config-defaults.yaml
@@ -243,8 +243,8 @@ dynamodb_service_link_enabled: "false"
 cluster_dns: "coredns"
 coredns_log_svc_names: "true"
 
-kuberuntu_image_v1_14: {{ amiID "zalando-ubuntu-kubernetes-production-v1.14.8-master-77" "861068367966" }}
 kuberuntu_image_v1_15: {{ amiID "zalando-ubuntu-kubernetes-production-v1.15.6-master-81" "861068367966" }}
+kuberuntu_image_v1_16: {{ amiID "zalando-ubuntu-kubernetes-production-v1.16.4-master-84" "861068367966" }}
 
 # Feature toggle to allow gradual decommissioning of ingress-template-controller
 enable_ingress_template_controller: "false"
@@ -265,14 +265,6 @@ audittrail_url: ""
 {{end}}
 audittrail_root_account_role: ""
 
-# Feature toggle for CustomResourceWebhookConversion (alpha in v1.13)
-# https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definition-versioning/#webhook-conversion
-custom_resource_webhook_conversion: "false"
-
-# Feature toggle for CustomResourcePublishOpenAPI (alpha in v1.14)
-# https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/#publish-validation-schema-in-openapi-v2
-custom_resource_publish_openapi: "false"
-
 # CIDR configuration for nodes and pods
 # Changing this will change the number of nodes and pods we can schedule in the
 # cluster: https://cloud.google.com/kubernetes-engine/docs/how-to/flexible-pod-cidr
@@ -309,3 +301,6 @@ custom_dns_zone_nameservers: "" # space seperated list of nameserver IP addresse
 
 # prefix prepended to ownership TXT records for external-dns
 external_dns_ownership_prefix: ""
+
+# Enable FeatureGate EndpointSlice
+enable_endpointslice: "false"
diff --git a/cluster/manifests/01-platformcredentialsset/customresourcedefinition.yaml b/cluster/manifests/01-platformcredentialsset/customresourcedefinition.yaml
@@ -1,6 +1,6 @@
 # A specification to declare needed OAuth credentials (tokens, clients) for the
 # Zalando Platform IAM system
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   name: platformcredentialssets.zalando.org

diff --git a/cluster/manifests/01-vertical-pod-autoscaler/01-crd.yaml b/cluster/manifests/01-vertical-pod-autoscaler/01-crd.yaml
@@ -1,5 +1,5 @@
 ---
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   name: verticalpodautoscalers.autoscaling.k8s.io
@@ -40,7 +40,7 @@ spec:
                 containerPolicies:
                   type: array
 ---
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   name: verticalpodautoscalercheckpoints.autoscaling.k8s.io

diff --git a/cluster/manifests/02-kube-aws-iam-controller/crd.yaml b/cluster/manifests/02-kube-aws-iam-controller/crd.yaml
@@ -1,4 +1,4 @@
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   name: awsiamroles.zalando.org

diff --git a/cluster/manifests/02-kube-aws-iam-controller/deployment.yaml b/cluster/manifests/02-kube-aws-iam-controller/deployment.yaml
@@ -36,8 +36,6 @@ spec:
             cpu: "{{.ConfigItems.kube_aws_iam_controller_cpu}}"
             memory: "{{.ConfigItems.kube_aws_iam_controller_mem}}"
       tolerations:
-      - key: node-role.kubernetes.io/master
-        effect: NoSchedule
       - key: node.kubernetes.io/role
         value: master
         effect: NoSchedule

diff --git a/cluster/manifests/admission-control/daemonset.yaml b/cluster/manifests/admission-control/daemonset.yaml
@@ -26,14 +26,12 @@ spec:
       dnsPolicy: Default
       hostNetwork: true
       tolerations:
-      - key: node-role.kubernetes.io/master
-        effect: NoSchedule
       - key: node.kubernetes.io/role
         value: master
         effect: NoSchedule
       containers:
       - name: cluster-autoscaler
-        image: registry.opensource.zalan.do/teapot/admission-controller:master-45
+        image: registry.opensource.zalan.do/teapot/admission-controller:master-49
         command:
           - /registry-proxy
           - --address=127.0.0.1:8285

diff --git a/cluster/manifests/cluster-lifecycle-controller/deployment.yaml b/cluster/manifests/cluster-lifecycle-controller/deployment.yaml
@@ -26,8 +26,6 @@ spec:
       priorityClassName: system-cluster-critical
       serviceAccountName: cluster-lifecycle-controller
       tolerations:
-      - key: node-role.kubernetes.io/master
-        effect: NoSchedule
       - key: node.kubernetes.io/role
         value: master
         effect: NoSchedule

diff --git a/cluster/manifests/etcd-backup/cronjob.yaml b/cluster/manifests/etcd-backup/cronjob.yaml
@@ -57,8 +57,6 @@ spec:
               readOnly: true
 {{ end }}
           tolerations:
-          - key: node-role.kubernetes.io/master
-            effect: NoSchedule
           - key: node.kubernetes.io/role
             value: master
             effect: NoSchedule

diff --git a/cluster/manifests/ingress-template-controller/crd.yaml b/cluster/manifests/ingress-template-controller/crd.yaml
@@ -1,5 +1,5 @@
 {{ if eq .ConfigItems.enable_ingress_template_controller "true"}}
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   name: ingresstemplates.zalando.org

diff --git a/cluster/manifests/kube-cluster-autoscaler/daemonset.yaml b/cluster/manifests/kube-cluster-autoscaler/daemonset.yaml
@@ -5,7 +5,7 @@ metadata:
   namespace: kube-system
   labels:
     application: kube-cluster-autoscaler
-    version: v1.12.2-internal-2.5
+    version: v1.12.2-internal-2.6
 spec:
   selector:
     matchLabels:
@@ -16,7 +16,7 @@ spec:
     metadata:
       labels:
         application: kube-cluster-autoscaler
-        version: v1.12.2-internal-2.5
+        version: v1.12.2-internal-2.6
       annotations:
         iam.amazonaws.com/role: "{{ .LocalID }}-app-autoscaler"
         config/pool-sizes: "{{range .NodePools}}{{.Name}}-{{.MinSize}}-{{.MaxSize}} {{end}}"
@@ -29,14 +29,12 @@ spec:
       serviceAccountName: cluster-autoscaler
       dnsPolicy: Default
       tolerations:
-      - key: node-role.kubernetes.io/master
-        effect: NoSchedule
       - key: node.kubernetes.io/role
         value: master
         effect: NoSchedule
       containers:
       - name: cluster-autoscaler
-        image: registry.opensource.zalan.do/teapot/kube-cluster-autoscaler:v1.12.2-internal-2.5
+        image: registry.opensource.zalan.do/teapot/kube-cluster-autoscaler:v1.12.2-internal-2.6
         command:
           - ./cluster-autoscaler
           - --v=4

diff --git a/cluster/node-pools/master-default/stack.yaml b/cluster/node-pools/master-default/stack.yaml
@@ -4,7 +4,7 @@ Description: Kubernetes default master node pool
 Mappings:
   Images:
     eu-central-1:
-      MachineImage: '{{ .Cluster.ConfigItems.kuberuntu_image_v1_15 }}'
+      MachineImage: '{{ .Cluster.ConfigItems.kuberuntu_image_v1_16 }}'
 
 Resources:
   AutoScalingGroup:
@@ -26,9 +26,6 @@ Resources:
       - Key: Name
         PropagateAtLaunch: true
         Value: "{{ .NodePool.Name }} ({{ .Cluster.ID }})"
-      - Key: kubernetes.io/role
-        PropagateAtLaunch: true
-        Value: master
       - Key: node.kubernetes.io/role
         PropagateAtLaunch: true
         Value: master

diff --git a/cluster/node-pools/master-default/userdata.yaml b/cluster/node-pools/master-default/userdata.yaml
@@ -3,8 +3,8 @@ write_files:
   - owner: root:root
     path: /etc/kubernetes/secrets.env
     content: |
-      NODEPOOL_TAINTS=node-role.kubernetes.io/master=:NoSchedule{{if index .NodePool.ConfigItems "taints"}},{{.NodePool.ConfigItems.taints}}{{end}}
-      NODE_LABELS=node-role.kubernetes.io/master,kubernetes.io/role=master,master=true,node.kubernetes.io/distro=ubuntu,cluster-lifecycle-controller.zalan.do/decommission-priority=999,{{ .Values.node_labels }}{{if index .NodePool.ConfigItems "labels"}},{{.NodePool.ConfigItems.labels}}{{end}}
+      NODEPOOL_TAINTS=node.kubernetes.io/role=master:NoSchedule{{if index .NodePool.ConfigItems "taints"}},{{.NodePool.ConfigItems.taints}}{{end}}
+      NODE_LABELS=master=true,node.kubernetes.io/distro=ubuntu,cluster-lifecycle-controller.zalan.do/decommission-priority=999,{{ .Values.node_labels }}{{if index .NodePool.ConfigItems "labels"}},{{.NodePool.ConfigItems.labels}}{{end}}
       NODEPOOL_NAME={{ .NodePool.Name }}
       KUBELET_ROLE=master
 
@@ -92,7 +92,8 @@ write_files:
       spec:
         priorityClassName: system-node-critical
         tolerations:
-        - key: node-role.kubernetes.io/master
+        - key: node.kubernetes.io/role
+          value: master
           effect: NoSchedule
         hostNetwork: true
         containers:
@@ -113,14 +114,14 @@ write_files:
           - --tls-cert-file=/etc/kubernetes/ssl/apiserver.pem
           - --tls-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem
           - --service-account-key-file=/etc/kubernetes/ssl/service-account-public-key.pem
-          - --runtime-config=extensions/v1beta1/networkpolicies=true,batch/v2alpha1=true,policy/v1beta1/podsecuritypolicy=true,imagepolicy.k8s.io/v1alpha1=true,authorization.k8s.io/v1beta1=true,scheduling.k8s.io/v1alpha1=true,admissionregistration.k8s.io/v1beta1=true
+          - --runtime-config=extensions/v1beta1/networkpolicies=true,batch/v2alpha1=true,policy/v1beta1/podsecuritypolicy=true,imagepolicy.k8s.io/v1alpha1=true,authorization.k8s.io/v1beta1=true,scheduling.k8s.io/v1alpha1=true,admissionregistration.k8s.io/v1beta1=true{{ if eq .Cluster.ConfigItems.enable_endpointslice "true" }},discovery.k8s.io/v1alpha1=true{{ end }}
           - --authentication-token-webhook-config-file=/etc/kubernetes/config/authn.yaml
           - --authentication-token-webhook-cache-ttl=10s
           - --cloud-provider=aws
           - --authorization-mode=Webhook,RBAC
           - --authorization-webhook-config-file=/etc/kubernetes/config/authz.yaml
           - --admission-control-config-file=/etc/kubernetes/config/image-policy-webhook.yaml
-          - --feature-gates=TaintNodesByCondition={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},ScheduleDaemonSetPods={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},TTLAfterFinished=true,CustomResourceWebhookConversion={{.Cluster.ConfigItems.custom_resource_webhook_conversion}},CustomResourcePublishOpenAPI={{.Cluster.ConfigItems.custom_resource_publish_openapi}}
+          - --feature-gates=TaintNodesByCondition={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},ScheduleDaemonSetPods={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},TTLAfterFinished=true{{ if eq .Cluster.ConfigItems.enable_endpointslice "true" }},EndpointSlice=true{{ end }}
           - --anonymous-auth=false
           {{ if ne .Cluster.ConfigItems.audittrail_url "" }}
           - --audit-webhook-config-file=/etc/kubernetes/config/audit.yaml
@@ -172,7 +173,7 @@ write_files:
             requests:
               cpu: 100m
               memory: 200Mi
-        - image: registry.opensource.zalan.do/teapot/admission-controller:master-48
+        - image: registry.opensource.zalan.do/teapot/admission-controller:master-49
           name: admission-controller
           readinessProbe:
             httpGet:
@@ -473,7 +474,8 @@ write_files:
       spec:
         priorityClassName: system-node-critical
         tolerations:
-        - key: node-role.kubernetes.io/master
+        - key: node.kubernetes.io/role
+          value: master
           effect: NoSchedule
         containers:
         - name: kube-controller-manager
@@ -499,6 +501,9 @@ write_files:
           - --horizontal-pod-autoscaler-sync-period={{ .Cluster.ConfigItems.horizontal_pod_autoscaler_sync_period }}
           - --horizontal-pod-autoscaler-tolerance={{ .Cluster.ConfigItems.horizontal_pod_autoscaler_tolerance }}
           - --horizontal-pod-autoscaler-upscale-delay={{ .Cluster.ConfigItems.horizontal_pod_autoscaler_upscale_delay }}
+          {{ if eq .Cluster.ConfigItems.enable_endpointslice "true" }}
+          - --controllers=endpointslice
+          {{ end }}
           resources:
             requests:
               cpu: 100m
@@ -539,7 +544,8 @@ write_files:
       spec:
         priorityClassName: system-node-critical
         tolerations:
-        - key: node-role.kubernetes.io/master
+        - key: node.kubernetes.io/role
+          value: master
           effect: NoSchedule
         hostNetwork: true
         containers:

diff --git a/cluster/node-pools/worker-default/stack.yaml b/cluster/node-pools/worker-default/stack.yaml
@@ -4,7 +4,7 @@ Description: Kubernetes default worker node pool
 Mappings:
   Images:
     eu-central-1:
-      MachineImage: '{{ .Cluster.ConfigItems.kuberuntu_image_v1_15 }}'
+      MachineImage: '{{ .Cluster.ConfigItems.kuberuntu_image_v1_16 }}'
 
 Resources:
   AutoScalingGroup:
@@ -42,9 +42,6 @@ Resources:
       - Key: k8s.io/role/node
         PropagateAtLaunch: true
         Value: worker
-      - Key: kubernetes.io/role
-        PropagateAtLaunch: true
-        Value: worker
       - Key: node.kubernetes.io/role
         PropagateAtLaunch: true
         Value: worker
@@ -57,15 +54,9 @@ Resources:
       - Key: k8s.io/cluster-autoscaler/enabled
         PropagateAtLaunch: true
         Value: ''
-      - Key: k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role
-        PropagateAtLaunch: true
-        Value: worker
       - Key: k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/role
         PropagateAtLaunch: true
         Value: worker
-      - Key: k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/node-pool
-        PropagateAtLaunch: true
-        Value: {{ .NodePool.Name }}
       - Key: k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/node-pool
         PropagateAtLaunch: true
         Value: {{ .NodePool.Name }}

diff --git a/cluster/node-pools/worker-splitaz/stack.yaml b/cluster/node-pools/worker-splitaz/stack.yaml
@@ -4,7 +4,7 @@ Description: Kubernetes default worker node pool
 Mappings:
   Images:
     eu-central-1:
-      MachineImage: '{{ .Cluster.ConfigItems.kuberuntu_image_v1_15 }}'
+      MachineImage: '{{ .Cluster.ConfigItems.kuberuntu_image_v1_16 }}'
 
 Resources:
 {{ with $data := . }}
@@ -46,9 +46,6 @@ Resources:
       - Key: k8s.io/role/node
         PropagateAtLaunch: true
         Value: worker
-      - Key: kubernetes.io/role
-        PropagateAtLaunch: true
-        Value: worker
       - Key: node.kubernetes.io/role
         PropagateAtLaunch: true
         Value: worker
@@ -61,15 +58,9 @@ Resources:
       - Key: k8s.io/cluster-autoscaler/enabled
         PropagateAtLaunch: true
         Value: ''
-      - Key: k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role
-        PropagateAtLaunch: true
-        Value: worker
       - Key: k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/role
         PropagateAtLaunch: true
         Value: worker
-      - Key: k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/node-pool
-        PropagateAtLaunch: true
-        Value: {{ $data.NodePool.Name }}
       - Key: k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/node-pool
         PropagateAtLaunch: true
         Value: {{ $data.NodePool.Name }}

diff --git a/test/e2e/Makefile b/test/e2e/Makefile
@@ -2,7 +2,7 @@
 
 BINARY       ?= kubernetes-on-aws-e2e
 VERSION      ?= $(shell git describe --tags --always --dirty)
-KUBE_VERSION ?= v1.15.6
+KUBE_VERSION ?= v1.16.4
 IMAGE        ?= registry-write.opensource.zalan.do/teapot/$(BINARY)
 TAG          ?= $(VERSION)
 DOCKERFILE   ?= Dockerfile

diff --git a/test/e2e/admission_controller.go b/test/e2e/admission_controller.go
@@ -32,6 +32,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
 	deploymentframework "k8s.io/kubernetes/test/e2e/framework/deployment"
+	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
 )
 
 const (
@@ -68,7 +69,7 @@ var _ = framework.KubeDescribe("Admission controller tests", func() {
 		Expect(err).NotTo(HaveOccurred())
 
 		//pods are not returned here
-		_, err = framework.WaitForPodsWithLabelRunningReady(cs, ns, labelSelector, int(replicas), 1*time.Minute)
+		_, err = e2epod.WaitForPodsWithLabelRunningReady(cs, ns, labelSelector, int(replicas), 1*time.Minute)
 		Expect(err).NotTo(HaveOccurred())
 
 		pods, err := cs.CoreV1().Pods(ns).List(metav1.ListOptions{LabelSelector: labelSelector.String()})
@@ -123,7 +124,7 @@ var _ = framework.KubeDescribe("Admission controller tests", func() {
 		_, err := cs.CoreV1().Pods(ns).Create(pod)
 		Expect(err).NotTo(HaveOccurred())
 
-		err = framework.WaitForPodSuccessInNamespaceSlow(cs, podName, ns)
+		err = e2epod.WaitForPodSuccessInNamespaceSlow(cs, podName, ns)
 		Expect(err).NotTo(HaveOccurred())
 	})
 })

diff --git a/test/e2e/apiserver.go b/test/e2e/apiserver.go
@@ -30,6 +30,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
 	deploymentframework "k8s.io/kubernetes/test/e2e/framework/deployment"
+	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
 )
 
 var _ = framework.KubeDescribe("API Server webhook tests", func() {
@@ -65,7 +66,7 @@ var _ = framework.KubeDescribe("API Server webhook tests", func() {
 		labelSelector := labels.SelectorFromSet(labels.Set(label))
 		err = deploymentframework.WaitForDeploymentWithCondition(cs, ns, deployment.Name, "MinimumReplicasAvailable", appsv1.DeploymentAvailable)
 		Expect(err).NotTo(HaveOccurred())
-		_, err = framework.WaitForPodsWithLabelRunningReady(cs, ns, labelSelector, int(replicas), 1*time.Minute)
+		_, err = e2epod.WaitForPodsWithLabelRunningReady(cs, ns, labelSelector, int(replicas), 1*time.Minute)
 		Expect(err).NotTo(HaveOccurred())
 	})
 

diff --git a/test/e2e/aws_iam.go b/test/e2e/aws_iam.go
@@ -20,7 +20,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	awsiamrole "github.com/mikkeloscar/kube-aws-iam-controller/pkg/client/clientset/versioned"
+	awsiamrole "github.com/zalando-incubator/kube-aws-iam-controller/pkg/client/clientset/versioned"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"