Update to Kubernetes v1.16

Signed-off-by: Mikkel Oscar Lyderik Larsen <[email protected]>

cluster/config-defaults.yaml

@@ -261,8 +261,8 @@ dynamodb_service_link_enabled: "false"
 cluster_dns: "coredns"
 coredns_log_svc_names: "true"
 
-kuberuntu_image_v1_14: {{ amiID "zalando-ubuntu-kubernetes-production-v1.14.8-master-77" "861068367966" }}
 kuberuntu_image_v1_15: {{ amiID "zalando-ubuntu-kubernetes-production-v1.15.6-master-81" "861068367966" }}
+kuberuntu_image_v1_16: {{ amiID "zalando-ubuntu-kubernetes-production-v1.16.4-master-84" "861068367966" }}
 
 # Feature toggle to allow gradual decommissioning of ingress-template-controller
 enable_ingress_template_controller: "false"
@@ -283,14 +283,6 @@ audittrail_url: ""
 {{end}}
 audittrail_root_account_role: ""
 
-# Feature toggle for CustomResourceWebhookConversion (alpha in v1.13)
-# https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definition-versioning/#webhook-conversion
-custom_resource_webhook_conversion: "false"
-
-# Feature toggle for CustomResourcePublishOpenAPI (alpha in v1.14)
-# https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/#publish-validation-schema-in-openapi-v2
-custom_resource_publish_openapi: "false"
-
 # CIDR configuration for nodes and pods
 # Changing this will change the number of nodes and pods we can schedule in the
 # cluster: https://cloud.google.com/kubernetes-engine/docs/how-to/flexible-pod-cidr

cluster/manifests/02-kube-aws-iam-controller/deployment.yaml

@@ -36,8 +36,6 @@ spec:
             cpu: "{{.ConfigItems.kube_aws_iam_controller_cpu}}"
             memory: "{{.ConfigItems.kube_aws_iam_controller_mem}}"
       tolerations:
-      - key: node-role.kubernetes.io/master
-        effect: NoSchedule
       - key: node.kubernetes.io/role
         value: master
         effect: NoSchedule

cluster/manifests/admission-control/daemonset.yaml

@@ -26,8 +26,6 @@ spec:
       dnsPolicy: Default
       hostNetwork: true
       tolerations:
-      - key: node-role.kubernetes.io/master
-        effect: NoSchedule
       - key: node.kubernetes.io/role
         value: master
         effect: NoSchedule

cluster/manifests/cluster-lifecycle-controller/deployment.yaml

@@ -26,8 +26,6 @@ spec:
       priorityClassName: system-cluster-critical
       serviceAccountName: cluster-lifecycle-controller
       tolerations:
-      - key: node-role.kubernetes.io/master
-        effect: NoSchedule
       - key: node.kubernetes.io/role
         value: master
         effect: NoSchedule

cluster/manifests/etcd-backup/cronjob.yaml

@@ -57,8 +57,6 @@ spec:
               readOnly: true
 {{ end }}
           tolerations:
-          - key: node-role.kubernetes.io/master
-            effect: NoSchedule
           - key: node.kubernetes.io/role
             value: master
             effect: NoSchedule

cluster/manifests/kube-cluster-autoscaler/daemonset.yaml

@@ -29,8 +29,6 @@ spec:
       serviceAccountName: cluster-autoscaler
       dnsPolicy: Default
       tolerations:
-      - key: node-role.kubernetes.io/master
-        effect: NoSchedule
       - key: node.kubernetes.io/role
         value: master
         effect: NoSchedule

cluster/node-pools/master-default/stack.yaml

@@ -4,7 +4,7 @@ Description: Kubernetes default master node pool
 Mappings:
   Images:
     eu-central-1:
-      MachineImage: '{{ .Cluster.ConfigItems.kuberuntu_image_v1_15 }}'
+      MachineImage: '{{ .Cluster.ConfigItems.kuberuntu_image_v1_16 }}'
 
 Resources:
   AutoScalingGroup:
@@ -26,9 +26,6 @@ Resources:
       - Key: Name
         PropagateAtLaunch: true
         Value: "{{ .NodePool.Name }} ({{ .Cluster.ID }})"
-      - Key: kubernetes.io/role
-        PropagateAtLaunch: true
-        Value: master
       - Key: node.kubernetes.io/role
         PropagateAtLaunch: true
         Value: master

cluster/node-pools/master-default/userdata.yaml

@@ -3,8 +3,8 @@ write_files:
   - owner: root:root
     path: /etc/kubernetes/secrets.env
     content: |
-      NODEPOOL_TAINTS=node-role.kubernetes.io/master=:NoSchedule{{if index .NodePool.ConfigItems "taints"}},{{.NodePool.ConfigItems.taints}}{{end}}
-      NODE_LABELS=node-role.kubernetes.io/master,kubernetes.io/role=master,master=true,node.kubernetes.io/distro=ubuntu,cluster-lifecycle-controller.zalan.do/decommission-priority=999,{{ .Values.node_labels }}{{if index .NodePool.ConfigItems "labels"}},{{.NodePool.ConfigItems.labels}}{{end}}
+      NODEPOOL_TAINTS=node.kubernetes.io/role=master:NoSchedule{{if index .NodePool.ConfigItems "taints"}},{{.NodePool.ConfigItems.taints}}{{end}}
+      NODE_LABELS=master=true,node.kubernetes.io/distro=ubuntu,cluster-lifecycle-controller.zalan.do/decommission-priority=999,{{ .Values.node_labels }}{{if index .NodePool.ConfigItems "labels"}},{{.NodePool.ConfigItems.labels}}{{end}}
       NODEPOOL_NAME={{ .NodePool.Name }}
       KUBELET_ROLE=master
 
@@ -92,7 +92,8 @@ write_files:
       spec:
         priorityClassName: system-node-critical
         tolerations:
-        - key: node-role.kubernetes.io/master
+        - key: node.kubernetes.io/role
+          value: master
           effect: NoSchedule
         hostNetwork: true
         containers:
@@ -113,14 +114,14 @@ write_files:
           - --tls-cert-file=/etc/kubernetes/ssl/apiserver.pem
           - --tls-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem
           - --service-account-key-file=/etc/kubernetes/ssl/service-account-public-key.pem
-          - --runtime-config=extensions/v1beta1/networkpolicies=true,batch/v2alpha1=true,policy/v1beta1/podsecuritypolicy=true,imagepolicy.k8s.io/v1alpha1=true,authorization.k8s.io/v1beta1=true,scheduling.k8s.io/v1alpha1=true,admissionregistration.k8s.io/v1beta1=true
+          - --runtime-config=extensions/v1beta1/networkpolicies=true,batch/v2alpha1=true,policy/v1beta1/podsecuritypolicy=true,imagepolicy.k8s.io/v1alpha1=true,authorization.k8s.io/v1beta1=true,scheduling.k8s.io/v1alpha1=true,admissionregistration.k8s.io/v1beta1=true,discovery.k8s.io/v1alpha1=true
           - --authentication-token-webhook-config-file=/etc/kubernetes/config/authn.yaml
           - --authentication-token-webhook-cache-ttl=10s
           - --cloud-provider=aws
           - --authorization-mode=Webhook,RBAC
           - --authorization-webhook-config-file=/etc/kubernetes/config/authz.yaml
           - --admission-control-config-file=/etc/kubernetes/config/image-policy-webhook.yaml
-          - --feature-gates=TaintNodesByCondition={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},ScheduleDaemonSetPods={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},TTLAfterFinished=true,CustomResourceWebhookConversion={{.Cluster.ConfigItems.custom_resource_webhook_conversion}},CustomResourcePublishOpenAPI={{.Cluster.ConfigItems.custom_resource_publish_openapi}}
+          - --feature-gates=TaintNodesByCondition={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},ScheduleDaemonSetPods={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},TTLAfterFinished=true,EndpointSlice=true
           - --anonymous-auth=false
           {{ if ne .Cluster.ConfigItems.audittrail_url "" }}
           - --audit-webhook-config-file=/etc/kubernetes/config/audit.yaml
@@ -473,7 +474,8 @@ write_files:
       spec:
         priorityClassName: system-node-critical
         tolerations:
-        - key: node-role.kubernetes.io/master
+        - key: node.kubernetes.io/role
+          value: master
           effect: NoSchedule
         containers:
         - name: kube-controller-manager
@@ -499,6 +501,7 @@ write_files:
           - --horizontal-pod-autoscaler-sync-period={{ .Cluster.ConfigItems.horizontal_pod_autoscaler_sync_period }}
           - --horizontal-pod-autoscaler-tolerance={{ .Cluster.ConfigItems.horizontal_pod_autoscaler_tolerance }}
           - --horizontal-pod-autoscaler-upscale-delay={{ .Cluster.ConfigItems.horizontal_pod_autoscaler_upscale_delay }}
+          - --controllers=endpointslice
           resources:
             requests:
               cpu: 100m
@@ -539,7 +542,8 @@ write_files:
       spec:
         priorityClassName: system-node-critical
         tolerations:
-        - key: node-role.kubernetes.io/master
+        - key: node.kubernetes.io/role
+          value: master
           effect: NoSchedule
         hostNetwork: true
         containers:

cluster/node-pools/worker-default/stack.yaml

@@ -4,7 +4,7 @@ Description: Kubernetes default worker node pool
 Mappings:
   Images:
     eu-central-1:
-      MachineImage: '{{ .Cluster.ConfigItems.kuberuntu_image_v1_15 }}'
+      MachineImage: '{{ .Cluster.ConfigItems.kuberuntu_image_v1_16 }}'
 
 Resources:
   AutoScalingGroup:
@@ -42,9 +42,6 @@ Resources:
       - Key: k8s.io/role/node
         PropagateAtLaunch: true
         Value: worker
-      - Key: kubernetes.io/role
-        PropagateAtLaunch: true
-        Value: worker
       - Key: node.kubernetes.io/role
         PropagateAtLaunch: true
         Value: worker
@@ -57,15 +54,9 @@ Resources:
       - Key: k8s.io/cluster-autoscaler/enabled
         PropagateAtLaunch: true
         Value: ''
-      - Key: k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role
-        PropagateAtLaunch: true
-        Value: worker
       - Key: k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/role
         PropagateAtLaunch: true
         Value: worker
-      - Key: k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/node-pool
-        PropagateAtLaunch: true
-        Value: {{ .NodePool.Name }}
       - Key: k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/node-pool
         PropagateAtLaunch: true
         Value: {{ .NodePool.Name }}

cluster/node-pools/worker-splitaz/stack.yaml

@@ -4,7 +4,7 @@ Description: Kubernetes default worker node pool
 Mappings:
   Images:
     eu-central-1:
-      MachineImage: '{{ .Cluster.ConfigItems.kuberuntu_image_v1_15 }}'
+      MachineImage: '{{ .Cluster.ConfigItems.kuberuntu_image_v1_16 }}'
 
 Resources:
 {{ with $data := . }}
@@ -46,9 +46,6 @@ Resources:
       - Key: k8s.io/role/node
         PropagateAtLaunch: true
         Value: worker
-      - Key: kubernetes.io/role
-        PropagateAtLaunch: true
-        Value: worker
       - Key: node.kubernetes.io/role
         PropagateAtLaunch: true
         Value: worker
@@ -61,15 +58,9 @@ Resources:
       - Key: k8s.io/cluster-autoscaler/enabled
         PropagateAtLaunch: true
         Value: ''
-      - Key: k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role
-        PropagateAtLaunch: true
-        Value: worker
       - Key: k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/role
         PropagateAtLaunch: true
         Value: worker
-      - Key: k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/node-pool
-        PropagateAtLaunch: true
-        Value: {{ $data.NodePool.Name }}
       - Key: k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/node-pool
         PropagateAtLaunch: true
         Value: {{ $data.NodePool.Name }}

test/e2e/Makefile

@@ -2,7 +2,7 @@
 
 BINARY       ?= kubernetes-on-aws-e2e
 VERSION      ?= $(shell git describe --tags --always --dirty)
-KUBE_VERSION ?= v1.15.6
+KUBE_VERSION ?= v1.16.4
 IMAGE        ?= registry-write.opensource.zalan.do/teapot/$(BINARY)
 TAG          ?= $(VERSION)
 DOCKERFILE   ?= Dockerfile

test/e2e/admission_controller.go

@@ -32,6 +32,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
 	deploymentframework "k8s.io/kubernetes/test/e2e/framework/deployment"
+	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
 )
 
 const (
@@ -68,7 +69,7 @@ var _ = framework.KubeDescribe("Admission controller tests", func() {
 		Expect(err).NotTo(HaveOccurred())
 
 		//pods are not returned here
-		_, err = framework.WaitForPodsWithLabelRunningReady(cs, ns, labelSelector, int(replicas), 1*time.Minute)
+		_, err = e2epod.WaitForPodsWithLabelRunningReady(cs, ns, labelSelector, int(replicas), 1*time.Minute)
 		Expect(err).NotTo(HaveOccurred())
 
 		pods, err := cs.CoreV1().Pods(ns).List(metav1.ListOptions{LabelSelector: labelSelector.String()})
@@ -123,7 +124,7 @@ var _ = framework.KubeDescribe("Admission controller tests", func() {
 		_, err := cs.CoreV1().Pods(ns).Create(pod)
 		Expect(err).NotTo(HaveOccurred())
 
-		err = framework.WaitForPodSuccessInNamespaceSlow(cs, podName, ns)
+		err = e2epod.WaitForPodSuccessInNamespaceSlow(cs, podName, ns)
 		Expect(err).NotTo(HaveOccurred())
 	})
 })

test/e2e/apiserver.go

@@ -30,6 +30,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
 	deploymentframework "k8s.io/kubernetes/test/e2e/framework/deployment"
+	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
 )
 
 var _ = framework.KubeDescribe("API Server webhook tests", func() {
@@ -65,7 +66,7 @@ var _ = framework.KubeDescribe("API Server webhook tests", func() {
 		labelSelector := labels.SelectorFromSet(labels.Set(label))
 		err = deploymentframework.WaitForDeploymentWithCondition(cs, ns, deployment.Name, "MinimumReplicasAvailable", appsv1.DeploymentAvailable)
 		Expect(err).NotTo(HaveOccurred())
-		_, err = framework.WaitForPodsWithLabelRunningReady(cs, ns, labelSelector, int(replicas), 1*time.Minute)
+		_, err = e2epod.WaitForPodsWithLabelRunningReady(cs, ns, labelSelector, int(replicas), 1*time.Minute)
 		Expect(err).NotTo(HaveOccurred())
 	})
 

test/e2e/aws_iam.go

@@ -20,7 +20,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	awsiamrole "github.com/mikkeloscar/kube-aws-iam-controller/pkg/client/clientset/versioned"
+	awsiamrole "github.com/zalando-incubator/kube-aws-iam-controller/pkg/client/clientset/versioned"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"

test/e2e/e2e_test.go

@@ -24,6 +24,7 @@ import (
 
 	"k8s.io/kubernetes/test/e2e"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/framework/config"
 	"k8s.io/kubernetes/test/e2e/framework/testfiles"
 	"k8s.io/kubernetes/test/e2e/framework/viperconfig"
 	"k8s.io/kubernetes/test/e2e/generated"
@@ -49,10 +50,19 @@ import (
 
 var viperConfig = flag.String("viper-config", "", "The name of a viper config file (https://github.com/spf13/viper#what-is-viper). All e2e command line parameters can also be configured in such a file. May contain a path and may or may not contain the file suffix. The default is to look for an optional file with `e2e` as base name. If a file is specified explicitly, it must be present.")
 
+// handleFlags sets up all flags and parses the command line.
+func handleFlags() {
+	config.CopyFlags(config.Flags, flag.CommandLine)
+	framework.RegisterCommonFlags(flag.CommandLine)
+	framework.RegisterClusterFlags(flag.CommandLine)
+	flag.Parse()
+}
+
 func TestMain(m *testing.M) {
-	// Register framework flags, then handle flags and Viper config.
-	framework.HandleFlags()
-	if err := viperconfig.ViperizeFlags(*viperConfig, ""); err != nil {
+	// Register test flags, then parse flags.
+	handleFlags()
+
+	if err := viperconfig.ViperizeFlags(*viperConfig, "e2e", flag.CommandLine); err != nil {
 		fmt.Fprintln(os.Stderr, err)
 		os.Exit(1)
 	}

test/e2e/go.mod

@@ -4,52 +4,37 @@ require (
 	github.com/NYTimes/gziphandler v1.0.1 // indirect
 	github.com/cloudflare/cfssl v0.0.0-20190616170404-1bf3e59ec1cf // indirect
 	github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50 // indirect
-	github.com/cyphar/filepath-securejoin v0.2.2 // indirect
 	github.com/d2g/dhcp4client v1.0.0 // indirect
-	github.com/docker/distribution v2.6.0-rc.1.0.20170726174610-edc3ab29cdff+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/libnetwork v0.8.0-dev.2.0.20180830151422-a9cd636e3789 // indirect
-	github.com/evanphx/json-patch v4.1.1-0.20190203023257-5858425f7550+incompatible
-	github.com/go-openapi/strfmt v0.17.2 // indirect
-	github.com/google/cadvisor v0.34.0 // indirect
+	github.com/evanphx/json-patch v4.2.0+incompatible
 	github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e // indirect
-	github.com/gorilla/websocket v1.4.0 // indirect
 	github.com/gregjones/httpcache v0.0.0-20181110185634-c63ab54fda8f // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.8.5 // indirect
 	github.com/imdario/mergo v0.3.6 // indirect
-	github.com/jonboulle/clockwork v0.1.0 // indirect
 	github.com/jteeuwen/go-bindata v0.0.0-20151023091102-a0ff2567cfb7
 	github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1 // indirect
 	github.com/karrick/godirwalk v1.8.0 // indirect
 	github.com/marstr/guid v1.1.0 // indirect
-	github.com/mattn/go-shellwords v1.0.3 // indirect
-	github.com/miekg/dns v1.1.1 // indirect
-	github.com/mikkeloscar/kube-aws-iam-controller v0.1.0
 	github.com/mindprince/gonvml v0.0.0-20180514031326-b364b296c732 // indirect
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
 	github.com/mrunalp/fileutils v0.0.0-20171103030105-7d4729fb3618 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20190414153302-2ae31c8b6b30 // indirect
 	github.com/onsi/ginkgo v1.8.0
-	github.com/onsi/gomega v1.4.3
-	github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
-	github.com/opencontainers/image-spec v1.0.1 // indirect
-	github.com/opencontainers/runc v1.0.0-rc2.0.20180424185634-871ba2e58e24 // indirect
+	github.com/onsi/gomega v1.5.0
 	github.com/opencontainers/runtime-spec v1.0.1 // indirect
-	github.com/opencontainers/selinux v1.0.0-rc1 // indirect
 	github.com/pkg/errors v0.8.1 // indirect
 	github.com/pquerna/ffjson v0.0.0-20181028064349-e517b90714f7 // indirect
 	github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 // indirect
-	github.com/spf13/cobra v0.0.3 // indirect
-	github.com/spf13/viper v1.2.1 // indirect
 	github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc // indirect
 	github.com/xanzy/go-cloudstack v2.3.3+incompatible // indirect
-	gopkg.in/square/go-jose.v2 v2.2.0 // indirect
+	github.com/zalando-incubator/kube-aws-iam-controller v0.1.1
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	k8s.io/api v0.0.0
 	k8s.io/apimachinery v0.0.0
 	k8s.io/apiserver v0.0.0
 	k8s.io/client-go v10.0.0+incompatible
-	k8s.io/kubernetes v1.15.4
+	k8s.io/kubernetes v1.16.4
 	k8s.io/utils v0.0.0-20190829053155-3a4a5477acf8 // indirect
 )
 
@@ -79,6 +64,8 @@ replace k8s.io/kube-proxy => ./e2e_modules/kubernetes/staging/src/k8s.io/kube-pr
 
 replace k8s.io/kube-scheduler => ./e2e_modules/kubernetes/staging/src/k8s.io/kube-scheduler
 
+replace k8s.io/kubectl => ./e2e_modules/kubernetes/staging/src/k8s.io/kubectl
+
 replace k8s.io/kubelet => ./e2e_modules/kubernetes/staging/src/k8s.io/kubelet
 
 replace k8s.io/sample-apiserver => ./e2e_modules/kubernetes/staging/src/k8s.io/sample-apiserver