Update to Kubernetes v1.16

cluster/config-defaults.yaml

@@ -176,9 +176,11 @@ teapot_admission_controller_validate_pod_template_resources: "true"
 {{end}}
 
 {{if eq .Environment "e2e"}}
-teapot_admission_controller_ignore_namespaces: "^kube-system|((downward-api|kubectl|projected|statefulset|pod-network)-.*)$"
+teapot_admission_controller_ignore_namespaces: "^kube-system|((downward-api|kubectl|projected|statefulset|pod-network|scope-selectors|resourcequota)-.*)$"
+teapot_admission_controller_crd_ensure_no_resources_on_delete: "false"
 {{else}}
 teapot_admission_controller_ignore_namespaces: "^kube-system$"
+teapot_admission_controller_crd_ensure_no_resources_on_delete: "true"
 {{end}}
 
 # etcd cluster
@@ -195,6 +197,7 @@ cluster_dns: "coredns"
 coredns_log_svc_names: "true"
 
 kuberuntu_image_v1_15: {{ amiID "zalando-ubuntu-kubernetes-production-v1.15.9-master-89" "861068367966" }}
+kuberuntu_image_v1_16: {{ amiID "zalando-ubuntu-kubernetes-production-v1.16.7-master-91" "861068367966" }}
 
 # Feature toggle to allow gradual decommissioning of ingress-template-controller
 enable_ingress_template_controller: "false"
@@ -215,14 +218,6 @@ audittrail_url: ""
 {{end}}
 audittrail_root_account_role: ""
 
-# Feature toggle for CustomResourceWebhookConversion (alpha in v1.13)
-# https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definition-versioning/#webhook-conversion
-custom_resource_webhook_conversion: "false"
-
-# Feature toggle for CustomResourcePublishOpenAPI (alpha in v1.14)
-# https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/#publish-validation-schema-in-openapi-v2
-custom_resource_publish_openapi: "false"
-
 # CIDR configuration for nodes and pods
 # Changing this will change the number of nodes and pods we can schedule in the
 # cluster: https://cloud.google.com/kubernetes-engine/docs/how-to/flexible-pod-cidr
@@ -286,12 +281,15 @@ enable_cdp_sa: "false"
 vm_dirty_background_bytes: ""
 vm_dirty_bytes: ""
 
+# Enable FeatureGate EndpointSlice
+enable_endpointslice: "false"
+
+# Enable FeatureGate HPAScaleToZero
+enable_hpa_scale_to_zero: "true"
+
 # temporary flag for kubernetes.io/node-pool node label
 legacy_node_pool_label_enabled: "false"
 
-# Disable legacy apiVersions which will be gone in Kubernetes v1.16
-disable_legacy_api_versions: "true"
-
 # setup supporting components to enable encryption
 # this flag must only be switched from true to false when enable_encryption is false and all secrets were decrypted
 support_encryption: "false"

cluster/manifests/01-platformcredentialsset/customresourcedefinition.yaml

@@ -1,13 +1,12 @@
 # A specification to declare needed OAuth credentials (tokens, clients) for the
 # Zalando Platform IAM system
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   name: platformcredentialssets.zalando.org
 spec:
   scope: Namespaced
   group: zalando.org
-  version: v1
   names:
     kind: PlatformCredentialsSet
     plural: platformcredentialssets
@@ -16,66 +15,72 @@ spec:
       - pcs
     categories:
       - all
-  additionalPrinterColumns:
-  - JSONPath: .spec.application
-    description: ID of application registered in application registry
-    name: Application
-    type: string
-  - JSONPath: .status.processingStatus
-    description: Processing status reported by Credentials Provider
-    name: Status
-    type: string
-  - JSONPath: .metadata.creationTimestamp
-    description: Age of the PlatformCredentialsSet
-    name: Age
-    type: date
-  validation:
-    openAPIV3Schema:
-      required:
+  versions:
+  - name: v1
+    served: true
+    storage: true
+    additionalPrinterColumns:
+    - jsonPath: .spec.application
+      description: ID of application registered in application registry
+      name: Application
+      type: string
+    - jsonPath: .status.processingStatus
+      description: Processing status reported by Credentials Provider
+      name: Status
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      description: Age of the PlatformCredentialsSet
+      name: Age
+      type: date
+    schema:
+      openAPIV3Schema:
+        type: object
+        required:
         - spec
-      properties:
-        spec:
-          required:
-            - application
-          properties:
-            application:
-              type: string
-              pattern: "^[a-z][a-z0-9-]*[a-z0-9]$"
-            clients:
-              type: object
-              additionalProperties:
+        properties:
+          spec:
+            type: object
+            required:
+              - application
+            properties:
+              application:
+                type: string
+                pattern: "^[a-z][a-z0-9-]*[a-z0-9]$"
+              clients:
                 type: object
-                properties:
-                  realm:
-                    type: string
-                    enum:
-                      - customers
-                      - users
-                      - services
-                  grant:
-                    type: string
-                    enum:
-                      - authorization-code
-                      - implicit
-                      - resource-owner-password-credentials
-                      - client-credentials
-                  redirectUri:
-                    type: string
-            tokens:
-              type: object
-              additionalProperties:
-                type: object
-                nullable: true
-                properties:
-                  privileges:
-                    type: array
-                    nullable: true
-                    items:
+                additionalProperties:
+                  type: object
+                  properties:
+                    realm:
+                      type: string
+                      enum:
+                        - customers
+                        - users
+                        - services
+                    grant:
                       type: string
-            token_version:
-              type: string
-              enum:
-                - v1
-                - v2
-  subresources:
-    status: {}
+                      enum:
+                        - authorization-code
+                        - implicit
+                        - resource-owner-password-credentials
+                        - client-credentials
+                    redirectUri:
+                      type: string
+              tokens:
+                type: object
+                additionalProperties:
+                  type: object
+                  nullable: true
+                  properties:
+                    privileges:
+                      type: array
+                      nullable: true
+                      items:
+                        type: string
+              token_version:
+                type: string
+                enum:
+                  - v1
+                  - v2
+    subresources:
+      status: {}

cluster/manifests/02-kube-aws-iam-controller/crd.yaml

@@ -1,56 +1,62 @@
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   name: awsiamroles.zalando.org
 spec:
   group: zalando.org
-  version: v1
   scope: Namespaced
   names:
     kind: AWSIAMRole
     singular: awsiamrole
     plural: awsiamroles
     categories:
     - all
-  additionalPrinterColumns:
-  - name: RoleARN
-    type: string
-    description: Full RoleARN
-    JSONPath: .status.roleARN
-  - name: Expiration
-    type: string
-    description: Expiration time of the current credentials provisioned for the role
-    JSONPath: .status.expiration
-  subresources:
-    # status enables the status subresource.
-    status: {}
-  # validation depends on Kubernetes >= v1.11.0
-  validation:
-    openAPIV3Schema:
-      properties:
-        spec:
-          properties:
-            roleReference:
-              description: |
-                Reference to an AWS IAM role which can either be a role name
-                or a full IAM role ARN.
-              type: string
-              minLength: 3
-            roleSessionDuration:
-              description: |
-                Specify the role session duration in seconds. Defaults to 3600
-                seconds (1 hour). This value must be less than or equal to the
-                `MaxSessionDuration` value of the IAM role.
-              type: integer
-              minimum: 900   # 15 minutes
-              maximum: 43200 # 12 hours
-        status:
-          properties:
-            observedGeneration:
-              type: integer
-            roleARN:
-              type: string
-            expiration:
-              type: string
-      required:
-      - spec
+  versions:
+  - name: v1
+    served: true
+    storage: true
+    additionalPrinterColumns:
+    - name: RoleARN
+      type: string
+      description: Full RoleARN
+      jsonPath: .status.roleARN
+    - name: Expiration
+      type: string
+      description: Expiration time of the current credentials provisioned for the role
+      jsonPath: .status.expiration
+    subresources:
+      # status enables the status subresource.
+      status: {}
+    # validation depends on Kubernetes >= v1.11.0
+    schema:
+      openAPIV3Schema:
+        type: object
+        properties:
+          spec:
+            type: object
+            properties:
+              roleReference:
+                description: |
+                  Reference to an AWS IAM role which can either be a role name
+                  or a full IAM role ARN.
+                type: string
+                minLength: 3
+              roleSessionDuration:
+                description: |
+                  Specify the role session duration in seconds. Defaults to 3600
+                  seconds (1 hour). This value must be less than or equal to the
+                  `MaxSessionDuration` value of the IAM role.
+                type: integer
+                minimum: 900   # 15 minutes
+                maximum: 43200 # 12 hours
+          status:
+            type: object
+            properties:
+              observedGeneration:
+                type: integer
+              roleARN:
+                type: string
+              expiration:
+                type: string
+        required:
+        - spec

cluster/manifests/02-kube-aws-iam-controller/deployment.yaml

@@ -36,8 +36,6 @@ spec:
             cpu: "{{.ConfigItems.kube_aws_iam_controller_cpu}}"
             memory: "{{.ConfigItems.kube_aws_iam_controller_mem}}"
       tolerations:
-      - key: node-role.kubernetes.io/master
-        effect: NoSchedule
       - key: node.kubernetes.io/role
         value: master
         effect: NoSchedule

cluster/manifests/admission-control-proxy/daemonset.yaml

@@ -26,14 +26,12 @@ spec:
       dnsPolicy: Default
       hostNetwork: true
       tolerations:
-      - key: node-role.kubernetes.io/master
-        effect: NoSchedule
       - key: node.kubernetes.io/role
         value: master
         effect: NoSchedule
       containers:
       - name: cluster-autoscaler
-        image: registry.opensource.zalan.do/teapot/admission-controller:master-57
+        image: registry.opensource.zalan.do/teapot/admission-controller:master-63
         command:
           - /registry-proxy
           - --address=127.0.0.1:8285

cluster/manifests/cluster-lifecycle-controller/deployment.yaml

@@ -26,8 +26,6 @@ spec:
       priorityClassName: system-cluster-critical
       serviceAccountName: cluster-lifecycle-controller
       tolerations:
-      - key: node-role.kubernetes.io/master
-        effect: NoSchedule
       - key: node.kubernetes.io/role
         value: master
         effect: NoSchedule

cluster/manifests/etcd-backup/cronjob.yaml

@@ -57,8 +57,6 @@ spec:
               readOnly: true
 {{ end }}
           tolerations:
-          - key: node-role.kubernetes.io/master
-            effect: NoSchedule
           - key: node.kubernetes.io/role
             value: master
             effect: NoSchedule

cluster/manifests/ingress-template-controller/crd.yaml

@@ -1,11 +1,14 @@
 {{ if eq .ConfigItems.enable_ingress_template_controller "true"}}
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   name: ingresstemplates.zalando.org
 spec:
   group: zalando.org
-  version: v1
+  versions:
+  - name: v1
+    served: true
+    storage: true
   scope: Namespaced
   names:
     kind: IngressTemplate

cluster/manifests/kube-cluster-autoscaler/daemonset.yaml

@@ -29,8 +29,6 @@ spec:
       serviceAccountName: cluster-autoscaler
       dnsPolicy: Default
       tolerations:
-      - key: node-role.kubernetes.io/master
-        effect: NoSchedule
       - key: node.kubernetes.io/role
         value: master
         effect: NoSchedule
@@ -53,6 +51,7 @@ spec:
           - --max-node-provision-time=7m
           - --max-nodes-total={{ nodeCIDRMaxNodes (parseInt64 .Cluster.ConfigItems.node_cidr_mask_size) (parseInt64 .Cluster.ConfigItems.reserved_nodes) }}
           - --scale-down-enabled={{ .ConfigItems.autoscaling_scale_down_enabled }}
+          - --scale-down-delay-after-add=-1s
         resources:
           requests:
             cpu: {{.Cluster.ConfigItems.cluster_autoscaler_cpu}}