From f2e511a393fc3e414061060ea362ba0f244a9a9e Mon Sep 17 00:00:00 2001 From: Arpad Ryszka Date: Tue, 15 Oct 2019 12:45:42 +0200 Subject: [PATCH 01/13] update skipper version Signed-off-by: Arpad Ryszka --- cluster/manifests/skipper/deployment.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cluster/manifests/skipper/deployment.yaml b/cluster/manifests/skipper/deployment.yaml index 1478d040b9..b49c1f75c4 100644 --- a/cluster/manifests/skipper/deployment.yaml +++ b/cluster/manifests/skipper/deployment.yaml @@ -5,7 +5,7 @@ metadata: namespace: kube-system labels: application: skipper-ingress - version: v0.10.295 + version: v0.11.1 component: ingress spec: strategy: @@ -18,7 +18,7 @@ spec: metadata: labels: application: skipper-ingress - version: v0.10.295 + version: v0.11.1 component: ingress annotations: kubernetes-log-watcher/scalyr-parser: | @@ -43,7 +43,7 @@ spec: hostNetwork: true containers: - name: skipper-ingress - image: registry.opensource.zalan.do/pathfinder/skipper:v0.10.295 + image: registry.opensource.zalan.do/pathfinder/skipper:v0.11.1 ports: - name: ingress-port containerPort: 9999 From c5b0274905cc1abc7c44e9c3821f699b9485c595 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Thu, 17 Oct 2019 09:27:24 +0200 Subject: [PATCH 02/13] cleanup defaults which are not used MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sandor Szücs --- cluster/config-defaults.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cluster/config-defaults.yaml b/cluster/config-defaults.yaml index 2447ae7696..18b3609e1b 100644 --- a/cluster/config-defaults.yaml +++ b/cluster/config-defaults.yaml @@ -17,11 +17,6 @@ cluster_autoscaler_memory: "300Mi" kube_aws_ingress_controller_ssl_policy: "ELBSecurityPolicy-TLS-1-2-2017-01" kube_aws_ingress_controller_idle_timeout: "1m" -# skipper resource settings -skipper_limits_mem: "250Mi" -skipper_requests_cpu: "150m" -skipper_requests_mem: "50Mi" - # skipper ingress settings skipper_ingress_target_average_utilization_cpu: "70" skipper_ingress_target_average_utilization_memory: "80" From 36673eb1c3e6572f8f1c2ec950d4d9e5a1c26081 Mon Sep 17 00:00:00 2001 From: Alexey Ermakov Date: Fri, 18 Oct 2019 17:45:33 +0200 Subject: [PATCH 03/13] CA: parameterize the version temporarily Signed-off-by: Alexey Ermakov --- cluster/config-defaults.yaml | 3 +++ cluster/manifests/kube-cluster-autoscaler/daemonset.yaml | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cluster/config-defaults.yaml b/cluster/config-defaults.yaml index 0eb00bd805..30b8df9cbe 100644 --- a/cluster/config-defaults.yaml +++ b/cluster/config-defaults.yaml @@ -13,6 +13,9 @@ autoscaling_buffer_pods: "0" cluster_autoscaler_cpu: "100m" cluster_autoscaler_memory: "300Mi" +# Temporarily moved to a config item so we could test the new version +cluster_autoscaler_version: "v1.12.2-internal.4" + # ALB config created by kube-aws-ingress-controller kube_aws_ingress_controller_ssl_policy: "ELBSecurityPolicy-TLS-1-2-2017-01" kube_aws_ingress_controller_idle_timeout: "1m" diff --git a/cluster/manifests/kube-cluster-autoscaler/daemonset.yaml b/cluster/manifests/kube-cluster-autoscaler/daemonset.yaml index 0116280f68..0fd163c8c3 100644 --- a/cluster/manifests/kube-cluster-autoscaler/daemonset.yaml +++ b/cluster/manifests/kube-cluster-autoscaler/daemonset.yaml @@ -5,7 +5,7 @@ metadata: namespace: kube-system labels: application: kube-cluster-autoscaler - version: v1.12.2-internal.4 + version: {{.Cluster.ConfigItems.cluster_autoscaler_version}} spec: selector: matchLabels: @@ -16,7 +16,7 @@ spec: metadata: labels: application: kube-cluster-autoscaler - version: v1.12.2-internal.4 + version: {{.Cluster.ConfigItems.cluster_autoscaler_version}} annotations: iam.amazonaws.com/role: "{{ .LocalID }}-app-autoscaler" config/pool-sizes: "{{range .NodePools}}{{.Name}}-{{.MinSize}}-{{.MaxSize}} {{end}}" @@ -33,7 +33,7 @@ spec: effect: NoSchedule containers: - name: cluster-autoscaler - image: registry.opensource.zalan.do/teapot/kube-cluster-autoscaler:v1.12.2-internal.4 + image: registry.opensource.zalan.do/teapot/kube-cluster-autoscaler:{{.Cluster.ConfigItems.cluster_autoscaler_version}} command: - ./cluster-autoscaler - --v=4 From 65431225481c1d31ec1ac37f6b86f41186902040 Mon Sep 17 00:00:00 2001 From: Mikkel Oscar Lyderik Larsen Date: Sun, 20 Oct 2019 23:25:47 +0200 Subject: [PATCH 04/13] Specify support-listener address for apiserver skippers Signed-off-by: Mikkel Oscar Lyderik Larsen --- cluster/node-pools/master-default/userdata.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cluster/node-pools/master-default/userdata.yaml b/cluster/node-pools/master-default/userdata.yaml index e4c18551f2..39177cae68 100644 --- a/cluster/node-pools/master-default/userdata.yaml +++ b/cluster/node-pools/master-default/userdata.yaml @@ -346,6 +346,7 @@ write_files: args: - skipper - -address=:9023 + - -support-listener=:9913 - -inline-routes - | health: Path("/healthz") -> inlineContent("ok") -> ; @@ -398,6 +399,7 @@ write_files: args: - skipper - -address=:8443 + - -support-listener=:9911 - -tls-cert=/etc/kubernetes/ssl/apiserver.pem - -tls-key=/etc/kubernetes/ssl/apiserver-key.pem - -insecure From 9b733c3f8fda2ce2c9014f041cc007d4bc823470 Mon Sep 17 00:00:00 2001 From: Mikkel Oscar Lyderik Larsen Date: Mon, 21 Oct 2019 09:02:16 +0200 Subject: [PATCH 05/13] Update to metrics-server v0.3.6 https://github.com/kubernetes-incubator/metrics-server/releases/tag/v0.3.6 Signed-off-by: Mikkel Oscar Lyderik Larsen --- cluster/manifests/metrics-server/deployment.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cluster/manifests/metrics-server/deployment.yaml b/cluster/manifests/metrics-server/deployment.yaml index 11d9575d23..bc8f43089b 100644 --- a/cluster/manifests/metrics-server/deployment.yaml +++ b/cluster/manifests/metrics-server/deployment.yaml @@ -5,7 +5,7 @@ metadata: namespace: kube-system labels: application: metrics-server - version: v0.3.5 + version: v0.3.6 spec: replicas: 1 selector: @@ -16,7 +16,7 @@ spec: name: metrics-server labels: application: metrics-server - version: v0.3.5 + version: v0.3.6 spec: dnsConfig: options: @@ -26,7 +26,7 @@ spec: serviceAccountName: metrics-server containers: - name: metrics-server - image: registry.opensource.zalan.do/teapot/metrics-server:v0.3.5 + image: registry.opensource.zalan.do/teapot/metrics-server:v0.3.6 resources: limits: cpu: "{{.ConfigItems.metrics_service_cpu}}" From 390af2421e40da0628aa4c26c56af187896e360d Mon Sep 17 00:00:00 2001 From: Alexey Ermakov Date: Tue, 22 Oct 2019 16:23:06 +0200 Subject: [PATCH 06/13] Update kube-metrics-adapter Signed-off-by: Alexey Ermakov --- cluster/manifests/kube-metrics-adapter/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/manifests/kube-metrics-adapter/deployment.yaml b/cluster/manifests/kube-metrics-adapter/deployment.yaml index 6a37bebae3..9fbd5f2f15 100644 --- a/cluster/manifests/kube-metrics-adapter/deployment.yaml +++ b/cluster/manifests/kube-metrics-adapter/deployment.yaml @@ -27,7 +27,7 @@ spec: serviceAccountName: custom-metrics-apiserver containers: - name: kube-metrics-adapter - image: registry.opensource.zalan.do/teapot/kube-metrics-adapter:v0.0.3 + image: registry.opensource.zalan.do/teapot/kube-metrics-adapter:v0.0.4 {{ if eq .ConfigItems.kube_aws_iam_controller_kube_system_enable "true"}} env: # must be set for the AWS SDK/AWS CLI to find the credentials file. From 53075d8f78c8ba404b7a79e60b7d6d4f1ae89eeb Mon Sep 17 00:00:00 2001 From: njuettner Date: Fri, 25 Oct 2019 14:35:19 +0200 Subject: [PATCH 07/13] OpenAPI v2 Publishing for CRD's --- cluster/config-defaults.yaml | 4 ++++ cluster/node-pools/master-default/userdata.yaml | 2 +- test/e2e/cluster_config.sh | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/cluster/config-defaults.yaml b/cluster/config-defaults.yaml index d60fcda24c..65ada71fd1 100644 --- a/cluster/config-defaults.yaml +++ b/cluster/config-defaults.yaml @@ -254,6 +254,10 @@ audit_pod_events: "true" # https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definition-versioning/#webhook-conversion custom_resource_webhook_conversion: "false" +# Feature toggle for CustomResourcePublishOpenAPI (alpha in v1.14) +# https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/#publish-validation-schema-in-openapi-v2 +custom_resource_publish_openapi: "false" + # CIDR configuration for nodes and pods # Changing this will change the number of nodes and pods we can schedule in the # cluster diff --git a/cluster/node-pools/master-default/userdata.yaml b/cluster/node-pools/master-default/userdata.yaml index 39177cae68..d8c0ad35ac 100644 --- a/cluster/node-pools/master-default/userdata.yaml +++ b/cluster/node-pools/master-default/userdata.yaml @@ -120,7 +120,7 @@ write_files: - --authorization-mode=Webhook,RBAC - --authorization-webhook-config-file=/etc/kubernetes/config/authz.yaml - --admission-control-config-file=/etc/kubernetes/config/image-policy-webhook.yaml - - --feature-gates=TaintNodesByCondition={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},ScheduleDaemonSetPods={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},TTLAfterFinished=true,CustomResourceWebhookConversion={{.Cluster.ConfigItems.custom_resource_webhook_conversion}} + - --feature-gates=TaintNodesByCondition={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},ScheduleDaemonSetPods={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},TTLAfterFinished=true,CustomResourceWebhookConversion={{.Cluster.ConfigItems.custom_resource_webhook_conversion}},CustomResourcePublishOpenAPI={{.Cluster.ConfigItems.custom_resource_publish_openapi}} - --anonymous-auth=false {{ if or (eq .Cluster.Environment "production") (index .Cluster.ConfigItems "audittrail_url") }} - --audit-webhook-config-file=/etc/kubernetes/config/audit.yaml diff --git a/test/e2e/cluster_config.sh b/test/e2e/cluster_config.sh index c444fec26b..cc9385d2a2 100755 --- a/test/e2e/cluster_config.sh +++ b/test/e2e/cluster_config.sh @@ -46,6 +46,7 @@ clusters: zmon_scheduler_replicas: '0' zmon_worker_replicas: '0' node_pool_feature_enabled: "true" + custom_resource_publish_openapi: "true" enable_rbac: "true" dynamodb_service_link_enabled: "false" skipper_ingress_cpu: 100m From e9cea423e63158f488bb7dfd2fbe1650c86850f2 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Fri, 25 Oct 2019 14:49:31 +0200 Subject: [PATCH 08/13] rename scalyr-region environment variable (bugfix) --- cluster/manifests/zmon-worker/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/manifests/zmon-worker/deployment.yaml b/cluster/manifests/zmon-worker/deployment.yaml index bffc81a4aa..f9d89e4411 100644 --- a/cluster/manifests/zmon-worker/deployment.yaml +++ b/cluster/manifests/zmon-worker/deployment.yaml @@ -66,7 +66,7 @@ spec: periodSeconds: 60 env: - - name: WORKER_SCALYR_REGION + - name: WORKER_PLUGIN_SCALYR_SCALYR_REGION value: {{.ConfigItems.zmon_scalyr_region}} - name: WORKER_ZMON_QUEUES value: zmon:queue:default/{{.ConfigItems.zmon_worker_count}} From 7e60142906d4d2d1611d094bf428211aba1e4879 Mon Sep 17 00:00:00 2001 From: njuettner Date: Fri, 25 Oct 2019 15:05:40 +0200 Subject: [PATCH 09/13] OpenAPI v2 Publishing for CRD's Signed-off-by: njuettner --- cluster/config-defaults.yaml | 2 +- test/e2e/cluster_config.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/cluster/config-defaults.yaml b/cluster/config-defaults.yaml index 65ada71fd1..34f14df028 100644 --- a/cluster/config-defaults.yaml +++ b/cluster/config-defaults.yaml @@ -256,7 +256,7 @@ custom_resource_webhook_conversion: "false" # Feature toggle for CustomResourcePublishOpenAPI (alpha in v1.14) # https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/#publish-validation-schema-in-openapi-v2 -custom_resource_publish_openapi: "false" +custom_resource_publish_openapi: "true" # CIDR configuration for nodes and pods # Changing this will change the number of nodes and pods we can schedule in the diff --git a/test/e2e/cluster_config.sh b/test/e2e/cluster_config.sh index cc9385d2a2..c444fec26b 100755 --- a/test/e2e/cluster_config.sh +++ b/test/e2e/cluster_config.sh @@ -46,7 +46,6 @@ clusters: zmon_scheduler_replicas: '0' zmon_worker_replicas: '0' node_pool_feature_enabled: "true" - custom_resource_publish_openapi: "true" enable_rbac: "true" dynamodb_service_link_enabled: "false" skipper_ingress_cpu: 100m From 29df2d111cc9791f06173c4abe76572316fe730f Mon Sep 17 00:00:00 2001 From: Alexey Ermakov Date: Mon, 28 Oct 2019 13:51:50 +0100 Subject: [PATCH 10/13] Update admission controller Signed-off-by: Alexey Ermakov --- cluster/manifests/admission-control/daemonset.yaml | 2 +- cluster/manifests/admission-control/teapot.yaml | 10 ++++++++++ cluster/node-pools/master-default/userdata.yaml | 2 +- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/cluster/manifests/admission-control/daemonset.yaml b/cluster/manifests/admission-control/daemonset.yaml index 967ee0f006..8947e8b164 100644 --- a/cluster/manifests/admission-control/daemonset.yaml +++ b/cluster/manifests/admission-control/daemonset.yaml @@ -30,7 +30,7 @@ spec: effect: NoSchedule containers: - name: cluster-autoscaler - image: registry.opensource.zalan.do/teapot/admission-controller:master-35 + image: registry.opensource.zalan.do/teapot/admission-controller:master-37 command: - /registry-proxy - --address=127.0.0.1:8285 diff --git a/cluster/manifests/admission-control/teapot.yaml b/cluster/manifests/admission-control/teapot.yaml index 8fa273ce6c..2df4ea0a2e 100644 --- a/cluster/manifests/admission-control/teapot.yaml +++ b/cluster/manifests/admission-control/teapot.yaml @@ -107,3 +107,13 @@ webhooks: apiGroups: ["apiextensions.k8s.io"] apiVersions: ["v1", "v1beta1"] resources: ["customresourcedefinitions"] + - name: ingress-admitter.teapot.zalan.do + clientConfig: + url: "https://localhost:8085/ingress" + caBundle: "{{ .ConfigItems.ca_cert_decompressed }}" + failurePolicy: Fail + rules: + - operations: ["CREATE", "UPDATE"] + apiGroups: ["extensions", "networking.k8s.io"] + apiVersions: ["v1beta1"] + resources: ["ingresses"] diff --git a/cluster/node-pools/master-default/userdata.yaml b/cluster/node-pools/master-default/userdata.yaml index d8c0ad35ac..fc1463f96b 100644 --- a/cluster/node-pools/master-default/userdata.yaml +++ b/cluster/node-pools/master-default/userdata.yaml @@ -172,7 +172,7 @@ write_files: requests: cpu: 100m memory: 200Mi - - image: registry.opensource.zalan.do/teapot/admission-controller:master-35 + - image: registry.opensource.zalan.do/teapot/admission-controller:master-37 name: admission-controller readinessProbe: httpGet: From d030c8e0720ca843cce64ffb93f201f9db6b963e Mon Sep 17 00:00:00 2001 From: Hanno Hecker Date: Tue, 29 Oct 2019 09:47:00 +0100 Subject: [PATCH 11/13] Update worker and scheduler * Scheduler: ignore unknown fields in check definitions * Worker: * fix utf-8 in http().prometheus_flat() * job metrics handling Signed-off-by: Hanno Hecker --- cluster/config-defaults.yaml | 2 +- cluster/manifests/zmon-scheduler/deployment.yaml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cluster/config-defaults.yaml b/cluster/config-defaults.yaml index 34f14df028..a07af301a8 100644 --- a/cluster/config-defaults.yaml +++ b/cluster/config-defaults.yaml @@ -130,7 +130,7 @@ zmon_worker_cpu: "750m" zmon_worker_count: "16" {{end}} zmon_scalyr_region: "eu" -zmon_worker_version: "v209-py2eol-2-gcd9ec1c-v251-py2eol" +zmon_worker_version: "v209-py2eol-11-g0a79b03-v251-py2eol" logging_watcher_mem: "200Mi" logging_scalyr_mem: "175Mi" logging_slo_heartbeat_mem: "25Mi" diff --git a/cluster/manifests/zmon-scheduler/deployment.yaml b/cluster/manifests/zmon-scheduler/deployment.yaml index d7c72be32e..4af312cd09 100644 --- a/cluster/manifests/zmon-scheduler/deployment.yaml +++ b/cluster/manifests/zmon-scheduler/deployment.yaml @@ -5,7 +5,7 @@ metadata: namespace: visibility labels: application: zmon-scheduler - version: "v46-51-g0a0ad78-0.1.6-master-88" + version: "v46-55-gd1b34bc-0.1.6-master-92" spec: replicas: {{.ConfigItems.zmon_scheduler_replicas}} selector: @@ -15,7 +15,7 @@ spec: metadata: labels: application: zmon-scheduler - version: "v46-51-g0a0ad78-0.1.6-master-88" + version: "v46-55-gd1b34bc-0.1.6-master-92" annotations: iam.amazonaws.com/role: "{{ .LocalID }}-app-zmon" spec: @@ -47,7 +47,7 @@ spec: containers: - name: zmon-scheduler - image: "pierone.stups.zalan.do/zmon/zmon-scheduler:v46-51-g0a0ad78-0.1.6-master-88" + image: "pierone.stups.zalan.do/zmon/zmon-scheduler:v46-55-gd1b34bc-0.1.6-master-92" resources: limits: cpu: 1000m From 700ee791c1ee5d79ed0a67e4de34c299581ea0f9 Mon Sep 17 00:00:00 2001 From: njuettner Date: Wed, 30 Oct 2019 13:09:53 +0100 Subject: [PATCH 12/13] Disable CustomResourcePublishOpenAPI by default Signed-off-by: njuettner --- cluster/config-defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/config-defaults.yaml b/cluster/config-defaults.yaml index a07af301a8..95ed1ba19e 100644 --- a/cluster/config-defaults.yaml +++ b/cluster/config-defaults.yaml @@ -256,7 +256,7 @@ custom_resource_webhook_conversion: "false" # Feature toggle for CustomResourcePublishOpenAPI (alpha in v1.14) # https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/#publish-validation-schema-in-openapi-v2 -custom_resource_publish_openapi: "true" +custom_resource_publish_openapi: "false" # CIDR configuration for nodes and pods # Changing this will change the number of nodes and pods we can schedule in the From 17957bc2cf1846dab28ff30889a0d78a3084c58f Mon Sep 17 00:00:00 2001 From: Mikkel Oscar Lyderik Larsen Date: Wed, 30 Oct 2019 13:35:54 +0100 Subject: [PATCH 13/13] Drop quota from visibility namespace Signed-off-by: Mikkel Oscar Lyderik Larsen --- cluster/manifests/01-visibility/quota.yaml | 8 -------- cluster/manifests/deletions.yaml | 5 +---- 2 files changed, 1 insertion(+), 12 deletions(-) delete mode 100644 cluster/manifests/01-visibility/quota.yaml diff --git a/cluster/manifests/01-visibility/quota.yaml b/cluster/manifests/01-visibility/quota.yaml deleted file mode 100644 index c15bbced0a..0000000000 --- a/cluster/manifests/01-visibility/quota.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: ResourceQuota -metadata: - name: compute-resources - namespace: visibility -spec: - hard: - pods: "500" diff --git a/cluster/manifests/deletions.yaml b/cluster/manifests/deletions.yaml index d61b1b68d3..b1af39d9e0 100644 --- a/cluster/manifests/deletions.yaml +++ b/cluster/manifests/deletions.yaml @@ -4,10 +4,7 @@ pre_apply: [] # everything defined under here will be deleted after applying the manifests post_apply: - name: compute-resources - namespace: default - kind: ResourceQuota -- name: compute-resources - namespace: kube-system + namespace: visibility kind: ResourceQuota {{ if eq .ConfigItems.teapot_admission_controller_process_resources "true" }} - name: limits