From d476dc22b051cd9bc03636a46300d7900cacdebb Mon Sep 17 00:00:00 2001 From: Stoyan Rachev Date: Mon, 17 Aug 2020 11:38:30 +0300 Subject: [PATCH] Add controlplane controller and webhook for cloud-controller-manager --- charts/images.yaml | 6 +- .../internal/cloud-provider-config/Chart.yaml | 4 + .../templates/cloud-provider-config.tpl | 13 + .../templates/cloud-provider-config.yaml | 8 + .../cloud-provider-config/values.yaml | 9 + .../seed/templates/deployment.yaml | 3 + charts/internal/seed-controlplane/Chart.yaml | 4 + .../cloud-controller-manager/Chart.yaml | 4 + .../ccm-monitoring-dashboard.json | 418 ++++++++++++++++++ .../charts/utils-tls-cipher-suites | 1 + .../templates/_helpers.tpl | 17 + .../cloud-controller-manager-svc.yaml | 18 + .../templates/cloud-controller-manager.yaml | 111 +++++ .../templates/configmap-monitoring.yaml | 62 +++ .../templates/vpa.yaml | 12 + .../cloud-controller-manager/values.yaml | 18 + .../seed-controlplane/requirements.yaml | 5 + charts/internal/seed-controlplane/values.yaml | 2 + .../internal/shoot-storageclasses/Chart.yaml | 4 + .../templates/_helpers.tpl | 7 + .../internal/shoot-storageclasses/values.yaml | 0 .../shoot-system-components/Chart.yaml | 4 + .../cloud-controller-manager/Chart.yaml | 4 + .../templates/rbac-cloud-controller.yaml | 86 ++++ .../templates/rbac-node-controller.yaml | 43 ++ .../templates/rbac-pvl-controller.yaml | 34 ++ .../cloud-controller-manager/values.yaml | 0 .../shoot-system-components/requirements.yaml | 5 + .../shoot-system-components/values.yaml | 2 + .../utils-tls-cipher-suites/Chart.yaml | 4 + .../templates/_tls_cipher_suites.tpl | 8 + .../app/app.go | 6 +- go.mod | 5 + hack/api-reference/api.md | 45 ++ pkg/apis/kubevirt/types_controlplane.go | 10 + .../kubevirt/v1alpha1/types_controlplane.go | 11 + .../v1alpha1/zz_generated.conversion.go | 32 ++ .../v1alpha1/zz_generated.deepcopy.go | 28 ++ pkg/apis/kubevirt/zz_generated.deepcopy.go | 28 ++ pkg/cmd/config.go | 5 - pkg/controller/controlplane/add.go | 54 +-- .../controlplane/controlplane_suite_test.go | 27 ++ pkg/controller/controlplane/valuesprovider.go | 259 +++++++++++ .../controlplane/valuesprovider_test.go | 214 +++++++++ pkg/controller/infrastructure/actuator.go | 8 +- pkg/controller/infrastructure/add.go | 4 +- pkg/kubevirt/types.go | 8 + pkg/kubevirt/util.go | 33 ++ pkg/webhook/controlplane/ensurer.go | 103 ++++- pkg/webhook/controlplane/ensurer_test.go | 354 +++++++++++++++ .../pkg/webhook/controlplane/test/matchers.go | 61 +++ vendor/modules.txt | 6 + 52 files changed, 2155 insertions(+), 62 deletions(-) create mode 100644 charts/internal/cloud-provider-config/Chart.yaml create mode 100644 charts/internal/cloud-provider-config/templates/cloud-provider-config.tpl create mode 100644 charts/internal/cloud-provider-config/templates/cloud-provider-config.yaml create mode 100644 charts/internal/cloud-provider-config/values.yaml create mode 100644 charts/internal/seed-controlplane/Chart.yaml create mode 100644 charts/internal/seed-controlplane/charts/cloud-controller-manager/Chart.yaml create mode 100644 charts/internal/seed-controlplane/charts/cloud-controller-manager/ccm-monitoring-dashboard.json create mode 120000 charts/internal/seed-controlplane/charts/cloud-controller-manager/charts/utils-tls-cipher-suites create mode 100644 charts/internal/seed-controlplane/charts/cloud-controller-manager/templates/_helpers.tpl create mode 100644 charts/internal/seed-controlplane/charts/cloud-controller-manager/templates/cloud-controller-manager-svc.yaml create mode 100644 charts/internal/seed-controlplane/charts/cloud-controller-manager/templates/cloud-controller-manager.yaml create mode 100644 charts/internal/seed-controlplane/charts/cloud-controller-manager/templates/configmap-monitoring.yaml create mode 100644 charts/internal/seed-controlplane/charts/cloud-controller-manager/templates/vpa.yaml create mode 100644 charts/internal/seed-controlplane/charts/cloud-controller-manager/values.yaml create mode 100644 charts/internal/seed-controlplane/requirements.yaml create mode 100644 charts/internal/seed-controlplane/values.yaml create mode 100644 charts/internal/shoot-storageclasses/Chart.yaml create mode 100644 charts/internal/shoot-storageclasses/templates/_helpers.tpl create mode 100644 charts/internal/shoot-storageclasses/values.yaml create mode 100644 charts/internal/shoot-system-components/Chart.yaml create mode 100644 charts/internal/shoot-system-components/charts/cloud-controller-manager/Chart.yaml create mode 100644 charts/internal/shoot-system-components/charts/cloud-controller-manager/templates/rbac-cloud-controller.yaml create mode 100644 charts/internal/shoot-system-components/charts/cloud-controller-manager/templates/rbac-node-controller.yaml create mode 100644 charts/internal/shoot-system-components/charts/cloud-controller-manager/templates/rbac-pvl-controller.yaml create mode 100644 charts/internal/shoot-system-components/charts/cloud-controller-manager/values.yaml create mode 100644 charts/internal/shoot-system-components/requirements.yaml create mode 100644 charts/internal/shoot-system-components/values.yaml create mode 100644 charts/internal/utils-tls-cipher-suites/Chart.yaml create mode 100644 charts/internal/utils-tls-cipher-suites/templates/_tls_cipher_suites.tpl create mode 100644 pkg/controller/controlplane/controlplane_suite_test.go create mode 100644 pkg/controller/controlplane/valuesprovider.go create mode 100644 pkg/controller/controlplane/valuesprovider_test.go create mode 100644 pkg/kubevirt/util.go create mode 100644 pkg/webhook/controlplane/ensurer_test.go create mode 100644 vendor/github.com/gardener/gardener/extensions/pkg/webhook/controlplane/test/matchers.go diff --git a/charts/images.yaml b/charts/images.yaml index 12c2df19..1a98d8bb 100644 --- a/charts/images.yaml +++ b/charts/images.yaml @@ -6,4 +6,8 @@ images: - name: machine-controller-manager-provider-kubevirt sourceRepository: github.com/gardener/machine-controller-manager-provider-kubevirt repository: eu.gcr.io/gardener-project/gardener/machine-controller-manager-provider-kubevirt - tag: latest \ No newline at end of file + tag: "v0.1.0" +- name: cloud-controller-manager + sourceRepository: https://github.com/kubevirt/cloud-provider-kubevirt + repository: eu.gcr.io/gardener-project/gardener/kubevirt-cloud-controller-manager + tag: "v0.0.8" \ No newline at end of file diff --git a/charts/internal/cloud-provider-config/Chart.yaml b/charts/internal/cloud-provider-config/Chart.yaml new file mode 100644 index 00000000..72db7545 --- /dev/null +++ b/charts/internal/cloud-provider-config/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: Helm chart for kubernetes cloud-provider-config +name: cloud-provider-config +version: 0.1.0 diff --git a/charts/internal/cloud-provider-config/templates/cloud-provider-config.tpl b/charts/internal/cloud-provider-config/templates/cloud-provider-config.tpl new file mode 100644 index 00000000..ac740336 --- /dev/null +++ b/charts/internal/cloud-provider-config/templates/cloud-provider-config.tpl @@ -0,0 +1,13 @@ +{{- define "cloud-provider-config" -}} +kubeconfig: | +{{ .Values.kubeconfig | indent 2 }} +loadBalancer: + enabled: {{ .Values.loadBalancer.enabled }} + creationPollInterval: {{ .Values.loadBalancer.creationPollInterval }} +instances: + enabled: {{ .Values.instances.enabled }} + enableInstanceTypes: {{ .Values.instances.enableInstanceTypes }} +zones: + enabled: {{ .Values.zones.enabled }} +{{- end -}} + diff --git a/charts/internal/cloud-provider-config/templates/cloud-provider-config.yaml b/charts/internal/cloud-provider-config/templates/cloud-provider-config.yaml new file mode 100644 index 00000000..c08c2602 --- /dev/null +++ b/charts/internal/cloud-provider-config/templates/cloud-provider-config.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: cloud-provider-config + namespace: {{ .Release.Namespace }} +type: Opaque +data: + cloudprovider.conf: {{ include "cloud-provider-config" . | b64enc }} diff --git a/charts/internal/cloud-provider-config/values.yaml b/charts/internal/cloud-provider-config/values.yaml new file mode 100644 index 00000000..aabed171 --- /dev/null +++ b/charts/internal/cloud-provider-config/values.yaml @@ -0,0 +1,9 @@ +kubeconfig: abc +loadBalancer: + enabled: true + creationPollInterval: 5 +instances: + enabled: true + enableInstanceTypes: false +zones: + enabled: true diff --git a/charts/internal/machine-controller-manager/seed/templates/deployment.yaml b/charts/internal/machine-controller-manager/seed/templates/deployment.yaml index 4d637319..cd47bb22 100644 --- a/charts/internal/machine-controller-manager/seed/templates/deployment.yaml +++ b/charts/internal/machine-controller-manager/seed/templates/deployment.yaml @@ -31,6 +31,9 @@ spec: networking.gardener.cloud/to-private-networks: allowed networking.gardener.cloud/to-seed-apiserver: allowed networking.gardener.cloud/to-shoot-apiserver: allowed + # This network policy is needed to access the provider shoot if it happens to be collocated on the same seed + # TODO Replace this with a network policy that allows egress only to the provider shoot + networking.gardener.cloud/to-all-shoot-apiservers: allowed networking.gardener.cloud/from-prometheus: allowed {{- if .Values.podLabels }} {{ toYaml .Values.podLabels | indent 8 }} diff --git a/charts/internal/seed-controlplane/Chart.yaml b/charts/internal/seed-controlplane/Chart.yaml new file mode 100644 index 00000000..cbddad66 --- /dev/null +++ b/charts/internal/seed-controlplane/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: An umbrella chart for control plane resources in the Seed cluster +name: seed-controlplane +version: 0.1.0 diff --git a/charts/internal/seed-controlplane/charts/cloud-controller-manager/Chart.yaml b/charts/internal/seed-controlplane/charts/cloud-controller-manager/Chart.yaml new file mode 100644 index 00000000..05b7e169 --- /dev/null +++ b/charts/internal/seed-controlplane/charts/cloud-controller-manager/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: Helm chart for cloud-controller-manager +name: cloud-controller-manager +version: 0.1.0 diff --git a/charts/internal/seed-controlplane/charts/cloud-controller-manager/ccm-monitoring-dashboard.json b/charts/internal/seed-controlplane/charts/cloud-controller-manager/ccm-monitoring-dashboard.json new file mode 100644 index 00000000..89ea38f1 --- /dev/null +++ b/charts/internal/seed-controlplane/charts/cloud-controller-manager/ccm-monitoring-dashboard.json @@ -0,0 +1,418 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 17, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "Shows the memory usage of the cloud-controller-manager.", + "fill": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "container_memory_working_set_bytes{pod=~\"cloud-controller-manager-(.+)\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "current", + "refId": "A" + }, + { + "expr": "kube_pod_container_resource_limits_memory_bytes{pod=~\"cloud-controller-manager-(.+)\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "limits", + "refId": "B" + }, + { + "expr": "kube_pod_container_resource_requests_memory_bytes{pod=~\"cloud-controller-manager-(.+)\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "requests", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cloud-controller-manager Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "Shows the CPU usage of the cloud-controller-manager and shows the requests and limits.", + "fill": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(container_cpu_usage_seconds_total{pod=~\"cloud-controller-manager-(.+)\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "current", + "refId": "A" + }, + { + "expr": "kube_pod_container_resource_limits_cpu_cores{pod=~\"cloud-controller-manager-(.+)\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "limits", + "refId": "C" + }, + { + "expr": "kube_pod_container_resource_requests_cpu_cores{pod=~\"cloud-controller-manager-(.+)\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "requests", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cloud-controller-manager CPU usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "The average http request rate of the last 5m executed by the Cloud Controller Manager to the cluster API server. The request are split by their response status codes.", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{host=~\".*kube-apiserver.*\", job=\"cloud-controller-manager\"}[5m])) by(code)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request Rate to Kube API", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "prometheus", + "description": "Current uptime status of the cloud controller managers.", + "editable": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 12, + "y": 6 + }, + "hideTimeOverride": false, + "id": 2, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(sum(up{job=\"cloud-controller-manager\"} == 1) / sum(up{job=\"kube-controller-manager\"})) * 100", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 600 + } + ], + "thresholds": "50, 80", + "title": "Cloud Controller Managers UP", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + } + ], + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Cloud Controller Manager", + "uid": "8Uz5D5FWz", + "version": 7 +} \ No newline at end of file diff --git a/charts/internal/seed-controlplane/charts/cloud-controller-manager/charts/utils-tls-cipher-suites b/charts/internal/seed-controlplane/charts/cloud-controller-manager/charts/utils-tls-cipher-suites new file mode 120000 index 00000000..e1e890fa --- /dev/null +++ b/charts/internal/seed-controlplane/charts/cloud-controller-manager/charts/utils-tls-cipher-suites @@ -0,0 +1 @@ +../../../../utils-tls-cipher-suites \ No newline at end of file diff --git a/charts/internal/seed-controlplane/charts/cloud-controller-manager/templates/_helpers.tpl b/charts/internal/seed-controlplane/charts/cloud-controller-manager/templates/_helpers.tpl new file mode 100644 index 00000000..00625acc --- /dev/null +++ b/charts/internal/seed-controlplane/charts/cloud-controller-manager/templates/_helpers.tpl @@ -0,0 +1,17 @@ +{{- define "cloud-controller-manager.featureGates" -}} +{{- if .Values.featureGates }} +- --feature-gates={{ range $feature, $enabled := .Values.featureGates }}{{ $feature }}={{ $enabled }},{{ end }} +{{- end }} +{{- end -}} + +{{- define "cloud-controller-manager.port" -}} +{{- if semverCompare ">= 1.13" .Values.kubernetesVersion -}} +10258 +{{- else -}} +10253 +{{- end -}} +{{- end -}} + +{{- define "deploymentversion" -}} +apps/v1 +{{- end -}} diff --git a/charts/internal/seed-controlplane/charts/cloud-controller-manager/templates/cloud-controller-manager-svc.yaml b/charts/internal/seed-controlplane/charts/cloud-controller-manager/templates/cloud-controller-manager-svc.yaml new file mode 100644 index 00000000..365fa6d1 --- /dev/null +++ b/charts/internal/seed-controlplane/charts/cloud-controller-manager/templates/cloud-controller-manager-svc.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: cloud-controller-manager + namespace: {{ .Release.Namespace }} + labels: + app: kubernetes + role: cloud-controller-manager +spec: + type: ClusterIP + clusterIP: None + ports: + - name: metrics + port: {{ include "cloud-controller-manager.port" . }} + protocol: TCP + selector: + app: kubernetes + role: cloud-controller-manager \ No newline at end of file diff --git a/charts/internal/seed-controlplane/charts/cloud-controller-manager/templates/cloud-controller-manager.yaml b/charts/internal/seed-controlplane/charts/cloud-controller-manager/templates/cloud-controller-manager.yaml new file mode 100644 index 00000000..d6f03432 --- /dev/null +++ b/charts/internal/seed-controlplane/charts/cloud-controller-manager/templates/cloud-controller-manager.yaml @@ -0,0 +1,111 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cloud-controller-manager + namespace: {{ .Release.Namespace }} + labels: + garden.sapcloud.io/role: controlplane + app: kubernetes + role: cloud-controller-manager +spec: + revisionHistoryLimit: 0 + replicas: {{ .Values.replicas }} + selector: + matchLabels: + app: kubernetes + role: cloud-controller-manager + template: + metadata: +{{- if .Values.podAnnotations }} + annotations: +{{ toYaml .Values.podAnnotations | indent 8 }} +{{- end }} + labels: + garden.sapcloud.io/role: controlplane + app: kubernetes + role: cloud-controller-manager + networking.gardener.cloud/to-dns: allowed + networking.gardener.cloud/to-public-networks: allowed + networking.gardener.cloud/to-shoot-apiserver: allowed + networking.gardener.cloud/from-prometheus: allowed + # This network policy is needed to access the provider shoot if it happens to be collocated on the same seed + # TODO Replace this with a network policy that allows egress only to the provider shoot + networking.gardener.cloud/to-all-shoot-apiservers: allowed +{{- if .Values.podLabels }} +{{ toYaml .Values.podLabels | indent 8 }} +{{- end }} + spec: + containers: + - name: kubevirt-cloud-controller-manager + image: {{ index .Values.images "cloud-controller-manager" }} + imagePullPolicy: IfNotPresent + command: + - /bin/kubevirt-cloud-controller-manager + - --allocate-node-cidrs=true + - --cloud-provider=kubevirt + - --cloud-config=/etc/kubernetes/cloudprovider/cloudprovider.conf + - --cluster-cidr={{ .Values.podNetwork }} + - --cluster-name={{ .Values.clusterName }} + - --concurrent-service-syncs=1 + - --configure-cloud-routes=true + {{- include "cloud-controller-manager.featureGates" . | trimSuffix "," | indent 8 }} + - --kubeconfig=/var/lib/cloud-controller-manager/kubeconfig + - --leader-elect=true + {{- if semverCompare ">= 1.13" .Values.kubernetesVersion }} + - --secure-port={{ include "cloud-controller-manager.port" . }} + - --port=0 + {{- end }} + {{- if semverCompare ">= 1.12" .Values.kubernetesVersion }} + - --authentication-kubeconfig=/var/lib/cloud-controller-manager/kubeconfig + - --authorization-kubeconfig=/var/lib/cloud-controller-manager/kubeconfig + - --tls-cert-file=/var/lib/cloud-controller-manager-server/cloud-controller-manager-server.crt + - --tls-private-key-file=/var/lib/cloud-controller-manager-server/cloud-controller-manager-server.key + {{- end }} + - --tls-cipher-suites={{ include "kubernetes.tlsCipherSuites" . | replace "\n" "," | trimPrefix "," }} + - --use-service-account-credentials + - --v=2 + livenessProbe: + httpGet: + path: /healthz + {{- if semverCompare ">= 1.13" .Values.kubernetesVersion }} + scheme: HTTPS + {{- else }} + scheme: HTTP + {{- end }} + port: {{ include "cloud-controller-manager.port" . }} + successThreshold: 1 + failureThreshold: 2 + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 15 + ports: + - containerPort: {{ include "cloud-controller-manager.port" . }} + name: metrics + protocol: TCP + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + {{- if .Values.resources }} + resources: +{{ toYaml .Values.resources | indent 10 }} + {{- end }} + volumeMounts: + - name: cloud-controller-manager + mountPath: /var/lib/cloud-controller-manager + - name: cloud-controller-manager-server + mountPath: /var/lib/cloud-controller-manager-server + - name: cloud-provider-config + mountPath: /etc/kubernetes/cloudprovider + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + terminationGracePeriodSeconds: 30 + volumes: + - name: cloud-controller-manager + secret: + secretName: cloud-controller-manager + - name: cloud-controller-manager-server + secret: + secretName: cloud-controller-manager-server + - name: cloud-provider-config + secret: + secretName: cloud-provider-config diff --git a/charts/internal/seed-controlplane/charts/cloud-controller-manager/templates/configmap-monitoring.yaml b/charts/internal/seed-controlplane/charts/cloud-controller-manager/templates/configmap-monitoring.yaml new file mode 100644 index 00000000..769936b1 --- /dev/null +++ b/charts/internal/seed-controlplane/charts/cloud-controller-manager/templates/configmap-monitoring.yaml @@ -0,0 +1,62 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: cloud-controller-manager-monitoring-config + namespace: {{ .Release.Namespace }} + labels: + extensions.gardener.cloud/configuration: monitoring +data: + scrape_config: | + - job_name: cloud-controller-manager + {{- if semverCompare ">= 1.13" .Values.kubernetesVersion }} + scheme: https + tls_config: + insecure_skip_verify: true + cert_file: /etc/prometheus/seed/prometheus.crt + key_file: /etc/prometheus/seed/prometheus.key + {{- end }} + honor_labels: false + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: [{{ .Release.Namespace }}] + relabel_configs: + - source_labels: + - __meta_kubernetes_service_name + - __meta_kubernetes_endpoint_port_name + action: keep + regex: cloud-controller-manager;metrics + # common metrics + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: [ __meta_kubernetes_pod_name ] + target_label: pod + metric_relabel_configs: + - source_labels: [ __name__ ] + regex: ^(rest_client_requests_total|process_max_fds|process_open_fds)$ + action: keep + + alerting_rules: | + cloud-controller-manager.rules.yaml: | + groups: + - name: cloud-controller-manager.rules + rules: + - alert: CloudControllerManagerDown + expr: absent(up{job="cloud-controller-manager"} == 1) + for: 15m + labels: + service: cloud-controller-manager + severity: critical + type: seed + visibility: all + annotations: + description: All infrastruture specific operations cannot be completed (e.g. creating loadbalancers or persistent volumes). + summary: Cloud controller manager is down. + + dashboard_operators: | + cloud-controller-manager-dashboard.json: |- +{{- .Files.Get "ccm-monitoring-dashboard.json" | nindent 6 }} + + dashboard_users: | + cloud-controller-manager-dashboard.json: |- +{{- .Files.Get "ccm-monitoring-dashboard.json" | nindent 6 }} diff --git a/charts/internal/seed-controlplane/charts/cloud-controller-manager/templates/vpa.yaml b/charts/internal/seed-controlplane/charts/cloud-controller-manager/templates/vpa.yaml new file mode 100644 index 00000000..472a971b --- /dev/null +++ b/charts/internal/seed-controlplane/charts/cloud-controller-manager/templates/vpa.yaml @@ -0,0 +1,12 @@ +apiVersion: autoscaling.k8s.io/v1beta2 +kind: VerticalPodAutoscaler +metadata: + name: cloud-controller-manager-vpa + namespace: {{ .Release.Namespace }} +spec: + targetRef: + apiVersion: {{ include "deploymentversion" . }} + kind: Deployment + name: cloud-controller-manager + updatePolicy: + updateMode: Auto diff --git a/charts/internal/seed-controlplane/charts/cloud-controller-manager/values.yaml b/charts/internal/seed-controlplane/charts/cloud-controller-manager/values.yaml new file mode 100644 index 00000000..35ebafef --- /dev/null +++ b/charts/internal/seed-controlplane/charts/cloud-controller-manager/values.yaml @@ -0,0 +1,18 @@ +replicas: 1 +clusterName: shoot-foo-bar +kubernetesVersion: 1.7.5 +podNetwork: 192.168.0.0/16 +podAnnotations: {} +podLabels: {} +featureGates: {} + # CustomResourceValidation: true + # RotateKubeletServerCertificate: false +images: + cloud-controller-manager: image-repository:image-tag +resources: + requests: + cpu: 11m + memory: 75Mi + limits: + cpu: 500m + memory: 512Mi diff --git a/charts/internal/seed-controlplane/requirements.yaml b/charts/internal/seed-controlplane/requirements.yaml new file mode 100644 index 00000000..c4784a4c --- /dev/null +++ b/charts/internal/seed-controlplane/requirements.yaml @@ -0,0 +1,5 @@ +dependencies: +- name: cloud-controller-manager + repository: http://localhost:10191 + version: 0.1.0 + condition: cloud-controller-manager.enabled diff --git a/charts/internal/seed-controlplane/values.yaml b/charts/internal/seed-controlplane/values.yaml new file mode 100644 index 00000000..8bd1e31c --- /dev/null +++ b/charts/internal/seed-controlplane/values.yaml @@ -0,0 +1,2 @@ +cloud-controller-manager: + enabled: true diff --git a/charts/internal/shoot-storageclasses/Chart.yaml b/charts/internal/shoot-storageclasses/Chart.yaml new file mode 100644 index 00000000..fe504597 --- /dev/null +++ b/charts/internal/shoot-storageclasses/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: A Helm chart for storageclasses that should be installed to the shoot +name: shoot-storageclasses +version: 0.1.0 diff --git a/charts/internal/shoot-storageclasses/templates/_helpers.tpl b/charts/internal/shoot-storageclasses/templates/_helpers.tpl new file mode 100644 index 00000000..50c126b0 --- /dev/null +++ b/charts/internal/shoot-storageclasses/templates/_helpers.tpl @@ -0,0 +1,7 @@ +{{- define "storageclassversion" -}} +{{- if semverCompare ">= 1.13-0" .Capabilities.KubeVersion.GitVersion -}} +storage.k8s.io/v1 +{{- else -}} +storage.k8s.io/v1beta1 +{{- end -}} +{{- end -}} diff --git a/charts/internal/shoot-storageclasses/values.yaml b/charts/internal/shoot-storageclasses/values.yaml new file mode 100644 index 00000000..e69de29b diff --git a/charts/internal/shoot-system-components/Chart.yaml b/charts/internal/shoot-system-components/Chart.yaml new file mode 100644 index 00000000..e44304be --- /dev/null +++ b/charts/internal/shoot-system-components/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: An umbrella chart for control plane resources in the Shoot cluster +name: shoot-system-components +version: 0.1.0 diff --git a/charts/internal/shoot-system-components/charts/cloud-controller-manager/Chart.yaml b/charts/internal/shoot-system-components/charts/cloud-controller-manager/Chart.yaml new file mode 100644 index 00000000..05b7e169 --- /dev/null +++ b/charts/internal/shoot-system-components/charts/cloud-controller-manager/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: Helm chart for cloud-controller-manager +name: cloud-controller-manager +version: 0.1.0 diff --git a/charts/internal/shoot-system-components/charts/cloud-controller-manager/templates/rbac-cloud-controller.yaml b/charts/internal/shoot-system-components/charts/cloud-controller-manager/templates/rbac-cloud-controller.yaml new file mode 100644 index 00000000..8a0664f8 --- /dev/null +++ b/charts/internal/shoot-system-components/charts/cloud-controller-manager/templates/rbac-cloud-controller.yaml @@ -0,0 +1,86 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:cloud-controller-manager +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - '*' +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +- apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - '*' +- apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - list + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:cloud-controller-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager +subjects: +- kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system \ No newline at end of file diff --git a/charts/internal/shoot-system-components/charts/cloud-controller-manager/templates/rbac-node-controller.yaml b/charts/internal/shoot-system-components/charts/cloud-controller-manager/templates/rbac-node-controller.yaml new file mode 100644 index 00000000..5790d597 --- /dev/null +++ b/charts/internal/shoot-system-components/charts/cloud-controller-manager/templates/rbac-node-controller.yaml @@ -0,0 +1,43 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:controller:cloud-node-controller +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - delete + - get + - patch + - update + - list +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:controller:cloud-node-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:controller:cloud-node-controller +subjects: +- kind: ServiceAccount + name: cloud-node-controller + namespace: kube-system diff --git a/charts/internal/shoot-system-components/charts/cloud-controller-manager/templates/rbac-pvl-controller.yaml b/charts/internal/shoot-system-components/charts/cloud-controller-manager/templates/rbac-pvl-controller.yaml new file mode 100644 index 00000000..313749e0 --- /dev/null +++ b/charts/internal/shoot-system-components/charts/cloud-controller-manager/templates/rbac-pvl-controller.yaml @@ -0,0 +1,34 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:controller:pvl-controller +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - '*' +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:controller:pvl-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:controller:pvl-controller +subjects: +- kind: ServiceAccount + name: pvl-controller + namespace: kube-system + \ No newline at end of file diff --git a/charts/internal/shoot-system-components/charts/cloud-controller-manager/values.yaml b/charts/internal/shoot-system-components/charts/cloud-controller-manager/values.yaml new file mode 100644 index 00000000..e69de29b diff --git a/charts/internal/shoot-system-components/requirements.yaml b/charts/internal/shoot-system-components/requirements.yaml new file mode 100644 index 00000000..c4784a4c --- /dev/null +++ b/charts/internal/shoot-system-components/requirements.yaml @@ -0,0 +1,5 @@ +dependencies: +- name: cloud-controller-manager + repository: http://localhost:10191 + version: 0.1.0 + condition: cloud-controller-manager.enabled diff --git a/charts/internal/shoot-system-components/values.yaml b/charts/internal/shoot-system-components/values.yaml new file mode 100644 index 00000000..8bd1e31c --- /dev/null +++ b/charts/internal/shoot-system-components/values.yaml @@ -0,0 +1,2 @@ +cloud-controller-manager: + enabled: true diff --git a/charts/internal/utils-tls-cipher-suites/Chart.yaml b/charts/internal/utils-tls-cipher-suites/Chart.yaml new file mode 100644 index 00000000..11d39bd1 --- /dev/null +++ b/charts/internal/utils-tls-cipher-suites/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: Util chart for cipher-suites +name: utils-tls-cipher-suites +version: 0.1.0 diff --git a/charts/internal/utils-tls-cipher-suites/templates/_tls_cipher_suites.tpl b/charts/internal/utils-tls-cipher-suites/templates/_tls_cipher_suites.tpl new file mode 100644 index 00000000..951a2bc7 --- /dev/null +++ b/charts/internal/utils-tls-cipher-suites/templates/_tls_cipher_suites.tpl @@ -0,0 +1,8 @@ +{{- define "kubernetes.tlsCipherSuites" }} +TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 +TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 +TLS_RSA_WITH_AES_128_CBC_SHA +TLS_RSA_WITH_AES_256_CBC_SHA +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA +{{- end -}} diff --git a/cmd/gardener-extension-provider-kubevirt/app/app.go b/cmd/gardener-extension-provider-kubevirt/app/app.go index 9daf076a..f50ce434 100644 --- a/cmd/gardener-extension-provider-kubevirt/app/app.go +++ b/cmd/gardener-extension-provider-kubevirt/app/app.go @@ -38,6 +38,7 @@ import ( machinev1alpha1 "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + autoscalingv1beta2 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1beta2" "sigs.k8s.io/controller-runtime/pkg/manager" ) @@ -135,6 +136,9 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { if err := druidv1alpha1.AddToScheme(scheme); err != nil { controllercmd.LogErrAndExit(err, "Could not update manager scheme") } + if err := autoscalingv1beta2.AddToScheme(scheme); err != nil { + controllercmd.LogErrAndExit(err, "Could not update manager scheme") + } if err := machinev1alpha1.AddToScheme(scheme); err != nil { controllercmd.LogErrAndExit(err, "Could not update manager scheme") } @@ -144,8 +148,6 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { // apply config options configFileOpts.Completed().ApplyETCDStorage(&kubevirtcontrolplaneexposure.DefaultAddOptions.ETCDStorage) - configFileOpts.Completed().ApplyGardenId(&kubevirtcontrolplane.DefaultAddOptions.GardenId) - configFileOpts.Completed().ApplyGardenId(&kubevirtinfra.DefaultAddOptions.GardenId) configFileOpts.Completed().ApplyHealthCheckConfig(&healthcheck.DefaultAddOptions.HealthCheckConfig) // apply controller options diff --git a/go.mod b/go.mod index a2c37057..7786d8d0 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.14 require ( github.com/ahmetb/gen-crd-api-reference-docs v0.2.0 + github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f github.com/gardener/etcd-druid v0.3.0 github.com/gardener/gardener v1.6.4 github.com/gardener/machine-controller-manager v0.27.0 @@ -18,9 +19,13 @@ require ( k8s.io/api v0.18.4 k8s.io/apiextensions-apiserver v0.17.6 k8s.io/apimachinery v0.18.4 + k8s.io/apiserver v0.17.6 + k8s.io/autoscaler v0.0.0-20190805135949-100e91ba756e k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible k8s.io/code-generator v0.18.4 k8s.io/component-base v0.18.4 + k8s.io/kubelet v0.17.6 + k8s.io/utils v0.0.0-20200327001022-6496210b90e8 sigs.k8s.io/controller-runtime v0.5.4 ) diff --git a/hack/api-reference/api.md b/hack/api-reference/api.md index 96d79571..9992af51 100644 --- a/hack/api-reference/api.md +++ b/hack/api-reference/api.md @@ -108,6 +108,20 @@ string ControlPlaneConfig + + +cloudControllerManager
+ + +CloudControllerManagerConfig + + + + +(Optional) +

CloudControllerManager contains configuration settings for the cloud-controller-manager.

+ +

InfrastructureConfig @@ -192,6 +206,37 @@ reconciliation is possible.

+

CloudControllerManagerConfig +

+

+(Appears on: +ControlPlaneConfig) +

+

+

CloudControllerManagerConfig contains configuration settings for the cloud-controller-manager.

+

+ + + + + + + + + + + + + +
FieldDescription
+featureGates
+ +map[string]bool + +		 +(Optional) +

FeatureGates contains information about enabled feature gates.

+

InfrastructureStatus

diff --git a/pkg/apis/kubevirt/types_controlplane.go b/pkg/apis/kubevirt/types_controlplane.go index 7442ff6e..96daa15f 100644 --- a/pkg/apis/kubevirt/types_controlplane.go +++ b/pkg/apis/kubevirt/types_controlplane.go @@ -23,4 +23,14 @@ import ( // ControlPlaneConfig contains configuration settings for the control plane. type ControlPlaneConfig struct { metav1.TypeMeta + + // CloudControllerManager contains configuration settings for the cloud-controller-manager. + // +optional + CloudControllerManager *CloudControllerManagerConfig +} + +// CloudControllerManagerConfig contains configuration settings for the cloud-controller-manager. +type CloudControllerManagerConfig struct { + // FeatureGates contains information about enabled feature gates. + FeatureGates map[string]bool } diff --git a/pkg/apis/kubevirt/v1alpha1/types_controlplane.go b/pkg/apis/kubevirt/v1alpha1/types_controlplane.go index b54de462..b4ba8956 100644 --- a/pkg/apis/kubevirt/v1alpha1/types_controlplane.go +++ b/pkg/apis/kubevirt/v1alpha1/types_controlplane.go @@ -24,4 +24,15 @@ import ( // ControlPlaneConfig contains configuration settings for the control plane. type ControlPlaneConfig struct { metav1.TypeMeta `json:",inline"` + + // CloudControllerManager contains configuration settings for the cloud-controller-manager. + // +optional + CloudControllerManager *CloudControllerManagerConfig `json:"cloudControllerManager,omitempty"` +} + +// CloudControllerManagerConfig contains configuration settings for the cloud-controller-manager. +type CloudControllerManagerConfig struct { + // FeatureGates contains information about enabled feature gates. + // +optional + FeatureGates map[string]bool `json:"featureGates,omitempty"` } diff --git a/pkg/apis/kubevirt/v1alpha1/zz_generated.conversion.go b/pkg/apis/kubevirt/v1alpha1/zz_generated.conversion.go index 6ac60021..6e1b80f9 100644 --- a/pkg/apis/kubevirt/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/kubevirt/v1alpha1/zz_generated.conversion.go @@ -35,6 +35,16 @@ func init() { // RegisterConversions adds conversion functions to the given scheme. // Public to allow building arbitrary schemes. func RegisterConversions(s *runtime.Scheme) error { + if err := s.AddGeneratedConversionFunc((*CloudControllerManagerConfig)(nil), (*kubevirt.CloudControllerManagerConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_CloudControllerManagerConfig_To_kubevirt_CloudControllerManagerConfig(a.(*CloudControllerManagerConfig), b.(*kubevirt.CloudControllerManagerConfig), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*kubevirt.CloudControllerManagerConfig)(nil), (*CloudControllerManagerConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kubevirt_CloudControllerManagerConfig_To_v1alpha1_CloudControllerManagerConfig(a.(*kubevirt.CloudControllerManagerConfig), b.(*CloudControllerManagerConfig), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*CloudProfileConfig)(nil), (*kubevirt.CloudProfileConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha1_CloudProfileConfig_To_kubevirt_CloudProfileConfig(a.(*CloudProfileConfig), b.(*kubevirt.CloudProfileConfig), scope) }); err != nil { @@ -128,6 +138,26 @@ func RegisterConversions(s *runtime.Scheme) error { return nil } +func autoConvert_v1alpha1_CloudControllerManagerConfig_To_kubevirt_CloudControllerManagerConfig(in *CloudControllerManagerConfig, out *kubevirt.CloudControllerManagerConfig, s conversion.Scope) error { + out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates)) + return nil +} + +// Convert_v1alpha1_CloudControllerManagerConfig_To_kubevirt_CloudControllerManagerConfig is an autogenerated conversion function. +func Convert_v1alpha1_CloudControllerManagerConfig_To_kubevirt_CloudControllerManagerConfig(in *CloudControllerManagerConfig, out *kubevirt.CloudControllerManagerConfig, s conversion.Scope) error { + return autoConvert_v1alpha1_CloudControllerManagerConfig_To_kubevirt_CloudControllerManagerConfig(in, out, s) +} + +func autoConvert_kubevirt_CloudControllerManagerConfig_To_v1alpha1_CloudControllerManagerConfig(in *kubevirt.CloudControllerManagerConfig, out *CloudControllerManagerConfig, s conversion.Scope) error { + out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates)) + return nil +} + +// Convert_kubevirt_CloudControllerManagerConfig_To_v1alpha1_CloudControllerManagerConfig is an autogenerated conversion function. +func Convert_kubevirt_CloudControllerManagerConfig_To_v1alpha1_CloudControllerManagerConfig(in *kubevirt.CloudControllerManagerConfig, out *CloudControllerManagerConfig, s conversion.Scope) error { + return autoConvert_kubevirt_CloudControllerManagerConfig_To_v1alpha1_CloudControllerManagerConfig(in, out, s) +} + func autoConvert_v1alpha1_CloudProfileConfig_To_kubevirt_CloudProfileConfig(in *CloudProfileConfig, out *kubevirt.CloudProfileConfig, s conversion.Scope) error { out.MachineImages = *(*[]kubevirt.MachineImages)(unsafe.Pointer(&in.MachineImages)) out.MachineDeploymentConfig = *(*[]kubevirt.MachineDeploymentConfig)(unsafe.Pointer(&in.MachineDeploymentConfig)) @@ -151,6 +181,7 @@ func Convert_kubevirt_CloudProfileConfig_To_v1alpha1_CloudProfileConfig(in *kube } func autoConvert_v1alpha1_ControlPlaneConfig_To_kubevirt_ControlPlaneConfig(in *ControlPlaneConfig, out *kubevirt.ControlPlaneConfig, s conversion.Scope) error { + out.CloudControllerManager = (*kubevirt.CloudControllerManagerConfig)(unsafe.Pointer(in.CloudControllerManager)) return nil } @@ -160,6 +191,7 @@ func Convert_v1alpha1_ControlPlaneConfig_To_kubevirt_ControlPlaneConfig(in *Cont } func autoConvert_kubevirt_ControlPlaneConfig_To_v1alpha1_ControlPlaneConfig(in *kubevirt.ControlPlaneConfig, out *ControlPlaneConfig, s conversion.Scope) error { + out.CloudControllerManager = (*CloudControllerManagerConfig)(unsafe.Pointer(in.CloudControllerManager)) return nil } diff --git a/pkg/apis/kubevirt/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/kubevirt/v1alpha1/zz_generated.deepcopy.go index 3c500afc..2e63eb5d 100644 --- a/pkg/apis/kubevirt/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/kubevirt/v1alpha1/zz_generated.deepcopy.go @@ -24,6 +24,29 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudControllerManagerConfig) DeepCopyInto(out *CloudControllerManagerConfig) { + *out = *in + if in.FeatureGates != nil { + in, out := &in.FeatureGates, &out.FeatureGates + *out = make(map[string]bool, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudControllerManagerConfig. +func (in *CloudControllerManagerConfig) DeepCopy() *CloudControllerManagerConfig { + if in == nil { + return nil + } + out := new(CloudControllerManagerConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CloudProfileConfig) DeepCopyInto(out *CloudProfileConfig) { *out = *in @@ -65,6 +88,11 @@ func (in *CloudProfileConfig) DeepCopyObject() runtime.Object { func (in *ControlPlaneConfig) DeepCopyInto(out *ControlPlaneConfig) { *out = *in out.TypeMeta = in.TypeMeta + if in.CloudControllerManager != nil { + in, out := &in.CloudControllerManager, &out.CloudControllerManager + *out = new(CloudControllerManagerConfig) + (*in).DeepCopyInto(*out) + } return } diff --git a/pkg/apis/kubevirt/zz_generated.deepcopy.go b/pkg/apis/kubevirt/zz_generated.deepcopy.go index 826c74ad..eb20f164 100644 --- a/pkg/apis/kubevirt/zz_generated.deepcopy.go +++ b/pkg/apis/kubevirt/zz_generated.deepcopy.go @@ -24,6 +24,29 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudControllerManagerConfig) DeepCopyInto(out *CloudControllerManagerConfig) { + *out = *in + if in.FeatureGates != nil { + in, out := &in.FeatureGates, &out.FeatureGates + *out = make(map[string]bool, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudControllerManagerConfig. +func (in *CloudControllerManagerConfig) DeepCopy() *CloudControllerManagerConfig { + if in == nil { + return nil + } + out := new(CloudControllerManagerConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CloudProfileConfig) DeepCopyInto(out *CloudProfileConfig) { *out = *in @@ -65,6 +88,11 @@ func (in *CloudProfileConfig) DeepCopyObject() runtime.Object { func (in *ControlPlaneConfig) DeepCopyInto(out *ControlPlaneConfig) { *out = *in out.TypeMeta = in.TypeMeta + if in.CloudControllerManager != nil { + in, out := &in.CloudControllerManager, &out.CloudControllerManager + *out = new(CloudControllerManagerConfig) + (*in).DeepCopyInto(*out) + } return } diff --git a/pkg/cmd/config.go b/pkg/cmd/config.go index e2b350b4..61e7d492 100644 --- a/pkg/cmd/config.go +++ b/pkg/cmd/config.go @@ -76,11 +76,6 @@ func (c *Config) ApplyETCDStorage(etcdStorage *config.ETCDStorage) { *etcdStorage = c.Config.ETCD.Storage } -// ApplyGardenId sets the gardenId. -func (c *Config) ApplyGardenId(gardenId *string) { - *gardenId = c.Config.GardenId -} - // Options initializes empty config.ControllerConfiguration, applies the set values and returns it. func (c *Config) Options() config.ControllerConfiguration { var cfg config.ControllerConfiguration diff --git a/pkg/controller/controlplane/add.go b/pkg/controller/controlplane/add.go index 439c6926..3bddcdde 100644 --- a/pkg/controller/controlplane/add.go +++ b/pkg/controller/controlplane/add.go @@ -15,15 +15,13 @@ package controlplane import ( - "context" - + "github.com/gardener/gardener-extension-provider-kubevirt/pkg/imagevector" "github.com/gardener/gardener-extension-provider-kubevirt/pkg/kubevirt" extensionscontroller "github.com/gardener/gardener/extensions/pkg/controller" - "github.com/gardener/gardener/extensions/pkg/controller/common" "github.com/gardener/gardener/extensions/pkg/controller/controlplane" - extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" - "github.com/go-logr/logr" + "github.com/gardener/gardener/extensions/pkg/controller/controlplane/genericactuator" + "github.com/gardener/gardener/extensions/pkg/util" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -32,23 +30,25 @@ import ( var ( // DefaultAddOptions are the default AddOptions for AddToManager. DefaultAddOptions = AddOptions{} + + logger = log.Log.WithName("kubevirt-controlplane-controller") ) -// AddOptions are options to apply when adding the KubeVirt controlplane controller to the manager. +// AddOptions are options to apply when adding the Kubevirt controlplane controller to the manager. type AddOptions struct { // Controller are the controller.Options. Controller controller.Options // IgnoreOperationAnnotation specifies whether to ignore the operation annotation or not. IgnoreOperationAnnotation bool - // GardenId is the Gardener garden identity - GardenId string } // AddToManagerWithOptions adds a controller with the given Options to the given manager. // The opts.Reconciler is being set with a newly instantiated actuator. func AddToManagerWithOptions(mgr manager.Manager, opts AddOptions) error { return controlplane.Add(mgr, controlplane.AddArgs{ - Actuator: NewActuator(opts.GardenId), + Actuator: genericactuator.NewActuator(kubevirt.Name, controlPlaneSecrets, nil, configChart, controlPlaneChart, controlPlaneShootChart, + storageClassChart, nil, NewValuesProvider(logger), extensionscontroller.ChartRendererFactoryFunc(util.NewChartRendererForShoot), + imagevector.ImageVector(), "", nil, mgr.GetWebhookServer().Port, logger), ControllerOptions: opts.Controller, Predicates: controlplane.DefaultPredicates(opts.IgnoreOperationAnnotation), Type: kubevirt.Type, @@ -59,39 +59,3 @@ func AddToManagerWithOptions(mgr manager.Manager, opts AddOptions) error { func AddToManager(mgr manager.Manager) error { return AddToManagerWithOptions(mgr, DefaultAddOptions) } - -// NewActuator creates a new Actuator that updates the status of the handled Infrastructure resources. -func NewActuator(gardenID string) controlplane.Actuator { - return &actuator{ - logger: log.Log.WithName("infrastructure-actuator"), - gardenID: gardenID, - } -} - -type actuator struct { - common.ChartRendererContext - - logger logr.Logger - gardenID string -} - -func (a *actuator) Reconcile(context.Context, *extensionsv1alpha1.ControlPlane, *extensionscontroller.Cluster) (bool, error) { - a.logger.Info("control-plane reconciled") - // TODO: install kubevirt-cloud-controller-manager here and related components, the genericActuator might be used here - return false, nil -} - -// Delete deletes the ControlPlane. -func (a *actuator) Delete(context.Context, *extensionsv1alpha1.ControlPlane, *extensionscontroller.Cluster) error { - return nil -} - -// Restore restores the ControlPlane. -func (a *actuator) Restore(context.Context, *extensionsv1alpha1.ControlPlane, *extensionscontroller.Cluster) (bool, error) { - return false, nil -} - -// Migrate migrates the ControlPlane. -func (a *actuator) Migrate(context.Context, *extensionsv1alpha1.ControlPlane, *extensionscontroller.Cluster) error { - return nil -} diff --git a/pkg/controller/controlplane/controlplane_suite_test.go b/pkg/controller/controlplane/controlplane_suite_test.go new file mode 100644 index 00000000..3dfd8023 --- /dev/null +++ b/pkg/controller/controlplane/controlplane_suite_test.go @@ -0,0 +1,27 @@ +// Copyright (c) 2020 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplane_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestControlplane(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Controlplane Suite") +} diff --git a/pkg/controller/controlplane/valuesprovider.go b/pkg/controller/controlplane/valuesprovider.go new file mode 100644 index 00000000..928919dc --- /dev/null +++ b/pkg/controller/controlplane/valuesprovider.go @@ -0,0 +1,259 @@ +// Copyright (c) 2020 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplane + +import ( + "context" + "path/filepath" + + apiskubevirt "github.com/gardener/gardener-extension-provider-kubevirt/pkg/apis/kubevirt" + "github.com/gardener/gardener-extension-provider-kubevirt/pkg/kubevirt" + + extensionscontroller "github.com/gardener/gardener/extensions/pkg/controller" + "github.com/gardener/gardener/extensions/pkg/controller/controlplane" + "github.com/gardener/gardener/extensions/pkg/controller/controlplane/genericactuator" + "github.com/gardener/gardener/extensions/pkg/util" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" + "github.com/gardener/gardener/pkg/utils/chart" + "github.com/gardener/gardener/pkg/utils/secrets" + "github.com/go-logr/logr" + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apiserver/pkg/authentication/user" + autoscalingv1beta2 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1beta2" +) + +var ( + controlPlaneSecrets = &secrets.Secrets{ + CertificateSecretConfigs: map[string]*secrets.CertificateSecretConfig{ + v1beta1constants.SecretNameCACluster: { + Name: v1beta1constants.SecretNameCACluster, + CommonName: "kubernetes", + CertType: secrets.CACert, + }, + }, + SecretConfigsFunc: func(cas map[string]*secrets.Certificate, clusterName string) []secrets.ConfigInterface { + return []secrets.ConfigInterface{ + &secrets.ControlPlaneSecretConfig{ + CertificateSecretConfig: &secrets.CertificateSecretConfig{ + Name: kubevirt.CloudControllerManagerName, + CommonName: "system:cloud-controller-manager", + Organization: []string{user.SystemPrivilegedGroup}, + CertType: secrets.ClientCert, + SigningCA: cas[v1beta1constants.SecretNameCACluster], + }, + KubeConfigRequest: &secrets.KubeConfigRequest{ + ClusterName: clusterName, + APIServerURL: v1beta1constants.DeploymentNameKubeAPIServer, + }, + }, + &secrets.ControlPlaneSecretConfig{ + CertificateSecretConfig: &secrets.CertificateSecretConfig{ + Name: kubevirt.CloudControllerManagerName + "-server", + CommonName: kubevirt.CloudControllerManagerName, + DNSNames: controlplane.DNSNamesForService(kubevirt.CloudControllerManagerName, clusterName), + CertType: secrets.ServerCert, + SigningCA: cas[v1beta1constants.SecretNameCACluster], + }, + }, + } + }, + } + + configChart = &chart.Chart{ + Name: "cloud-provider-config", + Path: filepath.Join(kubevirt.InternalChartsPath, "cloud-provider-config"), + Objects: []*chart.Object{ + { + Type: &corev1.Secret{}, + Name: kubevirt.CloudProviderConfigName, + }, + }, + } + + controlPlaneChart = &chart.Chart{ + Name: "seed-controlplane", + Path: filepath.Join(kubevirt.InternalChartsPath, "seed-controlplane"), + SubCharts: []*chart.Chart{ + { + Name: kubevirt.CloudControllerManagerName, + Images: []string{kubevirt.CloudControllerManagerImageName}, + Objects: []*chart.Object{ + {Type: &corev1.Service{}, Name: kubevirt.CloudControllerManagerName}, + {Type: &appsv1.Deployment{}, Name: kubevirt.CloudControllerManagerName}, + {Type: &corev1.ConfigMap{}, Name: kubevirt.CloudControllerManagerName + "-monitoring-config"}, + {Type: &autoscalingv1beta2.VerticalPodAutoscaler{}, Name: kubevirt.CloudControllerManagerName + "-vpa"}, + }, + }, + }, + } + + controlPlaneShootChart = &chart.Chart{ + Name: "shoot-system-components", + Path: filepath.Join(kubevirt.InternalChartsPath, "shoot-system-components"), + SubCharts: []*chart.Chart{ + { + Name: kubevirt.CloudControllerManagerName, + Objects: []*chart.Object{ + {Type: &rbacv1.ClusterRole{}, Name: "system:cloud-controller-manager"}, + {Type: &rbacv1.ClusterRoleBinding{}, Name: "system:cloud-controller-manager"}, + {Type: &rbacv1.ClusterRole{}, Name: "system:controller:cloud-node-controller"}, + {Type: &rbacv1.ClusterRoleBinding{}, Name: "system:controller:cloud-node-controller"}, + }, + }, + }, + } + + storageClassChart = &chart.Chart{ + Name: "shoot-storageclasses", + Path: filepath.Join(kubevirt.InternalChartsPath, "shoot-storageclasses"), + } +) + +// NewValuesProvider creates a new ValuesProvider for the generic actuator. +func NewValuesProvider(logger logr.Logger) genericactuator.ValuesProvider { + return &valuesProvider{ + logger: logger.WithName("kubevirt-values-provider"), + } +} + +// valuesProvider is a ValuesProvider that provides kubevirt-specific values for the 2 charts applied by the generic actuator. +type valuesProvider struct { + genericactuator.NoopValuesProvider + logger logr.Logger +} + +// GetConfigChartValues returns the values for the config chart applied by the generic actuator. +func (vp *valuesProvider) GetConfigChartValues( + ctx context.Context, + cp *extensionsv1alpha1.ControlPlane, + _ *extensionscontroller.Cluster, +) (map[string]interface{}, error) { + // Get kubeconfig + kubeconfig, err := kubevirt.GetKubeConfig(ctx, vp.Client(), cp.Spec.SecretRef) + if err != nil { + return nil, errors.Wrapf(err, "could not get kubeconfig from secret '%s/%s'", cp.Spec.SecretRef.Namespace, cp.Spec.SecretRef.Name) + } + + // Get config chart values + return getConfigChartValues(kubeconfig) +} + +// GetControlPlaneChartValues returns the values for the control plane chart applied by the generic actuator. +func (vp *valuesProvider) GetControlPlaneChartValues( + _ context.Context, + cp *extensionsv1alpha1.ControlPlane, + cluster *extensionscontroller.Cluster, + checksums map[string]string, + scaledDown bool, +) (map[string]interface{}, error) { + // Decode providerConfig + cpConfig := &apiskubevirt.ControlPlaneConfig{} + if cp.Spec.ProviderConfig != nil { + if _, _, err := vp.Decoder().Decode(cp.Spec.ProviderConfig.Raw, nil, cpConfig); err != nil { + return nil, errors.Wrapf(err, "could not decode providerConfig of controlplane '%s'", util.ObjectName(cp)) + } + } + + return getControlPlaneChartValues(cpConfig, cp, cluster, checksums, scaledDown) +} + +// GetControlPlaneShootChartValues returns the values for the control plane shoot chart applied by the generic actuator. +func (vp *valuesProvider) GetControlPlaneShootChartValues( + _ context.Context, + _ *extensionsv1alpha1.ControlPlane, + _ *extensionscontroller.Cluster, + _ map[string]string, +) (map[string]interface{}, error) { + return getControlPlaneShootChartValues(), nil +} + +// GetStorageClassesChartValues returns the values for the storage classes chart applied by the generic actuator. +func (vp *valuesProvider) GetStorageClassesChartValues( + _ context.Context, + _ *extensionsv1alpha1.ControlPlane, + _ *extensionscontroller.Cluster, +) (map[string]interface{}, error) { + return map[string]interface{}{}, nil +} + +// getConfigChartValues collects and returns the configuration chart values. +func getConfigChartValues(kubeconfig []byte) (map[string]interface{}, error) { + // Collect config chart values. + values := map[string]interface{}{ + "kubeconfig": string(kubeconfig), + } + + return values, nil +} + +// getControlPlaneChartValues collects and returns the control plane chart values. +func getControlPlaneChartValues( + cpConfig *apiskubevirt.ControlPlaneConfig, + cp *extensionsv1alpha1.ControlPlane, + cluster *extensionscontroller.Cluster, + checksums map[string]string, + scaledDown bool, +) (map[string]interface{}, error) { + ccm, err := getCCMChartValues(cpConfig, cp, cluster, checksums, scaledDown) + if err != nil { + return nil, err + } + + return map[string]interface{}{ + kubevirt.CloudControllerManagerName: ccm, + }, nil +} + +// getCCMChartValues collects and returns the CCM chart values. +func getCCMChartValues( + cpConfig *apiskubevirt.ControlPlaneConfig, + cp *extensionsv1alpha1.ControlPlane, + cluster *extensionscontroller.Cluster, + checksums map[string]string, + scaledDown bool, +) (map[string]interface{}, error) { + values := map[string]interface{}{ + "enabled": true, + "replicas": extensionscontroller.GetControlPlaneReplicas(cluster, scaledDown, 1), + "clusterName": cp.Namespace, + "kubernetesVersion": cluster.Shoot.Spec.Kubernetes.Version, + "podNetwork": extensionscontroller.GetPodNetwork(cluster), + "podAnnotations": map[string]interface{}{ + "checksum/secret-" + kubevirt.CloudControllerManagerName: checksums[kubevirt.CloudControllerManagerName], + "checksum/secret-" + kubevirt.CloudControllerManagerName + "-server": checksums[kubevirt.CloudControllerManagerName+"-server"], + "checksum/secret-" + kubevirt.CloudProviderConfigName: checksums[kubevirt.CloudProviderConfigName], + }, + "podLabels": map[string]interface{}{ + v1beta1constants.LabelPodMaintenanceRestart: "true", + }, + } + + if cpConfig.CloudControllerManager != nil { + values["featureGates"] = cpConfig.CloudControllerManager.FeatureGates + } + + return values, nil +} + +// getControlPlaneShootChartValues collects and returns the control plane shoot chart values. +func getControlPlaneShootChartValues() map[string]interface{} { + return map[string]interface{}{ + kubevirt.CloudControllerManagerName: map[string]interface{}{"enabled": true}, + } +} diff --git a/pkg/controller/controlplane/valuesprovider_test.go b/pkg/controller/controlplane/valuesprovider_test.go new file mode 100644 index 00000000..bf2b89f8 --- /dev/null +++ b/pkg/controller/controlplane/valuesprovider_test.go @@ -0,0 +1,214 @@ +// Copyright (c) 2020 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplane + +import ( + "context" + "encoding/json" + + apiskubevirt "github.com/gardener/gardener-extension-provider-kubevirt/pkg/apis/kubevirt" + "github.com/gardener/gardener-extension-provider-kubevirt/pkg/kubevirt" + + extensionscontroller "github.com/gardener/gardener/extensions/pkg/controller" + "github.com/gardener/gardener/extensions/pkg/controller/controlplane/genericactuator" + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" + mockclient "github.com/gardener/gardener/pkg/mock/controller-runtime/client" + "github.com/gardener/gardener/pkg/utils" + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/runtime/inject" +) + +const ( + namespace = "test" +) + +var _ = Describe("ValuesProvider", func() { + var ( + ctrl *gomock.Controller + ctx = context.TODO() + logger = log.Log.WithName("test") + + c *mockclient.MockClient + vp genericactuator.ValuesProvider + + scheme = runtime.NewScheme() + _ = apiskubevirt.AddToScheme(scheme) + + infrastructureStatus = &apiskubevirt.InfrastructureStatus{} + + cp = &extensionsv1alpha1.ControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "control-plane", + Namespace: namespace, + }, + Spec: extensionsv1alpha1.ControlPlaneSpec{ + Region: "local", + SecretRef: corev1.SecretReference{ + Name: v1beta1constants.SecretNameCloudProvider, + Namespace: namespace, + }, + DefaultSpec: extensionsv1alpha1.DefaultSpec{ + ProviderConfig: &runtime.RawExtension{ + Raw: encode(&apiskubevirt.ControlPlaneConfig{ + CloudControllerManager: &apiskubevirt.CloudControllerManagerConfig{ + FeatureGates: map[string]bool{ + "CustomResourceValidation": true, + }, + }, + }), + }, + }, + InfrastructureProviderStatus: &runtime.RawExtension{ + Raw: encode(infrastructureStatus), + }, + }, + } + + cidr = "10.250.0.0/19" + cluster = &extensionscontroller.Cluster{ + Shoot: &gardencorev1beta1.Shoot{ + Spec: gardencorev1beta1.ShootSpec{ + Networking: gardencorev1beta1.Networking{ + Pods: &cidr, + }, + Kubernetes: gardencorev1beta1.Kubernetes{ + Version: "1.13.4", + }, + }, + }, + } + + cpSecretKey = client.ObjectKey{Namespace: namespace, Name: v1beta1constants.SecretNameCloudProvider} + cpSecret = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: v1beta1constants.SecretNameCloudProvider, + Namespace: namespace, + }, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{ + "kubeconfig": []byte(`kubeconfig`), + }, + } + + checksums = map[string]string{ + kubevirt.CloudControllerManagerName: "3d791b164a808638da9a8df03924be2a41e34cd664e42231c00fe369e3588272", + kubevirt.CloudControllerManagerName + "-server": "6dff2a2e6f14444b66d8e4a351c049f7e89ee24ba3eaab95dbec40ba6bdebb52", + kubevirt.CloudProviderConfigName: "8bafb35ff1ac60275d62e1cbd495aceb511fb354f74a20f7d06ecb48b3a68432", + } + + enabledTrue = map[string]interface{}{"enabled": true} + ) + + BeforeEach(func() { + ctrl = gomock.NewController(GinkgoT()) + + c = mockclient.NewMockClient(ctrl) + vp = NewValuesProvider(logger) + + err := vp.(inject.Scheme).InjectScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + err = vp.(inject.Client).InjectClient(c) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + ctrl.Finish() + }) + + Describe("#GetConfigChartValues", func() { + It("should return correct config chart values", func() { + c.EXPECT().Get(ctx, cpSecretKey, &corev1.Secret{}).DoAndReturn(clientGet(cpSecret)) + + values, err := vp.GetConfigChartValues(ctx, cp, cluster) + Expect(err).NotTo(HaveOccurred()) + Expect(values).To(Equal(map[string]interface{}{ + "kubeconfig": "kubeconfig", + })) + }) + }) + + Describe("#GetControlPlaneChartValues", func() { + ccmChartValues := utils.MergeMaps(enabledTrue, map[string]interface{}{ + "replicas": 1, + "clusterName": namespace, + "kubernetesVersion": "1.17.5", + "podNetwork": cidr, + "podAnnotations": map[string]interface{}{ + "checksum/secret-cloud-controller-manager": "3d791b164a808638da9a8df03924be2a41e34cd664e42231c00fe369e3588272", + "checksum/secret-cloud-controller-manager-server": "6dff2a2e6f14444b66d8e4a351c049f7e89ee24ba3eaab95dbec40ba6bdebb52", + "checksum/secret-cloud-provider-config": "8bafb35ff1ac60275d62e1cbd495aceb511fb354f74a20f7d06ecb48b3a68432", + }, + "podLabels": map[string]interface{}{ + "maintenance.gardener.cloud/restart": "true", + }, + "featureGates": map[string]bool{ + "CustomResourceValidation": true, + }, + }) + + It("should return correct control plane chart values", func() { + values, err := vp.GetControlPlaneChartValues(ctx, cp, cluster, checksums, false) + + Expect(err).NotTo(HaveOccurred()) + Expect(values).To(Equal(map[string]interface{}{ + kubevirt.CloudControllerManagerName: utils.MergeMaps(ccmChartValues, map[string]interface{}{ + "kubernetesVersion": cluster.Shoot.Spec.Kubernetes.Version, + }), + })) + }) + }) + + Describe("#GetControlPlaneShootChartValues", func() { + It("should return correct control plane shoot chart values", func() { + values, err := vp.GetControlPlaneShootChartValues(ctx, cp, cluster, checksums) + Expect(err).NotTo(HaveOccurred()) + Expect(values).To(Equal(map[string]interface{}{ + kubevirt.CloudControllerManagerName: enabledTrue, + })) + }) + }) + + Describe("#GetStorageClassesChartValues()", func() { + It("should return empty storage class chart values", func() { + values, err := vp.GetStorageClassesChartValues(ctx, cp, cluster) + Expect(err).NotTo(HaveOccurred()) + Expect(values).To(Equal(map[string]interface{}{})) + }) + }) +}) + +func encode(obj runtime.Object) []byte { + data, _ := json.Marshal(obj) + return data +} + +func clientGet(result runtime.Object) interface{} { + return func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error { + switch obj.(type) { + case *corev1.Secret: + *obj.(*corev1.Secret) = *result.(*corev1.Secret) + } + return nil + } +} diff --git a/pkg/controller/infrastructure/actuator.go b/pkg/controller/infrastructure/actuator.go index f70b9f1a..2814babe 100644 --- a/pkg/controller/infrastructure/actuator.go +++ b/pkg/controller/infrastructure/actuator.go @@ -24,15 +24,13 @@ import ( type actuator struct { common.ChartRendererContext - logger logr.Logger - gardenID string + logger logr.Logger } // NewActuator creates a new Actuator that updates the status of the handled Infrastructure resources. -func NewActuator(gardenID string) infrastructure.Actuator { +func NewActuator() infrastructure.Actuator { return &actuator{ - logger: log.Log.WithName("infrastructure-actuator"), - gardenID: gardenID, + logger: log.Log.WithName("infrastructure-actuator"), } } diff --git a/pkg/controller/infrastructure/add.go b/pkg/controller/infrastructure/add.go index a2f21712..bdc5196a 100644 --- a/pkg/controller/infrastructure/add.go +++ b/pkg/controller/infrastructure/add.go @@ -33,15 +33,13 @@ type AddOptions struct { Controller controller.Options // IgnoreOperationAnnotation specifies whether to ignore the operation annotation or not. IgnoreOperationAnnotation bool - // GardenId is the Gardener garden identity - GardenId string } // AddToManagerWithOptions adds a controller with the given Options to the given manager. // The opts.Reconciler is being set with a newly instantiated actuator. func AddToManagerWithOptions(mgr manager.Manager, opts AddOptions) error { return infrastructure.Add(mgr, infrastructure.AddArgs{ - Actuator: NewActuator(opts.GardenId), + Actuator: NewActuator(), ControllerOptions: opts.Controller, Predicates: infrastructure.DefaultPredicates(opts.IgnoreOperationAnnotation), Type: kubevirt.Type, diff --git a/pkg/kubevirt/types.go b/pkg/kubevirt/types.go index 97bcb76e..9fdaf93a 100644 --- a/pkg/kubevirt/types.go +++ b/pkg/kubevirt/types.go @@ -21,6 +21,14 @@ import ( const ( // Name is the name of the KubeVirt provider controller. Name = "provider-kubevirt" + + // CloudControllerManagerImageName is the name of the cloud-controller-manager image. + CloudControllerManagerImageName = "cloud-controller-manager" + + // CloudProviderConfigName is the name of the secret containing the cloud provider config. + CloudProviderConfigName = "cloud-provider-config" + // CloudControllerManagerName is a constant for the name of the cloud-controller-manager. + CloudControllerManagerName = "cloud-controller-manager" // MachineControllerManagerName is a constant for the name of the machine-controller-manager. MachineControllerManagerName = "machine-controller-manager" // MachineControllerManagerImageName is the name of the MachineControllerManager image. diff --git a/pkg/kubevirt/util.go b/pkg/kubevirt/util.go new file mode 100644 index 00000000..4f690a43 --- /dev/null +++ b/pkg/kubevirt/util.go @@ -0,0 +1,33 @@ +// Copyright (c) 2020 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubevirt + +import ( + "context" + + extensionscontroller "github.com/gardener/gardener/extensions/pkg/controller" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// GetKubeConfig retrieves the kubeconfig specified by the secret reference. +func GetKubeConfig(ctx context.Context, c client.Client, secretRef corev1.SecretReference) ([]byte, error) { + secret, err := extensionscontroller.GetSecretByReference(ctx, c, &secretRef) + if err != nil { + return []byte(""), err + } + + return secret.Data["kubeconfig"], nil +} diff --git a/pkg/webhook/controlplane/ensurer.go b/pkg/webhook/controlplane/ensurer.go index bb27e613..baa3d22a 100644 --- a/pkg/webhook/controlplane/ensurer.go +++ b/pkg/webhook/controlplane/ensurer.go @@ -15,8 +15,17 @@ package controlplane import ( + "context" + "net" + + "github.com/coreos/go-systemd/unit" + extensionswebhook "github.com/gardener/gardener/extensions/pkg/webhook" "github.com/gardener/gardener/extensions/pkg/webhook/controlplane/genericmutator" "github.com/go-logr/logr" + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -39,4 +48,96 @@ func (e *ensurer) InjectClient(client client.Client) error { return nil } -// TODO: modify control-plane arguments here, like api-server args etc. +// EnsureKubeAPIServerDeployment ensures that the kube-apiserver deployment conforms to the provider requirements. +func (e *ensurer) EnsureKubeAPIServerDeployment(ctx context.Context, ectx genericmutator.EnsurerContext, new, _ *appsv1.Deployment) error { + template := &new.Spec.Template + ps := &template.Spec + if c := extensionswebhook.ContainerWithName(ps.Containers, "kube-apiserver"); c != nil { + ensureKubeAPIServerCommandLineArgs(c) + } + + cluster, err := ectx.GetCluster(ctx) + if err != nil { + return err + } + + if cluster.Shoot != nil && cluster.Seed != nil && cluster.Shoot.Spec.Networking.Nodes != nil { + shootNodesIP, _, err := net.ParseCIDR(*cluster.Shoot.Spec.Networking.Nodes) + if err != nil { + return errors.Wrapf(err, "could not parse shoot nodes CIDR %s", *cluster.Shoot.Spec.Networking.Nodes) + } + _, seedPodsIPNet, err := net.ParseCIDR(cluster.Seed.Spec.Networks.Pods) + if err != nil { + return errors.Wrapf(err, "could not parse seed pods CIDR %s", cluster.Seed.Spec.Networks.Pods) + } + + // If the seed pods CIDR contains the shoot nodes CIDR (the seed cluster hosts the shoot cluster), + // delete the vpn-seed pod since VPN is not needed in this case, and if established it causes connections + // from pods in the seed cluster to the kube-apiserver pod to fail. + if seedPodsIPNet.Contains(shootNodesIP) { + ps.Containers = extensionswebhook.EnsureNoContainerWithName(ps.Containers, "vpn-seed") + } + } + + return nil +} + +// EnsureKubeControllerManagerDeployment ensures that the kube-controller-manager deployment conforms to the provider requirements. +func (e *ensurer) EnsureKubeControllerManagerDeployment(ctx context.Context, _ genericmutator.EnsurerContext, new, _ *appsv1.Deployment) error { + template := &new.Spec.Template + ps := &template.Spec + if c := extensionswebhook.ContainerWithName(ps.Containers, "kube-controller-manager"); c != nil { + ensureKubeControllerManagerCommandLineArgs(c) + } + return nil +} + +func ensureKubeAPIServerCommandLineArgs(c *corev1.Container) { + c.Command = extensionswebhook.EnsureNoStringWithPrefix(c.Command, "--cloud-provider=") + + // Ensure CSI-related admission plugins + c.Command = extensionswebhook.EnsureNoStringWithPrefixContains(c.Command, "--enable-admission-plugins=", + "PersistentVolumeLabel", ",") + c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--disable-admission-plugins=", + "PersistentVolumeLabel", ",") + + // Ensure CSI-related feature gates + c.Command = extensionswebhook.EnsureNoStringWithPrefixContains(c.Command, "--feature-gates=", + "CSINodeInfo=false", ",") + c.Command = extensionswebhook.EnsureNoStringWithPrefixContains(c.Command, "--feature-gates=", + "CSIDriverRegistry=false", ",") +} + +func ensureKubeControllerManagerCommandLineArgs(c *corev1.Container) { + c.Command = extensionswebhook.EnsureStringWithPrefix(c.Command, "--cloud-provider=", "external") +} + +// EnsureKubeletServiceUnitOptions ensures that the kubelet.service unit options conform to the provider requirements. +func (e *ensurer) EnsureKubeletServiceUnitOptions(ctx context.Context, _ genericmutator.EnsurerContext, new, _ []*unit.UnitOption) ([]*unit.UnitOption, error) { + if opt := extensionswebhook.UnitOptionWithSectionAndName(new, "Service", "ExecStart"); opt != nil { + command := extensionswebhook.DeserializeCommandLine(opt.Value) + command = ensureKubeletCommandLineArgs(command) + opt.Value = extensionswebhook.SerializeCommandLine(command, 1, " \\\n ") + } + + new = extensionswebhook.EnsureUnitOption(new, &unit.UnitOption{ + Section: "Service", + Name: "ExecStartPre", + Value: `/bin/sh -c 'hostnamectl set-hostname $(cat /etc/hostname | cut -d '.' -f 1)'`, + }) + return new, nil +} + +func ensureKubeletCommandLineArgs(command []string) []string { + command = extensionswebhook.EnsureStringWithPrefix(command, "--cloud-provider=", "external") + return command +} + +// EnsureKubeletConfiguration ensures that the kubelet configuration conforms to the provider requirements. +func (e *ensurer) EnsureKubeletConfiguration(ctx context.Context, _ genericmutator.EnsurerContext, new, _ *kubeletconfigv1beta1.KubeletConfiguration) error { + // Ensure CSI-related feature gates + delete(new.FeatureGates, "VolumeSnapshotDataSource") + delete(new.FeatureGates, "CSINodeInfo") + delete(new.FeatureGates, "CSIDriverRegistry") + return nil +} diff --git a/pkg/webhook/controlplane/ensurer_test.go b/pkg/webhook/controlplane/ensurer_test.go new file mode 100644 index 00000000..4efd5f14 --- /dev/null +++ b/pkg/webhook/controlplane/ensurer_test.go @@ -0,0 +1,354 @@ +// Copyright (c) 2020 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplane + +import ( + "context" + "testing" + + "github.com/coreos/go-systemd/unit" + extensionscontroller "github.com/gardener/gardener/extensions/pkg/controller" + extensionswebhook "github.com/gardener/gardener/extensions/pkg/webhook" + "github.com/gardener/gardener/extensions/pkg/webhook/controlplane/genericmutator" + "github.com/gardener/gardener/extensions/pkg/webhook/controlplane/test" + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + mockclient "github.com/gardener/gardener/pkg/mock/controller-runtime/client" + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/runtime/inject" +) + +const ( + namespace = "test" +) + +func TestController(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Controlplane Webhook Suite") +} + +var _ = Describe("Ensurer", func() { + var ( + ctrl *gomock.Controller + + emptyContext = genericmutator.NewInternalEnsurerContext(&extensionscontroller.Cluster{}) + seedHostsShootContext = genericmutator.NewInternalEnsurerContext( + &extensionscontroller.Cluster{ + Shoot: &gardencorev1beta1.Shoot{ + Spec: gardencorev1beta1.ShootSpec{ + Networking: gardencorev1beta1.Networking{ + Nodes: pointer.StringPtr("10.225.128.0/17"), + }, + }, + }, + Seed: &gardencorev1beta1.Seed{ + Spec: gardencorev1beta1.SeedSpec{ + Networks: gardencorev1beta1.SeedNetworks{ + Pods: "10.225.0.0/16", + }, + }, + }, + }, + ) + ) + + BeforeEach(func() { + ctrl = gomock.NewController(GinkgoT()) + }) + + AfterEach(func() { + ctrl.Finish() + }) + + Describe("#EnsureKubeAPIServerDeployment", func() { + It("should add missing elements to kube-apiserver deployment", func() { + var ( + dep = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: v1beta1constants.DeploymentNameKubeAPIServer}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kube-apiserver", + }, + { + Name: "vpn-seed", + }, + }, + }, + }, + }, + } + ) + + // Create mock client + client := mockclient.NewMockClient(ctrl) + + // Create ensurer + ensurer := NewEnsurer(logger) + err := ensurer.(inject.Client).InjectClient(client) + Expect(err).To(Not(HaveOccurred())) + + // Call EnsureKubeAPIServerDeployment method and check the result + err = ensurer.EnsureKubeAPIServerDeployment(context.TODO(), emptyContext, dep, nil) + Expect(err).To(Not(HaveOccurred())) + checkKubeAPIServerDeployment(dep, true) + }) + + It("should modify existing elements of kube-apiserver deployment", func() { + var ( + dep = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: v1beta1constants.DeploymentNameKubeAPIServer}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kube-apiserver", + Command: []string{ + "--cloud-provider=?", + "--enable-admission-plugins=Priority,NamespaceLifecycle,PersistentVolumeLabel", + }, + }, + { + Name: "vpn-seed", + }, + }, + }, + }, + }, + } + ) + + // Create mock client + client := mockclient.NewMockClient(ctrl) + + // Create ensurer + ensurer := NewEnsurer(logger) + err := ensurer.(inject.Client).InjectClient(client) + Expect(err).To(Not(HaveOccurred())) + + // Call EnsureKubeAPIServerDeployment method and check the result + err = ensurer.EnsureKubeAPIServerDeployment(context.TODO(), emptyContext, dep, nil) + Expect(err).To(Not(HaveOccurred())) + checkKubeAPIServerDeployment(dep, true) + }) + + It("should delete the vpn-seed container if the seed hosts the shoot", func() { + var ( + dep = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: v1beta1constants.DeploymentNameKubeAPIServer}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kube-apiserver", + }, + { + Name: "vpn-seed", + }, + }, + }, + }, + }, + } + ) + + // Create mock client + client := mockclient.NewMockClient(ctrl) + + // Create ensurer + ensurer := NewEnsurer(logger) + err := ensurer.(inject.Client).InjectClient(client) + Expect(err).To(Not(HaveOccurred())) + + // Call EnsureKubeAPIServerDeployment method and check the result + err = ensurer.EnsureKubeAPIServerDeployment(context.TODO(), seedHostsShootContext, dep, nil) + Expect(err).To(Not(HaveOccurred())) + checkKubeAPIServerDeployment(dep, false) + }) + }) + + Describe("#EnsureKubeControllerManagerDeployment", func() { + It("should add missing elements to kube-controller-manager deployment", func() { + var ( + dep = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: v1beta1constants.DeploymentNameKubeControllerManager}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kube-controller-manager", + }, + }, + }, + }, + }, + } + ) + + // Create mock client + client := mockclient.NewMockClient(ctrl) + + // Create ensurer + ensurer := NewEnsurer(logger) + err := ensurer.(inject.Client).InjectClient(client) + Expect(err).To(Not(HaveOccurred())) + + // Call EnsureKubeControllerManagerDeployment method and check the result + err = ensurer.EnsureKubeControllerManagerDeployment(context.TODO(), emptyContext, dep, nil) + Expect(err).To(Not(HaveOccurred())) + checkKubeControllerManagerDeployment(dep) + }) + + It("should modify existing elements of kube-controller-manager deployment", func() { + var ( + dep = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: v1beta1constants.DeploymentNameKubeControllerManager}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + v1beta1constants.LabelNetworkPolicyToBlockedCIDRs: v1beta1constants.LabelNetworkPolicyAllowed, + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kube-controller-manager", + Command: []string{ + "--cloud-provider=?", + }, + }, + }, + }, + }, + }, + } + ) + + // Create mock client + client := mockclient.NewMockClient(ctrl) + + // Create ensurer + ensurer := NewEnsurer(logger) + err := ensurer.(inject.Client).InjectClient(client) + Expect(err).To(Not(HaveOccurred())) + + // Call EnsureKubeControllerManagerDeployment method and check the result + err = ensurer.EnsureKubeControllerManagerDeployment(context.TODO(), emptyContext, dep, nil) + Expect(err).To(Not(HaveOccurred())) + checkKubeControllerManagerDeployment(dep) + }) + }) + + Describe("#EnsureKubeletServiceUnitOptions", func() { + It("should modify existing elements of kubelet.service unit options", func() { + var ( + oldUnitOptions = []*unit.UnitOption{ + { + Section: "Service", + Name: "ExecStart", + Value: `/opt/bin/hyperkube kubelet \ + --config=/var/lib/kubelet/config/kubelet`, + }, + } + newUnitOptions = []*unit.UnitOption{ + { + Section: "Service", + Name: "ExecStart", + Value: `/opt/bin/hyperkube kubelet \ + --config=/var/lib/kubelet/config/kubelet \ + --cloud-provider=external`, + }, + { + Section: "Service", + Name: "ExecStartPre", + Value: `/bin/sh -c 'hostnamectl set-hostname $(cat /etc/hostname | cut -d '.' -f 1)'`, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(logger) + + // Call EnsureKubeletServiceUnitOptions method and check the result + opts, err := ensurer.EnsureKubeletServiceUnitOptions(context.TODO(), emptyContext, oldUnitOptions, nil) + Expect(err).To(Not(HaveOccurred())) + Expect(opts).To(Equal(newUnitOptions)) + }) + }) + + Describe("#EnsureKubeletConfiguration", func() { + It("should modify existing elements of kubelet configuration", func() { + var ( + oldKubeletConfig = &kubeletconfigv1beta1.KubeletConfiguration{ + FeatureGates: map[string]bool{ + "Foo": true, + "VolumeSnapshotDataSource": true, + "CSINodeInfo": true, + }, + } + newKubeletConfig = &kubeletconfigv1beta1.KubeletConfiguration{ + FeatureGates: map[string]bool{ + "Foo": true, + }, + } + ) + + // Create ensurer + ensurer := NewEnsurer(logger) + + // Call EnsureKubeletConfiguration method and check the result + kubeletConfig := *oldKubeletConfig + err := ensurer.EnsureKubeletConfiguration(context.TODO(), emptyContext, &kubeletConfig, nil) + Expect(err).To(Not(HaveOccurred())) + Expect(&kubeletConfig).To(Equal(newKubeletConfig)) + }) + }) +}) + +func checkKubeAPIServerDeployment(dep *appsv1.Deployment, vpnSeedContainerPresent bool) { + // Check that the kube-apiserver container still exists and contains all needed command line args, + // env vars, and volume mounts + c := extensionswebhook.ContainerWithName(dep.Spec.Template.Spec.Containers, "kube-apiserver") + Expect(c).To(Not(BeNil())) + Expect(c.Command).To(Not(test.ContainElementWithPrefixContaining("--enable-admission-plugins=", "PersistentVolumeLabel", ","))) + Expect(c.Command).To(test.ContainElementWithPrefixContaining("--disable-admission-plugins=", "PersistentVolumeLabel", ",")) + + vpnSeedContainer := extensionswebhook.ContainerWithName(dep.Spec.Template.Spec.Containers, "vpn-seed") + if vpnSeedContainerPresent { + Expect(vpnSeedContainer).To(Not(BeNil())) + } else { + Expect(vpnSeedContainer).To(BeNil()) + } +} + +func checkKubeControllerManagerDeployment(dep *appsv1.Deployment) { + // Check that the kube-controller-manager container still exists and contains all needed command line args, + // env vars, and volume mounts + c := extensionswebhook.ContainerWithName(dep.Spec.Template.Spec.Containers, "kube-controller-manager") + Expect(c).To(Not(BeNil())) +} diff --git a/vendor/github.com/gardener/gardener/extensions/pkg/webhook/controlplane/test/matchers.go b/vendor/github.com/gardener/gardener/extensions/pkg/webhook/controlplane/test/matchers.go new file mode 100644 index 00000000..6440f6df --- /dev/null +++ b/vendor/github.com/gardener/gardener/extensions/pkg/webhook/controlplane/test/matchers.go @@ -0,0 +1,61 @@ +// Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "fmt" + "strings" + + extensionswebhook "github.com/gardener/gardener/extensions/pkg/webhook" + + "github.com/onsi/gomega/types" +) + +// ContainElementWithPrefixContaining succeeds if actual contains a string having the given prefix +// and containing the given value in a list separated by sep. +// Actual must be a slice of strings. +func ContainElementWithPrefixContaining(prefix, value, sep string) types.GomegaMatcher { + return &containElementWithPrefixContainingMatcher{ + prefix: prefix, + value: value, + sep: sep, + } +} + +type containElementWithPrefixContainingMatcher struct { + prefix, value, sep string +} + +func (m *containElementWithPrefixContainingMatcher) Match(actual interface{}) (success bool, err error) { + items, ok := actual.([]string) + if !ok { + return false, fmt.Errorf("ContainElementWithPrefixContaining matcher expects []string") + } + i := extensionswebhook.StringWithPrefixIndex(items, m.prefix) + if i < 0 { + return false, nil + } + values := strings.Split(strings.TrimPrefix(items[i], m.prefix), m.sep) + j := extensionswebhook.StringIndex(values, m.value) + return j >= 0, nil +} + +func (m *containElementWithPrefixContainingMatcher) FailureMessage(actual interface{}) (message string) { + return fmt.Sprintf("Expected\n\t%#v\nto contain an element with prefix '%s' containing '%s'", actual, m.prefix, m.value) +} + +func (m *containElementWithPrefixContainingMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return fmt.Sprintf("Expected\n\t%#v\nnot to contain an element with prefix '%s' containing '%s'", actual, m.prefix, m.value) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index de9267d9..3b85ccfe 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -16,6 +16,7 @@ github.com/ahmetb/gen-crd-api-reference-docs # github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 github.com/beorn7/perks/quantile # github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f +## explicit github.com/coreos/go-systemd/unit # github.com/cyphar/filepath-securejoin v0.2.2 github.com/cyphar/filepath-securejoin @@ -78,6 +79,7 @@ github.com/gardener/gardener/extensions/pkg/webhook github.com/gardener/gardener/extensions/pkg/webhook/cmd github.com/gardener/gardener/extensions/pkg/webhook/controlplane github.com/gardener/gardener/extensions/pkg/webhook/controlplane/genericmutator +github.com/gardener/gardener/extensions/pkg/webhook/controlplane/test github.com/gardener/gardener/extensions/pkg/webhook/shoot github.com/gardener/gardener/hack github.com/gardener/gardener/hack/.ci @@ -647,8 +649,10 @@ k8s.io/apimachinery/third_party/forked/golang/json k8s.io/apimachinery/third_party/forked/golang/netutil k8s.io/apimachinery/third_party/forked/golang/reflect # k8s.io/apiserver v0.17.6 => k8s.io/apiserver v0.16.8 +## explicit k8s.io/apiserver/pkg/authentication/user # k8s.io/autoscaler v0.0.0-20190805135949-100e91ba756e +## explicit k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1beta2 # k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible => k8s.io/client-go v0.16.8 ## explicit @@ -812,8 +816,10 @@ k8s.io/kube-openapi/pkg/generators/rules k8s.io/kube-openapi/pkg/util/proto k8s.io/kube-openapi/pkg/util/sets # k8s.io/kubelet v0.17.6 +## explicit k8s.io/kubelet/config/v1beta1 # k8s.io/utils v0.0.0-20200327001022-6496210b90e8 +## explicit k8s.io/utils/buffer k8s.io/utils/integer k8s.io/utils/pointer