From 01b2d96e8ecb7b6a9821ee365ca15b42867eda0a Mon Sep 17 00:00:00 2001 From: Thomas George Hartland Date: Thu, 6 Aug 2020 14:11:52 +0200 Subject: [PATCH 1/2] Add magnum support in cluster autoscaler helm chart --- charts/cluster-autoscaler-chart/Chart.yaml | 2 +- charts/cluster-autoscaler-chart/README.md | 38 ++++++++++++++++--- .../cluster-autoscaler-chart/README.md.gotmpl | 29 ++++++++++++-- .../templates/deployment.yaml | 26 ++++++++++--- charts/cluster-autoscaler-chart/values.yaml | 24 ++++++++++-- 5 files changed, 100 insertions(+), 19 deletions(-) diff --git a/charts/cluster-autoscaler-chart/Chart.yaml b/charts/cluster-autoscaler-chart/Chart.yaml index 8bae90c93e6c..a188ee579f25 100644 --- a/charts/cluster-autoscaler-chart/Chart.yaml +++ b/charts/cluster-autoscaler-chart/Chart.yaml @@ -17,4 +17,4 @@ name: cluster-autoscaler-chart sources: - https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler type: application -version: 1.0.5 +version: 1.1.0 diff --git a/charts/cluster-autoscaler-chart/README.md b/charts/cluster-autoscaler-chart/README.md index 0b656f9a33d1..62666484c506 100644 --- a/charts/cluster-autoscaler-chart/README.md +++ b/charts/cluster-autoscaler-chart/README.md @@ -45,10 +45,15 @@ You must provide some minimal configuration, either to specify instance groups o Either: -- Set `autoDiscovery.clusterName` and tag your autoscaling groups appropriately (`--cloud-provider=aws` only) **or** -- Set at least one ASG as an element in the `autoscalingGroups` array with its three values: `name`, `minSize` and `maxSize`. +- Set `autoDiscovery.clusterName` and provide additional autodiscovery options if necessary **or** +- Set static node group configurations for one or more node groups (using `autoscalingGroups` or `autoscalingGroupsnamePrefix`). -To install the chart with the release name `my-release`: +To create a valid configuration, follow instructions for your cloud provider: + +* [AWS](#aws---using-auto-discovery-of-tagged-instance-groups) +* [GCE](#gce) +* [Azure AKS](#azure-aks) +* [OpenStack Magnum](#openstack-magnum) ### AWS - Using auto-discovery of tagged instance groups @@ -163,6 +168,24 @@ The following parameters are required: - `azureVMType: "AKS"` - `azureNodeResourceGroup: "your-aks-cluster-node-resource-group"` +### OpenStack Magnum + +`cloudProvider: magnum` must be set, and then one of + +- `magnumClusterName=` and `autoscalingGroups` with the names of node groups and min/max node counts +- or `autoDiscovery.clusterName=` with one or more `autoDiscovery.roles`. + +Additionally, `cloudConfigPath: "/etc/kubernetes/cloud-config"` must be set as this should be the location +of the cloud-config file on the host. + +Example values files can be found [here](../../cluster-autoscaler/cloudprovider/magnum/examples). + +Install the chart with + +``` +$ helm install my-release autoscaler/cluster-autoscaler-chart -f myvalues.yaml +``` + ## Uninstalling the Chart To uninstall `my-release`: @@ -290,9 +313,10 @@ Though enough for the majority of installations, the default PodSecurityPolicy _ | Key | Type | Default | Description | |-----|------|---------|-------------| | affinity | object | `{}` | Affinity for pod assignment | -| autoDiscovery.clusterName | string | `nil` | Enable autodiscovery for name in ASG tag (only `cloudProvider=aws`). Must be set for `cloudProvider=gce`, but no MIG tagging required. | +| autoDiscovery.clusterName | string | `nil` | Enable autodiscovery for `cloudProvider=aws`, for groups matching `autoDiscovery.tags`. Enable autodiscovery for `cloudProvider=gce`, but no MIG tagging required. Enable autodiscovery for `cloudProvider=magnum`, for groups matching `autoDiscovery.roles`. | +| autoDiscovery.roles | list | `["worker"]` | Magnum node group roles to match. | | autoDiscovery.tags | list | `["k8s.io/cluster-autoscaler/enabled","k8s.io/cluster-autoscaler/{{ .Values.autoDiscovery.clusterName }}"]` | ASG tags to match, run through `tpl`. | -| autoscalingGroups | list | `[]` | For AWS. At least one element is required if not using `autoDiscovery`. For example: 
 - name: asg1
   maxSize: 2
   minSize: 1
| +| autoscalingGroups | list | `[]` | For AWS, Azure AKS or Magnum. At least one element is required if not using `autoDiscovery`. For example: 
 - name: asg1
   maxSize: 2
   minSize: 1
| | autoscalingGroupsnamePrefix | list | `[]` | For GCE. At least one element is required if not using `autoDiscovery`. For example: 
 - name: ig01
   maxSize: 10
   minSize: 0
| | awsAccessKeyID | string | `""` | AWS access key ID ([if AWS user keys used](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#using-aws-credentials)) | | awsRegion | string | `"us-east-1"` | AWS region (required if `cloudProvider=aws`) | @@ -307,7 +331,7 @@ Though enough for the majority of installations, the default PodSecurityPolicy _ | azureUseManagedIdentityExtension | bool | `false` | Whether to use Azure's managed identity extension for credentials. If using MSI, ensure subscription ID and resource group are set. | | azureVMType | string | `"AKS"` | Azure VM type. | | cloudConfigPath | string | `"/etc/gce.conf"` | Configuration file for cloud provider. | -| cloudProvider | string | `"aws"` | The cloud provider where the autoscaler runs. Currently only `gce`, `aws`, and `azure` are supported. `aws` supported for AWS. `gce` for GCE. `azure` for Azure AKS. | +| cloudProvider | string | `"aws"` | The cloud provider where the autoscaler runs. Currently only `gce`, `aws`, `azure` and `magnum` are supported. `aws` supported for AWS. `gce` for GCE. `azure` for Azure AKS. `magnum` for OpenStack Magnum. | | containerSecurityContext | object | `{}` | [Security context for container](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) | | dnsPolicy | string | `"ClusterFirst"` | Defaults to `ClusterFirst`. Valid values are: `ClusterFirstWithHostNet`, `ClusterFirst`, `Default` or `None`. If autoscaler does not depend on cluster DNS, recommended to set this to `Default`. | | expanderPriorities | object | `{}` | The expanderPriorities is used if `extraArgs.expander` is set to `priority` and expanderPriorities is also set with the priorities. If `extraArgs.expander` is set to `priority`, then expanderPriorities is used to define cluster-autoscaler-priority-expander priorities. See: https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/expander/priority/readme.md | @@ -319,6 +343,8 @@ Though enough for the majority of installations, the default PodSecurityPolicy _ | image.repository | string | `"us.gcr.io/k8s-artifacts-prod/autoscaling/cluster-autoscaler"` | Image repository | | image.tag | string | `"v1.18.1"` | Image tag | | kubeTargetVersionOverride | string | `""` | Allow overridding the `.Capabilities.KubeVersion.GitVersion` check. Useful for `helm template` commands. | +| magnumCABundlePath | string | `"/etc/kubernetes/ca-bundle.crt"` | Path to the host's CA bundle, from `ca-file` in the cloud-config file. | +| magnumClusterName | string | `""` | Cluster name or ID in Magnum. Required if `cloudProvider=magnum` and not setting `autoDiscovery.clusterName`. | | nameOverride | string | `""` | String to partially override `cluster-autoscaler.fullname` template (will maintain the release name) | | nodeSelector | object | `{}` | Node labels for pod assignment. Ref: https://kubernetes.io/docs/user-guide/node-selection/. | | podAnnotations | object | `{}` | Annotations to add to each pod. | diff --git a/charts/cluster-autoscaler-chart/README.md.gotmpl b/charts/cluster-autoscaler-chart/README.md.gotmpl index 89092cc54148..ea23e5083568 100644 --- a/charts/cluster-autoscaler-chart/README.md.gotmpl +++ b/charts/cluster-autoscaler-chart/README.md.gotmpl @@ -45,10 +45,15 @@ You must provide some minimal configuration, either to specify instance groups o Either: -- Set `autoDiscovery.clusterName` and tag your autoscaling groups appropriately (`--cloud-provider=aws` only) **or** -- Set at least one ASG as an element in the `autoscalingGroups` array with its three values: `name`, `minSize` and `maxSize`. +- Set `autoDiscovery.clusterName` and provide additional autodiscovery options if necessary **or** +- Set static node group configurations for one or more node groups (using `autoscalingGroups` or `autoscalingGroupsnamePrefix`). -To install the chart with the release name `my-release`: +To create a valid configuration, follow instructions for your cloud provider: + +* [AWS](#aws---using-auto-discovery-of-tagged-instance-groups) +* [GCE](#gce) +* [Azure AKS](#azure-aks) +* [OpenStack Magnum](#openstack-magnum) ### AWS - Using auto-discovery of tagged instance groups @@ -163,6 +168,24 @@ The following parameters are required: - `azureVMType: "AKS"` - `azureNodeResourceGroup: "your-aks-cluster-node-resource-group"` +### OpenStack Magnum + +`cloudProvider: magnum` must be set, and then one of + +- `magnumClusterName=` and `autoscalingGroups` with the names of node groups and min/max node counts +- or `autoDiscovery.clusterName=` with one or more `autoDiscovery.roles`. + +Additionally, `cloudConfigPath: "/etc/kubernetes/cloud-config"` must be set as this should be the location +of the cloud-config file on the host. + +Example values files can be found [here](../../cluster-autoscaler/cloudprovider/magnum/examples). + +Install the chart with + +``` +$ helm install my-release autoscaler/cluster-autoscaler-chart -f myvalues.yaml +``` + ## Uninstalling the Chart To uninstall `my-release`: diff --git a/charts/cluster-autoscaler-chart/templates/deployment.yaml b/charts/cluster-autoscaler-chart/templates/deployment.yaml index e5ae127019e5..0943da720289 100644 --- a/charts/cluster-autoscaler-chart/templates/deployment.yaml +++ b/charts/cluster-autoscaler-chart/templates/deployment.yaml @@ -59,14 +59,20 @@ spec: - --node-group-auto-discovery=mig:namePrefix={{ .name }},min={{ .minSize }},max={{ .maxSize }} {{- end }} {{- end }} + {{- else if eq .Values.cloudProvider "magnum" }} + {{- if .Values.autoDiscovery.clusterName }} + - --cluster-name={{ .Values.autoDiscovery.clusterName }} + - --node-group-auto-discovery=magnum:role={{ tpl (join "," .Values.autoDiscovery.roles) . }} + {{- else }} + - --cluster-name={{ .Values.magnumClusterName }} + {{- end }} {{- end }} - {{- if eq .Values.cloudProvider "gce" }} + {{- if or (eq .Values.cloudProvider "gce") (eq .Values.cloudProvider "magnum") }} - --cloud-config={{ .Values.cloudConfigPath }} - {{- end }} + {{- end }} {{- range $key, $value := .Values.extraArgs }} - --{{ $key }}={{ $value }} {{- end }} - env: {{- if and (eq .Values.cloudProvider "aws") (ne .Values.awsRegion "") }} - name: AWS_REGION @@ -167,11 +173,16 @@ spec: securityContext: {{ toYaml .Values.containerSecurityContext | nindent 12 | trim }} {{- end }} - {{- if eq .Values.cloudProvider "gce" }} + {{- if or (eq .Values.cloudProvider "gce") (eq .Values.cloudProvider "magnum") }} volumeMounts: - name: cloudconfig mountPath: {{ .Values.cloudConfigPath }} readOnly: true + {{- if and (eq .Values.cloudProvider "magnum") (.Values.magnumCABundlePath) }} + - name: ca-bundle + mountPath: {{ .Values.magnumCABundlePath }} + readOnly: true + {{- end }} {{- end }} {{- if .Values.affinity }} affinity: @@ -188,11 +199,16 @@ spec: securityContext: {{ toYaml .Values.securityContext | nindent 8 | trim }} {{- end }} - {{- if eq .Values.cloudProvider "gce" }} + {{- if or (eq .Values.cloudProvider "gce") (eq .Values.cloudProvider "magnum") }} volumes: - name: cloudconfig hostPath: path: {{ .Values.cloudConfigPath }} + {{- if and (eq .Values.cloudProvider "magnum") (.Values.magnumCABundlePath) }} + - name: ca-bundle + hostPath: + path: {{ .Values.magnumCABundlePath }} + {{- end }} {{- end }} {{- if .Values.image.pullSecrets }} imagePullSecrets: diff --git a/charts/cluster-autoscaler-chart/values.yaml b/charts/cluster-autoscaler-chart/values.yaml index d360458dc1e3..445ca1ff5a08 100644 --- a/charts/cluster-autoscaler-chart/values.yaml +++ b/charts/cluster-autoscaler-chart/values.yaml @@ -3,17 +3,25 @@ affinity: {} autoDiscovery: - # Only cloudProvider `aws` and `gce` are supported by auto-discovery at this time + # cloudProviders `aws`, `gce` and `magnum` are supported by auto-discovery at this time # AWS: Set tags as described in https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#auto-discovery-setup - # autoDiscovery.clusterName -- Enable autodiscovery for name in ASG tag (only `cloudProvider=aws`). Must be set for `cloudProvider=gce`, but no MIG tagging required. + + # autoDiscovery.clusterName -- Enable autodiscovery for `cloudProvider=aws`, for groups matching `autoDiscovery.tags`. + # Enable autodiscovery for `cloudProvider=gce`, but no MIG tagging required. + # Enable autodiscovery for `cloudProvider=magnum`, for groups matching `autoDiscovery.roles`. clusterName: # cluster.local + # autoDiscovery.tags -- ASG tags to match, run through `tpl`. tags: - k8s.io/cluster-autoscaler/enabled - k8s.io/cluster-autoscaler/{{ .Values.autoDiscovery.clusterName }} # - kubernetes.io/cluster/{{ .Values.autoDiscovery.clusterName }} -# autoscalingGroups -- For AWS. At least one element is required if not using `autoDiscovery`. For example: + # autoDiscovery.roles -- Magnum node group roles to match. + roles: + - worker + +# autoscalingGroups -- For AWS, Azure AKS or Magnum. At least one element is required if not using `autoDiscovery`. For example: # 
 # - name: asg1

 #   maxSize: 2

@@ -84,12 +92,20 @@ azureNodeResourceGroup: ""
 # azureUseManagedIdentityExtension -- Whether to use Azure's managed identity extension for credentials. If using MSI, ensure subscription ID and resource group are set.
 azureUseManagedIdentityExtension: false
 
+# magnumClusterName -- Cluster name or ID in Magnum.
+# Required if `cloudProvider=magnum` and not setting `autoDiscovery.clusterName`.
+magnumClusterName: ""
+
+# magnumCABundlePath -- Path to the host's CA bundle, from `ca-file` in the cloud-config file.
+magnumCABundlePath: "/etc/kubernetes/ca-bundle.crt"
+
 # cloudConfigPath -- Configuration file for cloud provider.
 cloudConfigPath: /etc/gce.conf
 
 # cloudProvider -- The cloud provider where the autoscaler runs.
-# Currently only `gce`, `aws`, and `azure` are supported.
+# Currently only `gce`, `aws`, `azure` and `magnum` are supported.
 # `aws` supported for AWS. `gce` for GCE. `azure` for Azure AKS.
+# `magnum` for OpenStack Magnum.
 cloudProvider: aws
 
 # containerSecurityContext -- [Security context for container](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/)

From 5f08511b99feb3768c111513bd6b2e0cffb6497b Mon Sep 17 00:00:00 2001
From: Thomas George Hartland 
Date: Thu, 6 Aug 2020 16:35:50 +0200
Subject: [PATCH 2/2] Update magnum cloud provider README for helm installation

---
 .../cloudprovider/magnum/README.md            | 24 +++++++++++++++
 .../magnum/examples/values-autodiscovery.yaml | 29 ++++++++++++++++++
 .../magnum/examples/values-example.yaml       | 30 +++++++++++++++++++
 .../magnum/examples/values-minimal.yaml       | 10 +++++++
 4 files changed, 93 insertions(+)
 create mode 100644 cluster-autoscaler/cloudprovider/magnum/examples/values-autodiscovery.yaml
 create mode 100644 cluster-autoscaler/cloudprovider/magnum/examples/values-example.yaml
 create mode 100644 cluster-autoscaler/cloudprovider/magnum/examples/values-minimal.yaml

diff --git a/cluster-autoscaler/cloudprovider/magnum/README.md b/cluster-autoscaler/cloudprovider/magnum/README.md
index e37f773067cd..3f804030b41e 100644
--- a/cluster-autoscaler/cloudprovider/magnum/README.md
+++ b/cluster-autoscaler/cloudprovider/magnum/README.md
@@ -47,6 +47,30 @@ to match your cluster.
 | --nodes                     | Used to select a specific node group to autoscale and constrain its node count. Of the form `min:max:NodeGroupName`. Can be used multiple times. |
 | --node-group-auto-discovery | See below.                                                                                                                                       |
 
+#### Deployment with helm
+
+Alternatively, the autoscaler can be deployed with the cluster autoscaler helm chart.
+A minimal values.yaml file looks like:
+
+```yaml
+cloudProvider: "magnum"
+
+magnumClusterName: "cluster name or ID"
+
+autoscalingGroups:
+- name: default-worker
+  maxSize: 5
+  minSize: 1
+
+cloudConfigPath: "/etc/kubernetes/cloud-config"
+```
+
+For running on the master node and other suggested settings, see
+[examples/values-example.yaml](examples/values-example.yaml).
+To deploy with node group autodiscovery (for cluster autoscaler v1.19+), see
+[examples/values-autodiscovery.yaml](examples/values-autodiscovery.yaml).
+
+
 ## Node group auto discovery
 
 Instead of using `--nodes` to select specific node groups by name,
diff --git a/cluster-autoscaler/cloudprovider/magnum/examples/values-autodiscovery.yaml b/cluster-autoscaler/cloudprovider/magnum/examples/values-autodiscovery.yaml
new file mode 100644
index 000000000000..be7a4ae237d5
--- /dev/null
+++ b/cluster-autoscaler/cloudprovider/magnum/examples/values-autodiscovery.yaml
@@ -0,0 +1,29 @@
+cloudProvider: "magnum"
+
+autoDiscovery:
+  clusterName: "cluster name or ID"
+  roles:
+  - worker
+  - autoscaling
+
+image:
+  repository: docker.io/openstackmagnum/cluster-autoscaler
+  tag: v1.19.0
+
+nodeSelector:
+  node-role.kubernetes.io/master: ""
+
+tolerations:
+- key: CriticalAddonsOnly
+  value: "True"
+  effect: NoSchedule
+- key: dedicated
+  value: "master"
+  effect: NoSchedule
+- key: node-role.kubernetes.io/master
+  effect: NoSchedule
+
+cloudConfigPath: /etc/kubernetes/cloud-config
+
+extraArgs:
+  v: 2
diff --git a/cluster-autoscaler/cloudprovider/magnum/examples/values-example.yaml b/cluster-autoscaler/cloudprovider/magnum/examples/values-example.yaml
new file mode 100644
index 000000000000..7a9697e8cbea
--- /dev/null
+++ b/cluster-autoscaler/cloudprovider/magnum/examples/values-example.yaml
@@ -0,0 +1,30 @@
+cloudProvider: "magnum"
+
+magnumClusterName: "cluster name or ID"
+
+autoscalingGroups:
+- name: default-worker
+  maxSize: 5
+  minSize: 1
+
+image:
+  repository: docker.io/openstackmagnum/cluster-autoscaler
+  tag: v1.19.0
+
+nodeSelector:
+  node-role.kubernetes.io/master: ""
+
+tolerations:
+- key: CriticalAddonsOnly
+  value: "True"
+  effect: NoSchedule
+- key: dedicated
+  value: "master"
+  effect: NoSchedule
+- key: node-role.kubernetes.io/master
+  effect: NoSchedule
+
+cloudConfigPath: "/etc/kubernetes/cloud-config"
+
+extraArgs:
+  v: 2
diff --git a/cluster-autoscaler/cloudprovider/magnum/examples/values-minimal.yaml b/cluster-autoscaler/cloudprovider/magnum/examples/values-minimal.yaml
new file mode 100644
index 000000000000..5f11b0553625
--- /dev/null
+++ b/cluster-autoscaler/cloudprovider/magnum/examples/values-minimal.yaml
@@ -0,0 +1,10 @@
+cloudProvider: "magnum"
+
+magnumClusterName: "cluster name or ID"
+
+autoscalingGroups:
+- name: default-worker
+  maxSize: 5
+  minSize: 1
+
+cloudConfigPath: "/etc/kubernetes/cloud-config"