diff --git a/stable/awsebscsiprovisioner/Chart.yaml b/stable/awsebscsiprovisioner/Chart.yaml index ff5471c93..77c15ef26 100644 --- a/stable/awsebscsiprovisioner/Chart.yaml +++ b/stable/awsebscsiprovisioner/Chart.yaml @@ -1,12 +1,12 @@ apiVersion: v1 -appVersion: "0.4.0" +appVersion: "0.5.0" description: AWS EBS CSI driver and storage provisioner name: awsebscsiprovisioner maintainers: - name: alejandroEsc - name: gpaul - name: hectorj2f -version: 0.3.3 +version: 0.3.4 kubeVersion: ">=1.15.0" home: https://github.com/kubernetes-sigs/aws-ebs-csi-driver sources: diff --git a/stable/awsebscsiprovisioner/ci/annotations-kube2iam-usage.yaml b/stable/awsebscsiprovisioner/ci/annotations-kube2iam-usage.yaml new file mode 100644 index 000000000..3bde51a51 --- /dev/null +++ b/stable/awsebscsiprovisioner/ci/annotations-kube2iam-usage.yaml @@ -0,0 +1,7 @@ +# Check that statefulSet.podAnnoations is rendered +statefulSetCSIController: + podAnnotations: + iam.amazonaws.com/role: ebs-csi-driver-role +statefulSetCSISnapshotController: + podAnnotations: + iam.amazonaws.com/role: ebs-csi-snapshot-controller-role diff --git a/stable/awsebscsiprovisioner/ci/more-specific-settings.yaml b/stable/awsebscsiprovisioner/ci/more-specific-settings.yaml new file mode 100644 index 000000000..4764b983e --- /dev/null +++ b/stable/awsebscsiprovisioner/ci/more-specific-settings.yaml @@ -0,0 +1,21 @@ +# Check that statefulSet.podAnnoations is rendered +replicas: 2 +extraVolumeTags: + konvoy: cluster-name-random + konvoy-version: 1.4.2 +storageclass: + isDefault: true + reclaimPolicy: Delete + volumeBindingMode: WaitForFirstConsumer + type: io1 + fstype: xfs + iopsPerGB: 100 + encrypted: true + kmsKeyId: arn:aws:kms:us-west-2:123456789011:key/d72124e7-ffff-1111-zzzz-4f820a16908e +allowedTopologies: +- matchLabelExpressions: + - key: topology.ebs.csi.aws.com/zone + values: + - us-west-2a + - us-west-2b + - us-west-2c diff --git a/stable/awsebscsiprovisioner/templates/_helpers.tpl b/stable/awsebscsiprovisioner/templates/_helpers.tpl index cf6fc86c4..7fa13305a 100644 --- a/stable/awsebscsiprovisioner/templates/_helpers.tpl +++ b/stable/awsebscsiprovisioner/templates/_helpers.tpl @@ -42,4 +42,17 @@ app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end -}} \ No newline at end of file +{{- end -}} + +{{/* +Convert the `--extra-volume-tags` command line arg from a map. +*/}} +{{- define "aws-ebs-csi-driver.extra-volume-tags" -}} +{{- $result := dict "pairs" (list) -}} +{{- range $key, $value := .Values.extraVolumeTags -}} +{{- $noop := printf "%s=%s" $key $value | append $result.pairs | set $result "pairs" -}} +{{- end -}} +{{- if gt (len $result.pairs) 0 -}} +- --extra-volume-tags={{- join "," $result.pairs -}} +{{- end -}} +{{- end -}} diff --git a/stable/awsebscsiprovisioner/templates/csidriver.yaml b/stable/awsebscsiprovisioner/templates/csidriver.yaml new file mode 100644 index 000000000..aa9aca71f --- /dev/null +++ b/stable/awsebscsiprovisioner/templates/csidriver.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: storage.k8s.io/v1beta1 +kind: CSIDriver +metadata: + name: ebs.csi.aws.com +spec: + attachRequired: true + podInfoOnMount: false diff --git a/stable/awsebscsiprovisioner/templates/daemonset.yaml b/stable/awsebscsiprovisioner/templates/daemonset.yaml index 7399fa97f..f0a9ec67c 100644 --- a/stable/awsebscsiprovisioner/templates/daemonset.yaml +++ b/stable/awsebscsiprovisioner/templates/daemonset.yaml @@ -16,6 +16,9 @@ spec: app: ebs-csi-node app.kubernetes.io/name: {{ include "aws-ebs-csi-driver.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} + {{- if .Values.node.podAnnotations }} + annotations: {{ toYaml .Values.node.podAnnotations | nindent 8 }} + {{- end }} spec: nodeSelector: beta.kubernetes.io/os: linux @@ -31,6 +34,7 @@ spec: privileged: true image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" args: + - node - --endpoint=$(CSI_ENDPOINT) - --logtostderr - --v=5 diff --git a/stable/awsebscsiprovisioner/templates/roles.yaml b/stable/awsebscsiprovisioner/templates/roles.yaml index 991fe2bb0..ba8f8cedb 100644 --- a/stable/awsebscsiprovisioner/templates/roles.yaml +++ b/stable/awsebscsiprovisioner/templates/roles.yaml @@ -4,6 +4,19 @@ kind: ServiceAccount metadata: name: ebs-csi-controller-sa namespace: {{ .Release.Namespace }} + {{- with .Values.serviceAccount.controller.annotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end }} + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ebs-csi-snapshot-controller + namespace: {{ .Release.Namespace }} + {{- with .Values.serviceAccount.snapshot.annotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end }} --- kind: ClusterRole @@ -20,15 +33,24 @@ rules: - apiGroups: ["storage.k8s.io"] resources: ["storageclasses"] verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list"] - apiGroups: ["storage.k8s.io"] resources: ["csinodes"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] --- kind: ClusterRoleBinding @@ -95,7 +117,7 @@ rules: verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] + verbs: ["get", "list", "watch", "create", "update", "patch"] - apiGroups: [""] resources: ["secrets"] verbs: ["get", "list"] @@ -108,6 +130,12 @@ rules: - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshots"] verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] - apiGroups: ["apiextensions.k8s.io"] resources: ["customresourcedefinitions"] verbs: ["create", "list", "watch", "delete"] @@ -125,6 +153,77 @@ roleRef: kind: ClusterRole name: ebs-external-snapshotter-role apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-snapshot-controller-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-snapshot-controller-binding +subjects: + - kind: ServiceAccount + name: ebs-csi-snapshot-controller + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: ebs-csi-snapshot-controller-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-snapshot-controller-leaderelection + namespace: {{ .Release.Namespace }} +rules: +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-snapshot-controller-leaderelection + namespace: {{ .Release.Namespace }} +subjects: + - kind: ServiceAccount + name: ebs-csi-snapshot-controller + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: ebs-csi-snapshot-controller-leaderelection + apiGroup: rbac.authorization.k8s.io {{- end }} {{- if .Values.resizer.enabled }} @@ -153,7 +252,7 @@ rules: verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] + verbs: ["get", "list", "watch", "create", "update", "patch"] --- kind: ClusterRoleBinding @@ -169,12 +268,3 @@ roleRef: name: ebs-external-resizer-role apiGroup: rbac.authorization.k8s.io {{- end}} - ---- -apiVersion: storage.k8s.io/v1beta1 -kind: CSIDriver -metadata: - name: ebs.csi.aws.com -spec: - attachRequired: true - podInfoOnMount: false diff --git a/stable/awsebscsiprovisioner/templates/statefulset-snapshot-controller.yaml b/stable/awsebscsiprovisioner/templates/statefulset-snapshot-controller.yaml new file mode 100644 index 000000000..f48c92f30 --- /dev/null +++ b/stable/awsebscsiprovisioner/templates/statefulset-snapshot-controller.yaml @@ -0,0 +1,35 @@ +{{- if .Values.snapshotter.enabled }} +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: ebs-csi-snapshot-controller + namespace: {{ .Release.Namespace }} +spec: + serviceName: ebs-csi-snapshot-controller + replicas: 1 + selector: + matchLabels: + app: ebs-csi-snapshot-controller + app.kubernetes.io/name: {{ include "aws-ebs-csi-driver.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + {{- if .Values.statefulSetCSISnapshotController.podAnnotations }} + annotations: + {{- range $key, $value := .Values.statefulSetCSISnapshotController.podAnnotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} + labels: + app: ebs-csi-snapshot-controller + app.kubernetes.io/name: {{ include "aws-ebs-csi-driver.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + spec: + serviceAccount: ebs-csi-snapshot-controller + containers: + - name: ebs-csi-snapshot-controller + image: "{{ .Values.snapshotter.image.repository }}:{{ .Values.snapshotter.image.tag }}" + args: + - --v=5 + - --leader-election=false +{{- end }} diff --git a/stable/awsebscsiprovisioner/templates/statefulset.yaml b/stable/awsebscsiprovisioner/templates/statefulset.yaml index e646fe1ba..cd54f7039 100644 --- a/stable/awsebscsiprovisioner/templates/statefulset.yaml +++ b/stable/awsebscsiprovisioner/templates/statefulset.yaml @@ -6,12 +6,18 @@ metadata: namespace: {{ .Release.Namespace }} spec: serviceName: ebs-csi-controller - replicas: 1 + replicas: {{ .Values.replicas }} selector: matchLabels: app: ebs-csi-controller template: metadata: + {{- if .Values.statefulSetCSIController.podAnnotations }} + annotations: + {{- range $key, $value := .Values.statefulSetCSIController.podAnnotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} labels: app: ebs-csi-controller app.kubernetes.io/name: {{ include "aws-ebs-csi-driver.name" . }} @@ -28,7 +34,9 @@ spec: - name: ebs-plugin image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" args : + - controller - --endpoint=$(CSI_ENDPOINT) + {{ include "aws-ebs-csi-driver.extra-volume-tags" . }} - --logtostderr - --v=5 env: @@ -74,6 +82,8 @@ spec: {{- if .Values.provisioner.enableVolumeScheduling }} - --feature-gates=Topology=true {{- end}} + - --enable-leader-election + - --leader-election-type=leases env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock @@ -85,6 +95,7 @@ spec: args: - --csi-address=$(ADDRESS) - --v=5 + - --leader-election=true env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock @@ -97,6 +108,7 @@ spec: args: - --csi-address=$(ADDRESS) - --connection-timeout=15s + - --leader-election=true env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock diff --git a/stable/awsebscsiprovisioner/templates/storageclass.yaml b/stable/awsebscsiprovisioner/templates/storageclass.yaml index 8aeef4c76..032fe71be 100644 --- a/stable/awsebscsiprovisioner/templates/storageclass.yaml +++ b/stable/awsebscsiprovisioner/templates/storageclass.yaml @@ -24,6 +24,9 @@ parameters: {{- if .Values.storageclass.encrypted }} encrypted: {{ .Values.storageclass.encrypted | quote }} {{- end }} + {{- if .Values.storageclass.kmsKeyId }} + kmsKeyId: {{ .Values.storageclass.kmsKeyId | quote }} + {{- end }} {{- if .Values.storageclass.allowedTopologies }} {{- with .Values.storageclass.allowedTopologies }} allowedTopologies: diff --git a/stable/awsebscsiprovisioner/values.yaml b/stable/awsebscsiprovisioner/values.yaml index f1e0df7cc..7919c79bd 100644 --- a/stable/awsebscsiprovisioner/values.yaml +++ b/stable/awsebscsiprovisioner/values.yaml @@ -1,14 +1,17 @@ nameOverride: "" fullnameOverride: "" +# replicas of the CSI-Controller +replicas: 1 + image: repository: "amazon/aws-ebs-csi-driver" - tag: "v0.4.0" + tag: "v0.5.0" liveness: image: repository: "quay.io/k8scsi/livenessprobe" - tag: "v1.1.0" + tag: "v2.0.0" tolerations: - effect: NoSchedule @@ -20,38 +23,67 @@ tolerations: env: {} +node: + # annotations for the pods running on each node as started per DaemonSet + podAnnotations: {} + +statefulSetCSIController: + # if you want to use kube2iam or kiam roles define it here as podAnnotation for the CSI-Controller (statefulSet) + podAnnotations: {} + +statefulSetCSISnapshotController: + # if you want to use kube2iam or kiam roles define it here as podAnnotation for the CSI-Snapshot-Controller (statefulSet) + podAnnotations: {} + registrar: node: image: repository: "quay.io/k8scsi/csi-node-driver-registrar" - tag: "v1.1.0" - + tag: "v1.2.0" provisioner: # True if enable volume scheduling for dynamic volume provisioning enableVolumeScheduling: false image: repository: "quay.io/k8scsi/csi-provisioner" - tag: "v1.3.0" + tag: "v1.5.0" attacher: image: repository: "quay.io/k8scsi/csi-attacher" - tag: "v1.2.0" + tag: "v2.0.0" + +resizer: + # True if enable volume resizing + enabled: false + image: + repository: "quay.io/k8scsi/csi-resizer" + tag: "v0.4.0" snapshotter: # True if enable volume snapshot enabled: false image: repository: "quay.io/k8scsi/csi-snapshotter" - tag: "v1.1.0" + tag: "v1.2.2" -resizer: - # True if enable volume resizing - enabled: false +snapshot-controller: image: - repository: "quay.io/k8scsi/csi-resizer" - tag: "v0.2.0" + repository: "quay.io/k8scsi/snapshot-controller" + tag: "v2.0.1" + +# Extra volume tags to attach to each dynamically provisioned volume. +# --- +# extraVolumeTags: +# key1: value1 +# key2: value2 +extraVolumeTags: {} + +serviceAccount: + controller: + annotations: {} + snapshot: + annotations: {} # AWS key id and access key # these are optional and can be left as is @@ -65,11 +97,15 @@ storageclass: reclaimPolicy: Delete volumeBindingMode: WaitForFirstConsumer type: gp2 + fstype: ext4 + iopsPerGB: null + encrypted: false + kmsKeyId: null allowedTopologies: [] - allowVolumeExpansion: true # - matchLabelExpressions: # - key: topology.ebs.csi.aws.com/zone # values: # - us-west-2a # - us-west-2b # - us-west-2c + allowVolumeExpansion: true diff --git a/test/ct-e2e.yaml b/test/ct-e2e.yaml index fc6ba0561..eeba914f3 100644 --- a/test/ct-e2e.yaml +++ b/test/ct-e2e.yaml @@ -9,6 +9,7 @@ excluded-charts: - azuredisk-csi-driver # DCOS-62804 - defaultstorageclass # DCOS-62803 - dispatch # DCOS-62802 + - gcpdisk-csi-driver # D2IQ-65765 - gcpdiskprovisioner # DCOS-62801 - kommander # DCOS-62800 - kommander-karma # DCOS-62799 diff --git a/test/e2e-kind.sh b/test/e2e-kind.sh index bf18ee845..94fd367c6 100755 --- a/test/e2e-kind.sh +++ b/test/e2e-kind.sh @@ -130,6 +130,12 @@ install_dummylb() { echo } +replace_priority_class_name_system_x_critical() { + echo 'Replacing priorityClassName: system-X-critical' + grep -rl "priorityClassName: system-" . | xargs sed -i 's/system-.*-critical/null/g' + echo +} + main() { run_ct_container "$1" shift @@ -141,7 +147,11 @@ main() { install_dummylb install_certmanager - docker_exec ct lint-and-install --upgrade --debug "$@" + docker_exec ct lint --debug "$@" + + replace_priority_class_name_system_x_critical + + docker_exec ct install --upgrade --debug "$@" echo }