diff --git a/.gitignore b/.gitignore index 5cc62424..c884b281 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ + +# intellij idea +.idea + *.swp bin/ vendor/ diff --git a/deploy/kubernetes/helm/.helmignore b/deploy/kubernetes/helm/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/deploy/kubernetes/helm/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/deploy/kubernetes/helm/Chart.yaml b/deploy/kubernetes/helm/Chart.yaml new file mode 100644 index 00000000..d452ab55 --- /dev/null +++ b/deploy/kubernetes/helm/Chart.yaml @@ -0,0 +1,11 @@ +apiVersion: v2 +name: aws-fsx-csi-driver +description: The Amazon FSx for Lustre Container Storage Interface (CSI) Driver implements CSI specification for container orchestrators (CO) to manage lifecycle of Amazon FSx for Lustre filesystems. +type: application +version: 0.1.0 +appVersion: latest +keywords: + - aws-fsx-csi-driver + - aws + - fsx + - network file system diff --git a/deploy/kubernetes/helm/README.md b/deploy/kubernetes/helm/README.md new file mode 100644 index 00000000..ce2fe270 --- /dev/null +++ b/deploy/kubernetes/helm/README.md @@ -0,0 +1,76 @@ + +## Amazon FSx for Lustre CSI Driver Helm Chart +For more info view [aws-fsx-csi-driver](https://github.com/kubernetes-sigs/aws-fsx-csi-driver) + +## Prerequisites +- Helm 3+ +- Kubernetes > 1.17.x, can be deployed to any namespace. +- Kubernetes < 1.17.x, namespace **must** be `kube-system`, as `system-cluster-critical` hard coded to this namespace. +## Install chart +```shell script +helm install . --name aws-fsx-csi-driver +``` + +## Upgrade release +```shell script +helm upgrade aws-fsx-csi-driver \ + --install . \ + --version 0.1.0 \ + --namespace kube-system \ + -f values.yaml +``` +## Uninstalling the Chart +```shell script +helm delete aws-fsx-csi-driver --namespace [NAMESPACE] +``` +## Parameters + +The following table lists the configurable parameters of the aws-fsx-csi-driver chart and their default values. + +| Parameter | Description | Default | +| ------------------------------------------------------| ------------------------------------------------------------- | ------------------------------------------ | +| `controllerService.replicaCount` | Num of replicas for controller | `2` | +| `controllerService.nodeSelector` | Controllers node selector | `beta.kubernetes.io/os: linux` | +| `controllerService.podSecurityContext` | Security context for controller pods | `{}` | +| | | | +| `controllerService.fsxPlugin.image.repository` | aws-fsx-csi-driver image name | `amazon/aws-fsx-csi-driver` | +| `controllerService.fsxPlugin.image.tag` | aws-fsx-csi-driver image tag | `latest` | +| `controllerService.fsxPlugin.image.pullPolicy` | aws-fsx-csi-driver image pull policy | `IfNotPresent` | +| `controllerService.fsxPlugin.extraArgs` | Extra arguments to be passed to aws-fsx-csi-driver fsxPlugin | `--logtostderr --v=5` | +| `controllerService.fsxPlugin.securityContext` | Security context for the container | `{}` | +| `controllerService.fsxPlugin.resources` | CPU/Memory resource requests/limits | `{}` | +| | | | +| `controllerService.csiProvisioner.image.repository` | csi-provisioner image name | `quay.io/k8scsi/csi-provisioner` | +| `controllerService.csiProvisioner.image.tag` | csi-provisioner image tag | `v1.3.0` | +| `controllerService.csiProvisioner.image.pullPolicy` | csi-provisioner image pull policy | `IfNotPresent` | +| `controllerService.csiProvisioner.extraArgs` | Extra arguments to be passed to csi-provisioner | `--timeout=5m --v=5 --enable-leader-election --leader-election-type=leases`| +| `controllerService.csiProvisioner.securityContext` | Security context for the container | `{}` | +| `controllerService.csiProvisioner.resources` | CPU/Memory resource requests/limits | `{}` | +| | | | +| `controllerService.nodeSelector` | Controllers node selector | `beta.kubernetes.io/os: linux` | +| `nodeService.podSecurityContext` | Security context for controller pods | `{}` | +| | | | +| `nodeService.fsxPlugin.image.repository` | aws-fsx-csi-driver image name | `amazon/aws-fsx-csi-driver` | +| `nodeService.fsxPlugin.image.tag` | aws-fsx-csi-driver image tag | `latest` | +| `nodeService.fsxPlugin.image.pullPolicy` | aws-fsx-csi-driver image pull policy | `IfNotPresent` | +| `nodeService.fsxPlugin.extraArgs` | Extra arguments to be passed to aws-fsx-csi-driver fsxPlugin | `--logtostderr --v=5` | +| `nodeService.fsxPlugin.securityContext` | Security context for the container | `privileged: true` | +| `nodeService.fsxPlugin.resources` | CPU/Memory resource requests/limits | `{}` | +| | | | +| `nodeService.csiDriverRegistrar.image.repository` | csi-node-driver-registrar image name | `quay.io/k8scsi/csi-node-driver-registrar` | +| `nodeService.csiDriverRegistrar.image.tag` | csi-node-driver-registrar image tag | `v1.1.0` | +| `nodeService.csiDriverRegistrar.image.pullPolicy` | csi-node-driver-registrar image pull policy | `IfNotPresent` | +| `nodeService.csiDriverRegistrar.extraArgs` | Extra arguments to be passed to aws-fsx-csi-driver fsxPlugin | `--v=5` | +| `nodeService.csiDriverRegistrar.securityContext` | Security context for the container | `{}` | +| `nodeService.csiDriverRegistrar.resources` | CPU/Memory resource requests/limits | `{}` | +| | | | +| `nodeService.livenessProbe.image.repository` | livenessprobe image name | `quay.io/k8scsi/livenessprobe` | +| `nodeService.livenessProbe.image.tag` | livenessprobe image tag | `v1.1.0` | +| `nodeService.livenessProbe.image.pullPolicy` | livenessprobe image pull policy | `Always` | +| `nodeService.livenessProbe.resources` | CPU/Memory resource requests/limits | `{}` | +| | | | +| `nameOverride` | String to partially override aws-fsx-csi-driver.fullname | `""` | +| `fullnameOverride` | String to fully override aws-fsx-csi-driver.fullname | `""` | +| `serviceAccount.create` | Specifies whether a service account should be created | `true` | +| `serviceAccount.annotations` | Additional Service Account annotations | `{}` | +| `serviceAccount.name` | Service Account name | `fsx-csi-controller-sa` | diff --git a/deploy/kubernetes/helm/templates/_helpers.tpl b/deploy/kubernetes/helm/templates/_helpers.tpl new file mode 100644 index 00000000..5e2ca3be --- /dev/null +++ b/deploy/kubernetes/helm/templates/_helpers.tpl @@ -0,0 +1,63 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "helm.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "helm.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "helm.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "helm.labels" -}} +helm.sh/chart: {{ include "helm.chart" . }} +{{ include "helm.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "helm.selectorLabels" -}} +app.kubernetes.io/name: {{ include "helm.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "helm.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "helm.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/deploy/kubernetes/helm/templates/controller.yaml b/deploy/kubernetes/helm/templates/controller.yaml new file mode 100644 index 00000000..956cdae8 --- /dev/null +++ b/deploy/kubernetes/helm/templates/controller.yaml @@ -0,0 +1,73 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "helm.fullname" . }}-controller + labels: + {{- include "helm.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.controllerService.replicaCount }} + selector: + matchLabels: + {{- include "helm.selectorLabels" . | nindent 6 }}-controller + template: + metadata: + labels: + {{- include "helm.selectorLabels" . | nindent 8 }}-controller + spec: + serviceAccountName: {{ include "helm.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.controllerService.podSecurityContext | nindent 8 }} + priorityClassName: system-cluster-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + containers: + - name: fsx-plugin + securityContext: + {{- toYaml .Values.controllerService.fsxPlugin.securityContext | nindent 12 }} + image: "{{ .Values.controllerService.fsxPlugin.image.repository }}:{{ .Values.controllerService.fsxPlugin.image.tag }}" + imagePullPolicy: {{ .Values.controllerService.fsxPlugin.image.pullPolicy }} + args: + - --endpoint=$(CSI_ENDPOINT) + {{- toYaml .Values.controllerService.fsxPlugin.extraArgs | nindent 12 }} + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: aws-secret + key: key_id + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: aws-secret + key: access_key + optional: true + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + resources: + {{- toYaml .Values.controllerService.fsxPlugin.resources | nindent 12 }} + - name: csi-provisioner + image: "{{ .Values.controllerService.csiProvisioner.image.repository }}:{{ .Values.controllerService.csiProvisioner.image.tag }}" + args: + - --csi-address=$(ADDRESS) + {{- toYaml .Values.controllerService.csiProvisioner.extraArgs | nindent 12 }} + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + resources: + {{- toYaml .Values.controllerService.csiProvisioner.resources | nindent 12 }} + + volumes: + - name: socket-dir + emptyDir: {} + {{- with .Values.controllerService.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/deploy/kubernetes/helm/templates/csidriver.yaml b/deploy/kubernetes/helm/templates/csidriver.yaml new file mode 100644 index 00000000..f8479e86 --- /dev/null +++ b/deploy/kubernetes/helm/templates/csidriver.yaml @@ -0,0 +1,6 @@ +apiVersion: storage.k8s.io/v1beta1 +kind: CSIDriver +metadata: + name: fsx.csi.aws.com +spec: + attachRequired: false diff --git a/deploy/kubernetes/helm/templates/node.yaml b/deploy/kubernetes/helm/templates/node.yaml new file mode 100644 index 00000000..5176f1cf --- /dev/null +++ b/deploy/kubernetes/helm/templates/node.yaml @@ -0,0 +1,103 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "helm.fullname" . }}-daemonset + labels: + {{- include "helm.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "helm.selectorLabels" . | nindent 6 }}-daemonset + template: + metadata: + labels: + {{- include "helm.selectorLabels" . | nindent 8 }}-daemonset + spec: + securityContext: + {{- toYaml .Values.nodeService.podSecurityContext | nindent 8 }} + hostNetwork: true + containers: + - name: fsx-plugin + securityContext: + {{- toYaml .Values.nodeService.fsxPlugin.securityContext | nindent 12 }} + image: "{{ .Values.nodeService.fsxPlugin.image.repository }}:{{ .Values.nodeService.fsxPlugin.image.tag }}" + imagePullPolicy: {{ .Values.nodeService.fsxPlugin.image.pullPolicy }} + args: + - --endpoint=$(CSI_ENDPOINT) + {{- toYaml .Values.nodeService.fsxPlugin.extraArgs | nindent 12 }} + env: + - name: CSI_ENDPOINT + value: unix:/csi/csi.sock + volumeMounts: + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" + - name: plugin-dir + mountPath: /csi + ports: + - containerPort: 9810 + name: healthz + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 2 + resources: + {{- toYaml .Values.nodeService.fsxPlugin.resources | nindent 12 }} + - name: csi-driver-registrar + securityContext: + {{- toYaml .Values.nodeService.csiDriverRegistrar.securityContext | nindent 12 }} + image: "{{ .Values.nodeService.csiDriverRegistrar.image.repository }}:{{ .Values.nodeService.csiDriverRegistrar.image.tag }}" + imagePullPolicy: {{ .Values.nodeService.csiDriverRegistrar.image.pullPolicy }} + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + {{- toYaml .Values.nodeService.csiDriverRegistrar.extraArgs | nindent 12 }} + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/fsx.csi.aws.com/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + resources: + {{- toYaml .Values.nodeService.csiDriverRegistrar.resources | nindent 12 }} + - name: liveness-probe + imagePullPolicy: {{ .Values.nodeService.livenessProbe.image.pullPolicy }} + image: "{{ .Values.nodeService.livenessProbe.image.repository }}:{{ .Values.nodeService.livenessProbe.image.tag }}" + args: + - --csi-address=/csi/csi.sock + - --health-port=9810 + volumeMounts: + - mountPath: /csi + name: plugin-dir + resources: + {{- toYaml .Values.nodeService.livenessProbe.resources | nindent 12 }} + volumes: + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/fsx.csi.aws.com/ + type: DirectoryOrCreate + {{- with .Values.nodeService.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/deploy/kubernetes/helm/templates/serviceaccount.yaml b/deploy/kubernetes/helm/templates/serviceaccount.yaml new file mode 100644 index 00000000..7e4807d3 --- /dev/null +++ b/deploy/kubernetes/helm/templates/serviceaccount.yaml @@ -0,0 +1,58 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "helm.serviceAccountName" . }} + labels: + {{- include "helm.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: fsx-csi-external-provisioner-role + labels: + {{- include "helm.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: fsx-csi-external-provisioner-binding + labels: + {{- include "helm.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ include "helm.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: fsx-csi-external-provisioner-role + apiGroup: rbac.authorization.k8s.io +{{- end -}} \ No newline at end of file diff --git a/deploy/kubernetes/helm/values.yaml b/deploy/kubernetes/helm/values.yaml new file mode 100644 index 00000000..0b48fe48 --- /dev/null +++ b/deploy/kubernetes/helm/values.yaml @@ -0,0 +1,108 @@ +# Default values for helm. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +controllerService: + replicaCount: 2 + + nodeSelector: + beta.kubernetes.io/os: linux + + podSecurityContext: {} + # fsGroup: 2000 + + fsxPlugin: + image: + repository: amazon/aws-fsx-csi-driver + tag: latest + pullPolicy: IfNotPresent + + extraArgs: + - --logtostderr + - --v=5 + + securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + resources: {} + + csiProvisioner: + image: + repository: quay.io/k8scsi/csi-provisioner + tag: v1.3.0 + pullPolicy: IfNotPresent + + extraArgs: + - --timeout=5m + - --v=5 + - --enable-leader-election + - --leader-election-type=leases + + securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + resources: {} + +nodeService: + podSecurityContext: {} + # fsGroup: 2000 + + nodeSelector: + beta.kubernetes.io/os: linux + + fsxPlugin: + image: + repository: amazon/aws-fsx-csi-driver + tag: latest + pullPolicy: IfNotPresent + + extraArgs: + - --logtostderr + - --v=5 + + securityContext: + privileged: true + + resources: {} + + csiDriverRegistrar: + image: + repository: quay.io/k8scsi/csi-node-driver-registrar + tag: v1.1.0 + pullPolicy: IfNotPresent + + extraArgs: + - --v=5 + + securityContext: {} + + resources: {} + + livenessProbe: + image: + repository: quay.io/k8scsi/livenessprobe + tag: v1.1.0 + pullPolicy: Always + + resources: {} + +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + annotations: {} + ## Enable if EKS IAM for SA is used + # eks.amazonaws.com/role-arn: arn:aws:iam::111122223333:role/fsx-csi-role + name: fsx-csi-controller-sa diff --git a/tester/e2e-test-config.yaml b/tester/e2e-test-config.yaml index 3b3942d2..db5b4faa 100644 --- a/tester/e2e-test-config.yaml +++ b/tester/e2e-test-config.yaml @@ -39,18 +39,30 @@ build: | docker push $IMAGE_NAME:$IMAGE_TAG install: | + HELM_VERSION=3.1.1 + + echo "Installing Helm version: ${HELM_VERSION}" + wget "https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz" + tar -zxvf "helm-v${HELM_VERSION}-linux-amd64.tar.gz" + mv linux-amd64/helm /usr/local/bin/helm + echo "Deploying driver" AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) IMAGE_TAG={{TEST_ID}} IMAGE_NAME=$AWS_ACCOUNT_ID.dkr.ecr.us-west-2.amazonaws.com/aws-fsx-csi-driver - sed -i'' "s,chengpan/aws-fsx-csi-driver,$IMAGE_NAME," deploy/kubernetes/overlays/dev/kustomization.yaml - sed -i'' "s,newTag: latest,newTag: \"$IMAGE_TAG\"," deploy/kubernetes/overlays/dev/kustomization.yaml - kubectl apply -k deploy/kubernetes/overlays/dev/ + helm upgrade aws-fsx-csi-driver \ + --install deploy/kubernetes/helm/ \ + --namespace kube-system \ + -f deploy/kubernetes/helm/values.yaml \ + --set controllerService.fsxPlugin.image.repository=$IMAGE_NAME \ + --set controllerService.fsxPlugin.image.tag=$IMAGE_TAG \ + --set nodeService.fsxPlugin.image.repository=$IMAGE_NAME \ + --set nodeService.fsxPlugin.image.tag=$IMAGE_TAG uninstall: | echo "Removing driver" - kubectl delete -k deploy/kubernetes/overlays/dev/ + helm delete aws-fsx-csi-driver --namespace kube-system test: | go get -u github.com/onsi/ginkgo/ginkgo