Add Ansible Rook role (#2960)

* Add initial ansible rook role * Fix issue with no yet existing template * Add Rook schema related files * Remove already setup vars, add missing enters * Fix issue with feature mapping * Add configuration support * Move kubeconfig to rook role yaml * Update changelog * Add started dashes to yaml defaults for Rook * Add RH fix suggested by cicharka * Add rook cluster helm chart installation * Add separate chart values definitions * Limit fact checking in rook role * Add initial rook/ceph documentation * Update documentation for Rook * Update download requirements
hitachienergy · Mar 8, 2022 · 087f6c4 · 087f6c4
1 parent 364e12e
commit 087f6c4
Show file tree

Hide file tree

Showing 10 changed files with 226 additions and 4 deletions.
diff --git a/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/files.yml b/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/files.yml
@@ -38,3 +38,9 @@ files:
 
   'https://helm.elastic.co/helm/filebeat/filebeat-7.9.2.tgz':
     sha256: 5140b4c4473ca33a0af4c3f70545dcc89735c0a179d974ebc150f1f28ac229ab
+
+  'https://charts.rook.io/release/rook-ceph-v1.8.5.tgz':
+    sha256: 11a425c16a8513e9cf7a2d6de396ea6e5166a03a63a8bf883d4f2098ab35b268
+
+  'https://charts.rook.io/release/rook-ceph-cluster-v1.8.5.tgz':
+    sha256: 58695a740845091c10426266e7b345fe6185ee5a3b86f1f2d6e52a55af8503c1
diff --git a/...ble/playbooks/roles/repository/files/download-requirements/requirements/x86_64/images.yml b/...ble/playbooks/roles/repository/files/download-requirements/requirements/x86_64/images.yml
@@ -156,3 +156,28 @@ images:
 
   'calico/pod2daemon-flexvol:v3.20.3':
     sha1: 97c1b7ac90aa5a0f5c52e7f137549e598ff80f3e
+
+  # --- Rook ---
+  'k8s.gcr.io/sig-storage/csi-attacher:v3.4.0':
+    sha1: 8567876a11c527e9d406d3f3efa09e3cee437985
+
+  'k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.5.0':
+    sha1: 07104d23bbb224f81ef6b8fd379b01a7cbba0946
+
+  'k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0':
+    sha1: 55c9d55eb0f2cbe9e37e5464e578b2d7fa45f8f8
+
+  'k8s.gcr.io/sig-storage/csi-resizer:v1.4.0':
+    sha1: 289bafda08f126423dcf7ad78f08a5d8ed57a82e
+
+  'k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1':
+    sha1: 8deb564fde820c7f795d8b425f867d4194edd088
+
+  'quay.io/ceph/ceph:v16.2.7':
+    sha1: 039717f406243b516fead5c36d2d5698c458bf21
+
+  'quay.io/cephcsi/cephcsi:v3.5.1':
+    sha1: 2f89fc81c6665f4daf9df2edf3fbb45caa45891c
+
+  'rook/ceph:v1.8.5':
+    sha1: 96b3efbc626deec6c1db31e43d8d8d6fe8186174
diff --git a/ansible/playbooks/roles/rook/defaults/main.yml b/ansible/playbooks/roles/rook/defaults/main.yml
@@ -0,0 +1,6 @@
+---
+rook_helm_chart_file_name: rook-ceph-v1.8.5.tgz
+rook_helm_cluster_chart_file_name: rook-ceph-cluster-v1.8.5.tgz
+rook_chart_namespace: rook-ceph
+rook_helm_chart_name: rook-ceph
+rook_helm_cluster_chart_name: rook-ceph-cluster
diff --git a/ansible/playbooks/roles/rook/tasks/main.yml b/ansible/playbooks/roles/rook/tasks/main.yml
@@ -0,0 +1,67 @@
+---
+- name: Prepare configuration and upgrade/install Rook Helm chart
+  when: specification.enabled
+  become: true
+  run_once: true
+  block:
+
+    - name: RedHat fix | Create helm's binary symlink
+      file:
+        src: "/usr/local/bin/helm"
+        dest: "/usr/bin/helm"
+        state: link
+      when: ansible_os_family == 'RedHat'
+
+    - name: Download Rook's Chart Files
+      include_role:
+        name: download
+        tasks_from: download_file
+      vars:
+        file_name: "{{ item }}"
+      loop:
+        - "{{ rook_helm_chart_file_name }}"
+        - "{{ rook_helm_cluster_chart_file_name }}"
+
+    - name: Create custom configuration for operator Helm chart file (operator-custom-chart-values.yml)
+      when: specification.operator_chart_values is defined
+      copy:
+        content: "{{ specification.operator_chart_values }}"
+        dest: "{{ download_directory }}/operator-custom-chart-values.yml"
+
+    - name: Create custom configuration for cluster Helm chart file (cluster-custom-chart-values.yml)
+      when: specification.cluster_chart_values is defined
+      copy:
+        content: "{{ specification.cluster_chart_values }}"
+        dest: "{{ download_directory }}/cluster-custom-chart-values.yml"
+
+    - name: Install Rook operator using Helm chart with values from operator-custom-chart-values.yml
+      when: specification.operator_chart_values is defined
+      shell: |
+        helm -n {{ rook_chart_namespace }} upgrade --install \
+          -f {{ download_directory }}/operator-custom-chart-values.yml \
+          {{ rook_helm_chart_name }} \
+          {{ download_directory }}/{{ rook_helm_chart_file_name }} --create-namespace
+
+    - name: Install Rook operator using Helm chart with default values
+      when: not specification.operator_chart_values is defined
+      shell: |
+        helm -n {{ rook_chart_namespace }} upgrade --install \
+          {{ rook_helm_chart_name }} \
+          {{ download_directory }}/{{ rook_helm_chart_file_name }} --create-namespace
+
+    - name: Create Rook cluster with values from cluster-custom-chart-values.yml
+      when: specification.cluster_chart_values is defined
+      shell: |
+        helm -n {{ rook_chart_namespace }} upgrade --install \
+          --set operatorNamespace={{ rook_chart_namespace }} \
+          -f {{ download_directory }}/cluster-custom-chart-values.yml \
+          {{ rook_helm_cluster_chart_name }} \
+          {{ download_directory }}/{{ rook_helm_cluster_chart_file_name }} --create-namespace
+
+    - name: Create Rook cluster with default values
+      when: not specification.cluster_chart_values is defined
+      shell: |
+        helm -n {{ rook_chart_namespace }} upgrade --install \
+          --set operatorNamespace={{ rook_chart_namespace }} \
+          {{ rook_helm_cluster_chart_name }} \
+          {{ download_directory }}/{{ rook_helm_cluster_chart_file_name }} --create-namespace
diff --git a/ansible/playbooks/rook.yml b/ansible/playbooks/rook.yml
@@ -0,0 +1,12 @@
+---
+- hosts: repository
+  gather_facts: true
+  tasks: []
+
+- hosts: kubernetes_master
+  become: true
+  become_method: sudo
+  roles:
+    - rook
+  environment:
+    KUBECONFIG: "{{ kubeconfig.remote }}"
diff --git a/docs/changelogs/CHANGELOG-2.0.md b/docs/changelogs/CHANGELOG-2.0.md
@@ -19,6 +19,8 @@
 - [#2991](https://github.com/epiphany-platform/epiphany/issues/2991) - Add automatic backup creation for download requirements
 - [#2888](https://github.com/epiphany-platform/epiphany/issues/2888) - Define additional disks with defined sizes to VMs for Azure
 - [#2812](https://github.com/epiphany-platform/epiphany/issues/2812) - Extend K8s config validation
+- [#2890](https://github.com/epiphany-platform/epiphany/issues/2890) - Rook/Ceph Kubernetes Native Storage for Azure Kubernetes non-AKS cluster
+- [#1452](https://github.com/epiphany-platform/epiphany/issues/1452) - Create Kubernetes Native Storage Epiphany configuration for Azure provider
 
 ### Fixed
 

diff --git a/docs/home/howto/kubernetes/PERSISTENT_STORAGE.md b/docs/home/howto/kubernetes/PERSISTENT_STORAGE.md
@@ -1,8 +1,96 @@
-## Kubernetes persistent storage
+# Kubernetes Persistent Storage
 
-Epiphany supports [Azure Files](https://docs.microsoft.com/en-us/azure/storage/files/storage-files-introduction)
-and [Amazon EFS](https://docs.aws.amazon.com/efs/latest/ug/how-it-works.html) storage types to use as Kubernetes
-persistent volumes.
+In Epiphany there are two supported ways of setting up Kubernetes Persistent Storage:
+- Rook/Ceph Cluster Storage with disks resources created by Epiphany
+- [Azure Files](https://docs.microsoft.com/en-us/azure/storage/files/storage-files-introduction)
+or [Amazon EFS](https://docs.aws.amazon.com/efs/latest/ug/how-it-works.html) storage types to use as Kubernetes persistent volumes
+
+## Kubernetes Rook/Ceph Cluster Storage
+
+Rook provides distributed storage systems for Kubernetes installed with Epiphany.
+It provides capabilities:
+- self-managing
+- self-scaling
+- self-healing
+- upgrading
+- migration
+- disaster recovery
+- monitoring
+
+Epiphany supports Rook with Ceph storage, other options provided by Rook - Cassandra, NFS are not supported.
+
+### Rook/Ceph General Configuration
+
+To add Rook/Ceph support in Epiphany you need to add to your cluster configuration two elements:
+- storage (for cloud deployments - can be automatically created by Epiphany)
+- Rook/Ceph
+
+Adding the storage is described below in separate sections for Azure, AWS and on premise environments.
+
+To enable Rook support add to your cluster configuration the section like below:
+
+```yaml
+---
+kind: configuration/rook
+title: "Kubernetes Rook Config"
+provider: azure
+name: default
+specification:
+    enabled: true
+```
+
+The key `specification.enabled` must be set to true to install Rook/Ceph component. This will install Rook/Ceph with default values. To override default values provided by Rook you need to add to `configuration/rook` keys:
+- `specification.operator_chart_values` - to override Rook Operator Helm Chart default values
+- `specification.cluster_chart_values` - to override Rook Cluster Helm Chart default values
+
+```yaml
+---
+kind: configuration/rook
+title: "Kubernetes Rook Config"
+provider: azure
+name: default
+specification:
+    enabled: true
+    operator_chart_values: |
+      ...
+    cluster_chart_values: |
+      ...
+```
+Values nested below the `operator_chart_values` and `cluster_chart_values` keys are respectively Helm Chart values for Rook Operator and Rook Ceph Cluster.
+
+More information about Helm Chart values may be found:
+- [Helm Operator](https://github.com/rook/rook/blob/master/Documentation/helm-operator.md)
+- [Helm Ceph Cluster](https://github.com/rook/rook/blob/master/Documentation/helm-ceph-cluster.md)
+
+Sample configuration files that can be used in Epiphany `configuration/rook`:
+- [Helm Operator](https://raw.githubusercontent.com/rook/rook/v1.8.5/deploy/charts/rook-ceph/values.yaml)
+- [Helm Ceph Cluster](https://raw.githubusercontent.com/rook/rook/v1.8.5/deploy/charts/rook-ceph-cluster/values.yaml)
+
+More informations about Rook with Ceph storage may be found in the official Rook [documentation](https://rook.io/docs/rook/v1.8/).
+
+### Create disks for Rook/Ceph Cluster Storage - Azure
+
+To create Rook/Ceph Cluster Storage on Azure first you need to add empty disk resource to Kubernetes cluster in key `specification.additional_disks`.
+
+```yaml
+---
+kind: infrastructure/virtual-machine
+name: kubernetes-node-machine
+provider: azure
+based_on: kubernetes-node-machine
+specification:
+  storage_image_reference:
+    ..
+  storage_os_disk:
+    disk_size_gb: 64
+  additional_disks:
+    - storage_account_type: Premium_LRS
+      disk_size_gb: 128
+```
+
+#### Create disks for Rook/Ceph Cluster Storage - AWS
+
+#### Create disks for Rook/Ceph Cluster Storage - On Prem
 
 ### Azure
 

diff --git a/schema/common/defaults/configuration/feature-mapping.yml b/schema/common/defaults/configuration/feature-mapping.yml
@@ -49,6 +49,8 @@ specification:
       enabled: true
     - name: applications
       enabled: true
+    - name: rook
+      enabled: true
 
   roles_mapping:
     kafka:
@@ -113,6 +115,7 @@ specification:
       - kubernetes-master
       - helm
       - applications
+      - rook
       - node-exporter
       - filebeat
       - firewall

diff --git a/schema/common/defaults/configuration/rook.yml b/schema/common/defaults/configuration/rook.yml
@@ -0,0 +1,6 @@
+---
+kind: configuration/rook
+title: "Kubernetes Rook Config"
+name: default
+specification:
+    enabled: false
diff --git a/schema/common/validation/configuration/rook.yml b/schema/common/validation/configuration/rook.yml
@@ -0,0 +1,7 @@
+"$id": "#/specification"
+title: "Rook specification schema"
+description: "Rook specification schema"
+type: object
+properties:
+  enabled:
+    type: boolean