From 087f6c4dfdb1986ad46728c667765ceb0087b4b4 Mon Sep 17 00:00:00 2001 From: erzetpe <2865776+erzetpe@users.noreply.github.com> Date: Mon, 28 Feb 2022 15:15:45 +0100 Subject: [PATCH] Add Ansible Rook role (#2960) * Add initial ansible rook role * Fix issue with no yet existing template * Add Rook schema related files * Remove already setup vars, add missing enters * Fix issue with feature mapping * Add configuration support * Move kubeconfig to rook role yaml * Update changelog * Add started dashes to yaml defaults for Rook * Add RH fix suggested by cicharka * Add rook cluster helm chart installation * Add separate chart values definitions * Limit fact checking in rook role * Add initial rook/ceph documentation * Update documentation for Rook * Update download requirements --- .../requirements/x86_64/files.yml | 6 ++ .../requirements/x86_64/images.yml | 25 +++++ .../playbooks/roles/rook/defaults/main.yml | 6 ++ ansible/playbooks/roles/rook/tasks/main.yml | 67 +++++++++++++ ansible/playbooks/rook.yml | 12 +++ docs/changelogs/CHANGELOG-2.0.md | 2 + .../howto/kubernetes/PERSISTENT_STORAGE.md | 96 ++++++++++++++++++- .../configuration/feature-mapping.yml | 3 + schema/common/defaults/configuration/rook.yml | 6 ++ .../common/validation/configuration/rook.yml | 7 ++ 10 files changed, 226 insertions(+), 4 deletions(-) create mode 100644 ansible/playbooks/roles/rook/defaults/main.yml create mode 100644 ansible/playbooks/roles/rook/tasks/main.yml create mode 100644 ansible/playbooks/rook.yml create mode 100644 schema/common/defaults/configuration/rook.yml create mode 100644 schema/common/validation/configuration/rook.yml diff --git a/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/files.yml b/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/files.yml index 27734b80ec..dfc3e9f376 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/files.yml +++ b/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/files.yml @@ -38,3 +38,9 @@ files: 'https://helm.elastic.co/helm/filebeat/filebeat-7.9.2.tgz': sha256: 5140b4c4473ca33a0af4c3f70545dcc89735c0a179d974ebc150f1f28ac229ab + + 'https://charts.rook.io/release/rook-ceph-v1.8.5.tgz': + sha256: 11a425c16a8513e9cf7a2d6de396ea6e5166a03a63a8bf883d4f2098ab35b268 + + 'https://charts.rook.io/release/rook-ceph-cluster-v1.8.5.tgz': + sha256: 58695a740845091c10426266e7b345fe6185ee5a3b86f1f2d6e52a55af8503c1 diff --git a/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/images.yml b/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/images.yml index 7a7756cd61..c6c97fee70 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/images.yml +++ b/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/images.yml @@ -156,3 +156,28 @@ images: 'calico/pod2daemon-flexvol:v3.20.3': sha1: 97c1b7ac90aa5a0f5c52e7f137549e598ff80f3e + + # --- Rook --- + 'k8s.gcr.io/sig-storage/csi-attacher:v3.4.0': + sha1: 8567876a11c527e9d406d3f3efa09e3cee437985 + + 'k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.5.0': + sha1: 07104d23bbb224f81ef6b8fd379b01a7cbba0946 + + 'k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0': + sha1: 55c9d55eb0f2cbe9e37e5464e578b2d7fa45f8f8 + + 'k8s.gcr.io/sig-storage/csi-resizer:v1.4.0': + sha1: 289bafda08f126423dcf7ad78f08a5d8ed57a82e + + 'k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1': + sha1: 8deb564fde820c7f795d8b425f867d4194edd088 + + 'quay.io/ceph/ceph:v16.2.7': + sha1: 039717f406243b516fead5c36d2d5698c458bf21 + + 'quay.io/cephcsi/cephcsi:v3.5.1': + sha1: 2f89fc81c6665f4daf9df2edf3fbb45caa45891c + + 'rook/ceph:v1.8.5': + sha1: 96b3efbc626deec6c1db31e43d8d8d6fe8186174 diff --git a/ansible/playbooks/roles/rook/defaults/main.yml b/ansible/playbooks/roles/rook/defaults/main.yml new file mode 100644 index 0000000000..ea9d85a82a --- /dev/null +++ b/ansible/playbooks/roles/rook/defaults/main.yml @@ -0,0 +1,6 @@ +--- +rook_helm_chart_file_name: rook-ceph-v1.8.5.tgz +rook_helm_cluster_chart_file_name: rook-ceph-cluster-v1.8.5.tgz +rook_chart_namespace: rook-ceph +rook_helm_chart_name: rook-ceph +rook_helm_cluster_chart_name: rook-ceph-cluster diff --git a/ansible/playbooks/roles/rook/tasks/main.yml b/ansible/playbooks/roles/rook/tasks/main.yml new file mode 100644 index 0000000000..06c94b00c5 --- /dev/null +++ b/ansible/playbooks/roles/rook/tasks/main.yml @@ -0,0 +1,67 @@ +--- +- name: Prepare configuration and upgrade/install Rook Helm chart + when: specification.enabled + become: true + run_once: true + block: + + - name: RedHat fix | Create helm's binary symlink + file: + src: "/usr/local/bin/helm" + dest: "/usr/bin/helm" + state: link + when: ansible_os_family == 'RedHat' + + - name: Download Rook's Chart Files + include_role: + name: download + tasks_from: download_file + vars: + file_name: "{{ item }}" + loop: + - "{{ rook_helm_chart_file_name }}" + - "{{ rook_helm_cluster_chart_file_name }}" + + - name: Create custom configuration for operator Helm chart file (operator-custom-chart-values.yml) + when: specification.operator_chart_values is defined + copy: + content: "{{ specification.operator_chart_values }}" + dest: "{{ download_directory }}/operator-custom-chart-values.yml" + + - name: Create custom configuration for cluster Helm chart file (cluster-custom-chart-values.yml) + when: specification.cluster_chart_values is defined + copy: + content: "{{ specification.cluster_chart_values }}" + dest: "{{ download_directory }}/cluster-custom-chart-values.yml" + + - name: Install Rook operator using Helm chart with values from operator-custom-chart-values.yml + when: specification.operator_chart_values is defined + shell: | + helm -n {{ rook_chart_namespace }} upgrade --install \ + -f {{ download_directory }}/operator-custom-chart-values.yml \ + {{ rook_helm_chart_name }} \ + {{ download_directory }}/{{ rook_helm_chart_file_name }} --create-namespace + + - name: Install Rook operator using Helm chart with default values + when: not specification.operator_chart_values is defined + shell: | + helm -n {{ rook_chart_namespace }} upgrade --install \ + {{ rook_helm_chart_name }} \ + {{ download_directory }}/{{ rook_helm_chart_file_name }} --create-namespace + + - name: Create Rook cluster with values from cluster-custom-chart-values.yml + when: specification.cluster_chart_values is defined + shell: | + helm -n {{ rook_chart_namespace }} upgrade --install \ + --set operatorNamespace={{ rook_chart_namespace }} \ + -f {{ download_directory }}/cluster-custom-chart-values.yml \ + {{ rook_helm_cluster_chart_name }} \ + {{ download_directory }}/{{ rook_helm_cluster_chart_file_name }} --create-namespace + + - name: Create Rook cluster with default values + when: not specification.cluster_chart_values is defined + shell: | + helm -n {{ rook_chart_namespace }} upgrade --install \ + --set operatorNamespace={{ rook_chart_namespace }} \ + {{ rook_helm_cluster_chart_name }} \ + {{ download_directory }}/{{ rook_helm_cluster_chart_file_name }} --create-namespace diff --git a/ansible/playbooks/rook.yml b/ansible/playbooks/rook.yml new file mode 100644 index 0000000000..6226d34462 --- /dev/null +++ b/ansible/playbooks/rook.yml @@ -0,0 +1,12 @@ +--- +- hosts: repository + gather_facts: true + tasks: [] + +- hosts: kubernetes_master + become: true + become_method: sudo + roles: + - rook + environment: + KUBECONFIG: "{{ kubeconfig.remote }}" diff --git a/docs/changelogs/CHANGELOG-2.0.md b/docs/changelogs/CHANGELOG-2.0.md index 29d63d7916..99d55041fb 100644 --- a/docs/changelogs/CHANGELOG-2.0.md +++ b/docs/changelogs/CHANGELOG-2.0.md @@ -19,6 +19,8 @@ - [#2991](https://github.com/epiphany-platform/epiphany/issues/2991) - Add automatic backup creation for download requirements - [#2888](https://github.com/epiphany-platform/epiphany/issues/2888) - Define additional disks with defined sizes to VMs for Azure - [#2812](https://github.com/epiphany-platform/epiphany/issues/2812) - Extend K8s config validation +- [#2890](https://github.com/epiphany-platform/epiphany/issues/2890) - Rook/Ceph Kubernetes Native Storage for Azure Kubernetes non-AKS cluster +- [#1452](https://github.com/epiphany-platform/epiphany/issues/1452) - Create Kubernetes Native Storage Epiphany configuration for Azure provider ### Fixed diff --git a/docs/home/howto/kubernetes/PERSISTENT_STORAGE.md b/docs/home/howto/kubernetes/PERSISTENT_STORAGE.md index 37738d563a..fdcc6f57c1 100644 --- a/docs/home/howto/kubernetes/PERSISTENT_STORAGE.md +++ b/docs/home/howto/kubernetes/PERSISTENT_STORAGE.md @@ -1,8 +1,96 @@ -## Kubernetes persistent storage +# Kubernetes Persistent Storage -Epiphany supports [Azure Files](https://docs.microsoft.com/en-us/azure/storage/files/storage-files-introduction) -and [Amazon EFS](https://docs.aws.amazon.com/efs/latest/ug/how-it-works.html) storage types to use as Kubernetes -persistent volumes. +In Epiphany there are two supported ways of setting up Kubernetes Persistent Storage: +- Rook/Ceph Cluster Storage with disks resources created by Epiphany +- [Azure Files](https://docs.microsoft.com/en-us/azure/storage/files/storage-files-introduction) +or [Amazon EFS](https://docs.aws.amazon.com/efs/latest/ug/how-it-works.html) storage types to use as Kubernetes persistent volumes + +## Kubernetes Rook/Ceph Cluster Storage + +Rook provides distributed storage systems for Kubernetes installed with Epiphany. +It provides capabilities: +- self-managing +- self-scaling +- self-healing +- upgrading +- migration +- disaster recovery +- monitoring + +Epiphany supports Rook with Ceph storage, other options provided by Rook - Cassandra, NFS are not supported. + +### Rook/Ceph General Configuration + +To add Rook/Ceph support in Epiphany you need to add to your cluster configuration two elements: +- storage (for cloud deployments - can be automatically created by Epiphany) +- Rook/Ceph + +Adding the storage is described below in separate sections for Azure, AWS and on premise environments. + +To enable Rook support add to your cluster configuration the section like below: + +```yaml +--- +kind: configuration/rook +title: "Kubernetes Rook Config" +provider: azure +name: default +specification: + enabled: true +``` + +The key `specification.enabled` must be set to true to install Rook/Ceph component. This will install Rook/Ceph with default values. To override default values provided by Rook you need to add to `configuration/rook` keys: +- `specification.operator_chart_values` - to override Rook Operator Helm Chart default values +- `specification.cluster_chart_values` - to override Rook Cluster Helm Chart default values + +```yaml +--- +kind: configuration/rook +title: "Kubernetes Rook Config" +provider: azure +name: default +specification: + enabled: true + operator_chart_values: | + ... + cluster_chart_values: | + ... +``` +Values nested below the `operator_chart_values` and `cluster_chart_values` keys are respectively Helm Chart values for Rook Operator and Rook Ceph Cluster. + +More information about Helm Chart values may be found: +- [Helm Operator](https://github.com/rook/rook/blob/master/Documentation/helm-operator.md) +- [Helm Ceph Cluster](https://github.com/rook/rook/blob/master/Documentation/helm-ceph-cluster.md) + +Sample configuration files that can be used in Epiphany `configuration/rook`: +- [Helm Operator](https://raw.githubusercontent.com/rook/rook/v1.8.5/deploy/charts/rook-ceph/values.yaml) +- [Helm Ceph Cluster](https://raw.githubusercontent.com/rook/rook/v1.8.5/deploy/charts/rook-ceph-cluster/values.yaml) + +More informations about Rook with Ceph storage may be found in the official Rook [documentation](https://rook.io/docs/rook/v1.8/). + +### Create disks for Rook/Ceph Cluster Storage - Azure + +To create Rook/Ceph Cluster Storage on Azure first you need to add empty disk resource to Kubernetes cluster in key `specification.additional_disks`. + +```yaml +--- +kind: infrastructure/virtual-machine +name: kubernetes-node-machine +provider: azure +based_on: kubernetes-node-machine +specification: + storage_image_reference: + .. + storage_os_disk: + disk_size_gb: 64 + additional_disks: + - storage_account_type: Premium_LRS + disk_size_gb: 128 +``` + +#### Create disks for Rook/Ceph Cluster Storage - AWS + +#### Create disks for Rook/Ceph Cluster Storage - On Prem ### Azure diff --git a/schema/common/defaults/configuration/feature-mapping.yml b/schema/common/defaults/configuration/feature-mapping.yml index 8050b2b9e3..e4fac6626e 100644 --- a/schema/common/defaults/configuration/feature-mapping.yml +++ b/schema/common/defaults/configuration/feature-mapping.yml @@ -49,6 +49,8 @@ specification: enabled: true - name: applications enabled: true + - name: rook + enabled: true roles_mapping: kafka: @@ -113,6 +115,7 @@ specification: - kubernetes-master - helm - applications + - rook - node-exporter - filebeat - firewall diff --git a/schema/common/defaults/configuration/rook.yml b/schema/common/defaults/configuration/rook.yml new file mode 100644 index 0000000000..24d1aca767 --- /dev/null +++ b/schema/common/defaults/configuration/rook.yml @@ -0,0 +1,6 @@ +--- +kind: configuration/rook +title: "Kubernetes Rook Config" +name: default +specification: + enabled: false diff --git a/schema/common/validation/configuration/rook.yml b/schema/common/validation/configuration/rook.yml new file mode 100644 index 0000000000..59a51853d4 --- /dev/null +++ b/schema/common/validation/configuration/rook.yml @@ -0,0 +1,7 @@ +"$id": "#/specification" +title: "Rook specification schema" +description: "Rook specification schema" +type: object +properties: + enabled: + type: boolean