Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add Ansible Rook role (#2960)
Browse files Browse the repository at this point in the history
* Add initial ansible rook role

* Fix issue with no yet existing template

* Add Rook schema related files

* Remove already setup vars, add missing enters

* Fix issue with feature mapping

* Add configuration support

* Move kubeconfig to rook role yaml

* Update changelog

* Add started dashes to yaml defaults for Rook

* Add RH fix suggested by cicharka

* Add rook cluster helm chart installation

* Add separate chart values definitions

* Limit fact checking in rook role

* Add initial rook/ceph documentation

* Update documentation for Rook

* Update download requirements
erzetpe committed Apr 4, 2022
1 parent 82f65ae commit 436b638
Showing 10 changed files with 226 additions and 4 deletions.
Original file line number Diff line number Diff line change
@@ -38,3 +38,9 @@ files:

'https://helm.elastic.co/helm/filebeat/filebeat-7.9.2.tgz':
sha256: 5140b4c4473ca33a0af4c3f70545dcc89735c0a179d974ebc150f1f28ac229ab

'https://charts.rook.io/release/rook-ceph-v1.8.5.tgz':
sha256: 11a425c16a8513e9cf7a2d6de396ea6e5166a03a63a8bf883d4f2098ab35b268

'https://charts.rook.io/release/rook-ceph-cluster-v1.8.5.tgz':
sha256: 58695a740845091c10426266e7b345fe6185ee5a3b86f1f2d6e52a55af8503c1
Original file line number Diff line number Diff line change
@@ -156,3 +156,28 @@ images:

'calico/pod2daemon-flexvol:v3.20.3':
sha1: 97c1b7ac90aa5a0f5c52e7f137549e598ff80f3e

# --- Rook ---
'k8s.gcr.io/sig-storage/csi-attacher:v3.4.0':
sha1: 8567876a11c527e9d406d3f3efa09e3cee437985

'k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.5.0':
sha1: 07104d23bbb224f81ef6b8fd379b01a7cbba0946

'k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0':
sha1: 55c9d55eb0f2cbe9e37e5464e578b2d7fa45f8f8

'k8s.gcr.io/sig-storage/csi-resizer:v1.4.0':
sha1: 289bafda08f126423dcf7ad78f08a5d8ed57a82e

'k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1':
sha1: 8deb564fde820c7f795d8b425f867d4194edd088

'quay.io/ceph/ceph:v16.2.7':
sha1: 039717f406243b516fead5c36d2d5698c458bf21

'quay.io/cephcsi/cephcsi:v3.5.1':
sha1: 2f89fc81c6665f4daf9df2edf3fbb45caa45891c

'rook/ceph:v1.8.5':
sha1: 96b3efbc626deec6c1db31e43d8d8d6fe8186174
6 changes: 6 additions & 0 deletions ansible/playbooks/roles/rook/defaults/main.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
rook_helm_chart_file_name: rook-ceph-v1.8.5.tgz
rook_helm_cluster_chart_file_name: rook-ceph-cluster-v1.8.5.tgz
rook_chart_namespace: rook-ceph
rook_helm_chart_name: rook-ceph
rook_helm_cluster_chart_name: rook-ceph-cluster
67 changes: 67 additions & 0 deletions ansible/playbooks/roles/rook/tasks/main.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
---
- name: Prepare configuration and upgrade/install Rook Helm chart
when: specification.enabled
become: true
run_once: true
block:

- name: RedHat fix | Create helm's binary symlink
file:
src: "/usr/local/bin/helm"
dest: "/usr/bin/helm"
state: link
when: ansible_os_family == 'RedHat'

- name: Download Rook's Chart Files
include_role:
name: download
tasks_from: download_file
vars:
file_name: "{{ item }}"
loop:
- "{{ rook_helm_chart_file_name }}"
- "{{ rook_helm_cluster_chart_file_name }}"

- name: Create custom configuration for operator Helm chart file (operator-custom-chart-values.yml)
when: specification.operator_chart_values is defined
copy:
content: "{{ specification.operator_chart_values }}"
dest: "{{ download_directory }}/operator-custom-chart-values.yml"

- name: Create custom configuration for cluster Helm chart file (cluster-custom-chart-values.yml)
when: specification.cluster_chart_values is defined
copy:
content: "{{ specification.cluster_chart_values }}"
dest: "{{ download_directory }}/cluster-custom-chart-values.yml"

- name: Install Rook operator using Helm chart with values from operator-custom-chart-values.yml
when: specification.operator_chart_values is defined
shell: |
helm -n {{ rook_chart_namespace }} upgrade --install \
-f {{ download_directory }}/operator-custom-chart-values.yml \
{{ rook_helm_chart_name }} \
{{ download_directory }}/{{ rook_helm_chart_file_name }} --create-namespace
- name: Install Rook operator using Helm chart with default values
when: not specification.operator_chart_values is defined
shell: |
helm -n {{ rook_chart_namespace }} upgrade --install \
{{ rook_helm_chart_name }} \
{{ download_directory }}/{{ rook_helm_chart_file_name }} --create-namespace
- name: Create Rook cluster with values from cluster-custom-chart-values.yml
when: specification.cluster_chart_values is defined
shell: |
helm -n {{ rook_chart_namespace }} upgrade --install \
--set operatorNamespace={{ rook_chart_namespace }} \
-f {{ download_directory }}/cluster-custom-chart-values.yml \
{{ rook_helm_cluster_chart_name }} \
{{ download_directory }}/{{ rook_helm_cluster_chart_file_name }} --create-namespace
- name: Create Rook cluster with default values
when: not specification.cluster_chart_values is defined
shell: |
helm -n {{ rook_chart_namespace }} upgrade --install \
--set operatorNamespace={{ rook_chart_namespace }} \
{{ rook_helm_cluster_chart_name }} \
{{ download_directory }}/{{ rook_helm_cluster_chart_file_name }} --create-namespace
12 changes: 12 additions & 0 deletions ansible/playbooks/rook.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
---
- hosts: repository
gather_facts: true
tasks: []

- hosts: kubernetes_master
become: true
become_method: sudo
roles:
- rook
environment:
KUBECONFIG: "{{ kubeconfig.remote }}"
2 changes: 2 additions & 0 deletions docs/changelogs/CHANGELOG-2.0.md
Original file line number Diff line number Diff line change
@@ -21,6 +21,8 @@
- [#2821](https://github.com/epiphany-platform/epiphany/issues/2821) - Node Exporter preflight checks
- [#2888](https://github.com/epiphany-platform/epiphany/issues/2888) - Define additional disks with defined sizes to VMs for Azure
- [#2812](https://github.com/epiphany-platform/epiphany/issues/2812) - Extend K8s config validation
- [#2890](https://github.com/epiphany-platform/epiphany/issues/2890) - Rook/Ceph Kubernetes Native Storage for Azure Kubernetes non-AKS cluster
- [#1452](https://github.com/epiphany-platform/epiphany/issues/1452) - Create Kubernetes Native Storage Epiphany configuration for Azure provider

### Fixed

96 changes: 92 additions & 4 deletions docs/home/howto/kubernetes/PERSISTENT_STORAGE.md
Original file line number Diff line number Diff line change
@@ -1,8 +1,96 @@
## Kubernetes persistent storage
# Kubernetes Persistent Storage

Epiphany supports [Azure Files](https://docs.microsoft.com/en-us/azure/storage/files/storage-files-introduction)
and [Amazon EFS](https://docs.aws.amazon.com/efs/latest/ug/how-it-works.html) storage types to use as Kubernetes
persistent volumes.
In Epiphany there are two supported ways of setting up Kubernetes Persistent Storage:
- Rook/Ceph Cluster Storage with disks resources created by Epiphany
- [Azure Files](https://docs.microsoft.com/en-us/azure/storage/files/storage-files-introduction)
or [Amazon EFS](https://docs.aws.amazon.com/efs/latest/ug/how-it-works.html) storage types to use as Kubernetes persistent volumes

## Kubernetes Rook/Ceph Cluster Storage

Rook provides distributed storage systems for Kubernetes installed with Epiphany.
It provides capabilities:
- self-managing
- self-scaling
- self-healing
- upgrading
- migration
- disaster recovery
- monitoring

Epiphany supports Rook with Ceph storage, other options provided by Rook - Cassandra, NFS are not supported.

### Rook/Ceph General Configuration

To add Rook/Ceph support in Epiphany you need to add to your cluster configuration two elements:
- storage (for cloud deployments - can be automatically created by Epiphany)
- Rook/Ceph

Adding the storage is described below in separate sections for Azure, AWS and on premise environments.

To enable Rook support add to your cluster configuration the section like below:

```yaml
---
kind: configuration/rook
title: "Kubernetes Rook Config"
provider: azure
name: default
specification:
enabled: true
```
The key `specification.enabled` must be set to true to install Rook/Ceph component. This will install Rook/Ceph with default values. To override default values provided by Rook you need to add to `configuration/rook` keys:
- `specification.operator_chart_values` - to override Rook Operator Helm Chart default values
- `specification.cluster_chart_values` - to override Rook Cluster Helm Chart default values

```yaml
---
kind: configuration/rook
title: "Kubernetes Rook Config"
provider: azure
name: default
specification:
enabled: true
operator_chart_values: |
...
cluster_chart_values: |
...
```
Values nested below the `operator_chart_values` and `cluster_chart_values` keys are respectively Helm Chart values for Rook Operator and Rook Ceph Cluster.

More information about Helm Chart values may be found:
- [Helm Operator](https://github.com/rook/rook/blob/master/Documentation/helm-operator.md)
- [Helm Ceph Cluster](https://github.com/rook/rook/blob/master/Documentation/helm-ceph-cluster.md)

Sample configuration files that can be used in Epiphany `configuration/rook`:
- [Helm Operator](https://raw.githubusercontent.com/rook/rook/v1.8.5/deploy/charts/rook-ceph/values.yaml)
- [Helm Ceph Cluster](https://raw.githubusercontent.com/rook/rook/v1.8.5/deploy/charts/rook-ceph-cluster/values.yaml)

More informations about Rook with Ceph storage may be found in the official Rook [documentation](https://rook.io/docs/rook/v1.8/).

### Create disks for Rook/Ceph Cluster Storage - Azure

To create Rook/Ceph Cluster Storage on Azure first you need to add empty disk resource to Kubernetes cluster in key `specification.additional_disks`.

```yaml
---
kind: infrastructure/virtual-machine
name: kubernetes-node-machine
provider: azure
based_on: kubernetes-node-machine
specification:
storage_image_reference:
..
storage_os_disk:
disk_size_gb: 64
additional_disks:
- storage_account_type: Premium_LRS
disk_size_gb: 128
```

#### Create disks for Rook/Ceph Cluster Storage - AWS

#### Create disks for Rook/Ceph Cluster Storage - On Prem

### Azure

3 changes: 3 additions & 0 deletions schema/common/defaults/configuration/feature-mapping.yml
Original file line number Diff line number Diff line change
@@ -49,6 +49,8 @@ specification:
enabled: true
- name: applications
enabled: true
- name: rook
enabled: true

roles_mapping:
kafka:
@@ -113,6 +115,7 @@ specification:
- kubernetes-master
- helm
- applications
- rook
- node-exporter
- filebeat
- firewall
6 changes: 6 additions & 0 deletions schema/common/defaults/configuration/rook.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
kind: configuration/rook
title: "Kubernetes Rook Config"
name: default
specification:
enabled: false
7 changes: 7 additions & 0 deletions schema/common/validation/configuration/rook.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
"$id": "#/specification"
title: "Rook specification schema"
description: "Rook specification schema"
type: object
properties:
enabled:
type: boolean

0 comments on commit 436b638

Please sign in to comment.