From d0971e83f27299ed87ab306f9f0b05d7733c9f8d Mon Sep 17 00:00:00 2001
From: Seena Fallah <seena.fallah@ionos.com>
Date: Wed, 14 Feb 2024 15:29:03 +0100
Subject: [PATCH] ceph-config: introduce dedicated cluster config flow

Signed-off-by: Seena Fallah <seena.fallah@ionos.com>
---
 group_vars/all.yml.sample                | 10 +++++
 group_vars/rhcs.yml.sample               | 10 +++++
 plugins/filter/dict2dict.py              | 23 ++++++++++
 roles/ceph-config/tasks/main.yml         | 19 +++++----
 roles/ceph-config/templates/ceph.conf.j2 | 10 ++---
 roles/ceph-defaults/defaults/main.yml    | 10 +++++
 roles/ceph-rgw/tasks/pre_requisite.yml   |  9 ++--
 site-container.yml.sample                | 53 ++++++------------------
 site.yml.sample                          | 47 ++++++---------------
 9 files changed, 98 insertions(+), 93 deletions(-)
 create mode 100644 plugins/filter/dict2dict.py

diff --git a/group_vars/all.yml.sample b/group_vars/all.yml.sample
index 18ee24e034..aa96b27bde 100644
--- a/group_vars/all.yml.sample
+++ b/group_vars/all.yml.sample
@@ -273,6 +273,16 @@ dummy:
 
 #cephx: true
 
+# Cluster configuration
+#ceph_cluster_conf:
+#  global:
+#    public_network: "{{ public_network | default(omit) }}"
+#    cluster_network: "{{ cluster_network | default(omit) }}"
+#    osd_pool_default_crush_rule: "{{ osd_pool_default_crush_rule }}"
+#    ms_bind_ipv6: "{{ (ip_version == 'ipv6') | string }}"
+#    ms_bind_ipv4: "{{ (ip_version == 'ipv4') | string }}"
+#    osd_crush_chooseleaf_type: "{{ '0' if common_single_host_mode | default(false) else omit }}"
+
 ## Client options
 #
 #rbd_cache: "true"
diff --git a/group_vars/rhcs.yml.sample b/group_vars/rhcs.yml.sample
index 2fc078ea54..a8b492ac17 100644
--- a/group_vars/rhcs.yml.sample
+++ b/group_vars/rhcs.yml.sample
@@ -273,6 +273,16 @@ ceph_iscsi_config_dev: false
 
 #cephx: true
 
+# Cluster configuration
+#ceph_cluster_conf:
+#  global:
+#    public_network: "{{ public_network | default(omit) }}"
+#    cluster_network: "{{ cluster_network | default(omit) }}"
+#    osd_pool_default_crush_rule: "{{ osd_pool_default_crush_rule }}"
+#    ms_bind_ipv6: "{{ (ip_version == 'ipv6') | string }}"
+#    ms_bind_ipv4: "{{ (ip_version == 'ipv4') | string }}"
+#    osd_crush_chooseleaf_type: "{{ '0' if common_single_host_mode | default(false) else omit }}"
+
 ## Client options
 #
 #rbd_cache: "true"
diff --git a/plugins/filter/dict2dict.py b/plugins/filter/dict2dict.py
new file mode 100644
index 0000000000..5cf842f9e6
--- /dev/null
+++ b/plugins/filter/dict2dict.py
@@ -0,0 +1,23 @@
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+class FilterModule(object):
+    ''' Loop over nested dictionaries '''
+
+    def dict2dict(self, nested_dict):
+        items = []
+        for key, value in nested_dict.items():
+            for k, v in value.items():
+                items.append(
+                    (
+                        {'key': key, 'value': value},
+                        {'key': k, 'value': v},
+                    ),
+                )
+        return items
+
+    def filters(self):
+        return {
+            'dict2dict': self.dict2dict
+        }
diff --git a/roles/ceph-config/tasks/main.yml b/roles/ceph-config/tasks/main.yml
index 36b46f759d..c80e3345ca 100644
--- a/roles/ceph-config/tasks/main.yml
+++ b/roles/ceph-config/tasks/main.yml
@@ -96,14 +96,6 @@
 - name: set osd related config facts
   when: inventory_hostname in groups.get(osd_group_name, [])
   block:
-    - name: set_fact _osd_memory_target, override from ceph_conf_overrides
-      set_fact:
-        _osd_memory_target: "{{ item }}"
-      loop:
-        - "{{ ceph_conf_overrides.get('osd', {}).get('osd memory target', '') }}"
-        - "{{ ceph_conf_overrides.get('osd', {}).get('osd_memory_target', '') }}"
-      when: item
-
     - name: set_fact _osd_memory_target
       set_fact:
         _osd_memory_target: "{{ ((ansible_facts['memtotal_mb'] * 1048576 * safety_factor | float) / num_osds | float) | int }}"
@@ -112,6 +104,16 @@
         - num_osds | default(0) | int > 0
         - ((ansible_facts['memtotal_mb'] * 1048576 * safety_factor | float) / num_osds | float) > (osd_memory_target | float)
 
+    - name: Set osd_memory_target to cluster host config
+      ceph_config:
+        action: set
+        who: "osd.*/{{ ansible_hostname }}:host"
+        option: "osd_memory_target"
+        value: "{{ _osd_memory_target }}"
+      when:
+        - _osd_memory_target is defined
+        - ceph_conf_overrides.get('osd', {}).get('osd_memory_target', '') == ''
+
 - name: create ceph conf directory
   file:
     path: "/etc/ceph"
@@ -134,6 +136,7 @@
     owner: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}"
     group: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}"
     mode: "0644"
+    config_overrides: "{{ ceph_conf_overrides }}"
     config_type: ini
   notify:
     - restart ceph mons
diff --git a/roles/ceph-config/templates/ceph.conf.j2 b/roles/ceph-config/templates/ceph.conf.j2
index 69850ed08e..af5d1cb5de 100644
--- a/roles/ceph-config/templates/ceph.conf.j2
+++ b/roles/ceph-config/templates/ceph.conf.j2
@@ -2,11 +2,11 @@
 # {{ ansible_managed }}
 
 [global]
-#{% if not cephx | bool %}
-#auth cluster required = none
-#auth service required = none
-#auth client required = none
-#{% endif %}
+{% if not cephx | bool %}
+auth cluster required = none
+auth service required = none
+auth client required = none
+{% endif %}
 {# NOTE (leseb): the blank lines in-between are needed otherwise we won't get any line break #}
 
 {% set nb_mon = groups.get(mon_group_name, []) | length | int %}
diff --git a/roles/ceph-defaults/defaults/main.yml b/roles/ceph-defaults/defaults/main.yml
index 71882f6ddd..e327a57639 100644
--- a/roles/ceph-defaults/defaults/main.yml
+++ b/roles/ceph-defaults/defaults/main.yml
@@ -265,6 +265,16 @@ ceph_keyring_permissions: '0600'
 
 cephx: true
 
+# Cluster configuration
+ceph_cluster_conf:
+  global:
+    public_network: "{{ public_network | default(omit) }}"
+    cluster_network: "{{ cluster_network | default(omit) }}"
+    osd_pool_default_crush_rule: "{{ osd_pool_default_crush_rule }}"
+    ms_bind_ipv6: "{{ (ip_version == 'ipv6') | string }}"
+    ms_bind_ipv4: "{{ (ip_version == 'ipv4') | string }}"
+    osd_crush_chooseleaf_type: "{{ '0' if common_single_host_mode | default(false) else omit }}"
+
 ## Client options
 #
 rbd_cache: "true"
diff --git a/roles/ceph-rgw/tasks/pre_requisite.yml b/roles/ceph-rgw/tasks/pre_requisite.yml
index 11a49f763f..42bc739971 100644
--- a/roles/ceph-rgw/tasks/pre_requisite.yml
+++ b/roles/ceph-rgw/tasks/pre_requisite.yml
@@ -16,11 +16,15 @@
   loop: "{{ hostvars[inventory_hostname]['rgw_instances'] }}"
 
 - name: set rgw parameter (rgw_frontends)
+  vars:
+    _rgw_binding_socket: "{{ item.radosgw_address | default(_radosgw_address) | string + ':' + item.radosgw_frontend_port | default(radosgw_frontend_port) | string }}"
+    _rgw_beast_endpoint: "{{ 'ssl_' if radosgw_frontend_ssl_certificate else '' }}endpoint={{ _rgw_binding_socket }}"
+    _rgw_beast_ssl_option: "{{ ' ssl_certificate='+radosgw_frontend_ssl_certificate if radosgw_frontend_ssl_certificate else '' }}"
   ceph_config:
     action: set
     who: "client.rgw.{{ _rgw_hostname + '.' + item.instance_name }}"
     option: "rgw_frontends"
-    value: "beast port={{ item.radosgw_frontend_port | string }}"
+    value: "beast {{ _rgw_beast_endpoint }}{{ _rgw_beast_ssl_option }}"
   environment:
     CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}"
     CEPH_CONTAINER_BINARY: "{{ container_binary }}"
@@ -28,9 +32,6 @@
   loop: "{{ hostvars[inventory_hostname]['rgw_instances'] }}"
   notify: restart ceph rgws
 
-# rgw_frontends
-# {{ 'ssl_' if radosgw_frontend_ssl_certificate else '' }}endpoint={{ _rgw_binding_socket }}{{ ' ssl_certificate='+radosgw_frontend_ssl_certificate if radosgw_frontend_ssl_certificate else '' }}
-
 - name: create rados gateway directories
   file:
     path: "/var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_facts['hostname'] }}.{{ item.instance_name }}"
diff --git a/site-container.yml.sample b/site-container.yml.sample
index b100262634..ed7c1eac13 100644
--- a/site-container.yml.sample
+++ b/site-container.yml.sample
@@ -175,58 +175,29 @@
             end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - hosts: mons[0]
-  become: True
+  become: true
   gather_facts: false
   any_errors_fatal: true
   tasks:
-    - import_role:
+    - name: Import default role
+      ansible.builtin.import_role:
         name: ceph-defaults
 
-    - name: set global config
-      ceph_config:
-        action: set
-        who: "global"
-        option: "{{ item.key }}"
-        value: "{{ item.value }}"
-      environment:
-        CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}"
-        CEPH_CONTAINER_BINARY: "{{ container_binary }}"
-      with_dict:
-        "{{ { 
-           'public_network': public_network | default(False),
-           'cluster_network': cluster_network | default(False),
-           'osd pool default crush rule': osd_pool_default_crush_rule,
-           'ms bind ipv6': 'true' if ip_version == 'ipv6' else 'false',
-           'ms bind ipv4': 'false' if ip_version == 'ipv6' else 'true',
-           'osd crush chooseleaf type': '0' if common_single_host_mode | default(False) | bool else False,
-        } }}"
-      when:
-        - inventory_hostname == ansible_play_hosts_all | last
-        - item.value
-
-    - name: set global config overrides
-      ceph_config:
-        action: set
-        who: "global"
-        option: "{{ item.key }}"
-        value: "{{ item.value }}"
-      environment:
-        CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}"
-        CEPH_CONTAINER_BINARY: "{{ container_binary }}"
-      when: inventory_hostname == ansible_play_hosts_all | last
-      with_dict: "{{ ceph_conf_overrides['global'] }}"
+    - name: Import config role
+      ansible.builtin.import_role:
+        name: ceph-config
 
-    - name: set osd_memory_target
+    - name: Set clsuter configs
       ceph_config:
         action: set
-        who: "osd.*/{{ item }}:host"
-        option: "osd_memory_target"
-        value: "{{ _osd_memory_target | default(osd_memory_target) }}"
+        who: "{{ item.0.key }}"
+        option: "{{ item.1.key }}"
+        value: "{{ item.1.value }}"
+      when: item.1.value != omit
+      loop: "{{ ceph_cluster_conf | dict2dict }}"
       environment:
         CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}"
         CEPH_CONTAINER_BINARY: "{{ container_binary }}"
-      when: inventory_hostname == ansible_play_hosts_all | last
-      loop: "{{ groups[osd_group_name] | default([]) }}"
 
 - hosts: osds
   become: True
diff --git a/site.yml.sample b/site.yml.sample
index a5c2fdd225..9ed5947695 100644
--- a/site.yml.sample
+++ b/site.yml.sample
@@ -167,49 +167,26 @@
             end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - hosts: mons[0]
-  become: True
+  become: true
   gather_facts: false
   any_errors_fatal: true
   tasks:
-    - import_role:
+    - name: Import default role
+      ansible.builtin.import_role:
         name: ceph-defaults
 
-    - name: set global config
-      ceph_config:
-        action: set
-        who: "global"
-        option: "{{ item.key }}"
-        value: "{{ item.value }}"
-      with_dict:
-        "{{ {
-           'public_network': public_network | default(False),
-           'cluster_network': cluster_network | default(False),
-           'osd pool default crush rule': osd_pool_default_crush_rule,
-           'ms bind ipv6': 'true' if ip_version == 'ipv6' else 'false',
-           'ms bind ipv4': 'false' if ip_version == 'ipv6' else 'true',
-           'osd crush chooseleaf type': '0' if common_single_host_mode | default(False) | bool else False,
-        } }}"
-      when:
-        - inventory_hostname == ansible_play_hosts_all | last
-        - item.value
-
-    - name: set global config overrides
-      ceph_config:
-        action: set
-        who: "global"
-        option: "{{ item.key }}"
-        value: "{{ item.value }}"
-      when: inventory_hostname == ansible_play_hosts_all | last
-      with_dict: "{{ ceph_conf_overrides['global'] }}"
+    - name: Import config role
+      ansible.builtin.import_role:
+        name: ceph-config
 
-    - name: set osd_memory_target
+    - name: Set clsuter configs
       ceph_config:
         action: set
-        who: "osd.*/{{ item }}:host"
-        option: "osd_memory_target"
-        value: "{{ _osd_memory_target | default(osd_memory_target) }}"
-      when: inventory_hostname == ansible_play_hosts_all | last
-      loop: "{{ groups[osd_group_name] | default([]) }}"
+        who: "{{ item.0.key }}"
+        option: "{{ item.1.key }}"
+        value: "{{ item.1.value }}"
+      when: item.1.value != omit
+      loop: "{{ ceph_cluster_conf | dict2dict }}"
 
 - hosts: osds
   gather_facts: false